Add release 0.11.2 to CHANGELOG

Unit tests

.gitignore

@@ -32,7 +32,9 @@ coverage.html
 # Project specific
 weaver/weaver
 weavedns/weavedns
+weaveproxy/weaveproxy
 sigproxy/sigproxy
+weavewait/weavewait
 tools/bin
 tools/build
 releases
@@ -40,6 +42,8 @@ weave.tar
 weavedns.tar
 weaveexec.tar
 weaveexec/weave
+weaveexec/weaveproxy
+weaveexec/weavewait
 weaveexec/sigproxy
 weaveexec/docker*.tgz
 Vagrantfile.local

.travis.yml

@@ -3,6 +3,9 @@ sudo: false
 go:
   - 1.4
 
+env:
+  - PATH="${PATH}:${HOME}/.local/bin"
+
 addons:
   apt:
     packages:
@@ -11,6 +14,8 @@ addons:
 install:
   - go clean -i net
   - go install -tags netgo std
+  - pip install --user --upgrade gcloud gsutil
+  - bin/setup-ci-secrets $encrypted_5ba036e89377_key $encrypted_5ba036e89377_iv
   - make travis
 
 script:
@@ -20,3 +25,4 @@ script:
 after_success:
   - go get github.com/mattn/goveralls
   - $HOME/gopath/bin/goveralls -coverprofile=profile.cov -service=travis-ci
+  - test "${TRAVIS_PULL_REQUEST}" = "false" && site/build-and-publish.sh

CHANGELOG.md

@@ -1,3 +1,79 @@
+## Release 0.11.2
+
+This is a bug fix release, addressing the following issues:
+
+- `weave run` did not respect DOCKER_CLIENT_ARGS. #855
+- Negative result cache did not expire if requeried within TTL. #845
+
+More details in the [change
+log](https://github.com/weaveworks/weave/issues?q=milestone%3A0.11.2).
+
+*NB: This release does not change the weave protocol version.
+Therefore, when upgrading an existing 0.11 installation incrementally,
+connectivity between peers will be retained.*
+
+## Release 0.11.1
+
+This is a bug fix release, addressing the following issues:
+
+- The IP Allocator could crash in some relatively rare
+  circumstances. #782/#783.
+- When the proxy failed to attach a container to the weave network,
+  there was no failure indication and descriptive error anywhere, and
+  the application process would still start. Now an error is reported
+  to the client (i.e. typically the Docker CLI), recorded in the proxy
+  logs, and the container is terminated. #788/#799.
+- `weave launch-proxy --with-ipam` failed to configure the entrypoint
+  and DNS unless a (possibly blank) `WEAVE_CIDR` was
+  specified. Affected containers could start the application process
+  w/o the weave network interface being available, and without
+  functioning name resolution for the weaveDNS domain.
+  #744/#747/#751/#752
+- The `weave status` output for the IP Allocator was misleadingly
+  conveying a sense of brokenness when no IP allocation requests had
+  been made yet. #787/#801
+- When invoking `weave launch-proxy` twice, the second invocation
+  would output a blank line and terminate with a success exit
+  status. Now it reports that the proxy is already running and exits
+  with a non-zero status. #767/#780
+- `weave launch-proxy` was not respecting WEAVEPROXY_DOCKER_ARGS, so
+  any user-supplied custom configuration for the weaveproxy container
+  was ignored. #755/#780
+- The proxy was not intercepting API calls to the unversioned (1.0)
+  Docker Remote API. Hence none of the weave functionality was
+  available when Docker clients were using that version of the
+  API. #770/#774/#777,#809
+- The proxy would crash when certain elements of the
+  `/containers/create` JSON were missing. We are not aware of any
+  Docker clients doing this, but it's been fixed regardless. #778/#777
+
+More details in the [change
+log](https://github.com/weaveworks/weave/issues?q=milestone%3A0.11.1).
+
+*NB: This release does not change the weave protocol version.
+Therefore, when upgrading an existing 0.11 installation incrementally,
+connectivity between peers will be retained.*
+
+## Release 0.11.0
+
+**Highlights**:
+
+- **automatic IP Address Management (IPAM)**, which allows application
+  containers to be started and attached to the weave network without
+  needing to supply an IP address.
+- **proxy** for automatically attaching containers started with
+  ordinary `docker run`, or the Docker remote API, to the weave
+  network.
+- ability to **add/remove extra DNS records**.
+- performance and scalability improvements
+- fixes for a small number of bugs discovered during testing
+
+More detail in the [change log](https://github.com/weaveworks/weave/issues?q=milestone%3A0.11.0)
+
+*NB: This release changes the weave protocol version. Therefore, when
+upgrading an existing installation, all hosts need to be upgraded in
+order to for them to be able to communicate and form a network.*
+
 ## Release 0.10.0
 
 **Highlights**:

Makefile

@@ -1,7 +1,7 @@
 PUBLISH=publish_weave publish_weavedns publish_weaveexec
 
 .DEFAULT: all
-.PHONY: all update tests publish $(PUBLISH) clean prerequisites build travis
+.PHONY: all update tests publish $(PUBLISH) clean prerequisites build travis run-smoketests
 
 # If you can use docker without being root, you can do "make SUDO="
 SUDO=sudo
@@ -10,7 +10,9 @@ DOCKERHUB_USER=weaveworks
 WEAVE_VERSION=git-$(shell git rev-parse --short=12 HEAD)
 WEAVER_EXE=weaver/weaver
 WEAVEDNS_EXE=weavedns/weavedns
+WEAVEPROXY_EXE=weaveproxy/weaveproxy
 SIGPROXY_EXE=sigproxy/sigproxy
+WEAVEWAIT_EXE=weavewait/weavewait
 WEAVER_IMAGE=$(DOCKERHUB_USER)/weave
 WEAVEDNS_IMAGE=$(DOCKERHUB_USER)/weavedns
 WEAVEEXEC_IMAGE=$(DOCKERHUB_USER)/weaveexec
@@ -27,9 +29,9 @@ all: $(WEAVER_EXPORT) $(WEAVEDNS_EXPORT) $(WEAVEEXEC_EXPORT)
 travis: $(WEAVER_EXE) $(WEAVEDNS_EXE)
 
 update:
-	go get -u -f -v -tags -netgo ./$(dir $(WEAVER_EXE)) ./$(dir $(WEAVEDNS_EXE))
+	go get -u -f -v -tags -netgo ./$(dir $(WEAVER_EXE)) ./$(dir $(WEAVEDNS_EXE)) ./$(dir $(SIGPROXY_EXE)) ./$(dir $(WEAVEPROXY_EXE))
 
-$(WEAVER_EXE) $(WEAVEDNS_EXE): common/*.go
+$(WEAVER_EXE) $(WEAVEDNS_EXE) $(WEAVEPROXY_EXE) $(WEAVEWAIT_EXE): common/*.go
 	go get -tags netgo ./$(@D)
 	go build -ldflags "-extldflags \"-static\" -X main.version $(WEAVE_VERSION)" -tags netgo -o $@ ./$(@D)
 	@strings $@ | grep cgo_stub\\\.go >/dev/null || { \
@@ -41,8 +43,10 @@ $(WEAVER_EXE) $(WEAVEDNS_EXE): common/*.go
 		false; \
 	}
 
-$(WEAVER_EXE): router/*.go weaver/main.go
+$(WEAVER_EXE): router/*.go ipam/*.go ipam/*/*.go weaver/main.go
 $(WEAVEDNS_EXE): nameserver/*.go weavedns/main.go
+$(WEAVEPROXY_EXE): proxy/*.go weaveproxy/main.go
+$(WEAVEWAIT_EXE): weavewait/*.go weavewait/main.go
 
 # Sigproxy needs separate rule as it fails the netgo check in the main
 # build stanza due to not importing net package
@@ -57,9 +61,11 @@ $(WEAVEDNS_EXPORT): weavedns/Dockerfile $(WEAVEDNS_EXE)
 	$(SUDO) docker build -t $(WEAVEDNS_IMAGE) weavedns
 	$(SUDO) docker save $(WEAVEDNS_IMAGE):latest > $@
 
-$(WEAVEEXEC_EXPORT): weaveexec/Dockerfile $(DOCKER_DISTRIB) weave $(SIGPROXY_EXE)
+$(WEAVEEXEC_EXPORT): weaveexec/Dockerfile $(DOCKER_DISTRIB) weave $(SIGPROXY_EXE) $(WEAVEPROXY_EXE) $(WEAVEWAIT_EXE)
 	cp weave weaveexec/weave
-	cp sigproxy/sigproxy weaveexec/sigproxy
+	cp $(SIGPROXY_EXE) weaveexec/sigproxy
+	cp $(WEAVEWAIT_EXE) weaveexec/weavewait
+	cp $(WEAVEPROXY_EXE) weaveexec/weaveproxy
 	cp $(DOCKER_DISTRIB) weaveexec/docker.tgz
 	$(SUDO) docker build -t $(WEAVEEXEC_IMAGE) weaveexec
 	$(SUDO) docker save $(WEAVEEXEC_IMAGE):latest > $@
@@ -92,9 +98,12 @@ publish: $(PUBLISH)
 
 clean:
 	-$(SUDO) docker rmi $(WEAVER_IMAGE) $(WEAVEDNS_IMAGE) $(WEAVEEXEC_IMAGE)
-	rm -f $(WEAVER_EXE) $(WEAVEDNS_EXE) $(SIGPROXY_EXE) $(WEAVER_EXPORT) $(WEAVEDNS_EXPORT) $(WEAVEEXEC_EXPORT)
+	rm -f $(WEAVER_EXE) $(WEAVEDNS_EXE) $(SIGPROXY_EXE) $(WEAVEPROXY_EXE) $(WEAVEWAIT_EXE) $(WEAVER_EXPORT) $(WEAVEDNS_EXPORT) $(WEAVEEXEC_EXPORT)
 
 build:
 	$(SUDO) go clean -i net
 	$(SUDO) go install -tags netgo std
 	$(MAKE)
+
+run-smoketests: all
+	cd test && ./setup.sh && ./run_all.sh

README.md

@@ -1,9 +1,9 @@
 # Weave - the Docker network
 
-[![Build Status](https://travis-ci.org/weaveworks/weave.svg?branch=master)](https://travis-ci.org/weaveworks/weave) [![Coverage Status](https://coveralls.io/repos/weaveworks/weave/badge.svg)](https://coveralls.io/r/weaveworks/weave)
+[![Build Status](https://travis-ci.org/weaveworks/weave.svg?branch=master)](https://travis-ci.org/weaveworks/weave) [![Integration Tests](https://circleci.com/gh/weaveworks/weave/tree/master.svg?style=shield&circle-token=4933c7dabb3d0383e62117565cb9d16df7b1a811)](https://circleci.com/gh/weaveworks/weave) [![Coverage Status](https://coveralls.io/repos/weaveworks/weave/badge.svg)](https://coveralls.io/r/weaveworks/weave)
 
 Weave creates a virtual network that connects Docker containers
-deployed across multiple hosts.
+deployed across multiple hosts and enables their automatic discovery.
 
 ![Weave Virtual Network](/docs/virtual-network.png?raw=true "Weave Virtual Network")
 
@@ -29,7 +29,7 @@ capabilities, so these can continue to be used by containers.
 
 ## Installation
 
-Ensure you are running Linux (kernel 3.5 or later) and have Docker
+Ensure you are running Linux (kernel 3.8 or later) and have Docker
 (version 1.3.1 or later) installed. Then install weave with
 
     sudo wget -O /usr/local/bin/weave \
@@ -53,10 +53,10 @@ Say you have docker running on two hosts, accessible to each other as
 `$HOST1` and `$HOST2`, and want to deploy an application consisting of
 two containers, one on each host.
 
-On `$HOST1` run (as root)
+On `$HOST1` run
 
-    host1# weave launch
-    host1# C=$(weave run 10.2.1.1/24 -t -i ubuntu)
+    host1$ weave launch
+    host1$ C=$(weave run 10.2.1.1/24 -t -i ubuntu)
 
 The first line starts the weave router, in a container. This needs to
 be done once on each host. The required docker image for the weave
@@ -87,8 +87,8 @@ addresses must, of course, be unique.
 
 We repeat similar steps on `$HOST2`...
 
-    host2# weave launch $HOST1
-    host2# C=$(weave run 10.2.1.2/24 -t -i ubuntu)
+    host2$ weave launch $HOST1
+    host2$ C=$(weave run 10.2.1.2/24 -t -i ubuntu)
 
 The only difference, apart from the choice of IP address for the
 application container, is that we tell our weave that it should peer
@@ -103,14 +103,14 @@ Note that we could instead have told the weave on `$HOST1` to connect to
 weave automatically (re)connects to peers when they become
 available. Also, we can tell weave to connect to multiple peers by
 supplying multiple addresses, separated by spaces. And we can
-[add peers dynamically](http://weaveworks.github.io/weave/features.html#dynamic-topologies).
+[add peers dynamically](http://docs.weave.works/weave/latest_release/features.html#dynamic-topologies).
 
 Now that we've got everything set up, let's see whether our containers
 can talk to each other...
 
 On `$HOST1`...
 
-    host1# docker attach $C
+    host1$ docker attach $C
     root@28841bd02eff:/# ping -c 1 -q 10.2.1.2
     PING 10.2.1.2 (10.2.1.2): 48 data bytes
     --- 10.2.1.2 ping statistics ---
@@ -119,7 +119,7 @@ On `$HOST1`...
 
 Similarly, on `$HOST2`...
 
-    host2# docker attach $C
+    host2$ docker attach $C
     root@f76829496120:/# ping -c 1 -q 10.2.1.1
     PING 10.2.1.1 (10.2.1.1): 48 data bytes
     --- 10.2.1.1 ping statistics ---
@@ -131,12 +131,12 @@ to each other.
 
 ## Find out more
 
- * [Documentation homepage](http://weaveworks.github.io/weave/)
- * [Features](http://weaveworks.github.io/weave/features.html)
- * [Troubleshooting](http://weaveworks.github.io/weave/troubleshooting.html)
- * [Building](http://weaveworks.github.io/weave/building.html)
- * [How it works](http://weaveworks.github.io/weave/how-it-works.html)
- * [WeaveDNS README](https://github.com/weaveworks/weave/tree/master/weavedns#readme)
+ * [Documentation homepage](http://docs.weave.works/weave/latest_release/)
+ * [Features](http://docs.weave.works/weave/latest_release/features.html)
+ * [Automatic Discovery with WeaveDNS](http://docs.weave.works/weave/latest_release/weavedns.html)
+ * [Troubleshooting](http://docs.weave.works/weave/latest_release/troubleshooting.html)
+ * [Building](http://docs.weave.works/weave/latest_release/building.html)
+ * [How it works](http://docs.weave.works/weave/latest_release/how-it-works.html)
 
 ## Contact Us
 

Vagrantfile

@@ -1,12 +1,25 @@
 VAGRANTFILE_API_VERSION = "2"
 
+require './vagrant-common.rb'
+
 vm_ip = "172.16.0.3" # arbitrary private IP
-pkgs = "lxc-docker aufs-tools build-essential ethtool iputils-arping libpcap-dev git mercurial bc"
+
+pkgs = %w(
+  lxc-docker
+  aufs-tools
+  build-essential
+  ethtool
+  iputils-arping
+  libpcap-dev
+  git
+  mercurial
+  bc
+)
 
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
 
-  config.vm.box = "phusion/ubuntu-14.04-amd64"
-  config.vm.box_url = "https://oss-binaries.phusionpassenger.com/vagrant/boxes/latest/ubuntu-14.04-amd64-vbox.box"
+  config.vm.box = "ubuntu/ubuntu-14.10-amd64"
+  config.vm.box_url = "https://cloud-images.ubuntu.com/vagrant/utopic/current/utopic-server-cloudimg-amd64-vagrant-disk1.box"
 
   config.vm.network "private_network", ip: vm_ip
   config.vm.provider :virtualbox do |vb|
@@ -16,27 +29,12 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
 
   config.vm.synced_folder ".", "/vagrant", disabled: true
   config.vm.synced_folder ".", "/home/vagrant/src/github.com/weaveworks/weave"
-  config.vm.provision :shell, :inline => "ln -s /home/vagrant/src/github.com/weaveworks/weave /home/vagrant/weave"
 
-  config.vm.provision :shell, :inline => "sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 36A1D7869245C8950F966E92D8576A8BA88D21E9"
-  config.vm.provision :shell, :inline => "echo deb https://get.docker.io/ubuntu docker main > /etc/apt/sources.list.d/docker.list"
-
-  pkg_cmd = "export DEBIAN_FRONTEND=noninteractive; " \
-    "apt-get update -qq; " \
-    "apt-get install -qq --no-install-recommends #{pkgs}"
-
-  install_go_toolchain = "curl -s https://storage.googleapis.com/golang/go1.4.2.linux-amd64.tar.gz | tar xz -C /usr/local"
-
-  config.vm.provision :shell, :inline => pkg_cmd
-  config.vm.provision :shell, :inline => install_go_toolchain
-  config.vm.provision :shell, :inline => "usermod -a -G docker vagrant; "
-  config.vm.provision :shell, :inline => "echo export GOPATH=/home/vagrant >> /home/vagrant/.bashrc"
-  config.vm.provision :shell, :inline => "echo export PATH=/usr/local/go/bin:${PATH} >> /home/vagrant/.bashrc"
-  config.vm.provision :shell, :inline => "chown -R vagrant:vagrant /home/vagrant/src"
-  config.vm.provision :shell, :inline => "/usr/local/go/bin/go clean -i net; /usr/local/go/bin/go install -tags netgo std"
-
-  config.vm.provision :shell, :inline => "echo 'DOCKER_OPTS=\"-H unix:///var/run/docker.sock -H tcp://0.0.0.0:2375\"' >> /etc/default/docker"
-  config.vm.provision :shell, :inline => "service docker restart"
+  install_build_deps config.vm, pkgs
+  install_go_toochain config.vm
+  tweak_user_env config.vm
+  tweak_docker_daemon config.vm
+  cleanup config.vm
 
 end
 

bin/merge-into-gh-pages

@@ -1,20 +0,0 @@
-#! /bin/bash
-set -e
-
-# For info on why/how to use this script, see
-# https://github.com/weaveworks/weave/wiki/WorkingOnWeave#working-on-docs
-
-MERGE_ARGS="-X subtree=site"
-
-merge_dir=$(mktemp -d -t weave-merge.XXXXXX)/weave
-git clone -b gh-pages . $merge_dir
-
-pushd $merge_dir
-git pull --ff-only git@github.com:weaveworks/weave gh-pages
-git fetch origin master
-git merge $MERGE_ARGS -e -m "Merge master into gh-pages" origin/master
-
-popd
-git fetch $merge_dir gh-pages:gh-pages
-
-# rm -rf $merge_dir

bin/multiweave

@@ -14,14 +14,23 @@ set -e
 START=${FIRST_WEAVE:-1}
 COUNT=${NUM_WEAVES:-28}
 FINISH=$((START+COUNT-1))
+PORT=6783
 
 weavedir=$(dirname $0)/..
 
+# Generate a random MAC value - copied from 'weave' script
+random_mac() {
+    # In the first byte of the MAC, the 'multicast' bit should be
+    # clear and 'locally administered' bit should be set.  All other
+    # bits should be random.
+    od -txC -An -N6 /dev/urandom | sed 's|^ ||;s| |:|;s|[0-3]:|2:|;s|[4-7]:|6:|;s|[89ab]:|a:|;s|[c-f]:|e:|;s| |:|g'
+}
+
 case "$1" in
     launch)
         shift 1
         for i in $(seq $START $FINISH); do
-            WEAVE_CONTAINER_NAME=weave$i $weavedir/weave test-launch "$@"
+            WEAVE_CONTAINER_NAME=weave$i WEAVE_PORT=$((5000+i)) $weavedir/weave launch -iface '' -name $(random_mac) -nickname weave$i -port $PORT "$@"
         done
         if [ $START -ne 1 ] ; then
             status=$(docker inspect --format='{{.State.Running}} {{.NetworkSettings.IPAddress}}' weave$((START-1)))

bin/publish-gh-pages

@@ -1,3 +0,0 @@
-#! /bin/sh
-set -e
-git push git@github.com:weaveworks/weave gh-pages:gh-pages

bin/setup-ci-secrets

@@ -0,0 +1,57 @@
+#!/bin/sh
+
+## This is the shape of unencrypted `do-setup-ci-secrets`
+## 
+## cat > /tmp/boto_gsutil_cert.pem << EOF
+## -----BEGIN PRIVATE KEY-----
+## <...>
+## -----END PRIVATE KEY-----
+## EOF
+## 
+## cat > ~/.boto << EOF
+## [Credentials]
+## gs_service_client_id = <...>
+## gs_service_key_file = /tmp/boto_gsutil_cert.pem
+## 
+## [Boto]
+## https_validate_certificates = True
+## 
+## [GSUtil]
+## content_language = en
+## default_api_version = 2
+## default_project_id = <...>
+## EOF
+
+openssl base64 -d << EOF \
+  | openssl aes-256-cbc -d -K $1 -iv $2 \
+    -out bin/do-setup-ci-secrets
+mOYNzNsAdwva0LbB2YFMY9VgL0SITFu1bPrbOujyLtOrBNjmOYbzR5d/sG+ARMFr
+7NZ5OKcvMbItllz0n7TtNkzkCOMztMC870Wcx3nGJzbBephdcYwos4tBlp5bihTU
+AXgxLUiDT1broYAFKnbE1aZC/XyNxuLTN+nXJWIWeijaXTgvhkEEm3tab+z7ioCb
+tlg71W1BrUuocctAJAuVwH/uTZAaXaJGJj1s2S7F2DrFnG6sFlrp8rnCucPLWKnC
+4IwJFC5saR5gxK6wECffwmwbpeXVOoQ58OgzxoZTEyAr9nhVbaBt5UZz3tfHlx8W
+Rn5jHehACw5kg5eMtGPqMEksJ63LX4h3mfSTLJXJHEmOrzCRpFrigVrsbA408U2u
+8jJTHJmKrGfwBh5ZFzBg6NuV4VplOp+ZquJLgegv2WH/iu/ciopXqB2YDCdx5ZBi
+nHV2Puth7kjS4EMCKUGfeqpesjL9Djb8cygzv/J3ZBmLjNH9dqlSXWy2DcFZA1Iy
+pFTTHkVU3w5sNz3nYOuT2UPQvULBW0FUndDKX6olyVa0mlxPwGBP9N3yGg918/rw
+7rNV/r54lA334F5QMhDrVxar6sl9wfLA+Z+8qemBbNFlLog1uQGhuwWK2nFu1tRM
+9uyvQIq0oq5NHKpogWc7NEuk7CPe1JyfRIcbMVj8+ilmKjUPB2HPC43IDmKt9/56
+rY3IXAcxZ/ClGfGpeNHXqop9T/WhfaowponOZFcdGSUddG0OswqkTE4KIv71mgTF
+jXooMkTQBOG+gBnCe+8bCRdRCG3anxHX74za4dGD58cOJtl6DYWnjWuR91NWnxdT
+1Zr3iUe41txg9hCNY9LHo2QHGLPxe6pmNnSClQ1mcFhuY6NF8RmCX3HAGKJQkxJb
+vpLGbXTXjAIFgIYgLNgKP2RLvCif/LmSWgkvE3ZU9GXtCUbm6xgJodwOqXiI9uuO
+4H4+/vpGHnSFENiclkgMFrLG4yiMu8ArgmnCBzg9E2AoOX5zCrh/2ZRTU8Dk4mt7
+jUd5SKQLsjB9QtB+y3tRO7zY9RDvR6iBOPRsjx6t8EJMCfPb06KchfMJw+VGxvMM
+48jtmepREFnhd/8XDRGnw5bznjRuwohGik8BToltjIviMkkpifKWaLwlaOiDXaHh
+VBFNXnUKvOBwRMU+PZqv4+DP2qff1gSXnGKqA5GIURYrK9P7TqHHNTnmjBdfJLzc
+0x+W1DTiYhLT2vrLrkEpZ99o6qrg+lwvjat6IsJF8xoQEo4pLD/ZRPJ0WzoDyctT
+oWZ5rMsN1TzTvJrBrTAWIJyhIzZcuNwChMrU6qogMr0fd6TSxWhWn0sOJ5y8AaTR
+ej4JHx0qroS/9JPmgFYU61pX3c0ypdef/sSHWG23mcFzLTgwe2d4ty9VakypZy3T
+sXCl6EHVVXlUFWhY4zQCjdZpYJv/rtoKL1BL4fPE2NV5WTjCL2GNvaO5SfHR7pA0
+yQJR/iU+cS2n8UXsWNjyV+/qEn6+QXuyVHGLbxyH8EeqlliAbyHsc2rzVI8Xctzo
+W5DuIqxwHAkNNurhmx27ktIIQ6HhaCp028SWwvLttkbBBgF4v1mmd4u/2bKbKjUd
+dOaetaIUx1hU9OAeaePvVkeTDcc3mlFtPARCS9Fqj16hgJQ43onZyC0H1K2WAxmU
+GsmzjCHlTNquzFTuuQihYGZKfsWIh9z6HEOkvNJ3VEQ=
+EOF
+
+exec sh bin/do-setup-ci-secrets

bin/setup-circleci-secrets

@@ -0,0 +1,115 @@
+#!/bin/bash
+
+set -eu
+
+# openssl enc -in do-setup-circleci-secrets.orig -out setup-circleci-secrets.orig -e -aes256 -pass stdin
+# openssl base64 < setup-circleci-secrets.orig
+
+openssl base64 -d << EOF \
+  | openssl enc \
+    -out bin/do-setup-circleci-secrets \
+		-d -aes256 -pass pass:$1
+U2FsdGVkX1/3QHnDgpZn0srp+GiEQBTlag0CBdsw6Bp6iaIKy6Jy8gFshrsmvl3+
+c+7hn1nqsBVpzVH0S5QA3Ex5nx9rl7/dcGoogsySFcwMmBFcauT0UTMK1bvX0Hqx
+QrshYSndUkww72KA7EsSnjR1BBJWm3AhCDOTIcnO6nQhig1Fc+SBEj5FGS5xTQ5Z
+/8hDsKf54Ms49vOO6QeE5w5ndZ+k7IvVJfq9Evhb7htD20RntDhR86foAwMnLV7R
+Wh9qAqI3gnRq0GmbeRKaHmB7pcLq9f0OdScqM/Tn1D+q0556fqJjBziXpZiGbxVs
+DxEp8zDJ8yNDAi0Wb2NXjoA79SXJ8Q7iTieaST4y2OmXQpY+Vw8RVDXxi6ERyVnH
+41OeMzyLm0qvCoMl32pe1sSG5DRCUiAm7wOtDTVvNDRqmGJYUgwwexyayecJI1wQ
+LlhYLJavUvw3UGDZOld8BaMQhfOKBVCdFrjmA4HuVxmOf3029qkgzOCSZiQVNqs+
+fvM6J05zBuk+GPy8YInvRkzfxum/x00vmUyGjMs/lHDFfrmYDFK3ux41EoVqvWLG
+S5uYxdZJFdR3Xsrs+sMr51XNBQGMmz6KoEEzFYyHs5fZgtdpRdPAuQS1m6FqgZyi
+wSz6+EKsALtmGorEp+CF8JN5YzKJCrKzNkg0fILBqsqmJZGP64lmX69950lRLNGM
+/l3KVR2zH31b4AnnzTcvldVSCKDXjaMw3JA2Ce/MK6XhOWOcpEv4oDt/62lbhUKP
+QUQTPbfpQRvVfBcoIrfAm3ghi/CZc2TDVEuD7DhasLTZD3r7LoK261BWC9O5Wubi
+vlXuFIF+uyrpTDf6EJKE/+XywtArppffMP6RkHu1Iqnd8JtsqPaIZsmMmZbjgb6j
+BX4ZLb5U9R7jYoSgIke4+u6geFcrMQ+Lny4r7MxXaS0Y1UJ6Jo1bwpYw1TBIobZc
+eQ30+xhbSeRC9vEw1NXI6qM2Ai2gIHtAZvEiBkBBXTG5Ld7N/mBAmDIJEzVRxh6F
+jRkB3dzESF6eeTBm61OQepwV3aW6lslMaf26drPGv2W2q+K/WOpQOejfggtHUgVs
+9SWqtn7hHYSpIASe1Ihi14HAWv3kPhQien0zY1DLKJWf4ilbRaICn1QSJjNuKwtl
+A2ZwEw7FFddFcsF4okHPkxEoCre3shq0DEw9UA1gnwEaEF1Q+i9cvLtQg56zK3+R
+MfTzFUv2K1z3j1s5tC6Z6LzL8j682EFOll7MkihqiIWh+po1zcNs6Ln0SkjB3v9T
+OFRYVJis9GqXYO9PUW1mI69udR4nXbKlBl5tWNyAxn9hufJHLSAmbGMwActKLRM5
+kROapKrN1Ra/sHUmhoSSDUK9AXeYKkqPXVMbhly9xC1Kya4hFaFokjRQIMLPmXdQ
+lP46RL3HOq0v1BkXg05poSwDm+BtitZkcIhOzcmeucUqn1whsz39y3VoOhJqQu2j
+0dUcj19Fue1CDZfwzACb9mLZr643h0yCIHQaVT7is/yiN2vEt0S0zynvOA1D+Xgy
+4bFdjqed9DgBShLwF/lh4/hur5iZPpVr9A8rt3Vq0k5A1tdjsZpXP3qv5jLc7lJv
+cAQxdVLEuv8WPwwTV/EQsoH7lpoEdjktw/gu0v5PkUgR/WBxEqJq4liTekViQ+8X
+BAmdm/FGWeIxSSEm0a8I+muiBpIuKjuQoJit1f0k+lLUohoVDqpDDfeYHh1TxXHE
+xSKAKNmFSc2XOr5Zg2mOInxeH7WoxH6iylXKADBgEgAQ4C55DoAPRjFNtGGz2iJl
+WkzhQOfgsDss1VxpPZ4SWDQO+xxuuEnYcCYdJ6Tyi5hAwZ0+RZmXx0g9BCK+5qzc
+Ffei066kCItBcayekwCs0i86r8JvDtsMUUkzEKDQTSWHpP8ypIZoz3P+Skdqcs6p
+IcZHy2fODIsClctRAsHOjZ9BnWSY2rqf1lysqvRfOkNTViTOUVx2hf+LuL+8rT6g
+K4GvS39Q+c9FYSMjCNxv1m37NfHJv2W7AlqVV95gMBEFB97M3388T7PfvmSLsyPv
+qAKDE7genTOycQvEX9oeEACxPo08ai2Mgs/YXCig0g0btZoScoKPelQKtD0HLpZ+
+j7ruaKFy4/vySDMostB3AKSkCE/SkGu/W9oI37n1guSYC/9yJjtTcwZULXAClfwt
+LyHvLAw4+7BdGJqDe9Tf8z4zqVLax+BcBHkW7pbP+pvxC6fhIA4PO8k5HwwAVlRw
+68lG6e3z9+Vst4zrFLewHJvkDRSOTSecs9h9SPOjReTK0ovlDzMHVakfUvqpk9mv
+nCj8SWaMPF70z138AuRuKN7X4exNXPBFnOcseKp/NpT1DyvgoHm/oIO65mq9vvo+
+42uSbyyegYiTdVK4SaXBc2hixkhaTteqdEy6GxfWqKCvHtYuRD2LyUSjZn/0Y6CG
+3wZapylKL1sxfH+b1gogv2wAA8KtZAfiTlMISuymwQr5RbOFma7UVMVecCa7XrNG
+fZwO8R+6++ru0efDzeC/gtLOItjn+yVfsbk2gfu0FikHIV1sogOjnsv3dl6IpH/s
+w6WbuK/yZhyrrmhIIueHF30ES512Peogf/Gxp8U/ZAmqBa7p4qsv93XWDR52IPYz
+yU1MtboLD02J39daBNrYZuO4Yfo/xAvU7Ijg2KIR9HS0bSWNuSFAZaZXlJ2bSag3
+8l0dZMx2fZa0FALxWvY2+nd1tWcH+Wo99RSxX+4TH3guCpe1x/58yKOKL4d1xRpu
+02nGaqerV9bHU6WjCUYTrrZurTp6yqivpSFCALDMhAU5Ihd/Qjz69zg4uZ7iqbNd
+WqIYJ3SAOei2tSfoOJbdh8mohotnMFFbupck3Fv+lXAXYDp0V/sSuK32MSeJxan1
+q3t0eenAh6zoBP0Tj53N8B9YmrgZ1i1AljQqYsAdb8N5Z3vecuzQa946U8H4axEc
+BL5fgmqSfsbj1EjJVUrw3cE9hgqeqAYBBvZsyW5zSAsLq/RJKYtBxDGADqr0TOOu
+TjVqfPObb1u9EAQj15ALBhhPORa2gBAK5gnhJWW5deNgA0UOYoWARFYfATuQvr1T
+9xTVbqzZ0VJR4n7X5JyxJo2tZ44QYVzA0fqX0lt+uRkarrkbZMgplKp4MJTFCp2N
+tLciWS0YSHPbKTm7svw5mROazUvEoj1N3SYrJT+Ai9UgSL1cA4FmQ5Jsi+GOeQbH
+7E4pzjeST4b1aATY9ZZSy4TzK+UNYBcCVqY1dN5Xnk0e5yPMnrkkcYI41s/6oOaW
+lQkp/yrHkvpzQWZa7vF2yWg54j3eX8YWx/TsLvmuJmSQqiIHfu9TXcAIPDispCpF
+gp4ZbYPZ8RDQLw3ROc+bZ9E0J8e+FHwgXEELsMLW9n4d3PTj9pLFej1vmBKa03o3
+g+JViNxb1gUPNmC6ADk4o7TQzBrjV2pmmuCza16kOmq6oAbJPOeCLtVWA5HJTX8d
+AuOzBLO37tWP457dNqoNDaCMik3LCy8cslZjAz02D6VG/QGOAXP9x8bJ3zg9N7vW
+EOn1BMtIeMI4yIxpC6HgotNWDNbUBWremAFWYQRgYQqo9L1SdW4hz6X8FqdlBFvK
+ml7sdUwjEy2tNW+aiZI4cgAlI/VvMWYr5kPfhvaUu1wGYeS1TD1B+DPcjcXpOGWo
+vB1f/uvGc61zZWD0ph9f3S9OnvEko5qV3VhxOnxYUZTrqlM7LVgy/sHrSaPdR7Eo
+nb4QGdi5wEGMxVgQlDGwOmEmS5nCaA0RcfHbkQV4SebRleocLB9qcSwj5JzPICAB
+sn1qeSv45LnGdTzUnCQErw1EzgrK1FXOknujjZ3GX5dM7a6Gag0E6cAxoQeC8Mh0
+aUWolyBo9aSXltn7Vnywoe4X3v96eVH6tUQtN3wt51LBlfSDTWOQDeXLFyW57oYL
+/ErbmJLaSHpANBcC9InI+rxNYePTTba7K8rdPDppmMEmdOXjaL+5aPay2Bay9K8T
+FUyZdoyrbzxFZYT9/YyVJV/ukVcT9ru3HpzLTg9Vgc/CyRfIvCZY/Gzh+pPUbRsR
+pHYDT09BFS+mjszTkcdbceweYXJvknXpajMiISrPUUkb9rwCTmLS4QR4y7I9zAgJ
+9U72mCZB9yhhzS17kyKAANihXvOFrMXCO7QTWZbtSkobMBS2SybDxYP3rM5b6824
+HQaY4MZW5qCndydfz4fa908XbPTDka9Nu3CZfI3fhGiiGDFWViotGEYE4i9BuGZJ
+vmtzII7zeIoHwM4t5fb5mYDd9H+Rkct1yATGEIFEx7cu1fj4Ht2177EzhdkF3Q6I
+Fccn43yGDW06nFvEjnjb+Lz4EvtGp6dDefUhNBohdzd4Rm2o+etKobm0lZ1xOX74
+2gOGBLfXS6PF+msNkWP76mqKE6hG0+R1iPrN06Arti+ZRBhNGJDp1pnOERl7IyB0
+hiLUqae7xVSMVRDeq4O04EUpMXh5ixs3RJXjthYILt7vX+6qskJC3rMuyThrsvMi
+Kk0wErDpzR5U7KFnM2fk4jY5by/0b/jHZ/PWFGnaFrXNOiwK/mV1RIJtg5FCrvMv
+xJN5Qo7o2ZYzUL+IBf5mP5O49FCnz3e++9r4ySY/K57ybhQft1l+cQPWS7xuvVAg
+wZGM9DPl9AhBus0I3V3XzfkcWWHkMynMRcfTnkBHrmKjMopIOggsq05JPalM9b0V
+NfwZngVvTFbQejbGjgQQQ+G7spav/icCIA8Iu8JPHaKZt2IX77L1xMcQOouqMrx8
+JCXnJ7zsuU+8W3GQGm70H2kcen4bf4beA8TSkFmbVsaqDNtGaG+ZopWO8ctZX3CP
+jFgIYgEGi137IWvs1KD5QNFHZI6l0JMzglmvcfnViRPDHLi+EQXAoL/98j/YzA9t
+a9L4xMO0oucovKkqYOS2cdFaQKVIUM9L5ibcbW7Tol4ugMo5ibIsTgjZoyFohG6t
+acg+h2wTTyxtrIapvOOBeAo9V/iQowTAPQf38LKQZ5tApksiCG1qvJerKALTMp1J
+yUHjTCPCPsaiXVVO0mfa9OPtzVvi/icLmlz7umk82mew13djedgMS3/ghq5aEHbg
+e7rtI5snFlm/GQnV6IaCDytb+jYxqQ0DZ9Mb0CHcMpnLyiBO6nzHfadpZYlrxrpx
+dPx9MVqfLavj1UVjKuVj4Mgec9J38IBBRGMJZ4MGhde2hrw/s2jVzG1BzaIzcvFa
+bWiJOkKKRC4BY9DBDYfae5c/obwzkxLmvkbWAVzoFXc+46K1IK6bbC2a4EerOAxc
+C7B3X8BucD8OtiPORwXgjctd9AJ7cXq0EEVGoZZ8jkILnklsK6RXsxrT4gSOGFpx
+cBU7TF4kHpH+o4KGQmjlFA2VYP1tXirjEep57LOHfL+zy+HiSXdsbexf6odkqLYA
+1RR0TzSkmqJYvP0GfHPMvp2xredx1kP934fhTIUcWvErrG3NPd/Dod4BzQye1nNK
+8Y8aFYF6fACcD4SFZ11ecK9DJmB+WXGlsxtO3w1LeEysny2lVkkH6HuMRIzPyXAM
+/sEpwh3hwPJosAh6vGM6JxuL7CXP8hC1oUvK91VR0hxeDcO/LgdcHtDnlNMqt1wC
+jvYt9Hg+3aY5GehBnQPRppnIMaGhiBrRC6HjBg8Umt6UTUr5PXNY2aia5wRcx4M6
+Tj0O0dAt8WWZ9ySgzOoq4SRoaAp6m4E7CpRNMgXH031OWEeHqtt8rsYqAcvcQLuW
+vIwts5hiXZOfgXyR6SBY3ANE1SjDY46na2G4oiG8FI7eAqYgehjHrXqCnGrzjefE
+V1eVcIr7DBYuM8wq7C9x+qdMir6RNV4MrVD+7rO/drivnd3MaqDHlG0LfMZo6GIK
+QZGIy0z1W5lOXm9x3RAz7/Ea11ynkZyPDDxBE5NRcV6JfukdjDrvsaQA/Qk2Wo7h
+PdUZjv17H4syMdap6tUDTc8Cohut6n1UxnDnhaAYEDGR3TgPqI5vEaJ+yoaM9itv
+b2I/K+0uK6GfWwca7PeTtYn92/5etO4czG99xEBFzw7tiEmBTA8KfseZIUe/i3fr
+9LO+rKEGalFB4mVTYyCKPYYx4hOIvjdCox/jEBTFyYUEhgGnZvArgw9XabvII7eM
+c1FmwnN2CM/MHUzM0Kw7cS/oaoRFcG/PqioO/krL2Bd+xThwSH4Zn+Y1UVO7aXNy
+90MOehHSc2nhN984Y2re3ffVsjPhC3KdWxr8jbho/Az65rUkYg64roiO2ZqdqRvf
+aYtX9fCe8N8SzgLTlWZt+l8sjY8HP2k9NoajphV1nG4iC/pX0ZJiVQ+VBRNaGhSl
+6IhM9gM1ZsI86cULJpOTAhNu1fZBv+Mq8IXYD8SqlNVwVtjvshtlzBN4InSGEKiO
+HM7coo7aDuu+Go5AEiIVMbR09jAa0VINsCVScfSbTFhlGRTrZr2DSO/s8WxbV+Mb
+EQAfZmyrLF71Z3MfIukg+ps5iuqFhZ+gJSXXSFwumpg=
+EOF
+
+exec sh bin/do-setup-circleci-secrets

circle.yml

@@ -0,0 +1,46 @@
+general:
+  branches:
+    ignore:
+      - gh\-pages
+      - /pull\/.*/
+
+machine:
+  services:
+    - docker
+  environment:
+    GOPATH: /home/ubuntu:$GOPATH
+    SRCDIR: /home/ubuntu/src/github.com/weaveworks/weave
+    PATH: $PATH:$HOME/.local/bin:$HOME/google-cloud-sdk/bin/
+    CLOUDSDK_CORE_DISABLE_PROMPTS: 1
+    WEAVE_BUILD: $HOME/docker/weave-build.tar
+
+dependencies:
+  cache_directories:
+    - "~/docker"
+  override:
+    - if [[ -e "$WEAVE_BUILD" ]]; then
+        docker load -i $WEAVE_BUILD;
+      else
+        docker pull weaveworks/weave-build;
+        mkdir -p $(dirname "$WEAVE_BUILD");
+        docker save weaveworks/weave-build >$WEAVE_BUILD;
+      fi
+  post:
+    - curl https://sdk.cloud.google.com | bash
+    - bin/setup-circleci-secrets "$SECRET_PASSWORD"
+    - mkdir -p $(dirname $SRCDIR)
+    - cp -r $(pwd)/ $SRCDIR
+
+test:
+  override:
+    - docker run -v /var/run/docker.sock:/run/docker.sock -v /home/ubuntu:/home/go weaveworks/weave-build:
+        parallel: true
+    - cd $SRCDIR/test; ./gce.sh setup:
+        parallel: true
+    - cd $SRCDIR/test; . ./gce.sh hosts; ./setup.sh:
+        parallel: true
+    - cd $SRCDIR/test; . ./gce.sh hosts; ./run_all.sh:
+        parallel: true
+  post:
+    - cd $SRCDIR/test; ./gce.sh destroy:
+        parallel: true

common/errors.go

@@ -11,3 +11,10 @@ func CheckWarn(e error) {
 		Warning.Println(e)
 	}
 }
+
+// Assert test is true, panic otherwise
+func Assert(test bool) {
+	if !test {
+		panic("Assertion failure")
+	}
+}

common/signals.go

@@ -0,0 +1,37 @@
+package common
+
+import (
+	"os"
+	"os/signal"
+	"runtime"
+	"syscall"
+)
+
+// A subsystem/server/... that can be stopped or queried about the status with a signal
+type SignalReceiver interface {
+	Stop() error
+	Status() string
+}
+
+func SignalHandlerLoop(ss ...SignalReceiver) {
+	sigs := make(chan os.Signal, 1)
+	signal.Notify(sigs, syscall.SIGINT, syscall.SIGQUIT, syscall.SIGUSR1)
+	buf := make([]byte, 1<<20)
+	for {
+		switch <-sigs {
+		case syscall.SIGINT:
+			Info.Printf("=== received SIGINT ===\n*** exiting\n")
+			for _, subsystem := range ss {
+				subsystem.Stop()
+			}
+			return
+		case syscall.SIGQUIT:
+			stacklen := runtime.Stack(buf, true)
+			Info.Printf("=== received SIGQUIT ===\n*** goroutine dump...\n%s\n*** end\n", buf[:stacklen])
+		case syscall.SIGUSR1:
+			for _, subsystem := range ss {
+				Info.Printf("=== received SIGUSR1 ===\n*** status...\n%s\n*** end\n", subsystem.Status())
+			}
+		}
+	}
+}

docs/architecture.txt

@@ -160,7 +160,7 @@ if a node receives a packet that is bigger than the next MTU, it is
 required to drop the packet and send back an ICMP 3,4 packet which
 informs the sending node of the next hop MTU (RFC 1191). In theory,
 these ICMP packets should have all the reverse NAT and so forth
-applied, to make it all the way back to the sender. In practise this
+applied, to make it all the way back to the sender. In practice, this
 does often work, but in some cases all ICMP are sometimes blocked,
 often by idiots who don't understand that ICMP is really the error
 channel of all network traffic.
@@ -187,16 +187,16 @@ If weave determines that fragmentation is reliable then when weave
 sniffs large packets (i.e. packets larger than the MTU and thus don't
 have DF set) then weave will encapsulate these packets as necessary
 and similarly send them out as-is, without DF set, trusting to the
-network to do all necessary fragementation and reassembly.
+network to do all necessary fragmentation and reassembly.
 
-However, if weave determines that fragementation is not reliable
+However, if weave determines that fragmentation is not reliable
 between any two peers then it will manually fragment larger packets
 correctly according to the IP spec, and will then send them between
 weave peers with the DF flag set (i.e. the fragmentation will ensure
-that the encapsulatated traffic will not be greater than the PMTU
+that the encapsulated traffic will not be greater than the PMTU
 between weave peers). Because the fragmentation is done according to
 the IP spec, we don't need to do reassembly ourselves - on the
-receiving weave, we jus inject all the fragments and rely on the
+receiving weave, we just inject all the fragments and rely on the
 stack to do reassembly.
 
 Sometimes PMTU discovery doesn't work as ICMP packets sometimes get

docs/ipam.md

@@ -0,0 +1,214 @@
+See the [requirements](https://github.com/zettio/weave/wiki/IP-allocation-requirements).
+
+At its highest level, the idea is that we start with a certain IP
+address space, known to all peers, and divide it up between the
+peers. This allows peers to allocate and free individual IPs locally
+until they run out.
+
+We use a CRDT to represent shared knowledge about the space,
+transmitted over the Weave Gossip mechanism, together with
+point-to-point messages for one peer to request more space from
+another.
+
+The allocator running at each peer also has an http interface which
+the container infrastructure (e.g. the Weave script, or a Docker
+plugin) uses to request IP addresses.
+
+![Schematic of IP allocation in operation](https://docs.google.com/drawings/d/1-EUIRKYxwfKTpBJ7v_LMcdvSpodIMSz4lT3wgEfWKl4/pub?w=701&h=310)
+
+## Commands
+
+The commands supported by the allocator via the http interface are:
+
+- Allocate: request one IP address for a container
+- Free: return an IP address that is currently allocated
+- Claim: request a specific IP address for a container (e.g. because
+  it is already using that address)
+
+The allocator also watches via the Docker event mechanism: if a
+container dies then all IP addresses allocated to that container are
+freed.
+
+## Definitions
+
+1. Allocations. We use the word 'allocation' to refer to a specific
+   IP address being assigned, e.g. so it can be given to a container.
+
+2. Range. Most of the time, instead of dealing with individual IP
+   addresses, we operate on them in contiguous groups, for which we
+   use the word "range".
+
+3. Ring. We consider the address space as a ring, so ranges wrap
+   around from the highest address to the lowest address.
+
+4. Peer. A Peer is a node on the Weave network. It can own zero or
+   more ranges.
+
+### The Allocation Process
+
+When a peer owns some range(s), it can allocate freely from within
+those ranges to containers on the same machine. If it runs out of
+space (all owned ranges are full), it will ask another peer for space:
+  - it picks a peer to ask at random, weighted by the amount of space
+    owned by each peer
+    - if the target peer decides to give up space, it unicasts a message
+      back to the asker with the newly-updated ring.
+    - if the target peer has no space, it unicasts a message back to the
+      asker with its current copy of the ring, on the basis that the
+      requestor must have acted on out-of-date information.
+  - it will continue to ask for space until it receives some, or its
+    copy of the ring tells it all peers are full.
+
+### Claiming an address
+
+If a Weave process is restarted, in most cases it will hear from
+another peer which ranges it used to own, but it needs to know which
+individual IP addresses are assigned to containers in order to avoid
+giving the same address out in subsequent allocation requests. The
+weave script invokes the `claim` command in `populate_ipam` to do
+this.
+
+When the Allocator is told to claim an address, there are four
+possibilities:
+  - the address is outside of the space managed by this Allocator, in
+    which case we ignore the request.
+  - the address is owned by this peer, in which case we record the
+    address as being assigned to a particular container and return
+    success.
+  - the address is owned by a different peer, in which case we return
+    failure.
+  - we have not yet heard of any address ranges being owned by anyone,
+    in which case we wait until we do hear.
+
+This approach fails if the peer does not hear from another peer about
+the ranges it used to own, e.g. if all peers in a network partition
+are restarted at once.
+
+### The Ring CRDT
+
+We use a Convergent Replicated Data Type - a CRDT - so that peers can
+make changes concurrently and communicate them without central
+coordination. To achieve this, we arrange that peers only make changes
+to the data structure in ranges that they own (except under
+administrator command - see later).
+
+The data structure is a set of tokens, each containing the name of an
+owning peer. A peer can own many ranges. Each token is placed at the
+start address of a range, and the set is kept ordered so each range
+goes from one token to the next. Each range on the ring includes the
+start address but does not include the end address (which is the start
+of the next range).  Ranges wrap, so the 'next' token after the last
+one is the first token.
+
+![Tokens on the Ring](https://docs.google.com/drawings/d/1hp--q2vmxbBAnPjhza4Kqjr1ugrw2iS1M1GerhH-IKY/pub?w=960&h=288)
+
+In more detail:
+- Each token is a tuple {peer name, version}, placed
+  at an IP address.
+- Peer names are taken from Weave: they are unique and survive across restarts.
+- The contents of a token can only be updated by the owning peer, and
+  when this is done the version is incremented
+- The ring data structure is always gossiped in its entirety
+- The merge operation when a peer receives a ring via gossip is:
+  - Tokens with unique addresses are just copied into the combined ring
+  - For tokens at the same address, pick the one with the highest
+    version number
+- The data gossiped about the ring also includes the amount of free
+  space in each range: this is not essential but it improves the
+  selection of which peer to ask for space.
+- When a peer is asked for space, there are four scenarios:
+  1. It has an empty range; it can change the peer associated with
+     the token at the beginning of the range and increment the version.
+  2. It has a range which can be subdivided by a single token to form
+     a free range.  It inserts said token, owned by the peer requesting
+     space.
+  3. It has a 'hole' in the middle of a range; an empty range can be
+     created by inserting two tokens, one at the beginning of the hole
+     owned by the peer requesting the space, and one at the end of the
+     hole owned by the requestee.
+  4. It has no space.
+
+## Initialisation
+
+The previous sections describe how peers coordinate changes to the
+ring.  But how is the initial state of the ring established?  If a new
+peer joins a long-standing cluster, it can learn about the ring state
+from other peers.  But in a freshly started cluster, the initial ring
+state must be established from a clean slate.  And it must be
+consistent across all peers.  This is a distributed consensus problem,
+and we solve it using the Paxos algorithm.
+
+Although Paxos has a reputation for being hard to understand and to
+implement, the implementation used for ring initialization is
+relatively straightforward:
+
+- We only need to establish consensus for a single value, rather than
+  a succession of values (as in a replicated transaction log).  Thus
+  we implement basic Paxos, rather than multi-Paxos, and the
+  implementation closely follows the outline of basic Paxos described
+  [on
+  wikipedia](http://en.wikipedia.org/wiki/Paxos_%28computer_science%29#Basic_Paxos).
+
+- All peers play all the roles described in basic Paxos: Proposer,
+  acceptor and listener.
+
+- Paxos is usually described in terms of unicast messages from one
+  agent to another, although three of the four messages are fanned out
+  to all agents in a certain role ("prepare" and "accept request" from
+  a proposer to all acceptors; "accepted" from an acceptor to all
+  learners).  So most implementations involve a communications layer
+  to implement the required communication patterns.  But our
+  implementation is built on top of the weave gossip layer, which
+  gives us a broadcast medium.  Using broadcast for the one truly
+  unicast message ("promise") may seem wasteful, but as we only
+  establish consensus for a single value, it is not a major concern.
+
+The value the peers obtain consensus on is the set of peers that will
+be represented in the initial ring (with each peer getting an equal
+share of the address space).  When a proposing peer gets to choose the
+value, it includes all the peers it has heard from during the Paxos
+phase, which is at least a quorum.
+
+Once a consensus is reached, it is used to initialize the ring.  If a
+peer hears about an initialized ring via gossip, that implies that a
+consensus was reached, so it will stop participating in Paxos.
+
+Paxos requires that we know the quorum size for the cluster.  As weave
+clusters are a dynamic entity, there is no fixed quorum size.  But the
+Paxos phase used to initialize the ring should normally be
+short-lived, so we just need to know the initial cluster size.  A
+heuristic is used to derive this from the list of addresses passed to
+`weave launch`, but the user can set it explicitly.
+
+## Peer shutdown
+
+When a peer leaves (a `weave reset` command), it grants all its own
+tokens to another peer, then broadcasts the updated ring.
+
+After sending the message, the peer terminates - it does not wait for
+any response.
+
+Failures:
+- message lost
+  - the space will be unusable by other peers because it will still be
+    seen as owned.
+
+To cope with the situation where a peer has left or died without
+managing to tell its peers, an administrator may go to any other peer
+and command that it take over the dead peer's tokens (with `weave
+rmpeer`).  This information will then be gossipped out to the network.
+
+(This operation is not guaranteed to be safe - if the dead peer made
+some transfers which have are known to other peers but have not
+reached the peer that does the `rmpeer`, then we get an inconsistent
+ring.  We could make it safe, up to partitions, by inquiring of every
+live peer what its view of the dead peer is, and making sure we use
+the latest info.)
+
+## Limitations
+
+Nothing is persisted. If one peer restarts it should receive gossip
+from other peers showing what it used to own, and in that case it can
+recover quite well. If, however, there are no peers left alive, or
+this peer cannot establish network communication with those that are,
+then it cannot recover.

ipam/address/address.go

@@ -0,0 +1,61 @@
+package address
+
+import (
+	"github.com/weaveworks/weave/common"
+	"net"
+)
+
+// Using 32-bit integer to represent IPv4 address
+type Address uint32
+type Offset uint32
+
+type Range struct {
+	Start, End Address // [Start, End); Start <= End
+}
+
+func ParseIP(s string) (Address, error) {
+	if ip := net.ParseIP(s); ip == nil {
+		return 0, &net.ParseError{Type: "IP Address", Text: s}
+	} else {
+		return FromIP4(ip), nil
+	}
+}
+
+// FromIP4 converts an ipv4 address to our integer address type
+func FromIP4(ip4 net.IP) (r Address) {
+	for _, b := range ip4.To4() {
+		r <<= 8
+		r |= Address(b)
+	}
+	return
+}
+
+// IP4 converts our integer address type to an ipv4 address
+func (addr Address) IP4() (r net.IP) {
+	r = make([]byte, net.IPv4len)
+	for i := 3; i >= 0; i-- {
+		r[i] = byte(addr)
+		addr >>= 8
+	}
+	return
+}
+
+func (addr Address) String() string {
+	return addr.IP4().String()
+}
+
+func Add(addr Address, i Offset) Address {
+	return addr + Address(i)
+}
+
+func Subtract(a, b Address) Offset {
+	common.Assert(a >= b)
+	return Offset(a - b)
+}
+
+func Min(a, b Offset) Offset {
+	if a > b {
+		return b
+	}
+	return a
+}

ipam/allocate.go

@@ -0,0 +1,58 @@
+package ipam
+
+import (
+	"fmt"
+	"github.com/weaveworks/weave/ipam/address"
+)
+
+type allocateResult struct {
+	addr address.Address
+	err  error
+}
+
+type allocate struct {
+	resultChan       chan<- allocateResult
+	hasBeenCancelled func() bool
+	ident            string
+}
+
+// Try returns true if the request is completed, false if pending
+func (g *allocate) Try(alloc *Allocator) bool {
+	if g.hasBeenCancelled() {
+		g.Cancel()
+		return true
+	}
+
+	// If we have previously stored an address for this container, return it.
+	if addr, found := alloc.owned[g.ident]; found {
+		g.resultChan <- allocateResult{addr, nil}
+		return true
+	}
+
+	if ok, addr := alloc.space.Allocate(); ok {
+		alloc.debugln("Allocated", addr, "for", g.ident)
+		alloc.addOwned(g.ident, addr)
+		g.resultChan <- allocateResult{addr, nil}
+		return true
+	}
+
+	// out of space
+	if donor, err := alloc.ring.ChoosePeerToAskForSpace(); err == nil {
+		alloc.debugln("Decided to ask peer", donor, "for space")
+		alloc.sendRequest(donor, msgSpaceRequest)
+	}
+
+	return false
+}
+
+func (g *allocate) Cancel() {
+	g.resultChan <- allocateResult{0, fmt.Errorf("Allocate request for %s cancelled", g.ident)}
+}
+
+func (g *allocate) String() string {
+	return fmt.Sprintf("Allocate for %s", g.ident)
+}
+
+func (g *allocate) ForContainer(ident string) bool {
+	return g.ident == ident
+}

ipam/allocator.go

@@ -0,0 +1,696 @@
+package ipam
+
+import (
+	"bytes"
+	"encoding/gob"
+	"errors"
+	"fmt"
+	"net"
+	"sort"
+	"time"
+
+	"github.com/weaveworks/weave/common"
+	"github.com/weaveworks/weave/ipam/address"
+	"github.com/weaveworks/weave/ipam/paxos"
+	"github.com/weaveworks/weave/ipam/ring"
+	"github.com/weaveworks/weave/ipam/space"
+	"github.com/weaveworks/weave/router"
+)
+
+// Kinds of message we can unicast to other peers
+const (
+	msgSpaceRequest = iota
+	msgRingUpdate
+
+	paxosInterval = time.Second * 5
+)
+
+// operation represents something which Allocator wants to do, but
+// which may need to wait until some other message arrives.
+type operation interface {
+	// Try attempts this operations and returns false if needs to be tried again.
+	Try(alloc *Allocator) bool
+
+	Cancel()
+
+	String() string
+
+	// Does this operation pertain to the given container id?
+	// Used for tidying up pending operations when containers die.
+	ForContainer(ident string) bool
+}
+
+// Allocator brings together Ring and space.Set, and does the
+// necessary plumbing.  Runs as a single-threaded Actor, so no locks
+// are used around data structures.
+type Allocator struct {
+	actionChan       chan<- func()
+	ourName          router.PeerName
+	subnetStart      address.Address            // start address of space all peers are allocating from
+	subnetSize       address.Offset             // length of space all peers are allocating from
+	prefixLen        int                        // network prefix length, e.g. 24 for a /24 network
+	ring             *ring.Ring                 // information on ranges owned by all peers
+	space            space.Space                // more detail on ranges owned by us
+	owned            map[string]address.Address // who owns what address, indexed by container-ID
+	nicknames        map[router.PeerName]string // so we can map nicknames for rmpeer
+	pendingAllocates []operation                // held until we get some free space
+	pendingClaims    []operation                // held until we know who owns the space
+	gossip           router.Gossip              // our link to the outside world for sending messages
+	paxos            *paxos.Node
+	paxosTicker      *time.Ticker
+	shuttingDown     bool // to avoid doing any requests while trying to shut down
+	now              func() time.Time
+}
+
+// NewAllocator creates and initialises a new Allocator
+func NewAllocator(ourName router.PeerName, ourUID router.PeerUID, ourNickname string, subnetCIDR string, quorum uint) (*Allocator, error) {
+	_, subnet, err := net.ParseCIDR(subnetCIDR)
+	if err != nil {
+		return nil, err
+	}
+	if subnet.IP.To4() == nil {
+		return nil, errors.New("Non-IPv4 address not supported")
+	}
+	// Get the size of the network from the mask
+	ones, bits := subnet.Mask.Size()
+	var subnetSize address.Offset = 1 << uint(bits-ones)
+	if subnetSize < 4 {
+		return nil, errors.New("Allocation subnet too small")
+	}
+	subnetStart := address.FromIP4(subnet.IP)
+	alloc := &Allocator{
+		ourName:     ourName,
+		subnetStart: subnetStart,
+		subnetSize:  subnetSize,
+		prefixLen:   ones,
+		// per RFC 1122, don't allocate the first and last address in the subnet
+		ring:      ring.New(address.Add(subnetStart, 1), address.Add(subnetStart, subnetSize-1), ourName),
+		owned:     make(map[string]address.Address),
+		paxos:     paxos.NewNode(ourName, ourUID, quorum),
+		nicknames: map[router.PeerName]string{ourName: ourNickname},
+		now:       time.Now,
+	}
+	return alloc, nil
+}
+
+// Start runs the allocator goroutine
+func (alloc *Allocator) Start() {
+	actionChan := make(chan func(), router.ChannelSize)
+	alloc.actionChan = actionChan
+	go alloc.actorLoop(actionChan)
+}
+
+// Stop makes the actor routine exit, for test purposes ONLY because any
+// calls after this is processed will hang. Async.
+func (alloc *Allocator) Stop() {
+	alloc.actionChan <- nil
+}
+
+// Operation life cycle
+
+// Given an operation, try it, and add it to the pending queue if it didn't succeed
+func (alloc *Allocator) doOperation(op operation, ops *[]operation) {
+	alloc.actionChan <- func() {
+		if alloc.shuttingDown {
+			op.Cancel()
+			return
+		}
+		alloc.establishRing()
+		if !op.Try(alloc) {
+			*ops = append(*ops, op)
+		}
+	}
+}
+
+// Given an operation, remove it from the pending queue
+//  Note the op may not be on the queue; it may have
+//  already succeeded.  If it is on the queue, we call
+//  cancel on it, allowing callers waiting for the resultChans
+//  to unblock.
+func (alloc *Allocator) cancelOp(op operation, ops *[]operation) {
+	for i, op := range *ops {
+		if op == op {
+			*ops = append((*ops)[:i], (*ops)[i+1:]...)
+			op.Cancel()
+			break
+		}
+	}
+}
+
+// Cancel all operations in a queue
+func (alloc *Allocator) cancelOps(ops *[]operation) {
+	for _, op := range *ops {
+		op.Cancel()
+	}
+	*ops = []operation{}
+}
+
+// Cancel all operations for a given container id, returns true
+// if we found any.
+func (alloc *Allocator) cancelOpsFor(ops *[]operation, ident string) bool {
+	var found bool
+	for i, op := range *ops {
+		if op.ForContainer(ident) {
+			found = true
+			op.Cancel()
+			*ops = append((*ops)[:i], (*ops)[i+1:]...)
+		}
+	}
+	return found
+}
+
+// Try all pending operations
+func (alloc *Allocator) tryPendingOps() {
+	// The slightly different semantics requires us to operate on 'claims' and
+	// 'allocates' separately:
+	// Claims must be tried before Allocates
+	for i := 0; i < len(alloc.pendingClaims); {
+		op := alloc.pendingClaims[i]
+		if !op.Try(alloc) {
+			i++
+			continue
+		}
+		alloc.pendingClaims = append(alloc.pendingClaims[:i], alloc.pendingClaims[i+1:]...)
+	}
+
+	// When the first Allocate fails, bail - no need to
+	// send too many begs for space.
+	for i := 0; i < len(alloc.pendingAllocates); {
+		op := alloc.pendingAllocates[i]
+		if !op.Try(alloc) {
+			break
+		}
+		alloc.pendingAllocates = append(alloc.pendingAllocates[:i], alloc.pendingAllocates[i+1:]...)
+	}
+}
+
+func hasBeenCancelled(cancelChan <-chan bool) func() bool {
+	return func() bool {
+		select {
+		case <-cancelChan:
+			return true
+		default:
+			return false
+		}
+	}
+}
+
+// Actor client API
+
+// Allocate (Sync) - get IP address for container with given name
+// if there isn't any space we block indefinitely
+func (alloc *Allocator) Allocate(ident string, cancelChan <-chan bool) (address.Address, error) {
+	resultChan := make(chan allocateResult)
+	op := &allocate{resultChan: resultChan, ident: ident,
+		hasBeenCancelled: hasBeenCancelled(cancelChan)}
+	alloc.doOperation(op, &alloc.pendingAllocates)
+	result := <-resultChan
+	return result.addr, result.err
+}
+
+// Claim an address that we think we should own (Sync)
+func (alloc *Allocator) Claim(ident string, addr address.Address, cancelChan <-chan bool) error {
+	resultChan := make(chan error)
+	op := &claim{resultChan: resultChan, ident: ident, addr: addr,
+		hasBeenCancelled: hasBeenCancelled(cancelChan)}
+	alloc.doOperation(op, &alloc.pendingClaims)
+	return <-resultChan
+}
+
+// Free (Sync) - release IP address for container with given name
+func (alloc *Allocator) Free(ident string) error {
+	return alloc.free(ident)
+}
+
+// ContainerDied is provided to satisfy the updater interface; does a free underneath.  Async.
+func (alloc *Allocator) ContainerDied(ident string) error {
+	alloc.debugln("Container", ident, "died; releasing addresses")
+	return alloc.free(ident)
+}
+
+func (alloc *Allocator) free(ident string) error {
+	errChan := make(chan error)
+	alloc.actionChan <- func() {
+		addr, found := alloc.owned[ident]
+		if found {
+			alloc.space.Free(addr)
+		}
+		delete(alloc.owned, ident)
+
+		// Also remove any pending ops
+		found = alloc.cancelOpsFor(&alloc.pendingAllocates, ident) || found
+		found = alloc.cancelOpsFor(&alloc.pendingClaims, ident) || found
+
+		if !found {
+			errChan <- fmt.Errorf("Free: no addresses for %s", ident)
+			return
+		}
+		errChan <- nil
+	}
+	return <-errChan
+}
+
+// Sync.
+func (alloc *Allocator) String() string {
+	resultChan := make(chan string)
+	alloc.actionChan <- func() {
+		resultChan <- alloc.string()
+	}
+	return <-resultChan
+}
+
+// Shutdown (Sync)
+func (alloc *Allocator) Shutdown() {
+	alloc.infof("Shutdown")
+	doneChan := make(chan struct{})
+	alloc.actionChan <- func() {
+		alloc.shuttingDown = true
+		alloc.cancelOps(&alloc.pendingClaims)
+		alloc.cancelOps(&alloc.pendingAllocates)
+		if heir := alloc.ring.PickPeerForTransfer(); heir != router.UnknownPeerName {
+			alloc.ring.Transfer(alloc.ourName, heir)
+			alloc.space.Clear()
+			alloc.gossip.GossipBroadcast(alloc.Gossip())
+			time.Sleep(100 * time.Millisecond)
+		}
+		doneChan <- struct{}{}
+	}
+	<-doneChan
+}
+
+// AdminTakeoverRanges (Sync) - take over the ranges owned by a given peer.
+// Only done on adminstrator command.
+func (alloc *Allocator) AdminTakeoverRanges(peerNameOrNickname string) error {
+	resultChan := make(chan error)
+	alloc.actionChan <- func() {
+		peername, err := alloc.lookupPeername(peerNameOrNickname)
+		if err != nil {
+			resultChan <- fmt.Errorf("Cannot find peer '%s'", peerNameOrNickname)
+			return
+		}
+
+		alloc.debugln("AdminTakeoverRanges:", peername)
+		if peername == alloc.ourName {
+			resultChan <- fmt.Errorf("Cannot take over ranges from yourself!")
+			return
+		}
+
+		delete(alloc.nicknames, peername)
+		err, newRanges := alloc.ring.Transfer(peername, alloc.ourName)
+		alloc.space.AddRanges(newRanges)
+		resultChan <- err
+	}
+	return <-resultChan
+}
+
+// Lookup a PeerName by nickname or stringified PeerName.  We can't
+// call into the router for this because we are interested in peers
+// that have gone away but are still in the ring, which is why we
+// maintain our own nicknames map.
+func (alloc *Allocator) lookupPeername(name string) (router.PeerName, error) {
+	for peername, nickname := range alloc.nicknames {
+		if nickname == name {
+			return peername, nil
+		}
+	}
+
+	return router.PeerNameFromString(name)
+}
+
+// Restrict the peers in "nicknames" to those in the ring and our own
+func (alloc *Allocator) pruneNicknames() {
+	ringPeers := alloc.ring.PeerNames()
+	for name, _ := range alloc.nicknames {
+		if _, ok := ringPeers[name]; !ok && name != alloc.ourName {
+			delete(alloc.nicknames, name)
+		}
+	}
+}
+
+// OnGossipUnicast (Sync)
+func (alloc *Allocator) OnGossipUnicast(sender router.PeerName, msg []byte) error {
+	alloc.debugln("OnGossipUnicast from", sender, ": ", len(msg), "bytes")
+	resultChan := make(chan error)
+	alloc.actionChan <- func() {
+		switch msg[0] {
+		case msgSpaceRequest:
+			// some other peer asked us for space
+			alloc.donateSpace(sender)
+			resultChan <- nil
+		case msgRingUpdate:
+			resultChan <- alloc.update(msg[1:])
+		}
+	}
+	return <-resultChan
+}
+
+// OnGossipBroadcast (Sync)
+func (alloc *Allocator) OnGossipBroadcast(msg []byte) (router.GossipData, error) {
+	alloc.debugln("OnGossipBroadcast:", len(msg), "bytes")
+	resultChan := make(chan error)
+	alloc.actionChan <- func() {
+		resultChan <- alloc.update(msg)
+	}
+	return alloc.Gossip(), <-resultChan
+}
+
+type gossipState struct {
+	// We send a timstamp along with the information to be
+	// gossipped in order to do detect skewed clocks
+	Now       int64
+	Nicknames map[router.PeerName]string
+
+	Paxos paxos.GossipState
+	Ring  *ring.Ring
+}
+
+func (alloc *Allocator) encode() []byte {
+	data := gossipState{
+		Now:       alloc.now().Unix(),
+		Nicknames: alloc.nicknames,
+	}
+
+	// We're only interested in Paxos until we have a Ring.
+	if alloc.ring.Empty() {
+		data.Paxos = alloc.paxos.GossipState()
+	} else {
+		data.Ring = alloc.ring
+	}
+	buf := new(bytes.Buffer)
+	enc := gob.NewEncoder(buf)
+	if err := enc.Encode(data); err != nil {
+		panic(err)
+	}
+	return buf.Bytes()
+}
+
+// Encode (Sync)
+func (alloc *Allocator) Encode() []byte {
+	resultChan := make(chan []byte)
+	alloc.actionChan <- func() {
+		resultChan <- alloc.encode()
+	}
+	return <-resultChan
+}
+
+// OnGossip (Sync)
+func (alloc *Allocator) OnGossip(msg []byte) (router.GossipData, error) {
+	alloc.debugln("Allocator.OnGossip:", len(msg), "bytes")
+	resultChan := make(chan error)
+	alloc.actionChan <- func() {
+		resultChan <- alloc.update(msg)
+	}
+	return nil, <-resultChan // for now, we never propagate updates. TBD
+}
+
+// GossipData implementation is trivial - we always gossip the latest
+// data we have at time of sending
+type ipamGossipData struct {
+	alloc *Allocator
+}
+
+func (d *ipamGossipData) Merge(other router.GossipData) {
+	// no-op
+}
+
+func (d *ipamGossipData) Encode() []byte {
+	return d.alloc.Encode()
+}
+
+// Gossip returns a GossipData implementation, which in this case always
+// returns the latest ring state (and does nothing on merge)
+func (alloc *Allocator) Gossip() router.GossipData {
+	return &ipamGossipData{alloc}
+}
+
+// SetInterfaces gives the allocator two interfaces for talking to the outside world
+func (alloc *Allocator) SetInterfaces(gossip router.Gossip) {
+	alloc.gossip = gossip
+}
+
+// ACTOR server
+
+func (alloc *Allocator) actorLoop(actionChan <-chan func()) {
+	for {
+		var tickChan <-chan time.Time
+		if alloc.paxosTicker != nil {
+			tickChan = alloc.paxosTicker.C
+		}
+
+		select {
+		case action := <-actionChan:
+			if action == nil {
+				return
+			}
+			action()
+		case <-tickChan:
+			alloc.propose()
+		}
+
+		alloc.assertInvariants()
+		alloc.reportFreeSpace()
+	}
+}
+
+// Helper functions
+
+func (alloc *Allocator) string() string {
+	var buf bytes.Buffer
+	fmt.Fprintf(&buf, "Allocator subnet %s/%d", alloc.subnetStart.String(), alloc.prefixLen)
+
+	if alloc.ring.Empty() {
+		if alloc.paxosTicker != nil {
+			fmt.Fprintf(&buf, " awaiting consensus: %s", alloc.paxos.String())
+		}
+	} else {
+		localFreeSpace := alloc.space.NumFreeAddresses()
+		remoteFreeSpace := alloc.ring.TotalRemoteFree()
+		percentFree := 100 * float64(localFreeSpace+remoteFreeSpace) / float64(alloc.subnetSize)
+		fmt.Fprintf(&buf, "  Free IPs: ~%.1f%%, %d local, ~%d remote\n",
+			percentFree, localFreeSpace, remoteFreeSpace)
+
+		fmt.Fprint(&buf, "Owned Ranges:")
+		alloc.ring.FprintWithNicknames(&buf, alloc.nicknames)
+	}
+	if len(alloc.pendingAllocates)+len(alloc.pendingClaims) > 0 {
+		fmt.Fprintf(&buf, "\nPending requests:")
+		for _, op := range alloc.pendingAllocates {
+			fmt.Fprintf(&buf, "\n  %s", op.String())
+		}
+		for _, op := range alloc.pendingClaims {
+			fmt.Fprintf(&buf, "\n  %s", op.String())
+		}
+	}
+	return buf.String()
+}
+
+func (alloc *Allocator) consensus() []router.PeerName {
+	_, val := alloc.paxos.Consensus()
+	return val.Value
+}
+
+// Ensure we are making progress towards an established ring
+func (alloc *Allocator) establishRing() {
+	if !alloc.ring.Empty() || alloc.paxosTicker != nil {
+		return
+	}
+
+	alloc.propose()
+	if cons := alloc.consensus(); cons != nil {
+		// If the quorum was 1, then proposing immediately
+		// leads to consensus
+		alloc.createRing(cons)
+	} else {
+		// re-try until we get consensus
+		alloc.paxosTicker = time.NewTicker(paxosInterval)
+	}
+}
+
+func (alloc *Allocator) createRing(peers []router.PeerName) {
+	alloc.debugln("Paxos consensus:", peers)
+	alloc.ring.ClaimForPeers(normalizeConsensus(peers))
+	alloc.gossip.GossipBroadcast(alloc.Gossip())
+	alloc.ringUpdated()
+}
+
+func (alloc *Allocator) ringUpdated() {
+	// When we have a ring, we don't need paxos any more
+	if alloc.paxos != nil {
+		alloc.paxos = nil
+
+		if alloc.paxosTicker != nil {
+			alloc.paxosTicker.Stop()
+			alloc.paxosTicker = nil
+		}
+	}
+
+	alloc.space.UpdateRanges(alloc.ring.OwnedRanges())
+	alloc.tryPendingOps()
+}
+
+// For compatibility with sort.Interface
+type peerNames []router.PeerName
+
+func (a peerNames) Len() int           { return len(a) }
+func (a peerNames) Less(i, j int) bool { return a[i] < a[j] }
+func (a peerNames) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
+
+// When we get a consensus from Paxos, the peer names are not in a
+// defined order and may contain duplicates.  This function sorts them
+// and de-dupes.
+func normalizeConsensus(consensus []router.PeerName) []router.PeerName {
+	if len(consensus) == 0 {
+		return nil
+	}
+
+	peers := make(peerNames, len(consensus))
+	copy(peers, consensus)
+	sort.Sort(peers)
+
+	dst := 0
+	for src := 1; src < len(peers); src++ {
+		if peers[dst] != peers[src] {
+			dst++
+			peers[dst] = peers[src]
+		}
+	}
+
+	return peers[:dst+1]
+}
+
+func (alloc *Allocator) propose() {
+	alloc.debugf("Paxos proposing")
+	alloc.paxos.Propose()
+	alloc.gossip.GossipBroadcast(alloc.Gossip())
+}
+
+func (alloc *Allocator) sendRequest(dest router.PeerName, kind byte) {
+	msg := router.Concat([]byte{kind}, alloc.encode())
+	alloc.gossip.GossipUnicast(dest, msg)
+}
+
+func (alloc *Allocator) update(msg []byte) error {
+	reader := bytes.NewReader(msg)
+	decoder := gob.NewDecoder(reader)
+	var data gossipState
+	var err error
+
+	if err := decoder.Decode(&data); err != nil {
+		return err
+	}
+
+	deltat := time.Unix(data.Now, 0).Sub(alloc.now())
+	if deltat > time.Hour || -deltat > time.Hour {
+		return fmt.Errorf("clock skew of %v detected, ignoring update", deltat)
+	}
+
+	// Merge nicknames
+	for peer, nickname := range data.Nicknames {
+		alloc.nicknames[peer] = nickname
+	}
+
+	// only one of Ring and Paxos should be present.  And we
+	// shouldn't get updates for a empty Ring. But tolerate
+	// them just in case.
+	if data.Ring != nil {
+		err = alloc.ring.Merge(*data.Ring)
+		if !alloc.ring.Empty() {
+			alloc.pruneNicknames()
+			alloc.ringUpdated()
+		}
+		return err
+	}
+
+	if data.Paxos != nil && alloc.ring.Empty() {
+		if alloc.paxos.Update(data.Paxos) {
+			if alloc.paxos.Think() {
+				// If something important changed, broadcast
+				alloc.gossip.GossipBroadcast(alloc.Gossip())
+			}
+
+			if cons := alloc.consensus(); cons != nil {
+				alloc.createRing(cons)
+			}
+		}
+	}
+
+	return nil
+}
+
+func (alloc *Allocator) donateSpace(to router.PeerName) {
+	// No matter what we do, we'll send a unicast gossip
+	// of our ring back to tha chap who asked for space.
+	// This serves to both tell him of any space we might
+	// have given him, or tell him where he might find some
+	// more.
+	defer alloc.sendRequest(to, msgRingUpdate)
+
+	alloc.debugln("Peer", to, "asked me for space")
+	start, size, ok := alloc.space.Donate()
+	if !ok {
+		free := alloc.space.NumFreeAddresses()
+		common.Assert(free == 0)
+		alloc.debugln("No space to give to peer", to)
+		return
+	}
+	end := address.Add(start, size)
+	alloc.debugln("Giving range", start, end, size, "to", to)
+	alloc.ring.GrantRangeToHost(start, end, to)
+}
+
+func (alloc *Allocator) assertInvariants() {
+	// We need to ensure all ranges the ring thinks we own have
+	// a corresponding space in the space set, and vice versa
+	checkSpace := space.New()
+	checkSpace.AddRanges(alloc.ring.OwnedRanges())
+	ranges := checkSpace.OwnedRanges()
+	spaces := alloc.space.OwnedRanges()
+
+	common.Assert(len(ranges) == len(spaces))
+
+	for i := 0; i < len(ranges); i++ {
+		r := ranges[i]
+		s := spaces[i]
+		common.Assert(s.Start == r.Start && s.End == r.End)
+	}
+}
+
+func (alloc *Allocator) reportFreeSpace() {
+	ranges := alloc.ring.OwnedRanges()
+	if len(ranges) == 0 {
+		return
+	}
+
+	freespace := make(map[address.Address]address.Offset)
+	for _, r := range ranges {
+		freespace[r.Start] = alloc.space.NumFreeAddressesInRange(r.Start, r.End)
+	}
+	alloc.ring.ReportFree(freespace)
+}
+
+// Owned addresses
+
+func (alloc *Allocator) addOwned(ident string, addr address.Address) {
+	alloc.owned[ident] = addr
+}
+
+func (alloc *Allocator) findOwner(addr address.Address) string {
+	for ident, candidate := range alloc.owned {
+		if candidate == addr {
+			return ident
+		}
+	}
+	return ""
+}
+
+// Logging
+
+func (alloc *Allocator) infof(fmt string, args ...interface{}) {
+	common.Info.Printf("[allocator %s] "+fmt, append([]interface{}{alloc.ourName}, args...)...)
+}
+func (alloc *Allocator) debugln(args ...interface{}) {
+	common.Debug.Println(append([]interface{}{fmt.Sprintf("[allocator %s]:", alloc.ourName)}, args...)...)
+}
+func (alloc *Allocator) debugf(fmt string, args ...interface{}) {
+	common.Debug.Printf("[allocator %s] "+fmt, append([]interface{}{alloc.ourName}, args...)...)
+}

ipam/allocator_test.go

@@ -0,0 +1,453 @@
+package ipam
+
+import (
+	"fmt"
+	"math/rand"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/weaveworks/weave/common"
+	"github.com/weaveworks/weave/ipam/address"
+	"github.com/weaveworks/weave/router"
+	wt "github.com/weaveworks/weave/testing"
+)
+
+const (
+	testStart1 = "10.0.1.0"
+	testStart2 = "10.0.2.0"
+	testStart3 = "10.0.3.0"
+)
+
+func TestAllocFree(t *testing.T) {
+	const (
+		container1 = "abcdef"
+		container2 = "baddf00d"
+		container3 = "b01df00d"
+		universe   = "10.0.3.0/28"
+		testAddr1  = "10.0.3.1" // first address allocated should be .1 because .0 is network addr
+		spaceSize  = 14         // 16 IP addresses in /28, minus .0 and .15
+	)
+
+	alloc := makeAllocatorWithMockGossip(t, "01:00:00:01:00:00", universe, 1)
+	defer alloc.Stop()
+
+	alloc.claimRingForTesting()
+	addr1, _ := alloc.Allocate(container1, nil)
+	wt.AssertEqualString(t, addr1.String(), testAddr1, "address")
+
+	// Ask for another address for a different container and check it's different
+	addr2, _ := alloc.Allocate(container2, nil)
+	if addr2.String() == testAddr1 {
+		t.Fatalf("Expected different address but got %s", addr2.String())
+	}
+
+	// Ask for the first container again and we should get the same address again
+	addr1a, _ := alloc.Allocate(container1, nil)
+	wt.AssertEqualString(t, addr1a.String(), testAddr1, "address")
+
+	// Now free the first one, and we should get it back when we ask
+	wt.AssertSuccess(t, alloc.Free(container1))
+	addr3, _ := alloc.Allocate(container3, nil)
+	wt.AssertEqualString(t, addr3.String(), testAddr1, "address")
+
+	alloc.ContainerDied(container2)
+	alloc.ContainerDied(container3)
+	alloc.String() // force sync-up after async call
+	wt.AssertEquals(t, alloc.space.NumFreeAddresses(), address.Offset(spaceSize))
+}
+
+func TestBootstrap(t *testing.T) {
+	common.InitDefaultLogging(false)
+	const (
+		donateSize     = 5
+		donateStart    = "10.0.1.7"
+		ourNameString  = "01:00:00:01:00:00"
+		peerNameString = "02:00:00:02:00:00"
+	)
+
+	alloc1 := makeAllocatorWithMockGossip(t, ourNameString, testStart1+"/22", 2)
+	defer alloc1.Stop()
+
+	// Simulate another peer on the gossip network
+	alloc2 := makeAllocatorWithMockGossip(t, peerNameString, testStart1+"/22", 2)
+	defer alloc2.Stop()
+
+	alloc1.OnGossipBroadcast(alloc2.encode())
+
+	alloc1.tryPendingOps()
+
+	ExpectBroadcastMessage(alloc1, nil) // alloc1 will try to form consensus
+	done := make(chan bool)
+	go func() {
+		alloc1.Allocate("somecontainer", nil)
+		done <- true
+	}()
+	time.Sleep(100 * time.Millisecond)
+	AssertNothingSent(t, done)
+
+	CheckAllExpectedMessagesSent(alloc1, alloc2)
+
+	alloc1.tryPendingOps()
+	AssertNothingSent(t, done)
+
+	CheckAllExpectedMessagesSent(alloc1, alloc2)
+
+	// alloc2 receives paxos update and broadcasts its reply
+	ExpectBroadcastMessage(alloc2, nil)
+	alloc2.OnGossipBroadcast(alloc1.encode())
+
+	ExpectBroadcastMessage(alloc1, nil)
+	alloc1.OnGossipBroadcast(alloc2.encode())
+
+	// both nodes will get consensus now so initialize the ring
+	ExpectBroadcastMessage(alloc2, nil)
+	ExpectBroadcastMessage(alloc2, nil)
+	alloc2.OnGossipBroadcast(alloc1.encode())
+
+	CheckAllExpectedMessagesSent(alloc1, alloc2)
+
+	alloc1.OnGossipBroadcast(alloc2.encode())
+	// now alloc1 should have space
+
+	AssertSent(t, done)
+
+	CheckAllExpectedMessagesSent(alloc1, alloc2)
+}
+
+func TestAllocatorClaim(t *testing.T) {
+	const (
+		container1 = "abcdef"
+		container2 = "baddf00d"
+		container3 = "b01df00d"
+		universe   = "10.0.3.0/30"
+		testAddr1  = "10.0.3.1" // first address allocated should be .1 because .0 is network addr
+	)
+
+	alloc := makeAllocatorWithMockGossip(t, "01:00:00:01:00:00", universe, 1)
+	defer alloc.Stop()
+
+	alloc.claimRingForTesting()
+	addr1, _ := alloc.Allocate(container1, nil)
+	alloc.Allocate(container2, nil)
+
+	// Now free the first one, and try to claim it
+	wt.AssertSuccess(t, alloc.Free(container1))
+	t.Log(alloc)
+	err := alloc.Claim(container3, addr1, nil)
+	wt.AssertNoErr(t, err)
+	addr3, _ := alloc.Allocate(container3, nil)
+	wt.AssertEqualString(t, addr3.String(), testAddr1, "address")
+}
+
+func (alloc *Allocator) pause() func() {
+	paused := make(chan struct{})
+	alloc.actionChan <- func() {
+		paused <- struct{}{}
+		<-paused
+	}
+	<-paused
+	return func() {
+		close(paused)
+	}
+}
+
+func TestCancel(t *testing.T) {
+	common.InitDefaultLogging(false)
+	const (
+		CIDR = "10.0.1.7/26"
+	)
+
+	router := TestGossipRouter{make(map[router.PeerName]chan gossipMessage), 0.0}
+
+	alloc1 := makeAllocator("01:00:00:02:00:00", CIDR, 2)
+	alloc1.SetInterfaces(router.connect(alloc1.ourName, alloc1))
+
+	alloc2 := makeAllocator("02:00:00:02:00:00", CIDR, 2)
+	alloc2.SetInterfaces(router.connect(alloc2.ourName, alloc2))
+	alloc1.claimRingForTesting(alloc1, alloc2)
+	alloc2.claimRingForTesting(alloc1, alloc2)
+
+	alloc1.Start()
+	alloc2.Start()
+
+	// tell peers about each other
+	alloc1.OnGossipBroadcast(alloc2.encode())
+
+	// Get some IPs, so each allocator has some space
+	res1, _ := alloc1.Allocate("foo", nil)
+	common.Debug.Printf("res1 = %s", res1.String())
+	res2, _ := alloc2.Allocate("bar", nil)
+	common.Debug.Printf("res2 = %s", res2.String())
+	if res1 == res2 {
+		wt.Fatalf(t, "Error: got same ips!")
+	}
+
+	// Now we're going to pause alloc2 and ask alloc1
+	// for an allocation
+	unpause := alloc2.pause()
+
+	// Use up all the IPs that alloc1 owns, so the allocation after this will prompt a request to alloc2
+	for i := 0; alloc1.space.NumFreeAddresses() > 0; i++ {
+		alloc1.Allocate(fmt.Sprintf("tmp%d", i), nil)
+	}
+	cancelChan := make(chan bool, 1)
+	doneChan := make(chan bool)
+	go func() {
+		_, ok := alloc1.Allocate("baz", cancelChan)
+		doneChan <- ok == nil
+	}()
+
+	AssertNothingSent(t, doneChan)
+	time.Sleep(100 * time.Millisecond)
+	AssertNothingSent(t, doneChan)
+
+	cancelChan <- true
+	unpause()
+	if <-doneChan {
+		wt.Fatalf(t, "Error: got result from Allocate")
+	}
+}
+
+func TestGossipShutdown(t *testing.T) {
+	const (
+		container1 = "abcdef"
+		container2 = "baddf00d"
+		universe   = "10.0.3.0/30"
+		testAddr1  = "10.0.3.1" // first address allocated should be .1 because .0 is network addr
+	)
+
+	alloc := makeAllocatorWithMockGossip(t, "01:00:00:01:00:00", universe, 1)
+	defer alloc.Stop()
+
+	alloc.claimRingForTesting()
+	addr1, _ := alloc.Allocate(container1, nil)
+	wt.AssertEqualString(t, addr1.String(), testAddr1, "address")
+
+	alloc.Shutdown()
+
+	_, err := alloc.Allocate(container2, nil) // trying to allocate after shutdown should fail
+	wt.AssertFalse(t, err == nil, "no address")
+
+	CheckAllExpectedMessagesSent(alloc)
+}
+
+func TestTransfer(t *testing.T) {
+	const (
+		cidr = "10.0.1.7/22"
+	)
+	allocs, router := makeNetworkOfAllocators(3, cidr)
+	alloc1 := allocs[0]
+	alloc2 := allocs[1]
+	alloc3 := allocs[2] // This will be 'master' and get the first range
+
+	_, err := alloc2.Allocate("foo", nil)
+	wt.AssertTrue(t, err == nil, "Failed to get address")
+
+	_, err = alloc3.Allocate("bar", nil)
+	wt.AssertTrue(t, err == nil, "Failed to get address")
+
+	router.GossipBroadcast(alloc2.Gossip())
+	router.GossipBroadcast(alloc3.Gossip())
+	router.removePeer(alloc2.ourName)
+	router.removePeer(alloc3.ourName)
+	alloc2.Stop()
+	alloc3.Stop()
+	wt.AssertSuccess(t, alloc1.AdminTakeoverRanges(alloc2.ourName.String()))
+	wt.AssertSuccess(t, alloc1.AdminTakeoverRanges(alloc3.ourName.String()))
+
+	wt.AssertEquals(t, alloc1.space.NumFreeAddresses(), address.Offset(1022))
+
+	_, err = alloc1.Allocate("foo", nil)
+	wt.AssertTrue(t, err == nil, "Failed to get address")
+	alloc1.Stop()
+}
+
+func TestFakeRouterSimple(t *testing.T) {
+	common.InitDefaultLogging(false)
+	const (
+		cidr = "10.0.1.7/22"
+	)
+	allocs, _ := makeNetworkOfAllocators(2, cidr)
+	defer stopNetworkOfAllocators(allocs)
+
+	alloc1 := allocs[0]
+	//alloc2 := allocs[1]
+
+	addr, _ := alloc1.Allocate("foo", nil)
+	println("Got addr", addr)
+}
+
+func TestAllocatorFuzz(t *testing.T) {
+	common.InitDefaultLogging(false)
+	const (
+		firstpass    = 1000
+		secondpass   = 5000
+		nodes        = 10
+		maxAddresses = 1000
+		concurrency  = 10
+		cidr         = "10.0.1.7/22"
+	)
+	allocs, _ := makeNetworkOfAllocators(nodes, cidr)
+	defer stopNetworkOfAllocators(allocs)
+
+	// Test state
+	// For each IP issued we store the allocator
+	// that issued it and the name of the container
+	// it was issued to.
+	type result struct {
+		name  string
+		alloc int32
+	}
+	stateLock := sync.Mutex{}
+	state := make(map[string]result)
+	// Keep a list of addresses issued, so we
+	// Can pick random ones
+	var addrs []string
+	numPending := 0
+
+	rand.Seed(0)
+
+	// Remove item from list by swapping it with last
+	// and reducing slice length by 1
+	rm := func(xs []string, i int32) []string {
+		ls := len(xs) - 1
+		xs[i] = xs[ls]
+		return xs[:ls]
+	}
+
+	// Do a Allocate and check the address
+	// is unique.  Needs a unique container
+	// name.
+	allocate := func(name string) {
+		stateLock.Lock()
+		if len(addrs)+numPending >= maxAddresses {
+			stateLock.Unlock()
+			return
+		}
+		numPending++
+		stateLock.Unlock()
+
+		allocIndex := rand.Int31n(nodes)
+		alloc := allocs[allocIndex]
+		//common.Info.Printf("Allocate: asking allocator %d", allocIndex)
+		addr, err := alloc.Allocate(name, nil)
+
+		if err != nil {
+			panic(fmt.Sprintf("Could not allocate addr"))
+		}
+
+		//common.Info.Printf("Allocate: got address %s for name %s", addr, name)
+		addrStr := addr.String()
+
+		stateLock.Lock()
+		defer stateLock.Unlock()
+
+		if res, existing := state[addrStr]; existing {
+			panic(fmt.Sprintf("Dup found for address %s - %s and %s", addrStr,
+				name, res.name))
+		}
+
+		state[addrStr] = result{name, allocIndex}
+		addrs = append(addrs, addrStr)
+		numPending--
+	}
+
+	// Free a random address.
+	free := func() {
+		stateLock.Lock()
+		if len(addrs) == 0 {
+			stateLock.Unlock()
+			return
+		}
+		// Delete an existing allocation
+		// Pick random addr
+		addrIndex := rand.Int31n(int32(len(addrs)))
+		addr := addrs[addrIndex]
+		res := state[addr]
+		addrs = rm(addrs, addrIndex)
+		delete(state, addr)
+		stateLock.Unlock()
+
+		alloc := allocs[res.alloc]
+		//common.Info.Printf("Freeing %s on allocator %d", addr, res.alloc)
+
+		wt.AssertSuccess(t, alloc.Free(res.name))
+	}
+
+	// Do a Allocate on an existing container & allocator
+	// and check we get the right answer.
+	allocateAgain := func() {
+		stateLock.Lock()
+		addrIndex := rand.Int31n(int32(len(addrs)))
+		addr := addrs[addrIndex]
+		res := state[addr]
+		stateLock.Unlock()
+		alloc := allocs[res.alloc]
+
+		//common.Info.Printf("Asking for %s on allocator %d again", addr, res.alloc)
+
+		newAddr, _ := alloc.Allocate(res.name, nil)
+		oldAddr, _ := address.ParseIP(addr)
+		if newAddr != oldAddr {
+			panic(fmt.Sprintf("Got different address for repeat request"))
+		}
+	}
+
+	// Run function _f_ _iterations_ times, in _concurrency_
+	// number of goroutines
+	doConcurrentIterations := func(iterations int, f func(int)) {
+		iterationsPerThread := iterations / concurrency
+
+		wg := sync.WaitGroup{}
+		for i := 0; i < concurrency; i++ {
+			wg.Add(1)
+			go func(j int) {
+				defer wg.Done()
+				for k := 0; k < iterationsPerThread; k++ {
+					f((j * iterationsPerThread) + k)
+				}
+			}(i)
+		}
+		wg.Wait()
+	}
+
+	// First pass, just allocate a bunch of ips
+	doConcurrentIterations(firstpass, func(iteration int) {
+		name := fmt.Sprintf("first%d", iteration)
+		allocate(name)
+	})
+
+	// Second pass, random ask for more allocations,
+	// or remove existing ones, or ask for allocation
+	// again.
+	doConcurrentIterations(secondpass, func(iteration int) {
+		r := rand.Float32()
+		switch {
+		case 0.0 <= r && r < 0.4:
+			// Ask for a new allocation
+			name := fmt.Sprintf("second%d", iteration)
+			allocate(name)
+
+		case (0.4 <= r && r < 0.8):
+			// free a random addr
+			free()
+
+		case 0.8 <= r && r < 1.0:
+			// ask for an existing name again, check we get same ip
+			allocateAgain()
+		}
+	})
+}
+
+func TestGossipSkew(t *testing.T) {
+	alloc1 := makeAllocatorWithMockGossip(t, "01:00:00:01:00:00", "10.0.1.0/22", 2)
+	defer alloc1.Stop()
+	alloc2 := makeAllocatorWithMockGossip(t, "02:00:00:02:00:00", "10.0.1.0/22", 2)
+	alloc2.now = func() time.Time { return time.Now().Add(time.Hour * 2) }
+	defer alloc2.Stop()
+
+	if _, err := alloc1.OnGossipBroadcast(alloc2.encode()); err == nil {
+		t.Fail()
+	}
+}

ipam/claim.go

@@ -0,0 +1,79 @@
+package ipam
+
+import (
+	"fmt"
+
+	"github.com/weaveworks/weave/ipam/address"
+	"github.com/weaveworks/weave/router"
+)
+
+type claim struct {
+	resultChan       chan<- error
+	hasBeenCancelled func() bool
+	ident            string
+	addr             address.Address
+}
+
+// Try returns true for success (or failure), false if we need to try again later
+func (c *claim) Try(alloc *Allocator) bool {
+	if (c.hasBeenCancelled)() {
+		c.Cancel()
+		return true
+	}
+
+	if !alloc.ring.Contains(c.addr) {
+		// Address not within our universe; assume user knows what they are doing
+		alloc.infof("Ignored address %s claimed by %s - not in our universe\n", c.addr, c.ident)
+		c.resultChan <- nil
+		return true
+	}
+
+	// If our ring doesn't know, it must be empty.  We will have initiated the
+	// bootstrap of the ring, so wait until we find some owner for this
+	// range (might be us).
+	owner := alloc.ring.Owner(c.addr)
+	if owner == router.UnknownPeerName {
+		alloc.infof("Ring is empty; will try later.\n", c.addr, owner)
+		return false
+	}
+	if owner != alloc.ourName {
+		name, found := alloc.nicknames[owner]
+		if found {
+			name = " (" + name + ")"
+		}
+		c.resultChan <- fmt.Errorf("address %s is owned by other peer %s%s", c.addr.String(), owner, name)
+		return true
+	}
+	// We are the owner, check we haven't given it to another container
+	existingIdent := alloc.findOwner(c.addr)
+	if existingIdent == c.ident {
+		// same identifier is claiming same address; that's OK
+		c.resultChan <- nil
+		return true
+	}
+	if existingIdent == "" {
+		err := alloc.space.Claim(c.addr)
+		if err != nil {
+			c.resultChan <- err
+			return true
+		}
+		alloc.addOwned(c.ident, c.addr)
+		c.resultChan <- nil
+		return true
+	}
+	// Addr already owned by container on this machine
+	c.resultChan <- fmt.Errorf("address %s is already owned by %s", c.addr.String(), existingIdent)
+	return true
+}
+
+func (c *claim) Cancel() {
+	c.resultChan <- fmt.Errorf("Operation cancelled.")
+}
+
+func (c *claim) String() string {
+	return fmt.Sprintf("Claim %s -> %s", c.ident, c.addr.String())
+}
+
+func (c *claim) ForContainer(ident string) bool {
+	return c.ident == ident
+}

ipam/dummy.go

@@ -0,0 +1,36 @@
+package ipam
+
+import (
+	"errors"
+
+	"github.com/weaveworks/weave/router"
+)
+
+// Exists solely for the purpose of detecting a network in which some
+// peers are configured with IPAM on and some with IPAM off.
+type DummyAllocator struct {
+}
+
+var (
+	ErrMixedConfig = errors.New("Mixed IPAM/non-IPAM configuration not supported")
+)
+
+func (alloc *DummyAllocator) OnGossipUnicast(sender router.PeerName, msg []byte) error {
+	return ErrMixedConfig
+}
+
+func (alloc *DummyAllocator) OnGossipBroadcast(msg []byte) (router.GossipData, error) {
+	return nil, ErrMixedConfig
+}
+
+func (alloc *DummyAllocator) OnGossip(msg []byte) (router.GossipData, error) {
+	return nil, ErrMixedConfig
+}
+
+func (alloc *DummyAllocator) Gossip() router.GossipData {
+	return nil
+}
+
+func (alloc *DummyAllocator) Encode() []byte {
+	return nil
+}

ipam/http.go

@@ -0,0 +1,72 @@
+package ipam
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/gorilla/mux"
+
+	"github.com/weaveworks/weave/common"
+	"github.com/weaveworks/weave/ipam/address"
+)
+
+func badRequest(w http.ResponseWriter, err error) {
+	http.Error(w, err.Error(), http.StatusBadRequest)
+	common.Warning.Println("[allocator]:", err.Error())
+}
+
+// HandleHTTP wires up ipams HTTP endpoints to the provided mux.
+func (alloc *Allocator) HandleHTTP(router *mux.Router) {
+	router.Methods("PUT").Path("/ip/{id}/{ip}").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		closedChan := w.(http.CloseNotifier).CloseNotify()
+		vars := mux.Vars(r)
+		ident := vars["id"]
+		ipStr := vars["ip"]
+		if ip, err := address.ParseIP(ipStr); err != nil {
+			badRequest(w, err)
+			return
+		} else if err := alloc.Claim(ident, ip, closedChan); err != nil {
+			badRequest(w, fmt.Errorf("Unable to claim: %s", err))
+			return
+		}
+
+		w.WriteHeader(204)
+	})
+
+	router.Methods("POST").Path("/ip/{id}").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		closedChan := w.(http.CloseNotifier).CloseNotify()
+		ident := mux.Vars(r)["id"]
+		newAddr, err := alloc.Allocate(ident, closedChan)
+		if err != nil {
+			badRequest(w, err)
+			return
+		}
+
+		fmt.Fprintf(w, "%s/%d", newAddr.String(), alloc.prefixLen)
+	})
+
+	router.Methods("DELETE").Path("/ip/{id}").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		ident := mux.Vars(r)["id"]
+		if err := alloc.Free(ident); err != nil {
+			badRequest(w, err)
+			return
+		}
+
+		w.WriteHeader(204)
+	})
+
+	router.Methods("DELETE").Path("/peer").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		alloc.Shutdown()
+		w.WriteHeader(204)
+	})
+
+	router.Methods("DELETE").Path("/peer/{id}").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		ident := mux.Vars(r)["id"]
+		if err := alloc.AdminTakeoverRanges(ident); err != nil {
+			badRequest(w, err)
+			return
+		}
+
+		w.WriteHeader(204)
+	})
+}

ipam/http_test.go

@@ -0,0 +1,164 @@
+package ipam
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"math/rand"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/gorilla/mux"
+
+	"github.com/weaveworks/weave/common"
+	wt "github.com/weaveworks/weave/testing"
+)
+
+func HTTPPost(t *testing.T, url string) string {
+	resp, err := http.Post(url, "", nil)
+	wt.AssertNoErr(t, err)
+	wt.AssertStatus(t, resp.StatusCode, http.StatusOK, "http response")
+	defer resp.Body.Close()
+	body, _ := ioutil.ReadAll(resp.Body)
+	return string(body)
+}
+
+func doHTTP(method string, url string) (resp *http.Response, err error) {
+	req, _ := http.NewRequest(method, url, nil)
+	return http.DefaultClient.Do(req)
+}
+
+func listenHTTP(port int, alloc *Allocator) {
+	router := mux.NewRouter()
+	router.HandleFunc("/status", func(w http.ResponseWriter, r *http.Request) {
+		io.WriteString(w, fmt.Sprintln(alloc))
+	})
+	alloc.HandleHTTP(router)
+
+	srv := &http.Server{
+		Addr:    fmt.Sprintf(":%d", port),
+		Handler: router,
+	}
+	if err := srv.ListenAndServe(); err != nil {
+		common.Error.Fatal("Unable to create http listener: ", err)
+	}
+}
+
+func allocURL(port int, containerID string) string {
+	return fmt.Sprintf("http://localhost:%d/ip/%s", port, containerID)
+}
+
+func TestHttp(t *testing.T) {
+	var (
+		containerID = "deadbeef"
+		container2  = "baddf00d"
+		container3  = "b01df00d"
+		testAddr1   = "10.0.3.9"
+		netSuffix   = "/29"
+		testCIDR1   = "10.0.3.8" + netSuffix
+	)
+
+	alloc := makeAllocatorWithMockGossip(t, "08:00:27:01:c3:9a", testCIDR1, 1)
+	port := rand.Intn(10000) + 32768
+	fmt.Println("Http test on port", port)
+	go listenHTTP(port, alloc)
+
+	time.Sleep(100 * time.Millisecond) // Allow for http server to get going
+
+	alloc.claimRingForTesting()
+	// Ask the http server for a new address
+	cidr1 := HTTPPost(t, allocURL(port, containerID))
+	wt.AssertEqualString(t, cidr1, testAddr1+netSuffix, "address")
+
+	// Ask the http server for another address and check it's different
+	cidr2 := HTTPPost(t, allocURL(port, container2))
+	wt.AssertFalse(t, cidr2 == testAddr1+netSuffix, "address")
+
+	// Ask for the first container again and we should get the same address again
+	cidr1a := HTTPPost(t, allocURL(port, containerID))
+	wt.AssertEqualString(t, cidr1a, testAddr1+netSuffix, "address")
+
+	// Now free the first one, and we should get it back when we ask
+	doHTTP("DELETE", allocURL(port, containerID))
+	cidr3 := HTTPPost(t, allocURL(port, container3))
+	wt.AssertEqualString(t, cidr3, testAddr1+netSuffix, "address")
+
+	// Would like to shut down the http server at the end of this test
+	// but it's complicated.
+	// See https://groups.google.com/forum/#!topic/golang-nuts/vLHWa5sHnCE
+}
+
+func TestBadHttp(t *testing.T) {
+	var (
+		containerID = "deadbeef"
+		testCIDR1   = "10.0.0.0/8"
+	)
+
+	alloc := makeAllocatorWithMockGossip(t, "08:00:27:01:c3:9a", testCIDR1, 1)
+	defer alloc.Stop()
+	port := rand.Intn(10000) + 32768
+	fmt.Println("BadHttp test on port", port)
+	go listenHTTP(port, alloc)
+
+	alloc.claimRingForTesting()
+	cidr1 := HTTPPost(t, allocURL(port, containerID))
+	parts := strings.Split(cidr1, "/")
+	testAddr1 := parts[0]
+	// Verb that's not handled
+	resp, err := doHTTP("HEAD", fmt.Sprintf("http://localhost:%d/ip/%s/%s", port, containerID, testAddr1))
+	wt.AssertNoErr(t, err)
+	wt.AssertStatus(t, resp.StatusCode, http.StatusNotFound, "http response")
+	// Mis-spelled URL
+	resp, err = doHTTP("POST", fmt.Sprintf("http://localhost:%d/xip/%s/", port, containerID))
+	wt.AssertNoErr(t, err)
+	wt.AssertStatus(t, resp.StatusCode, http.StatusNotFound, "http response")
+	// Malformed URL
+	resp, err = doHTTP("POST", fmt.Sprintf("http://localhost:%d/ip/%s/foo/bar/baz", port, containerID))
+	wt.AssertNoErr(t, err)
+	wt.AssertStatus(t, resp.StatusCode, http.StatusNotFound, "http response")
+}
+
+func TestHTTPCancel(t *testing.T) {
+	wt.RunWithTimeout(t, 2*time.Second, func() {
+		impTestHTTPCancel(t)
+	})
+}
+
+func impTestHTTPCancel(t *testing.T) {
+	common.InitDefaultLogging(true)
+	var (
+		containerID = "deadbeef"
+		testCIDR1   = "10.0.3.5/29"
+	)
+
+	alloc := makeAllocatorWithMockGossip(t, "08:00:27:01:c3:9a", testCIDR1, 1)
+	defer alloc.Stop()
+	alloc.claimRingForTesting()
+	port := rand.Intn(10000) + 32768
+	fmt.Println("Http test on port", port)
+	go listenHTTP(port, alloc)
+
+	time.Sleep(100 * time.Millisecond) // Allow for http server to get going
+
+	// Stop the alloc so nothing actually works
+	unpause := alloc.pause()
+
+	// Ask the http server for a new address
+	done := make(chan *http.Response)
+	req, _ := http.NewRequest("POST", allocURL(port, containerID), nil)
+	go func() {
+		res, _ := http.DefaultClient.Do(req)
+		done <- res
+	}()
+
+	time.Sleep(100 * time.Millisecond)
+	fmt.Println("Cancelling allocate")
+	http.DefaultTransport.(*http.Transport).CancelRequest(req)
+	unpause()
+	res := <-done
+	if res != nil {
+		wt.Fatalf(t, "Error: Allocate returned non-nil")
+	}
+}

ipam/paxos/paxos.go

@@ -0,0 +1,261 @@
+package paxos
+
+import (
+	"fmt"
+	"github.com/weaveworks/weave/router"
+)
+
+// The node identifier.  The use of the UID here is important: Paxos
+// acceptors must not forget their promises, so it's important that a
+// node does not restart and lose its Paxos state but claim to have
+// the same ID.
+type NodeID struct {
+	Name router.PeerName
+	UID  router.PeerUID
+}
+
+// note all fields exported in structs so we can Gob them
+type ProposalID struct {
+	// round numbers begin at 1.  round 0 indicates an
+	// uninitialized ProposalID, and precedes all other ProposalIDs
+	Round    uint
+	Proposer NodeID
+}
+
+func (a ProposalID) precedes(b ProposalID) bool {
+	switch {
+	case a.Round != b.Round:
+		return a.Round < b.Round
+	case a.Proposer.Name != b.Proposer.Name:
+		return a.Proposer.Name < b.Proposer.Name
+	case a.Proposer.UID != b.Proposer.UID:
+		return a.Proposer.UID < b.Proposer.UID
+	default:
+		return false
+	}
+}
+
+func (a ProposalID) valid() bool {
+	return a.Round > 0
+}
+
+// For seeding IPAM, the value we want consensus on is a set of peer names
+type Value []router.PeerName
+
+// An AcceptedValue is a Value plus the proposal which originated that
+// Value.  The origin is not essential, but makes comparing
+// AcceptedValues easy even if comparing Values is not.
+type AcceptedValue struct {
+	Value  Value
+	Origin ProposalID
+}
+
+type NodeClaims struct {
+	// The node promises not to accept a proposal with id less
+	// than this.
+	Promise ProposalID
+
+	// The accepted proposal, if valid
+	Accepted    ProposalID
+	AcceptedVal AcceptedValue
+}
+
+func (a NodeClaims) equals(b NodeClaims) bool {
+	return a.Promise == b.Promise && a.Accepted == b.Accepted &&
+		a.AcceptedVal.Origin == b.AcceptedVal.Origin
+}
+
+type GossipState map[NodeID]NodeClaims
+
+type Node struct {
+	id     NodeID
+	quorum uint
+	knows  GossipState
+}
+
+func NewNode(name router.PeerName, uid router.PeerUID, quorum uint) *Node {
+	return &Node{
+		id:     NodeID{name, uid},
+		quorum: quorum,
+		knows:  map[NodeID]NodeClaims{},
+	}
+}
+
+func (node *Node) GossipState() GossipState {
+	return node.knows
+}
+
+// Update this node's information about what other nodes know.
+// Returns true if we learned something new.
+func (node *Node) Update(from GossipState) bool {
+	changed := false
+
+	for i, fromClaims := range from {
+		claims, ok := node.knows[i]
+		if ok {
+			if claims.Promise.precedes(fromClaims.Promise) {
+				claims.Promise = fromClaims.Promise
+				changed = true
+			}
+
+			if claims.Accepted.precedes(fromClaims.Accepted) {
+				claims.Accepted = fromClaims.Accepted
+				claims.AcceptedVal = fromClaims.AcceptedVal
+				changed = true
+			}
+		} else {
+			claims = fromClaims
+			changed = true
+		}
+
+		node.knows[i] = claims
+	}
+
+	return changed
+}
+
+func max(a uint, b uint) uint {
+	if a > b {
+		return a
+	} else {
+		return b
+	}
+}
+
+// Initiate a new proposal, i.e. the Paxos "Prepare" step.  This is
+// simply a matter of gossipping a new proposal that supersedes all
+// others.
+func (node *Node) Propose() {
+	// Find the highest round number around
+	round := uint(0)
+
+	for _, claims := range node.knows {
+		round = max(round, claims.Promise.Round)
+		round = max(round, claims.Accepted.Round)
+	}
+
+	ourClaims := node.knows[node.id]
+	ourClaims.Promise = ProposalID{
+		Round:    round + 1,
+		Proposer: node.id,
+	}
+	node.knows[node.id] = ourClaims
+
+	// With a quorum of 1, we can immediately accept our proposal
+	if node.quorum == 1 {
+		node.Think()
+	}
+}
+
+// The heart of the consensus algorithm. Return true if we have
+// changed our claims.
+func (node *Node) Think() bool {
+	ourClaims := node.knows[node.id]
+
+	// The "Promise" step of Paxos: Copy the highest known
+	// promise.
+	for _, claims := range node.knows {
+		if ourClaims.Promise.precedes(claims.Promise) {
+			ourClaims.Promise = claims.Promise
+		}
+	}
+
+	// The "Accept Request" step of Paxos: Acting as a proposer,
+	// do we have a proposal that has been promised by a quorum?
+	//
+	// In Paxos, the "proposer" and "acceptor" roles are distinct,
+	// so in principle a node acting as a proposer could continue
+	// trying to get its proposal acccepted even after the same
+	// node as an acceptor has superseded that proposal.  But
+	// that's pointless in a gossip context: If our promise
+	// supersedes our own proposal, then anyone who hears about
+	// that promise will not accept that proposal.  So our
+	// proposal is only in the running if it is also our promise.
+	if ourClaims.Promise.Proposer == node.id {
+		// Determine whether a quorum has promised, and the
+		// best previously-accepted value if there is one.
+		count := uint(0)
+		var accepted ProposalID
+		var acceptedVal AcceptedValue
+
+		for _, claims := range node.knows {
+			if claims.Promise == ourClaims.Promise {
+				count++
+
+				if accepted.precedes(claims.Accepted) {
+					accepted = claims.Accepted
+					acceptedVal = claims.AcceptedVal
+				}
+			}
+		}
+
+		if count >= node.quorum {
+			if !accepted.valid() {
+				acceptedVal.Value = node.pickValue()
+				acceptedVal.Origin = ourClaims.Promise
+			}
+
+			// We automatically accept our own proposal,
+			// and that's how we communicate the "accept
+			// request" to other nodes.
+			ourClaims.Accepted = ourClaims.Promise
+			ourClaims.AcceptedVal = acceptedVal
+		}
+	}
+
+	// The "Accepted" step of Paxos: If the proposal we promised
+	// on got accepted by some other node, we accept it too.
+	for _, claims := range node.knows {
+		if claims.Accepted == ourClaims.Promise {
+			ourClaims.Accepted = claims.Accepted
+			ourClaims.AcceptedVal = claims.AcceptedVal
+			break
+		}
+	}
+
+	if ourClaims.equals(node.knows[node.id]) {
+		return false
+	}
+
+	node.knows[node.id] = ourClaims
+	return true
+}
+
+// When we get to pick a value, we use the set of peer names we know
+// about.  This is not necessarily all peer names, but it is at least
+// a quorum, and so good enough for seeding the ring.
+func (node *Node) pickValue() Value {
+	val := make([]router.PeerName, len(node.knows))
+	i := 0
+	for id := range node.knows {
+		val[i] = id.Name
+		i++
+	}
+	return val
+}
+
+// Has a consensus been reached, based on the known claims of other nodes?
+func (node *Node) Consensus() (bool, AcceptedValue) {
+	counts := map[ProposalID]uint{}
+
+	for _, claims := range node.knows {
+		if claims.Accepted.valid() {
+			origin := claims.AcceptedVal.Origin
+			count := counts[origin] + 1
+			counts[origin] = count
+			if count >= node.quorum {
+				return true, claims.AcceptedVal
+			}
+		}
+	}
+
+	return false, AcceptedValue{}
+}
+
+func (node *Node) String() string {
+	if ok, val := node.Consensus(); ok {
+		return fmt.Sprintf("Consensus reached with size %d", len(val.Value))
+	} else {
+		return fmt.Sprintf("Nodes known: %d, Quorum size: %d", len(node.knows), node.quorum)
+	}
+}

ipam/paxos/paxos_test.go

@@ -0,0 +1,372 @@
+package paxos
+
+import (
+	"fmt"
+	"github.com/weaveworks/weave/router"
+	"math/rand"
+	"testing"
+	"time"
+)
+
+type TestNode struct {
+	*Node
+
+	// Topology
+	links    []*Link
+	isolated bool
+
+	// The first consensus the Node observed
+	firstConsensus AcceptedValue
+}
+
+type Link struct {
+	from     *TestNode
+	to       *TestNode
+	converse *Link
+
+	// A link is considered ready if it is worthwhile gossipping
+	// along it (i.e. unless we already gossipped along it and the
+	// "from" node didn't change since then).  If this is <0, then
+	// the link is not ready.  Otherwise, this gives the index in
+	// the Model's readyLinks slice.
+	ready int
+}
+
+type Model struct {
+	t          *testing.T
+	r          *rand.Rand
+	quorum     uint
+	nodes      []TestNode
+	readyLinks []*Link
+	nextID     uint
+}
+
+func (m *Model) addLink(a, b *TestNode) {
+	if a == b {
+		// Never link a node to itself
+		return
+	}
+
+	ab := Link{from: a, to: b, ready: len(m.readyLinks)}
+	ba := Link{from: b, to: a, converse: &ab, ready: len(m.readyLinks) + 1}
+	ab.converse = &ba
+	a.links = append(a.links, &ab)
+	b.links = append(b.links, &ba)
+	m.readyLinks = append(m.readyLinks, &ab, &ba)
+}
+
+func (m *Model) linkExists(a, b *TestNode) bool {
+	for _, l := range a.links {
+		if l.to == b {
+			return true
+		}
+	}
+
+	return false
+}
+
+type TestParams struct {
+	// Number of nodes
+	nodeCount uint
+
+	// Probability that two nodes are connected.
+	connectedProb float32
+
+	// Probability that some node will re-propose at each
+	// step. Setting this too high makes it likely that we'll fail
+	// to converge.
+	reproposeProb float32
+
+	// Probability that some node will be isolated at each step.
+	isolateProb float32
+
+	// Probability that some node willbe restarted at each step.
+	restartProb float32
+}
+
+// Make a network of nodes with random topology
+func makeRandomModel(params *TestParams, r *rand.Rand, t *testing.T) *Model {
+	m := Model{
+		t:          t,
+		r:          r,
+		quorum:     params.nodeCount/2 + 1,
+		nodes:      make([]TestNode, params.nodeCount),
+		readyLinks: []*Link{},
+		nextID:     params.nodeCount + 1,
+	}
+
+	for i := range m.nodes {
+		m.nodes[i].Node = NewNode(router.PeerName(i/2+1),
+			router.PeerUID(r.Int63()), m.quorum)
+		m.nodes[i].Propose()
+	}
+
+	for i := 1; i < len(m.nodes); i++ {
+		// was node i connected to the other nodes yet?
+		connected := false
+
+		for j := 0; j < i; j++ {
+			if r.Float32() < params.connectedProb {
+				connected = true
+				m.addLink(&m.nodes[i], &m.nodes[j])
+			}
+		}
+
+		if !connected {
+			// node i must be connected into the graph
+			// somewhere.  So if we didn't connect it
+			// already, this is a last resort.
+			m.addLink(&m.nodes[i], &m.nodes[r.Intn(i)])
+		}
+	}
+
+	return &m
+}
+
+// Mark a link as unready
+func (m *Model) unreadyLink(link *Link) {
+	i := link.ready
+	if i >= 0 {
+		m.readyLinks[i] = m.readyLinks[len(m.readyLinks)-1]
+		m.readyLinks[i].ready = i
+		m.readyLinks = m.readyLinks[:len(m.readyLinks)-1]
+		link.ready = -1
+	}
+}
+
+// Mark a link as ready
+func (m *Model) readyLink(link *Link) {
+	if link.ready < 0 {
+		link.ready = len(m.readyLinks)
+		m.readyLinks = append(m.readyLinks, link)
+	}
+}
+
+// Mark all the outgoing links from a node as ready
+func (m *Model) nodeChanged(node *TestNode) {
+	for _, l := range node.links {
+		m.readyLink(l)
+	}
+}
+
+// Isolate a node
+func (m *Model) isolateNode(node *TestNode) {
+	node.isolated = true
+	for _, l := range node.links {
+		if l.ready >= 0 {
+			m.unreadyLink(l)
+			m.unreadyLink(l.converse)
+		}
+	}
+
+	// Isolating a node could partition the network.  We don't
+	// want to test such a case (because it could prevent
+	// consensus).  Checking for partitions would be more code
+	// than its worth, so just add some links to prevent the
+	// possibility of partitions.
+	for i := 1; i < len(node.links); i++ {
+		m.addLink(node.links[i-1].to, node.links[i].to)
+	}
+}
+
+// Restart a node
+func (m *Model) restart(node *TestNode) {
+	node.Node = NewNode(router.PeerName(m.nextID),
+		router.PeerUID(m.r.Int63()), m.quorum)
+	m.nextID++
+	node.Propose()
+
+	// The node is now ignorant, so we need to mark the links into
+	// the node as ready.
+	for _, l := range node.links {
+		if !l.to.isolated {
+			m.readyLink(l.converse)
+		}
+	}
+
+	// If a consensus was just accepted due to this node accepting
+	// it, without other nodes hearing of it, and we then restart
+	// this node, then a different consensus can occur later on.
+	// If a tree falls with no one to hear it, does it make a
+	// sound?
+	node.firstConsensus = AcceptedValue{}
+}
+
+func (m *Model) pickNode() *TestNode {
+	for {
+		node := &m.nodes[m.r.Intn(len(m.nodes))]
+		if !node.isolated {
+			return node
+		}
+	}
+}
+
+func (m *Model) simulate(params *TestParams) bool {
+	nodesLeft := uint(len(m.nodes))
+	restarts := uint(0)
+
+	for step := 0; step < 1000000; step++ {
+		if len(m.readyLinks) == 0 {
+			// Everything has converged.  This can be
+			// because consensus was reached, or because
+			// consensus became impossible, e.g. everyone
+			// promised on a particular proposal, but then
+			// the proposer restarted or was isolated.  So
+			// we detect the latter cases and force a new
+			// proposal
+			for i := range m.nodes {
+				ok, _ := m.nodes[i].Consensus()
+				if ok {
+					return true
+				}
+			}
+
+			node := m.pickNode()
+			node.Propose()
+			m.nodeChanged(node)
+		}
+
+		// Pick a ready link at random
+		i := m.r.Intn(len(m.readyLinks))
+		link := m.readyLinks[i]
+		if link.ready != i {
+			m.t.Fatal("Link in readyLinks was not ready")
+		}
+
+		// gossip across link
+		node := link.to
+		if node.Update(link.from.GossipState()) {
+			node.Think()
+
+			if !node.firstConsensus.Origin.valid() {
+				ok, val := node.Consensus()
+				if ok {
+					node.firstConsensus = val
+				}
+			}
+
+			m.nodeChanged(node)
+		}
+
+		m.unreadyLink(link)
+
+		// Re-propose?
+		if m.r.Float32() < params.reproposeProb {
+			node := m.pickNode()
+			node.Propose()
+			m.nodeChanged(node)
+		}
+
+		// Isolate?
+		if nodesLeft > m.quorum && m.r.Float32() < params.isolateProb {
+			m.isolateNode(m.pickNode())
+			nodesLeft--
+
+			// We isolated a node, so get another node to
+			// re-propose.  In reality the lack of
+			// consensus would be detected via a timeout
+			node := m.pickNode()
+			node.Propose()
+			m.nodeChanged(node)
+		}
+
+		// Restart?
+		if restarts < m.quorum && m.r.Float32() < params.restartProb {
+			restarts++
+			node := m.pickNode()
+			m.restart(node)
+			m.nodeChanged(node)
+		}
+	}
+
+	return false
+}
+
+func (m *Model) dump() {
+	for i := range m.nodes {
+		node := &m.nodes[i]
+		fmt.Println(node.id)
+		for n, claims := range node.knows {
+			fmt.Printf("\t%d %v\n", n, claims)
+		}
+	}
+}
+
+// Validate the final model state
+func (m *Model) validate() {
+	var origin ProposalID
+
+	for i := range m.nodes {
+		ok, val := m.nodes[i].Consensus()
+		if !ok {
+			//m.dump()
+			m.t.Fatal("Node doesn't know about consensus")
+		}
+
+		firstConsensus := m.nodes[i].firstConsensus
+		if firstConsensus.Origin.valid() &&
+			firstConsensus.Origin != val.Origin {
+			//m.dump()
+			m.t.Fatal("Consensus mismatch")
+		}
+
+		if i == 0 {
+			origin = val.Origin
+		} else if val.Origin != origin {
+			m.t.Fatal("Node disagrees about consensus")
+		}
+	}
+}
+
+func TestSingleNode(t *testing.T) {
+	r := rand.New(rand.NewSource(time.Now().UnixNano()))
+
+	// Test the single node case
+	params := &TestParams{
+		nodeCount:     1,
+		connectedProb: 0,
+		reproposeProb: 0,
+		isolateProb:   0,
+		restartProb:   0,
+	}
+
+	m := makeRandomModel(params, r, t)
+
+	if !m.simulate(params) {
+		m.t.Fatal("Failed to converge")
+	}
+
+	m.validate()
+}
+
+func TestPaxos(t *testing.T) {
+	r := rand.New(rand.NewSource(time.Now().UnixNano()))
+
+	params := &TestParams{
+		nodeCount:     10,
+		connectedProb: 0.5,
+		reproposeProb: 0.01,
+		isolateProb:   0.01,
+
+		// Restarts cause failures due to the case where 1)
+		// the gossip ordering is such that there is an
+		// effective partition from the start, with a quorum
+		// on one side and one node less than a quorum on the
+		// other; 2) the quorum reaches a consensus; 3) a node
+		// is restarted, and at the same time jumps the
+		// partition, so the side that was now just less than
+		// a quorum becomes a quorum; 4) the new quorum
+		// reaches a contradictory consensus.
+		restartProb: 0,
+	}
+
+	for i := 0; i < 1000; i++ {
+		m := makeRandomModel(params, r, t)
+
+		if !m.simulate(params) {
+			m.t.Fatal("Failed to converge")
+		}
+
+		m.validate()
+	}
+}

ipam/ring/entry.go

@@ -0,0 +1,102 @@
+package ring
+
+import (
+	"sort"
+
+	"github.com/weaveworks/weave/common"
+	"github.com/weaveworks/weave/ipam/address"
+	"github.com/weaveworks/weave/router"
+)
+
+// Entry represents entries around the ring
+type entry struct {
+	Token   address.Address // The start of this range
+	Peer    router.PeerName // Who owns this range
+	Version uint32          // Version of this range
+	Free    address.Offset  // Number of free IPs in this range
+}
+
+func (e *entry) Equal(e2 *entry) bool {
+	return e.Token == e2.Token && e.Peer == e2.Peer &&
+		e.Version == e2.Version
+}
+
+func (e *entry) update(peername router.PeerName, free address.Offset) {
+	e.Peer = peername
+	e.Version++
+	e.Free = free
+}
+
+// For compatibility with sort.Interface
+type entries []*entry
+
+func (es entries) Len() int           { return len(es) }
+func (es entries) Less(i, j int) bool { return es[i].Token < es[j].Token }
+func (es entries) Swap(i, j int)      { panic("Should never be swapping entries!") }
+
+func (es entries) entry(i int) *entry {
+	i = i % len(es)
+	if i < 0 {
+		i += len(es)
+	}
+	return es[i]
+}
+
+func (es *entries) insert(e entry) {
+	i := sort.Search(len(*es), func(j int) bool {
+		return (*es)[j].Token >= e.Token
+	})
+
+	if i < len(*es) && (*es)[i].Token == e.Token {
+		panic("Trying to insert an existing token!")
+	}
+
+	*es = append(*es, &entry{})
+	copy((*es)[i+1:], (*es)[i:])
+	(*es)[i] = &e
+}
+
+func (es entries) get(token address.Address) (*entry, bool) {
+	i := sort.Search(len(es), func(j int) bool {
+		return es[j].Token >= token
+	})
+
+	if i < len(es) && es[i].Token == token {
+		return es[i], true
+	}
+
+	return nil, false
+}
+
+func (es *entries) remove(i int) {
+	*es = (*es)[:i+copy((*es)[i:], (*es)[i+1:])]
+}
+
+// Is token between entries at i and j?
+// NB i and j can overflow and will wrap
+// NBB this function does not work very well if there is only one
+//     token on the ring; luckily an accurate answer is not needed
+//     by the call sites in this case.
+func (es entries) between(token address.Address, i, j int) bool {
+	common.Assert(i < j)
+
+	first := es.entry(i)
+	second := es.entry(j)
+
+	switch {
+	case first.Token == second.Token:
+		// This implies there is only one token
+		// on the ring (i < j and i.token == j.token)
+		// In which case everything is between, expect
+		// this one token
+		return token != first.Token
+
+	case first.Token < second.Token:
+		return first.Token <= token && token < second.Token
+
+	case second.Token < first.Token:
+		return first.Token <= token || token < second.Token
+	}
+
+	panic("Should never get here - switch covers all possibilities.")
+}

ipam/ring/expvar.go

@@ -0,0 +1,31 @@
+package ring
+
+/* Exported variables for monitoring and management */
+
+import (
+	"expvar"
+	"strconv"
+)
+
+var (
+	expRingSize    = expvar.NewMap("ipam.ringSize")
+	expRingEntries = expvar.NewMap("ipam.ringEntries")
+)
+
+type _int int
+
+func (i _int) String() string {
+	return strconv.Itoa(int(i))
+}
+
+type _uint32 uint32
+
+func (i _uint32) String() string {
+	return strconv.FormatUint(uint64(i), 10)
+}
+
+func (r *Ring) updateExportedVariables() {
+	ringName := r.Start.String()
+	expRingSize.Set(ringName, _uint32(r.End-r.Start))
+	expRingEntries.Set(ringName, _int(len(r.Entries)))
+}

ipam/ring/ring.go

@@ -0,0 +1,520 @@
+/*
+Package ring implements a simple ring CRDT.
+*/
+package ring
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"math/rand"
+	"sort"
+
+	"github.com/weaveworks/weave/common"
+	"github.com/weaveworks/weave/ipam/address"
+	"github.com/weaveworks/weave/router"
+)
+
+// Ring represents the ring itself
+type Ring struct {
+	Start, End address.Address // [min, max) tokens in this ring.  Due to wrapping, min == max (effectively)
+	Peer       router.PeerName // name of peer owning this ring instance
+	Entries    entries         // list of entries sorted by token
+}
+
+func (r *Ring) assertInvariants() {
+	err := r.checkInvariants()
+	if err != nil {
+		panic(err.Error())
+	}
+}
+
+// Errors returned by Merge
+var (
+	ErrNotSorted        = errors.New("Ring not sorted")
+	ErrTokenRepeated    = errors.New("Token appears twice in ring")
+	ErrTokenOutOfRange  = errors.New("Token is out of range")
+	ErrDifferentSubnets = errors.New("IP Allocator with different subnet detected")
+	ErrNewerVersion     = errors.New("Received new version for entry I own!")
+	ErrInvalidEntry     = errors.New("Received invalid state update!")
+	ErrEntryInMyRange   = errors.New("Received new entry in my range!")
+	ErrNoFreeSpace      = errors.New("No free space found!")
+	ErrInvalidTimeout   = errors.New("dt must be greater than 0")
+	ErrNotFound         = errors.New("No entries for peer found")
+)
+
+func (r *Ring) checkInvariants() error {
+	if !sort.IsSorted(r.Entries) {
+		return ErrNotSorted
+	}
+
+	// Check no token appears twice
+	// We know it's sorted...
+	for i := 1; i < len(r.Entries); i++ {
+		if r.Entries[i-1].Token == r.Entries[i].Token {
+			return ErrTokenRepeated
+		}
+	}
+
+	if len(r.Entries) == 0 {
+		return nil
+	}
+
+	// Check tokens are in range
+	if r.Entries.entry(0).Token < r.Start {
+		return ErrTokenOutOfRange
+	}
+	if r.Entries.entry(-1).Token >= r.End {
+		return ErrTokenOutOfRange
+	}
+
+	// Check all the freespaces are in range
+	for i, entry := range r.Entries {
+		next := r.Entries.entry(i + 1)
+		distance := r.distance(entry.Token, next.Token)
+
+		if entry.Free > distance {
+			return fmt.Errorf("Entry %s-%s reporting too much free space: %d > %d", entry.Token, next.Token, entry.Free, distance)
+		}
+	}
+
+	return nil
+}
+
+// New creates an empty ring belonging to peer.
+func New(start, end address.Address, peer router.PeerName) *Ring {
+	common.Assert(start < end)
+
+	ring := &Ring{Start: start, End: end, Peer: peer, Entries: make([]*entry, 0)}
+	ring.updateExportedVariables()
+	return ring
+}
+
+// TotalRemoteFree returns the approximate number of free IPs
+// on other hosts.
+func (r *Ring) TotalRemoteFree() address.Offset {
+	result := address.Offset(0)
+	for _, entry := range r.Entries {
+		if entry.Peer != r.Peer {
+			result += entry.Free
+		}
+	}
+	return result
+}
+
+// Returns the distance between two tokens on this ring, dealing
+// with ranges which cross the origin
+func (r *Ring) distance(start, end address.Address) address.Offset {
+	if end > start {
+		return address.Offset(end - start)
+	}
+
+	return address.Offset((r.End - start) + (end - r.Start))
+}
+
+// GrantRangeToHost modifies the ring such that range [start, end)
+// is assigned to peer.  This may insert up to two new tokens.
+// Preconditions:
+// - start < end
+// - [start, end) must be owned by the calling peer
+func (r *Ring) GrantRangeToHost(start, end address.Address, peer router.PeerName) {
+	//fmt.Printf("%s GrantRangeToHost [%v,%v) -> %s\n", r.Peer, start, end, peer)
+
+	r.assertInvariants()
+	defer r.assertInvariants()
+	defer r.updateExportedVariables()
+
+	// ----------------- Start of Checks -----------------
+
+	common.Assert(start < end)
+	common.Assert(r.Start <= start && start < r.End)
+	common.Assert(r.Start < end && end <= r.End)
+	common.Assert(len(r.Entries) > 0)
+
+	// Look for the left-most entry greater than start, then go one previous
+	// to get the right-most entry less than or equal to start
+	preceedingPos := sort.Search(len(r.Entries), func(j int) bool {
+		return r.Entries[j].Token > start
+	})
+	preceedingPos--
+
+	// Check all tokens up to end are owned by us
+	for pos := preceedingPos; pos < len(r.Entries) && r.Entries.entry(pos).Token < end; pos++ {
+		common.Assert(r.Entries.entry(pos).Peer == r.Peer)
+	}
+
+	// ----------------- End of Checks -----------------
+
+	// Free space at start is max(length of range, distance to next token)
+	startFree := r.distance(start, r.Entries.entry(preceedingPos+1).Token)
+	if length := r.distance(start, end); startFree > length {
+		startFree = length
+	}
+	// Is there already a token at start, update it
+	if previousEntry := r.Entries.entry(preceedingPos); previousEntry.Token == start {
+		previousEntry.update(peer, startFree)
+	} else {
+		// Otherwise, these isn't a token here, insert a new one.
+		r.Entries.insert(entry{Token: start, Peer: peer, Free: startFree})
+		preceedingPos++
+		// Reset free space on previous entry, which we own.
+		previousEntry.update(r.Peer, r.distance(previousEntry.Token, start))
+	}
+
+	// Give all intervening tokens to the other peer
+	pos := preceedingPos + 1
+	for ; pos < len(r.Entries) && r.Entries.entry(pos).Token < end; pos++ {
+		entry := r.Entries.entry(pos)
+		entry.update(peer, address.Min(entry.Free, r.distance(entry.Token, end)))
+	}
+
+	// There is never an entry with a token of r.End, as the end of
+	// the ring is exclusive.
+	if end == r.End {
+		end = r.Start
+	}
+
+	//  If there is a token equal to the end of the range, we don't need to do anything further
+	if _, found := r.Entries.get(end); found {
+		return
+	}
+
+	// If not, we need to insert a token such that we claim this bit on the end.
+	endFree := r.distance(end, r.Entries.entry(pos).Token)
+	r.Entries.insert(entry{Token: end, Peer: r.Peer, Free: endFree})
+}
+
+// Merge the given ring into this ring and return any new ranges added
+func (r *Ring) Merge(gossip Ring) error {
+	r.assertInvariants()
+	defer r.assertInvariants()
+	defer r.updateExportedVariables()
+
+	// Don't panic when checking the gossiped in ring.
+	// In this case just return any error found.
+	if err := gossip.checkInvariants(); err != nil {
+		return err
+	}
+
+	if r.Start != gossip.Start || r.End != gossip.End {
+		return ErrDifferentSubnets
+	}
+
+	// Now merge their ring with yours, in a temporary ring.
+	var result entries
+	addToResult := func(e entry) { result = append(result, &e) }
+
+	var mine, theirs *entry
+	var previousOwner *router.PeerName
+	// i is index into r.Entries; j is index into gossip.Entries
+	var i, j int
+	for i < len(r.Entries) && j < len(gossip.Entries) {
+		mine, theirs = r.Entries[i], gossip.Entries[j]
+		switch {
+		case mine.Token < theirs.Token:
+			addToResult(*mine)
+			previousOwner = &mine.Peer
+			i++
+		case mine.Token > theirs.Token:
+			// insert, checking that a range owned by us hasn't been split
+			if previousOwner != nil && *previousOwner == r.Peer && theirs.Peer != r.Peer {
+				return ErrEntryInMyRange
+			}
+			addToResult(*theirs)
+			previousOwner = nil
+			j++
+		case mine.Token == theirs.Token:
+			// merge
+			switch {
+			case mine.Version >= theirs.Version:
+				if mine.Version == theirs.Version && !mine.Equal(theirs) {
+					common.Debug.Printf("Error merging entries at %s - %v != %v\n", mine.Token, mine, theirs)
+					return ErrInvalidEntry
+				}
+				addToResult(*mine)
+				previousOwner = &mine.Peer
+			case mine.Version < theirs.Version:
+				if mine.Peer == r.Peer { // We shouldn't receive updates to our own tokens
+					return ErrNewerVersion
+				}
+				addToResult(*theirs)
+				previousOwner = nil
+			}
+			i++
+			j++
+		}
+	}
+
+	// At this point, either i is at the end of r or j is at the end
+	// of gossip, so copy over the remaining entries.
+
+	for ; i < len(r.Entries); i++ {
+		mine = r.Entries[i]
+		addToResult(*mine)
+	}
+
+	for ; j < len(gossip.Entries); j++ {
+		theirs = gossip.Entries[j]
+		if previousOwner != nil && *previousOwner == r.Peer && theirs.Peer != r.Peer {
+			return ErrEntryInMyRange
+		}
+		addToResult(*theirs)
+		previousOwner = nil
+	}
+
+	r.Entries = result
+	return nil
+}
+
+// Empty returns true if the ring has no entries
+func (r *Ring) Empty() bool {
+	return len(r.Entries) == 0
+}
+
+// Given a slice of ranges which are all in the right order except
+// possibly the last one spans zero, fix that up and return the slice
+func (r *Ring) splitRangesOverZero(ranges []address.Range) []address.Range {
+	if len(ranges) == 0 {
+		return ranges
+	}
+	lastRange := ranges[len(ranges)-1]
+	// if end token == start (ie last) entry on ring, we want to actually use r.End
+	if lastRange.End == r.Start {
+		ranges[len(ranges)-1].End = r.End
+	} else if lastRange.End <= lastRange.Start {
+		// We wrapped; want to split around 0
+		// First shuffle everything up as we want results to be sorted
+		ranges = append(ranges, address.Range{})
+		copy(ranges[1:], ranges[:len(ranges)-1])
+		ranges[0] = address.Range{Start: r.Start, End: lastRange.End}
+		ranges[len(ranges)-1].End = r.End
+	}
+	return ranges
+}
+
+// OwnedRanges returns slice of Ranges, ordered by IP, indicating which
+// ranges are owned by this peer.  Will split ranges which
+// span 0 in the ring.
+func (r *Ring) OwnedRanges() (result []address.Range) {
+	r.assertInvariants()
+
+	for i, entry := range r.Entries {
+		if entry.Peer == r.Peer {
+			nextEntry := r.Entries.entry(i + 1)
+			result = append(result, address.Range{Start: entry.Token, End: nextEntry.Token})
+		}
+	}
+
+	return r.splitRangesOverZero(result)
+}
+
+// ClaimForPeers claims the entire ring for the array of peers passed
+// in.  Only works for empty rings.
+func (r *Ring) ClaimForPeers(peers []router.PeerName) {
+	common.Assert(r.Empty())
+	defer r.assertInvariants()
+	defer r.updateExportedVariables()
+
+	totalSize := r.distance(r.Start, r.End)
+	share := totalSize/address.Offset(len(peers)) + 1
+	remainder := totalSize % address.Offset(len(peers))
+	pos := r.Start
+
+	for i, peer := range peers {
+		if address.Offset(i) == remainder {
+			share--
+			if share == 0 {
+				break
+			}
+		}
+
+		if e, found := r.Entries.get(pos); found {
+			e.update(peer, share)
+		} else {
+			r.Entries.insert(entry{Token: pos, Peer: peer, Free: share})
+		}
+
+		pos += address.Address(share)
+	}
+
+	common.Assert(pos == r.End)
+}
+
+func (r *Ring) ClaimItAll() {
+	r.ClaimForPeers([]router.PeerName{r.Peer})
+}
+
+func (r *Ring) FprintWithNicknames(w io.Writer, m map[router.PeerName]string) {
+	for _, entry := range r.Entries {
+		nickname, found := m[entry.Peer]
+		if found {
+			nickname = fmt.Sprintf(" (%s)", nickname)
+		}
+
+		fmt.Fprintf(w, "\n  %s -> %s%s (v%d)", entry.Token,
+			entry.Peer, nickname, entry.Version)
+	}
+}
+
+func (r *Ring) String() string {
+	var buffer bytes.Buffer
+	fmt.Fprintf(&buffer, "Ring [%s, %s)", r.Start, r.End)
+	r.FprintWithNicknames(&buffer, make(map[router.PeerName]string))
+	return buffer.String()
+}
+
+// ReportFree is used by the allocator to tell the ring
+// how many free ips are in a given range, so that ChoosePeerToAskForSpace
+// can make more intelligent decisions.
+func (r *Ring) ReportFree(freespace map[address.Address]address.Offset) {
+	r.assertInvariants()
+	defer r.assertInvariants()
+	defer r.updateExportedVariables()
+
+	common.Assert(!r.Empty())
+	entries := r.Entries
+
+	// As OwnedRanges splits around the origin, we need to
+	// detect that here and fix up freespace
+	if free, found := freespace[r.Start]; found && entries.entry(0).Token != r.Start {
+		lastToken := entries.entry(-1).Token
+		prevFree, found := freespace[lastToken]
+		common.Assert(found)
+		freespace[lastToken] = prevFree + free
+		delete(freespace, r.Start)
+	}
+
+	for start, free := range freespace {
+		// Look for entry
+		i := sort.Search(len(entries), func(j int) bool {
+			return entries[j].Token >= start
+		})
+
+		// Are you trying to report free on space I don't own?
+		common.Assert(i < len(entries) && entries[i].Token == start && entries[i].Peer == r.Peer)
+
+		// Check we're not reporting more space than the range
+		entry, next := entries.entry(i), entries.entry(i+1)
+		maxSize := r.distance(entry.Token, next.Token)
+		common.Assert(free <= maxSize)
+
+		if entries[i].Free == free {
+			return
+		}
+
+		entries[i].Free = free
+		entries[i].Version++
+	}
+}
+
+// ChoosePeerToAskForSpace chooses a weighted-random peer to ask
+// for space.
+func (r *Ring) ChoosePeerToAskForSpace() (result router.PeerName, err error) {
+	var (
+		sum               address.Offset
+		totalSpacePerPeer = make(map[router.PeerName]address.Offset) // Compute total free space per peer
+	)
+
+	// iterate through tokens
+	for _, entry := range r.Entries {
+		// Ignore ranges with no free space
+		if entry.Free <= 0 {
+			continue
+		}
+
+		// Don't talk to yourself
+		if entry.Peer == r.Peer {
+			continue
+		}
+
+		totalSpacePerPeer[entry.Peer] += entry.Free
+		sum += entry.Free
+	}
+
+	if sum == 0 {
+		err = ErrNoFreeSpace
+		return
+	}
+
+	// Pick random peer, weighted by total free space
+	rn := rand.Int63n(int64(sum))
+	for peername, space := range totalSpacePerPeer {
+		rn -= int64(space)
+		if rn < 0 {
+			return peername, nil
+		}
+	}
+
+	panic("Should never reach this point")
+}
+
+func (r *Ring) PickPeerForTransfer() router.PeerName {
+	for _, entry := range r.Entries {
+		if entry.Peer != r.Peer {
+			return entry.Peer
+		}
+	}
+	return router.UnknownPeerName
+}
+
+// Transfer will mark all entries associated with 'from' peer as owned by 'to' peer
+// and return ranges indicating the new space we picked up
+func (r *Ring) Transfer(from, to router.PeerName) (error, []address.Range) {
+	r.assertInvariants()
+	defer r.assertInvariants()
+	defer r.updateExportedVariables()
+
+	var newRanges []address.Range
+	found := false
+
+	for i, entry := range r.Entries {
+		if entry.Peer == from {
+			found = true
+			entry.Peer = to
+			entry.Version++
+			newRanges = append(newRanges, address.Range{Start: entry.Token, End: r.Entries.entry(i + 1).Token})
+		}
+	}
+
+	if !found {
+		return ErrNotFound, nil
+	}
+
+	return nil, r.splitRangesOverZero(newRanges)
+}
+
+// Contains returns true if addr is in this ring
+func (r *Ring) Contains(addr address.Address) bool {
+	return addr >= r.Start && addr < r.End
+}
+
+// Owner returns the peername which owns the range containing addr
+func (r *Ring) Owner(token address.Address) router.PeerName {
+	common.Assert(r.Start <= token && token < r.End)
+
+	r.assertInvariants()
+	// There can be no owners on an empty ring
+	if r.Empty() {
+		return router.UnknownPeerName
+	}
+
+	// Look for the right-most entry, less than or equal to token
+	preceedingEntry := sort.Search(len(r.Entries), func(j int) bool {
+		return r.Entries[j].Token > token
+	})
+	preceedingEntry--
+	entry := r.Entries.entry(preceedingEntry)
+	return entry.Peer
+}
+
+// Get the set of PeerNames mentioned in the ring
+func (r *Ring) PeerNames() map[router.PeerName]struct{} {
+	res := make(map[router.PeerName]struct{})
+
+	for _, entry := range r.Entries {
+		res[entry.Peer] = struct{}{}
+	}
+
+	return res
+}

ipam/ring/ring_test.go

@@ -0,0 +1,724 @@
+package ring
+
+import (
+	"bytes"
+	"fmt"
+	"math/rand"
+	"sort"
+	"testing"
+
+	"github.com/weaveworks/weave/common"
+	"github.com/weaveworks/weave/ipam/address"
+	"github.com/weaveworks/weave/router"
+	wt "github.com/weaveworks/weave/testing"
+)
+
+var (
+	peer1name, _ = router.PeerNameFromString("01:00:00:00:02:00")
+	peer2name, _ = router.PeerNameFromString("02:00:00:00:02:00")
+	peer3name, _ = router.PeerNameFromString("03:00:00:00:02:00")
+
+	start, end    = ParseIP("10.0.0.0"), ParseIP("10.0.0.255")
+	dot10, dot245 = ParseIP("10.0.0.10"), ParseIP("10.0.0.245")
+	dot250        = ParseIP("10.0.0.250")
+	middle        = ParseIP("10.0.0.128")
+)
+
+func ParseIP(s string) address.Address {
+	addr, _ := address.ParseIP(s)
+	return addr
+}
+
+func TestInvariants(t *testing.T) {
+	ring := New(start, end, peer1name)
+
+	// Check ring is sorted
+	ring.Entries = []*entry{{Token: dot245, Peer: peer1name}, {Token: dot10, Peer: peer2name}}
+	wt.AssertTrue(t, ring.checkInvariants() == ErrNotSorted, "Expected error")
+
+	// Check tokens don't appear twice
+	ring.Entries = []*entry{{Token: dot245, Peer: peer1name}, {Token: dot245, Peer: peer2name}}
+	wt.AssertTrue(t, ring.checkInvariants() == ErrTokenRepeated, "Expected error")
+
+	// Check tokens are in bounds
+	ring = New(dot10, dot245, peer1name)
+	ring.Entries = []*entry{{Token: start, Peer: peer1name}}
+	wt.AssertTrue(t, ring.checkInvariants() == ErrTokenOutOfRange, "Expected error")
+
+	ring.Entries = []*entry{{Token: end, Peer: peer1name}}
+	wt.AssertTrue(t, ring.checkInvariants() == ErrTokenOutOfRange, "Expected error")
+}
+
+func TestInsert(t *testing.T) {
+	ring := New(start, end, peer1name)
+	ring.Entries = []*entry{{Token: start, Peer: peer1name, Free: 255}}
+
+	wt.AssertPanic(t, func() {
+		ring.Entries.insert(entry{Token: start, Peer: peer1name})
+	})
+
+	ring.Entries.entry(0).Free = 0
+	ring.Entries.insert(entry{Token: dot245, Peer: peer1name})
+	ring2 := New(start, end, peer1name)
+	ring2.Entries = []*entry{{Token: start, Peer: peer1name, Free: 0}, {Token: dot245, Peer: peer1name}}
+	wt.AssertEquals(t, ring, ring2)
+
+	ring.Entries.insert(entry{Token: dot10, Peer: peer1name})
+	ring2.Entries = []*entry{{Token: start, Peer: peer1name, Free: 0}, {Token: dot10, Peer: peer1name}, {Token: dot245, Peer: peer1name}}
+	wt.AssertEquals(t, ring, ring2)
+}
+
+func TestBetween(t *testing.T) {
+	ring1 := New(start, end, peer1name)
+	ring1.Entries = []*entry{{Token: start, Peer: peer1name, Free: 255}}
+
+	// First off, in a ring where everything is owned by the peer
+	// between should return true for everything
+	for i := 1; i <= 255; i++ {
+		ip := ParseIP(fmt.Sprintf("10.0.0.%d", i))
+		wt.AssertTrue(t, ring1.Entries.between(ip, 0, 1), "between should be true!")
+	}
+
+	// Now, construct a ring with entries at +10 and -10
+	// And check the correct behaviour
+
+	ring1.Entries = []*entry{{Token: dot10, Peer: peer1name}, {Token: dot245, Peer: peer2name}}
+	ring1.assertInvariants()
+	for i := 10; i <= 244; i++ {
+		ipStr := fmt.Sprintf("10.0.0.%d", i)
+		ip := ParseIP(ipStr)
+		wt.AssertTrue(t, ring1.Entries.between(ip, 0, 1),
+			fmt.Sprintf("Between should be true for %s!", ipStr))
+		wt.AssertFalse(t, ring1.Entries.between(ip, 1, 2),
+			fmt.Sprintf("Between should be false for %s!", ipStr))
+	}
+	for i := 0; i <= 9; i++ {
+		ipStr := fmt.Sprintf("10.0.0.%d", i)
+		ip := ParseIP(ipStr)
+		wt.AssertFalse(t, ring1.Entries.between(ip, 0, 1),
+			fmt.Sprintf("Between should be false for %s!", ipStr))
+		wt.AssertTrue(t, ring1.Entries.between(ip, 1, 2),
+			fmt.Sprintf("Between should be true for %s!", ipStr))
+	}
+	for i := 245; i <= 255; i++ {
+		ipStr := fmt.Sprintf("10.0.0.%d", i)
+		ip := ParseIP(ipStr)
+		wt.AssertFalse(t, ring1.Entries.between(ip, 0, 1),
+			fmt.Sprintf("Between should be false for %s!", ipStr))
+		wt.AssertTrue(t, ring1.Entries.between(ip, 1, 2),
+			fmt.Sprintf("Between should be true for %s!", ipStr))
+	}
+}
+
+func TestGrantSimple(t *testing.T) {
+	ring1 := New(start, end, peer1name)
+	ring2 := New(start, end, peer2name)
+
+	// Claim everything for peer1
+	ring1.ClaimItAll()
+	wt.AssertEquals(t, ring1.Entries, entries{{Token: start, Peer: peer1name, Free: 255}})
+
+	// Now grant everything to peer2
+	ring1.GrantRangeToHost(start, end, peer2name)
+	ring2.Entries = []*entry{{Token: start, Peer: peer2name, Free: 255, Version: 1}}
+	wt.AssertEquals(t, ring1.Entries, ring2.Entries)
+
+	// Now spint back to peer 1
+	ring2.GrantRangeToHost(dot10, end, peer1name)
+	ring1.Entries = []*entry{{Token: start, Peer: peer2name, Free: 10, Version: 2},
+		{Token: dot10, Peer: peer1name, Free: 245}}
+	wt.AssertEquals(t, ring1.Entries, ring2.Entries)
+
+	// And spint back to peer 2 again
+	ring1.GrantRangeToHost(dot245, end, peer2name)
+	wt.AssertEquals(t, ring1.Entries, entries{{Token: start, Peer: peer2name, Free: 10, Version: 2},
+		{Token: dot10, Peer: peer1name, Free: 235, Version: 1},
+		{Token: dot245, Peer: peer2name, Free: 10}})
+
+	// Grant range spanning a live token
+	ring1.Entries = []*entry{{Token: start, Peer: peer1name, Free: 10, Version: 2},
+		{Token: dot10, Peer: peer1name, Free: 235}, {Token: dot245, Peer: peer1name, Free: 10}}
+	ring1.GrantRangeToHost(dot10, end, peer2name)
+	wt.AssertEquals(t, ring1.Entries, entries{{Token: start, Peer: peer1name, Free: 10, Version: 2},
+		{Token: dot10, Peer: peer2name, Free: 235, Version: 1},
+		{Token: dot245, Peer: peer2name, Free: 10, Version: 1}})
+}
+
+func TestGrantSplit(t *testing.T) {
+	ring1 := New(start, end, peer1name)
+	ring2 := New(start, end, peer2name)
+
+	// Claim everything for peer1
+	ring1.Entries = []*entry{{Token: start, Peer: peer1name, Free: 255}}
+	ring2.Merge(*ring1)
+	wt.AssertEquals(t, ring1.Entries, ring2.Entries)
+
+	// Now grant a split range to peer2
+	ring1.GrantRangeToHost(dot10, dot245, peer2name)
+	wt.AssertEquals(t, ring1.Entries, entries{{Token: start, Peer: peer1name, Free: 10, Version: 1},
+		{Token: dot10, Peer: peer2name, Free: 235},
+		{Token: dot245, Peer: peer1name, Free: 10}})
+	ring1.assertInvariants()
+
+	// Grant range spanning a live token, and inserting a new token
+	ring1.Entries = []*entry{{Token: start, Peer: peer1name, Free: 10, Version: 2},
+		{Token: dot10, Peer: peer1name, Free: 118}, {Token: middle, Peer: peer1name, Free: 127}}
+	ring1.GrantRangeToHost(dot10, dot245, peer2name)
+	wt.AssertEquals(t, ring1.Entries, entries{{Token: start, Peer: peer1name, Free: 10, Version: 2},
+		{Token: dot10, Peer: peer2name, Free: 118, Version: 1},
+		{Token: middle, Peer: peer2name, Free: 117, Version: 1},
+		{Token: dot245, Peer: peer1name, Free: 10, Version: 0}})
+	ring1.assertInvariants()
+}
+
+func TestMergeSimple(t *testing.T) {
+	ring1 := New(start, end, peer1name)
+	ring2 := New(start, end, peer2name)
+
+	// Claim everything for peer1
+	ring1.ClaimItAll()
+	ring1.GrantRangeToHost(middle, end, peer2name)
+	wt.AssertSuccess(t, ring2.Merge(*ring1))
+
+	wt.AssertEquals(t, ring1.Entries, entries{{Token: start, Peer: peer1name, Free: 128, Version: 1},
+		{Token: middle, Peer: peer2name, Free: 127}})
+	wt.AssertEquals(t, ring1.Entries, ring2.Entries)
+
+	// Now to two different operations on either side,
+	// check we can Merge again
+	ring1.GrantRangeToHost(start, middle, peer2name)
+	ring2.GrantRangeToHost(middle, end, peer1name)
+
+	wt.AssertSuccess(t, ring2.Merge(*ring1))
+	wt.AssertSuccess(t, ring1.Merge(*ring2))
+
+	wt.AssertEquals(t, ring1.Entries, entries{{Token: start, Peer: peer2name, Free: 128, Version: 2},
+		{Token: middle, Peer: peer1name, Free: 127, Version: 1}})
+	wt.AssertEquals(t, ring1.Entries, ring2.Entries)
+}
+
+func TestMergeErrors(t *testing.T) {
+	// Cannot Merge in an invalid ring
+	ring1 := New(start, end, peer1name)
+	ring2 := New(start, end, peer2name)
+	ring2.Entries = []*entry{{Token: middle, Peer: peer2name}, {Token: start, Peer: peer2name}}
+	wt.AssertTrue(t, ring1.Merge(*ring2) == ErrNotSorted, "Expected ErrNotSorted")
+
+	// Should Merge two rings for different ranges
+	ring2 = New(start, middle, peer2name)
+	ring2.Entries = []*entry{}
+	wt.AssertTrue(t, ring1.Merge(*ring2) == ErrDifferentSubnets, "Expected ErrDifferentSubnets")
+
+	// Cannot Merge newer version of entry I own
+	ring2 = New(start, end, peer2name)
+	ring1.Entries = []*entry{{Token: start, Peer: peer1name}}
+	ring2.Entries = []*entry{{Token: start, Peer: peer1name, Version: 1}}
+	wt.AssertTrue(t, ring1.Merge(*ring2) == ErrNewerVersion, "Expected ErrNewerVersion")
+
+	// Cannot Merge two entries with same version but different hosts
+	ring1.Entries = []*entry{{Token: start, Peer: peer1name}}
+	ring2.Entries = []*entry{{Token: start, Peer: peer2name}}
+	wt.AssertTrue(t, ring1.Merge(*ring2) == ErrInvalidEntry, "Expected ErrInvalidEntry")
+
+	// Cannot Merge an entry into a range I own
+	ring1.Entries = []*entry{{Token: start, Peer: peer1name}}
+	ring2.Entries = []*entry{{Token: middle, Peer: peer2name}}
+	wt.AssertTrue(t, ring1.Merge(*ring2) == ErrEntryInMyRange, "Expected ErrEntryInMyRange")
+}
+
+func TestMergeMore(t *testing.T) {
+	ring1 := New(start, end, peer1name)
+	ring2 := New(start, end, peer2name)
+
+	assertRing := func(ring *Ring, entries entries) {
+		wt.AssertEquals(t, ring.Entries, entries)
+	}
+
+	assertRing(ring1, []*entry{})
+	assertRing(ring2, []*entry{})
+
+	// Claim everything for peer1
+	ring1.ClaimItAll()
+	assertRing(ring1, []*entry{{Token: start, Peer: peer1name, Free: 255}})
+	assertRing(ring2, []*entry{})
+
+	// Check the Merge sends it to the other ring
+	wt.AssertSuccess(t, ring2.Merge(*ring1))
+	assertRing(ring1, []*entry{{Token: start, Peer: peer1name, Free: 255}})
+	assertRing(ring2, []*entry{{Token: start, Peer: peer1name, Free: 255}})
+
+	// Give everything to peer2
+	ring1.GrantRangeToHost(start, end, peer2name)
+	assertRing(ring1, []*entry{{Token: start, Peer: peer2name, Free: 255, Version: 1}})
+	assertRing(ring2, []*entry{{Token: start, Peer: peer1name, Free: 255}})
+
+	wt.AssertSuccess(t, ring2.Merge(*ring1))
+	assertRing(ring1, []*entry{{Token: start, Peer: peer2name, Free: 255, Version: 1}})
+	assertRing(ring2, []*entry{{Token: start, Peer: peer2name, Free: 255, Version: 1}})
+
+	// And carve off some space
+	ring2.GrantRangeToHost(middle, end, peer1name)
+	assertRing(ring2, []*entry{{Token: start, Peer: peer2name, Free: 128, Version: 2},
+		{Token: middle, Peer: peer1name, Free: 127}})
+	assertRing(ring1, []*entry{{Token: start, Peer: peer2name, Free: 255, Version: 1}})
+
+	// And Merge back
+	wt.AssertSuccess(t, ring1.Merge(*ring2))
+	assertRing(ring1, []*entry{{Token: start, Peer: peer2name, Free: 128, Version: 2},
+		{Token: middle, Peer: peer1name, Free: 127}})
+	assertRing(ring2, []*entry{{Token: start, Peer: peer2name, Free: 128, Version: 2},
+		{Token: middle, Peer: peer1name, Free: 127}})
+
+	// This should be a no-op
+	wt.AssertSuccess(t, ring2.Merge(*ring1))
+	assertRing(ring1, []*entry{{Token: start, Peer: peer2name, Free: 128, Version: 2},
+		{Token: middle, Peer: peer1name, Free: 127}})
+	assertRing(ring2, []*entry{{Token: start, Peer: peer2name, Free: 128, Version: 2},
+		{Token: middle, Peer: peer1name, Free: 127}})
+}
+
+func TestMergeSplit(t *testing.T) {
+	ring1 := New(start, end, peer1name)
+	ring2 := New(start, end, peer2name)
+
+	// Claim everything for peer2
+	ring1.Entries = []*entry{{Token: start, Peer: peer2name, Free: 255}}
+	wt.AssertSuccess(t, ring2.Merge(*ring1))
+	wt.AssertEquals(t, ring1.Entries, ring2.Entries)
+
+	// Now grant a split range to peer1
+	ring2.GrantRangeToHost(dot10, dot245, peer1name)
+	wt.AssertEquals(t, ring2.Entries, entries{{Token: start, Peer: peer2name, Free: 10, Version: 1},
+		{Token: dot10, Peer: peer1name, Free: 235},
+		{Token: dot245, Peer: peer2name, Free: 10}})
+	wt.AssertSuccess(t, ring1.Merge(*ring2))
+	wt.AssertEquals(t, ring1.Entries, entries{{Token: start, Peer: peer2name, Free: 10, Version: 1},
+		{Token: dot10, Peer: peer1name, Free: 235},
+		{Token: dot245, Peer: peer2name, Free: 10}})
+	wt.AssertEquals(t, ring1.Entries, ring2.Entries)
+}
+
+func TestMergeSplit2(t *testing.T) {
+	ring1 := New(start, end, peer1name)
+	ring2 := New(start, end, peer2name)
+
+	// Claim everything for peer2
+	ring1.Entries = []*entry{{Token: start, Peer: peer2name, Free: 250}, {Token: dot250, Peer: peer2name, Free: 5}}
+	wt.AssertSuccess(t, ring2.Merge(*ring1))
+	wt.AssertEquals(t, ring1.Entries, ring2.Entries)
+
+	// Now grant a split range to peer1
+	ring2.GrantRangeToHost(dot10, dot245, peer1name)
+	wt.AssertEquals(t, ring2.Entries, entries{{Token: start, Peer: peer2name, Free: 10, Version: 1},
+		{Token: dot10, Peer: peer1name, Free: 235},
+		{Token: dot245, Peer: peer2name, Free: 5}, {Token: dot250, Peer: peer2name, Free: 5}})
+	wt.AssertSuccess(t, ring1.Merge(*ring2))
+	wt.AssertEquals(t, ring1.Entries, entries{{Token: start, Peer: peer2name, Free: 10, Version: 1},
+		{Token: dot10, Peer: peer1name, Free: 235},
+		{Token: dot245, Peer: peer2name, Free: 5}, {Token: dot250, Peer: peer2name, Free: 5}})
+	wt.AssertEquals(t, ring1.Entries, ring2.Entries)
+}
+
+// A simple test, very similar to above, but using the marshalling to byte[]s
+func TestGossip(t *testing.T) {
+	ring1 := New(start, end, peer1name)
+	ring2 := New(start, end, peer2name)
+
+	assertRing := func(ring *Ring, entries entries) {
+		wt.AssertEquals(t, ring.Entries, entries)
+	}
+
+	assertRing(ring1, []*entry{})
+	assertRing(ring2, []*entry{})
+
+	// Claim everything for peer1
+	ring1.ClaimItAll()
+	assertRing(ring1, []*entry{{Token: start, Peer: peer1name, Free: 255}})
+	assertRing(ring2, []*entry{})
+
+	// Check the Merge sends it to the other ring
+	wt.AssertSuccess(t, ring2.Merge(*ring1))
+	assertRing(ring1, []*entry{{Token: start, Peer: peer1name, Free: 255}})
+	assertRing(ring2, []*entry{{Token: start, Peer: peer1name, Free: 255}})
+}
+
+func TestFindFree(t *testing.T) {
+	ring1 := New(start, end, peer1name)
+
+	_, err := ring1.ChoosePeerToAskForSpace()
+	wt.AssertTrue(t, err == ErrNoFreeSpace, "Expected ErrNoFreeSpace")
+
+	ring1.Entries = []*entry{{Token: start, Peer: peer1name}}
+	_, err = ring1.ChoosePeerToAskForSpace()
+	wt.AssertTrue(t, err == ErrNoFreeSpace, "Expected ErrNoFreeSpace")
+
+	// We shouldn't return outselves
+	ring1.ReportFree(map[address.Address]address.Offset{start: 10})
+	_, err = ring1.ChoosePeerToAskForSpace()
+	wt.AssertTrue(t, err == ErrNoFreeSpace, "Expected ErrNoFreeSpace")
+
+	ring1.Entries = []*entry{{Token: start, Peer: peer1name, Free: 1},
+		{Token: start, Peer: peer1name, Free: 1}}
+	_, err = ring1.ChoosePeerToAskForSpace()
+	wt.AssertTrue(t, err == ErrNoFreeSpace, "Expected ErrNoFreeSpace")
+
+	// We should return others
+	ring1.Entries = []*entry{{Token: start, Peer: peer2name, Free: 1}}
+	peer, err := ring1.ChoosePeerToAskForSpace()
+	wt.AssertSuccess(t, err)
+	wt.AssertEquals(t, peer, peer2name)
+
+	ring1.Entries = []*entry{{Token: start, Peer: peer2name, Free: 1},
+		{Token: start, Peer: peer2name, Free: 1}}
+	peer, err = ring1.ChoosePeerToAskForSpace()
+	wt.AssertSuccess(t, err)
+	wt.AssertEquals(t, peer, peer2name)
+}
+
+func TestReportFree(t *testing.T) {
+	ring1 := New(start, end, peer1name)
+	ring2 := New(start, end, peer2name)
+
+	ring1.ClaimItAll()
+	ring1.GrantRangeToHost(middle, end, peer2name)
+	wt.AssertSuccess(t, ring2.Merge(*ring1))
+
+	freespace := make(map[address.Address]address.Offset)
+	for _, r := range ring2.OwnedRanges() {
+		freespace[r.Start] = 0
+	}
+	ring2.ReportFree(freespace)
+}
+
+func TestMisc(t *testing.T) {
+	ring := New(start, end, peer1name)
+
+	wt.AssertTrue(t, ring.Empty(), "empty")
+
+	ring.ClaimItAll()
+	println(ring.String())
+}
+
+func TestEmptyGossip(t *testing.T) {
+	ring1 := New(start, end, peer1name)
+	ring2 := New(start, end, peer2name)
+
+	ring1.ClaimItAll()
+	// This used to panic, and it shouldn't
+	wt.AssertSuccess(t, ring1.Merge(*ring2))
+}
+
+func TestMergeOldMessage(t *testing.T) {
+	ring1 := New(start, end, peer1name)
+	ring2 := New(start, end, peer2name)
+
+	ring1.ClaimItAll()
+	wt.AssertSuccess(t, ring2.Merge(*ring1))
+
+	ring1.GrantRangeToHost(middle, end, peer1name)
+	wt.AssertSuccess(t, ring1.Merge(*ring2))
+}
+
+func TestSplitRangeAtBeginning(t *testing.T) {
+	ring1 := New(start, end, peer1name)
+	ring2 := New(start, end, peer2name)
+
+	ring1.ClaimItAll()
+	wt.AssertSuccess(t, ring2.Merge(*ring1))
+
+	ring1.GrantRangeToHost(start, middle, peer2name)
+	wt.AssertSuccess(t, ring2.Merge(*ring1))
+}
+
+func TestOwnedRange(t *testing.T) {
+	ring1 := New(start, end, peer1name)
+	ring1.ClaimItAll()
+
+	wt.AssertEquals(t, ring1.OwnedRanges(), []address.Range{{Start: start, End: end}})
+
+	ring1.GrantRangeToHost(middle, end, peer2name)
+	wt.AssertEquals(t, ring1.OwnedRanges(), []address.Range{{Start: start, End: middle}})
+
+	ring2 := New(start, end, peer2name)
+	ring2.Merge(*ring1)
+	wt.AssertEquals(t, ring2.OwnedRanges(), []address.Range{{Start: middle, End: end}})
+
+	ring2.Entries = []*entry{{Token: middle, Peer: peer2name}}
+	wt.AssertEquals(t, ring2.OwnedRanges(),
+		[]address.Range{{Start: start, End: middle}, {Start: middle, End: end}})
+
+	ring2.Entries = []*entry{{Token: dot10, Peer: peer2name}, {Token: middle, Peer: peer2name}}
+	wt.AssertEquals(t, ring2.OwnedRanges(),
+		[]address.Range{{Start: start, End: dot10}, {Start: dot10, End: middle},
+			{Start: middle, End: end}})
+}
+
+func TestTransfer(t *testing.T) {
+	// First test just checks if we can grant some range to a host, when we transfer it, we get it back
+	ring1 := New(start, end, peer1name)
+	ring1.ClaimItAll()
+	ring1.GrantRangeToHost(middle, end, peer2name)
+	ring1.Transfer(peer2name, peer1name)
+	wt.AssertEquals(t, ring1.OwnedRanges(), []address.Range{{start, middle}, {middle, end}})
+
+	// Second test is what happens when a token exists at the end of a range but is transferred
+	// - does it get resurrected correctly?
+	ring1 = New(start, end, peer1name)
+	ring1.ClaimItAll()
+	ring1.GrantRangeToHost(middle, end, peer2name)
+	ring1.Transfer(peer2name, peer1name)
+	ring1.GrantRangeToHost(dot10, middle, peer2name)
+	wt.AssertEquals(t, ring1.OwnedRanges(), []address.Range{{start, dot10}, {middle, end}})
+}
+
+func TestOwner(t *testing.T) {
+	ring1 := New(start, end, peer1name)
+	wt.AssertTrue(t, ring1.Contains(start), "start should be in ring")
+	wt.AssertFalse(t, ring1.Contains(end), "end should not be in ring")
+
+	wt.AssertEquals(t, ring1.Owner(start), router.UnknownPeerName)
+
+	ring1.ClaimItAll()
+	ring1.GrantRangeToHost(middle, end, peer2name)
+	wt.AssertEquals(t, ring1.Owner(start), peer1name)
+	wt.AssertEquals(t, ring1.Owner(middle), peer2name)
+	wt.AssertPanic(t, func() {
+		ring1.Owner(end)
+	})
+}
+
+type addressSlice []address.Address
+
+func (s addressSlice) Len() int           { return len(s) }
+func (s addressSlice) Less(i, j int) bool { return s[i] < s[j] }
+func (s addressSlice) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
+
+func TestFuzzRing(t *testing.T) {
+	var (
+		numPeers   = 25
+		iterations = 1000
+	)
+
+	peers := make([]router.PeerName, numPeers)
+	for i := 0; i < numPeers; i++ {
+		peer, _ := router.PeerNameFromString(fmt.Sprintf("%02d:00:00:00:02:00", i))
+		peers[i] = peer
+	}
+
+	// Make a valid, random ring
+	makeGoodRandomRing := func() *Ring {
+		addressSpace := end - start
+		numTokens := rand.Intn(int(addressSpace))
+
+		tokenMap := make(map[address.Address]bool)
+		for i := 0; i < numTokens; i++ {
+			tokenMap[address.Address(rand.Intn(int(addressSpace)))] = true
+		}
+		var tokens []address.Address
+		for token := range tokenMap {
+			tokens = append(tokens, token)
+		}
+		sort.Sort(addressSlice(tokens))
+
+		peer := peers[rand.Intn(len(peers))]
+		ring := New(start, end, peer)
+		for _, token := range tokens {
+			peer = peers[rand.Intn(len(peers))]
+			ring.Entries = append(ring.Entries, &entry{Token: start + token, Peer: peer})
+		}
+
+		ring.assertInvariants()
+		return ring
+	}
+
+	for i := 0; i < iterations; i++ {
+		// make 2 random rings
+		ring1 := makeGoodRandomRing()
+		ring2 := makeGoodRandomRing()
+
+		// Merge them - this might fail, we don't care
+		// We just want to make sure it doesn't panic
+		ring1.Merge(*ring2)
+
+		// Check whats left still passes assertions
+		ring1.assertInvariants()
+		ring2.assertInvariants()
+	}
+
+	// Make an invalid, random ring
+	makeBadRandomRing := func() *Ring {
+		addressSpace := end - start
+		numTokens := rand.Intn(int(addressSpace))
+		tokens := make([]address.Address, numTokens)
+		for i := 0; i < numTokens; i++ {
+			tokens[i] = address.Address(rand.Intn(int(addressSpace)))
+		}
+
+		peer := peers[rand.Intn(len(peers))]
+		ring := New(start, end, peer)
+		for _, token := range tokens {
+			peer = peers[rand.Intn(len(peers))]
+			ring.Entries = append(ring.Entries, &entry{Token: start + token, Peer: peer})
+		}
+
+		return ring
+	}
+
+	for i := 0; i < iterations; i++ {
+		// make 2 random rings
+		ring1 := makeGoodRandomRing()
+		ring2 := makeBadRandomRing()
+
+		// Merge them - this might fail, we don't care
+		// We just want to make sure it doesn't panic
+		ring1.Merge(*ring2)
+
+		// Check whats left still passes assertions
+		ring1.assertInvariants()
+	}
+}
+
+func TestFuzzRingHard(t *testing.T) {
+	//common.InitDefaultLogging(true)
+	var (
+		numPeers   = 100
+		iterations = 3000
+		peers      []router.PeerName
+		rings      []*Ring
+		nextPeerID = 0
+	)
+
+	addPeer := func() {
+		peer, _ := router.PeerNameFromString(fmt.Sprintf("%02d:%02d:00:00:00:00", nextPeerID/10, nextPeerID%10))
+		common.Debug.Printf("%s: Adding peer", peer)
+		nextPeerID++
+		peers = append(peers, peer)
+		rings = append(rings, New(start, end, peer))
+	}
+
+	for i := 0; i < numPeers; i++ {
+		addPeer()
+	}
+
+	rings[0].ClaimItAll()
+
+	randomPeer := func(exclude int) (int, router.PeerName, *Ring) {
+		var peerIndex int
+		if exclude >= 0 {
+			peerIndex = rand.Intn(len(peers) - 1)
+			if peerIndex == exclude {
+				peerIndex++
+			}
+		} else {
+			peerIndex = rand.Intn(len(peers))
+		}
+		return peerIndex, peers[peerIndex], rings[peerIndex]
+	}
+
+	// Keep a map of index -> ranges, as these are a little expensive to
+	// calculate for every ring on every iteration.
+	var theRanges = make(map[int][]address.Range)
+	theRanges[0] = rings[0].OwnedRanges()
+
+	addOrRmPeer := func() {
+		if len(peers) < numPeers {
+			addPeer()
+			return
+		}
+
+		peerIndex, peername, _ := randomPeer(-1)
+		// Remove peer from our state
+		peers = append(peers[:peerIndex], peers[peerIndex+1:]...)
+		rings = append(rings[:peerIndex], rings[peerIndex+1:]...)
+		theRanges = make(map[int][]address.Range)
+
+		// Transfer the space for this peer on another peer, but not this one
+		_, otherPeername, otherRing := randomPeer(peerIndex)
+
+		// We need to be in a ~converged ring to rmpeer
+		for _, ring := range rings {
+			wt.AssertSuccess(t, otherRing.Merge(*ring))
+		}
+
+		common.Debug.Printf("%s: transferring from peer %s", otherPeername, peername)
+		otherRing.Transfer(peername, peername)
+
+		// And now tell everyone about the transfer - rmpeer is
+		// not partition safe
+		for i, ring := range rings {
+			wt.AssertSuccess(t, ring.Merge(*otherRing))
+			theRanges[i] = ring.OwnedRanges()
+		}
+	}
+
+	doGrantOrGossip := func() {
+		var ringsWithRanges = make([]int, 0, len(rings))
+		for index, ranges := range theRanges {
+			if len(ranges) > 0 {
+				ringsWithRanges = append(ringsWithRanges, index)
+			}
+		}
+
+		if len(ringsWithRanges) > 0 {
+			// Produce a random split in a random owned range, given to a random peer
+			indexWithRanges := ringsWithRanges[rand.Intn(len(ringsWithRanges))]
+			ownedRanges := theRanges[indexWithRanges]
+			ring := rings[indexWithRanges]
+
+			rangeToSplit := ownedRanges[rand.Intn(len(ownedRanges))]
+			size := address.Subtract(rangeToSplit.End, rangeToSplit.Start)
+			ipInRange := address.Add(rangeToSplit.Start, address.Offset(rand.Intn(int(size))))
+			_, peerToGiveTo, _ := randomPeer(-1)
+			common.Debug.Printf("%s: Granting [%v, %v) to %s", ring.Peer, ipInRange, rangeToSplit.End, peerToGiveTo)
+			ring.GrantRangeToHost(ipInRange, rangeToSplit.End, peerToGiveTo)
+
+			// Now 'gossip' this to a random host (note, note could be same host as above)
+			otherIndex, _, otherRing := randomPeer(-1)
+			common.Debug.Printf("%s: 'Gossiping' to %s", ring.Peer, otherRing.Peer)
+			wt.AssertSuccess(t, otherRing.Merge(*ring))
+
+			theRanges[indexWithRanges] = ring.OwnedRanges()
+			theRanges[otherIndex] = otherRing.OwnedRanges()
+			return
+		}
+
+		// No rings think they own anything (as gossip might be behind)
+		// We're going to pick a random host (which has entries) and gossip
+		// it to a random host (which may or may not have entries).
+		var ringsWithEntries = make([]*Ring, 0, len(rings))
+		for _, ring := range rings {
+			if len(ring.Entries) > 0 {
+				ringsWithEntries = append(ringsWithEntries, ring)
+			}
+		}
+		ring1 := ringsWithEntries[rand.Intn(len(ringsWithEntries))]
+		ring2index, _, ring2 := randomPeer(-1)
+		common.Debug.Printf("%s: 'Gossiping' to %s", ring1.Peer, ring2.Peer)
+		wt.AssertSuccess(t, ring2.Merge(*ring1))
+		theRanges[ring2index] = ring2.OwnedRanges()
+	}
+
+	for i := 0; i < iterations; i++ {
+		// about 1 in 10 times, rmpeer or add host
+		n := rand.Intn(10)
+		switch {
+		case n < 1:
+			addOrRmPeer()
+		default:
+			doGrantOrGossip()
+		}
+	}
+}
+
+func (es entries) String() string {
+	var buffer bytes.Buffer
+	fmt.Fprintf(&buffer, "[")
+	for i, entry := range es {
+		fmt.Fprintf(&buffer, "%+v", *entry)
+		if i+1 < len(es) {
+			fmt.Fprintf(&buffer, " ")
+		}
+	}
+	fmt.Fprintf(&buffer, "]")
+	return buffer.String()
+}

ipam/space/space.go

@@ -0,0 +1,248 @@
+package space
+
+import (
+	"bytes"
+	"fmt"
+	"sort"
+
+	"github.com/weaveworks/weave/common"
+	"github.com/weaveworks/weave/ipam/address"
+)
+
+type Addr address.Address
+
+type Space struct {
+	// ours and free represent a set of addresses as a sorted
+	// sequences of ranges.  Even elements give the inclusive
+	// starting points of ranges, and odd elements give the
+	// exclusive ending points.  Ranges in an array do not
+	// overlap, and neighbouring ranges are always coalesced if
+	// possible, so the arrays consist of sorted Addrs without
+	// repetition.
+	ours []address.Address
+	free []address.Address
+}
+
+func New() *Space {
+	return &Space{}
+}
+
+func (s *Space) Add(start address.Address, size address.Offset) {
+	s.free = add(s.free, start, address.Add(start, size))
+}
+
+// Clear removes all spaces from this space set.  Used during node shutdown.
+func (s *Space) Clear() {
+	s.free = s.free[:0]
+	s.ours = s.ours[:0]
+}
+
+func (s *Space) Allocate() (bool, address.Address) {
+	if len(s.free) == 0 {
+		return false, 0
+	} else {
+		res := s.free[0]
+		s.ours = add(s.ours, res, res+1)
+		s.free = subtract(s.free, res, res+1)
+		return true, res
+	}
+}
+
+func (s *Space) Claim(addr address.Address) error {
+	if !contains(s.free, addr) {
+		return fmt.Errorf("Address %v is not free to claim", addr)
+	}
+
+	s.ours = add(s.ours, addr, addr+1)
+	s.free = subtract(s.free, addr, addr+1)
+	return nil
+}
+
+func (s *Space) NumFreeAddresses() address.Offset {
+	res := address.Offset(0)
+	for i := 0; i < len(s.free); i += 2 {
+		res += address.Subtract(s.free[i+1], s.free[i])
+	}
+	return res
+}
+
+func (s *Space) NumFreeAddressesInRange(start, end address.Address) address.Offset {
+	res := address.Offset(0)
+	for i := 0; i < len(s.free); i += 2 {
+		s, e := s.free[i], s.free[i+1]
+		if s < start {
+			s = start
+		}
+		if e > end {
+			e = end
+		}
+		if s >= e {
+			continue
+		}
+		res += address.Subtract(e, s)
+	}
+	return res
+}
+
+func (s *Space) Free(addr address.Address) error {
+	if !contains(s.ours, addr) {
+		return fmt.Errorf("Address %v is not ours", addr)
+	}
+	if contains(s.free, addr) {
+		return fmt.Errorf("Address %v is already free", addr)
+	}
+
+	s.ours = subtract(s.ours, addr, addr+1)
+	s.free = add(s.free, addr, addr+1)
+	return nil
+}
+
+func (s *Space) biggestFreeRange() (int, address.Offset) {
+	pos := -1
+	biggest := address.Offset(0)
+
+	for i := 0; i < len(s.free); i += 2 {
+		size := address.Subtract(s.free[i+1], s.free[i])
+		if size >= biggest {
+			pos = i
+			biggest = size
+		}
+	}
+	return pos, biggest
+}
+
+func (s *Space) Donate() (address.Address, address.Offset, bool) {
+	if len(s.free) == 0 {
+		return 0, 0, false
+	}
+
+	pos, biggest := s.biggestFreeRange()
+
+	// Donate half of that biggest free range, rounding up so
+	// that the donation can't be empty
+	end := s.free[pos+1]
+	start := end - address.Address((biggest+1)/2)
+
+	s.ours = subtract(s.ours, start, end)
+	s.free = subtract(s.free, start, end)
+	return start, address.Subtract(end, start), true
+}
+
+func firstGreater(a []address.Address, x address.Address) int {
+	return sort.Search(len(a), func(i int) bool { return a[i] > x })
+}
+
+func firstGreaterOrEq(a []address.Address, x address.Address) int {
+	return sort.Search(len(a), func(i int) bool { return a[i] >= x })
+}
+
+// Do the ranges contain the given address?
+func contains(addrs []address.Address, addr address.Address) bool {
+	return firstGreater(addrs, addr)&1 != 0
+}
+
+// Take the union of the range [start, end) with the ranges in the array
+func add(addrs []address.Address, start address.Address, end address.Address) []address.Address {
+	return addSub(addrs, start, end, 0)
+}
+
+// Subtract the range [start, end) from the ranges in the array
+func subtract(addrs []address.Address, start address.Address, end address.Address) []address.Address {
+	return addSub(addrs, start, end, 1)
+}
+
+func addSub(addrs []address.Address, start address.Address, end address.Address, sense int) []address.Address {
+	start_pos := firstGreaterOrEq(addrs, start)
+	end_pos := firstGreater(addrs[start_pos:], end) + start_pos
+
+	// Boundaries up to start_pos are unaffected
+	res := make([]address.Address, start_pos, len(addrs)+2)
+	copy(res, addrs)
+
+	// Include start and end as new boundaries if they lie
+	// outside/inside existing ranges (according to sense).
+	if start_pos&1 == sense {
+		res = append(res, start)
+	}
+
+	if end_pos&1 == sense {
+		res = append(res, end)
+	}
+
+	// Boundaries after end_pos are unaffected
+	return append(res, addrs[end_pos:]...)
+}
+
+func (s *Space) String() string {
+	var buf bytes.Buffer
+	if len(s.ours) > 0 {
+		fmt.Fprint(&buf, "owned:")
+		for i := 0; i < len(s.ours); i += 2 {
+			fmt.Fprintf(&buf, " %s+%d ", s.ours[i], s.ours[i+1]-s.ours[i])
+		}
+	}
+	if len(s.free) > 0 {
+		fmt.Fprintf(&buf, "free:")
+		for i := 0; i < len(s.free); i += 2 {
+			fmt.Fprintf(&buf, " %s+%d ", s.free[i], s.free[i+1]-s.free[i])
+		}
+	}
+	if len(s.ours) == 0 && len(s.free) == 0 {
+		fmt.Fprintf(&buf, "No address ranges owned")
+	}
+	return buf.String()
+}
+
+type addressSlice []address.Address
+
+func (p addressSlice) Len() int           { return len(p) }
+func (p addressSlice) Less(i, j int) bool { return p[i] < p[j] }
+func (p addressSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
+
+func (s *Space) assertInvariants() {
+	common.Assert(sort.IsSorted(addressSlice(s.ours)))
+	common.Assert(sort.IsSorted(addressSlice(s.free)))
+}
+
+// Return a slice representing everything we own, whether it is free or not
+func (s *Space) everything() []address.Address {
+	a := make([]address.Address, len(s.ours))
+	copy(a, s.ours)
+	for i := 0; i < len(s.free); i += 2 {
+		a = add(a, s.free[i], s.free[i+1])
+	}
+	return a
+}
+
+// OwnedRanges returns slice of Ranges, ordered by IP, gluing together
+// contiguous sequences of owned and free addresses
+func (s *Space) OwnedRanges() []address.Range {
+	everything := s.everything()
+	result := make([]address.Range, len(everything)/2)
+	for i := 0; i < len(everything); i += 2 {
+		result[i/2] = address.Range{Start: everything[i], End: everything[i+1]}
+	}
+	return result
+}
+
+// Create a Space that has free space in all the supplied Ranges.
+func (s *Space) AddRanges(ranges []address.Range) {
+	for _, r := range ranges {
+		s.free = add(s.free, r.Start, r.End)
+	}
+}
+
+// Taking ranges to be a set of all space we should own, add in any excess as free space
+func (s *Space) UpdateRanges(ranges []address.Range) {
+	new := []address.Address{}
+	for _, r := range ranges {
+		new = add(new, r.Start, r.End)
+	}
+	current := s.everything()
+	for i := 0; i < len(current); i += 2 {
+		new = subtract(new, current[i], current[i+1])
+	}
+	for i := 0; i < len(new); i += 2 {
+		s.free = add(s.free, new[i], new[i+1])
+	}
+}

ipam/space/space_test.go

@@ -0,0 +1,249 @@
+package space
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/weaveworks/weave/ipam/address"
+	wt "github.com/weaveworks/weave/testing"
+)
+
+func makeSpace(start address.Address, size address.Offset) *Space {
+	s := New()
+	s.Add(start, size)
+	return s
+}
+
+func ip(s string) address.Address {
+	addr, _ := address.ParseIP(s)
+	return addr
+}
+
+func TestLowlevel(t *testing.T) {
+	a := []address.Address{}
+	a = add(a, 100, 200)
+	wt.AssertEquals(t, a, []address.Address{100, 200})
+	wt.AssertTrue(t, !contains(a, 99), "")
+	wt.AssertTrue(t, contains(a, 100), "")
+	wt.AssertTrue(t, contains(a, 199), "")
+	wt.AssertTrue(t, !contains(a, 200), "")
+	a = add(a, 700, 800)
+	wt.AssertEquals(t, a, []address.Address{100, 200, 700, 800})
+	a = add(a, 300, 400)
+	wt.AssertEquals(t, a, []address.Address{100, 200, 300, 400, 700, 800})
+	a = add(a, 400, 500)
+	wt.AssertEquals(t, a, []address.Address{100, 200, 300, 500, 700, 800})
+	a = add(a, 600, 700)
+	wt.AssertEquals(t, a, []address.Address{100, 200, 300, 500, 600, 800})
+	a = add(a, 500, 600)
+	wt.AssertEquals(t, a, []address.Address{100, 200, 300, 800})
+	a = subtract(a, 500, 600)
+	wt.AssertEquals(t, a, []address.Address{100, 200, 300, 500, 600, 800})
+	a = subtract(a, 600, 700)
+	wt.AssertEquals(t, a, []address.Address{100, 200, 300, 500, 700, 800})
+	a = subtract(a, 400, 500)
+	wt.AssertEquals(t, a, []address.Address{100, 200, 300, 400, 700, 800})
+	a = subtract(a, 300, 400)
+	wt.AssertEquals(t, a, []address.Address{100, 200, 700, 800})
+	a = subtract(a, 700, 800)
+	wt.AssertEquals(t, a, []address.Address{100, 200})
+	a = subtract(a, 100, 200)
+	wt.AssertEquals(t, a, []address.Address{})
+
+	s := New()
+	wt.AssertEquals(t, s.NumFreeAddresses(), address.Offset(0))
+	ok, got := s.Allocate()
+	wt.AssertFalse(t, ok, "allocate in empty space should fail")
+
+	s.Add(100, 100)
+	wt.AssertEquals(t, s.NumFreeAddresses(), address.Offset(100))
+	ok, got = s.Allocate()
+	wt.AssertTrue(t, ok && got == 100, "allocate")
+	wt.AssertEquals(t, s.NumFreeAddresses(), address.Offset(99))
+	wt.AssertNoErr(t, s.Claim(150))
+	wt.AssertEquals(t, s.NumFreeAddresses(), address.Offset(98))
+	wt.AssertNoErr(t, s.Free(100))
+	wt.AssertEquals(t, s.NumFreeAddresses(), address.Offset(99))
+	wt.AssertErrorInterface(t, s.Free(0), (*error)(nil), "free not allocated")
+	wt.AssertErrorInterface(t, s.Free(100), (*error)(nil), "double free")
+
+	start, size, ok := s.Donate()
+	wt.AssertTrue(t, ok && start == 125 && size == 25, "donate")
+
+	// test Donate when addresses are scarce
+	s = New()
+	start, size, ok = s.Donate()
+	wt.AssertTrue(t, !ok, "donate on empty space should fail")
+	s.Add(0, 3)
+	wt.AssertNoErr(t, s.Claim(0))
+	wt.AssertNoErr(t, s.Claim(2))
+	start, size, ok = s.Donate()
+	wt.AssertTrue(t, ok && start == 1 && size == 1, "donate")
+	start, size, ok = s.Donate()
+	wt.AssertTrue(t, !ok, "donate should fail")
+}
+
+func TestSpaceAllocate(t *testing.T) {
+	const (
+		testAddr1   = "10.0.3.4"
+		testAddrx   = "10.0.3.19"
+		testAddry   = "10.0.9.19"
+		containerID = "deadbeef"
+	)
+
+	space1 := makeSpace(ip(testAddr1), 20)
+	wt.AssertEquals(t, space1.NumFreeAddresses(), address.Offset(20))
+	space1.assertInvariants()
+
+	_, addr1 := space1.Allocate()
+	wt.AssertEqualString(t, addr1.String(), testAddr1, "address")
+	wt.AssertEquals(t, space1.NumFreeAddresses(), address.Offset(19))
+	space1.assertInvariants()
+
+	_, addr2 := space1.Allocate()
+	wt.AssertFalse(t, addr2.String() == testAddr1, "address")
+	wt.AssertEquals(t, space1.NumFreeAddresses(), address.Offset(18))
+	wt.AssertEquals(t, space1.NumFreeAddressesInRange(ip(testAddr1), ip(testAddrx)), address.Offset(13))
+	wt.AssertEquals(t, space1.NumFreeAddressesInRange(ip(testAddr1), ip(testAddry)), address.Offset(18))
+	space1.assertInvariants()
+
+	space1.Free(addr2)
+	space1.assertInvariants()
+
+	wt.AssertErrorInterface(t, space1.Free(addr2), (*error)(nil), "double free")
+	wt.AssertErrorInterface(t, space1.Free(ip(testAddrx)), (*error)(nil), "address not allocated")
+	wt.AssertErrorInterface(t, space1.Free(ip(testAddry)), (*error)(nil), "wrong out of range")
+
+	space1.assertInvariants()
+}
+
+func TestSpaceFree(t *testing.T) {
+	const (
+		testAddr1   = "10.0.3.4"
+		testAddrx   = "10.0.3.19"
+		testAddry   = "10.0.9.19"
+		containerID = "deadbeef"
+	)
+
+	space := makeSpace(ip(testAddr1), 20)
+
+	// Check we are prepared to give up the entire space
+	pos, size := space.biggestFreeRange()
+	wt.AssertTrue(t, space.free[pos] == ip(testAddr1) && size == 20, "Wrong space")
+
+	for i := 0; i < 20; i++ {
+		ok, _ := space.Allocate()
+		wt.AssertTrue(t, ok, "Failed to get address")
+	}
+
+	// Check we are full
+	ok, _ := space.Allocate()
+	wt.AssertTrue(t, !ok, "Should have failed to get address")
+	_, size, ok = space.Donate()
+	wt.AssertTrue(t, size == 0, "Wrong space")
+
+	// Free in the middle
+	wt.AssertSuccess(t, space.Free(ip("10.0.3.13")))
+	pos, size = space.biggestFreeRange()
+	wt.AssertTrue(t, space.free[pos] == ip("10.0.3.13") && size == 1, "Wrong space")
+
+	// Free one at the end
+	wt.AssertSuccess(t, space.Free(ip("10.0.3.23")))
+	pos, size = space.biggestFreeRange()
+	wt.AssertTrue(t, space.free[pos] == ip("10.0.3.23") && size == 1, "Wrong space")
+
+	// Now free a few at the end
+	wt.AssertSuccess(t, space.Free(ip("10.0.3.22")))
+	wt.AssertSuccess(t, space.Free(ip("10.0.3.21")))
+
+	wt.AssertEquals(t, space.NumFreeAddresses(), address.Offset(4))
+
+	// Now get the biggest free space; should be 3.21
+	pos, size = space.biggestFreeRange()
+	wt.AssertTrue(t, space.free[pos] == ip("10.0.3.21") && size == 3, "Wrong space")
+
+	// Now free a few in the middle
+	wt.AssertSuccess(t, space.Free(ip("10.0.3.12")))
+	wt.AssertSuccess(t, space.Free(ip("10.0.3.11")))
+	wt.AssertSuccess(t, space.Free(ip("10.0.3.10")))
+
+	wt.AssertEquals(t, space.NumFreeAddresses(), address.Offset(7))
+
+	// Now get the biggest free space; should be 3.21
+	pos, size = space.biggestFreeRange()
+	wt.AssertTrue(t, space.free[pos] == ip("10.0.3.10") && size == 4, "Wrong space")
+
+	wt.AssertEquals(t, space.OwnedRanges(), []address.Range{address.Range{Start: ip("10.0.3.4"), End: ip("10.0.3.24")}})
+}
+
+func (s1 *Space) Equal(s2 *Space) bool {
+	return reflect.DeepEqual(s1.ours, s2.ours) && reflect.DeepEqual(s1.free, s2.free)
+}
+
+func TestDonateSimple(t *testing.T) {
+	const (
+		testAddr1 = "10.0.1.0"
+		testAddr2 = "10.0.1.32"
+	)
+
+	var (
+		ipAddr1 = ip(testAddr1)
+	)
+
+	ps1 := makeSpace(ipAddr1, 48)
+
+	// Empty space set should split in two and give me the second half
+	start, numGivenUp, ok := ps1.Donate()
+	wt.AssertTrue(t, ok, "Donate result")
+	wt.AssertEqualString(t, start.String(), "10.0.1.24", "Invalid start")
+	wt.AssertEquals(t, numGivenUp, address.Offset(24))
+	wt.AssertEquals(t, ps1.NumFreeAddresses(), address.Offset(24))
+
+	// Now check we can give the rest up.
+	count := 0 // count to avoid infinite loop
+	for ; count < 1000; count++ {
+		_, size, ok := ps1.Donate()
+		if !ok {
+			break
+		}
+		numGivenUp += size
+	}
+	wt.AssertEquals(t, ps1.NumFreeAddresses(), address.Offset(0))
+	wt.AssertEquals(t, numGivenUp, address.Offset(48))
+}
+
+func TestDonateHard(t *testing.T) {
+	//common.InitDefaultLogging(true)
+	var (
+		start                = ip("10.0.1.0")
+		size  address.Offset = 48
+	)
+
+	// Fill a fresh space
+	spaceset := makeSpace(start, size)
+	for i := address.Offset(0); i < size; i++ {
+		ok, _ := spaceset.Allocate()
+		wt.AssertTrue(t, ok, "Failed to get IP!")
+	}
+
+	wt.AssertEquals(t, spaceset.NumFreeAddresses(), address.Offset(0))
+
+	// Now free all but the last address
+	// this will force us to split the free list
+	for i := address.Offset(0); i < size-1; i++ {
+		wt.AssertSuccess(t, spaceset.Free(address.Add(start, i)))
+	}
+
+	// Now split
+	newRange, numGivenUp, ok := spaceset.Donate()
+	wt.AssertTrue(t, ok, "GiveUpSpace result")
+	wt.AssertEquals(t, newRange, ip("10.0.1.23"))
+	wt.AssertEquals(t, numGivenUp, address.Offset(24))
+	wt.AssertEquals(t, spaceset.NumFreeAddresses(), address.Offset(23))
+
+	//Space set should now have 2 spaces
+	expected := New()
+	expected.Add(start, 23)
+	expected.ours = add(nil, ip("10.0.1.47"), ip("10.0.1.48"))
+	wt.AssertEquals(t, spaceset, expected)
+}

ipam/testutils_test.go

@@ -0,0 +1,286 @@
+package ipam
+
+import (
+	"fmt"
+	"math/rand"
+	"testing"
+	"time"
+
+	"github.com/weaveworks/weave/common"
+	"github.com/weaveworks/weave/router"
+	wt "github.com/weaveworks/weave/testing"
+)
+
+type mockMessage struct {
+	dst     router.PeerName
+	msgType byte
+	buf     []byte
+}
+
+func (m *mockMessage) String() string {
+	return fmt.Sprintf("-> %s [%x]", m.dst, m.buf)
+}
+
+func toStringArray(messages []mockMessage) []string {
+	out := make([]string, len(messages))
+	for i := range out {
+		out[i] = messages[i].String()
+	}
+	return out
+}
+
+type mockGossipComms struct {
+	t        *testing.T
+	name     string
+	messages []mockMessage
+}
+
+func (m *mockGossipComms) String() string {
+	return fmt.Sprintf("[mockGossipComms %s]", m.name)
+}
+
+// Note: this style of verification, using equalByteBuffer, requires
+// that the contents of messages are never re-ordered.  Which, for instance,
+// requires they are not based off iterating through a map.
+
+func (m *mockGossipComms) GossipBroadcast(update router.GossipData) error {
+	buf := []byte{}
+	if len(m.messages) == 0 {
+		m.Fatalf("%s: Gossip broadcast message unexpected: \n%x", m.name, buf)
+	} else if msg := m.messages[0]; msg.dst != router.UnknownPeerName {
+		m.Fatalf("%s: Expected Gossip message to %s but got broadcast", m.name, msg.dst)
+	} else if msg.buf != nil && !equalByteBuffer(msg.buf, buf) {
+		m.Fatalf("%s: Gossip message not sent as expected: \nwant: %x\ngot : %x", m.name, msg.buf, buf)
+	} else {
+		// Swallow this message
+		m.messages = m.messages[1:]
+	}
+	return nil
+}
+
+func equalByteBuffer(a, b []byte) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for i := 0; i < len(a); i++ {
+		if a[i] != b[i] {
+			return false
+		}
+	}
+	return true
+}
+
+func (m *mockGossipComms) Fatalf(format string, args ...interface{}) {
+	// this sometimes hangs: wt.Fatalf(m.t, args...)
+	panic(fmt.Sprintf(format, args...))
+}
+
+func (m *mockGossipComms) GossipUnicast(dstPeerName router.PeerName, buf []byte) error {
+	if len(m.messages) == 0 {
+		m.Fatalf("%s: Gossip message to %s unexpected: \n%s", m.name, dstPeerName, buf)
+	} else if msg := m.messages[0]; msg.dst == router.UnknownPeerName {
+		m.Fatalf("%s: Expected Gossip broadcast message but got dest %s", m.name, dstPeerName)
+	} else if msg.dst != dstPeerName {
+		m.Fatalf("%s: Expected Gossip message to %s but got dest %s", m.name, msg.dst, dstPeerName)
+	} else if buf[0] != msg.msgType {
+		m.Fatalf("%s: Expected Gossip message of type %d but got type %d", m.name, msg.msgType, buf[0])
+	} else if msg.buf != nil && !equalByteBuffer(msg.buf, buf[1:]) {
+		m.Fatalf("%s: Gossip message not sent as expected: \nwant: %x\ngot : %x", m.name, msg.buf, buf[1:])
+	} else {
+		// Swallow this message
+		m.messages = m.messages[1:]
+	}
+	return nil
+}
+
+func ExpectMessage(alloc *Allocator, dst string, msgType byte, buf []byte) {
+	m := alloc.gossip.(*mockGossipComms)
+	dstPeerName, _ := router.PeerNameFromString(dst)
+	m.messages = append(m.messages, mockMessage{dstPeerName, msgType, buf})
+}
+
+func ExpectBroadcastMessage(alloc *Allocator, buf []byte) {
+	m := alloc.gossip.(*mockGossipComms)
+	m.messages = append(m.messages, mockMessage{router.UnknownPeerName, 0, buf})
+}
+
+func CheckAllExpectedMessagesSent(allocs ...*Allocator) {
+	for _, alloc := range allocs {
+		m := alloc.gossip.(*mockGossipComms)
+		if len(m.messages) > 0 {
+			wt.Fatalf(m.t, "%s: Gossip message(s) not sent as expected: \n%x", m.name, m.messages)
+		}
+	}
+}
+
+func makeAllocator(name string, cidr string, quorum uint) *Allocator {
+	peername, err := router.PeerNameFromString(name)
+	if err != nil {
+		panic(err)
+	}
+
+	alloc, err := NewAllocator(peername, router.PeerUID(rand.Int63()),
+		"nick-"+name, cidr, quorum)
+	if err != nil {
+		panic(err)
+	}
+
+	return alloc
+}
+
+func makeAllocatorWithMockGossip(t *testing.T, name string, universeCIDR string, quorum uint) *Allocator {
+	alloc := makeAllocator(name, universeCIDR, quorum)
+	gossip := &mockGossipComms{t: t, name: name}
+	alloc.SetInterfaces(gossip)
+	alloc.Start()
+	return alloc
+}
+
+func (alloc *Allocator) claimRingForTesting(allocs ...*Allocator) {
+	peers := []router.PeerName{alloc.ourName}
+	for _, alloc2 := range allocs {
+		peers = append(peers, alloc2.ourName)
+	}
+	alloc.ring.ClaimForPeers(normalizeConsensus(peers))
+	alloc.space.AddRanges(alloc.ring.OwnedRanges())
+}
+
+// Check whether or not something was sent on a channel
+func AssertSent(t *testing.T, ch <-chan bool) {
+	timeout := time.After(10 * time.Second)
+	select {
+	case <-ch:
+		// This case is ok
+	case <-timeout:
+		wt.Fatalf(t, "Nothing sent on channel")
+	}
+}
+
+func AssertNothingSent(t *testing.T, ch <-chan bool) {
+	select {
+	case val := <-ch:
+		wt.Fatalf(t, "Unexpected value on channel: %t", val)
+	default:
+		// no message received
+	}
+}
+
+func AssertNothingSentErr(t *testing.T, ch <-chan error) {
+	select {
+	case val := <-ch:
+		wt.Fatalf(t, "Unexpected value on channel: %t", val)
+	default:
+		// no message received
+	}
+}
+
+// Router to convey gossip from one gossiper to another, for testing
+type gossipMessage struct {
+	isUnicast bool
+	sender    *router.PeerName
+	buf       []byte
+	exitChan  chan bool
+}
+
+type TestGossipRouter struct {
+	gossipChans map[router.PeerName]chan gossipMessage
+	loss        float32 // 0.0 means no loss
+}
+
+func (grouter *TestGossipRouter) GossipBroadcast(update router.GossipData) error {
+	for _, gossipChan := range grouter.gossipChans {
+		select {
+		case gossipChan <- gossipMessage{buf: update.(*ipamGossipData).alloc.encode()}:
+		default: // drop the message if we cannot send it
+		}
+	}
+	return nil
+}
+
+func (grouter *TestGossipRouter) removePeer(peer router.PeerName) {
+	gossipChan := grouter.gossipChans[peer]
+	resultChan := make(chan bool)
+	gossipChan <- gossipMessage{exitChan: resultChan}
+	<-resultChan
+	delete(grouter.gossipChans, peer)
+}
+
+type TestGossipRouterClient struct {
+	router *TestGossipRouter
+	sender router.PeerName
+}
+
+func (grouter *TestGossipRouter) connect(sender router.PeerName, gossiper router.Gossiper) router.Gossip {
+	gossipChan := make(chan gossipMessage, 100)
+
+	go func() {
+		gossipTimer := time.Tick(10 * time.Second)
+		for {
+			select {
+			case message := <-gossipChan:
+				if message.exitChan != nil {
+					message.exitChan <- true
+					return
+				}
+
+				if rand.Float32() > (1.0 - grouter.loss) {
+					continue
+				}
+
+				if message.isUnicast {
+					err := gossiper.OnGossipUnicast(*message.sender, message.buf)
+					if err != nil {
+						panic(fmt.Sprintf("Error doing gossip unicast to %s: %s", sender, err))
+					}
+				} else {
+					_, err := gossiper.OnGossipBroadcast(message.buf)
+					if err != nil {
+						panic(fmt.Sprintf("Error doing gossip broadcast to %s: %s", sender, err))
+					}
+				}
+			case <-gossipTimer:
+				grouter.GossipBroadcast(gossiper.Gossip())
+			}
+		}
+	}()
+
+	grouter.gossipChans[sender] = gossipChan
+	return TestGossipRouterClient{grouter, sender}
+}
+
+func (client TestGossipRouterClient) GossipUnicast(dstPeerName router.PeerName, buf []byte) error {
+	select {
+	case client.router.gossipChans[dstPeerName] <- gossipMessage{isUnicast: true, sender: &client.sender, buf: buf}:
+	default: // drop the message if we cannot send it
+		common.Error.Printf("Dropping message")
+	}
+	return nil
+}
+
+func (client TestGossipRouterClient) GossipBroadcast(update router.GossipData) error {
+	return client.router.GossipBroadcast(update)
+}
+
+func makeNetworkOfAllocators(size int, cidr string) ([]*Allocator, TestGossipRouter) {
+
+	gossipRouter := TestGossipRouter{make(map[router.PeerName]chan gossipMessage), 0.0}
+	allocs := make([]*Allocator, size)
+
+	for i := 0; i < size; i++ {
+		alloc := makeAllocator(fmt.Sprintf("%02d:00:00:02:00:00", i),
+			cidr, uint(size/2+1))
+		alloc.SetInterfaces(gossipRouter.connect(alloc.ourName, alloc))
+		alloc.Start()
+		allocs[i] = alloc
+	}
+
+	gossipRouter.GossipBroadcast(allocs[size-1].Gossip())
+	time.Sleep(1000 * time.Millisecond)
+	return allocs, gossipRouter
+}
+
+func stopNetworkOfAllocators(allocs []*Allocator) {
+	for _, alloc := range allocs {
+		alloc.Stop()
+	}
+}

nameserver/addrs.go

@@ -0,0 +1,125 @@
+package nameserver
+
+import (
+	"bytes"
+	"fmt"
+	"net"
+)
+
+// A zone record
+// Note: we will try to keep all the zone records information is concentrated here. Maybe not all queries will
+//       use things like priorities or weights, but we do not want to create a hierarchy for dealing with all
+//       possible queries...
+type ZoneRecord interface {
+	Name() string  // The name for this IP
+	IP() net.IP    // The IP (v4) address
+	Priority() int // The priority
+	Weight() int   // The weight
+	TTL() int      // TTL
+}
+
+type ZoneLookup interface {
+	// Lookup for a name
+	LookupName(name string) ([]ZoneRecord, error)
+	// Lookup for an address
+	LookupInaddr(inaddr string) ([]ZoneRecord, error)
+}
+
+type ZoneObserver interface {
+	// Observe anything that affects a particular name in the zone
+	ObserveName(name string, observer ZoneRecordObserver) error
+	// Observe anything that affects a particular IP in the zone
+	ObserveInaddr(inaddr string, observer ZoneRecordObserver) error
+}
+
+type ZoneRecordObserver func()
+
+/////////////////////////////////////////////////////////////
+
+// A basic record struct (satisfying the ZoneRecord interface)
+type Record struct {
+	name     string
+	ip       net.IP
+	priority int
+	weight   int
+	ttl      int
+}
+
+func (r Record) Name() string  { return r.name }
+func (r Record) IP() net.IP    { return r.ip }
+func (r Record) Priority() int { return r.priority }
+func (r Record) Weight() int   { return r.weight }
+func (r Record) TTL() int      { return r.ttl }
+
+func (r Record) String() string {
+	var buf bytes.Buffer
+	if len(r.Name()) > 0 {
+		fmt.Fprintf(&buf, "%s", r.Name())
+	}
+	if !r.IP().IsUnspecified() {
+		fmt.Fprintf(&buf, "[%s]", r.IP())
+	}
+	if r.Priority() > 0 {
+		fmt.Fprintf(&buf, "/P:%d", r.Priority())
+	}
+	if r.Weight() > 0 {
+		fmt.Fprintf(&buf, "/W:%d", r.Weight())
+	}
+	if r.TTL() > 0 {
+		fmt.Fprintf(&buf, "/TTL:%d", r.TTL())
+	}
+
+	return buf.String()
+}
+
+/////////////////////////////////////////////////////////////
+
+func newParseError(reason string, addr string) *net.ParseError {
+	return &net.ParseError{Type: reason, Text: addr}
+}
+
+// simple IPv4 type that can be used as a key in a map (in contrast with net.IP), used for sets of IPs, etc...
+type IPv4 [4]byte
+
+func (ip IPv4) toNetIP() net.IP { return net.IPv4(ip[0], ip[1], ip[2], ip[3]) }
+func (ip IPv4) String() string  { return ip.toNetIP().String() }
+func ipToIPv4(nip net.IP) IPv4  { ip := nip.To4(); return IPv4([4]byte{ip[0], ip[1], ip[2], ip[3]}) }
+
+// Parse an address (eg, "10.13.12.11") and return the corresponding IPv4 (eg, "10.13.12.11")
+func addrToIPv4(addr string) (IPv4, error) {
+	ip := net.ParseIP(addr)
+	if ip == nil {
+		return IPv4{}, newParseError("IP address", addr)
+	}
+	return ipToIPv4(ip), nil
+}
+
+// Parse a reverse address (eg, "11.12.13.10.in-addr.arpa.") and return the corresponding IPv4 (eg, "10.13.12.11")
+func raddrToIPv4(addr string) (IPv4, error) {
+	l := len(addr)
+	if l < rdnsDomainLen+1 {
+		return IPv4{}, newParseError("too short reverse IP address", addr)
+	}
+
+	suffixLen := l - rdnsDomainLen
+	suffix := addr[suffixLen+1:]
+	if suffix != RDNSDomain {
+		return IPv4{}, newParseError("suffix of reverse IP address", addr)
+	}
+	ipStr := addr[:suffixLen]
+	revIP := net.ParseIP(ipStr)
+	if revIP == nil {
+		return IPv4{}, newParseError("reverse IP address", addr)
+	}
+
+	revIP4 := revIP.To4()
+	return IPv4([4]byte{revIP4[3], revIP4[2], revIP4[1], revIP4[0]}), nil
+}
+
+func raddrToIP(addr string) (net.IP, error) {
+	r, err := raddrToIPv4(addr)
+	if err != nil {
+		return net.IP{}, err
+	}
+	return r.toNetIP(), nil
+}

nameserver/addrs_test.go

@@ -0,0 +1,31 @@
+package nameserver
+
+import (
+	. "github.com/weaveworks/weave/common"
+	wt "github.com/weaveworks/weave/testing"
+	"net"
+	"testing"
+)
+
+func TestAddrs(t *testing.T) {
+	InitDefaultLogging(testing.Verbose())
+	Info.Println("TestAddrs starting")
+
+	ip, err := addrToIPv4("10.13.12.11")
+	wt.AssertNoErr(t, err)
+	wt.AssertTrue(t, net.ParseIP("10.13.12.11").Equal(ip.toNetIP()), "IP")
+
+	ip, err = raddrToIPv4("11.12.13.10.in-addr.arpa.")
+	wt.AssertNoErr(t, err)
+	wt.AssertTrue(t, net.ParseIP("10.13.12.11").Equal(ip.toNetIP()), "IP")
+
+	// some malformed addresses
+	ip, err = addrToIPv4("10.13.12")
+	wt.AssertTrue(t, err != nil, "when parsing malformed address")
+	ip, err = addrToIPv4("10.13.AA.12")
+	wt.AssertTrue(t, err != nil, "when parsing malformed address")
+	ip, err = raddrToIPv4("11.12.13.10.in-axxx.arpa.")
+	wt.AssertTrue(t, err != nil, "when parsing malformed address")
+	ip, err = raddrToIPv4("11.12.13.10.in-addr")
+	wt.AssertTrue(t, err != nil, "when parsing malformed address")
+}

nameserver/cache.go

@@ -1,14 +1,18 @@
 package nameserver
 
 import (
+	"bytes"
 	"container/heap"
 	"errors"
-	"github.com/miekg/dns"
-	. "github.com/weaveworks/weave/common"
+	"fmt"
 	"math"
 	"math/rand"
 	"sync"
 	"time"
+
+	"github.com/benbjohnson/clock"
+	"github.com/miekg/dns"
+	. "github.com/weaveworks/weave/common"
 )
 
 var (
@@ -31,6 +35,11 @@ const (
 	stResolved entryStatus = iota // resolved
 )
 
+var statusToString = map[entryStatus]string{
+	stPending:  "pending",
+	stResolved: "resolved",
+}
+
 const (
 	CacheNoLocalReplies uint8 = 1 << iota // not found in local network (stored in the cache so we skip another local lookup or some time)
 )
@@ -137,9 +146,12 @@ func (e *cacheEntry) setReply(reply *dns.Msg, ttl int, flags uint8, now time.Tim
 		prevValidUntil = e.validUntil
 	}
 
-	e.Status = stResolved
-	e.Flags = flags
-	e.putTime = now
+	// make sure we do not overwrite noLocalReplies entries
+	if flags&CacheNoLocalReplies != 0 {
+		if e.Flags&CacheNoLocalReplies != 0 {
+			return false
+		}
+	}
 
 	if ttl != nullTTL {
 		e.validUntil = now.Add(time.Second * time.Duration(ttl))
@@ -153,8 +165,14 @@ func (e *cacheEntry) setReply(reply *dns.Msg, ttl int, flags uint8, now time.Tim
 			}
 		}
 		e.validUntil = now.Add(time.Second * time.Duration(minTTL))
+	} else {
+		Warning.Printf("[cache] no valid TTL could be calculated")
 	}
 
+	e.Status = stResolved
+	e.Flags = flags
+	e.putTime = now
+
 	if reply != nil {
 		e.reply = *reply
 		e.ReplyLen = reply.Len()
@@ -163,6 +181,19 @@ func (e *cacheEntry) setReply(reply *dns.Msg, ttl int, flags uint8, now time.Tim
 	return (prevValidUntil != e.validUntil)
 }
 
+func (e *cacheEntry) String() string {
+	var buf bytes.Buffer
+	q := e.question
+	fmt.Fprintf(&buf, "'%s'[%s]: ", q.Name, dns.TypeToString[q.Qtype])
+	if e.Flags&CacheNoLocalReplies != 0 {
+		fmt.Fprintf(&buf, "neg-local")
+	} else {
+		fmt.Fprintf(&buf, "%s", statusToString[e.Status])
+	}
+	fmt.Fprintf(&buf, "(%d bytes)", e.ReplyLen)
+	return buf.String()
+}
+
 //////////////////////////////////////////////////////////////////////////////////////
 
 // An entriesPtrHeap is a min-heap of cache entries.
@@ -196,26 +227,49 @@ func (h *entriesPtrsHeap) Pop() interface{} {
 
 //////////////////////////////////////////////////////////////////////////////////////
 
+// The cache interface
+type ZoneCache interface {
+	// Get from the cache
+	Get(request *dns.Msg, maxLen int) (reply *dns.Msg, err error)
+	// Put in the cache
+	Put(request *dns.Msg, reply *dns.Msg, ttl int, flags uint8) int
+	// Remove from the cache
+	Remove(question *dns.Question)
+	// Purge all expired entries from the cache
+	Purge()
+	// Remove all entries
+	Clear()
+	// Return the cache length
+	Len() int
+	// Return the max capacity
+	Capacity() int
+}
+
 type cacheKey dns.Question
 type entries map[cacheKey]*cacheEntry
 
 // Cache is a thread-safe fixed capacity LRU cache.
 type Cache struct {
-	Capacity int
-
+	capacity int
 	entries  entries
 	entriesH entriesPtrsHeap // len(entriesH) <= len(entries), as pending entries can be in entries but not in entriesH
+	clock    clock.Clock
 	lock     sync.RWMutex
 }
 
 // NewCache creates a cache of the given capacity
-func NewCache(capacity int) (*Cache, error) {
+func NewCache(capacity int, clk clock.Clock) (*Cache, error) {
 	if capacity <= 0 {
 		return nil, errInvalidCapacity
 	}
 	c := &Cache{
-		Capacity: capacity,
+		capacity: capacity,
 		entries:  make(entries, capacity),
+		clock:    clk,
+	}
+
+	if c.clock == nil {
+		c.clock = clock.New()
 	}
 
 	heap.Init(&c.entriesH)
@@ -227,15 +281,16 @@ func (c *Cache) Clear() {
 	c.lock.Lock()
 	defer c.lock.Unlock()
 
-	c.entries = make(entries, c.Capacity)
+	c.entries = make(entries, c.capacity)
 	heap.Init(&c.entriesH)
 }
 
 // Purge removes the old elements in the cache
-func (c *Cache) Purge(now time.Time) {
+func (c *Cache) Purge() {
 	c.lock.Lock()
 	defer c.lock.Unlock()
 
+	now := c.clock.Now()
 	for i, entry := range c.entriesH {
 		if entry.hasExpired(now) {
 			heap.Remove(&c.entriesH, i)
@@ -247,10 +302,12 @@ func (c *Cache) Purge(now time.Time) {
 }
 
 // Add adds a reply to the cache.
-func (c *Cache) Put(request *dns.Msg, reply *dns.Msg, ttl int, flags uint8, now time.Time) int {
+// When `ttl` is equal to `nullTTL`, the cache entry will be valid until the closest TTL in the `reply`
+func (c *Cache) Put(request *dns.Msg, reply *dns.Msg, ttl int, flags uint8) int {
 	c.lock.Lock()
 	defer c.lock.Unlock()
 
+	now := c.clock.Now()
 	question := request.Question[0]
 	key := cacheKey(question)
 	ent, found := c.entries[key]
@@ -261,7 +318,7 @@ func (c *Cache) Put(request *dns.Msg, reply *dns.Msg, ttl int, flags uint8, now
 		}
 	} else {
 		// If we will add a new item and the capacity has been exceeded, make some room...
-		if len(c.entriesH) >= c.Capacity {
+		if len(c.entriesH) >= c.capacity {
 			lowestEntry := heap.Pop(&c.entriesH).(*cacheEntry)
 			delete(c.entries, cacheKey(lowestEntry.question))
 		}
@@ -274,12 +331,12 @@ func (c *Cache) Put(request *dns.Msg, reply *dns.Msg, ttl int, flags uint8, now
 }
 
 // Look up for a question's reply from the cache.
-// If no reply is stored in the cache, it returns a `nil` reply and no error. The caller can then `Wait()`
-// for another goroutine `Put`ing a reply in the cache.
-func (c *Cache) Get(request *dns.Msg, maxLen int, now time.Time) (reply *dns.Msg, err error) {
+// If no reply is stored in the cache, it returns a `nil` reply and no error.
+func (c *Cache) Get(request *dns.Msg, maxLen int) (reply *dns.Msg, err error) {
 	c.lock.Lock()
 	defer c.lock.Unlock()
 
+	now := c.clock.Now()
 	question := request.Question[0]
 	key := cacheKey(question)
 	if ent, found := c.entries[key]; found {
@@ -307,6 +364,7 @@ func (c *Cache) Remove(question *dns.Question) {
 
 	key := cacheKey(*question)
 	if entry, found := c.entries[key]; found {
+		Debug.Printf("[cache] removing %s-response for '%s'", dns.TypeToString[question.Qtype], question.Name)
 		if entry.index > 0 {
 			heap.Remove(&c.entriesH, entry.index)
 		}
@@ -321,3 +379,23 @@ func (c *Cache) Len() int {
 
 	return len(c.entries)
 }
+
+// Return the max capacity
+func (c *Cache) Capacity() int {
+	c.lock.RLock()
+	defer c.lock.RUnlock()
+
+	return c.capacity
+}
+
+// Return the max capacity
+func (c *Cache) String() string {
+	c.lock.RLock()
+	defer c.lock.RUnlock()
+
+	var buf bytes.Buffer
+	for _, entry := range c.entries {
+		fmt.Fprintf(&buf, "%s\n", entry)
+	}
+	return buf.String()
+}

nameserver/cache_test.go

@@ -2,39 +2,44 @@ package nameserver
 
 import (
 	"fmt"
-	"github.com/miekg/dns"
-	. "github.com/weaveworks/weave/common"
-	wt "github.com/weaveworks/weave/testing"
 	"net"
 	"testing"
 	"time"
+
+	"github.com/benbjohnson/clock"
+	"github.com/miekg/dns"
+	. "github.com/weaveworks/weave/common"
+	wt "github.com/weaveworks/weave/testing"
 )
 
 // Check that the cache keeps its intended capacity constant
 func TestCacheLength(t *testing.T) {
-	InitDefaultLogging(true)
+	InitDefaultLogging(testing.Verbose())
+	Info.Println("TestCacheLength starting")
 
 	const cacheLen = 128
 
-	l, err := NewCache(cacheLen)
+	l, err := NewCache(cacheLen, nil)
 	wt.AssertNoErr(t, err)
 
 	insTime := time.Now()
 
 	t.Logf("Inserting 256 questions in the cache at '%s', with TTL from 0 to 255", insTime)
 	for i := 0; i < cacheLen*2; i++ {
+		name := fmt.Sprintf("name%d", i)
 		questionMsg := new(dns.Msg)
-		questionMsg.SetQuestion(fmt.Sprintf("name%d", i), dns.TypeA)
+		questionMsg.SetQuestion(name, dns.TypeA)
 		questionMsg.RecursionDesired = true
 
 		question := &questionMsg.Question[0]
 
 		ip := net.ParseIP(fmt.Sprintf("10.0.1.%d", i))
-		ips := []net.IP{ip}
-		reply := makeAddressReply(questionMsg, question, ips)
+		records := []ZoneRecord{Record{name, ip, 0, 0, 0}}
+
+		reply := makeAddressReply(questionMsg, question, records)
 		reply.Answer[0].Header().Ttl = uint32(i)
 
-		l.Put(questionMsg, reply, 0, 0, insTime)
+		l.Put(questionMsg, reply, 0, 0)
 	}
 
 	wt.AssertEqualInt(t, l.Len(), cacheLen, "cache length")
@@ -50,13 +55,15 @@ func TestCacheLength(t *testing.T) {
 
 // Check that the cache entries are ok
 func TestCacheEntries(t *testing.T) {
-	InitDefaultLogging(true)
+	InitDefaultLogging(testing.Verbose())
+	Info.Println("TestCacheEntries starting")
 
 	Info.Println("Checking cache consistency")
 
 	const cacheLen = 128
+	clk := clock.NewMock()
 
-	l, err := NewCache(cacheLen)
+	l, err := NewCache(cacheLen, clk)
 	wt.AssertNoErr(t, err)
 
 	questionMsg := new(dns.Msg)
@@ -66,14 +73,14 @@ func TestCacheEntries(t *testing.T) {
 	question := &questionMsg.Question[0]
 
 	t.Logf("Trying to get a name")
-	resp, err := l.Get(questionMsg, minUDPSize, time.Now())
+	resp, err := l.Get(questionMsg, minUDPSize)
 	wt.AssertNoErr(t, err)
 	if resp != nil {
 		t.Logf("Got\n%s", resp)
 		t.Fatalf("ERROR: Did not expect a reponse from Get() yet")
 	}
 	t.Logf("Trying to get it again")
-	resp, err = l.Get(questionMsg, minUDPSize, time.Now())
+	resp, err = l.Get(questionMsg, minUDPSize)
 	wt.AssertNoErr(t, err)
 	if resp != nil {
 		t.Logf("Got\n%s", resp)
@@ -81,21 +88,21 @@ func TestCacheEntries(t *testing.T) {
 	}
 
 	t.Logf("Inserting the reply")
-	reply1 := makeAddressReply(questionMsg, question, []net.IP{net.ParseIP("10.0.1.1")})
-	l.Put(questionMsg, reply1, nullTTL, 0, time.Now())
+	records := []ZoneRecord{Record{"some.name", net.ParseIP("10.0.1.1"), 0, 0, 0}}
+	reply1 := makeAddressReply(questionMsg, question, records)
+	l.Put(questionMsg, reply1, nullTTL, 0)
 
-	timeGet1 := time.Now()
 	t.Logf("Checking we can Get() the reply now")
-	resp, err = l.Get(questionMsg, minUDPSize, timeGet1)
+	resp, err = l.Get(questionMsg, minUDPSize)
 	wt.AssertNoErr(t, err)
 	wt.AssertTrue(t, resp != nil, "reponse from Get()")
 	t.Logf("Received '%s'", resp.Answer[0])
 	wt.AssertType(t, resp.Answer[0], (*dns.A)(nil), "DNS record")
 	ttlGet1 := resp.Answer[0].Header().Ttl
 
-	timeGet2 := timeGet1.Add(time.Duration(1) * time.Second)
+	clk.Add(time.Duration(1) * time.Second)
 	t.Logf("Checking that a second Get(), after 1 second, gets the same result, but with reduced TTL")
-	resp, err = l.Get(questionMsg, minUDPSize, timeGet2)
+	resp, err = l.Get(questionMsg, minUDPSize)
 	wt.AssertNoErr(t, err)
 	wt.AssertTrue(t, resp != nil, "reponse from a second Get()")
 	t.Logf("Received '%s'", resp.Answer[0])
@@ -103,9 +110,9 @@ func TestCacheEntries(t *testing.T) {
 	ttlGet2 := resp.Answer[0].Header().Ttl
 	wt.AssertEqualInt(t, int(ttlGet1-ttlGet2), 1, "TTL difference")
 
-	timeGet3 := timeGet1.Add(time.Duration(localTTL) * time.Second)
+	clk.Add(time.Duration(localTTL) * time.Second)
 	t.Logf("Checking that a third Get(), after %d second, gets no result", localTTL)
-	resp, err = l.Get(questionMsg, minUDPSize, timeGet3)
+	resp, err = l.Get(questionMsg, minUDPSize)
 	wt.AssertNoErr(t, err)
 	if resp != nil {
 		t.Logf("Got\n%s", resp)
@@ -113,28 +120,30 @@ func TestCacheEntries(t *testing.T) {
 	}
 
 	t.Logf("Checking that an Remove() results in Get() returning nothing")
-	replyTemp := makeAddressReply(questionMsg, question, []net.IP{net.ParseIP("10.0.9.9")})
-	l.Put(questionMsg, replyTemp, nullTTL, 0, time.Now())
+	records = []ZoneRecord{Record{"some.name", net.ParseIP("10.0.9.9"), 0, 0, 0}}
+	replyTemp := makeAddressReply(questionMsg, question, records)
+	l.Put(questionMsg, replyTemp, nullTTL, 0)
 	lenBefore := l.Len()
 	l.Remove(question)
 	wt.AssertEqualInt(t, l.Len(), lenBefore-1, "cache length")
 	l.Remove(question) // do it again: should have no effect...
 	wt.AssertEqualInt(t, l.Len(), lenBefore-1, "cache length")
 
-	resp, err = l.Get(questionMsg, minUDPSize, timeGet1)
+	resp, err = l.Get(questionMsg, minUDPSize)
 	wt.AssertNoErr(t, err)
 	wt.AssertTrue(t, resp == nil, "reponse from the Get() after a Remove()")
 
 	t.Logf("Inserting a two replies for the same query")
-	timePut2 := time.Now()
-	reply2 := makeAddressReply(questionMsg, question, []net.IP{net.ParseIP("10.0.1.2")})
-	l.Put(questionMsg, reply2, nullTTL, 0, timePut2)
-	timePut3 := timePut2.Add(time.Duration(1) * time.Second)
-	reply3 := makeAddressReply(questionMsg, question, []net.IP{net.ParseIP("10.0.1.3")})
-	l.Put(questionMsg, reply3, nullTTL, 0, timePut3)
+	records = []ZoneRecord{Record{"some.name", net.ParseIP("10.0.1.2"), 0, 0, 0}}
+	reply2 := makeAddressReply(questionMsg, question, records)
+	l.Put(questionMsg, reply2, nullTTL, 0)
+	clk.Add(time.Duration(1) * time.Second)
+	records = []ZoneRecord{Record{"some.name", net.ParseIP("10.0.1.3"), 0, 0, 0}}
+	reply3 := makeAddressReply(questionMsg, question, records)
+	l.Put(questionMsg, reply3, nullTTL, 0)
 
 	t.Logf("Checking we get the last one...")
-	resp, err = l.Get(questionMsg, minUDPSize, timePut3)
+	resp, err = l.Get(questionMsg, minUDPSize)
 	wt.AssertNoErr(t, err)
 	wt.AssertTrue(t, resp != nil, "reponse from the Get()")
 	t.Logf("Received '%s'", resp.Answer[0])
@@ -142,7 +151,8 @@ func TestCacheEntries(t *testing.T) {
 	wt.AssertEqualString(t, resp.Answer[0].(*dns.A).A.String(), "10.0.1.3", "IP address")
 	wt.AssertEqualInt(t, int(resp.Answer[0].Header().Ttl), int(localTTL), "TTL")
 
-	resp, err = l.Get(questionMsg, minUDPSize, timePut3.Add(time.Duration(localTTL-1)*time.Second))
+	clk.Add(time.Duration(localTTL-1) * time.Second)
+	resp, err = l.Get(questionMsg, minUDPSize)
 	wt.AssertNoErr(t, err)
 	wt.AssertTrue(t, resp != nil, "reponse from the Get()")
 	t.Logf("Received '%s'", resp.Answer[0])
@@ -152,7 +162,8 @@ func TestCacheEntries(t *testing.T) {
 
 	t.Logf("Checking we get empty replies when they are expired...")
 	lenBefore = l.Len()
-	resp, err = l.Get(questionMsg, minUDPSize, timePut3.Add(time.Duration(localTTL)*time.Second))
+	clk.Add(time.Duration(localTTL) * time.Second)
+	resp, err = l.Get(questionMsg, minUDPSize)
 	wt.AssertNoErr(t, err)
 	if resp != nil {
 		t.Logf("Got\n%s", resp.Answer[0])
@@ -166,15 +177,16 @@ func TestCacheEntries(t *testing.T) {
 	question2 := &questionMsg2.Question[0]
 
 	t.Logf("Trying to Get() a name")
-	resp, err = l.Get(questionMsg2, minUDPSize, time.Now())
+	resp, err = l.Get(questionMsg2, minUDPSize)
 	wt.AssertNoErr(t, err)
 	wt.AssertNil(t, resp, "reponse from Get() yet")
 
 	t.Logf("Checking that an Remove() between Get() and Put() does not break things")
-	replyTemp2 := makeAddressReply(questionMsg2, question2, []net.IP{net.ParseIP("10.0.9.9")})
+	records = []ZoneRecord{Record{"some.name", net.ParseIP("10.0.9.9"), 0, 0, 0}}
+	replyTemp2 := makeAddressReply(questionMsg2, question2, records)
 	l.Remove(question2)
-	l.Put(questionMsg2, replyTemp2, nullTTL, 0, time.Now())
-	resp, err = l.Get(questionMsg2, minUDPSize, time.Now())
+	l.Put(questionMsg2, replyTemp2, nullTTL, 0)
+	resp, err = l.Get(questionMsg2, minUDPSize)
 	wt.AssertNoErr(t, err)
 	wt.AssertNotNil(t, resp, "reponse from Get()")
 
@@ -184,23 +196,33 @@ func TestCacheEntries(t *testing.T) {
 	question3 := &questionMsg3.Question[0]
 
 	t.Logf("Checking that a entry with CacheNoLocalReplies return an error")
-	timePut3 = time.Now()
-	l.Put(questionMsg3, nil, nullTTL, CacheNoLocalReplies, timePut3)
-	resp, err = l.Get(questionMsg3, minUDPSize, timePut3)
+	l.Put(questionMsg3, nil, negLocalTTL, CacheNoLocalReplies)
+	resp, err = l.Get(questionMsg3, minUDPSize)
 	wt.AssertNil(t, resp, "Get() response with CacheNoLocalReplies")
-	wt.AssertNotNil(t, err, "Get() error with CacheNoLocalReplies")
+	wt.AssertTrue(t, err == errNoLocalReplies, "Get() error with CacheNoLocalReplies")
 
-	timeExpiredGet3 := timePut3.Add(time.Second * time.Duration(negLocalTTL+1))
-	t.Logf("Checking that we get an expired response after %f seconds", timeExpiredGet3.Sub(timePut3).Seconds())
-	resp, err = l.Get(questionMsg3, minUDPSize, timeExpiredGet3)
+	t.Logf("Checking that more Put's do not make the negative TTL longer")
+	clk.Add(time.Duration(negLocalTTL/2) * time.Second)
+	l.Put(questionMsg3, nil, negLocalTTL, CacheNoLocalReplies)
+	clk.Add(time.Duration(1) * time.Second)
+	l.Put(questionMsg3, nil, negLocalTTL, CacheNoLocalReplies)
+	clk.Add(time.Duration(1) * time.Second)
+	l.Put(questionMsg3, nil, negLocalTTL, CacheNoLocalReplies)
+	resp, err = l.Get(questionMsg3, minUDPSize)
+	wt.AssertNil(t, resp, "Get() response with CacheNoLocalReplies")
+	wt.AssertTrue(t, err == errNoLocalReplies, "Get() error with CacheNoLocalReplies")
+
+	clk.Add(time.Duration(negLocalTTL/2) * time.Second)
+	t.Logf("Checking that we get an expired response after %f seconds", negLocalTTL)
+	resp, err = l.Get(questionMsg3, minUDPSize)
 	wt.AssertNil(t, resp, "expired Get() response with CacheNoLocalReplies")
 	wt.AssertNil(t, err, "expired Get() error with CacheNoLocalReplies")
 
 	l.Remove(question3)
 	t.Logf("Checking that Put&Get with CacheNoLocalReplies with a Remove in the middle returns nothing")
-	l.Put(questionMsg3, nil, nullTTL, CacheNoLocalReplies, time.Now())
+	l.Put(questionMsg3, nil, nullTTL, CacheNoLocalReplies)
 	l.Remove(question3)
-	resp, err = l.Get(questionMsg3, minUDPSize, time.Now())
+	resp, err = l.Get(questionMsg3, minUDPSize)
 	wt.AssertNil(t, resp, "Get() reponse with CacheNoLocalReplies")
 	wt.AssertNil(t, err, "Get() error with CacheNoLocalReplies")
 }

nameserver/dns.go

@@ -2,7 +2,6 @@ package nameserver
 
 import (
 	"github.com/miekg/dns"
-	"net"
 )
 
 const (
@@ -34,20 +33,23 @@ func makeTruncatedReply(r *dns.Msg) *dns.Msg {
 	return reply
 }
 
-func makeAddressReply(r *dns.Msg, q *dns.Question, addrs []net.IP) *dns.Msg {
+func makeAddressReply(r *dns.Msg, q *dns.Question, addrs []ZoneRecord) *dns.Msg {
 	answers := make([]dns.RR, len(addrs))
 	header := makeHeader(r, q)
 	count := 0
 	for _, addr := range addrs {
+		ip := addr.IP()
+		ip4 := ip.To4()
+
 		switch q.Qtype {
 		case dns.TypeA:
-			if ip4 := addr.To4(); ip4 != nil {
-				answers[count] = &dns.A{Hdr: *header, A: addr}
+			if ip4 != nil {
+				answers[count] = &dns.A{Hdr: *header, A: ip}
 				count++
 			}
 		case dns.TypeAAAA:
-			if ip4 := addr.To4(); ip4 == nil {
-				answers[count] = &dns.AAAA{Hdr: *header, AAAA: addr}
+			if ip4 == nil {
+				answers[count] = &dns.AAAA{Hdr: *header, AAAA: ip}
 				count++
 			}
 		}
@@ -55,11 +57,11 @@ func makeAddressReply(r *dns.Msg, q *dns.Question, addrs []net.IP) *dns.Msg {
 	return makeReply(r, answers[:count])
 }
 
-func makePTRReply(r *dns.Msg, q *dns.Question, names []string) *dns.Msg {
+func makePTRReply(r *dns.Msg, q *dns.Question, names []ZoneRecord) *dns.Msg {
 	answers := make([]dns.RR, len(names))
 	header := makeHeader(r, q)
 	for i, name := range names {
-		answers[i] = &dns.PTR{Hdr: *header, Ptr: name}
+		answers[i] = &dns.PTR{Hdr: *header, Ptr: name.Name()}
 	}
 	return makeReply(r, answers)
 }
@@ -72,14 +74,6 @@ func makeDNSFailResponse(r *dns.Msg) *dns.Msg {
 	return m
 }
 
-func makeDNSNotImplResponse(r *dns.Msg) *dns.Msg {
-	m := new(dns.Msg)
-	m.SetReply(r)
-	m.RecursionAvailable = true
-	m.Rcode = dns.RcodeNotImplemented
-	return m
-}
-
 // get the maximum UDP-reply length
 func getMaxReplyLen(r *dns.Msg, proto dnsProtocol) int {
 	maxLen := minUDPSize

nameserver/http_test.go

@@ -27,7 +27,8 @@ func TestHttp(t *testing.T) {
 		dockerIP        = "9.8.7.6"
 	)
 
-	var zone = new(ZoneDb)
+	zone := NewZoneDb(DefaultLocalDomain)
+
 	port := rand.Intn(10000) + 32768
 	fmt.Println("Http test on port", port)
 	go ListenHTTP("", nil, testDomain, zone, port)
@@ -43,11 +44,11 @@ func TestHttp(t *testing.T) {
 	wt.AssertStatus(t, resp.StatusCode, http.StatusOK, "http response")
 
 	// Check that the address is now there.
+	ip, _, _ := net.ParseCIDR(testAddr1)
 	foundIP, err := zone.LookupName(successTestName)
 	wt.AssertNoErr(t, err)
-	ip, _, _ := net.ParseCIDR(testAddr1)
-	if !foundIP.Equal(ip) {
-		t.Fatal("Unexpected result for", successTestName, foundIP)
+	if !foundIP[0].IP().Equal(ip) {
+		t.Fatalf("Unexpected result for %s: received %s, expected %s", successTestName, foundIP, ip)
 	}
 
 	// Adding exactly the same address should be OK
@@ -70,7 +71,8 @@ func TestHttp(t *testing.T) {
 	wt.AssertStatus(t, resp.StatusCode, http.StatusOK, "http response")
 
 	// Check that the address is still resolvable.
-	_, err = zone.LookupName(successTestName)
+	x, err := zone.LookupName(successTestName)
+	t.Logf("Got %s", x)
 	wt.AssertNoErr(t, err)
 
 	// Delete the address record mentioning the other container
@@ -79,7 +81,8 @@ func TestHttp(t *testing.T) {
 	wt.AssertStatus(t, resp.StatusCode, http.StatusOK, "http response")
 
 	// Check that the address is gone
-	_, err = zone.LookupName(successTestName)
+	x, err = zone.LookupName(successTestName)
+	t.Logf("Got %s", x)
 	wt.AssertErrorType(t, err, (*LookupError)(nil), "fully-removed address")
 
 	// Would like to shut down the http server at the end of this test

nameserver/mdns_client.go

@@ -1,6 +1,8 @@
 package nameserver
 
 import (
+	"bytes"
+	"fmt"
 	"github.com/miekg/dns"
 	"math"
 	"net"
@@ -27,14 +29,53 @@ var (
 )
 
 type Response struct {
-	Name string
-	Addr net.IP
-	Err  error
+	name string
+	addr net.IP
+	ttl  int
+	err  error
+}
+
+func (r Response) Name() string  { return r.name }
+func (r Response) IP() net.IP    { return r.addr }
+func (r Response) Priority() int { return 0 }
+func (r Response) Weight() int   { return 0 }
+func (r Response) TTL() int      { return r.ttl }
+
+func (r Response) Equal(r2 *Response) bool {
+	if r.name != r2.name {
+		return false
+	}
+	if !r.addr.Equal(r2.addr) {
+		return false
+	}
+	if r.err != r2.err {
+		return false
+	}
+	return true
+}
+
+func (r Response) String() string {
+	var buf bytes.Buffer
+	if r.err != nil {
+		fmt.Fprintf(&buf, "%s", r.err)
+	} else {
+		if len(r.Name()) > 0 {
+			fmt.Fprintf(&buf, "%s", r.Name())
+		}
+		if !r.IP().IsUnspecified() {
+			fmt.Fprintf(&buf, "[%s]", r.IP())
+		}
+		if r.ttl > 0 {
+			fmt.Fprintf(&buf, "(TTL:%d)", r.TTL())
+		}
+	}
+	return buf.String()
 }
 
 type responseInfo struct {
-	timeout time.Time // if no answer by this time, give up
-	ch      chan<- *Response
+	timeout   time.Time // if no answer by this time, give up
+	insistent bool      // insistent queries are not removed on the first reply
+	ch        chan<- *Response
 }
 
 // Represents one query that we have sent for one name.
@@ -55,6 +96,7 @@ type MDNSClient struct {
 	addr       *net.UDPAddr
 	inflight   map[string]*inflightQuery
 	actionChan chan<- MDNSAction
+	running    bool
 }
 
 type mDNSQueryInfo struct {
@@ -64,17 +106,21 @@ type mDNSQueryInfo struct {
 }
 
 func NewMDNSClient() (*MDNSClient, error) {
-	if conn, err := net.ListenUDP("udp4", &net.UDPAddr{IP: net.IPv4zero, Port: 0}); err != nil {
-		return nil, err
-	} else {
-		return &MDNSClient{
-			conn:     conn,
-			addr:     ipv4Addr,
-			inflight: make(map[string]*inflightQuery)}, nil
-	}
+	return &MDNSClient{
+		addr:     ipv4Addr,
+		running:  false,
+		inflight: make(map[string]*inflightQuery)}, nil
 }
 
-func (c *MDNSClient) Start(ifi *net.Interface) error {
+func (c *MDNSClient) Start(ifi *net.Interface) (err error) {
+	if c.running {
+		return nil
+	}
+
+	if c.conn, err = net.ListenUDP("udp4", &net.UDPAddr{IP: net.IPv4zero, Port: 0}); err != nil {
+		return err
+	}
+
 	multicast, err := LinkLocalMulticastListener(ifi)
 	if err != nil {
 		return err
@@ -93,7 +139,13 @@ func (c *MDNSClient) Start(ifi *net.Interface) error {
 	actionChan := make(chan MDNSAction, MailboxSize)
 	c.actionChan = actionChan
 	go c.actorLoop(actionChan)
+	return nil
+}
 
+func (c *MDNSClient) Stop() error {
+	if c.running {
+		c.actionChan <- nil
+	}
 	return nil
 }
 
@@ -106,7 +158,7 @@ func LinkLocalMulticastListener(ifi *net.Interface) (net.PacketConn, error) {
 type MDNSAction func()
 
 // Async
-func (c *MDNSClient) SendQuery(name string, querytype uint16, responseCh chan<- *Response) {
+func (c *MDNSClient) SendQuery(name string, querytype uint16, insistent bool, responseCh chan<- *Response) {
 	c.actionChan <- func() {
 		query, found := c.inflight[name]
 		if !found {
@@ -115,7 +167,7 @@ func (c *MDNSClient) SendQuery(name string, querytype uint16, responseCh chan<-
 			m.RecursionDesired = false
 			buf, err := m.Pack()
 			if err != nil {
-				responseCh <- &Response{Err: err}
+				responseCh <- &Response{err: err}
 				close(responseCh)
 				return
 			}
@@ -124,15 +176,16 @@ func (c *MDNSClient) SendQuery(name string, querytype uint16, responseCh chan<-
 				id:   m.Id,
 			}
 			if _, err = c.conn.WriteTo(buf, c.addr); err != nil {
-				responseCh <- &Response{Err: err}
+				responseCh <- &Response{err: err}
 				close(responseCh)
 				return
 			}
 			c.inflight[name] = query
 		}
 		info := &responseInfo{
-			ch:      responseCh,
-			timeout: time.Now().Add(mDNSTimeout),
+			ch:        responseCh,
+			timeout:   time.Now().Add(mDNSTimeout),
+			insistent: insistent,
 		}
 		// Invariant on responseInfos: they are in ascending order of
 		// timeout.  Since we use a fixed interval from Now(), this
@@ -151,20 +204,31 @@ func (c *MDNSClient) ResponseCallback(r *dns.Msg) {
 			switch rr := answer.(type) {
 			case *dns.A:
 				name = rr.Hdr.Name
-				res = &Response{Addr: rr.A}
+				res = &Response{name: name, addr: rr.A, ttl: int(rr.Hdr.Ttl)}
 			case *dns.PTR:
 				name = rr.Hdr.Name
-				res = &Response{Name: rr.Ptr}
+				raddr, _ := raddrToIP(name)
+				res = &Response{name: rr.Ptr, addr: raddr, ttl: int(rr.Hdr.Ttl)}
 			default:
 				return
 			}
 
 			if query, found := c.inflight[name]; found {
+				var newResponseInfos []*responseInfo
 				for _, resp := range query.responseInfos {
 					resp.ch <- res
-					close(resp.ch)
+					// insistent queries are not removed on the first reply, but on the timeout
+					if resp.insistent {
+						newResponseInfos = append(newResponseInfos, resp)
+					} else {
+						close(resp.ch)
+					}
+				}
+				if len(newResponseInfos) == 0 {
+					delete(c.inflight, name)
+				} else {
+					query.responseInfos = newResponseInfos
 				}
-				delete(c.inflight, name)
 			} else {
 				// We've received a response that didn't match a query
 				// Do we want to cache it?
@@ -207,13 +271,13 @@ func (c *MDNSClient) checkInFlightQueries() time.Duration {
 func (c *MDNSClient) actorLoop(actionChan <-chan MDNSAction) {
 	timer := time.NewTimer(MaxDuration)
 	run := func() { timer.Reset(c.checkInFlightQueries()) }
-	terminate := false
-	for !terminate {
+	c.running = true
+	for c.running {
 		select {
 		case action := <-actionChan:
 			if action == nil {
 				c.listener.Shutdown()
-				terminate = true
+				c.running = false
 			} else {
 				action()
 				run()

nameserver/mdns_client_test.go

@@ -76,6 +76,8 @@ func RunLocalMulticastServer() (*dns.Server, error) {
 }
 
 func setup(t *testing.T) (*MDNSClient, *dns.Server, error) {
+	InitDefaultLogging(testing.Verbose())
+
 	server, err := RunLocalMulticastServer()
 	if err != nil {
 		t.Fatalf("Unable to run test server: %s. No default multicast interface?", err)
@@ -108,9 +110,9 @@ func (c *testContext) checkResponse(t *testing.T, channelOk bool, resp *Response
 		c.channel = nil
 		return
 	}
-	wt.AssertNoErr(t, resp.Err)
-	log.Printf("Got address response %s addr %s", resp.Name, resp.Addr)
-	c.receivedAddr = resp.Addr
+	wt.AssertNoErr(t, resp.err)
+	log.Printf("Got address response %s addr %s", resp.Name(), resp.IP())
+	c.receivedAddr = resp.IP()
 	c.receivedCount++
 }
 
@@ -123,7 +125,7 @@ func TestSimpleQuery(t *testing.T) {
 	context := newTestContext()
 
 	// First, a test we expect to succeed
-	mdnsClient.SendQuery(successTestName, dns.TypeA, context.channel)
+	mdnsClient.SendQuery(successTestName, dns.TypeA, false, context.channel)
 	for resp := range context.channel {
 		context.checkResponse(t, true, resp)
 	}
@@ -134,7 +136,7 @@ func TestSimpleQuery(t *testing.T) {
 
 	// Now, a test we expect to time out with no responses
 	context = newTestContext()
-	mdnsClient.SendQuery("test2.weave.", dns.TypeA, context.channel)
+	mdnsClient.SendQuery("test2.weave.", dns.TypeA, false, context.channel)
 	for resp := range context.channel {
 		context.checkResponse(t, true, resp)
 	}
@@ -153,8 +155,8 @@ func TestParallelQuery(t *testing.T) {
 	context1 := newTestContext()
 	context2 := newTestContext()
 
-	go mdnsClient.SendQuery(successTestName, dns.TypeA, context1.channel)
-	go mdnsClient.SendQuery(successTestName, dns.TypeA, context2.channel)
+	go mdnsClient.SendQuery(successTestName, dns.TypeA, false, context1.channel)
+	go mdnsClient.SendQuery(successTestName, dns.TypeA, false, context2.channel)
 	timeout := time.After(2 * time.Second)
 outerloop:
 	for context1.channel != nil || context2.channel != nil {
@@ -178,18 +180,18 @@ func TestAsLookup(t *testing.T) {
 	defer mdnsClient.Shutdown()
 	defer server.Shutdown()
 
-	ip, err := mdnsClient.LookupName(successTestName)
+	ips, err := mdnsClient.LookupName(successTestName)
 	wt.AssertNoErr(t, err)
-	if !testAddr.Equal(ip) {
-		t.Fatalf("Returned address incorrect %s", ip)
+	if !testAddr.Equal(ips[0].IP()) {
+		t.Fatalf("Returned address incorrect %s", ips)
 	}
 
-	ip, err = mdnsClient.LookupName("foo.example.com.")
+	ips, err = mdnsClient.LookupName("foo.example.com.")
 	wt.AssertErrorType(t, err, (*LookupError)(nil), "unknown hostname")
 
-	name, err := mdnsClient.LookupInaddr(testInAddr)
+	names, err := mdnsClient.LookupInaddr(testInAddr)
 	wt.AssertNoErr(t, err)
-	if !(successTestName == name) {
-		t.Fatalf("Expected name %s, got %s", successTestName, name)
+	if !(successTestName == names[0].Name()) {
+		t.Fatalf("Expected name %s, got %s", successTestName, names)
 	}
 }

nameserver/mdns_lookup.go

@@ -3,37 +3,69 @@ package nameserver
 import (
 	"github.com/miekg/dns"
 	. "github.com/weaveworks/weave/common"
-	"net"
 )
 
-func mdnsLookup(client *MDNSClient, name string, qtype uint16) (*Response, error) {
+func mdnsLookup(client *MDNSClient, name string, qtype uint16, insistent bool) ([]ZoneRecord, error) {
+	var responses []*Response
 	channel := make(chan *Response)
-	client.SendQuery(name, qtype, channel)
+	client.SendQuery(name, qtype, insistent, channel)
+
 	for resp := range channel {
-		if err := resp.Err; err != nil {
-			Debug.Printf("[mdns] Error for query type %s name %s: %s",
-				dns.TypeToString[qtype], name, err)
+		if err := resp.err; err != nil {
+			Debug.Printf("[mdns] Error for query type %s name %s: %s", dns.TypeToString[qtype], name, err)
 			return nil, err
 		}
-		Debug.Printf("[mdns] Got response name for query type %s name %s: name %s, addr %s",
-			dns.TypeToString[qtype], name, resp.Name, resp.Addr)
-		return resp, nil
+		Debug.Printf("[mdns] Got response name for %s-query about name '%s': name '%s', addr '%s'",
+			dns.TypeToString[qtype], name, resp.name, resp.addr)
+
+		if !insistent {
+			return []ZoneRecord{resp}, nil // non-background queries return the first response...
+		}
+
+		// add the response, avoiding duplicates
+		// TODO: replace this linear search by something better... (not a big deal as we don't expect long responses)
+		duplicate := false
+		for _, r := range responses {
+			if r.Equal(resp) {
+				duplicate = true
+				break
+			}
+		}
+		if !duplicate {
+			responses = append(responses, resp)
+		}
 	}
-	return nil, LookupError(name)
+
+	if len(responses) == 0 {
+		return nil, LookupError(name)
+	}
+
+	// convert []Response to []ZoneRecord
+	res := make([]ZoneRecord, len(responses))
+	for i, rr := range responses {
+		res[i] = ZoneRecord(rr)
+	}
+	return res, nil
 }
 
-func (client *MDNSClient) LookupName(name string) (net.IP, error) {
-	if r, e := mdnsLookup(client, name, dns.TypeA); r != nil {
-		return r.Addr, nil
-	} else {
-		return nil, e
-	}
+// Perform a lookup for a name, returning either an IP address or an error
+func (client *MDNSClient) LookupName(name string) ([]ZoneRecord, error) {
+	return mdnsLookup(client, name, dns.TypeA, false)
 }
 
-func (client *MDNSClient) LookupInaddr(inaddr string) (string, error) {
-	if r, e := mdnsLookup(client, inaddr, dns.TypePTR); r != nil {
-		return r.Name, nil
-	} else {
-		return "", e
-	}
+// Perform a lookup for an IP address, returning either a name or an error
+func (client *MDNSClient) LookupInaddr(inaddr string) ([]ZoneRecord, error) {
+	return mdnsLookup(client, inaddr, dns.TypePTR, false)
+}
+
+// Perform lookup for a name, returning either a list of IP addresses or an error
+// Insistent queries will wait up to `mDNSTimeout` milliseconds for responses
+func (client *MDNSClient) InsistentLookupName(name string) ([]ZoneRecord, error) {
+	return mdnsLookup(client, name, dns.TypeA, true)
+}
+
+// Perform an insistent lookup for an IP address, returning either a list of names or an error
+// Insistent queries will wait up to `mDNSTimeout` milliseconds for responses
+func (client *MDNSClient) InsistentLookupInaddr(inaddr string) ([]ZoneRecord, error) {
+	return mdnsLookup(client, inaddr, dns.TypePTR, true)
 }

nameserver/mdns_server.go

@@ -11,15 +11,13 @@ type MDNSServer struct {
 	sendconn   *net.UDPConn
 	srv        *dns.Server
 	zone       Zone
+	running    bool
 }
 
+// Create a new mDNS server
+// Nothing will be done (including port bindings) until you `Start()` the server
 func NewMDNSServer(zone Zone) (*MDNSServer, error) {
-	// This is a bit of a kludge - per the RFC we should send responses from 5353, but that doesn't seem to work
-	sendconn, err := net.ListenUDP("udp4", &net.UDPAddr{IP: net.IPv4zero, Port: 0})
-	if err != nil {
-		return nil, err
-	}
-	return &MDNSServer{sendconn: sendconn, zone: zone}, nil
+	return &MDNSServer{zone: zone}, nil
 }
 
 // Return true if testaddr is a UDP address with IP matching my local i/f
@@ -36,7 +34,19 @@ func (s *MDNSServer) addrIsLocal(testaddr net.Addr) bool {
 	return false
 }
 
-func (s *MDNSServer) Start(ifi *net.Interface, localDomain string) error {
+// Start the mDNS server
+func (s *MDNSServer) Start(ifi *net.Interface) (err error) {
+	// skip double initialization
+	if s.running {
+		return nil
+	}
+
+	// This is a bit of a kludge - per the RFC we should send responses from 5353, but that doesn't seem to work
+	s.sendconn, err = net.ListenUDP("udp4", &net.UDPAddr{IP: net.IPv4zero, Port: 0})
+	if err != nil {
+		return err
+	}
+
 	conn, err := LinkLocalMulticastListener(ifi)
 	if err != nil {
 		return err
@@ -52,23 +62,23 @@ func (s *MDNSServer) Start(ifi *net.Interface, localDomain string) error {
 	}
 
 	handleLocal := s.makeHandler(dns.TypeA,
-		func(zone Lookup, r *dns.Msg, q *dns.Question) *dns.Msg {
-			if ip, err := zone.LookupName(q.Name); err == nil {
-				return makeAddressReply(r, q, []net.IP{ip})
+		func(zone ZoneLookup, r *dns.Msg, q *dns.Question) *dns.Msg {
+			if ips, err := zone.LookupName(q.Name); err == nil {
+				return makeAddressReply(r, q, ips)
 			}
 			return nil
 		})
 
 	handleReverse := s.makeHandler(dns.TypePTR,
-		func(zone Lookup, r *dns.Msg, q *dns.Question) *dns.Msg {
-			if name, err := zone.LookupInaddr(q.Name); err == nil {
-				return makePTRReply(r, q, []string{name})
+		func(zone ZoneLookup, r *dns.Msg, q *dns.Question) *dns.Msg {
+			if names, err := zone.LookupInaddr(q.Name); err == nil {
+				return makePTRReply(r, q, names)
 			}
 			return nil
 		})
 
 	mux := dns.NewServeMux()
-	mux.HandleFunc(localDomain, handleLocal)
+	mux.HandleFunc(s.zone.Domain(), handleLocal)
 	mux.HandleFunc(RDNSDomain, handleReverse)
 
 	s.srv = &dns.Server{
@@ -77,17 +87,24 @@ func (s *MDNSServer) Start(ifi *net.Interface, localDomain string) error {
 		Handler:    mux,
 	}
 	go s.srv.ActivateAndServe()
+	s.running = true
 	return err
 }
 
+// Stop the mDNS server
 func (s *MDNSServer) Stop() error {
+	s.running = false
 	return s.srv.Shutdown()
 }
 
-type LookupFunc func(Lookup, *dns.Msg, *dns.Question) *dns.Msg
+func (s *MDNSServer) Zone() Zone {
+	return s.zone
+}
+
+type LookupFunc func(ZoneLookup, *dns.Msg, *dns.Question) *dns.Msg
 
 func (s *MDNSServer) makeHandler(qtype uint16, lookup LookupFunc) dns.HandlerFunc {
-	return func(_ dns.ResponseWriter, r *dns.Msg) {
+	return func(rw dns.ResponseWriter, r *dns.Msg) {
 		// Handle only questions, ignore answers. We might also ignore
 		// questions that arise locally (i.e., that come from an IP we
 		// think is local), but in the interest of avoiding
@@ -95,18 +112,20 @@ func (s *MDNSServer) makeHandler(qtype uint16, lookup LookupFunc) dns.HandlerFun
 		// assumption that the client wouldn't ask if it already knew
 		// the answer, and if it does ask, it'll be happy to get an
 		// answer.
+		// TODO: use any Answer in the query for adding records in the Zone database...
 		if len(r.Answer) == 0 && len(r.Question) > 0 {
 			q := &r.Question[0]
 			if q.Qtype == qtype {
+				Debug.Printf("[mdns msgid %d] srv: trying to answer to mDNS query '%s'", r.MsgHdr.Id, q.Name)
 				if m := lookup(s.zone, r, q); m != nil {
-					Debug.Printf("[mdns msgid %d] Found local answer to mDNS query %s",
-						r.MsgHdr.Id, q.Name)
+					Debug.Printf("[mdns msgid %d] srv: found local answer to mDNS query '%s'", r.MsgHdr.Id, q.Name)
 					if err := s.sendResponse(m); err != nil {
-						Warning.Printf("[mdns msgid %d] Error writing to %v",
-							r.MsgHdr.Id, s.sendconn)
+						Warning.Printf("[mdns msgid %d] srv: error writing mDNS response to %v", r.MsgHdr.Id, s.sendconn)
+					} else {
+						Debug.Printf("[mdns msgid %d] srv: response sent: %d answers", r.MsgHdr.Id, len(m.Answer))
 					}
 				} else {
-					Debug.Printf("[mdns msgid %d] No local answer for mDNS query %s",
+					Debug.Printf("[mdns msgid %d] srv: no local answer for answering mDNS query '%s'",
 						r.MsgHdr.Id, q.Name)
 				}
 			}

nameserver/mdns_server_test.go

@@ -2,82 +2,51 @@ package nameserver
 
 import (
 	"github.com/miekg/dns"
-	"github.com/weaveworks/weave/common"
+	. "github.com/weaveworks/weave/common"
 	wt "github.com/weaveworks/weave/testing"
-	"log"
 	"net"
 	"testing"
 	"time"
 )
 
-var (
-	containerID = "deadbeef"
-	testName    = "test.weave.local."
-	testAddr1   = "10.2.2.1/24"
-	testInAddr1 = "1.2.2.10.in-addr.arpa."
-)
-
-func sendQuery(name string, querytype uint16) error {
-	m := new(dns.Msg)
-	m.SetQuestion(name, querytype)
-	m.RecursionDesired = false
-	buf, err := m.Pack()
-	if err != nil {
-		return err
-	}
-	conn, err := net.ListenUDP("udp4", &net.UDPAddr{IP: net.IPv4zero, Port: 0})
-	if err != nil {
-		return err
-	}
-	_, err = conn.WriteTo(buf, ipv4Addr)
-	return err
-}
-
 func TestServerSimpleQuery(t *testing.T) {
-	// The ff can be handy for debugging (obvs)
-	common.InitDefaultLogging(true)
+	var (
+		testRecord1 = Record{"test.weave.local.", net.ParseIP("10.20.20.10"), 0, 0, 0}
+		testRecord2 = Record{"test.weave.local.", net.ParseIP("10.20.20.20"), 0, 0, 0}
+		testInAddr1 = "10.20.20.10.in-addr.arpa."
+	)
 
-	log.Println("TestServerSimpleQuery starting")
-	var zone = new(ZoneDb)
-	ip, _, _ := net.ParseCIDR(testAddr1)
-	zone.AddRecord(containerID, testName, ip)
+	InitDefaultLogging(testing.Verbose())
+	Info.Println("TestServerSimpleQuery starting")
 
-	var receivedAddr net.IP
-	var receivedName string
-	var recvChan chan interface{}
+	mzone := newMockedZoneWithRecords([]ZoneRecord{testRecord1, testRecord2})
+	mdnsServer, err := NewMDNSServer(mzone)
+	wt.AssertNoErr(t, err)
+	err = mdnsServer.Start(nil)
+	wt.AssertNoErr(t, err)
+	defer mdnsServer.Stop()
+
+	var receivedAddrs []net.IP
+	receivedName := ""
+	recvChan := make(chan interface{})
 	receivedCount := 0
 
-	reset := func() {
-		receivedAddr = nil
-		receivedName = ""
-		receivedCount = 0
-		recvChan = make(chan interface{})
-	}
-
-	wait := func() {
-		select {
-		case <-recvChan:
-			return
-		case <-time.After(100 * time.Millisecond):
-			return
-		}
-	}
-
 	// Implement a minimal listener for responses
 	multicast, err := LinkLocalMulticastListener(nil)
-	if err != nil {
-		t.Fatalf("Unable to run test server: %s. No default multicast interface?", err)
-	}
+	wt.AssertNoErr(t, err)
 
 	handleMDNS := func(w dns.ResponseWriter, r *dns.Msg) {
 		// Only handle responses here
 		if len(r.Answer) > 0 {
+			t.Logf("Received %d answer(s)", len(r.Answer))
 			for _, answer := range r.Answer {
 				switch rr := answer.(type) {
 				case *dns.A:
-					receivedAddr = rr.A
+					t.Logf("... A:\n%+v", rr)
+					receivedAddrs = append(receivedAddrs, rr.A)
 					receivedCount++
 				case *dns.PTR:
+					t.Logf("... PTR:\n%+v", rr)
 					receivedName = rr.Ptr
 					receivedCount++
 				}
@@ -86,6 +55,33 @@ func TestServerSimpleQuery(t *testing.T) {
 		}
 	}
 
+	sendQuery := func(name string, querytype uint16) {
+		receivedAddrs = make([]net.IP, 0)
+		receivedName = ""
+		receivedCount = 0
+		recvChan = make(chan interface{})
+
+		m := new(dns.Msg)
+		m.SetQuestion(name, querytype)
+		m.RecursionDesired = false
+		buf, err := m.Pack()
+		wt.AssertNoErr(t, err)
+		conn, err := net.ListenUDP("udp4", &net.UDPAddr{IP: net.IPv4zero, Port: 0})
+		wt.AssertNoErr(t, err)
+		Debug.Printf("Sending UDP packet to %s", ipv4Addr)
+		_, err = conn.WriteTo(buf, ipv4Addr)
+		wt.AssertNoErr(t, err)
+
+		Debug.Printf("Waiting for response")
+		select {
+		case <-recvChan:
+			return
+		case <-time.After(100 * time.Millisecond):
+			Debug.Printf("Timeout while waiting for response")
+			return
+		}
+	}
+
 	listener := &dns.Server{
 		Unsafe:      true,
 		PacketConn:  multicast,
@@ -94,39 +90,31 @@ func TestServerSimpleQuery(t *testing.T) {
 	go listener.ActivateAndServe()
 	defer listener.Shutdown()
 
-	mdnsServer, err := NewMDNSServer(zone)
-	wt.AssertNoErr(t, err)
-	err = mdnsServer.Start(nil, DefaultLocalDomain)
-	wt.AssertNoErr(t, err)
-
 	time.Sleep(100 * time.Millisecond) // Allow for server to get going
 
-	reset()
-	sendQuery(testName, dns.TypeA)
-	wait()
-
-	if receivedCount != 1 {
-		t.Fatalf("Unexpected result count %d for %s", receivedCount, testName)
+	Debug.Printf("Checking that we get 2 IPs fo name '%s' [A]", testRecord1.Name())
+	sendQuery(testRecord1.Name(), dns.TypeA)
+	if receivedCount != 2 {
+		t.Fatalf("Unexpected result count %d for %s", receivedCount, testRecord1.Name())
 	}
-	if !receivedAddr.Equal(ip) {
-		t.Fatalf("Unexpected result %s for %s", receivedAddr, testName)
+	if !(receivedAddrs[0].Equal(testRecord1.IP()) || receivedAddrs[0].Equal(testRecord2.IP())) {
+		t.Fatalf("Unexpected result %s for %s", receivedAddrs, testRecord1.Name())
+	}
+	if !(receivedAddrs[1].Equal(testRecord1.IP()) || receivedAddrs[1].Equal(testRecord2.IP())) {
+		t.Fatalf("Unexpected result %s for %s", receivedAddrs, testRecord1.Name())
 	}
 
-	reset()
+	Debug.Printf("Checking that 'testfail.weave.' [A] gets no answers")
 	sendQuery("testfail.weave.", dns.TypeA)
-	wait()
-
 	if receivedCount != 0 {
 		t.Fatalf("Unexpected result count %d for testfail.weave", receivedCount)
 	}
 
-	reset()
+	Debug.Printf("Checking that '%s' [PTR] gets one name", testInAddr1)
 	sendQuery(testInAddr1, dns.TypePTR)
-	wait()
-
 	if receivedCount != 1 {
 		t.Fatalf("Expected an answer to %s, got %d answers", testInAddr1, receivedCount)
-	} else if !(testName == receivedName) {
-		t.Fatalf("Expected answer %s to query for %s, got %s", testName, testInAddr1, receivedName)
+	} else if !(testRecord1.Name() == receivedName) {
+		t.Fatalf("Expected answer %s to query for %s, got %s", testRecord1.Name(), testInAddr1, receivedName)
 	}
 }

nameserver/mocks_test.go

@@ -0,0 +1,91 @@
+package nameserver
+
+import (
+	. "github.com/weaveworks/weave/common"
+	"net"
+	"sync"
+)
+
+const (
+	testSocketTimeout = 100 // in millisecs
+)
+
+// Warn about some methods that some day should be implemented...
+func notImplWarn() { Warning.Printf("Mocked method. Not implemented.") }
+
+// A mocked Zone that always returns the same records
+// * it does not send/receive any mDNS query
+type mockedZoneWithRecords struct {
+	sync.RWMutex
+
+	records []ZoneRecord
+
+	// Statistics
+	NumLookupsName   int
+	NumLookupsInaddr int
+}
+
+func newMockedZoneWithRecords(zr []ZoneRecord) *mockedZoneWithRecords {
+	return &mockedZoneWithRecords{records: zr}
+}
+func (mz *mockedZoneWithRecords) Domain() string { return DefaultLocalDomain }
+func (mz *mockedZoneWithRecords) LookupName(name string) ([]ZoneRecord, error) {
+	Debug.Printf("[mocked zone]: LookupName: returning records %s", mz.records)
+	mz.Lock()
+	defer mz.Unlock()
+
+	mz.NumLookupsName++
+	var res []ZoneRecord
+	for _, r := range mz.records {
+		if r.Name() == name {
+			res = append(res, r)
+		}
+	}
+	return res, nil
+}
+
+func (mz *mockedZoneWithRecords) LookupInaddr(inaddr string) ([]ZoneRecord, error) {
+	Debug.Printf("[mocked zone]: LookupInaddr: returning records %s", mz.records)
+	mz.Lock()
+	defer mz.Unlock()
+
+	mz.NumLookupsInaddr++
+	var res []ZoneRecord
+	for _, r := range mz.records {
+		revIP, err := raddrToIP(inaddr)
+		if err != nil {
+			return nil, newParseError("lookup address", inaddr)
+		}
+		if r.IP().Equal(revIP) {
+			res = append(res, r)
+		}
+	}
+	return res, nil
+}
+
+func (mz *mockedZoneWithRecords) DomainLookupName(name string) ([]ZoneRecord, error) {
+	return mz.LookupName(name)
+}
+func (mz *mockedZoneWithRecords) DomainLookupInaddr(inaddr string) ([]ZoneRecord, error) {
+	return mz.LookupInaddr(inaddr)
+}
+
+// the following methods are not currently needed...
+func (mz *mockedZoneWithRecords) AddRecord(ident string, name string, ip net.IP) error {
+	notImplWarn()
+	return nil
+}
+func (mz *mockedZoneWithRecords) DeleteRecord(ident string, ip net.IP) error {
+	notImplWarn()
+	return nil
+}
+func (mz *mockedZoneWithRecords) DeleteRecordsFor(ident string) error { notImplWarn(); return nil }
+func (mz *mockedZoneWithRecords) Status() string                      { notImplWarn(); return "nothing" }
+func (mz *mockedZoneWithRecords) ObserveName(name string, observer ZoneRecordObserver) error {
+	notImplWarn()
+	return nil
+}
+func (mz *mockedZoneWithRecords) ObserveInaddr(inaddr string, observer ZoneRecordObserver) error {
+	notImplWarn()
+	return nil
+}

nameserver/server.go

@@ -7,7 +7,6 @@ import (
 	. "github.com/weaveworks/weave/common"
 	"net"
 	"sync"
-	"time"
 )
 
 const (
@@ -130,7 +129,7 @@ func NewDNSServer(config DNSServerConfig, zone Zone, iface *net.Interface) (s *D
 		cacheLen = config.CacheLen
 	}
 	Debug.Printf("[dns] Initializing cache: %d entries", config.CacheLen)
-	s.cache, err = NewCache(cacheLen)
+	s.cache, err = NewCache(cacheLen, nil)
 	if err != nil {
 		return
 	}
@@ -156,7 +155,7 @@ func (s *DNSServer) Start() error {
 	Info.Printf("Using mDNS on %v", s.Iface)
 	err := s.mdnsCli.Start(s.Iface)
 	CheckFatal(err)
-	err = s.mdnsSrv.Start(s.Iface, s.Domain)
+	err = s.mdnsSrv.Start(s.Iface)
 	CheckFatal(err)
 
 	s.listenersWg.Add(2)
@@ -215,75 +214,53 @@ func (s *DNSServer) Stop() error {
 }
 
 func (s *DNSServer) queryHandler(proto dnsProtocol) dns.HandlerFunc {
-	return func(w dns.ResponseWriter, r *dns.Msg) {
-		now := time.Now()
-		q := r.Question[0]
-		maxLen := getMaxReplyLen(r, proto)
-		lookups := []Lookup{s.Zone, s.mdnsCli}
-		Debug.Printf("Query: %+v", q)
-
-		reply, err := s.cache.Get(r, maxLen, time.Now())
+	zoneLookup := func(lookup ZoneLookup, q *dns.Question, r *dns.Msg) (*dns.Msg, []ZoneRecord, error) {
+		ips, err := lookup.LookupName(q.Name)
 		if err != nil {
-			if err == errNoLocalReplies {
-				Debug.Printf("[dns msgid %d] Cached 'no local replies' - skipping local lookup", r.MsgHdr.Id)
-				lookups = []Lookup{s.Zone}
-			} else {
-				Debug.Printf("[dns msgid %d] Error from cache: %s", r.MsgHdr.Id, err)
-				w.WriteMsg(makeDNSFailResponse(r))
-				return
-			}
-		}
-		if reply != nil {
-			Debug.Printf("[dns msgid %d] Returning reply from cache: %s/%d answers",
-				r.MsgHdr.Id, dns.RcodeToString[reply.MsgHdr.Rcode], len(reply.Answer))
-			w.WriteMsg(reply)
-			return
+			return nil, nil, err
 		}
+		return makeAddressReply(r, q, ips), ips, nil
+	}
 
-		// catch unsupported queries
-		if q.Qtype != dns.TypeA {
-			Debug.Printf("[dns msgid %d] Unsuported query type %s", r.MsgHdr.Id, dns.TypeToString[q.Qtype])
-			m := makeDNSNotImplResponse(r)
-			s.cache.Put(r, m, negLocalTTL, 0, now)
-			w.WriteMsg(m)
-			return
-		}
-
-		for _, lookup := range lookups {
-			if ip, err := lookup.LookupName(q.Name); err == nil {
-				m := makeAddressReply(r, &q, []net.IP{ip})
-				m.Authoritative = true
-
-				Debug.Printf("[dns msgid %d] Caching response for %s-query for \"%s\": %s [code:%s]",
-					m.MsgHdr.Id, dns.TypeToString[q.Qtype], q.Name, ip, dns.RcodeToString[m.Rcode])
-				s.cache.Put(r, m, nullTTL, 0, now)
-				w.WriteMsg(m)
-				return
-			}
-			now = time.Now()
-		}
-
-		Info.Printf("[dns msgid %d] No results for type %s query %s [caching no-local]",
-			r.MsgHdr.Id, dns.TypeToString[q.Qtype], q.Name)
-		s.cache.Put(r, nil, negLocalTTL, CacheNoLocalReplies, now)
+	fallback := func(w dns.ResponseWriter, r *dns.Msg) {
 		w.WriteMsg(makeDNSFailResponse(r))
 	}
+
+	return s.commonQueryHandler(proto, "Query", dns.TypeA, zoneLookup, fallback)
 }
 
 func (s *DNSServer) rdnsHandler(proto dnsProtocol) dns.HandlerFunc {
-	fallback := s.notUsHandler(proto)
-	return func(w dns.ResponseWriter, r *dns.Msg) {
-		now := time.Now()
-		q := r.Question[0]
-		lookups := []Lookup{s.Zone, s.mdnsCli}
-		maxLen := getMaxReplyLen(r, proto)
+	zoneLookup := func(lookup ZoneLookup, q *dns.Question, r *dns.Msg) (*dns.Msg, []ZoneRecord, error) {
+		names, err := lookup.LookupInaddr(q.Name)
+		if err != nil {
+			return nil, nil, err
+		}
+		return makePTRReply(r, q, names), names, nil
+	}
 
-		Debug.Printf("Reverse query: %+v", q)
-		reply, err := s.cache.Get(r, maxLen, time.Now())
+	notUsHandler := s.notUsHandler(proto)
+	fallback := func(w dns.ResponseWriter, r *dns.Msg) {
+		Info.Printf("[dns msgid %d] -> sending to fallback server", r.MsgHdr.Id)
+		notUsHandler(w, r)
+	}
+
+	return s.commonQueryHandler(proto, "Reverse query", dns.TypePTR, zoneLookup, fallback)
+}
+
+func (s *DNSServer) commonQueryHandler(proto dnsProtocol, kind string, qtype uint16,
+	zoneLookup func(ZoneLookup, *dns.Question, *dns.Msg) (*dns.Msg, []ZoneRecord, error), fallback dns.HandlerFunc) dns.HandlerFunc {
+
+	return func(w dns.ResponseWriter, r *dns.Msg) {
+		q := r.Question[0]
+		Debug.Printf("[dns msgid %d] %s: %+v", r.MsgHdr.Id, kind, q)
+		maxLen := getMaxReplyLen(r, proto)
+		lookups := []ZoneLookup{s.Zone, s.mdnsCli}
+
+		reply, err := s.cache.Get(r, maxLen)
 		if err != nil {
 			if err == errNoLocalReplies {
 				Debug.Printf("[dns msgid %d] Cached 'no local replies' - skipping local lookup", r.MsgHdr.Id)
-				lookups = []Lookup{s.Zone}
+				lookups = []ZoneLookup{s.Zone}
 			} else {
 				Debug.Printf("[dns msgid %d] Error from cache: %s", r.MsgHdr.Id, err)
 				w.WriteMsg(makeDNSFailResponse(r))
@@ -298,32 +275,29 @@ func (s *DNSServer) rdnsHandler(proto dnsProtocol) dns.HandlerFunc {
 		}
 
 		// catch unsupported queries
-		if q.Qtype != dns.TypePTR {
-			Warning.Printf("[dns msgid %d] Unexpected reverse query type %s: %+v",
-				r.MsgHdr.Id, dns.TypeToString[q.Qtype], q)
-			m := makeDNSNotImplResponse(r)
-			s.cache.Put(r, m, negLocalTTL, 0, now)
+		if q.Qtype != qtype {
+			Debug.Printf("[dns msgid %d] Unsupported query type %s", r.MsgHdr.Id, dns.TypeToString[q.Qtype])
+			m := makeDNSFailResponse(r)
+			s.cache.Put(r, m, negLocalTTL, 0)
 			w.WriteMsg(m)
 			return
 		}
 
 		for _, lookup := range lookups {
-			if name, err := lookup.LookupInaddr(q.Name); err == nil {
-				m := makePTRReply(r, &q, []string{name})
+			if m, answers, err := zoneLookup(lookup, &q, r); err == nil {
 				m.Authoritative = true
-
-				Debug.Printf("[dns msgid %d] Caching response for %s-query for \"%s\": %s [code:%s]",
-					m.MsgHdr.Id, dns.TypeToString[q.Qtype], q.Name, name, dns.RcodeToString[m.Rcode])
-				s.cache.Put(r, m, nullTTL, 0, now)
+				Debug.Printf("[dns msgid %d] Caching response for type %s query for '%s': %s [code:%s]",
+					m.MsgHdr.Id, dns.TypeToString[q.Qtype], q.Name, answers, dns.RcodeToString[m.Rcode])
+				s.cache.Put(r, m, nullTTL, 0)
 				w.WriteMsg(m)
 				return
 			}
-			now = time.Now()
 		}
 
-		Info.Printf("[dns msgid %d] No results for %s-query about '%s' [caching no-local] -> sending to fallback server",
+		Info.Printf("[dns msgid %d] No results for type %s query for '%s' [caching no-local]",
 			r.MsgHdr.Id, dns.TypeToString[q.Qtype], q.Name)
-		s.cache.Put(r, nil, negLocalTTL, CacheNoLocalReplies, now)
+		s.cache.Put(r, nil, negLocalTTL, CacheNoLocalReplies)
+
 		fallback(w, r)
 	}
 }

nameserver/server_test.go

@@ -36,6 +36,7 @@ func setupForTest(t *testing.T) {
 func TestUDPDNSServer(t *testing.T) {
 	setupForTest(t)
 	const (
+		containerID     = "foobar"
 		successTestName = "test1.weave.local."
 		failTestName    = "test2.weave.local."
 		nonLocalName    = "weave.works."
@@ -44,7 +45,7 @@ func TestUDPDNSServer(t *testing.T) {
 	testCIDR1 := testAddr1 + "/24"
 
 	InitDefaultLogging(true)
-	var zone = new(ZoneDb)
+	var zone = NewZoneDb(DefaultLocalDomain)
 	ip, _, _ := net.ParseCIDR(testCIDR1)
 	zone.AddRecord(containerID, successTestName, ip)
 
@@ -100,8 +101,8 @@ func TestUDPDNSServer(t *testing.T) {
 
 	assertExchange(t, testRDNSfail, dns.TypePTR, 0, 0, dns.RcodeNameError)
 
-	// This should fail because we don't handle MX records
-	assertExchange(t, successTestName, dns.TypeMX, 0, 0, dns.RcodeNotImplemented)
+	// This should fail because we don't have information about MX records
+	assertExchange(t, successTestName, dns.TypeMX, 0, 0, dns.RcodeNameError)
 
 	// This non-local query for an MX record should succeed by being
 	// passed on to the fallback server
@@ -125,14 +126,14 @@ func TestTCPDNSServer(t *testing.T) {
 	dnsAddr := fmt.Sprintf("localhost:%d", testPort)
 
 	InitDefaultLogging(true)
-	var zone = new(ZoneDb)
+	var zone = NewZoneDb(DefaultLocalDomain)
 
 	// generate a list of `numAnswers` IP addresses
-	var addrs []net.IP
+	var addrs []ZoneRecord
 	bs := make([]byte, 4)
 	for i := 0; i < numAnswers; i++ {
 		binary.LittleEndian.PutUint32(bs, uint32(i))
-		addrs = append(addrs, net.IPv4(bs[0], bs[1], bs[2], bs[3]))
+		addrs = append(addrs, Record{"", net.IPv4(bs[0], bs[1], bs[2], bs[3]), 0, 0, 0})
 	}
 
 	// handler for the fallback server: it will just return a very long response

nameserver/zone.go

@@ -16,19 +16,15 @@ const (
 // +1 to also exclude a dot
 var rdnsDomainLen = len(RDNSDomain) + 1
 
-type Lookup interface {
-	LookupName(name string) (net.IP, error)
-	LookupInaddr(inaddr string) (string, error)
-}
-
 type Zone interface {
 	AddRecord(ident string, name string, ip net.IP) error
 	DeleteRecord(ident string, ip net.IP) error
 	DeleteRecordsFor(ident string) error
-	Lookup
+	Domain() string
+	ZoneLookup
 }
 
-type Record struct {
+type dbRecord struct {
 	Ident string
 	Name  string
 	IP    net.IP
@@ -37,8 +33,9 @@ type Record struct {
 // Very simple data structure for now, with linear searching.
 // TODO: make more sophisticated to improve performance.
 type ZoneDb struct {
-	mx   sync.RWMutex
-	recs []Record
+	mx     sync.RWMutex
+	recs   []dbRecord
+	domain string
 }
 
 type LookupError string
@@ -54,7 +51,17 @@ func (dup DuplicateError) Error() string {
 	return "Tried to add a duplicate entry"
 }
 
-func (zone *ZoneDb) indexOf(match func(Record) bool) int {
+func NewZoneDb(domain string) *ZoneDb {
+	return &ZoneDb{
+		domain: domain,
+	}
+}
+
+func (zone *ZoneDb) Domain() string {
+	return zone.domain
+}
+
+func (zone *ZoneDb) indexOf(match func(dbRecord) bool) int {
 	for i, r := range zone.recs {
 		if match(r) {
 			return i
@@ -73,18 +80,18 @@ func (zone *ZoneDb) String() string {
 	return buf.String()
 }
 
-func (zone *ZoneDb) LookupName(name string) (net.IP, error) {
+func (zone *ZoneDb) LookupName(name string) ([]ZoneRecord, error) {
 	zone.mx.RLock()
 	defer zone.mx.RUnlock()
 	for _, r := range zone.recs {
 		if r.Name == name {
-			return r.IP, nil
+			return []ZoneRecord{Record{r.Name, r.IP, 0, 0, 0}}, nil
 		}
 	}
 	return nil, LookupError(name)
 }
 
-func (zone *ZoneDb) LookupInaddr(inaddr string) (string, error) {
+func (zone *ZoneDb) LookupInaddr(inaddr string) ([]ZoneRecord, error) {
 	if revIP := net.ParseIP(inaddr[:len(inaddr)-rdnsDomainLen]); revIP != nil {
 		revIP4 := revIP.To4()
 		ip := []byte{revIP4[3], revIP4[2], revIP4[1], revIP4[0]}
@@ -93,31 +100,31 @@ func (zone *ZoneDb) LookupInaddr(inaddr string) (string, error) {
 		defer zone.mx.RUnlock()
 		for _, r := range zone.recs {
 			if r.IP.Equal(ip) {
-				return r.Name, nil
+				return []ZoneRecord{Record{r.Name, r.IP, 0, 0, 0}}, nil
 			}
 		}
-		return "", LookupError(inaddr)
+		return nil, LookupError(inaddr)
 	}
 	Warning.Printf("[zonedb] Asked to reverse lookup %s", inaddr)
-	return "", LookupError(inaddr)
+	return nil, LookupError(inaddr)
 }
 
 func (zone *ZoneDb) AddRecord(ident string, name string, ip net.IP) error {
 	zone.mx.Lock()
 	defer zone.mx.Unlock()
 	fqdn := dns.Fqdn(name)
-	pred := func(r Record) bool { return r.Name == fqdn && r.IP.Equal(ip) && r.Ident == ident }
+	pred := func(r dbRecord) bool { return r.Name == fqdn && r.IP.Equal(ip) && r.Ident == ident }
 	if index := zone.indexOf(pred); index != -1 {
 		return DuplicateError{}
 	}
-	zone.recs = append(zone.recs, Record{ident, fqdn, ip})
+	zone.recs = append(zone.recs, dbRecord{ident, fqdn, ip})
 	return nil
 }
 
 func (zone *ZoneDb) DeleteRecord(ident string, ip net.IP) error {
 	zone.mx.Lock()
 	defer zone.mx.Unlock()
-	pred := func(r Record) bool { return r.Ident == ident && r.IP.Equal(ip) }
+	pred := func(r dbRecord) bool { return r.Ident == ident && r.IP.Equal(ip) }
 	if index := zone.indexOf(pred); index != -1 {
 		zone.recs = append(zone.recs[:index], zone.recs[index+1:]...)
 		return nil

nameserver/zone_test.go

@@ -14,7 +14,7 @@ func TestZone(t *testing.T) {
 		testAddr1        = "10.2.2.1/24"
 	)
 
-	var zone = new(ZoneDb)
+	var zone = NewZoneDb(DefaultLocalDomain)
 
 	ip, _, _ := net.ParseCIDR(testAddr1)
 	err := zone.AddRecord(containerID, successTestName, ip)
@@ -30,7 +30,7 @@ func TestZone(t *testing.T) {
 	foundIP, err := zone.LookupName(successTestName)
 	wt.AssertNoErr(t, err)
 
-	if !foundIP.Equal(ip) {
+	if !foundIP[0].IP().Equal(ip) {
 		t.Fatal("Unexpected result for", successTestName, foundIP)
 	}
 
@@ -38,7 +38,7 @@ func TestZone(t *testing.T) {
 	foundName, err := zone.LookupInaddr("1.2.2.10.in-addr.arpa.")
 	wt.AssertNoErr(t, err)
 
-	if foundName != successTestName {
+	if foundName[0].Name() != successTestName {
 		t.Fatal("Unexpected result for", ip, foundName)
 	}
 
@@ -72,7 +72,7 @@ func TestDeleteFor(t *testing.T) {
 		addr1 = "10.2.2.3/24"
 		addr2 = "10.2.7.8/24"
 	)
-	zone := new(ZoneDb)
+	zone := NewZoneDb(DefaultLocalDomain)
 	for _, addr := range []string{addr1, addr2} {
 		ip, _, _ := net.ParseCIDR(addr)
 		err := zone.AddRecord(id, name, ip)

proxy/client.go

@@ -0,0 +1,170 @@
+package proxy
+
+import (
+	"bufio"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"net/http/httputil"
+
+	"github.com/fsouza/go-dockerclient"
+	. "github.com/weaveworks/weave/common"
+)
+
+type client struct {
+	interceptor
+	Dial func() (net.Conn, error)
+}
+
+func newClient(dial func() (net.Conn, error), i interceptor) *client {
+	return &client{
+		interceptor: i,
+		Dial:        dial,
+	}
+}
+
+func (c *client) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if err := c.InterceptRequest(r); err != nil {
+		_, isNoSuchContainer := err.(*docker.NoSuchContainer)
+		switch {
+		case isNoSuchContainer:
+			http.Error(w, err.Error(), http.StatusNotFound)
+		case err == docker.ErrNoSuchImage:
+			http.Error(w, err.Error(), http.StatusNotFound)
+		default:
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			Warning.Print("Error intercepting request: ", err)
+		}
+		return
+	}
+
+	conn, err := c.Dial()
+	if err != nil {
+		http.Error(w, "Could not connect to target", http.StatusInternalServerError)
+		Warning.Print(err)
+		return
+	}
+	client := httputil.NewClientConn(conn, nil)
+	defer client.Close()
+
+	resp, err := client.Do(r)
+	if err != nil && err != httputil.ErrPersistEOF {
+		http.Error(w, fmt.Sprintf("Could not make request to target: %v", err), http.StatusInternalServerError)
+		Warning.Print("Error forwarding request: ", err)
+		return
+	}
+	err = c.InterceptResponse(resp)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		Warning.Print("Error intercepting response: ", err)
+		return
+	}
+
+	hdr := w.Header()
+	for k, vs := range resp.Header {
+		for _, v := range vs {
+			hdr.Add(k, v)
+		}
+	}
+	Debug.Printf("Response from target: %s %v", resp.Status, w.Header())
+
+	if resp.Header.Get("Content-Type") == "application/vnd.docker.raw-stream" {
+		doRawStream(w, resp, client)
+	} else if resp.TransferEncoding != nil && resp.TransferEncoding[0] == "chunked" {
+		doChunkedResponse(w, resp, client)
+	} else {
+		w.WriteHeader(resp.StatusCode)
+		if _, err := io.Copy(w, resp.Body); err != nil {
+			Warning.Print(err)
+		}
+	}
+}
+
+func doRawStream(w http.ResponseWriter, resp *http.Response, client *httputil.ClientConn) {
+	down, downBuf, up, rem, err := hijack(w, client)
+	if err != nil {
+		http.Error(w, "Unable to hijack connection for raw stream mode", http.StatusInternalServerError)
+		return
+	}
+	defer down.Close()
+	defer up.Close()
+
+	if _, err := down.Write([]byte("HTTP/1.1 200 OK\n")); err != nil {
+		Warning.Print(err)
+		return
+	}
+
+	if err := resp.Header.Write(down); err != nil {
+		Warning.Print(err)
+		return
+	}
+
+	if _, err := down.Write([]byte("\n")); err != nil {
+		Warning.Print(err)
+		return
+	}
+
+	upDone := make(chan struct{})
+	downDone := make(chan struct{})
+	go copyStream(down, io.MultiReader(rem, up), upDone)
+	go copyStream(up, downBuf, downDone)
+	<-upDone
+	<-downDone
+}
+
+func copyStream(dst io.Writer, src io.Reader, done chan struct{}) {
+	defer close(done)
+	if _, err := io.Copy(dst, src); err != nil {
+		Warning.Print(err)
+	}
+	if c, ok := dst.(interface {
+		CloseWrite() error
+	}); ok {
+		if err := c.CloseWrite(); err != nil {
+			Warning.Printf("Error closing connection: %s", err)
+		}
+	}
+}
+
+func doChunkedResponse(w http.ResponseWriter, resp *http.Response, client *httputil.ClientConn) {
+	// Because we can't go back to request/response after we
+	// hijack the connection, we need to close it and make the
+	// client open another.
+	w.Header().Add("Connection", "close")
+	w.WriteHeader(resp.StatusCode)
+
+	down, _, up, rem, err := hijack(w, client)
+	if err != nil {
+		http.Error(w, "Unable to hijack response stream for chunked response", http.StatusInternalServerError)
+		return
+	}
+	defer up.Close()
+	defer down.Close()
+	// Copy the chunked response body to downstream,
+	// stopping at the end of the chunked section.
+	rawResponseBody := io.MultiReader(rem, up)
+	if _, err := io.Copy(ioutil.Discard, httputil.NewChunkedReader(io.TeeReader(rawResponseBody, down))); err != nil {
+		http.Error(w, "Error copying chunked response body", http.StatusInternalServerError)
+		return
+	}
+	resp.Trailer.Write(down)
+	// a chunked response ends with a CRLF
+	down.Write([]byte("\r\n"))
+}
+
+func hijack(w http.ResponseWriter, client *httputil.ClientConn) (down net.Conn, downBuf *bufio.ReadWriter, up net.Conn, rem io.Reader, err error) {
+	hj, ok := w.(http.Hijacker)
+	if !ok {
+		err = errors.New("Unable to cast to Hijack")
+		return
+	}
+	down, downBuf, err = hj.Hijack()
+	if err != nil {
+		return
+	}
+	up, rem = client.Hijack()
+	return
+}

proxy/common.go

@@ -0,0 +1,64 @@
+package proxy
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os/exec"
+	"regexp"
+	"strings"
+
+	"github.com/fsouza/go-dockerclient"
+	. "github.com/weaveworks/weave/common"
+)
+
+var (
+	containerIDRegexp   = regexp.MustCompile("^(/v[0-9\\.]*)?/containers/([^/]*)/.*")
+	weaveWaitEntrypoint = []string{"/home/weavewait/weavewait"}
+)
+
+func callWeave(args ...string) ([]byte, error) {
+	args = append([]string{"--local"}, args...)
+	Debug.Print("Calling weave", args)
+	cmd := exec.Command("./weave", args...)
+	cmd.Env = []string{"PROCFS=/hostproc", "PATH=/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"}
+	out, err := cmd.CombinedOutput()
+	return out, err
+}
+
+func weaveCIDRsFromConfig(config *docker.Config) ([]string, bool) {
+	for _, e := range config.Env {
+		if strings.HasPrefix(e, "WEAVE_CIDR=") {
+			return strings.Fields(e[11:]), true
+		}
+	}
+	return nil, false
+}
+
+func marshalRequestBody(r *http.Request, body interface{}) error {
+	newBody, err := json.Marshal(body)
+	if err != nil {
+		return err
+	}
+	r.Body = ioutil.NopCloser(bytes.NewReader(newBody))
+	r.ContentLength = int64(len(newBody))
+	return nil
+}
+
+func inspectContainerInPath(client *docker.Client, path string) (*docker.Container, error) {
+	subs := containerIDRegexp.FindStringSubmatch(path)
+	if subs == nil {
+		err := fmt.Errorf("No container id found in request with path %s", path)
+		Warning.Println(err)
+		return nil, err
+	}
+	containerID := subs[2]
+
+	container, err := client.InspectContainer(containerID)
+	if err != nil {
+		Warning.Printf("Error inspecting container %s: %v", containerID, err)
+	}
+	return container, err
+}

proxy/create_container_interceptor.go

@@ -0,0 +1,108 @@
+package proxy
+
+import (
+	"bytes"
+	"encoding/json"
+	"io/ioutil"
+	"net/http"
+	"strings"
+
+	"github.com/fsouza/go-dockerclient"
+	. "github.com/weaveworks/weave/common"
+)
+
+type createContainerInterceptor struct {
+	client         *docker.Client
+	withDNS        bool
+	dockerBridgeIP string
+	withIPAM       bool
+}
+
+type createContainerRequestBody struct {
+	*docker.Config
+	HostConfig *docker.HostConfig `json:"HostConfig,omitempty" yaml:"HostConfig,omitempty"`
+	MacAddress string             `json:"MacAddress,omitempty" yaml:"MacAddress,omitempty"`
+}
+
+func (i *createContainerInterceptor) InterceptRequest(r *http.Request) error {
+	body, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		return err
+	}
+	r.Body.Close()
+
+	container := createContainerRequestBody{}
+	if err := json.Unmarshal(body, &container); err != nil {
+		return err
+	}
+
+	if cidrs, ok := weaveCIDRsFromConfig(container.Config); ok || i.withIPAM {
+		Info.Printf("Creating container with WEAVE_CIDR \"%s\"", strings.Join(cidrs, " "))
+		if container.HostConfig == nil {
+			container.HostConfig = &docker.HostConfig{}
+		}
+		container.HostConfig.VolumesFrom = append(container.HostConfig.VolumesFrom, "weaveproxy")
+		if container.Config == nil {
+			container.Config = &docker.Config{}
+		}
+		if err := i.setWeaveWaitEntrypoint(container.Config); err != nil {
+			return err
+		}
+		if err := i.setWeaveDNS(&container); err != nil {
+			return err
+		}
+	}
+
+	newBody, err := json.Marshal(container)
+	if err != nil {
+		return err
+	}
+	r.Body = ioutil.NopCloser(bytes.NewReader(newBody))
+	r.ContentLength = int64(len(newBody))
+
+	return nil
+}
+
+func (i *createContainerInterceptor) setWeaveWaitEntrypoint(container *docker.Config) error {
+	if len(container.Entrypoint) == 0 {
+		image, err := i.client.InspectImage(container.Image)
+		if err != nil {
+			return err
+		}
+
+		if len(container.Cmd) == 0 {
+			container.Cmd = image.Config.Cmd
+		}
+
+		if container.Entrypoint == nil {
+			container.Entrypoint = image.Config.Entrypoint
+		}
+	}
+
+	container.Entrypoint = append(weaveWaitEntrypoint, container.Entrypoint...)
+	return nil
+}
+
+func configHasEntrypoint(c *docker.Config) bool {
+	return c != nil && len(c.Entrypoint) > 0
+}
+func configHasCmd(c *docker.Config) bool {
+	return c != nil && len(c.Cmd) > 0
+}
+
+func (i *createContainerInterceptor) setWeaveDNS(container *createContainerRequestBody) error {
+	if !i.withDNS {
+		return nil
+	}
+
+	container.HostConfig.DNS = append(container.HostConfig.DNS, i.dockerBridgeIP)
+
+	if len(container.HostConfig.DNSSearch) == 0 {
+		container.HostConfig.DNSSearch = []string{"."}
+	}
+	return nil
+}
+
+func (i *createContainerInterceptor) InterceptResponse(r *http.Response) error {
+	return nil
+}

proxy/create_exec_interceptor.go

@@ -0,0 +1,55 @@
+package proxy
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"net/http"
+	"strings"
+
+	"github.com/fsouza/go-dockerclient"
+	. "github.com/weaveworks/weave/common"
+)
+
+type createExecInterceptor struct {
+	client   *docker.Client
+	withIPAM bool
+}
+
+type createExecRequestBody struct {
+	*docker.Config
+	HostConfig *docker.HostConfig `json:"HostConfig,omitempty" yaml:"HostConfig,omitempty"`
+	MacAddress string             `json:"MacAddress,omitempty" yaml:"MacAddress,omitempty"`
+}
+
+func (i *createExecInterceptor) InterceptRequest(r *http.Request) error {
+	body, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		return err
+	}
+	r.Body.Close()
+
+	options := docker.CreateExecOptions{}
+	if err := json.Unmarshal(body, &options); err != nil {
+		return err
+	}
+
+	container, err := inspectContainerInPath(i.client, r.URL.Path)
+	if err != nil {
+		return err
+	}
+
+	if cidrs, ok := weaveCIDRsFromConfig(container.Config); ok || i.withIPAM {
+		Info.Printf("Exec in container %s with WEAVE_CIDR \"%s\"", container.ID, strings.Join(cidrs, " "))
+		options.Cmd = append(weaveWaitEntrypoint, options.Cmd...)
+	}
+
+	if err := marshalRequestBody(r, options); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (i *createExecInterceptor) InterceptResponse(r *http.Response) error {
+	return nil
+}

proxy/interceptor.go

@@ -0,0 +1,19 @@
+package proxy
+
+import "net/http"
+
+type interceptor interface {
+	InterceptRequest(*http.Request) error
+	InterceptResponse(*http.Response) error
+}
+
+type nullInterceptor struct {
+}
+
+func (i nullInterceptor) InterceptRequest(r *http.Request) error {
+	return nil
+}
+
+func (i nullInterceptor) InterceptResponse(r *http.Response) error {
+	return nil
+}

proxy/proxy.go

@@ -0,0 +1,85 @@
+package proxy
+
+import (
+	"net"
+	"net/http"
+	"net/url"
+	"regexp"
+	"strings"
+
+	"github.com/fsouza/go-dockerclient"
+	. "github.com/weaveworks/weave/common"
+)
+
+var (
+	containerCreateRegexp = regexp.MustCompile("^(/v[0-9\\.]*)?/containers/create$")
+	containerStartRegexp  = regexp.MustCompile("^(/v[0-9\\.]*)?/containers/[^/]*/(re)?start$")
+	execCreateRegexp      = regexp.MustCompile("^(/v[0-9\\.]*)?/containers/[^/]*/exec$")
+)
+
+type Proxy struct {
+	Dial           func() (net.Conn, error)
+	client         *docker.Client
+	withDNS        bool
+	dockerBridgeIP string
+	withIPAM       bool
+}
+
+func NewProxy(targetURL string, withDNS, withIPAM bool) (*Proxy, error) {
+	u, err := url.Parse(targetURL)
+	if err != nil {
+		return nil, err
+	}
+
+	var dockerBridgeIP []byte
+	if withDNS {
+		dockerBridgeIP, err = callWeave("docker-bridge-ip")
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	client, err := docker.NewClient(targetURL)
+	if err != nil {
+		return nil, err
+	}
+
+	targetAddr := ""
+	switch u.Scheme {
+	case "tcp":
+		targetAddr = u.Host
+	case "unix":
+		targetAddr = u.Path
+	}
+
+	return &Proxy{
+		Dial: func() (net.Conn, error) {
+			return net.Dial(u.Scheme, targetAddr)
+		},
+		client:         client,
+		withDNS:        withDNS,
+		dockerBridgeIP: string(dockerBridgeIP),
+		withIPAM:       withIPAM,
+	}, nil
+}
+
+func (proxy *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	Info.Printf("%s %s", r.Method, r.URL)
+	path := r.URL.Path
+	switch {
+	case containerCreateRegexp.MatchString(path):
+		proxy.serveWithInterceptor(&createContainerInterceptor{proxy.client, proxy.withDNS, proxy.dockerBridgeIP, proxy.withIPAM}, w, r)
+	case containerStartRegexp.MatchString(path):
+		proxy.serveWithInterceptor(&startContainerInterceptor{proxy.client, proxy.withDNS, proxy.withIPAM}, w, r)
+	case execCreateRegexp.MatchString(path):
+		proxy.serveWithInterceptor(&createExecInterceptor{proxy.client, proxy.withIPAM}, w, r)
+	case strings.HasPrefix(path, "/weave"):
+		w.WriteHeader(http.StatusOK)
+	default:
+		proxy.serveWithInterceptor(&nullInterceptor{}, w, r)
+	}
+}
+
+func (proxy *Proxy) serveWithInterceptor(i interceptor, w http.ResponseWriter, r *http.Request) {
+	newClient(proxy.Dial, i).ServeHTTP(w, r)
+}

proxy/start_container_interceptor.go

@@ -0,0 +1,43 @@
+package proxy
+
+import (
+	"errors"
+	"net/http"
+	"strings"
+
+	"github.com/fsouza/go-dockerclient"
+	. "github.com/weaveworks/weave/common"
+)
+
+type startContainerInterceptor struct {
+	client   *docker.Client
+	withDNS  bool
+	withIPAM bool
+}
+
+func (i *startContainerInterceptor) InterceptRequest(r *http.Request) error {
+	return nil
+}
+
+func (i *startContainerInterceptor) InterceptResponse(r *http.Response) error {
+	container, err := inspectContainerInPath(i.client, r.Request.URL.Path)
+	if err != nil {
+		return err
+	}
+
+	cidrs, ok := weaveCIDRsFromConfig(container.Config)
+	if !ok && !i.withIPAM {
+		Debug.Print("No Weave CIDR, ignoring")
+		return nil
+	}
+	Info.Printf("Attaching container %s with WEAVE_CIDR \"%s\" to weave network", container.ID, strings.Join(cidrs, " "))
+	args := []string{"attach"}
+	args = append(args, cidrs...)
+	args = append(args, container.ID)
+	if output, err := callWeave(args...); err != nil {
+		i.client.KillContainer(docker.KillContainerOptions{ID: container.ID})
+		Warning.Printf("Attaching container %s to weave network failed: %s", container.ID, string(output))
+		return errors.New(string(output))
+	}
+	return nil
+}

router/connection.go

@@ -82,17 +82,17 @@ func (conn *RemoteConnection) BreakTie(Connection) ConnectionTieBreak { return T
 func (conn *RemoteConnection) Shutdown(error)                         {}
 
 func (conn *RemoteConnection) Log(args ...interface{}) {
-	log.Println(append(append([]interface{}{}, fmt.Sprintf("->[%s|%s]:", conn.remoteTCPAddr, conn.remote.FullName())), args...)...)
+	log.Println(append(append([]interface{}{}, fmt.Sprintf("->[%s|%s]:", conn.remoteTCPAddr, conn.remote)), args...)...)
 }
 
 func (conn *RemoteConnection) String() string {
 	from := "<nil>"
 	if conn.local != nil {
-		from = conn.local.FullName()
+		from = conn.local.String()
 	}
 	to := "<nil>"
 	if conn.remote != nil {
-		to = conn.remote.FullName()
+		to = conn.remote.String()
 	}
 	return fmt.Sprint("Connection ", from, "->", to)
 }
@@ -196,14 +196,13 @@ func (conn *LocalConnection) SendProtocolMsg(m ProtocolMsg) {
 
 // Async
 func (conn *LocalConnection) Shutdown(err error) {
+	// err should always be a real error, even if only io.EOF
+	if err == nil {
+		panic("nil error")
+	}
+
 	// Run on its own goroutine in case the channel is backed up
-	go func() {
-		if err == nil {
-			conn.sendAction(nil)
-		} else {
-			conn.sendAction(func() error { return err })
-		}
-	}()
+	go func() { conn.sendAction(func() error { return err }) }()
 }
 
 // Async
@@ -231,7 +230,7 @@ func (conn *LocalConnection) ReceivedHeartbeat(remoteUDPAddr *net.UDPAddr, connU
 		if oldRemoteUDPAddr == nil {
 			return conn.sendFastHeartbeats()
 		} else if oldRemoteUDPAddr.String() != remoteUDPAddr.String() {
-			log.Println("Peer", conn.remote.FullName(), "moved from", old, "to", remoteUDPAddr)
+			log.Println("Peer", conn.remote, "moved from", old, "to", remoteUDPAddr)
 		}
 		return nil
 	})
@@ -379,9 +378,6 @@ func (conn *LocalConnection) actorLoop(actionChan <-chan ConnectionAction) (err
 	for err == nil {
 		select {
 		case action := <-actionChan:
-			if action == nil {
-				break
-			}
 			err = action()
 		case <-conn.heartbeatTCP.C:
 			err = conn.sendSimpleProtocolMsg(ProtocolHeartbeat)
@@ -398,13 +394,10 @@ func (conn *LocalConnection) actorLoop(actionChan <-chan ConnectionAction) (err
 }
 
 func (conn *LocalConnection) shutdown(err error) {
-	switch {
-	case err != nil && conn.remote == nil:
+	if conn.remote == nil {
 		log.Printf("->[%s] connection shutting down due to error during handshake: %v\n", conn.remoteTCPAddr, err)
-	case err != nil:
+	} else {
 		conn.Log("connection shutting down due to error:", err)
-	default:
-		conn.Log("connection shutting down")
 	}
 
 	if conn.TCPConn != nil {
@@ -412,7 +405,7 @@ func (conn *LocalConnection) shutdown(err error) {
 	}
 
 	if conn.remote != nil {
-		conn.remote.DecrementLocalRefCount()
+		conn.Router.Peers.Dereference(conn.remote)
 		conn.Router.Ourself.DeleteConnection(conn)
 	}
 
@@ -442,11 +435,10 @@ func (conn *LocalConnection) sendProtocolMsg(m ProtocolMsg) error {
 }
 
 func (conn *LocalConnection) receiveTCP(decoder *gob.Decoder) {
-	defer conn.Decryptor.Shutdown()
 	usingPassword := conn.SessionKey != nil
 	var receiver TCPReceiver
 	if usingPassword {
-		receiver = NewEncryptedTCPReceiver(conn)
+		receiver = NewEncryptedTCPReceiver(conn.SessionKey, conn.outbound)
 	} else {
 		receiver = NewSimpleTCPReceiver()
 	}
@@ -489,11 +481,6 @@ func (conn *LocalConnection) handleProtocolMsg(tag ProtocolTag, payload []byte)
 			nil)
 	case ProtocolFragmentationReceived:
 		conn.setStackFrag(true)
-	case ProtocolNonce:
-		if conn.SessionKey == nil {
-			return fmt.Errorf("unexpected nonce on unencrypted connection")
-		}
-		conn.Decryptor.ReceiveNonce(payload)
 	case ProtocolPMTUVerified:
 		conn.pmtuVerified(int(binary.BigEndian.Uint16(payload)))
 	case ProtocolGossipUnicast, ProtocolGossipBroadcast, ProtocolGossip:

router/connection_maker.go

@@ -206,8 +206,13 @@ func (cm *ConnectionMaker) ourConnections() (PeerNameSet, map[string]struct{}, m
 
 func (cm *ConnectionMaker) addPeerTargets(ourConnectedPeers PeerNameSet, addTarget func(string)) {
 	cm.peers.ForEach(func(peer *Peer) {
-		for conn := range peer.Connections() {
-			otherPeer := conn.Remote().Name
+		if peer == cm.ourself.Peer {
+			return
+		}
+		// Modifying peer.connections requires a write lock on Peers,
+		// and since we are holding a read lock (due to the ForEach),
+		// access without locking the peer is safe.
+		for otherPeer, conn := range peer.connections {
 			if otherPeer == cm.ourself.Name {
 				continue
 			}
@@ -215,12 +220,13 @@ func (cm *ConnectionMaker) addPeerTargets(ourConnectedPeers PeerNameSet, addTarg
 				continue
 			}
 			address := conn.RemoteTCPAddr()
-			// Try both port of connection and standard port.  Don't
-			// use remote side of inbound connection.
 			if conn.Outbound() {
 				addTarget(address)
-			}
-			if ip, _, err := net.SplitHostPort(address); err == nil {
+			} else if ip, _, err := net.SplitHostPort(address); err == nil {
+				// There is no point connecting to the (likely
+				// ephemeral) remote port of an inbound connection
+				// that some peer has. Let's try to connect to on the
+				// weave port instead.
 				addTarget(fmt.Sprintf("%s:%d", ip, cm.port))
 			}
 		}

router/consts.go

@@ -13,7 +13,6 @@ const (
 	DefaultPMTU         = 65535
 	MaxUDPPacketSize    = 65536
 	ChannelSize         = 16
-	UDPNonceSendAt      = 8192
 	FragTestSize        = 60001
 	PMTUDiscoverySize   = 60000
 	TCPHeartbeat        = 30 * time.Second
@@ -23,7 +22,7 @@ const (
 	PMTUVerifyAttempts  = 8
 	PMTUVerifyTimeout   = 10 * time.Millisecond // gets doubled with every attempt
 	MaxDuration         = time.Duration(math.MaxInt64)
-	MaxMissedHeartbeats = 3
+	MaxMissedHeartbeats = 6
 	HeartbeatTimeout    = MaxMissedHeartbeats * SlowHeartbeat
 )
 

router/crypto.go

@@ -1,7 +1,6 @@
 package router
 
 import (
-	"bytes"
 	"code.google.com/p/go-bit/bit"
 	"code.google.com/p/go.crypto/nacl/box"
 	"code.google.com/p/go.crypto/nacl/secretbox"
@@ -10,7 +9,6 @@ import (
 	"encoding/binary"
 	"encoding/gob"
 	"fmt"
-	"log"
 	"sync"
 )
 
@@ -27,85 +25,13 @@ func FormSessionKey(remotePublicKey, localPrivateKey *[32]byte, secretKey []byte
 	return &sessionKey
 }
 
-func GenerateRandomNonce() ([24]byte, error) {
-	var nonce [24]byte
-	n, err := rand.Read(nonce[:])
-	if err != nil {
-		return nonce, err
-	}
-	if n != 24 {
-		return nonce, fmt.Errorf("Did not read enough - wanted 24, got %v", n)
-	}
-	return nonce, nil
-}
-
-func EncryptPrefixNonce(plaintxt []byte, nonce *[24]byte, secret *[32]byte) []byte {
-	buf := make([]byte, 24, 24+len(plaintxt)+secretbox.Overhead)
-	copy(buf, nonce[:])
-	// Seal *appends* to buf
-	return secretbox.Seal(buf, plaintxt, nonce, secret)
-}
-
-func DecryptPrefixNonce(ciphertxt []byte, secret *[32]byte) ([]byte, bool) {
-	if len(ciphertxt) < secretbox.Overhead+24 {
-		return nil, false
-	}
-	// There is no way to nicely convert from a slice to an
-	// array. So have to used the following loop.
-	var nonce [24]byte
-	for idx, e := range ciphertxt[0:24] {
-		nonce[idx] = e
-	}
-	ciphertxt = ciphertxt[24:]
-	return secretbox.Open(nil, ciphertxt, &nonce, secret)
-}
-
-func SetNonceLow15Bits(nonce *[24]byte, offset uint16) {
-	// ensure top bit of offset is 0
-	offset = offset & ((1 << 15) - 1)
-	// grab top bit of nonce[22:24] (and clear out lower bits)
-	nonceBits := binary.BigEndian.Uint16(nonce[22:24]) & (1 << 15)
-	// Big endian => the MSB is stored at the *lowest* address. So
-	// that top bit in nonce[22] should stay as the top bit in
-	// nonce[22]
-	binary.BigEndian.PutUint16(nonce[22:24], nonceBits+offset)
-}
-
-// Nonce encoding/decoding
-
-func EncodeNonce(df bool) (*[24]byte, []byte, error) {
-	nonce, err := GenerateRandomNonce()
-	if err != nil {
-		return nil, []byte{}, err
-	}
-	// wipe out lowest 15 bits, but encode the df right at the bottom
-	flags := uint16(0)
-	if df {
-		flags = flags | 1
-	}
-	SetNonceLow15Bits(&nonce, flags)
-	// NB: need to make a copy since callers may modify the array
-	return &nonce, Concat(nonce[:]), nil
-}
-
-func DecodeNonce(msg []byte) (bool, *[24]byte) {
-	flags := uint16(msg[23])
-	df := 0 != (flags & 1)
-	nonce := [24]byte{}
-	// upper bound is exclusive so this avoids copying the flags
-	for idx, elem := range msg[:23] {
-		nonce[idx] = elem
-	}
-	return df, &nonce
-}
-
 // Frame Encryptors
 
 type Encryptor interface {
 	FrameOverhead() int
 	PacketOverhead() int
 	IsEmpty() bool
-	Bytes() []byte
+	Bytes() ([]byte, error)
 	AppendFrame(src []byte, dst []byte, frame []byte)
 	TotalLen() int
 }
@@ -119,14 +45,12 @@ type NonEncryptor struct {
 
 type NaClEncryptor struct {
 	NonEncryptor
-	buf       []byte
-	offset    uint16
-	nonce     *[24]byte
-	nonceChan chan *[24]byte
-	flags     uint16
-	prefixLen int
-	conn      *LocalConnection
-	df        bool
+	buf        []byte
+	prefixLen  int
+	sessionKey *[32]byte
+	nonce      [24]byte
+	seqNo      uint64
+	df         bool
 }
 
 func NewNonEncryptor(prefix []byte) *NonEncryptor {
@@ -151,11 +75,11 @@ func (ne *NonEncryptor) IsEmpty() bool {
 	return ne.buffered == ne.prefixLen
 }
 
-func (ne *NonEncryptor) Bytes() []byte {
+func (ne *NonEncryptor) Bytes() ([]byte, error) {
 	buf := ne.buf[:ne.buffered]
 	ne.buffered = ne.prefixLen
 	ne.bufTail = ne.buf[ne.prefixLen:]
-	return buf
+	return buf, nil
 }
 
 func (ne *NonEncryptor) AppendFrame(src []byte, dst []byte, frame []byte) {
@@ -175,66 +99,47 @@ func (ne *NonEncryptor) TotalLen() int {
 	return ne.buffered
 }
 
-func NewNaClEncryptor(prefix []byte, conn *LocalConnection, df bool) *NaClEncryptor {
+func NewNaClEncryptor(prefix []byte, sessionKey *[32]byte, outbound bool, df bool) *NaClEncryptor {
 	buf := make([]byte, MaxUDPPacketSize)
 	prefixLen := copy(buf, prefix)
-	flags := uint16(0)
-	if df {
-		flags = flags | (1 << 15)
-	}
-	return &NaClEncryptor{
+	ne := &NaClEncryptor{
 		NonEncryptor: *NewNonEncryptor([]byte{}),
 		buf:          buf,
-		offset:       0,
-		nonce:        nil,
-		nonceChan:    make(chan *[24]byte, ChannelSize),
-		flags:        flags,
 		prefixLen:    prefixLen,
-		conn:         conn,
+		sessionKey:   sessionKey,
 		df:           df}
+	if outbound {
+		ne.nonce[0] |= (1 << 7)
+	}
+	return ne
 }
 
-func (ne *NaClEncryptor) Bytes() []byte {
-	plaintext := ne.NonEncryptor.Bytes()
-	offsetFlags := ne.offset | ne.flags
+func (ne *NaClEncryptor) Bytes() ([]byte, error) {
+	plaintext, err := ne.NonEncryptor.Bytes()
+	if err != nil {
+		return nil, err
+	}
+	// We carry the DF flag in the (unencrypted portion of the)
+	// payload, rather than just extracting it from the packet headers
+	// at the receiving end, since we do not trust routers not to mess
+	// with headers. As we have different decryptors for non-DF and
+	// DF, that would result in hard to track down packet drops due to
+	// crypto errors.
+	seqNoAndDF := ne.seqNo
+	if ne.df {
+		seqNoAndDF |= (1 << 63)
+	}
 	ciphertext := ne.buf
-	binary.BigEndian.PutUint16(ciphertext[ne.prefixLen:], offsetFlags)
-	nonce := ne.nonce
-	if nonce == nil {
-		freshNonce, encodedNonce, err := EncodeNonce(ne.df)
-		if err != nil {
-			ne.conn.Shutdown(err)
-			return []byte{}
-		}
-		ne.conn.SendProtocolMsg(ProtocolMsg{ProtocolNonce, encodedNonce})
-		ne.nonce = freshNonce
-		nonce = freshNonce
-	}
-	offset := ne.offset
-	SetNonceLow15Bits(nonce, offset)
+	binary.BigEndian.PutUint64(ciphertext[ne.prefixLen:], seqNoAndDF)
+	binary.BigEndian.PutUint64(ne.nonce[16:24], seqNoAndDF)
 	// Seal *appends* to ciphertext
-	ciphertext = secretbox.Seal(ciphertext[:ne.prefixLen+2], plaintext, nonce, ne.conn.SessionKey)
-
-	offset = (offset + 1) & ((1 << 15) - 1)
-	if offset == 0 {
-		// need a new nonce please
-		ne.nonce = <-ne.nonceChan
-	} else if offset == 1<<14 { // half way through range, send new nonce
-		nonce, encodedNonce, err := EncodeNonce(ne.df)
-		if err != nil {
-			ne.conn.Shutdown(err)
-			return []byte{}
-		}
-		ne.nonceChan <- nonce
-		ne.conn.SendProtocolMsg(ProtocolMsg{ProtocolNonce, encodedNonce})
-	}
-	ne.offset = offset
-
-	return ciphertext
+	ciphertext = secretbox.Seal(ciphertext[:ne.prefixLen+8], plaintext, &ne.nonce, ne.sessionKey)
+	ne.seqNo++
+	return ciphertext, nil
 }
 
 func (ne *NaClEncryptor) PacketOverhead() int {
-	return ne.prefixLen + 2 + secretbox.Overhead + ne.NonEncryptor.PacketOverhead()
+	return ne.prefixLen + 8 + secretbox.Overhead + ne.NonEncryptor.PacketOverhead()
 }
 
 func (ne *NaClEncryptor) TotalLen() int {
@@ -243,39 +148,43 @@ func (ne *NaClEncryptor) TotalLen() int {
 
 // Frame Decryptors
 
-type FrameConsumer func(conn *LocalConnection, src []byte, dst []byte, frame []byte)
+type FrameConsumer func(src []byte, dst []byte, frame []byte)
 
 type Decryptor interface {
 	IterateFrames([]byte, FrameConsumer) error
-	ReceiveNonce([]byte)
-	Shutdown()
 }
 
 type NonDecryptor struct {
-	conn *LocalConnection
 }
 
 type NaClDecryptor struct {
 	NonDecryptor
+	sessionKey *[32]byte
 	instance   *NaClDecryptorInstance
 	instanceDF *NaClDecryptorInstance
 }
 
 type NaClDecryptorInstance struct {
-	nonce               *[24]byte
-	previousNonce       *[24]byte
+	nonce               [24]byte
+	currentWindow       uint64
 	usedOffsets         *bit.Set
 	previousUsedOffsets *bit.Set
-	highestOffsetSeen   uint16
-	nonceChan           chan *[24]byte
+}
+
+func NewNaClDecryptorInstance(outbound bool) *NaClDecryptorInstance {
+	di := &NaClDecryptorInstance{usedOffsets: bit.New()}
+	if !outbound {
+		di.nonce[0] |= (1 << 7)
+	}
+	return di
 }
 
 type PacketDecodingError struct {
 	Desc string
 }
 
-func NewNonDecryptor(conn *LocalConnection) *NonDecryptor {
-	return &NonDecryptor{conn: conn}
+func NewNonDecryptor() *NonDecryptor {
+	return &NonDecryptor{}
 }
 
 func (nd *NonDecryptor) IterateFrames(packet []byte, consumer FrameConsumer) error {
@@ -291,7 +200,7 @@ func (nd *NonDecryptor) IterateFrames(packet []byte, consumer FrameConsumer) err
 		}
 		frame := packet[:length]
 		packet = packet[length:]
-		consumer(nd.conn, srcNameByte, dstNameByte, frame)
+		consumer(srcNameByte, dstNameByte, frame)
 	}
 	if len(packet) > 0 {
 		return PacketDecodingError{Desc: fmt.Sprintf("%d octets of trailing garbage", len(packet))}
@@ -299,127 +208,125 @@ func (nd *NonDecryptor) IterateFrames(packet []byte, consumer FrameConsumer) err
 	return nil
 }
 
-func (nd *NonDecryptor) Shutdown() {
-}
-
-func (nd *NonDecryptor) ReceiveNonce(msg []byte) {
-	log.Println("Received Nonce on non-encrypted channel. Ignoring.")
-}
-
-func NewNaClDecryptor(conn *LocalConnection) *NaClDecryptor {
+func NewNaClDecryptor(sessionKey *[32]byte, outbound bool) *NaClDecryptor {
 	return &NaClDecryptor{
-		NonDecryptor: *NewNonDecryptor(conn),
-		instance: &NaClDecryptorInstance{
-			usedOffsets: bit.New(),
-			nonceChan:   make(chan *[24]byte, ChannelSize)},
-		instanceDF: &NaClDecryptorInstance{
-			usedOffsets: bit.New(),
-			nonceChan:   make(chan *[24]byte, ChannelSize)}}
-}
-
-func (nd *NaClDecryptor) Shutdown() {
-	close(nd.instance.nonceChan)
-	close(nd.instanceDF.nonceChan)
-}
-
-func (nd *NaClDecryptor) ReceiveNonce(msg []byte) {
-	df, nonce := DecodeNonce(msg)
-	if df {
-		nd.instanceDF.nonceChan <- nonce
-	} else {
-		nd.instance.nonceChan <- nonce
-	}
+		NonDecryptor: *NewNonDecryptor(),
+		sessionKey:   sessionKey,
+		instance:     NewNaClDecryptorInstance(outbound),
+		instanceDF:   NewNaClDecryptorInstance(outbound)}
 }
 
 func (nd *NaClDecryptor) IterateFrames(packet []byte, consumer FrameConsumer) error {
-	buf, err := nd.decrypt(packet)
-	if err != nil {
-		return PacketDecodingError{Desc: fmt.Sprint("decryption failed; ", err)}
+	if len(packet) < 8 {
+		return PacketDecodingError{Desc: fmt.Sprintf("encrypted UDP packet too short; expected length >= 8, got %d", len(packet))}
+	}
+	buf, success := nd.decrypt(packet)
+	if !success {
+		return PacketDecodingError{Desc: fmt.Sprint("UDP packet decryption failed")}
 	}
 	return nd.NonDecryptor.IterateFrames(buf, consumer)
 }
 
-func (nd *NaClDecryptor) decrypt(buf []byte) ([]byte, error) {
-	offset := binary.BigEndian.Uint16(buf[:2])
-	df := (offset & (1 << 15)) != 0
-	offsetNoFlags := offset & ((1 << 15) - 1)
+func (nd *NaClDecryptor) decrypt(buf []byte) ([]byte, bool) {
+	seqNoAndDF := binary.BigEndian.Uint64(buf[:8])
+	df := (seqNoAndDF & (1 << 63)) != 0
+	seqNo := seqNoAndDF & ((1 << 63) - 1)
 	var di *NaClDecryptorInstance
 	if df {
 		di = nd.instanceDF
 	} else {
 		di = nd.instance
 	}
-	nonce, usedOffsets, err := di.advanceState(offsetNoFlags)
-	if err != nil {
-		return nil, err
-	}
-	offsetNoFlagsInt := int(offsetNoFlags)
-	if usedOffsets.Contains(offsetNoFlagsInt) {
-		return nil, fmt.Errorf("Suspected replay attack detected when decrypting UDP packet")
-	}
-	SetNonceLow15Bits(nonce, offsetNoFlags)
-	result, success := secretbox.Open(nil, buf[2:], nonce, nd.conn.SessionKey)
+	binary.BigEndian.PutUint64(di.nonce[16:24], seqNoAndDF)
+	result, success := secretbox.Open(nil, buf[8:], &di.nonce, nd.sessionKey)
 	if !success {
-		return nil, fmt.Errorf("Unable to decrypt UDP packet")
+		return nil, false
 	}
-	usedOffsets.Add(offsetNoFlagsInt)
-	return result, nil
+	// Drop duplicates. We do this *after* decryption since we must
+	// not advance our state unless decryption succeeded. Doing so
+	// would open an easy attack vector where an adversary could
+	// inject a packet with a sequence number of (1 << 63) - 1,
+	// causing all subsequent genuine packets to get dropped.
+	offset, usedOffsets := di.advanceState(seqNo)
+	if usedOffsets == nil || usedOffsets.Contains(offset) {
+		// We have detected a possible replay attack, but it is
+		// possible we may have just received a very old packet, or
+		// duplication may have occurred in the network. So let's just
+		// drop the packet silently.
+		return nil, true
+	}
+	usedOffsets.Add(offset)
+	return result, success
 }
 
-func (di *NaClDecryptorInstance) advanceState(offsetNoFlags uint16) (*[24]byte, *bit.Set, error) {
-	var ok bool
-	if di.nonce == nil {
-		if offsetNoFlags > (1 << 13) {
-			// offset is already beyond the first quarter and it's the
-			// first thing we've seen?! I don't think so.
-			return nil, nil, fmt.Errorf("Unexpected offset when decrypting UDP packet")
-		}
-		di.nonce, ok = <-di.nonceChan
-		if !ok {
-			return nil, nil, fmt.Errorf("Nonce chan closed")
-		}
-		di.highestOffsetSeen = offsetNoFlags
-	} else {
-		highestOffsetSeen := di.highestOffsetSeen
-		switch {
-		case offsetNoFlags < (1<<13) && highestOffsetSeen > ((1<<14)+(1<<13)) &&
-			(highestOffsetSeen-offsetNoFlags) > ((1<<14)+(1<<13)):
-			// offset is in the first quarter, highestOffsetSeen is in
-			// the top quarter and under a quarter behind us. We
-			// interpret this as we need to move to the next nonce
-			di.previousUsedOffsets = di.usedOffsets
-			di.usedOffsets = bit.New()
-			di.previousNonce = di.nonce
-			di.nonce, ok = <-di.nonceChan
-			if !ok {
-				return nil, nil, fmt.Errorf("Nonce chan closed")
-			}
-			di.highestOffsetSeen = offsetNoFlags
-		case offsetNoFlags > highestOffsetSeen &&
-			(offsetNoFlags-highestOffsetSeen) < (1<<13):
-			// offset is under a quarter above highestOffsetSeen. This
-			// is ok - maybe some packet loss
-			di.highestOffsetSeen = offsetNoFlags
-		case offsetNoFlags <= highestOffsetSeen &&
-			(highestOffsetSeen-offsetNoFlags) < (1<<13):
-			// offset is within a quarter of the highest we've
-			// seen. This is ok - just assuming some out-of-order
-			// delivery.
-		case highestOffsetSeen < (1<<13) && offsetNoFlags > ((1<<14)+(1<<13)) &&
-			(offsetNoFlags-highestOffsetSeen) > ((1<<14)+(1<<13)):
-			// offset is in the last quarter, highestOffsetSeen is in
-			// the first quarter, and offset is under a quarter behind
-			// us. This is ok - as above, just some out of order. But
-			// here it means we're dealing with the previous nonce
-			return di.previousNonce, di.previousUsedOffsets, nil
-		default:
-			return nil, nil, fmt.Errorf("Unexpected offset when decrypting UDP packet")
-		}
+// We record seen message sequence numbers in a sliding window of
+// 2*WindowSize which slides in WindowSize increments. This allows us
+// to process out-of-order delivery within the window, while
+// accurately discarding duplicates. By contrast any messages with
+// sequence numbers below the window are discarded as potential
+// duplicates.
+//
+// There are two sets, corresponding to the lower and upper half of
+// the window. We slide the window s.t. that 2nd set always contains
+// the highest seen sequence number. We do this regardless of how far
+// ahead of the current window that sequence number might be, so we
+// can cope with large gaps resulting from packet loss.
+
+const (
+	WindowSize = 20 // bits
+)
+
+func (di *NaClDecryptorInstance) advanceState(seqNo uint64) (int, *bit.Set) {
+	var (
+		offset = int(seqNo & ((1 << WindowSize) - 1))
+		window = seqNo >> WindowSize
+	)
+	switch delta := int64(window - di.currentWindow); {
+	case delta < -1:
+		return offset, nil
+	case delta == -1:
+		return offset, di.previousUsedOffsets
+	default:
+		return offset, di.usedOffsets
+	case delta == +1:
+		di.currentWindow = window
+		di.previousUsedOffsets = di.usedOffsets
+		di.usedOffsets = bit.New()
+		return offset, di.usedOffsets
+	case delta > +1:
+		di.currentWindow = window
+		di.previousUsedOffsets = bit.New()
+		di.usedOffsets = bit.New()
+		return offset, di.usedOffsets
 	}
-	return di.nonce, di.usedOffsets, nil
 }
 
-// TCP Senders
+// TCP Senders/Receivers
+
+// The lowest 64 bits of the nonce contain the message sequence
+// number. The top most bit indicates the connection polarity at the
+// sender - '1' for outbound. The remaining 127 bits are zero. The
+// polarity is needed so that the two ends of a connection do not use
+// the same nonces. This is a requirement of the NaCl Security Model;
+// see http://nacl.cr.yp.to/box.html.
+type TCPCryptoState struct {
+	sessionKey *[32]byte
+	nonce      [24]byte
+	seqNo      uint64
+}
+
+func NewTCPCryptoState(sessionKey *[32]byte, outbound bool) *TCPCryptoState {
+	s := &TCPCryptoState{sessionKey: sessionKey}
+	if outbound {
+		s.nonce[0] |= (1 << 7)
+	}
+	return s
+}
+
+func (s *TCPCryptoState) advance() {
+	s.seqNo++
+	binary.BigEndian.PutUint64(s.nonce[16:24], s.seqNo)
+}
 
 type TCPSender interface {
 	Send([]byte) error
@@ -430,17 +337,9 @@ type SimpleTCPSender struct {
 }
 
 type EncryptedTCPSender struct {
+	SimpleTCPSender
 	sync.RWMutex
-	outerEncoder *gob.Encoder
-	innerEncoder *gob.Encoder
-	buffer       *bytes.Buffer
-	conn         *LocalConnection
-	msgCount     int
-}
-
-type EncryptedTCPMessage struct {
-	Number int
-	Body   []byte
+	state *TCPCryptoState
 }
 
 func NewSimpleTCPSender(encoder *gob.Encoder) *SimpleTCPSender {
@@ -451,37 +350,20 @@ func (sender *SimpleTCPSender) Send(msg []byte) error {
 	return sender.encoder.Encode(msg)
 }
 
-func NewEncryptedTCPSender(encoder *gob.Encoder, conn *LocalConnection) *EncryptedTCPSender {
-	buffer := new(bytes.Buffer)
+func NewEncryptedTCPSender(encoder *gob.Encoder, sessionKey *[32]byte, outbound bool) *EncryptedTCPSender {
 	return &EncryptedTCPSender{
-		outerEncoder: encoder,
-		innerEncoder: gob.NewEncoder(buffer),
-		buffer:       buffer,
-		conn:         conn,
-		msgCount:     0}
+		SimpleTCPSender: *NewSimpleTCPSender(encoder),
+		state:           NewTCPCryptoState(sessionKey, outbound)}
 }
 
 func (sender *EncryptedTCPSender) Send(msg []byte) error {
-	nonce, err := GenerateRandomNonce()
-	if err != nil {
-		return err
-	}
 	sender.Lock()
 	defer sender.Unlock()
-	wrappedMsg := EncryptedTCPMessage{Number: sender.msgCount, Body: msg}
-	buffer := sender.buffer
-	buffer.Reset()
-	err = sender.innerEncoder.Encode(wrappedMsg)
-	if err != nil {
-		return err
-	}
-	sender.msgCount = sender.msgCount + 1
-	return sender.outerEncoder.Encode(
-		EncryptPrefixNonce(buffer.Bytes(), &nonce, sender.conn.SessionKey))
+	encodedMsg := secretbox.Seal(nil, msg, &sender.state.nonce, sender.state.sessionKey)
+	sender.state.advance()
+	return sender.SimpleTCPSender.Send(encodedMsg)
 }
 
-// TCP Receivers
-
 type TCPReceiver interface {
 	Decode([]byte) ([]byte, error)
 }
@@ -490,10 +372,8 @@ type SimpleTCPReceiver struct {
 }
 
 type EncryptedTCPReceiver struct {
-	conn     *LocalConnection
-	decoder  *gob.Decoder
-	buffer   *bytes.Buffer
-	msgCount int
+	SimpleTCPReceiver
+	state *TCPCryptoState
 }
 
 func NewSimpleTCPReceiver() *SimpleTCPReceiver {
@@ -504,33 +384,17 @@ func (receiver *SimpleTCPReceiver) Decode(msg []byte) ([]byte, error) {
 	return msg, nil
 }
 
-func NewEncryptedTCPReceiver(conn *LocalConnection) *EncryptedTCPReceiver {
-	buffer := new(bytes.Buffer)
+func NewEncryptedTCPReceiver(sessionKey *[32]byte, outbound bool) *EncryptedTCPReceiver {
 	return &EncryptedTCPReceiver{
-		conn:     conn,
-		decoder:  gob.NewDecoder(buffer),
-		buffer:   buffer,
-		msgCount: 0}
+		SimpleTCPReceiver: *NewSimpleTCPReceiver(),
+		state:             NewTCPCryptoState(sessionKey, !outbound)}
 }
 
 func (receiver *EncryptedTCPReceiver) Decode(msg []byte) ([]byte, error) {
-	plaintext, success := DecryptPrefixNonce(msg, receiver.conn.SessionKey)
+	decodedMsg, success := secretbox.Open(nil, msg, &receiver.state.nonce, receiver.state.sessionKey)
 	if !success {
-		return msg, fmt.Errorf("Unable to decrypt TCP msg")
+		return nil, fmt.Errorf("Unable to decrypt TCP msg")
 	}
-	receiver.buffer.Reset()
-	_, err := receiver.buffer.Write(plaintext)
-	if err != nil {
-		return msg, err
-	}
-	wrappedMsg := new(EncryptedTCPMessage)
-	err = receiver.decoder.Decode(wrappedMsg)
-	if err != nil {
-		return msg, err
-	}
-	if wrappedMsg.Number != receiver.msgCount {
-		return msg, fmt.Errorf("Received TCP message with wrong sequence number; possible replay attack")
-	}
-	receiver.msgCount = receiver.msgCount + 1
-	return wrappedMsg.Body, nil
+	receiver.state.advance()
+	return decodedMsg, nil
 }

router/forwarder.go

@@ -30,8 +30,8 @@ func (conn *LocalConnection) ensureForwarders() error {
 	usingPassword := conn.SessionKey != nil
 	var encryptor, encryptorDF Encryptor
 	if usingPassword {
-		encryptor = NewNaClEncryptor(conn.local.NameByte, conn, false)
-		encryptorDF = NewNaClEncryptor(conn.local.NameByte, conn, true)
+		encryptor = NewNaClEncryptor(conn.local.NameByte, conn.SessionKey, conn.outbound, false)
+		encryptorDF = NewNaClEncryptor(conn.local.NameByte, conn.SessionKey, conn.outbound, true)
 	} else {
 		encryptor = NewNonEncryptor(conn.local.NameByte)
 		encryptorDF = NewNonEncryptor(conn.local.NameByte)
@@ -287,7 +287,11 @@ func (fwd *Forwarder) appendFrame(frame *ForwardedFrame) bool {
 }
 
 func (fwd *Forwarder) flush() {
-	err := fwd.processSendError(fwd.udpSender.Send(fwd.enc.Bytes()))
+	msg, err := fwd.enc.Bytes()
+	if err != nil {
+		fwd.conn.Shutdown(err)
+	}
+	err = fwd.processSendError(fwd.udpSender.Send(msg))
 	if err != nil && PosixError(err) != syscall.ENOBUFS {
 		fwd.conn.Shutdown(err)
 	}

router/gossip.go

@@ -107,13 +107,17 @@ func (router *Router) NewGossip(channelName string, g Gossiper) Gossip {
 
 func (router *Router) SendAllGossip() {
 	for _, channel := range router.GossipChannels {
-		channel.Send(router.Ourself.Name, channel.gossiper.Gossip())
+		if gossip := channel.gossiper.Gossip(); gossip != nil {
+			channel.Send(router.Ourself.Name, gossip)
+		}
 	}
 }
 
 func (router *Router) SendAllGossipDown(conn Connection) {
 	for _, channel := range router.GossipChannels {
-		channel.SendDown(conn, channel.gossiper.Gossip())
+		if gossip := channel.gossiper.Gossip(); gossip != nil {
+			channel.SendDown(conn, channel.gossiper.Gossip())
+		}
 	}
 }
 

router/gossip_test.go

@@ -55,8 +55,8 @@ func (router *Router) DeleteTestChannelConnection(r *Router) {
 	fromPeer, _ := r.Peers.Fetch(fromName)
 	toPeer, _ := router.Peers.Fetch(toName)
 
-	fromPeer.DecrementLocalRefCount()
-	toPeer.DecrementLocalRefCount()
+	r.Peers.Dereference(fromPeer)
+	router.Peers.Dereference(toPeer)
 
 	conn, _ := router.Ourself.ConnectionTo(toName)
 	router.Ourself.handleDeleteConnection(conn)
@@ -79,7 +79,8 @@ func (router *Router) tp(routers ...*Router) *Peer {
 		p := NewPeer(r.Ourself.Peer.Name, "", r.Ourself.Peer.UID, r.Ourself.Peer.version)
 		connections[r.Ourself.Peer.Name] = newMockConnection(peer, p)
 	}
-	peer.SetVersionAndConnections(router.Ourself.Peer.version, connections)
+	peer.version = router.Ourself.Peer.version
+	peer.connections = connections
 	return peer
 }
 

router/handshake.go

@@ -75,10 +75,10 @@ func (conn *LocalConnection) handshake(enc *gob.Encoder, dec *gob.Decoder, accep
 			return fmt.Errorf("Found unknown remote name: %s at %s", name, conn.remoteTCPAddr)
 		}
 	}
-	if existingConn, found := conn.local.ConnectionTo(name); found && existingConn.Established() {
-		return fmt.Errorf("Already have connection to %s at %s", existingConn.Remote().FullName(), existingConn.RemoteTCPAddr())
+	if existingConn, found := conn.Router.Ourself.ConnectionTo(name); found && existingConn.Established() {
+		return fmt.Errorf("Already have connection to %s at %s", existingConn.Remote(), existingConn.RemoteTCPAddr())
 	}
-	uid, err := strconv.ParseUint(uidStr, 10, 64)
+	uid, err := ParsePeerUID(uidStr)
 	if err != nil {
 		return err
 	}
@@ -102,14 +102,14 @@ func (conn *LocalConnection) handshake(enc *gob.Encoder, dec *gob.Decoder, accep
 			remotePublic[idx] = elem
 		}
 		conn.SessionKey = FormSessionKey(&remotePublic, private, conn.Router.Password)
-		conn.tcpSender = NewEncryptedTCPSender(enc, conn)
-		conn.Decryptor = NewNaClDecryptor(conn)
+		conn.tcpSender = NewEncryptedTCPSender(enc, conn.SessionKey, conn.outbound)
+		conn.Decryptor = NewNaClDecryptor(conn.SessionKey, conn.outbound)
 	} else {
 		if rpErr == nil {
 			return fmt.Errorf("Remote network is encrypted. Password required.")
 		}
 		conn.tcpSender = NewSimpleTCPSender(enc)
-		conn.Decryptor = NewNonDecryptor(conn)
+		conn.Decryptor = NewNonDecryptor()
 	}
 
 	return conn.setRemote(NewPeer(name, nickNameStr, uid, 0))

router/json.go

@@ -6,7 +6,7 @@ import (
 	"time"
 )
 
-func (router *Router) GenerateStatusJSON(version, encryption string) ([]byte, error) {
+func (router *Router) StatusJSON(version, encryption string) ([]byte, error) {
 	return json.Marshal(struct {
 		Version    string
 		Encryption string
@@ -35,8 +35,30 @@ func (cache *MacCache) MarshalJSON() ([]byte, error) {
 }
 
 func (peers *Peers) MarshalJSON() ([]byte, error) {
-	var ps []*Peer
-	peers.ForEach(func(peer *Peer) { ps = append(ps, peer) })
+	type p struct {
+		Name        string
+		NickName    string
+		UID         PeerUID
+		Version     uint64
+		Connections []Connection
+	}
+	var ps []*p
+	peers.ForEach(func(peer *Peer) {
+		var connections []Connection
+		if peer == peers.ourself.Peer {
+			for conn := range peers.ourself.Connections() {
+				connections = append(connections, conn)
+			}
+		} else {
+			// Modifying peer.connections requires a write lock on
+			// Peers, and since we are holding a read lock (due to the
+			// ForEach), access without locking the peer is safe.
+			for _, conn := range peer.connections {
+				connections = append(connections, conn)
+			}
+		}
+		ps = append(ps, &p{peer.Name.String(), peer.NickName, peer.UID, peer.version, connections})
+	})
 	return json.Marshal(ps)
 }
 
@@ -63,21 +85,6 @@ func (routes *Routes) MarshalJSON() ([]byte, error) {
 	return json.Marshal(r)
 }
 
-func (peer *Peer) MarshalJSON() ([]byte, error) {
-	conns := peer.Connections()
-	connections := make([]Connection, 0, len(conns))
-	for conn := range conns {
-		connections = append(connections, conn)
-	}
-	return json.Marshal(struct {
-		Name        string
-		NickName    string
-		UID         uint64
-		Version     uint64
-		Connections []Connection
-	}{peer.Name.String(), peer.NickName, peer.UID, peer.version, connections})
-}
-
 func (conn *RemoteConnection) MarshalJSON() ([]byte, error) {
 	return json.Marshal(struct {
 		Name     string

router/local_peer.go

@@ -4,10 +4,12 @@ import (
 	"fmt"
 	"log"
 	"net"
+	"sync"
 	"time"
 )
 
 type LocalPeer struct {
+	sync.RWMutex
 	*Peer
 	router     *Router
 	actionChan chan<- LocalPeerAction
@@ -38,7 +40,7 @@ func (peer *LocalPeer) Relay(srcPeer, dstPeer *Peer, df bool, frame []byte, dec
 	if !found {
 		// Not necessarily an error as there could be a race with the
 		// dst disappearing whilst the frame is in flight
-		log.Println("Received packet for unknown destination:", dstPeer.FullName())
+		log.Println("Received packet for unknown destination:", dstPeer)
 		return nil
 	}
 	conn, found := peer.ConnectionTo(relayPeerName)
@@ -75,6 +77,29 @@ func (peer *LocalPeer) RelayBroadcast(srcPeer *Peer, df bool, frame []byte, dec
 	}
 }
 
+func (peer *LocalPeer) Info() string {
+	peer.RLock()
+	defer peer.RUnlock()
+	return peer.Peer.Info()
+}
+
+func (peer *LocalPeer) Connections() ConnectionSet {
+	connections := make(ConnectionSet)
+	peer.RLock()
+	defer peer.RUnlock()
+	for _, conn := range peer.connections {
+		connections[conn] = void
+	}
+	return connections
+}
+
+func (peer *LocalPeer) ConnectionTo(name PeerName) (Connection, bool) {
+	peer.RLock()
+	defer peer.RUnlock()
+	conn, found := peer.connections[name]
+	return conn, found // yes, you really can't inline that. FFS.
+}
+
 func (peer *LocalPeer) ConnectionsTo(names []PeerName) []Connection {
 	conns := make([]Connection, 0, len(names))
 	peer.RLock()
@@ -162,7 +187,7 @@ func (peer *LocalPeer) handleAddConnection(conn Connection) error {
 		log.Fatal("Attempt made to add connection to peer with unknown remote peer")
 	}
 	toName := conn.Remote().Name
-	dupErr := fmt.Errorf("Multiple connections to %s added to %s", conn.Remote().FullName(), peer.FullName())
+	dupErr := fmt.Errorf("Multiple connections to %s added to %s", conn.Remote(), peer.String())
 	// deliberately non symmetrical
 	if dupConn, found := peer.connections[toName]; found {
 		if dupConn == conn {
@@ -228,6 +253,8 @@ func (peer *LocalPeer) handleDeleteConnection(conn Connection) {
 	peer.broadcastPeerUpdate()
 }
 
+// helpers
+
 func (peer *LocalPeer) broadcastPeerUpdate(peers ...*Peer) {
 	peer.router.Routes.Recalculate()
 	peer.router.TopologyGossip.GossipBroadcast(NewTopologyGossipData(peer.router.Peers, append(peers, peer.Peer)...))
@@ -235,8 +262,34 @@ func (peer *LocalPeer) broadcastPeerUpdate(peers ...*Peer) {
 
 func (peer *LocalPeer) checkConnectionLimit() error {
 	limit := peer.router.ConnLimit
-	if 0 != limit && peer.ConnectionCount() >= limit {
+	if 0 != limit && peer.connectionCount() >= limit {
 		return fmt.Errorf("Connection limit reached (%v)", limit)
 	}
 	return nil
 }
+
+func (peer *LocalPeer) addConnection(conn Connection) {
+	peer.Lock()
+	defer peer.Unlock()
+	peer.connections[conn.Remote().Name] = conn
+	peer.version++
+}
+
+func (peer *LocalPeer) deleteConnection(conn Connection) {
+	peer.Lock()
+	defer peer.Unlock()
+	delete(peer.connections, conn.Remote().Name)
+	peer.version++
+}
+
+func (peer *LocalPeer) connectionEstablished(conn Connection) {
+	peer.Lock()
+	defer peer.Unlock()
+	peer.version++
+}
+
+func (peer *LocalPeer) connectionCount() int {
+	peer.RLock()
+	defer peer.RUnlock()
+	return len(peer.connections)
+}

router/mac_cache.go

@@ -89,7 +89,7 @@ func (cache *MacCache) String() string {
 	cache.RLock()
 	defer cache.RUnlock()
 	for key, entry := range cache.table {
-		fmt.Fprintf(&buf, "%v -> %s (%v)\n", intmac(key), entry.peer.FullName(), entry.lastSeen)
+		fmt.Fprintf(&buf, "%v -> %s (%v)\n", intmac(key), entry.peer, entry.lastSeen)
 	}
 	return buf.String()
 }

router/mocks_test.go

@@ -14,7 +14,7 @@ import (
 func (peers *Peers) AddTestConnection(p *Peer) {
 	toPeer := NewPeer(p.Name, "", p.UID, 0)
 	peers.FetchWithDefault(toPeer) // Has side-effect of incrementing refcount
-	conn := newMockConnection(peers.ourself, toPeer)
+	conn := newMockConnection(peers.ourself.Peer, toPeer)
 	peers.ourself.addConnection(conn)
 	peers.ourself.connectionEstablished(conn)
 }
@@ -33,7 +33,7 @@ func (peers *Peers) AddTestRemoteConnection(p1, p2 *Peer) {
 func (peers *Peers) DeleteTestConnection(p *Peer) {
 	toName := p.Name
 	toPeer, _ := peers.Fetch(toName)
-	toPeer.DecrementLocalRefCount()
+	peers.Dereference(toPeer)
 	conn, _ := peers.ourself.ConnectionTo(toName)
 	peers.ourself.deleteConnection(conn)
 }

router/pcap.go

@@ -2,6 +2,7 @@ package router
 
 import (
 	"code.google.com/p/gopacket/pcap"
+	"fmt"
 )
 
 type PcapIO struct {
@@ -44,6 +45,16 @@ func newPcapIO(ifName string, promisc bool, snaplen int, bufSz int) (handle *Pca
 		return
 	}
 	if err = inactive.SetImmediateMode(true); err != nil {
+		// If gopacket is compiled against an older pcap.h that
+		// doesn't have pcap_set_immediate_mode, it supplies a dummy
+		// definition that always returns PCAP_ERROR.  That becomes
+		// "Generic error", which is not very helpful.  The real
+		// pcap_set_immediate_mode never returns PCAP_ERROR, so this
+		// turns it into a more informative message.
+		if fmt.Sprint(err) == "Generic error" {
+			err = fmt.Errorf("compiled against an old version of libpcap; please compile against libpcap-1.5.0 or later")
+		}
+
 		return
 	}
 	if err = inactive.SetBufferSize(bufSz); err != nil {

router/peer.go

@@ -3,25 +3,31 @@ package router
 import (
 	"fmt"
 	"sort"
-	"sync"
+	"strconv"
 )
 
+type PeerUID uint64
+
+func ParsePeerUID(s string) (PeerUID, error) {
+	uid, err := strconv.ParseUint(s, 10, 64)
+	return PeerUID(uid), err
+}
+
 type Peer struct {
-	sync.RWMutex
 	Name          PeerName
 	NameByte      []byte
 	NickName      string
-	UID           uint64
+	UID           PeerUID
 	version       uint64
-	localRefCount uint64
+	localRefCount uint64 // maintained by Peers
 	connections   map[PeerName]Connection
 }
 
 type ConnectionSet map[Connection]struct{}
 
-func NewPeer(name PeerName, nickName string, uid uint64, version uint64) *Peer {
+func NewPeer(name PeerName, nickName string, uid PeerUID, version uint64) *Peer {
 	if uid == 0 {
-		uid = randUint64()
+		uid = PeerUID(randUint64())
 	}
 	return &Peer{
 		Name:        name,
@@ -33,87 +39,11 @@ func NewPeer(name PeerName, nickName string, uid uint64, version uint64) *Peer {
 }
 
 func (peer *Peer) String() string {
-	peer.RLock()
-	defer peer.RUnlock()
-	return fmt.Sprint(peer.FullName(), " (v", peer.version, ") (UID ", peer.UID, ")")
-}
-
-func (peer *Peer) FullName() string {
 	return fmt.Sprint(peer.Name, "(", peer.NickName, ")")
 }
 
-func (peer *Peer) Version() uint64 {
-	peer.RLock()
-	defer peer.RUnlock()
-	return peer.version
-}
-
-func (peer *Peer) IncrementLocalRefCount() {
-	peer.Lock()
-	defer peer.Unlock()
-	peer.localRefCount++
-}
-
-func (peer *Peer) DecrementLocalRefCount() {
-	peer.Lock()
-	defer peer.Unlock()
-	peer.localRefCount--
-}
-
-func (peer *Peer) IsLocallyReferenced() bool {
-	peer.RLock()
-	defer peer.RUnlock()
-	return peer.localRefCount != 0
-}
-
-func (peer *Peer) ConnectionCount() int {
-	peer.RLock()
-	defer peer.RUnlock()
-	return len(peer.connections)
-}
-
-func (peer *Peer) ConnectionTo(name PeerName) (Connection, bool) {
-	peer.RLock()
-	defer peer.RUnlock()
-	conn, found := peer.connections[name]
-	return conn, found // yes, you really can't inline that. FFS.
-}
-
-func (peer *Peer) Connections() ConnectionSet {
-	connections := make(ConnectionSet)
-	peer.RLock()
-	defer peer.RUnlock()
-	for _, conn := range peer.connections {
-		connections[conn] = void
-	}
-	return connections
-}
-
-func (peer *Peer) SetVersionAndConnections(version uint64, connections map[PeerName]Connection) {
-	peer.Lock()
-	defer peer.Unlock()
-	peer.version = version
-	peer.connections = connections
-}
-
-func (peer *Peer) addConnection(conn Connection) {
-	peer.Lock()
-	defer peer.Unlock()
-	peer.connections[conn.Remote().Name] = conn
-	peer.version++
-}
-
-func (peer *Peer) deleteConnection(conn Connection) {
-	peer.Lock()
-	defer peer.Unlock()
-	delete(peer.connections, conn.Remote().Name)
-	peer.version++
-}
-
-func (peer *Peer) connectionEstablished(conn Connection) {
-	peer.Lock()
-	defer peer.Unlock()
-	peer.version++
+func (peer *Peer) Info() string {
+	return fmt.Sprint(peer.String(), " (v", peer.version, ") (UID ", peer.UID, ")")
 }
 
 // Calculate the routing table from this peer to all peers reachable
@@ -137,13 +67,9 @@ func (peer *Peer) connectionEstablished(conn Connection) {
 // reaches that peer. The boolean return indicates whether that has
 // happened.
 //
-// We acquire read locks on peers as we encounter them during the
-// traversal. This prevents the connectivity graph from changing
-// underneath us in ways that would invalidate the result. Thus the
-// answer returned may be out of date, but never inconsistent.
+// NB: This function should generally be invoked while holding a read
+// lock on Peers and LocalPeer.
 func (peer *Peer) Routes(stopAt *Peer, establishedAndSymmetric bool) (bool, map[PeerName]PeerName) {
-	peer.RLock()
-	defer peer.RUnlock()
 	routes := make(map[PeerName]PeerName)
 	routes[peer.Name] = UnknownPeerName
 	nextWorklist := []*Peer{peer}
@@ -155,21 +81,10 @@ func (peer *Peer) Routes(stopAt *Peer, establishedAndSymmetric bool) (bool, map[
 			if curPeer == stopAt {
 				return true, routes
 			}
-			curName := curPeer.Name
-			for remoteName, conn := range curPeer.connections {
-				if establishedAndSymmetric && !conn.Established() {
-					continue
-				}
-				if _, found := routes[remoteName]; found {
-					continue
-				}
-				remote := conn.Remote()
-				remote.RLock()
-				// DANGER holding rlock on remote, possibly taking
-				// rlock on remoteConn
-				if remoteConn, found := remote.connections[curName]; !establishedAndSymmetric || (found && remoteConn.Established()) {
-					defer remote.RUnlock()
-					nextWorklist = append(nextWorklist, remote)
+			curPeer.ForEachConnectedPeer(establishedAndSymmetric, routes,
+				func(remotePeer *Peer) {
+					nextWorklist = append(nextWorklist, remotePeer)
+					remoteName := remotePeer.Name
 					// We now know how to get to remoteName: the same
 					// way we get to curPeer. Except, if curPeer is
 					// the starting peer in which case we know we can
@@ -177,13 +92,25 @@ func (peer *Peer) Routes(stopAt *Peer, establishedAndSymmetric bool) (bool, map[
 					if curPeer == peer {
 						routes[remoteName] = remoteName
 					} else {
-						routes[remoteName] = routes[curName]
+						routes[remoteName] = routes[curPeer.Name]
 					}
-				} else {
-					remote.RUnlock()
-				}
-			}
+				})
 		}
 	}
 	return false, routes
 }
+
+func (peer *Peer) ForEachConnectedPeer(establishedAndSymmetric bool, exclude map[PeerName]PeerName, f func(*Peer)) {
+	for remoteName, conn := range peer.connections {
+		if establishedAndSymmetric && !conn.Established() {
+			continue
+		}
+		if _, found := exclude[remoteName]; found {
+			continue
+		}
+		remotePeer := conn.Remote()
+		if remoteConn, found := remotePeer.connections[peer.Name]; !establishedAndSymmetric || (found && remoteConn.Established()) {
+			f(remotePeer)
+		}
+	}
+}

router/peer_name_hash.go

@@ -2,6 +2,7 @@
 
 // Let peer names be sha256 hashes...of anything (as long as it's
 // unique).
+
 package router
 
 import (

router/peer_name_mac.go

@@ -14,6 +14,7 @@
 // Note that we do not mandate *what* MAC should be used as the peer
 // name. In particular it doesn't actually have to be the MAC of, say,
 // the network interface the peer is sniffing on.
+
 package router
 
 import (

router/peers.go

@@ -10,7 +10,7 @@ import (
 
 type Peers struct {
 	sync.RWMutex
-	ourself *Peer
+	ourself *LocalPeer
 	table   map[PeerName]*Peer
 	onGC    func(*Peer)
 }
@@ -25,7 +25,21 @@ type NameCollisionError struct {
 
 type PeerNameSet map[PeerName]struct{}
 
-func NewPeers(ourself *Peer, onGC func(*Peer)) *Peers {
+type PeerSummary struct {
+	NameByte []byte
+	NickName string
+	UID      PeerUID
+	Version  uint64
+}
+
+type ConnectionSummary struct {
+	NameByte      []byte
+	RemoteTCPAddr string
+	Outbound      bool
+	Established   bool
+}
+
+func NewPeers(ourself *LocalPeer, onGC func(*Peer)) *Peers {
 	return &Peers{
 		ourself: ourself,
 		table:   make(map[PeerName]*Peer),
@@ -33,20 +47,17 @@ func NewPeers(ourself *Peer, onGC func(*Peer)) *Peers {
 }
 
 func (peers *Peers) FetchWithDefault(peer *Peer) *Peer {
-	peers.RLock()
-	res, found := peers.fetchAlias(peer)
-	peers.RUnlock()
-	if found {
-		return res
-	}
 	peers.Lock()
 	defer peers.Unlock()
-	res, found = peers.fetchAlias(peer)
-	if found {
-		return res
+	if existingPeer, found := peers.table[peer.Name]; found {
+		if existingPeer.UID != peer.UID {
+			return nil
+		}
+		existingPeer.localRefCount++
+		return existingPeer
 	}
 	peers.table[peer.Name] = peer
-	peer.IncrementLocalRefCount()
+	peer.localRefCount++
 	return peer
 }
 
@@ -57,6 +68,12 @@ func (peers *Peers) Fetch(name PeerName) (*Peer, bool) {
 	return peer, found // GRRR, why can't I inline this!?
 }
 
+func (peers *Peers) Dereference(peer *Peer) {
+	peers.Lock()
+	defer peers.Unlock()
+	peer.localRefCount--
+}
+
 func (peers *Peers) ForEach(fun func(*Peer)) {
 	peers.RLock()
 	defer peers.RUnlock()
@@ -111,19 +128,21 @@ func (peers *Peers) Names() PeerNameSet {
 }
 
 func (peers *Peers) EncodePeers(names PeerNameSet) []byte {
+	buf := new(bytes.Buffer)
+	enc := gob.NewEncoder(buf)
 	peers.RLock()
+	defer peers.RUnlock()
 	peerList := make([]*Peer, 0, len(names))
 	for name := range names {
 		if peer, found := peers.table[name]; found {
+			if peer == peers.ourself.Peer {
+				peers.ourself.Peer.Encode(enc)
+			} else {
+				peer.Encode(enc)
+			}
 			peerList = append(peerList, peer)
 		}
 	}
-	peers.RUnlock() // release lock so we don't hold it while encoding
-	buf := new(bytes.Buffer)
-	enc := gob.NewEncoder(buf)
-	for _, peer := range peerList {
-		peer.encode(enc)
-	}
 	return buf.Bytes()
 }
 
@@ -135,35 +154,39 @@ func (peers *Peers) GarbageCollect() []*Peer {
 
 func (peers *Peers) String() string {
 	var buf bytes.Buffer
+	printConnection := func(conn Connection) {
+		established := ""
+		if !conn.Established() {
+			established = " (unestablished)"
+		}
+		fmt.Fprintf(&buf, "   -> %s [%v%s]\n", conn.Remote(), conn.RemoteTCPAddr(), established)
+	}
 	peers.ForEach(func(peer *Peer) {
-		fmt.Fprint(&buf, peer, "\n")
-		for conn := range peer.Connections() {
-			established := ""
-			if !conn.Established() {
-				established = " (unestablished)"
+		if peer == peers.ourself.Peer {
+			fmt.Fprintln(&buf, peers.ourself.Info())
+			for conn := range peers.ourself.Connections() {
+				printConnection(conn)
+			}
+		} else {
+			fmt.Fprintln(&buf, peer.Info())
+			// Modifying peer.connections requires a write lock on
+			// Peers, and since we are holding a read lock (due to the
+			// ForEach), access without locking the peer is safe.
+			for _, conn := range peer.connections {
+				printConnection(conn)
 			}
-			fmt.Fprintf(&buf, "   -> %s [%v%s]\n", conn.Remote().FullName(), conn.RemoteTCPAddr(), established)
 		}
 	})
 	return buf.String()
 }
 
-func (peers *Peers) fetchAlias(peer *Peer) (*Peer, bool) {
-	if existingPeer, found := peers.table[peer.Name]; found {
-		if existingPeer.UID != peer.UID {
-			return nil, true
-		}
-		existingPeer.IncrementLocalRefCount()
-		return existingPeer, true
-	}
-	return nil, false
-}
-
 func (peers *Peers) garbageCollect() []*Peer {
 	removed := []*Peer{}
+	peers.ourself.RLock()
 	_, reached := peers.ourself.Routes(nil, false)
+	peers.ourself.RUnlock()
 	for name, peer := range peers.table {
-		if _, found := reached[peer.Name]; !found && !peer.IsLocallyReferenced() {
+		if _, found := reached[peer.Name]; !found && peer.localRefCount == 0 {
 			delete(peers.table, name)
 			peers.onGC(peer)
 			removed = append(removed, peer)
@@ -180,27 +203,27 @@ func setFromPeersMap(peers map[PeerName]*Peer) PeerNameSet {
 	return names
 }
 
-func (peers *Peers) decodeUpdate(update []byte) (newPeers map[PeerName]*Peer, decodedUpdate []*Peer, decodedConns [][]byte, err error) {
+func (peers *Peers) decodeUpdate(update []byte) (newPeers map[PeerName]*Peer, decodedUpdate []*Peer, decodedConns [][]ConnectionSummary, err error) {
 	newPeers = make(map[PeerName]*Peer)
 	decodedUpdate = []*Peer{}
-	decodedConns = [][]byte{}
+	decodedConns = [][]ConnectionSummary{}
 
 	updateBuf := new(bytes.Buffer)
 	updateBuf.Write(update)
 	decoder := gob.NewDecoder(updateBuf)
 
 	for {
-		nameByte, nickName, uid, version, connsBuf, decErr := decodePeerNoConns(decoder)
+		peerSummary, connSummaries, decErr := decodePeer(decoder)
 		if decErr == io.EOF {
 			break
 		} else if decErr != nil {
 			err = decErr
 			return
 		}
-		name := PeerNameFromBin(nameByte)
-		newPeer := NewPeer(name, nickName, uid, version)
+		name := PeerNameFromBin(peerSummary.NameByte)
+		newPeer := NewPeer(name, peerSummary.NickName, peerSummary.UID, peerSummary.Version)
 		decodedUpdate = append(decodedUpdate, newPeer)
-		decodedConns = append(decodedConns, connsBuf)
+		decodedConns = append(decodedConns, connSummaries)
 		existingPeer, found := peers.table[name]
 		if !found {
 			newPeers[name] = newPeer
@@ -210,38 +233,33 @@ func (peers *Peers) decodeUpdate(update []byte) (newPeers map[PeerName]*Peer, de
 		}
 	}
 
-	for _, connsBuf := range decodedConns {
-		decErr := connsIterator(connsBuf, func(remoteNameByte []byte, _ string, _, _ bool) {
-			remoteName := PeerNameFromBin(remoteNameByte)
+	for _, connSummaries := range decodedConns {
+		for _, connSummary := range connSummaries {
+			remoteName := PeerNameFromBin(connSummary.NameByte)
 			if _, found := newPeers[remoteName]; found {
-				return
+				continue
 			}
 			if _, found := peers.table[remoteName]; found {
-				return
+				continue
 			}
 			// Update refers to a peer which we have no knowledge
 			// of. Thus we can't apply the update. Abort.
 			err = UnknownPeerError{remoteName}
-		})
-		if decErr != nil && decErr != io.EOF {
-			err = decErr
-		}
-		if err != nil {
 			return
 		}
 	}
 	return
 }
 
-func (peers *Peers) applyUpdate(decodedUpdate []*Peer, decodedConns [][]byte) map[PeerName]*Peer {
+func (peers *Peers) applyUpdate(decodedUpdate []*Peer, decodedConns [][]ConnectionSummary) map[PeerName]*Peer {
 	newUpdate := make(map[PeerName]*Peer)
 	for idx, newPeer := range decodedUpdate {
-		connsBuf := decodedConns[idx]
+		connSummaries := decodedConns[idx]
 		name := newPeer.Name
 		// guaranteed to find peer in the peers.table
 		peer := peers.table[name]
 		if peer != newPeer &&
-			(peer == peers.ourself || peer.Version() >= newPeer.Version()) {
+			(peer == peers.ourself.Peer || peer.version >= newPeer.version) {
 			// Nobody but us updates us. And if we know more about a
 			// peer than what's in the the update, we ignore the
 			// latter.
@@ -257,89 +275,57 @@ func (peers *Peers) applyUpdate(decodedUpdate []*Peer, decodedConns [][]byte) ma
 		// for an update would be someone else calling
 		// router.Peers.ApplyUpdate. But ApplyUpdate takes the Lock on
 		// the router.Peers, so there can be no race here.
-		conns := readConnsMap(peer, connsBuf, peers.table)
-		peer.SetVersionAndConnections(newPeer.Version(), conns)
+		peer.version = newPeer.version
+		peer.connections = makeConnsMap(peer, connSummaries, peers.table)
 		newUpdate[name] = peer
 	}
 	return newUpdate
 }
 
-func (peer *Peer) encode(enc *gob.Encoder) {
-	peer.RLock()
-	defer peer.RUnlock()
+func (peer *Peer) Encode(enc *gob.Encoder) {
+	checkFatal(enc.Encode(PeerSummary{
+		peer.NameByte,
+		peer.NickName,
+		peer.UID,
+		peer.version}))
 
-	checkFatal(enc.Encode(peer.NameByte))
-	checkFatal(enc.Encode(peer.NickName))
-	checkFatal(enc.Encode(peer.UID))
-	checkFatal(enc.Encode(peer.version))
-
-	connsBuf := new(bytes.Buffer)
-	connsEnc := gob.NewEncoder(connsBuf)
+	connSummaries := []ConnectionSummary{}
 	for _, conn := range peer.connections {
-		checkFatal(connsEnc.Encode(conn.Remote().NameByte))
-		checkFatal(connsEnc.Encode(conn.RemoteTCPAddr()))
-		checkFatal(connsEnc.Encode(conn.Outbound()))
-		// DANGER holding rlock on peer, going to take rlock on conn
-		checkFatal(connsEnc.Encode(conn.Established()))
+		connSummaries = append(connSummaries, ConnectionSummary{
+			conn.Remote().NameByte,
+			conn.RemoteTCPAddr(),
+			conn.Outbound(),
+			// DANGER holding rlock on peer, going to take rlock on conn
+			conn.Established(),
+		})
 	}
-	checkFatal(enc.Encode(connsBuf.Bytes()))
+
+	checkFatal(enc.Encode(connSummaries))
 }
 
-func decodePeerNoConns(dec *gob.Decoder) (nameByte []byte, nickName string, uid uint64, version uint64, conns []byte, err error) {
-	if err = dec.Decode(&nameByte); err != nil {
+func (peer *LocalPeer) Encode(enc *gob.Encoder) {
+	peer.RLock()
+	defer peer.RUnlock()
+	peer.Peer.Encode(enc)
+}
+
+func decodePeer(dec *gob.Decoder) (peerSummary PeerSummary, connSummaries []ConnectionSummary, err error) {
+	if err = dec.Decode(&peerSummary); err != nil {
 		return
 	}
-	if err = dec.Decode(&nickName); err != nil {
-		return
-	}
-	if err = dec.Decode(&uid); err != nil {
-		return
-	}
-	if err = dec.Decode(&version); err != nil {
-		return
-	}
-	if err = dec.Decode(&conns); err != nil {
+	if err = dec.Decode(&connSummaries); err != nil {
 		return
 	}
 	return
 }
 
-func connsIterator(input []byte, fun func([]byte, string, bool, bool)) error {
-	buf := new(bytes.Buffer)
-	buf.Write(input)
-	dec := gob.NewDecoder(buf)
-	for {
-		var nameByte []byte
-		if err := dec.Decode(&nameByte); err != nil {
-			return err
-		}
-		var foundAt string
-		if err := dec.Decode(&foundAt); err != nil {
-			return err
-		}
-		var outbound bool
-		if err := dec.Decode(&outbound); err != nil {
-			return err
-		}
-		var established bool
-		if err := dec.Decode(&established); err != nil {
-			return err
-		}
-		fun(nameByte, foundAt, outbound, established)
-	}
-}
-
-func readConnsMap(peer *Peer, buf []byte, table map[PeerName]*Peer) map[PeerName]Connection {
+func makeConnsMap(peer *Peer, connSummaries []ConnectionSummary, table map[PeerName]*Peer) map[PeerName]Connection {
 	conns := make(map[PeerName]Connection)
-	if err := connsIterator(buf, func(nameByte []byte, remoteTCPAddr string, outbound bool, established bool) {
-		name := PeerNameFromBin(nameByte)
+	for _, connSummary := range connSummaries {
+		name := PeerNameFromBin(connSummary.NameByte)
 		remotePeer := table[name]
-		conn := NewRemoteConnection(peer, remotePeer, remoteTCPAddr, outbound, established)
+		conn := NewRemoteConnection(peer, remotePeer, connSummary.RemoteTCPAddr, connSummary.Outbound, connSummary.Established)
 		conns[name] = conn
-	}); err != io.EOF {
-		// this should never happen since we've already successfully
-		// decoded the same data in decodeUpdate
-		checkFatal(err)
 	}
 	return conns
 }

router/peers_test.go

@@ -17,10 +17,10 @@ import (
 // - non-gc of peers that are only referenced locally
 
 func newNode(name PeerName) (*Peer, *Peers) {
-	peer := NewPeer(name, "", 0, 0)
+	peer := NewLocalPeer(name, "", nil)
 	peers := NewPeers(peer, func(*Peer) {})
-	peers.FetchWithDefault(peer)
-	return peer, peers
+	peers.FetchWithDefault(peer.Peer)
+	return peer.Peer, peers
 }
 
 // Check that ApplyUpdate copies the whole topology from peers
@@ -29,7 +29,7 @@ func checkApplyUpdate(t *testing.T, peers *Peers) {
 	// We need a new node outside of the network, with a connection
 	// into it.
 	_, testBedPeers := newNode(dummyName)
-	testBedPeers.AddTestConnection(peers.ourself)
+	testBedPeers.AddTestConnection(peers.ourself.Peer)
 	testBedPeers.ApplyUpdate(peers.EncodePeers(peers.Names()))
 
 	checkTopologyPeers(t, true, testBedPeers.allPeersExcept(dummyName), peers.allPeers()...)
@@ -52,7 +52,7 @@ func TestPeersEncoding(t *testing.T) {
 		case 0:
 			from, to := rand.Intn(numNodes), rand.Intn(numNodes)
 			if from != to {
-				if _, found := peer[from].ConnectionTo(peer[to].Name); !found {
+				if _, found := peer[from].connections[peer[to].Name]; !found {
 					ps[from].AddTestConnection(peer[to])
 					conns = append(conns, struct{ from, to int }{from, to})
 					checkApplyUpdate(t, ps[from])

router/protocol.go

@@ -2,7 +2,7 @@ package router
 
 const (
 	Protocol        = "weave"
-	ProtocolVersion = 15
+	ProtocolVersion = 17
 )
 
 type ProtocolTag byte
@@ -12,7 +12,6 @@ const (
 	ProtocolConnectionEstablished
 	ProtocolFragmentationReceived
 	ProtocolStartFragmentationTest
-	ProtocolNonce
 	ProtocolPMTUVerified
 	ProtocolGossip
 	ProtocolGossipUnicast

router/router.go

@@ -56,17 +56,17 @@ type PacketSourceSink interface {
 func NewRouter(config RouterConfig, name PeerName, nickName string) *Router {
 	router := &Router{RouterConfig: config, GossipChannels: make(map[uint32]*GossipChannel)}
 	onMacExpiry := func(mac net.HardwareAddr, peer *Peer) {
-		log.Println("Expired MAC", mac, "at", peer.FullName())
+		log.Println("Expired MAC", mac, "at", peer)
 	}
 	onPeerGC := func(peer *Peer) {
 		router.Macs.Delete(peer)
-		log.Println("Removed unreachable peer", peer.FullName())
+		log.Println("Removed unreachable peer", peer)
 	}
 	router.Ourself = NewLocalPeer(name, nickName, router)
 	router.Macs = NewMacCache(macMaxAge, onMacExpiry)
-	router.Peers = NewPeers(router.Ourself.Peer, onPeerGC)
+	router.Peers = NewPeers(router.Ourself, onPeerGC)
 	router.Peers.FetchWithDefault(router.Ourself.Peer)
-	router.Routes = NewRoutes(router.Ourself.Peer, router.Peers)
+	router.Routes = NewRoutes(router.Ourself, router.Peers)
 	router.ConnectionMaker = NewConnectionMaker(router.Ourself, router.Peers, router.Port)
 	router.TopologyGossip = router.NewGossip("topology", router)
 	return router
@@ -94,13 +94,18 @@ func (router *Router) Start() {
 	}
 }
 
+func (router *Router) Stop() error {
+	// TODO: perform graceful shutdown...
+	return nil
+}
+
 func (router *Router) UsingPassword() bool {
 	return router.Password != nil
 }
 
 func (router *Router) Status() string {
 	var buf bytes.Buffer
-	fmt.Fprintln(&buf, "Our name is", router.Ourself.FullName())
+	fmt.Fprintln(&buf, "Our name is", router.Ourself)
 	fmt.Fprintln(&buf, "Sniffing traffic on", router.Iface)
 	fmt.Fprintf(&buf, "MACs:\n%s", router.Macs)
 	fmt.Fprintf(&buf, "Peers:\n%s", router.Peers)
@@ -250,7 +255,7 @@ func (router *Router) udpReader(conn *net.UDPConn, po PacketSink) {
 		if !ok {
 			continue
 		}
-		if err := relayConn.Decryptor.IterateFrames(packet, router.handleUDPPacketFunc(dec, sender, po)); err != nil {
+		if err := relayConn.Decryptor.IterateFrames(packet, router.handleUDPPacketFunc(relayConn, dec, sender, po)); err != nil {
 			// Errors during UDP packet decoding / processing are
 			// non-fatal. One common cause is that we receive and
 			// attempt to decrypt a "stray" packet. This can actually
@@ -267,8 +272,8 @@ func (router *Router) udpReader(conn *net.UDPConn, po PacketSink) {
 	}
 }
 
-func (router *Router) handleUDPPacketFunc(dec *EthernetDecoder, sender *net.UDPAddr, po PacketSink) FrameConsumer {
-	return func(relayConn *LocalConnection, srcNameByte, dstNameByte []byte, frame []byte) {
+func (router *Router) handleUDPPacketFunc(relayConn *LocalConnection, dec *EthernetDecoder, sender *net.UDPAddr, po PacketSink) FrameConsumer {
+	return func(srcNameByte, dstNameByte []byte, frame []byte) {
 		srcPeer, found := router.Peers.Fetch(PeerNameFromBin(srcNameByte))
 		if !found {
 			return
@@ -323,7 +328,7 @@ func (router *Router) handleUDPPacketFunc(dec *EthernetDecoder, sender *net.UDPA
 		dstMac := dec.eth.DstMAC
 
 		if router.Macs.Enter(srcMac, srcPeer) {
-			log.Println("Discovered remote MAC", srcMac, "at", srcPeer.FullName())
+			log.Println("Discovered remote MAC", srcMac, "at", srcPeer)
 		}
 		if po != nil {
 			router.LogFrame("Injecting", frame, &dec.eth)

router/routes.go

@@ -9,7 +9,7 @@ import (
 
 type Routes struct {
 	sync.RWMutex
-	ourself      *Peer
+	ourself      *LocalPeer
 	peers        *Peers
 	unicast      map[PeerName]PeerName
 	unicastAll   map[PeerName]PeerName // [1]
@@ -21,7 +21,7 @@ type Routes struct {
 	// symmetric ones
 }
 
-func NewRoutes(ourself *Peer, peers *Peers) *Routes {
+func NewRoutes(ourself *LocalPeer, peers *Peers) *Routes {
 	routes := &Routes{
 		ourself:      ourself,
 		peers:        peers,
@@ -166,12 +166,17 @@ func (routes *Routes) run(recalculate <-chan *struct{}, wait <-chan chan struct{
 }
 
 func (routes *Routes) calculate() {
+	routes.peers.RLock()
+	routes.ourself.RLock()
 	var (
 		unicast      = routes.calculateUnicast(true)
 		unicastAll   = routes.calculateUnicast(false)
 		broadcast    = routes.calculateBroadcast(true)
 		broadcastAll = routes.calculateBroadcast(false)
 	)
+	routes.ourself.RUnlock()
+	routes.peers.RUnlock()
+
 	routes.Lock()
 	routes.unicast = unicast
 	routes.unicastAll = unicastAll
@@ -214,29 +219,13 @@ func (routes *Routes) calculateUnicast(establishedAndSymmetric bool) map[PeerNam
 // where <= is the subset relationship on keys of the returned map.
 func (routes *Routes) calculateBroadcast(establishedAndSymmetric bool) map[PeerName][]PeerName {
 	broadcast := make(map[PeerName][]PeerName)
-	ourself := routes.ourself
-	ourConnections := ourself.Connections()
-
-	routes.peers.ForEach(func(peer *Peer) {
+	for _, peer := range routes.peers.table {
 		hops := []PeerName{}
-		if found, reached := peer.Routes(ourself, establishedAndSymmetric); found {
-			// This is rather similar to the inner loop on
-			// peer.Routes(...); the main difference is in the
-			// locking.
-			for conn := range ourConnections {
-				if establishedAndSymmetric && !conn.Established() {
-					continue
-				}
-				remoteName := conn.Remote().Name
-				if _, found := reached[remoteName]; found {
-					continue
-				}
-				if remoteConn, found := conn.Remote().ConnectionTo(ourself.Name); !establishedAndSymmetric || (found && remoteConn.Established()) {
-					hops = append(hops, remoteName)
-				}
-			}
+		if found, reached := peer.Routes(routes.ourself.Peer, establishedAndSymmetric); found {
+			routes.ourself.ForEachConnectedPeer(establishedAndSymmetric, reached,
+				func(remotePeer *Peer) { hops = append(hops, remotePeer.Name) })
 		}
 		broadcast[peer.Name] = hops
-	})
+	}
 	return broadcast
 }

site/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gem 'jekyll'

site/_config.yml

@@ -2,3 +2,9 @@ markdown: redcarpet
 
 redcarpet:
   extensions: ['fenced_code_blocks', 'smart']
+
+exclude:
+  - 'Gemfile'
+  - 'Gemfile.lock'
+  - 'params.json'
+  - 'build-and-publish.sh'

site/boot2docker.md

@@ -23,7 +23,7 @@ example above, the 'launch' command would be run like this:
 
 and the 'run' command like this:
 
-    host1# C=$(boot2docker ssh "sudo ./weave run 10.2.1.1/24 -t -i ubuntu")
+    host1$ C=$(boot2docker ssh "sudo ./weave run 10.2.1.1/24 -t -i ubuntu")
 
 For more information about how to access services running in
 Boot2Docker from the host or other machines (i.e. outside of the weave

site/build-and-publish.sh

@@ -0,0 +1,36 @@
+#!/bin/sh -ex
+
+cd $(dirname $0)/
+
+## Use GCS until we set up Route53/CloudFront/S3
+
+BRANCH=${TRAVIS_BRANCH:-`git rev-parse --abbrev-ref @`}
+COMMIT=${TRAVIS_COMMIT:-`git rev-parse @`}
+OUTPUT="weave/${BRANCH}/${COMMIT}"
+
+if [ -n "${TRAVIS_TAG}" ]; then
+  ## Travis will run separate build for commit and tag pushes,
+  OUTPUT="weave/${TRAVIS_TAG}"
+fi
+
+if [ "${BRANCH}" = "latest_release_doc_updates" ]; then
+  OUTPUT="weave/latest_release"
+fi
+
+export BRANCH COMMIT OUTPUT
+
+bundle install --path=.bundle
+bundle exec jekyll build --verbose
+
+gsutil -m rsync -r -d _site "gs://docs.weave.works/${OUTPUT}"
+
+echo "Published at http://docs.weave.works/${OUTPUT}"
+
+if [ -z "${TRAVIS_TAG}" -a ! "${BRANCH}" = "latest_release_doc_updates" ]; then
+  echo "<meta http-equiv=\"refresh\" content=\"0; url=http://docs.weave.works/${OUTPUT}\" />" \
+    | gsutil \
+      -h "Content-Type:text/html" \
+      -h "Cache-Control:private, max-age=0, no-transform" \
+      cp -a "public-read" - "gs://docs.weave.works/weave/${BRANCH}/index.html"
+  gsutil -m rsync -r -d -p gs://docs.weave.works/${OUTPUT} gs://docs.weave.works/weave/${BRANCH}/head
+fi

site/building.md

@@ -162,6 +162,24 @@ vm$ sudo docker ps
 
 and so on.
 
+If you are looking to just do a build and not run anything on this VM,
+you can do so with
+
+```bash
+$ vagrant ssh -c 'make -C src/github.com/weaveworks/weave'
+```
+
+you should then find container snapshot tarballs in the top-level
+directory. You can use these snapshots with `docker load` against a
+different host, e.g.
+
+```bash
+$ export DOCKER_HOST=tcp://<HOST:PORT>
+$ docker load < weave.tar
+$ docker load < weavedns.tar
+$ docker load < weaveexec.tar
+```
+
 You can provide extra Vagrant configuration by putting a file
 `Vagrant.local` in the same place as `Vagrantfile`; for instance, to
 forward additional ports.

site/features.md

@@ -22,7 +22,9 @@ example](https://github.com/weaveworks/weave#example):
  * [Dynamic topologies](#dynamic-topologies)
  * [Container mobility](#container-mobility)
  * [Fault tolerance](#fault-tolerance)
- * [DNS](#dns)
+ * [Automatic IP address management](#ipam)
+ * [Automatic discovery with WeaveDNS](#dns)
+ * [Starting containers with Docker](#proxy)
 
 ### <a name="virtual-ethernet-switch"></a>Virtual Ethernet Switch
 
@@ -63,13 +65,13 @@ To accomplish that, we assign each application a different subnet. So,
 in the above example, if we wanted to add another application similar
 to, but isolated from, our first, we'd launch the containers with...
 
-    host1# D=$(weave run 10.2.2.1/24 -t -i ubuntu)
-    host2# D=$(weave run 10.2.2.2/24 -t -i ubuntu)
+    host1$ D=$(weave run 10.2.2.1/24 -t -i ubuntu)
+    host2$ D=$(weave run 10.2.2.2/24 -t -i ubuntu)
 
 A quick 'ping' test in the containers confirms that they can talk to
 each other but not the containers of our first application...
 
-    host1# docker attach $D
+    host1$ docker attach $D
     
     root@da50502598d5:/# ping -c 1 -q 10.2.2.2
     PING 10.2.2.2 (10.2.2.2): 48 data bytes
@@ -93,7 +95,7 @@ well-known technique from the 'on metal' days to containers.
 If desired, a container can be attached to multiple subnets when it is
 started:
 
-    host1# weave run 10.2.2.1/24 10.2.3.1/24 -t -i ubuntu
+    host1$ weave run 10.2.2.1/24 10.2.3.1/24 -t -i ubuntu
 
 NB: By default docker permits communication between containers on the
 same host, via their docker-assigned IP addresses. For complete
@@ -114,30 +116,30 @@ situations, weave allows an existing, running container to be attached
 to the weave network. To illustrate, we can achieve the same effect as
 the first example with
 
-    host1# C=$(docker run -d -t -i ubuntu)
-    host1# weave attach 10.2.1.1/24 $C
+    host1$ C=$(docker run -d -t -i ubuntu)
+    host1$ weave attach 10.2.1.1/24 $C
 
 There is a matching `weave detach` command:
 
-    host1# weave detach 10.2.1.1/24 $C
+    host1$ weave detach 10.2.1.1/24 $C
 
 You can detach a container from one application network and attach it
 to another:
 
-    host1# weave detach 10.2.1.1/24 $C
-    host1# weave attach 10.2.2.1/24 $C
+    host1$ weave detach 10.2.1.1/24 $C
+    host1$ weave attach 10.2.2.1/24 $C
 
 or attach a container to multiple application networks, effectively
 sharing it between applications:
 
-    host1# weave attach 10.2.1.1/24 $C
-    host1# weave attach 10.2.2.1/24 $C
+    host1$ weave attach 10.2.1.1/24 $C
+    host1$ weave attach 10.2.2.1/24 $C
 
 Finally, multiple addresses can be attached or detached with a single
 invocation:
 
-    host1# weave attach 10.2.1.1/24 10.2.2.1/24 10.2.3.1/24 $C
-    host1# weave detach 10.2.1.1/24 10.2.2.1/24 10.2.3.1/24 $C
+    host1$ weave attach 10.2.1.1/24 10.2.2.1/24 10.2.3.1/24 $C
+    host1$ weave detach 10.2.1.1/24 10.2.2.1/24 10.2.3.1/24 $C
 
 ### <a name="security"></a>Security
 
@@ -145,12 +147,12 @@ In order to connect containers across untrusted networks, weave peers
 can be told to encrypt traffic by supplying a `-password` option or
 `WEAVE_PASSWORD` environment variable when launching weave, e.g.
 
-    host1# weave launch -password wEaVe
+    host1$ weave launch -password wEaVe
 
 or
 
-    host1# export WEAVE_PASSWORD=wEaVe
-    host1# weave launch
+    host1$ export WEAVE_PASSWORD=wEaVe
+    host1$ weave launch
 
 _NOTE: The command line option takes precedence over the environment
 variable._
@@ -169,31 +171,31 @@ anywhere.
 Let's say that in our example we want `$HOST2` to have access to the
 application containers. On `$HOST2` we run
 
-    host2# weave expose 10.2.1.102/24
+    host2$ weave expose 10.2.1.102/24
 
 choosing an unused IP address in the application subnet. (There is a
 corresponding 'hide' command to revert this step.)
 
 Now
 
-    host2# ping 10.2.1.2
+    host2$ ping 10.2.1.2
 
 will work. And, more interestingly,
 
-    host2# ping 10.2.1.1
+    host2$ ping 10.2.1.1
 
 will work too, which is talking to a container that resides on `$HOST1`.
 
 Multiple subnet addresses can be exposed or hidden with a single
 invocation:
 
-    host2# weave expose 10.2.1.102/24 10.2.2.102/24
-    host2# weave hide 10.2.1.102/24 10.2.2.102/24
+    host2$ weave expose 10.2.1.102/24 10.2.2.102/24
+    host2$ weave hide   10.2.1.102/24 10.2.2.102/24
 
 Finally, exposed addresses can be added to weaveDNS by supplying a
-hostname:
+fully-qualified domain name:
 
-    host2# weave expose 10.2.1.102/24 -h exposed.weave.local
+    host2$ weave expose 10.2.1.102/24 -h exposed.weave.local
 
 ### <a name="service-export"></a>Service export
 
@@ -208,12 +210,12 @@ a container on `$HOST1`, accessible to the outside world via `$HOST2`.
 First we need to expose the application network to `$HOST2`, as
 explained [above](#host-network-integration), i.e.
 
-    host2# weave expose 10.2.1.102/24
+    host2$ weave expose 10.2.1.102/24
 
 Then we add a NAT rule to route from the outside world to the
 destination container service.
 
-    host2# iptables -t nat -A PREROUTING -p tcp -i eth0 --dport 2211 \
+    host2$ iptables -t nat -A PREROUTING -p tcp -i eth0 --dport 2211 \
            -j DNAT --to-destination 10.2.1.1:4422
 
 Here we are assuming that the "outside world" is connecting to `$HOST2`
@@ -248,19 +250,19 @@ First we need to expose the application network to the host, as
 explained [above](#host-network-integration), this time on `$HOST1`,
 i.e.
 
-    host1# weave expose 10.2.1.101/24
+    host1$ weave expose 10.2.1.101/24
 
 Then we add a NAT rule to route from the above IP to the destination
 service.
 
-    host1# iptables -t nat -A PREROUTING -p tcp -d 10.2.1.101 --dport 3322 \
+    host1$ iptables -t nat -A PREROUTING -p tcp -d 10.2.1.101 --dport 3322 \
            -j DNAT --to-destination $HOST3:2211
 
 This allows any application container to reach the service by
 connecting to 10.2.1.101:3322. So if `$HOST3` is indeed running a netcat
 service on port 2211, e.g.
 
-    host3# nc -lk -p 2211
+    host3$ nc -lk -p 2211
 
 then we can connect to it from our application container on `$HOST2` with
 
@@ -295,13 +297,13 @@ In our example above, the netcat service on `$HOST3` is imported into
 weave via `$HOST1`. We can export it on `$HOST2` by first exposing the
 application network with
 
-    host2# weave expose 10.2.1.102/24
+    host2$ weave expose 10.2.1.102/24
 
 and then adding a NAT rule which routes traffic from the `$HOST2`
 network (i.e. anything which can connect to `$HOST2`) to the service
 endpoint in the weave network
 
-    host2# iptables -t nat -A PREROUTING -p tcp -i eth0 --dport 4433 \
+    host2$ iptables -t nat -A PREROUTING -p tcp -i eth0 --dport 4433 \
            -j DNAT --to-destination 10.2.1.101:3322
 
 Now any host on the same network as `$HOST2` can access the service with
@@ -390,9 +392,23 @@ restarted in that event, and indeed may not even experience a
 temporary connectivity failure if the weave container is restarted
 quickly enough.
 
-### <a name="dns"></a>DNS
+### <a name="ipam"></a>Automatic IP Address Management
+
+Weave can automatically assign unique IP addresses to each container
+across the network. The `run`, `start`, `attach`, `expose` and `hide`
+commands will then fetch an address automatically if none is
+specified. See the [IPAM](ipam.html) documentation for details on how
+to enable and use this feature.
+
+### <a name="dns"></a>Automatic discovery with WeaveDNS
 
 WeaveDNS is a distributed DNS service for weave networks, enabling
-containers to address each other by name rather than IP address. A
-preview release of weaveDNS is described [on our
-blog](http://weaveblog.com/2014/11/04/have-you-met-weavedns/).
+containers to address each other by name rather than IP address. Find
+out more about WeaveDNS from its [documentation](weavedns.html).
+
+### <a name="proxy"></a>Starting containers with Docker
+
+Instead of the `weave` command-line utility, you may prefer to use the
+standard Docker command-line interface, or the Docker remote API, for
+starting weave-enabled containers. This can be accomplished with the
+[Weave Proxy](proxy.html).

site/how-it-works.md

@@ -5,6 +5,14 @@ layout: default
 
 ## How does it work?
 
+ * [Overview](#overview)
+ * [Encapsulation](#encapsulation)
+ * [Topology](#topology)
+ * [Crypto](#crypto)
+ * [Further reading](#further-reading)
+
+### <a name="overview"></a>Overview
+
 A weave network consists of a number of 'peers' - weave routers
 residing on different hosts. Each peer has a name, which tends to
 remain the same over restarts, a human friendly nickname for use in
@@ -102,30 +110,34 @@ MAC discovery.
 
 The topology information captures which peers are connected to which
 other peers. Weave peers communicate their knowledge of the topology
-(and changes to it) to their neighbours, who then pass this
-information on to their own neighbours, and so on, until the entire
-network knows about any change.
+(and changes to it) to others, so that all peers learn about the
+entire topology. This communication occurs over the TCP links between
+peers, using a) spanning-tree based broadcast mechanism, and b) a
+neighour gossip mechanism.
 
-Topology is communicated over the TCP links between peers, using a
-Gossip mechanism.  Topology messages are sent by a peer...
+Topology messages are sent by a peer...
 
-- when a connection has been added; the entire topology is sent to the
-  remote peer, and an incremental update, containing information on
-  just the two peers at the ends of the connection, is sent to all
-  neighbours,
+- when a connection has been added; if the remote peer appears to be
+  new to the network, the entire topology is sent to it, and an
+  incremental update, containing information on just the two peers at
+  the ends of the connection, is broadcast,
 - when a connection has been marked as 'established', indicating that
   the remote peer can receive UDP traffic from the peer; an update
-  containing just information about the local peer is sent to all
-  neighbours,
+  containing just information about the local peer is broadcast,
 - when a connection has been torn down; an update containing just
-  information about the local peer is sent to all neighbours,
-- periodically, on a timer, in case someone has missed an update.
+  information about the local peer is broadcast,
+- periodically, on a timer, the entire topology is "gossiped" to a
+  subset of neighbours, based on a topology-sensitive random
+  distribution. This is done in case some of the aforementioned
+  broadcasts do not reach all peers, due to rapid changes in the
+  topology causing broadcast routing tables to become outdated.
 
 The receiver of a topology update merges that update with its own
 topology model, adding peers hitherto unknown to it, and updating
 peers for which the update contains a more recent version than known
-to it. If there were any such new/updated peers, then an improved
-update containing them is sent out on all connections.
+to it. If there were any such new/updated peers, and the topology
+update was received over gossip (rather than broadcast), then an
+improved update containing them is gossiped.
 
 If the update mentions a peer that the receiver does not know, then
 the entire update is ignored.
@@ -206,8 +218,8 @@ no longer has any connections within the network, it drops all
 knowledge of that second peer.
 
 #### Out-of-date topology
-The peer-to-peer gossiping of updates is not instantaneous, so it is
-very possible for a node elsewhere in the network to have an
+The propagation of topology changes to all peers is not instantaneous,
+so it is very possible for a node elsewhere in the network to have an
 out-of-date view.
 
 If the destination peer for a packet is still reachable, then
@@ -235,6 +247,16 @@ quite difficult to use NaCl incorrectly. Contrast this with libraries
 such as OpenSSL where the library and its APIs are vast in size,
 poorly documented, and easily used wrongly.
 
+There are some similarities between weave's crypto and
+[TLS](https://tools.ietf.org/html/rfc4346). We do not need to cater
+for multiple cipher suites, certificate exchange and other
+requirements emanating from X509, and a number of other features. This
+simplifies the protocol and implementation considerably. On the other
+hand, we need to support UDP transports, and while there are
+extensions to TLS such as [DTLS](https://tools.ietf.org/html/rfc4347)
+which can operate over UDP, these are not widely implemented and
+deployed.
+
 #### Establishing the Ephemeral Session Key
 
 For every connection between peers, a fresh public/private key pair is
@@ -268,25 +290,23 @@ between two peers.
 #### TCP
 
 TCP connection are only used to exchange topology information between
-peers, via a message-based protocol. The router generates a fresh
-192-bit random nonce for every message to be sent, and prepends the
-nonce to the encrypted message, as is normal in NaCl, so that the
-receiver knows the nonce. Encryption of each message is carried out
-using NaCl's `secretbox.Seal` function using the ephemeral session
-key. Each TCP connection has a monotonically incrementing message
-counter, the current value of which is included in the encrypted part
-of the message. Given the assumption that TCP is reliable and ordered,
-a message received via TCP is only acted upon if the message counter
-in the received message is the expected message counter. This prevents
-replay attacks on the TCP connection.
+peers, via a message-based protocol. Encryption of each message is
+carried out by NaCl's `secretbox.Seal` function using the ephemeral
+session key and a nonce. The nonce contains the message sequence
+number, which is incremented for every message sent, and a bit
+indicating the polarity of the connection at the sender ('1' for
+outbound). The latter is required by the
+[NaCl Security Model](http://nacl.cr.yp.to/box.html) in order to
+ensure that the two ends of the connection do not use the same nonces.
 
-As TCP connections do not carry captured traffic, minimising message
-size or latency is not a major concern, so the potentially substantial
-increase in length of messages by prepending the full nonce, or the
-cost of generating a fresh random nonce for each message is not
-considered likely to cause problems. The random nonces are created by
-the go `crypto/rand` package, which implements a cryptographically
-secure pseudorandom number generator.
+Decryption of a message at the receiver is carried out by NaCl's
+`secretbox.Open` function using the ephemeral session key and a
+nonce. The receiver maintains its own message sequence number, which
+it increments for every message it decrypted successfully. The nonce
+is constructed from that sequence number and the connection
+polarity. As a result the receiver will only be able to decrypt a
+message if it has the expected sequence number. This prevents replay
+attacks.
 
 #### UDP
 
@@ -297,7 +317,7 @@ follows:
     +-----------------------------------+
     | Name of sending peer              |
     +-----------------------------------+
-    | Nonce offset and flags            |
+    | Message Sequence No and flags     |
     +-----------------------------------+
     | NaCl SecretBox overheads          |
     +-----------------------------------+ -+
@@ -330,71 +350,38 @@ follows:
 
 This is very similar to the [non-crypto encapsulation](#encapsulation).
 
-All of the frames are encrypted with the same ephemeral session key
-and all must be decrypted by the receiving peer. Frames which are to
-be forwarded on to some further peer will be re-encrypted with the
-relevant ephemeral session keys for the onward connections. Thus all
-traffic is fully decrypted on every peer it passes through.
-Encryption is again done with the NaCl `secretbox.Seal` function.
+All of the frames on a connection are encrypted with the same
+ephemeral session key, and a nonce constructed from a message sequence
+number, flags and the connection polarity. This is very similar to the
+TCP encryption scheme, and encryption is again done with the NaCl
+`secretbox.Seal` function. The main difference is that the message
+sequence number and flags are transmitted as part of the message,
+unencrypted.
 
-The name of the sending peer enables the receiver to identify the peer
-who sent this UDP packet, and in turn to determine which ephemeral
-session key was used and which nonce, and perform decryption. To avoid
-sending a fresh 192-bit nonce with every UDP packet, which would pose
-an unacceptable overhead, each UDP packet only carries the lowest 15
-bits of the nonce, which is treated as an offset from the "established
-nonce". The lifecycle of a nonce for UDP is:
+The receiver uses the name of the sending peer to determine which
+ephemeral session key and local cryptographic state to use for
+decryption. Frames which are to be forwarded on to some further peer
+will be re-encrypted with the relevant ephemeral session keys for the
+onward connections. Thus all traffic is fully decrypted on every peer
+it passes through.
 
-1. A fresh nonce is generated and the most significant 177 bits are
-sent to the receiving peer over the TCP connection. These upper most
-177 bits are used for the next 2^15 (32768) UDP messages. This is the
-"established nonce".
+Decryption is once again carried out by NaCl's `secretbox.Open`
+function using the ephemeral session key and nonce. The latter is
+constructed from the message sequence number and flags that appeared
+in the unencrypted portion of the received message, and the connection
+polarity.
 
-2. Each UDP message carries the lowest 15 bits as a unique counter. Thus
-the lowest 15 bits are combined with (appended to) the uppermost 177
-bits to form the unique nonce for that message.
+To guard against replay attacks, the receiver maintains some state in
+which it remembers the highest message sequence number seen. It could
+simply reject messages with lower sequence numbers, but that could
+result in excessive message loss when messages are re-ordered. The
+receiver therefore additionally maintains a set of received message
+sequence numbers in a window below the highest number seen, and only
+rejects messages with a sequence number below that window, or
+contained in the set. The window spans at least 2^20 message sequence
+numbers, and hence any re-ordering between the most recent ~1 million
+messages is handled without dropping messages.
 
-3. Once the sending side has sent the 16384'th message on the current
-nonce (50% of the way through the available range), it generates a new
-nonce (upper 177 bits only) and sends that over the TCP connection,
-thus hopefully ensuring it arrives and is ready before it is
-needed. The sending side will switch to using the new nonce once the
-full 32768 messages of the current nonce have been used. At this
-point, the sending side resets the offset to 0.
-
-4. The receiving side must deal with the fact that UDP is unreliable
-and unordered. The receiving side keeps track of the highest offset
-seen for each established nonce. The condition to switching to the new
-nonce it has received via the TCP connection is: the highest offset
-seen for the current nonce must be above 24576 (75% of the available
-range) _and_ the current message just received must have an offset
-less than 8192 (25% of the available range) _and_ the new offset must
-be less than 8192 messages *ahead* of the highest seen offset so far
-(assuming modulo 32768). If these conditions are met, the highest
-offset is set to the offset of the current message and the new nonce
-is used. If those conditions are not met, the receiving side continues
-with the current nonce (according to the rules below), updating the
-highest offset seen as appropriate. Thus in order for the nonce to not
-be updated correctly would require a loss of at least 8192 messages.
-
-5. When the receiving side switches to the new nonce, it does not
-discard the old nonce. If the highest offset seen is below 8192 (25% of
-the range) _and_ the current message offset is above 24576 (75% of the
-range), _and_ the current message offset is less than 8192 behind the
-highest seen nonce (assuming modulo 32768), then the current message is
-decoded with the old nonce.
-
-6. In the remaining cases, the current message is only decoded if its
-offset is within 8192 either side of the highest seen offset.
-
-7. To avoid replay attacks, the receiving side keeps a set of which
-offsets have been used with the current and previous nonce. If the
-offset doesn't exist in the set, and the message can be correctly
-decoded, the offset is added to the relevant set (thus we avoid
-poisoning attacks). If the offset already exists in the set, or the
-message cannot be correctly decoded, the message is not processed
-further.
-
-### Further reading
+### <a name="further-reading"></a>Further reading
 More details on the inner workings of weave can be found in the
 [architecture documentation](https://github.com/weaveworks/weave/blob/master/docs/architecture.txt).

site/index.md

@@ -5,12 +5,12 @@ layout: default
 
 # weave documentation
 
- * [Introduction and example (in README)](https://github.com/weaveworks/weave#readme)
+ * [Introduction, Installation and Example](https://github.com/weaveworks/weave#readme)
+ * [Getting Started Guides](http://weave.works/guides/)
  * [Features](features.html)
+ * [Automatic IP Address Management](ipam.html)
+ * [Automatic Discovery with WeaveDNS](weavedns.html)
  * [Troubleshooting](troubleshooting.html)
  * [Building](building.html)
  * [How it works](how-it-works.html)
  * [Contact information](https://github.com/weaveworks/weave#contact-us)
- * *Installation Notes*
-   * [Using with boot2docker](boot2docker.html)
-   * [Using with systemd](systemd.html)

site/ipam.md

@@ -0,0 +1,139 @@
+---
+title: Automatic IP Address Management
+layout: default
+---
+
+# Automatic IP Address Management
+
+Weave can automatically assign unique IP addresses to each container
+across the network. To make this work, weave must be told on startup
+what range of addresses to allocate from, for example:
+
+    host1# weave launch -iprange 10.2.3.0/24
+
+The `run`, `start`, `attach`, `expose` and `hide` commands will then
+fetch an address automatically if none is specified, i.e.:
+
+    host1# C=$(weave run -ti ubuntu)
+
+You can see which address was allocated with
+[`weave ps`](troubleshooting.html#list-attached-containers):
+
+    host1# weave ps $C
+    a7aff7249393 7a:51:d1:09:21:78 10.2.3.1/24
+
+The `-iprange` parameter is given in [CIDR
+notation](http://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing)
+format - in this example "/24" means the first 24 bits of the address
+form the network address and the allocator is to allocate container
+addresses that all start 10.2.3. The ".0" and ".-1" addresses in a
+subnet are not used, as required by [RFC
+1122](https://tools.ietf.org/html/rfc1122#page-29).
+
+Once you have given a range of addresses to the IP allocator, you must
+not use any addresses in that range for anything else.  If, in our
+example, you subsequently did `weave run 10.2.3.x/24`, you run the
+risk that that address will be allocated to another container, which
+will make network traffic delivery intermittent or non-existent for
+the containers that share the same IP address. No diagnostic message
+is output by weave if you break this rule.
+
+Weave will automatically learn when a container has exited
+and hence can release its IP address.
+
+Weave shares the IP range across all peers, dynamically according to
+their needs.  If a group of peers becomes isolated from the rest (a
+partition), they can continue to work with the IP ranges they had
+before isolation, and can be re-connected to the rest of the network
+and carry on. Note that you must specify the same range with
+`-iprange` on each host, and you cannot mix weaves started with and
+without -iprange.
+
+### Initialisation
+
+Just once, when you start up the whole network, weave needs a majority
+of peers to agree in order to avoid isolated groups starting off
+inconsistently. Therefore, you must either supply the list of all
+peers in the network to `weave launch` or add the `-initpeercount`
+flag to specify how many peers there will be.  It isn't a problem to
+over-estimate by a bit, but if you supply a number that is too small
+then multiple independent groups may form.
+
+To illustrate, suppose you have three hosts, accessible to each other
+as `$HOST1`, `$HOST2` and `$HOST3`. You can start weave on those three
+hosts with these three commands:
+
+    host1$ weave launch -iprange 10.3.0.0/16 $HOST2 $HOST3
+
+    host2$ weave launch -iprange 10.3.0.0/16 $HOST1 $HOST3
+
+    host3$ weave launch -iprange 10.3.0.0/16 $HOST1 $HOST2
+
+Or, if it is not convenient to name all the other hosts at launch
+time, you can give the number of peers like this:
+
+    host1$ weave launch -iprange 10.3.0.0/16 -initpeercount 3
+
+    host2$ weave launch -iprange 10.3.0.0/16 -initpeercount 3 $HOST3
+
+    host3$ weave launch -iprange 10.3.0.0/16 -initpeercount 3 $HOST2
+
+### Stopping and removing peers
+
+You may wish to `weave stop` and re-launch to change some config or to
+upgrade to a new version; provided the underlying protocol hasn't
+changed it will pick up where it left off and learn from peers in the
+network which address ranges it was previously using. If, however, you
+run `weave reset` this will remove the peer from the network so
+if Weave is run again on that node it will start from scratch.
+
+For failed peers, the `weave rmpeer` command can be used to
+permanently remove the ranges allocated to said peer.  This will allow
+other peers to allocate IPs in the ranges previously owner by the rm'd
+peer, and as such should be used with extreme caution - if the rm'd
+peer had transferred some range of IP addresses to another peer but
+this is not known to the whole network, or if it later rejoins
+the Weave network, the same IP address may be allocated twice.
+
+Assuming we had started the three peers in the example earlier, and
+host3 has caught fire, we can go to one of the other hosts and run:
+
+    host1$ weave rmpeer host3
+
+Weave will take all the IP address ranges owned by host3 and transfer
+them to be owned by host1. The name "host3" is resolved via the
+'nickname' feature of weave, which defaults to the local host
+name. Alternatively, one can supply a peer name as shown in `weave
+status`.
+
+## <a name="troubleshooting"></a>Troubleshooting
+
+The command
+
+    weave status
+
+reports on the current status of the weave router and IP allocator:
+
+````
+weave router git-8f675f15c0b5
+...
+Allocator subnet 10.2.1.0/24
+  Free IPs: ~98.0%, 62 local, ~189 remote
+Owned Ranges:
+  10.2.1.1 -> 96:e9:e2:2e:2d:bc (host1) (v3)
+  10.2.1.128 -> ea:84:25:9b:31:2e (host2) (v3)
+  10.2.1.192 -> ea:6c:21:09:cf:f0 (host3) (v9)
+````
+
+The first section covers the router; see the [troubleshooting
+guide](troubleshooting.html#status-report) for full details.
+
+The 'Allocator' section, which is only present if weave has been
+started with the `-iprange` option, summarises the overall position and
+lists which address ranges have been assigned to which peer. Each
+range begins at the address shown and ends just before the next
+address, or wraps around at the end of the subnet. The 'v' number
+denotes how many times that entry has been updated.
+
+The 'Free IPs' information may be out of date with respect to changes
+happening elsewhere in the network.

site/proxy.md

@@ -0,0 +1,108 @@
+---
+title: Weave Proxy
+layout: default
+---
+
+# Proxy
+
+Instead of the `weave` command-line utility, you may prefer to use the
+standard [Docker command-line
+interface](https://docs.docker.com/reference/commandline/cli/), or the
+[Docker remote
+API](https://docs.docker.com/reference/api/docker_remote_api/). The
+weave proxy sits between the `docker` command or API and the Docker
+daemon, so that it can automatically attach containers to the weave
+network.
+
+Using the proxy brings some additional benefits over `weave run`. The
+proxy ensures that the weave network interface is available before
+starting a container's application process. Furthermore, containers
+can be started in foreground mode, and can be automatically removed
+(with the ususal `--rm`).
+
+## Setup
+
+Start the proxy with
+
+    host1$ weave launch-proxy
+
+By default, the proxy will connect to docker at
+`unix:///var/run/docker.sock`, and listen on port 12375. However, we
+can adjust the connection to docker via the `-H` argument. All docker
+commands can be run via the proxy, so it is safe to globally adjust
+your `DOCKER_HOST`.
+
+    host1$ export DOCKER_HOST=tcp://host1:12375
+    host1$ docker ps
+
+## Usage
+
+When containers are created via the weave proxy, their entrypoint will
+be modified to wait for the weave network interface to become
+available. When they are started via the weave proxy, any IP addresses
+and networks specified in the `WEAVE_CIDR` environment variable will
+be attached. We can create and start a container via the weave proxy
+with
+
+    host1$ docker run -e WEAVE_CIDR=10.2.1.1/24 -ti ubuntu /bin/sh
+
+or, equivalently with
+
+    host1$ docker create -e WEAVE_CIDR=10.2.1.1/24 -ti ubuntu /bin/sh
+    5ef831df61d50a1a49272357155a976595e7268e590f0a2c75693337b14e1382
+    host1$ docker start 5ef831df61d50a1a49272357155a976595e7268e590f0a2c75693337b14e1382
+
+Multiple IP addresses and networks can be supplied in the `WEAVE_CIDR`
+variable by space-separating them, as in
+`WEAVE_CIDR="10.2.1.1/24 10.2.2.1/24"`.
+
+## Usage with IPAM
+
+To automatically assign a unique IP address to a container, weave must
+be told on startup what range of addresses to allocate from. For
+example:
+
+    host1# weave launch -iprange 10.2.3.0/24
+    host1$ weave launch-proxy
+    host1$ export DOCKER_HOST=tcp://host1:12375
+
+With this done, we can automatically assign an address to a container
+by providing a blank `WEAVE_CIDR` value, as in
+
+    host1$ docker run -e WEAVE_CIDR= -ti ubuntu /bin/sh
+
+Alternatively, to enable automatic allocation of all containers
+without a `WEAVE_CIDR`, we can launch the proxy with the `--with-ipam`
+option. For example:
+
+    host1$ weave launch-proxy --with-ipam
+    host1$ export DOCKER_HOST=tcp://host1:12375
+
+Now, containers will always be automatically assigned an IP address
+and attached to the weave network when we start them. For example:
+
+    host1$ docker run -ti ubuntu /bin/sh
+
+More details on IPAM can be found in the [IPAM
+documentation](ipam.html).
+
+## Usage with WeaveDNS
+
+Containers started via the proxy can be automatically configured to
+use WeaveDNS for name resolution. To accomplish this we need to launch
+the proxy with the `--with-dns` option
+
+    host1$ weave launch
+    host1$ weave launch-dns 10.2.254.1/24
+    host1$ weave launch-proxy --with-dns
+
+With this done, any containers connected via IPAM or `WEAVE_CIDR=...`,
+which are launched through the proxy will use weaveDNS for name
+resolution. WeaveDNS is used in addition to any dns servers specified
+via the `--dns` option. More details on weaveDNS can be found in the
+[weaveDNS documentation](weavedns.html).
+
+## Limitations
+
+* The proxy does not currently support TLS.
+* If you have a firewall, you will need to make sure port 12375 is open.

site/systemd.md

@@ -15,7 +15,7 @@ normally place it in `/etc/systemd/system/weave.service`.
 
     [Unit]
     Description=Weave Network
-    Documentation=http://weaveworks.github.io/weave/
+    Documentation=http://docs.weave.works/weave/latest_release/
     Requires=docker.service
     After=docker.service
     [Service]
@@ -65,6 +65,6 @@ context with the commands shown below. You will need to have the
     sudo restorecon /usr/local/bin/weave
 
 [readme]: https://github.com/weaveworks/weave/blob/master/README.md#installation
-[connect]: http://weaveworks.github.io/weave/features.html#dynamic-topologies
+[connect]: features.html#dynamic-topologies
 [systemd]: http://www.freedesktop.org/wiki/Software/systemd/
-[encryption]: http://weaveworks.github.io/weave/how-it-works.html#crypto
+[encryption]: how-it-works.html#crypto

site/troubleshooting.md

@@ -21,8 +21,8 @@ Check the weave container logs with
 A reasonable amount of information, and all errors, get logged there.
 
 The log verbosity can be increased by supplying the `-debug` flag when
-launching weave. Be warned, this will log information on a per-packet
-basis, so can produce a lot of output.
+launching weave. To log information on a per-packet basis use
+`-pktdebug` - be warned, this can produce a lot of output.
 
 Another useful debugging technique is to attach standard packet
 capture and analysis tools, such as tcpdump and wireshark, to the
@@ -126,6 +126,10 @@ not currently connected to.  Each line contains some information about
 what went wrong the last time; whether it is attempting to connect or
 is waiting for a while before connecting again.
 
+There may also be further sections for
+[IP allocator](ipam.html#troubleshooting) and
+[weaveDNS](weavedns.html#troubleshooting).
+
 ### <a name="list-attached-containers"></a>List attached containers
 
     weave ps

site/weavedns-design.md

@@ -1,3 +1,8 @@
+---
+title: WeaveDNS Design Notes
+layout: default
+---
+
 # Weave DNS (service discovery) design
 
 The model is that each host has a service that is notified of

site/weavedns.md

@@ -1,3 +1,8 @@
+---
+title: Automatic Discovery with WeaveDNS
+layout: default
+---
+
 # Weave DNS server
 
 The Weave DNS server answers name queries in a Weave network. This
@@ -6,13 +11,24 @@ them hostnames and tell other containers to connect to those names.
 Unlike Docker 'links', this requires no code changes and works across
 hosts.
 
-## Using weaveDNS
+* [Using weaveDNS](#usage)
+* [How it works](#how-it-works)
+* [Adding and removing extra DNS entries](#add-remove)
+* [Hot-swapping service containers](#hot-swapping)
+* [Retaining DNS entries when containers stop](#retain-stopped)
+* [Configuring the domain search path](#domain-search-path)
+* [Using a different local domain](#local-domain)
+* [Using weaveDNS without `weave run`](#without-run)
+* [Troubleshooting](#troubleshooting)
+* [Present limitations](#limitations)
+
+## <a name="usage"></a>Using weaveDNS
 
 WeaveDNS is deployed as a set of containers that communicate with each
 other over the weave network. One such container needs to be started
 on every weave host, by invoking the weave script command
 `launch-dns`. Application containers are then instructed to use
-WeaveDNS as their nameserver by supplying the `--with-dns` option when
+weaveDNS as their nameserver by supplying the `--with-dns` option when
 starting them. Giving any container a hostname in the `.weave.local`
 domain registers it in weaveDNS.  For example:
 
@@ -23,20 +39,41 @@ $ weave run 10.2.1.25/24 -ti -h pingme.weave.local ubuntu
 $ shell1=$(weave run --with-dns 10.2.1.26/24 -ti -h ubuntu.weave.local ubuntu)
 $ docker attach $shell1
 
-# ping pingme
+root@ubuntu:/# ping pingme
 ...
 ```
 
+Note that it is permissible to register multiple containers with the
+same name. weaveDNS picks one address to return when asked for the
+name. Since weaveDNS removes any container that dies, this is a simple
+way to implement redundancy.  In the current implementation it does
+not attempt to do load-balancing.
+
 Each weaveDNS container started with `launch-dns` needs to be given
 its own, unique, IP address, in a subnet that is a) common to all
 weaveDNS containers, b) disjoint from the application subnets, and c)
-not in use on any of the hosts. In our example the weaveDNS address is
-in subnet 10.2.254.0/24 and the application containers are in
-subnet 10.2.1.0/24.
+not in use on any of the hosts.
+
+In our example the weaveDNS address is in subnet 10.2.254.0/24 and the
+application containers are in subnet 10.2.1.0/24. So, to launch and
+use weaveDNS on a second host we would run:
+
+```bash
+host2$ weave launch $HOST1
+host2$ weave launch-dns 10.2.254.2/24
+host2$ shell2=$(weave run --with-dns 10.2.1.36/24 -ti -h ubuntu2.weave.local ubuntu)
+host2$ docker attach $shell2
+
+root@ubuntu2:/# ping pingme
+...
+```
+
+Notice the different IP address in the 10.2.254.0/24 subnet we gave to
+`weave launch-dns`, compared to what we supplied on the first host.
 
 WeaveDNS containers can be stopped with `stop-dns`.
 
-## How it works
+## <a name="how-it-works"></a>How it works
 
 The weaveDNS container running on every host acts as the nameserver
 for containers on that host. It is told about hostnames for local
@@ -52,7 +89,62 @@ When weaveDNS is queried for a name in a domain other than
 `.weave.local`, it queries the host's configured nameserver,
 which is the standard behaviour for Docker containers.
 
-## Domain search paths
+So that containers can connect to a stable and always routable IP
+address, weaveDNS publishes its port 53 to the Docker bridge device,
+which is assumed to be `docker0`. Some configurations may use a
+different Docker bridge device. To supply a different bridge device,
+use the environment variable `DOCKER_BRIDGE`, e.g.,
+
+```bash
+$ sudo DOCKER_BRIDGE=someother weave launch-dns 10.2.254.1/24
+```
+
+In the event that weaveDNS is launched in this way, it's important that
+other calls to `weave` also specify the bridge device:
+
+```bash
+$ sudo DOCKER_BRIDGE=someother weave run --with-dns ...
+```
+
+## <a name="add-remove"></a>Adding and removing extra DNS entries
+
+If you want to give the container a name in DNS *other* than its
+hostname, you can register it using the `dns-add` command. For example:
+
+```bash
+$ shell2=(weave run 10.2.1.27/24 -ti ubuntu)
+$ weave dns-add 10.2.1.28 $shell2 -h pingme2.weave.local
+```
+
+You can also use `dns-add` to add the container's configured hostname
+and domain, simply by omitting `-h <fqdn>`.
+
+The inverse operation can be carried out using the `dns-remove` command:
+
+```bash
+$ weave dns-remove 10.2.1.27 $shell2
+```
+
+## <a name="hot-swapping"></a>Hot-swapping service containers
+
+If you would like to deploy a new version of a service, keep the old
+one running because it has active connections but make all new
+requests go to the new version, then you can simply start the new
+server container and then [remove](#add-remove) the entry for the old
+server container. Later, when all connections to the old server have
+terminated, stop the container as normal.
+
+## <a name="retain-stopped"></a>Retaining DNS entries when containers stop
+
+By default, weaveDNS watches docker events and removes entries for any
+containers that die. You can tell it not to, by adding `--watch=false`
+to the container args:
+
+```bash
+$ weave launch-dns 10.2.254.1/24 --watch=false
+```
+
+## <a name="domain-search-path"></a>Configuring the domain search paths
 
 If you don't supply a domain search path (with `--dns-search=`),
 `weave run ...` tells a container to look for "bare" hostnames, like
@@ -73,31 +165,32 @@ weave run --with-dns 10.2.1.4/24 -ti \
   --dns-search=weave.local ubuntu
 ```
 
-## Doing things more manually
+## <a name="local-domain"></a>Using a different local domain
 
-If you use the `--with-dns` option, `weave run` automatically supplies
-the DNS server address to the new container. And both `weave run` and
-`weave attach` register the hostname of the given container against
-the given weave network IP address.
-
-In some circumstances, you may not want to use the `weave`
-command. You can still take advantage of a running weaveDNS, with some
-extra manual steps.
-
-### Using a different docker bridge
-
-So that containers can connect to a stable and always routable IP
-address, weaveDNS publishes its port 53 to the Docker bridge device,
-which is assumed to be `docker0`.
-
-Some configurations may use a different Docker bridge device. To
-supply a different bridge device, use the environment variable
-`DOCKER_BRIDGE`, e.g.,
+By default, weaveDNS uses `weave.local.` as the domain for names on the
+Weave network. In general users do not need to change this domain, but you
+can force weaveDNS to use a different domain by launching it
+with the `--domain` argument. For example,
 
 ```bash
-$ sudo DOCKER_BRIDGE=someother weave launch-dns 10.2.254.1/24
+$ weave launch-dns 10.2.254.1/24 --domain="mycompany.local."
 ```
 
+The local domain should end with `local.`, since these names are
+link-local as per [RFC6762](https://tools.ietf.org/html/rfc6762),
+(though this is not strictly necessary).
+
+## <a name="without-run"></a>Using weaveDNS without `weave run`
+
+When weaveDNS is running, both `weave run` and `weave attach` register
+the hostname of the given container against the given weave network IP
+address. And if you use the `--with-dns` option, `weave run`
+automatically supplies the DNS server address to the new container.
+
+In some circumstances, you may not want to use the `weave run` command
+to start containers. You can still take advantage of a running
+weaveDNS, with some extra manual steps.
+
 ### Supplying the DNS server
 
 If you want to start containers with `docker run` rather than `weave
@@ -134,65 +227,7 @@ that you can use unqualified hostnames. Use `--dns-search=.` to make
 the resolver use the container's domain, or e.g.,
 `--dns-search=weave.local` to make it look in `weave.local`.
 
-### Using a different local domain
-
-By default, WeaveDNS uses `weave.local.` as the domain for names on the
-Weave network. In general users do not need to change this domain, but you
-can force WeaveDNS to use a different domain by launching it
-with the `--domain` argument. For example,
-
-```bash
-$ weave launch-dns 10.2.254.1/24 --domain="mycompany.local."
-```
-
-The local domain should end with `local.`, since these names are
-link-local as per [RFC6762](https://tools.ietf.org/html/rfc6762),
-(though this is not strictly neccessary).
-
-### Adding containers to DNS
-
-If DNS is started after you've attached a container to the weave
-network, or you want to give the container a name in DNS *other* than
-its hostname, you can register it using the HTTP API:
-
-```bash
-$ docker start $shell2
-$ curl -X PUT "http://$dns_ip:6785/name/$shell2/10.2.1.27" -d fqdn=shell2.weave.local
-```
-
-### Registering multiple containers with the same name
-
-This is supported; weaveDNS picks one address to return when you ask
-for the name. Since weaveDNS removes any container that dies, this is
-a simple way to implement redundancy.  In the current implementation
-it does not attempt to do load-balancing.
-
-### Replacing one container with another at the same name
-
-If you would like to deploy a new version of a service, keep the old one running because it has active connections but make all new requests go to the new version, then you can simply start the new server container and then [unregister](https://github.com/weaveworks/weave/tree/master/weavedns#unregistering) the old one from DNS. And finally, when all connections to the old server have terminated, stop the container as normal.
-
-### Not watching docker events
-
-By default, weaveDNS watches docker events and removes entries for any
-containers that die. You can tell it not to, by adding `--watch=false`
-to the container args:
-
-```bash
-$ weave launch-dns 10.2.254.1/24 --watch=false
-```
-
-### Unregistering
-
-You can manually delete entries for a host, by poking weaveDNS's HTTP
-API with e.g., `curl`:
-
-```bash
-$ docker stop $shell2
-$ dns_ip=$(docker inspect --format='{{ .NetworkSettings.IPAddress }}' weavedns)
-$ curl -X DELETE "http://$dns_ip:6785/name/$shell2/10.2.1.27"
-```
-
-## Troubleshooting
+## <a name="troubleshooting"></a>Troubleshooting
 
 The command
 
@@ -216,8 +251,8 @@ e8d85b1dcdb1 10.2.1.27 waff.weave.local.
 1ab0e8b17c39 10.2.1.29 pong.weave.local.
 ````
 
-The first section covers the router; see the troubleshooting guide in
-the main documentation for more detail.
+The first section covers the router; see the [troubleshooting
+guide](troubleshooting.html#status-report) for more detail.
 
 The second section is pertinent to weaveDNS, and includes:
 
@@ -227,7 +262,7 @@ The second section is pertinent to weaveDNS, and includes:
 * The fallback DNS which will be used to resolve non local names
 * The names known to the local weaveDNS server. Each entry comprises the container ID, IP address and its fully qualified domain name
 
-## Present limitations
+## <a name="limitations"></a>Present limitations
 
  * The server will not know about restarted containers, but if you
    re-attach a restarted container to the weave network, it will be
@@ -237,4 +272,4 @@ The second section is pertinent to weaveDNS, and includes:
    hostnames with the subnets.
  * We use UDP multicast to find out about remote names (from weaveDNS
    servers on other hosts); this likely won't scale well beyond a
-   certain point T.B.D., so we'll have to come up with another scheme.
+   certain point T.B.D.

test/100_cross_hosts_test.sh

@@ -4,21 +4,20 @@
 
 C1=10.2.1.4
 C2=10.2.1.7
+UNIVERSE=10.2.2.0/24
 
-start_suite "Ping over cross-host weave network"
+start_suite "Ping over cross-host weave network (with and without IPAM)"
 
-for HOST in $HOST1 $HOST2; do
-    weave_on $HOST stop || true
-    weave_on $HOST stop-dns || true
-    docker_on $HOST rm -f c1 c2 || true
-done
+weave_on $HOST1 launch -iprange $UNIVERSE
+weave_on $HOST2 launch -iprange $UNIVERSE $HOST1
 
-weave_on $HOST1 launch
-weave_on $HOST2 launch $HOST1
+start_container $HOST1 $C1/24 --name=c1
+start_container $HOST2 $C2/24 --name=c2
+assert_raises "exec_on $HOST1 c1 $PING $C2"
 
-weave_on $HOST2 run $C2/24 -t --name=c2 ubuntu
-weave_on $HOST1 run $C1/24 -t --name=c1 ubuntu
-ok=$(docker -H tcp://$HOST1:2375 exec c1 sh -c "ping -q -c 4 $C2 >&2 && echo ok")
-assert "echo $ok" "ok"
+start_container $HOST1 --name=c3
+start_container $HOST2 --name=c4
+C4=$(container_ip $HOST2 c4)
+assert_raises "exec_on $HOST1 c3 $PING $C4"
 
 end_suite

test/110_encryption_test.sh

@@ -7,18 +7,11 @@ C2=10.2.1.7
 
 start_suite "Ping over encrypted cross-host weave network"
 
-for HOST in $HOST1 $HOST2; do
-    weave_on $HOST stop || true
-    weave_on $HOST stop-dns || true
-    docker_on $HOST rm -f c1 c2 || true
-done
-
 weave_on $HOST1 launch -password notverysecure
 weave_on $HOST2 launch -password notverysecure $HOST1
 
-weave_on $HOST2 run $C2/24 -t --name=c2 ubuntu
-weave_on $HOST1 run $C1/24 -t --name=c1 ubuntu
-ok=$(docker -H tcp://$HOST1:2375 exec c1 sh -c "ping -q -c 4 $C2 >&2 && echo ok")
-assert "echo $ok" "ok"
+start_container $HOST1 $C1/24 --name=c1
+start_container $HOST2 $C2/24 --name=c2
+assert_raises "exec_on $HOST1 c1 $PING $C2"
 
 end_suite

test/130_expose_test.sh

@@ -0,0 +1,81 @@
+#! /bin/bash
+
+. ./config.sh
+
+C1=10.2.1.34
+C2=10.2.1.37
+C3=10.2.2.34
+C4=10.2.2.37
+EXP=10.2.2.101
+UNIVERSE=10.2.3.0/24
+
+weave_on1() {
+    assert_raises "weave_on $HOST1 $@"
+}
+
+run_on1() {
+    assert_raises "run_on   $HOST1 $@"
+}
+
+exec_on1() {
+    assert_raises "exec_on  $HOST1 $@"
+}
+
+# Containers in the same subnet should be able to talk; different subnet not.
+check_container_connectivity() {
+    exec_on1 "c1 $PING $C2"
+    exec_on1 "c3 $PING $C4"
+    exec_on1 "c5 $PING $C6"
+    exec_on1 "c1 sh -c \"! $PING $C3\""
+    exec_on1 "c3 sh -c \"! $PING $C5\""
+    exec_on1 "c5 sh -c \"! $PING $C1\""
+}
+
+start_suite "exposing weave network to host"
+
+# expose/hide with a CIDR work prior to launching weave
+run_on1   "! $PING $EXP"
+weave_on1 "expose  $EXP/24"
+run_on1   "  $PING $EXP"
+weave_on1 "hide    $EXP/24"
+run_on1   "! $PING $EXP"
+
+weave_on $HOST1 launch -iprange $UNIVERSE
+
+start_container $HOST1 $C1/24 --name=c1
+start_container $HOST1 $C2/24 --name=c2
+start_container $HOST1 $C3/24 --name=c3
+start_container $HOST1 $C4/24 --name=c4
+start_container $HOST1        --name=c5
+start_container $HOST1        --name=c6
+C5=$(container_ip $HOST1 c5)
+C6=$(container_ip $HOST1 c6)
+
+# absence of host connectivity by default
+run_on1 "! $PING $C3"
+run_on1 "! $PING $C5"
+check_container_connectivity
+
+# host connectivity after 'expose', and idempotence of 'expose'
+weave_on1 "expose $EXP/24"
+weave_on1 "expose $EXP/24"
+run_on1   "! $PING $C1"
+run_on1   "  $PING $C3"
+run_on1   "! $PING $C5"
+weave_on1 "expose"
+weave_on1 "expose"
+run_on1   "! $PING $C1"
+run_on1   "  $PING $C3"
+run_on1   "  $PING $C5"
+check_container_connectivity
+
+# no host connectivity after 'hide', and idempotence of 'hide'
+weave_on1 "hide $EXP/24"
+weave_on1 "hide $EXP/24"
+run_on1   "! $PING $C3"
+run_on1   "  $PING $C5"
+weave_on1 "hide"
+weave_on1 "hide"
+run_on1   "! $PING $C5"
+
+end_suite

test/140_weave_local_test.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+. ./config.sh
+
+start_suite "Run weave with --local"
+
+run_on $HOST1 sudo ./weave --local reset
+
+run_on $HOST1 sudo ./weave --local launch -iprange 10.2.5.0/24
+assert_raises "docker_on $HOST1 ps | grep weave"
+
+run_on $HOST1 sudo ./weave --local run 10.2.6.5/24 -ti --name=c1 $SMALL_IMAGE /bin/sh
+assert_raises "exec_on $HOST1 c1 $CHECK_ETHWE_UP"
+
+run_on $HOST1 sudo ./weave --local run             -ti --name=c2 $SMALL_IMAGE /bin/sh
+assert_raises "exec_on $HOST1 c2 $CHECK_ETHWE_UP"
+
+end_suite

test/200_dns_test.sh

@@ -4,22 +4,17 @@
 
 C1=10.2.0.78
 C2=10.2.0.34
+NAME=seetwo.weave.local
 
 start_suite "Resolve names on a single host"
 
-weave_on $HOST1 stop || true
-weave_on $HOST1 stop-dns || true
 weave_on $HOST1 launch-dns 10.2.254.1/24
 
-docker_on $HOST1 rm -f c1 c2 || true
+start_container          $HOST1 $C2/24 --name=c2 -h $NAME
+start_container_with_dns $HOST1 $C1/24 --name=c1
 
-weave_on $HOST1 run $C2/24 -t --name=c2 -h seetwo.weave.local ubuntu
-weave_on $HOST1 run --with-dns $C1/24 -t --name=c1 aanand/docker-dnsutils /bin/sh
+assert_dns_record $HOST1 c1 $NAME $C2
 
-ok=$(docker -H tcp://$HOST1:2375 exec c1 sh -c "dig +short seetwo.weave.local")
-assert "echo $ok" "$C2"
-
-ok=$(docker -H tcp://$HOST1:2375 exec c1 sh -c "dig +short -x $C2")
-assert "echo $ok" "seetwo.weave.local."
+assert_raises "exec_on $HOST1 c1 dig MX $NAME | grep -q \"status: NXDOMAIN\""
 
 end_suite

test/210_dns_multicast_test.sh

@@ -4,31 +4,28 @@
 
 C1=10.2.3.78
 C2=10.2.3.34
+UNIVERSE=10.2.4.0/24
+NAME2=seetwo.weave.local
+NAME4=seefour.weave.local
 
-start_suite "Resolve names across hosts"
+start_suite "Resolve names across hosts (with and without IPAM)"
 
-for host in $HOST1 $HOST2; do
-    weave_on $host stop || true
-    weave_on $host stop-dns || true
-    docker_on $host rm -f c1 c2 || true
-done
+weave_on $HOST1 launch -iprange $UNIVERSE
+weave_on $HOST2 launch -iprange $UNIVERSE $HOST1
 
-weave_on $HOST1 launch
-weave_on $HOST2 launch $HOST1
+weave_on $HOST1 launch-dns 10.2.254.1/24
+weave_on $HOST2 launch-dns 10.2.254.2/24
 
-weave_on $HOST1 launch-dns 10.2.254.1/24 -debug
-weave_on $HOST2 launch-dns 10.2.254.2/24 -debug
+start_container          $HOST2 $C2/24 --name=c2 -h $NAME2
+start_container          $HOST2        --name=c4 -h $NAME4
+start_container_with_dns $HOST1 $C1/24 --name=c1
+start_container_with_dns $HOST1        --name=c3
+C4=$(container_ip $HOST2 c4)
 
-weave_on $HOST2 run $C2/24 -t --name=c2 -h seetwo.weave.local ubuntu
-weave_on $HOST1 run --with-dns $C1/24 --name=c1 -t aanand/docker-dnsutils /bin/sh
+assert_dns_record $HOST1 c1 $NAME2 $C2
+assert_dns_record $HOST1 c3 $NAME4 $C4
 
-ok=$(docker -H tcp://$HOST1:2375 exec c1 dig +short seetwo.weave.local)
-assert "echo $ok" "$C2"
-
-ok=$(docker -H tcp://$HOST1:2375 exec c1 dig +short -x $C2)
-assert "echo $ok" "seetwo.weave.local."
-
-ok=$(docker -H tcp://$HOST1:2375 exec c1 dig +short -x 8.8.8.8)
-assert "test -n \"$ok\" && echo pass" "pass"
+assert_raises "exec_on $HOST1 c1 getent hosts 8.8.8.8 | grep google"
+assert_raises "exec_on $HOST1 c3 getent hosts 8.8.8.8 | grep google"
 
 end_suite

test/220_dns_custom_test.sh

@@ -4,22 +4,16 @@
 
 C1=10.2.56.34
 C2=10.2.54.91
+DOMAIN=foo.bar
+NAME=seetwo.$DOMAIN
 
 start_suite "Resolve names in custom domain"
 
-weave_on $HOST1 stop || true
-weave_on $HOST1 stop-dns || true
-weave_on $HOST1 launch-dns 10.2.254.1/24 --domain foo.bar.
+weave_on $HOST1 launch-dns 10.2.254.1/24 --domain $DOMAIN.
 
-docker_on $HOST1 rm -f c1 c2 || true
+start_container          $HOST1 $C2/24 --name=c2 -h $NAME
+start_container_with_dns $HOST1 $C1/24 --name=c1
 
-weave_on $HOST1 run $C2/24 -t --name=c2 -h seetwo.foo.bar ubuntu
-weave_on $HOST1 run --with-dns $C1/24 -t --name=c1 aanand/docker-dnsutils /bin/sh
-
-ok=$(docker -H tcp://$HOST1:2375 exec c1 sh -c "dig +short seetwo.foo.bar.")
-assert "echo $ok" "$C2"
-
-ok=$(docker -H tcp://$HOST1:2375 exec c1 sh -c "dig +short -x $C2")
-assert "echo $ok" "seetwo.foo.bar."
+assert_dns_record $HOST1 c1 $NAME $C2
 
 end_suite

test/230_dns_unqualified_test.sh

@@ -0,0 +1,22 @@
+#! /bin/bash
+
+. ./config.sh
+
+C1=10.2.0.78
+C2=10.2.0.34
+C3=10.2.0.57
+DOMAIN=weave.local
+NAME=seeone.$DOMAIN
+
+start_suite "Resolve unqualified names"
+
+weave_on $HOST1 launch-dns 10.2.254.1/24
+
+start_container          $HOST1 $C1/24 --name=c1 -h $NAME
+start_container_with_dns $HOST1 $C2/24 --name=c2 -h seetwo.$DOMAIN
+start_container_with_dns $HOST1 $C3/24 --name=c3 --dns-search=$DOMAIN
+
+assert "exec_on $HOST1 c2 getent hosts seeone | tr -s ' '" "$C1 $NAME"
+assert "exec_on $HOST1 c3 getent hosts seeone | tr -s ' '" "$C1 $NAME"
+
+end_suite

test/240_dns_add_name_test.sh

@@ -0,0 +1,29 @@
+#! /bin/bash
+
+. ./config.sh
+
+C1=10.2.0.78
+C2=10.2.0.34
+NAME1=seeone.weave.local
+NAME2=seetwo.weave.local
+
+start_suite "Add and remove names on a single host"
+
+weave_on $HOST1 launch-dns 10.2.254.1/24
+
+start_container          $HOST1 $C2/24 --name=c2
+start_container_with_dns $HOST1 $C1/24 --name=c1
+
+weave_on $HOST1 dns-add $C2 c2 -h $NAME2
+
+assert_dns_record $HOST1 c1 $NAME2 $C2
+
+weave_on $HOST1 dns-add $C1 c1 -h $NAME1
+
+assert "exec_on $HOST1 c1 getent hosts $C1 | tr -s ' '" "$C1 $NAME1"
+
+weave_on $HOST1 dns-remove $C1 c1
+
+assert_raises "exec_on $HOST1 c1 getent hosts $NAME1" 2
+
+end_suite