add changelog entry for 0.10.0

see #576.

.gitignore

@@ -6,6 +6,7 @@
 # Folders
 _obj
 _test
+.vagrant
 
 # Architecture specific extensions/prefixes
 *.[568vq]
@@ -22,10 +23,23 @@ _testmain.go
 *.exe
 *.test
 *.prof
+profile.cov
+coverage.html
 
 # Emacs backup files
 *~
 
 # Project specific
+weaver/weaver
 weavedns/weavedns
+sigproxy/sigproxy
+tools/bin
+tools/build
 releases
+weave.tar
+weavedns.tar
+weaveexec.tar
+weaveexec/weave
+weaveexec/sigproxy
+weaveexec/docker*.tgz
+Vagrantfile.local

.travis.yml

@@ -0,0 +1,22 @@
+language: go
+sudo: false
+go:
+  - 1.4
+
+addons:
+  apt:
+    packages:
+    - libpcap-dev
+
+install:
+  - go clean -i net
+  - go install -tags netgo std
+  - make travis
+
+script:
+  - bin/lint .
+  - make tests
+
+after_success:
+  - go get github.com/mattn/goveralls
+  - $HOME/gopath/bin/goveralls -coverprofile=profile.cov -service=travis-ci

CHANGELOG.md

@@ -1,4 +1,72 @@
-== Release 0.7.0
+## Release 0.10.0
 
-This is the first release assigned a version number. See the README at
-https://github.com/zettio/weave#readme for more information.
+**Highlights**:
+
+- **bug fixes**, in particular eradicating several deadlocks that could
+  cause peer connection failures and hangs.
+- **performance and scalability improvements** to the weave control plane,
+  as a result of which it is now possible to construct much larger
+  weave networks than previously.
+- **improved installation and administration**, particularly the
+  introduction of remote execution of the weave script in a container,
+  permitting fully containerised deployment of weave.
+- **improved diagnostics**, such as the reporting of connection failures
+  in `weave status`.
+- **new weaveDNS features**, most notably the caching of DNS records
+  for application containers, which makes finding container IP
+  addresses via weaveDNS much faster.
+
+More detail in the [change log](https://github.com/weaveworks/weave/issues?q=milestone%3A0.10.0)
+
+*NB: This release changes the Weave Docker image names. To upgrade
+from an older version, 1) stop all application containers, 2) run
+`weave reset` from the old version to remove all traces of weave, and
+only then 3) install the new version.*
+
+## Release 0.9.0
+
+- Improve WeaveDNS to the point where it can act as the name server
+  for containers in nearly all situations.
+
+- Diagnose and report peer connectivity more comprehensively.
+
+- Adapt to changes in topology - adding & removing of weave peers,
+  disruption of connectivity - more rapidly.
+
+- Cope with delays in downloading/running docker images/containers
+  required for weave operation.
+
+See the
+[complete change log](https://github.com/weaveworks/weave/issues?q=milestone%3A0.9.0)
+for more details.
+
+## Release 0.8.0
+
+- Align script and image version. When the `weave` script has a
+  version number, i.e. it is part of an official release, it runs
+  docker images matching that version. Thus the script and image
+  versions are automatically aligned. Unversioned/unreleased `weave`
+  scripts run the 'latest'-tagged image versions.
+
+- Eliminate dependency on ethtool and conntrack. Instead of requiring
+  these to be installed on the host, weave invokes them via a
+  `weavetools` docker image that contains minimally packaged versions
+  of these utilities.
+
+- New `weave setup` command. This downloads all docker images used by
+  weave. Invoking this is strictly optional; its main purpose is to
+  facilitate automated installation of weave and preventing delays in
+  subsequent weave command execution due to image downloading.
+
+## Release 0.7.0
+
+This is the first release assigned a version number.
+
+When downloading weave you now have the following choices...
+
+1. a specific version, e.g. https://github.com/weaveworks/weave/releases/download/v0.7.0/weave
+2. latest released version: https://github.com/weaveworks/weave/releases/download/latest_release/weave
+3. most recent 'master' commit: https://raw.githubusercontent.com/weaveworks/weave/master/weave
+
+Previously the only documented download location was (3). We recommend
+that any automated scripts using that be changed to either (1) or (2).

LICENSE

@@ -176,7 +176,7 @@
 
    END OF TERMS AND CONDITIONS
 
-   Copyright 2014 Zettio Ltd.
+   Copyright 2014-2015 Weaveworks Ltd.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

Makefile

@@ -1,22 +1,37 @@
+PUBLISH=publish_weave publish_weavedns publish_weaveexec
+
 .DEFAULT: all
-.PHONY: all publish clean tests
+.PHONY: all update tests publish $(PUBLISH) clean prerequisites build travis
 
 # If you can use docker without being root, you can do "make SUDO="
 SUDO=sudo
 
+DOCKERHUB_USER=weaveworks
 WEAVE_VERSION=git-$(shell git rev-parse --short=12 HEAD)
 WEAVER_EXE=weaver/weaver
 WEAVEDNS_EXE=weavedns/weavedns
-WEAVER_IMAGE=zettio/weave
-WEAVEDNS_IMAGE=zettio/weavedns
-WEAVER_EXPORT=/var/tmp/weave.tar
-WEAVEDNS_EXPORT=/var/tmp/weavedns.tar
+SIGPROXY_EXE=sigproxy/sigproxy
+WEAVER_IMAGE=$(DOCKERHUB_USER)/weave
+WEAVEDNS_IMAGE=$(DOCKERHUB_USER)/weavedns
+WEAVEEXEC_IMAGE=$(DOCKERHUB_USER)/weaveexec
+WEAVER_EXPORT=weave.tar
+WEAVEDNS_EXPORT=weavedns.tar
+WEAVEEXEC_EXPORT=weaveexec.tar
 
-all: $(WEAVER_EXPORT) $(WEAVEDNS_EXPORT)
+WEAVEEXEC_DOCKER_VERSION=1.3.1
+DOCKER_DISTRIB=weaveexec/docker-$(WEAVEEXEC_DOCKER_VERSION).tgz
+DOCKER_DISTRIB_URL=https://get.docker.com/builds/Linux/x86_64/docker-$(WEAVEEXEC_DOCKER_VERSION).tgz
 
-$(WEAVER_EXE) $(WEAVEDNS_EXE):
-	go get -tags netgo ./$(shell dirname $@)
-	go build -ldflags "-extldflags \"-static\" -X main.version $(WEAVE_VERSION)" -tags netgo -o $@ ./$(shell dirname $@)
+all: $(WEAVER_EXPORT) $(WEAVEDNS_EXPORT) $(WEAVEEXEC_EXPORT)
+
+travis: $(WEAVER_EXE) $(WEAVEDNS_EXE)
+
+update:
+	go get -u -f -v -tags -netgo ./$(dir $(WEAVER_EXE)) ./$(dir $(WEAVEDNS_EXE))
+
+$(WEAVER_EXE) $(WEAVEDNS_EXE): common/*.go
+	go get -tags netgo ./$(@D)
+	go build -ldflags "-extldflags \"-static\" -X main.version $(WEAVE_VERSION)" -tags netgo -o $@ ./$(@D)
 	@strings $@ | grep cgo_stub\\\.go >/dev/null || { \
 		rm $@; \
 		echo "\nYour go standard library was built without the 'netgo' build tag."; \
@@ -29,6 +44,11 @@ $(WEAVER_EXE) $(WEAVEDNS_EXE):
 $(WEAVER_EXE): router/*.go weaver/main.go
 $(WEAVEDNS_EXE): nameserver/*.go weavedns/main.go
 
+# Sigproxy needs separate rule as it fails the netgo check in the main
+# build stanza due to not importing net package
+$(SIGPROXY_EXE): sigproxy/main.go
+	go build -o $@ ./$(@D)
+
 $(WEAVER_EXPORT): weaver/Dockerfile $(WEAVER_EXE)
 	$(SUDO) docker build -t $(WEAVER_IMAGE) weaver
 	$(SUDO) docker save $(WEAVER_IMAGE):latest > $@
@@ -37,10 +57,44 @@ $(WEAVEDNS_EXPORT): weavedns/Dockerfile $(WEAVEDNS_EXE)
 	$(SUDO) docker build -t $(WEAVEDNS_IMAGE) weavedns
 	$(SUDO) docker save $(WEAVEDNS_IMAGE):latest > $@
 
-# Add more directories in here as more tests are created
+$(WEAVEEXEC_EXPORT): weaveexec/Dockerfile $(DOCKER_DISTRIB) weave $(SIGPROXY_EXE)
+	cp weave weaveexec/weave
+	cp sigproxy/sigproxy weaveexec/sigproxy
+	cp $(DOCKER_DISTRIB) weaveexec/docker.tgz
+	$(SUDO) docker build -t $(WEAVEEXEC_IMAGE) weaveexec
+	$(SUDO) docker save $(WEAVEEXEC_IMAGE):latest > $@
+
+$(DOCKER_DISTRIB):
+	curl -o $(DOCKER_DISTRIB) $(DOCKER_DISTRIB_URL)
+
 tests:
-	cd nameserver; go test -tags netgo
+	echo "mode: count" > profile.cov
+	fail=0 ;                                                                              \
+	for dir in $$(find . -type f -name '*_test.go' | xargs -n1 dirname | sort -u); do     \
+	    output=$$(mktemp cover.XXXXXXXXXX) ;                                              \
+	    if ! go test -tags netgo -covermode=count -coverprofile=$$output $$dir ; then     \
+            fail=1 ;                                                                          \
+        fi ;                                                                                  \
+	    if [ -f $$output ]; then                                                          \
+	        tail -n +2 <$$output >>profile.cov;                                           \
+	        rm $$output;                                                                  \
+	    fi                                                                                \
+	done ;                                                                                \
+	exit $$fail
+	go tool cover -html=profile.cov -o=coverage.html
+
+$(PUBLISH): publish_%:
+	$(SUDO) docker tag -f $(DOCKERHUB_USER)/$* $(DOCKERHUB_USER)/$*:$(WEAVE_VERSION)
+	$(SUDO) docker push   $(DOCKERHUB_USER)/$*:$(WEAVE_VERSION)
+	$(SUDO) docker push   $(DOCKERHUB_USER)/$*:latest
+
+publish: $(PUBLISH)
 
 clean:
-	-$(SUDO) docker rmi $(WEAVER_IMAGE) $(WEAVEDNS_IMAGE)
-	rm -f $(WEAVER_EXE) $(WEAVEDNS_EXE) $(WEAVER_EXPORT) $(WEAVEDNS_EXPORT)
+	-$(SUDO) docker rmi $(WEAVER_IMAGE) $(WEAVEDNS_IMAGE) $(WEAVEEXEC_IMAGE)
+	rm -f $(WEAVER_EXE) $(WEAVEDNS_EXE) $(SIGPROXY_EXE) $(WEAVER_EXPORT) $(WEAVEDNS_EXPORT) $(WEAVEEXEC_EXPORT)
+
+build:
+	$(SUDO) go clean -i net
+	$(SUDO) go install -tags netgo std
+	$(MAKE)

NOTICE

@@ -1,3 +1,3 @@
 Weave
-Copyright 2012-2014 Zettio Ltd.
-This product includes software developed at Zettio Ltd.
+Copyright 2014-2015 Weaveworks Ltd.
+This product includes software developed at Weaveworks Ltd.

README.md

@@ -1,5 +1,7 @@
 # Weave - the Docker network
 
+[![Build Status](https://travis-ci.org/weaveworks/weave.svg?branch=master)](https://travis-ci.org/weaveworks/weave) [![Coverage Status](https://coveralls.io/repos/weaveworks/weave/badge.svg)](https://coveralls.io/r/weaveworks/weave)
+
 Weave creates a virtual network that connects Docker containers
 deployed across multiple hosts.
 
@@ -27,26 +29,17 @@ capabilities, so these can continue to be used by containers.
 
 ## Installation
 
-To run weave on a host, you need to install...
+Ensure you are running Linux (kernel 3.5 or later) and have Docker
+(version 1.3.1 or later) installed. Then install weave with
 
-1. Linux and Docker. We've tested with Docker versions 0.9.1 through
-   1.3.1, but other versions should work too. Linux kernels after 3.5
-   are known to work; the newer the better.
-2. weave. Install this with
+    sudo wget -O /usr/local/bin/weave \
+      https://github.com/weaveworks/weave/releases/download/latest_release/weave
+    sudo chmod a+x /usr/local/bin/weave
 
-        sudo wget -O /usr/local/bin/weave \
-          https://raw.githubusercontent.com/zettio/weave/master/weave
-        sudo chmod a+x /usr/local/bin/weave
+CoreOS users see [here](https://github.com/fintanr/weave-gs/blob/master/coreos-simple/user-data) for an example of installing weave using cloud-config.
 
-3. (recommended) ethtool. On many systems this is installed already;
-   if not then grab it via your favourite package manager. On some
-   systems, weave application container networking may not operate
-   correctly unless ethtool is available.
-
-4. (optional) conntrack. Install this via your favourite package
-   manager. Without conntrack, the weave network may not re-establish
-   itself fully when individual weave instances are stopped (with
-   `weave stop`) and restarted quickly (typically within ~3 minutes).
+Weave respects the environment variable `DOCKER_HOST`, so you can run
+it locally to control a weave network on a remote host.
 
 ## Quick Start Screencast
 
@@ -63,10 +56,16 @@ two containers, one on each host.
 On `$HOST1` run (as root)
 
     host1# weave launch
-    host1# C=$(weave run 10.0.1.1/24 -t -i ubuntu)
+    host1# C=$(weave run 10.2.1.1/24 -t -i ubuntu)
 
 The first line starts the weave router, in a container. This needs to
-be done once on each host.
+be done once on each host. The required docker image for the weave
+router container is downloaded automatically. There is also a `weave
+setup` command for downloading this and other images required for
+weave operation; this is a strictly optional step which is especially
+useful for automated installation of weave and ensures that any
+subsequent weave commands do not encounter delays due to image
+downloading.
 
 The second line runs our application container. We give it an IP
 address and network, in
@@ -89,21 +88,22 @@ addresses must, of course, be unique.
 We repeat similar steps on `$HOST2`...
 
     host2# weave launch $HOST1
-    host2# C=$(weave run 10.0.1.2/24 -t -i ubuntu)
+    host2# C=$(weave run 10.2.1.2/24 -t -i ubuntu)
 
 The only difference, apart from the choice of IP address for the
 application container, is that we tell our weave that it should peer
-with the weave on `$HOST1` (specified as the IP address or hostname, and
-optional `:port`, by which `$HOST2` can reach it). NB: if there is a
-firewall between `$HOST1` and `$HOST2`, you must open port 6783 for TCP
-and UDP.
+with the weave on `$HOST1` (specified as the IP address or hostname,
+and optional `:port`, by which `$HOST2` can reach it). NB: if there is
+a firewall between `$HOST1` and `$HOST2`, you must open the weave port
+(6783 by default; this can be overriden by setting `WEAVE_PORT`) for
+TCP and UDP.
 
 Note that we could instead have told the weave on `$HOST1` to connect to
 `$HOST2`, or told both about each other. Order does not matter here;
 weave automatically (re)connects to peers when they become
 available. Also, we can tell weave to connect to multiple peers by
 supplying multiple addresses, separated by spaces. And we can
-[add peers dynamically](http://zettio.github.io/weave/features.html#dynamic-topologies).
+[add peers dynamically](http://weaveworks.github.io/weave/features.html#dynamic-topologies).
 
 Now that we've got everything set up, let's see whether our containers
 can talk to each other...
@@ -111,18 +111,18 @@ can talk to each other...
 On `$HOST1`...
 
     host1# docker attach $C
-    root@28841bd02eff:/# ping -c 1 -q 10.0.1.2
-    PING 10.0.1.2 (10.0.1.2): 48 data bytes
-    --- 10.0.1.2 ping statistics ---
+    root@28841bd02eff:/# ping -c 1 -q 10.2.1.2
+    PING 10.2.1.2 (10.2.1.2): 48 data bytes
+    --- 10.2.1.2 ping statistics ---
     1 packets transmitted, 1 packets received, 0% packet loss
     round-trip min/avg/max/stddev = 1.048/1.048/1.048/0.000 ms
 
 Similarly, on `$HOST2`...
 
     host2# docker attach $C
-    root@f76829496120:/# ping -c 1 -q 10.0.1.1
-    PING 10.0.1.1 (10.0.1.1): 48 data bytes
-    --- 10.0.1.1 ping statistics ---
+    root@f76829496120:/# ping -c 1 -q 10.2.1.1
+    PING 10.2.1.1 (10.2.1.1): 48 data bytes
+    --- 10.2.1.1 ping statistics ---
     1 packets transmitted, 1 packets received, 0% packet loss
     round-trip min/avg/max/stddev = 1.034/1.034/1.034/0.000 ms
 
@@ -131,22 +131,22 @@ to each other.
 
 ## Find out more
 
- * [Documentation homepage](http://zettio.github.io/weave/)
- * [Features](http://zettio.github.io/weave/features.html)
- * [Troubleshooting](http://zettio.github.io/weave/troubleshooting.html)
- * [Building](http://zettio.github.io/weave/building.html)
- * [How it works](http://zettio.github.io/weave/how-it-works.html)
- * [Preview of weaveDNS](http://weaveblog.com/2014/11/04/have-you-met-weavedns/)
+ * [Documentation homepage](http://weaveworks.github.io/weave/)
+ * [Features](http://weaveworks.github.io/weave/features.html)
+ * [Troubleshooting](http://weaveworks.github.io/weave/troubleshooting.html)
+ * [Building](http://weaveworks.github.io/weave/building.html)
+ * [How it works](http://weaveworks.github.io/weave/how-it-works.html)
+ * [WeaveDNS README](https://github.com/weaveworks/weave/tree/master/weavedns#readme)
 
 ## Contact Us
 
 Found a bug, want to suggest a feature, or have a question?
-[File an issue](https://github.com/zettio/weave/issues), or email
+[File an issue](https://github.com/weaveworks/weave/issues), or email
 help@weave.works. When reporting a bug, please include which version of
 weave you are running, as shown by `weave version`.
 
 Follow weave on Twitter:
-[@weavenetwork](https://twitter.com/weavenetwork).
+[@weaveworks](https://twitter.com/weaveworks).
 
 Read the Weave blog:
 [Weaveblog](http://weaveblog.com/).

Vagrantfile

@@ -1,27 +1,42 @@
 VAGRANTFILE_API_VERSION = "2"
 
 vm_ip = "172.16.0.3" # arbitrary private IP
-pkgs = "lxc-docker golang ethtool libpcap-dev git mercurial"
+pkgs = "lxc-docker aufs-tools build-essential ethtool iputils-arping libpcap-dev git mercurial bc"
 
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
+
   config.vm.box = "phusion/ubuntu-14.04-amd64"
   config.vm.box_url = "https://oss-binaries.phusionpassenger.com/vagrant/boxes/latest/ubuntu-14.04-amd64-vbox.box"
 
   config.vm.network "private_network", ip: vm_ip
+  config.vm.provider :virtualbox do |vb|
+    vb.customize ["modifyvm", :id, "--natdnshostresolver1", "off"]
+    vb.customize ["modifyvm", :id, "--natdnsproxy1", "off"]
+  end
 
-  config.vm.synced_folder "./", "/home/vagrant/src/github.com/zettio/weave"
+  config.vm.synced_folder ".", "/vagrant", disabled: true
+  config.vm.synced_folder ".", "/home/vagrant/src/github.com/weaveworks/weave"
+  config.vm.provision :shell, :inline => "ln -s /home/vagrant/src/github.com/weaveworks/weave /home/vagrant/weave"
 
   config.vm.provision :shell, :inline => "sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 36A1D7869245C8950F966E92D8576A8BA88D21E9"
   config.vm.provision :shell, :inline => "echo deb https://get.docker.io/ubuntu docker main > /etc/apt/sources.list.d/docker.list"
 
-  pkg_cmd = "apt-get update -qq; " \
-  "apt-get install -q -y --force-yes --no-install-recommends "
-  pkg_cmd << pkgs
+  pkg_cmd = "export DEBIAN_FRONTEND=noninteractive; " \
+    "apt-get update -qq; " \
+    "apt-get install -qq --no-install-recommends #{pkgs}"
+
+  install_go_toolchain = "curl -s https://storage.googleapis.com/golang/go1.4.2.linux-amd64.tar.gz | tar xz -C /usr/local"
+
   config.vm.provision :shell, :inline => pkg_cmd
+  config.vm.provision :shell, :inline => install_go_toolchain
   config.vm.provision :shell, :inline => "usermod -a -G docker vagrant; "
   config.vm.provision :shell, :inline => "echo export GOPATH=/home/vagrant >> /home/vagrant/.bashrc"
+  config.vm.provision :shell, :inline => "echo export PATH=/usr/local/go/bin:${PATH} >> /home/vagrant/.bashrc"
   config.vm.provision :shell, :inline => "chown -R vagrant:vagrant /home/vagrant/src"
-  config.vm.provision :shell, :inline => "sudo go install -a -tags netgo std"
+  config.vm.provision :shell, :inline => "/usr/local/go/bin/go clean -i net; /usr/local/go/bin/go install -tags netgo std"
+
+  config.vm.provision :shell, :inline => "echo 'DOCKER_OPTS=\"-H unix:///var/run/docker.sock -H tcp://0.0.0.0:2375\"' >> /etc/default/docker"
+  config.vm.provision :shell, :inline => "service docker restart"
 
 end
 

bin/lint

@@ -0,0 +1,81 @@
+#!/bin/bash
+# This scipt lints go files for common errors.
+#
+# Its runs gofmt and go vet, and optionally golint and
+# gocyclo, if they are installed.
+#
+# With no arguments, it lints the current files staged
+# for git commit.  Or you can pass it explicit filenames
+# (or directories) and it will lint them.
+#
+# To use this script automatically, run:
+#   ln -s ../../bin/lint .git/hooks/pre-commit
+
+set -eu
+
+function lint_go {
+	filename="$1"
+	result=0
+
+	if [ -n "$(gofmt -l "${filename}")" ]; then
+		result=1
+		echo "${filename}: run gofmt -w ${filename}!"
+	fi
+
+	go tool vet "${filename}" || result=$?
+
+	# golint is completely optional.  If you don't like it
+	# don't have it installed.
+	if type golint >/dev/null 2>&1; then
+		# golint doesn't set an exit code it seems
+		lintoutput=$(golint "${filename}")
+		if [ "$lintoutput" != "" ]; then
+			result=1
+			echo "$lintoutput"
+		fi
+	fi
+
+	# gocyclo is completely optional.  If you don't like it
+	# don't have it installed.  Also never blocks a commit,
+	# it just warns.
+	if type gocyclo >/dev/null 2>&1; then
+		gocyclo -over 25 "${filename}" | while read line; do
+			echo "${filename}": higher than 25 cyclomatic complexity - "${line}"
+		done
+	fi
+
+	return $result
+}
+
+function lint {
+	filename="$1"
+	ext="${filename##*\.}"
+
+	# Don't lint this script
+	if [ "${filename}" = "bin/lint" ]; then
+		return
+	fi
+
+	case "$ext" in
+		go) lint_go "${filename}"
+		;;
+	esac
+}
+
+function lint_files {
+	lint_result=0
+	while read filename; do
+		lint "${filename}" || lint_result=1
+	done
+	exit $lint_result
+}
+
+function list_files {
+	if [ $# -gt 0 ]; then
+		find "$@" -type f
+	else
+		git diff --cached --name-only
+	fi
+}
+
+list_files "$@" | lint_files

bin/merge-into-gh-pages

@@ -2,7 +2,7 @@
 set -e
 
 # For info on why/how to use this script, see
-# https://github.com/zettio/weave/wiki/WorkingOnWeave#working-on-docs
+# https://github.com/weaveworks/weave/wiki/WorkingOnWeave#working-on-docs
 
 MERGE_ARGS="-X subtree=site"
 
@@ -10,7 +10,7 @@ merge_dir=$(mktemp -d -t weave-merge.XXXXXX)/weave
 git clone -b gh-pages . $merge_dir
 
 pushd $merge_dir
-git pull --ff-only git@github.com:zettio/weave gh-pages
+git pull --ff-only git@github.com:weaveworks/weave gh-pages
 git fetch origin master
 git merge $MERGE_ARGS -e -m "Merge master into gh-pages" origin/master
 

bin/multiweave

@@ -0,0 +1,45 @@
+#!/bin/bash
+set -e
+
+# You may need to adjust
+#    echo 4096 >/proc/sys/net/ipv4/neigh/default/gc_thresh3
+# where the number is greater than NUM_WEAVES*(NUM_WEAVES-1) plus some
+# fudge.
+#
+# Without that, some connections between the weaves may fail to get
+# established, and networking on the host in general becomes
+# broken. There will also be some "net_ratelimit: nnn callbacks
+# suppressed" messages in the syslog.
+
+START=${FIRST_WEAVE:-1}
+COUNT=${NUM_WEAVES:-28}
+FINISH=$((START+COUNT-1))
+
+weavedir=$(dirname $0)/..
+
+case "$1" in
+    launch)
+        shift 1
+        for i in $(seq $START $FINISH); do
+            WEAVE_CONTAINER_NAME=weave$i $weavedir/weave test-launch "$@"
+        done
+        if [ $START -ne 1 ] ; then
+            status=$(docker inspect --format='{{.State.Running}} {{.NetworkSettings.IPAddress}}' weave$((START-1)))
+            PREV="${status#true }"
+        fi
+        for i in $(seq $START $FINISH); do
+            if [ -n "$PREV" ] ; then
+                echo connecting weave$i to $PREV
+                WEAVE_CONTAINER_NAME=weave$i $weavedir/weave connect $PREV
+            fi
+            status=$(docker inspect --format='{{.State.Running}} {{.NetworkSettings.IPAddress}}' weave$i)
+            PREV="${status#true }"
+        done
+        ;;
+    stop)
+        for i in $(seq $START $FINISH); do
+            echo Stopping weave$i
+            WEAVE_CONTAINER_NAME=weave$i $weavedir/weave stop
+        done
+        ;;
+esac

bin/publish-gh-pages

@@ -1,3 +1,3 @@
 #! /bin/sh
 set -e
-git push git@github.com:zettio/weave gh-pages:gh-pages
+git push git@github.com:weaveworks/weave gh-pages:gh-pages

bin/release

@@ -2,11 +2,13 @@
 
 set -e
 
+SUDO=${SUDO-sudo}
+GITHUB_USER=${GITHUB_USER:-weaveworks}
+DOCKERHUB_USER=${DOCKERHUB_USER:-weaveworks}
+RELEASE_NAME=${RELEASE_NAME:-"Weave"}
+RELEASE_DESCRIPTION=${RELEASE_DESCRIPTION:-"Weaving Containers into Applications"}
+
 PWD=`pwd`
-GITHUB_USER=zettio
-DOCKERHUB_USER=zettio
-RELEASE_NAME="Weave"
-RELEASE_DESCRIPTION="Weaving Containers into Applications"
 
 setup() {
     ## Get the new version number from the most recent tag
@@ -37,7 +39,7 @@ build() {
     cd $RELEASE_DIR
 
     ## Check that the top changelog entry is this version
-    if ! latest_changelog=$(grep -o "== Release [0-9].*" -m1 ./CHANGELOG.md) || \
+    if ! latest_changelog=$(grep -o "## Release [0-9].*" -m1 ./CHANGELOG.md) || \
        ! [ `echo "$latest_changelog" | grep -o '[0-9][^ ]*'` == "$VERSION" ]; then
         echo -e "\u2757 Latest changelog entry \"$latest_changelog\" does not match the release version $VERSION" >&2
         exit 1
@@ -49,10 +51,7 @@ build() {
     ## Inject the version numbers and build the distributables
     ## (library versions?)
     sed -i "/SCRIPT_VERSION=/ c\SCRIPT_VERSION=\"$VERSION\"" ./weave
-    make \
-        WEAVE_VERSION=$VERSION \
-        WEAVER_IMAGE=$DOCKERHUB_USER/weave \
-        WEAVEDNS_IMAGE=$DOCKERHUB_USER/weavedns
+    make SUDO=$SUDO WEAVE_VERSION=$VERSION DOCKERHUB_USER=$DOCKERHUB_USER
 
     if make tests; then
         echo -e '\u2713 Tests pass'
@@ -121,14 +120,8 @@ publish() {
 
     echo '** Sanity checks OK for publishing tag' $LATEST_TAG as $DOCKERHUB_USER/weave:$VERSION
 
-    echo == Tagging $DOCKERHUB_USER/weave and $DOCKERHUB_USER/weavedns as $VERSION
-    docker tag $DOCKERHUB_USER/weave $DOCKERHUB_USER/weave:$VERSION
-    docker tag $DOCKERHUB_USER/weavedns $DOCKERHUB_USER/weavedns:$VERSION
-
-    docker push $DOCKERHUB_USER/weave:$VERSION
-    echo "** Pushed $DOCKERHUB_USER/weave:$VERSION to docker hub"
-    docker push $DOCKERHUB_USER/weavedns:$VERSION
-    echo "** Pushed $DOCKERHUB_USER/weavedns:$VERSION to docker hub"
+    echo "== Tagging and publishing images on docker hub as user $DOCKERHUB_USER"
+    make SUDO=$SUDO WEAVE_VERSION=$VERSION DOCKERHUB_USER=$DOCKERHUB_USER publish
 
     echo "== Creating GitHub release $RELEASE_NAME $VERSION"
     # This cannot be done as a draft because of a bug in
@@ -173,7 +166,7 @@ publish() {
         --file "./weave"
 
     echo "** Published $RELEASE_NAME as latest_release at"
-    echo -e "\thttps://github.com/$GITHUB_USER/weave/releases/latest_release"    
+    echo -e "\thttps://github.com/$GITHUB_USER/weave/releases/latest_release"
 }
 
 usage() {

build/Dockerfile

@@ -0,0 +1,21 @@
+FROM ubuntu
+
+RUN apt-get -y update && apt-get -y install --no-install-recommends build-essential git ca-certificates docker.io mercurial libpcap-dev curl make pkg-config gcc bison flex
+
+# When doing a build in a container, "apt-get update" happens twice,
+# which can be a very significant overhead for incremental builds.
+# And it's unnecessary, because if any of the prerequisites do change,
+# the right thing to do is to rebuild the container image.  Rather
+# than suppressing the "apt-get update" commands explicitly, we clear
+# out sources.list so that 'apt-get update' doesn't do anything.
+RUN echo >/etc/apt/sources.list
+
+ENV GO_VERSION 1.4.2
+RUN curl -sSL https://golang.org/dl/go${GO_VERSION}.linux-amd64.tar.gz | tar -C /usr/local -xz
+ENV PATH /usr/local/go/bin:$PATH
+
+# So that sudo does not trample on PATH
+RUN sed -ie '/secure_path/d' /etc/sudoers
+
+COPY build.sh /
+ENTRYPOINT ["sh", "/build.sh"]

build/build.sh

@@ -0,0 +1,44 @@
+#!/bin/sh
+set -e
+
+GOPATH=/home/go
+export GOPATH
+
+WEAVE_SRC=$GOPATH/src/github.com/weaveworks/weave
+
+if [ $# -eq 0 ] ; then
+    # No arguments.  Expect that the weave repo will be bind-mounted
+    # into $GOPATH
+    if ! [ -e $WEAVE_SRC ] ; then
+        cat 2>&1 <<EOF
+No container arguments supplied, and nothing at ${WEAVE_SRC}.  Please
+either bind-mount the golang workspace containing weave with the
+docker run -v option, e.g.:
+
+    $ docker run -v <host gopath>:${GOPATH} \\
+          -v /var/run/docker.sock:/var/run/docker.sock weaveworks/weave-build
+
+Or supply git clone arguments to retrieve it, e.g.:
+
+    $ docker run -v /var/run/docker.sock:/var/run/docker.sock \\
+          weaveworks/weave-build https://github.com/weaveworks/weave.git
+EOF
+        exit 1
+    fi
+
+    # If we run make directly, any files created on the bind mount
+    # will have awkward ownership.  So we switch to a user with the
+    # same user and group IDs as source directory.  We have to set a
+    # few things up so that sudo works without complaining later on.
+    uid=$(stat --format="%u" $WEAVE_SRC)
+    gid=$(stat --format="%g" $WEAVE_SRC)
+    echo "weave:x:$uid:$gid::$WEAVE_SRC:/bin/sh" >>/etc/passwd
+    echo "weave:*:::::::" >>/etc/shadow
+    echo "weave	ALL=(ALL)	NOPASSWD: ALL" >>/etc/sudoers
+    su weave -c "PATH=$PATH make -C $WEAVE_SRC build"
+else
+    # There are arguments to pass to git-clone
+    mkdir -p ${WEAVE_SRC%/*}
+    git clone "$@" $WEAVE_SRC
+    make -C $WEAVE_SRC build
+fi

common/errors.go

@@ -0,0 +1,13 @@
+package common
+
+func CheckFatal(e error) {
+	if e != nil {
+		Error.Fatal(e)
+	}
+}
+
+func CheckWarn(e error) {
+	if e != nil {
+		Warning.Println(e)
+	}
+}

common/init.go

@@ -0,0 +1,10 @@
+package common
+
+import (
+	"io/ioutil"
+	"os"
+)
+
+func init() {
+	InitLogging(ioutil.Discard, os.Stdout, os.Stdout, os.Stderr)
+}

common/logging.go

@@ -1,4 +1,4 @@
-package nameserver
+package common
 
 import (
 	"io"
@@ -32,6 +32,10 @@ func InitLogging(debugHandle io.Writer,
 	Error = log.New(errorHandle, "ERROR: ", standardLogFlags)
 }
 
-func init() {
-	InitLogging(ioutil.Discard, os.Stdout, os.Stdout, os.Stderr)
+func InitDefaultLogging(debug bool) {
+	debugOut := ioutil.Discard
+	if debug {
+		debugOut = os.Stderr
+	}
+	InitLogging(debugOut, os.Stdout, os.Stdout, os.Stderr)
 }

common/updater/updater.go

@@ -0,0 +1,47 @@
+package updater
+
+import (
+	"github.com/fsouza/go-dockerclient"
+	. "github.com/weaveworks/weave/common"
+)
+
+type ContainerObserver interface {
+	ContainerDied(ident string) error
+}
+
+func checkError(err error, apiPath string) {
+	if err != nil {
+		Error.Fatalf("[updater] Unable to connect to Docker API on %s: %s",
+			apiPath, err)
+	}
+}
+
+func Start(apiPath string, ob ContainerObserver) error {
+	client, err := docker.NewClient(apiPath)
+	checkError(err, apiPath)
+
+	env, err := client.Version()
+	checkError(err, apiPath)
+
+	events := make(chan *docker.APIEvents)
+	err = client.AddEventListener(events)
+	checkError(err, apiPath)
+
+	Info.Printf("[updater] Using Docker API on %s: %v", apiPath, env)
+
+	go func() {
+		for event := range events {
+			handleEvent(ob, event, client)
+		}
+	}()
+	return nil
+}
+
+func handleEvent(ob ContainerObserver, event *docker.APIEvents, client *docker.Client) error {
+	switch event.Status {
+	case "die":
+		id := event.ID
+		ob.ContainerDied(id)
+	}
+	return nil
+}

docs/architecture.txt

@@ -83,13 +83,17 @@ Router:
     local peer. It will tell the local connection (communicating to
     the actor thread) about the UDP address of the remote peer. The
     local connection will then start its forwarder threads as
-    described in 6a, and start sending slow heartbeats. The local connection
-    is marked as established which means it is included in network updates
-    broadcast to our peers. We also send to the remote peer via TCP a
-    ConnectionEstablished message. The remote peer receives this (on
-    the TCP receiver process), tells the connection actor process,
-    which then replaces the fast heartbeater with a slow heartbeater
-    and similarly marks the connection as established.
+    described in 6a, and start sending fast heartbeats. We send to the
+    remote peer via TCP a ConnectionEstablished message. The remote
+    peer receives this (on the TCP receiver process), tells the
+    connection actor process, which then replaces the fast heartbeater
+    with a slow heartbeater and marks the connection as established
+    (which means it is included in network updates broadcast to our
+    peers).
+  6c. When the connection initiator receives the fast heartbeat from
+    the remote peer it sends to the remote peer via a TCP a
+    ConnectionEstablished message. This is handled by the remote peer
+    as described in 6b.
   7. Whenever a connection is established or terminated, the local
     peer's version is incremented. Whenever this happens, the peer
     generates a network update message which is broadcast to its

docs/dockerless.txt

@@ -126,7 +126,7 @@ If all is well, you should now be able to launch weave on VM1
 then on VM2:
 
 VM1> weaver -iface veth6 -name vm1
-VM2> weaver -iface veth6 -name vm2 192.168.122.1
+VM2> weaver -iface veth6 -name vm2 192.168.122.2
 
 Once the connection is up and running, you should be able to connect
 from VM1/ns2 to VM2/ns2 and vice versa:

nameserver/cache.go

@@ -0,0 +1,323 @@
+package nameserver
+
+import (
+	"container/heap"
+	"errors"
+	"github.com/miekg/dns"
+	. "github.com/weaveworks/weave/common"
+	"math"
+	"math/rand"
+	"sync"
+	"time"
+)
+
+var (
+	errInvalidCapacity = errors.New("Invalid cache capacity")
+	errCouldNotResolve = errors.New("Could not resolve")
+	errTimeout         = errors.New("Timeout while waiting for resolution")
+	errNoLocalReplies  = errors.New("No local replies")
+)
+
+const (
+	defPendingTimeout int = 5 // timeout for a resolution
+)
+
+const nullTTL = 0 // a null TTL
+
+type entryStatus uint8
+
+const (
+	stPending  entryStatus = iota // someone is waiting for the resolution
+	stResolved entryStatus = iota // resolved
+)
+
+const (
+	CacheNoLocalReplies uint8 = 1 << iota // not found in local network (stored in the cache so we skip another local lookup or some time)
+)
+
+// shuffleAnswers reorders answers for very basic load balancing
+func shuffleAnswers(answers []dns.RR) []dns.RR {
+	n := len(answers)
+	if n > 1 {
+		rand.Seed(time.Now().UTC().UnixNano())
+
+		for i := 0; i < n; i++ {
+			r := i + rand.Intn(n-i)
+			answers[r], answers[i] = answers[i], answers[r]
+		}
+	}
+
+	return answers
+}
+
+// a cache entry
+type cacheEntry struct {
+	Status   entryStatus // status of the entry
+	Flags    uint8       // some extra flags
+	ReplyLen int
+
+	question dns.Question
+	protocol dnsProtocol
+	reply    dns.Msg
+
+	validUntil time.Time // obtained from the reply and stored here for convenience/speed
+	putTime    time.Time
+
+	index int // for fast lookups in the heap
+}
+
+func newCacheEntry(question *dns.Question, now time.Time) *cacheEntry {
+	e := &cacheEntry{
+		Status:     stPending,
+		validUntil: now.Add(time.Second * time.Duration(defPendingTimeout)),
+		question:   *question,
+		index:      -1,
+	}
+
+	return e
+}
+
+// Get a copy of the reply stored in the entry, but with some values adjusted like the TTL
+func (e *cacheEntry) getReply(request *dns.Msg, maxLen int, now time.Time) (*dns.Msg, error) {
+	if e.Status != stResolved {
+		return nil, nil
+	}
+
+	// if the reply has expired or is invalid, force the caller to start a new resolution
+	if e.hasExpired(now) {
+		return nil, nil
+	}
+
+	if e.Flags&CacheNoLocalReplies != 0 {
+		return nil, errNoLocalReplies
+	}
+
+	if e.ReplyLen >= maxLen {
+		Debug.Printf("[cache msgid %d] returning truncated reponse: %d > %d", request.MsgHdr.Id, e.ReplyLen, maxLen)
+		return makeTruncatedReply(request), nil
+	}
+
+	// create a copy of the reply, with values for this particular query
+	reply := e.reply.Copy()
+	reply.SetReply(request)
+
+	// adjust the TTLs
+	passedSecs := uint32(now.Sub(e.putTime).Seconds())
+	for _, rr := range reply.Answer {
+		hdr := rr.Header()
+		ttl := hdr.Ttl
+		if passedSecs < ttl {
+			hdr.Ttl = ttl - passedSecs
+		} else {
+			return nil, nil // it is expired: do not spend more time and return nil...
+		}
+	}
+
+	reply.Rcode = e.reply.Rcode
+	reply.Authoritative = true
+
+	// shuffle the values, etc...
+	reply.Answer = shuffleAnswers(reply.Answer)
+
+	return reply, nil
+}
+
+func (e cacheEntry) hasExpired(now time.Time) bool {
+	return e.validUntil.Before(now) || e.validUntil == now
+}
+
+// set the reply for the entry
+// returns True if the entry has changed the validUntil time
+func (e *cacheEntry) setReply(reply *dns.Msg, ttl int, flags uint8, now time.Time) bool {
+	var prevValidUntil time.Time
+	if e.Status == stResolved {
+		if reply != nil {
+			Debug.Printf("[cache msgid %d] replacing response in cache", reply.MsgHdr.Id)
+		}
+		prevValidUntil = e.validUntil
+	}
+
+	e.Status = stResolved
+	e.Flags = flags
+	e.putTime = now
+
+	if ttl != nullTTL {
+		e.validUntil = now.Add(time.Second * time.Duration(ttl))
+	} else if reply != nil {
+		// calculate the validUntil from the reply TTL
+		var minTTL uint32 = math.MaxUint32
+		for _, rr := range reply.Answer {
+			ttl := rr.Header().Ttl
+			if ttl < minTTL {
+				minTTL = ttl // TODO: improve the minTTL calculation (maybe we should skip some RRs)
+			}
+		}
+		e.validUntil = now.Add(time.Second * time.Duration(minTTL))
+	}
+
+	if reply != nil {
+		e.reply = *reply
+		e.ReplyLen = reply.Len()
+	}
+
+	return (prevValidUntil != e.validUntil)
+}
+
+//////////////////////////////////////////////////////////////////////////////////////
+
+// An entriesPtrHeap is a min-heap of cache entries.
+type entriesPtrsHeap []*cacheEntry
+
+func (h entriesPtrsHeap) Len() int           { return len(h) }
+func (h entriesPtrsHeap) Less(i, j int) bool { return h[i].validUntil.Before(h[j].validUntil) }
+func (h entriesPtrsHeap) Swap(i, j int) {
+	h[i], h[j] = h[j], h[i]
+	h[i].index = i
+	h[j].index = j
+}
+
+func (h *entriesPtrsHeap) Push(x interface{}) {
+	// Push and Pop use pointer receivers because they modify the slice's length,
+	// not just its contents.
+	n := len(*h)
+	entry := x.(*cacheEntry)
+	entry.index = n
+	*h = append(*h, entry)
+}
+
+func (h *entriesPtrsHeap) Pop() interface{} {
+	old := *h
+	n := len(old)
+	item := old[n-1]
+	item.index = -1 // for safety
+	*h = old[0 : n-1]
+	return item
+}
+
+//////////////////////////////////////////////////////////////////////////////////////
+
+type cacheKey dns.Question
+type entries map[cacheKey]*cacheEntry
+
+// Cache is a thread-safe fixed capacity LRU cache.
+type Cache struct {
+	Capacity int
+
+	entries  entries
+	entriesH entriesPtrsHeap // len(entriesH) <= len(entries), as pending entries can be in entries but not in entriesH
+	lock     sync.RWMutex
+}
+
+// NewCache creates a cache of the given capacity
+func NewCache(capacity int) (*Cache, error) {
+	if capacity <= 0 {
+		return nil, errInvalidCapacity
+	}
+	c := &Cache{
+		Capacity: capacity,
+		entries:  make(entries, capacity),
+	}
+
+	heap.Init(&c.entriesH)
+	return c, nil
+}
+
+// Clear removes all the entries in the cache
+func (c *Cache) Clear() {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+
+	c.entries = make(entries, c.Capacity)
+	heap.Init(&c.entriesH)
+}
+
+// Purge removes the old elements in the cache
+func (c *Cache) Purge(now time.Time) {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+
+	for i, entry := range c.entriesH {
+		if entry.hasExpired(now) {
+			heap.Remove(&c.entriesH, i)
+			delete(c.entries, cacheKey(entry.question))
+		} else {
+			return // all remaining entries must be still valid...
+		}
+	}
+}
+
+// Add adds a reply to the cache.
+func (c *Cache) Put(request *dns.Msg, reply *dns.Msg, ttl int, flags uint8, now time.Time) int {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+
+	question := request.Question[0]
+	key := cacheKey(question)
+	ent, found := c.entries[key]
+	if found {
+		updated := ent.setReply(reply, ttl, flags, now)
+		if updated {
+			heap.Fix(&c.entriesH, ent.index)
+		}
+	} else {
+		// If we will add a new item and the capacity has been exceeded, make some room...
+		if len(c.entriesH) >= c.Capacity {
+			lowestEntry := heap.Pop(&c.entriesH).(*cacheEntry)
+			delete(c.entries, cacheKey(lowestEntry.question))
+		}
+		ent = newCacheEntry(&question, now)
+		ent.setReply(reply, ttl, flags, now)
+		heap.Push(&c.entriesH, ent)
+		c.entries[key] = ent
+	}
+	return ent.ReplyLen
+}
+
+// Look up for a question's reply from the cache.
+// If no reply is stored in the cache, it returns a `nil` reply and no error. The caller can then `Wait()`
+// for another goroutine `Put`ing a reply in the cache.
+func (c *Cache) Get(request *dns.Msg, maxLen int, now time.Time) (reply *dns.Msg, err error) {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+
+	question := request.Question[0]
+	key := cacheKey(question)
+	if ent, found := c.entries[key]; found {
+		reply, err = ent.getReply(request, maxLen, now)
+		if ent.hasExpired(now) {
+			Debug.Printf("[cache msgid %d] expired: removing", request.MsgHdr.Id)
+			if ent.index > 0 {
+				heap.Remove(&c.entriesH, ent.index)
+			}
+			delete(c.entries, key)
+			reply = nil
+		}
+	} else {
+		// we are the first asking for this name: create an entry with no reply... the caller must wait
+		Debug.Printf("[cache msgid %d] addind in pending state", request.MsgHdr.Id)
+		c.entries[key] = newCacheEntry(&question, now)
+	}
+	return
+}
+
+// Remove removes the provided question from the cache.
+func (c *Cache) Remove(question *dns.Question) {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+
+	key := cacheKey(*question)
+	if entry, found := c.entries[key]; found {
+		if entry.index > 0 {
+			heap.Remove(&c.entriesH, entry.index)
+		}
+		delete(c.entries, key)
+	}
+}
+
+// Len returns the number of entries in the cache.
+func (c *Cache) Len() int {
+	c.lock.RLock()
+	defer c.lock.RUnlock()
+
+	return len(c.entries)
+}

nameserver/cache_test.go

@@ -0,0 +1,206 @@
+package nameserver
+
+import (
+	"fmt"
+	"github.com/miekg/dns"
+	. "github.com/weaveworks/weave/common"
+	wt "github.com/weaveworks/weave/testing"
+	"net"
+	"testing"
+	"time"
+)
+
+// Check that the cache keeps its intended capacity constant
+func TestCacheLength(t *testing.T) {
+	InitDefaultLogging(true)
+
+	const cacheLen = 128
+
+	l, err := NewCache(cacheLen)
+	wt.AssertNoErr(t, err)
+
+	insTime := time.Now()
+
+	t.Logf("Inserting 256 questions in the cache at '%s', with TTL from 0 to 255", insTime)
+	for i := 0; i < cacheLen*2; i++ {
+		questionMsg := new(dns.Msg)
+		questionMsg.SetQuestion(fmt.Sprintf("name%d", i), dns.TypeA)
+		questionMsg.RecursionDesired = true
+
+		question := &questionMsg.Question[0]
+
+		ip := net.ParseIP(fmt.Sprintf("10.0.1.%d", i))
+		ips := []net.IP{ip}
+		reply := makeAddressReply(questionMsg, question, ips)
+		reply.Answer[0].Header().Ttl = uint32(i)
+
+		l.Put(questionMsg, reply, 0, 0, insTime)
+	}
+
+	wt.AssertEqualInt(t, l.Len(), cacheLen, "cache length")
+
+	minExpectedTime := insTime.Add(time.Duration(cacheLen) * time.Second)
+	t.Logf("Checking all remaining entries expire after insert_time + %d secs='%s'", cacheLen, minExpectedTime)
+	for _, entry := range l.entries {
+		if entry.validUntil.Before(minExpectedTime) {
+			t.Fatalf("Entry valid until %s", entry.validUntil)
+		}
+	}
+}
+
+// Check that the cache entries are ok
+func TestCacheEntries(t *testing.T) {
+	InitDefaultLogging(true)
+
+	Info.Println("Checking cache consistency")
+
+	const cacheLen = 128
+
+	l, err := NewCache(cacheLen)
+	wt.AssertNoErr(t, err)
+
+	questionMsg := new(dns.Msg)
+	questionMsg.SetQuestion("some.name", dns.TypeA)
+	questionMsg.RecursionDesired = true
+
+	question := &questionMsg.Question[0]
+
+	t.Logf("Trying to get a name")
+	resp, err := l.Get(questionMsg, minUDPSize, time.Now())
+	wt.AssertNoErr(t, err)
+	if resp != nil {
+		t.Logf("Got\n%s", resp)
+		t.Fatalf("ERROR: Did not expect a reponse from Get() yet")
+	}
+	t.Logf("Trying to get it again")
+	resp, err = l.Get(questionMsg, minUDPSize, time.Now())
+	wt.AssertNoErr(t, err)
+	if resp != nil {
+		t.Logf("Got\n%s", resp)
+		t.Fatalf("ERROR: Did not expect a reponse from Get() yet")
+	}
+
+	t.Logf("Inserting the reply")
+	reply1 := makeAddressReply(questionMsg, question, []net.IP{net.ParseIP("10.0.1.1")})
+	l.Put(questionMsg, reply1, nullTTL, 0, time.Now())
+
+	timeGet1 := time.Now()
+	t.Logf("Checking we can Get() the reply now")
+	resp, err = l.Get(questionMsg, minUDPSize, timeGet1)
+	wt.AssertNoErr(t, err)
+	wt.AssertTrue(t, resp != nil, "reponse from Get()")
+	t.Logf("Received '%s'", resp.Answer[0])
+	wt.AssertType(t, resp.Answer[0], (*dns.A)(nil), "DNS record")
+	ttlGet1 := resp.Answer[0].Header().Ttl
+
+	timeGet2 := timeGet1.Add(time.Duration(1) * time.Second)
+	t.Logf("Checking that a second Get(), after 1 second, gets the same result, but with reduced TTL")
+	resp, err = l.Get(questionMsg, minUDPSize, timeGet2)
+	wt.AssertNoErr(t, err)
+	wt.AssertTrue(t, resp != nil, "reponse from a second Get()")
+	t.Logf("Received '%s'", resp.Answer[0])
+	wt.AssertType(t, resp.Answer[0], (*dns.A)(nil), "DNS record")
+	ttlGet2 := resp.Answer[0].Header().Ttl
+	wt.AssertEqualInt(t, int(ttlGet1-ttlGet2), 1, "TTL difference")
+
+	timeGet3 := timeGet1.Add(time.Duration(localTTL) * time.Second)
+	t.Logf("Checking that a third Get(), after %d second, gets no result", localTTL)
+	resp, err = l.Get(questionMsg, minUDPSize, timeGet3)
+	wt.AssertNoErr(t, err)
+	if resp != nil {
+		t.Logf("Got\n%s", resp)
+		t.Fatalf("ERROR: Did NOT expect a reponse from the second Get()")
+	}
+
+	t.Logf("Checking that an Remove() results in Get() returning nothing")
+	replyTemp := makeAddressReply(questionMsg, question, []net.IP{net.ParseIP("10.0.9.9")})
+	l.Put(questionMsg, replyTemp, nullTTL, 0, time.Now())
+	lenBefore := l.Len()
+	l.Remove(question)
+	wt.AssertEqualInt(t, l.Len(), lenBefore-1, "cache length")
+	l.Remove(question) // do it again: should have no effect...
+	wt.AssertEqualInt(t, l.Len(), lenBefore-1, "cache length")
+
+	resp, err = l.Get(questionMsg, minUDPSize, timeGet1)
+	wt.AssertNoErr(t, err)
+	wt.AssertTrue(t, resp == nil, "reponse from the Get() after a Remove()")
+
+	t.Logf("Inserting a two replies for the same query")
+	timePut2 := time.Now()
+	reply2 := makeAddressReply(questionMsg, question, []net.IP{net.ParseIP("10.0.1.2")})
+	l.Put(questionMsg, reply2, nullTTL, 0, timePut2)
+	timePut3 := timePut2.Add(time.Duration(1) * time.Second)
+	reply3 := makeAddressReply(questionMsg, question, []net.IP{net.ParseIP("10.0.1.3")})
+	l.Put(questionMsg, reply3, nullTTL, 0, timePut3)
+
+	t.Logf("Checking we get the last one...")
+	resp, err = l.Get(questionMsg, minUDPSize, timePut3)
+	wt.AssertNoErr(t, err)
+	wt.AssertTrue(t, resp != nil, "reponse from the Get()")
+	t.Logf("Received '%s'", resp.Answer[0])
+	wt.AssertType(t, resp.Answer[0], (*dns.A)(nil), "DNS record")
+	wt.AssertEqualString(t, resp.Answer[0].(*dns.A).A.String(), "10.0.1.3", "IP address")
+	wt.AssertEqualInt(t, int(resp.Answer[0].Header().Ttl), int(localTTL), "TTL")
+
+	resp, err = l.Get(questionMsg, minUDPSize, timePut3.Add(time.Duration(localTTL-1)*time.Second))
+	wt.AssertNoErr(t, err)
+	wt.AssertTrue(t, resp != nil, "reponse from the Get()")
+	t.Logf("Received '%s'", resp.Answer[0])
+	wt.AssertType(t, resp.Answer[0], (*dns.A)(nil), "DNS record")
+	wt.AssertEqualString(t, resp.Answer[0].(*dns.A).A.String(), "10.0.1.3", "IP address")
+	wt.AssertEqualInt(t, int(resp.Answer[0].Header().Ttl), 1, "TTL")
+
+	t.Logf("Checking we get empty replies when they are expired...")
+	lenBefore = l.Len()
+	resp, err = l.Get(questionMsg, minUDPSize, timePut3.Add(time.Duration(localTTL)*time.Second))
+	wt.AssertNoErr(t, err)
+	if resp != nil {
+		t.Logf("Got\n%s", resp.Answer[0])
+		t.Fatalf("ERROR: Did NOT expect a reponse from the Get()")
+	}
+	wt.AssertEqualInt(t, l.Len(), lenBefore-1, "cache length (after getting an expired entry)")
+
+	questionMsg2 := new(dns.Msg)
+	questionMsg2.SetQuestion("some.other.name", dns.TypeA)
+	questionMsg2.RecursionDesired = true
+	question2 := &questionMsg2.Question[0]
+
+	t.Logf("Trying to Get() a name")
+	resp, err = l.Get(questionMsg2, minUDPSize, time.Now())
+	wt.AssertNoErr(t, err)
+	wt.AssertNil(t, resp, "reponse from Get() yet")
+
+	t.Logf("Checking that an Remove() between Get() and Put() does not break things")
+	replyTemp2 := makeAddressReply(questionMsg2, question2, []net.IP{net.ParseIP("10.0.9.9")})
+	l.Remove(question2)
+	l.Put(questionMsg2, replyTemp2, nullTTL, 0, time.Now())
+	resp, err = l.Get(questionMsg2, minUDPSize, time.Now())
+	wt.AssertNoErr(t, err)
+	wt.AssertNotNil(t, resp, "reponse from Get()")
+
+	questionMsg3 := new(dns.Msg)
+	questionMsg3.SetQuestion("some.other.name", dns.TypeA)
+	questionMsg3.RecursionDesired = true
+	question3 := &questionMsg3.Question[0]
+
+	t.Logf("Checking that a entry with CacheNoLocalReplies return an error")
+	timePut3 = time.Now()
+	l.Put(questionMsg3, nil, nullTTL, CacheNoLocalReplies, timePut3)
+	resp, err = l.Get(questionMsg3, minUDPSize, timePut3)
+	wt.AssertNil(t, resp, "Get() response with CacheNoLocalReplies")
+	wt.AssertNotNil(t, err, "Get() error with CacheNoLocalReplies")
+
+	timeExpiredGet3 := timePut3.Add(time.Second * time.Duration(negLocalTTL+1))
+	t.Logf("Checking that we get an expired response after %f seconds", timeExpiredGet3.Sub(timePut3).Seconds())
+	resp, err = l.Get(questionMsg3, minUDPSize, timeExpiredGet3)
+	wt.AssertNil(t, resp, "expired Get() response with CacheNoLocalReplies")
+	wt.AssertNil(t, err, "expired Get() error with CacheNoLocalReplies")
+
+	l.Remove(question3)
+	t.Logf("Checking that Put&Get with CacheNoLocalReplies with a Remove in the middle returns nothing")
+	l.Put(questionMsg3, nil, nullTTL, CacheNoLocalReplies, time.Now())
+	l.Remove(question3)
+	resp, err = l.Get(questionMsg3, minUDPSize, time.Now())
+	wt.AssertNil(t, resp, "Get() reponse with CacheNoLocalReplies")
+	wt.AssertNil(t, err, "Get() error with CacheNoLocalReplies")
+}

nameserver/dns.go

@@ -6,8 +6,10 @@ import (
 )
 
 const (
-	localTTL uint32 = 300 // somewhat arbitrary; we don't expect anyone
-	// downstream to cache results
+	localTTL    uint32 = 30 // somewhat arbitrary; we don't expect anyone downstream to cache results
+	negLocalTTL        = 30 // TTL for negative local resolutions
+	minUDPSize         = 512
+	maxUDPSize         = 65535
 )
 
 func makeHeader(r *dns.Msg, q *dns.Question) *dns.RR_Header {
@@ -24,6 +26,14 @@ func makeReply(r *dns.Msg, as []dns.RR) *dns.Msg {
 	return m
 }
 
+func makeTruncatedReply(r *dns.Msg) *dns.Msg {
+	// for truncated response, we create a minimal reply with the Truncated bit set
+	reply := new(dns.Msg)
+	reply.SetReply(r)
+	reply.Truncated = true
+	return reply
+}
+
 func makeAddressReply(r *dns.Msg, q *dns.Question, addrs []net.IP) *dns.Msg {
 	answers := make([]dns.RR, len(addrs))
 	header := makeHeader(r, q)
@@ -32,12 +42,12 @@ func makeAddressReply(r *dns.Msg, q *dns.Question, addrs []net.IP) *dns.Msg {
 		switch q.Qtype {
 		case dns.TypeA:
 			if ip4 := addr.To4(); ip4 != nil {
-				answers[count] = &dns.A{*header, addr}
+				answers[count] = &dns.A{Hdr: *header, A: addr}
 				count++
 			}
 		case dns.TypeAAAA:
 			if ip4 := addr.To4(); ip4 == nil {
-				answers[count] = &dns.AAAA{*header, addr}
+				answers[count] = &dns.AAAA{Hdr: *header, AAAA: addr}
 				count++
 			}
 		}
@@ -49,7 +59,34 @@ func makePTRReply(r *dns.Msg, q *dns.Question, names []string) *dns.Msg {
 	answers := make([]dns.RR, len(names))
 	header := makeHeader(r, q)
 	for i, name := range names {
-		answers[i] = &dns.PTR{*header, name}
+		answers[i] = &dns.PTR{Hdr: *header, Ptr: name}
 	}
 	return makeReply(r, answers)
 }
+
+func makeDNSFailResponse(r *dns.Msg) *dns.Msg {
+	m := new(dns.Msg)
+	m.SetReply(r)
+	m.RecursionAvailable = true
+	m.Rcode = dns.RcodeNameError
+	return m
+}
+
+func makeDNSNotImplResponse(r *dns.Msg) *dns.Msg {
+	m := new(dns.Msg)
+	m.SetReply(r)
+	m.RecursionAvailable = true
+	m.Rcode = dns.RcodeNotImplemented
+	return m
+}
+
+// get the maximum UDP-reply length
+func getMaxReplyLen(r *dns.Msg, proto dnsProtocol) int {
+	maxLen := minUDPSize
+	if proto == protTCP {
+		maxLen = maxUDPSize
+	} else if opt := r.IsEdns0(); opt != nil {
+		maxLen = int(opt.UDPSize())
+	}
+	return maxLen
+}

nameserver/http.go

@@ -1,107 +1,95 @@
 package nameserver
 
 import (
-	"errors"
 	"fmt"
+	"github.com/gorilla/mux"
 	"github.com/miekg/dns"
-	"io"
+	. "github.com/weaveworks/weave/common"
 	"log"
 	"net"
 	"net/http"
-	"strings"
 )
 
-// Parse a URL of the form PUT /name/<identifier>/<ip-address>
-func parseUrl(url string) (identifier string, ipaddr string, err error) {
-	parts := strings.Split(url, "/")
-	if len(parts) != 4 {
-		return "", "", errors.New("Unable to parse url: " + url)
-	}
-	return parts[2], parts[3], nil
-}
-
 func httpErrorAndLog(level *log.Logger, w http.ResponseWriter, msg string,
 	status int, logmsg string, logargs ...interface{}) {
 	http.Error(w, msg, status)
-	level.Printf(logmsg, logargs...)
+	level.Printf("[http] "+logmsg, logargs...)
 }
 
-func ListenHttp(domain string, db Zone, port int) {
+func ListenHTTP(version string, server *DNSServer, domain string, db Zone, port int) {
 
-	http.HandleFunc("/status", func(w http.ResponseWriter, r *http.Request) {
-		if r.Method != "GET" {
-			http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
-		} else {
-			io.WriteString(w, "ok")
-		}
+	muxRouter := mux.NewRouter()
+
+	muxRouter.Methods("GET").Path("/status").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintln(w, "weave DNS", version)
+		fmt.Fprintln(w, server.Status())
 	})
 
-	http.HandleFunc("/name/", func(w http.ResponseWriter, r *http.Request) {
-
+	muxRouter.Methods("PUT").Path("/name/{id:.+}/{ip:.+}").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		reqError := func(msg string, logmsg string, logargs ...interface{}) {
-			httpErrorAndLog(Warning, w, msg, http.StatusBadRequest,
-				logmsg, logargs...)
+			httpErrorAndLog(Warning, w, msg, http.StatusBadRequest, logmsg, logargs...)
 		}
 
-		switch r.Method {
-		case "PUT":
-			ident, ipStr, err := parseUrl(r.URL.Path)
-			name := r.FormValue("fqdn")
-			if ident == "" || ipStr == "" || name == "" {
-				reqError("Invalid request", "Invalid request: %s, %s", r.URL, r.Form)
-				return
-			}
+		vars := mux.Vars(r)
+		idStr := vars["id"]
+		ipStr := vars["ip"]
+		name := r.FormValue("fqdn")
 
-			ip := net.ParseIP(ipStr)
-			if ip == nil {
-				reqError("Invalid IP", "Invalid IP in request: %s", ipStr)
-				return
-			}
+		if name == "" {
+			reqError("Invalid FQDN", "Invalid FQDN in request: %s, %s", r.URL, r.Form)
+			return
+		}
 
-			if dns.IsSubDomain(domain, name) {
-				Info.Printf("Adding %s -> %s", name, ipStr)
-				if err = db.AddRecord(ident, name, ip); err != nil {
-					if _, ok := err.(DuplicateError); !ok {
-						httpErrorAndLog(
-							Error, w, "Internal error", http.StatusInternalServerError,
-							"Unexpected error from DB: %s", err)
-						return
-					} // oh, I already know this. whatever.
-				}
-			} else {
-				Info.Printf("Ignoring name %s, not in %s", name, domain)
-			}
+		ip := net.ParseIP(ipStr)
+		if ip == nil {
+			reqError("Invalid IP", "Invalid IP in request: %s", ipStr)
+			return
+		}
 
-		case "DELETE":
-			ident, ipStr, err := parseUrl(r.URL.Path)
-			if ident == "" || ipStr == "" {
-				reqError("Invalid Request", "Invalid request: %s, %s", r.URL, r.Form)
-				return
-			}
-
-			ip := net.ParseIP(ipStr)
-			if ip == nil {
-				reqError("Invalid IP in request", "Invalid IP in request: %s", ipStr)
-				return
-			}
-			Info.Printf("Deleting %s (%s)", ident, ipStr)
-			if err = db.DeleteRecord(ident, ip); err != nil {
-				if _, ok := err.(LookupError); !ok {
+		if dns.IsSubDomain(domain, name) {
+			Info.Printf("[http] Adding %s -> %s", name, ipStr)
+			if err := db.AddRecord(idStr, name, ip); err != nil {
+				if _, ok := err.(DuplicateError); !ok {
 					httpErrorAndLog(
 						Error, w, "Internal error", http.StatusInternalServerError,
 						"Unexpected error from DB: %s", err)
 					return
-				}
+				} // oh, I already know this. whatever.
 			}
-		default:
-			msg := "Unexpected http method: " + r.Method
-			reqError(msg, msg)
-			return
+		} else {
+			Info.Printf("[http] Ignoring name %s, not in %s", name, domain)
 		}
 	})
 
+	muxRouter.Methods("DELETE").Path("/name/{id:.+}/{ip:.+}").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		reqError := func(msg string, logmsg string, logargs ...interface{}) {
+			httpErrorAndLog(Warning, w, msg, http.StatusBadRequest, logmsg, logargs...)
+		}
+
+		vars := mux.Vars(r)
+		idStr := vars["id"]
+		ipStr := vars["ip"]
+
+		ip := net.ParseIP(ipStr)
+		if ip == nil {
+			reqError("Invalid IP in request", "Invalid IP in request: %s", ipStr)
+			return
+		}
+		Info.Printf("[http] Deleting %s (%s)", idStr, ipStr)
+		if err := db.DeleteRecord(idStr, ip); err != nil {
+			if _, ok := err.(LookupError); !ok {
+				httpErrorAndLog(
+					Error, w, "Internal error", http.StatusInternalServerError,
+					"Unexpected error from DB: %s", err)
+				return
+			}
+		}
+	})
+
+	http.Handle("/", muxRouter)
+
 	address := fmt.Sprintf(":%d", port)
 	if err := http.ListenAndServe(address, nil); err != nil {
-		Error.Fatal("Unable to create http listener: ", err)
+		Error.Fatal("[http] Unable to create http listener: ", err)
 	}
 }

nameserver/http_test.go

@@ -2,6 +2,7 @@ package nameserver
 
 import (
 	"fmt"
+	wt "github.com/weaveworks/weave/testing"
 	"math/rand"
 	"net"
 	"net/http"
@@ -22,64 +23,64 @@ func TestHttp(t *testing.T) {
 		containerID     = "deadbeef"
 		testDomain      = "weave.local."
 		successTestName = "test1." + testDomain
-		testAddr1       = "10.0.2.1/24"
+		testAddr1       = "10.2.2.1/24"
 		dockerIP        = "9.8.7.6"
 	)
 
 	var zone = new(ZoneDb)
 	port := rand.Intn(10000) + 32768
 	fmt.Println("Http test on port", port)
-	go ListenHttp(testDomain, zone, port)
+	go ListenHTTP("", nil, testDomain, zone, port)
 
 	time.Sleep(100 * time.Millisecond) // Allow for http server to get going
 
 	// Ask the http server to add our test address into the database
 	addrParts := strings.Split(testAddr1, "/")
-	addrUrl := fmt.Sprintf("http://localhost:%d/name/%s/%s", port, containerID, addrParts[0])
-	resp, err := genForm("PUT", addrUrl,
+	addrURL := fmt.Sprintf("http://localhost:%d/name/%s/%s", port, containerID, addrParts[0])
+	resp, err := genForm("PUT", addrURL,
 		url.Values{"fqdn": {successTestName}, "local_ip": {dockerIP}, "routing_prefix": {addrParts[1]}})
-	assertNoErr(t, err)
-	assertStatus(t, resp.StatusCode, http.StatusOK, "http response")
+	wt.AssertNoErr(t, err)
+	wt.AssertStatus(t, resp.StatusCode, http.StatusOK, "http response")
 
 	// Check that the address is now there.
-	foundIP, err := zone.LookupLocal(successTestName)
-	assertNoErr(t, err)
+	foundIP, err := zone.LookupName(successTestName)
+	wt.AssertNoErr(t, err)
 	ip, _, _ := net.ParseCIDR(testAddr1)
 	if !foundIP.Equal(ip) {
 		t.Fatal("Unexpected result for", successTestName, foundIP)
 	}
 
 	// Adding exactly the same address should be OK
-	resp, err = genForm("PUT", addrUrl,
+	resp, err = genForm("PUT", addrURL,
 		url.Values{"fqdn": {successTestName}, "local_ip": {dockerIP}, "routing_prefix": {addrParts[1]}})
-	assertNoErr(t, err)
-	assertStatus(t, resp.StatusCode, http.StatusOK, "http success response for duplicate add")
+	wt.AssertNoErr(t, err)
+	wt.AssertStatus(t, resp.StatusCode, http.StatusOK, "http success response for duplicate add")
 
 	// Now try adding the same address again with a different ident --
 	// again should be fine
-	otherUrl := fmt.Sprintf("http://localhost:%d/name/%s/%s", port, "other", addrParts[0])
-	resp, err = genForm("PUT", otherUrl,
+	otherURL := fmt.Sprintf("http://localhost:%d/name/%s/%s", port, "other", addrParts[0])
+	resp, err = genForm("PUT", otherURL,
 		url.Values{"fqdn": {successTestName}, "local_ip": {dockerIP}, "routing_prefix": {addrParts[1]}})
-	assertNoErr(t, err)
-	assertStatus(t, resp.StatusCode, http.StatusOK, "http response")
+	wt.AssertNoErr(t, err)
+	wt.AssertStatus(t, resp.StatusCode, http.StatusOK, "http response")
 
 	// Delete the address
-	resp, err = genForm("DELETE", addrUrl, nil)
-	assertNoErr(t, err)
-	assertStatus(t, resp.StatusCode, http.StatusOK, "http response")
+	resp, err = genForm("DELETE", addrURL, nil)
+	wt.AssertNoErr(t, err)
+	wt.AssertStatus(t, resp.StatusCode, http.StatusOK, "http response")
 
 	// Check that the address is still resolvable.
-	_, err = zone.LookupLocal(successTestName)
-	assertNoErr(t, err)
+	_, err = zone.LookupName(successTestName)
+	wt.AssertNoErr(t, err)
 
 	// Delete the address record mentioning the other container
-	resp, err = genForm("DELETE", otherUrl, nil)
-	assertNoErr(t, err)
-	assertStatus(t, resp.StatusCode, http.StatusOK, "http response")
+	resp, err = genForm("DELETE", otherURL, nil)
+	wt.AssertNoErr(t, err)
+	wt.AssertStatus(t, resp.StatusCode, http.StatusOK, "http response")
 
 	// Check that the address is gone
-	_, err = zone.LookupLocal(successTestName)
-	assertErrorType(t, err, (*LookupError)(nil), "fully-removed address")
+	_, err = zone.LookupName(successTestName)
+	wt.AssertErrorType(t, err, (*LookupError)(nil), "fully-removed address")
 
 	// Would like to shut down the http server at the end of this test
 	// but it's complicated.

nameserver/mdns_client.go

@@ -26,7 +26,7 @@ var (
 	}
 )
 
-type ResponseA struct {
+type Response struct {
 	Name string
 	Addr net.IP
 	Err  error
@@ -34,7 +34,7 @@ type ResponseA struct {
 
 type responseInfo struct {
 	timeout time.Time // if no answer by this time, give up
-	ch      chan<- *ResponseA
+	ch      chan<- *Response
 }
 
 // Represents one query that we have sent for one name.
@@ -50,17 +50,17 @@ type inflightQuery struct {
 type MDNSClient struct {
 	// note unorthodox use of 'Server' class in client logic - using it to
 	// listen on a multicast socket and call callback whenever a message comes in.
-	listener  *dns.Server
-	conn      *net.UDPConn
-	addr      *net.UDPAddr
-	inflight  map[string]*inflightQuery
-	queryChan chan<- *MDNSInteraction
+	listener   *dns.Server
+	conn       *net.UDPConn
+	addr       *net.UDPAddr
+	inflight   map[string]*inflightQuery
+	actionChan chan<- MDNSAction
 }
 
 type mDNSQueryInfo struct {
 	name       string
 	querytype  uint16
-	responseCh chan<- *ResponseA
+	responseCh chan<- *Response
 }
 
 func NewMDNSClient() (*MDNSClient, error) {
@@ -90,9 +90,9 @@ func (c *MDNSClient) Start(ifi *net.Interface) error {
 	c.listener = &dns.Server{Unsafe: true, PacketConn: multicast, Handler: dns.HandlerFunc(handleMDNS)}
 	go c.listener.ActivateAndServe()
 
-	queryChan := make(chan *MDNSInteraction, MailboxSize)
-	c.queryChan = queryChan
-	go c.queryLoop(queryChan)
+	actionChan := make(chan MDNSAction, MailboxSize)
+	c.actionChan = actionChan
+	go c.actorLoop(actionChan)
 
 	return nil
 }
@@ -103,34 +103,74 @@ func LinkLocalMulticastListener(ifi *net.Interface) (net.PacketConn, error) {
 
 // ACTOR client API
 
-const (
-	CSendQuery       = iota
-	CShutdown        = iota
-	CMessageReceived = iota
-)
-
-type MDNSInteraction struct {
-	code       int
-	resultChan chan<- interface{}
-	payload    interface{}
-}
+type MDNSAction func()
 
 // Async
-func (c *MDNSClient) Shutdown() {
-	c.queryChan <- &MDNSInteraction{code: CShutdown}
-}
-
-// Async
-func (c *MDNSClient) SendQuery(name string, querytype uint16, responseCh chan<- *ResponseA) {
-	c.queryChan <- &MDNSInteraction{
-		code:    CSendQuery,
-		payload: mDNSQueryInfo{name, querytype, responseCh},
+func (c *MDNSClient) SendQuery(name string, querytype uint16, responseCh chan<- *Response) {
+	c.actionChan <- func() {
+		query, found := c.inflight[name]
+		if !found {
+			m := new(dns.Msg)
+			m.SetQuestion(name, querytype)
+			m.RecursionDesired = false
+			buf, err := m.Pack()
+			if err != nil {
+				responseCh <- &Response{Err: err}
+				close(responseCh)
+				return
+			}
+			query = &inflightQuery{
+				name: name,
+				id:   m.Id,
+			}
+			if _, err = c.conn.WriteTo(buf, c.addr); err != nil {
+				responseCh <- &Response{Err: err}
+				close(responseCh)
+				return
+			}
+			c.inflight[name] = query
+		}
+		info := &responseInfo{
+			ch:      responseCh,
+			timeout: time.Now().Add(mDNSTimeout),
+		}
+		// Invariant on responseInfos: they are in ascending order of
+		// timeout.  Since we use a fixed interval from Now(), this
+		// must be after all existing timeouts.
+		query.responseInfos = append(query.responseInfos, info)
 	}
 }
 
 // Async - called from dns library multiplexer
 func (c *MDNSClient) ResponseCallback(r *dns.Msg) {
-	c.queryChan <- &MDNSInteraction{code: CMessageReceived, payload: r}
+	c.actionChan <- func() {
+		for _, answer := range r.Answer {
+			var name string
+			var res *Response
+
+			switch rr := answer.(type) {
+			case *dns.A:
+				name = rr.Hdr.Name
+				res = &Response{Addr: rr.A}
+			case *dns.PTR:
+				name = rr.Hdr.Name
+				res = &Response{Name: rr.Ptr}
+			default:
+				return
+			}
+
+			if query, found := c.inflight[name]; found {
+				for _, resp := range query.responseInfos {
+					resp.ch <- res
+					close(resp.ch)
+				}
+				delete(c.inflight, name)
+			} else {
+				// We've received a response that didn't match a query
+				// Do we want to cache it?
+			}
+		}
+	}
 }
 
 // ACTOR server
@@ -164,28 +204,18 @@ func (c *MDNSClient) checkInFlightQueries() time.Duration {
 	return after
 }
 
-func (c *MDNSClient) queryLoop(queryChan <-chan *MDNSInteraction) {
+func (c *MDNSClient) actorLoop(actionChan <-chan MDNSAction) {
 	timer := time.NewTimer(MaxDuration)
-	run := func() {
-		timer.Reset(c.checkInFlightQueries())
-	}
-
+	run := func() { timer.Reset(c.checkInFlightQueries()) }
 	terminate := false
 	for !terminate {
 		select {
-		case query, ok := <-queryChan:
-			if !ok {
-				break
-			}
-			switch query.code {
-			case CShutdown:
+		case action := <-actionChan:
+			if action == nil {
 				c.listener.Shutdown()
 				terminate = true
-			case CSendQuery:
-				c.handleSendQuery(query.payload.(mDNSQueryInfo))
-				run()
-			case CMessageReceived:
-				c.handleResponse(query.payload.(*dns.Msg))
+			} else {
+				action()
 				run()
 			}
 		case <-timer.C:
@@ -200,54 +230,3 @@ func (c *MDNSClient) queryLoop(queryChan <-chan *MDNSInteraction) {
 		}
 	}
 }
-
-func (c *MDNSClient) handleSendQuery(q mDNSQueryInfo) {
-	query, found := c.inflight[q.name]
-	if !found {
-		m := new(dns.Msg)
-		m.SetQuestion(q.name, q.querytype)
-		m.RecursionDesired = false
-		buf, err := m.Pack()
-		if err != nil {
-			q.responseCh <- &ResponseA{Err: err}
-			close(q.responseCh)
-			return
-		}
-		query = &inflightQuery{
-			name: q.name,
-			id:   m.Id,
-		}
-		if _, err = c.conn.WriteTo(buf, c.addr); err != nil {
-			q.responseCh <- &ResponseA{Err: err}
-			close(q.responseCh)
-			return
-		}
-		c.inflight[q.name] = query
-	}
-	info := &responseInfo{
-		ch:      q.responseCh,
-		timeout: time.Now().Add(mDNSTimeout),
-	}
-	// Invariant on responseInfos: they are in ascending order of timeout.
-	// Since we use a fixed interval from Now(), this must be after all existing timeouts.
-	query.responseInfos = append(query.responseInfos, info)
-}
-
-func (c *MDNSClient) handleResponse(r *dns.Msg) {
-	for _, answer := range r.Answer {
-		switch rr := answer.(type) {
-		case *dns.A:
-			name := rr.Hdr.Name
-			if query, found := c.inflight[name]; found {
-				for _, resp := range query.responseInfos {
-					resp.ch <- &ResponseA{Name: rr.Hdr.Name, Addr: rr.A}
-					close(resp.ch)
-				}
-				delete(c.inflight, name)
-			} else {
-				// We've received a response that didn't match a query
-				// Do we want to cache it?
-			}
-		}
-	}
-}

nameserver/mdns_client_test.go

@@ -2,6 +2,8 @@ package nameserver
 
 import (
 	"github.com/miekg/dns"
+	. "github.com/weaveworks/weave/common"
+	wt "github.com/weaveworks/weave/testing"
 	"log"
 	"net"
 	"testing"
@@ -12,9 +14,27 @@ var (
 	successTestName = "test1.weave."
 	failTestName    = "test2.weave."
 	testAddr        = net.ParseIP("9.8.7.6")
+	testInAddr      = "6.7.8.9.in-addr.arpa."
 )
 
 func minimalServer(w dns.ResponseWriter, req *dns.Msg) {
+
+	sendAnswer := func(m *dns.Msg) {
+		buf, err := m.Pack()
+		CheckFatal(err)
+		if buf == nil {
+			log.Fatal("Nil buffer")
+		}
+		//log.Println("minimalServer sending:", buf)
+		// This is a bit of a kludge - per the RFC we should send responses from 5353, but that doesn't seem to work
+		sendconn, err := net.DialUDP("udp", nil, ipv4Addr)
+		CheckFatal(err)
+
+		_, err = sendconn.Write(buf)
+		sendconn.Close()
+		CheckFatal(err)
+	}
+
 	//log.Println("minimalServer received:", req)
 	if len(req.Answer) > 0 {
 		return // Only interested in questions.
@@ -22,72 +42,73 @@ func minimalServer(w dns.ResponseWriter, req *dns.Msg) {
 	if len(req.Question) != 1 {
 		return // We only handle single-question messages
 	}
-	if req.Question[0].Name != successTestName {
-		return // This is not the DNS record you are looking for
+
+	if req.Question[0].Qtype == dns.TypeA && req.Question[0].Name == successTestName {
+		m := new(dns.Msg)
+		m.SetReply(req)
+		hdr := dns.RR_Header{Name: m.Question[0].Name, Rrtype: dns.TypeA,
+			Class: dns.ClassINET, Ttl: 3600}
+		a := &dns.A{Hdr: hdr, A: testAddr}
+		m.Answer = append(m.Answer, a)
+		sendAnswer(m)
+	} else if req.Question[0].Qtype == dns.TypePTR && req.Question[0].Name == testInAddr {
+		m := new(dns.Msg)
+		m.SetReply(req)
+		hdr := dns.RR_Header{Name: m.Question[0].Name, Rrtype: dns.TypePTR,
+			Class: dns.ClassINET, Ttl: 3600}
+		a := &dns.PTR{Hdr: hdr, Ptr: successTestName}
+		m.Answer = append(m.Answer, a)
+		sendAnswer(m)
 	}
-
-	m := new(dns.Msg)
-	m.SetReply(req)
-
-	hdr := dns.RR_Header{Name: m.Question[0].Name, Rrtype: dns.TypeA,
-		Class: dns.ClassINET, Ttl: 3600}
-	a := &dns.A{hdr, testAddr}
-	m.Answer = append(m.Answer, a)
-
-	buf, err := m.Pack()
-	checkFatal(err)
-	if buf == nil {
-		log.Fatal("Nil buffer")
-	}
-	//log.Println("minimalServer sending:", buf)
-	// This is a bit of a kludge - per the RFC we should send responses from 5353, but that doesn't seem to work
-	sendconn, err := net.DialUDP("udp", nil, ipv4Addr)
-	checkFatal(err)
-
-	_, err = sendconn.Write(buf)
-	sendconn.Close()
-	checkFatal(err)
 }
 
 func RunLocalMulticastServer() (*dns.Server, error) {
-	multicast, err := net.ListenMulticastUDP("udp", nil, ipv4Addr)
+	multicast, err := LinkLocalMulticastListener(nil)
 	if err != nil {
 		return nil, err
 	}
-	server := &dns.Server{Listener: nil, PacketConn: multicast, Handler: dns.HandlerFunc(minimalServer)}
+	server := &dns.Server{
+		PacketConn:  multicast,
+		Handler:     dns.HandlerFunc(minimalServer),
+		ReadTimeout: 100 * time.Millisecond}
 	go server.ActivateAndServe()
 	return server, nil
 }
 
 func setup(t *testing.T) (*MDNSClient, *dns.Server, error) {
-	mdnsClient, err := NewMDNSClient()
-	assertNoErr(t, err)
-	err = mdnsClient.Start(nil)
-	assertNoErr(t, err)
-
 	server, err := RunLocalMulticastServer()
 	if err != nil {
-		t.Fatalf("Unable to run test server: %s", err)
+		t.Fatalf("Unable to run test server: %s. No default multicast interface?", err)
 	}
+
+	mdnsClient, err := NewMDNSClient()
+	wt.AssertNoErr(t, err)
+	err = mdnsClient.Start(nil)
+	wt.AssertNoErr(t, err)
+
 	return mdnsClient, server, err
 }
 
+func (c *MDNSClient) Shutdown() {
+	c.actionChan <- nil
+}
+
 type testContext struct {
 	receivedAddr  net.IP
 	receivedCount int
-	channel       chan *ResponseA
+	channel       chan *Response
 }
 
 func newTestContext() *testContext {
-	return &testContext{channel: make(chan *ResponseA, 4)}
+	return &testContext{channel: make(chan *Response, 4)}
 }
 
-func (c *testContext) checkResponse(t *testing.T, channelOk bool, resp *ResponseA) {
+func (c *testContext) checkResponse(t *testing.T, channelOk bool, resp *Response) {
 	if !channelOk {
 		c.channel = nil
 		return
 	}
-	assertNoErr(t, resp.Err)
+	wt.AssertNoErr(t, resp.Err)
 	log.Printf("Got address response %s addr %s", resp.Name, resp.Addr)
 	c.receivedAddr = resp.Addr
 	c.receivedCount++
@@ -151,3 +172,24 @@ outerloop:
 		t.Fatal("Unexpected result for", successTestName, context1.receivedAddr, context2.receivedAddr, context1.receivedCount, context2.receivedCount)
 	}
 }
+
+func TestAsLookup(t *testing.T) {
+	mdnsClient, server, _ := setup(t)
+	defer mdnsClient.Shutdown()
+	defer server.Shutdown()
+
+	ip, err := mdnsClient.LookupName(successTestName)
+	wt.AssertNoErr(t, err)
+	if !testAddr.Equal(ip) {
+		t.Fatalf("Returned address incorrect %s", ip)
+	}
+
+	ip, err = mdnsClient.LookupName("foo.example.com.")
+	wt.AssertErrorType(t, err, (*LookupError)(nil), "unknown hostname")
+
+	name, err := mdnsClient.LookupInaddr(testInAddr)
+	wt.AssertNoErr(t, err)
+	if !(successTestName == name) {
+		t.Fatalf("Expected name %s, got %s", successTestName, name)
+	}
+}

nameserver/mdns_lookup.go

@@ -0,0 +1,39 @@
+package nameserver
+
+import (
+	"github.com/miekg/dns"
+	. "github.com/weaveworks/weave/common"
+	"net"
+)
+
+func mdnsLookup(client *MDNSClient, name string, qtype uint16) (*Response, error) {
+	channel := make(chan *Response)
+	client.SendQuery(name, qtype, channel)
+	for resp := range channel {
+		if err := resp.Err; err != nil {
+			Debug.Printf("[mdns] Error for query type %s name %s: %s",
+				dns.TypeToString[qtype], name, err)
+			return nil, err
+		}
+		Debug.Printf("[mdns] Got response name for query type %s name %s: name %s, addr %s",
+			dns.TypeToString[qtype], name, resp.Name, resp.Addr)
+		return resp, nil
+	}
+	return nil, LookupError(name)
+}
+
+func (client *MDNSClient) LookupName(name string) (net.IP, error) {
+	if r, e := mdnsLookup(client, name, dns.TypeA); r != nil {
+		return r.Addr, nil
+	} else {
+		return nil, e
+	}
+}
+
+func (client *MDNSClient) LookupInaddr(inaddr string) (string, error) {
+	if r, e := mdnsLookup(client, inaddr, dns.TypePTR); r != nil {
+		return r.Name, nil
+	} else {
+		return "", e
+	}
+}

nameserver/mdns_server.go

@@ -2,12 +2,14 @@ package nameserver
 
 import (
 	"github.com/miekg/dns"
+	. "github.com/weaveworks/weave/common"
 	"net"
 )
 
 type MDNSServer struct {
 	localAddrs []net.Addr
 	sendconn   *net.UDPConn
+	srv        *dns.Server
 	zone       Zone
 }
 
@@ -34,7 +36,7 @@ func (s *MDNSServer) addrIsLocal(testaddr net.Addr) bool {
 	return false
 }
 
-func (s *MDNSServer) Start(ifi *net.Interface) error {
+func (s *MDNSServer) Start(ifi *net.Interface, localDomain string) error {
 	conn, err := LinkLocalMulticastListener(ifi)
 	if err != nil {
 		return err
@@ -49,24 +51,67 @@ func (s *MDNSServer) Start(ifi *net.Interface) error {
 		return err
 	}
 
-	handleMDNS := func(w dns.ResponseWriter, r *dns.Msg) {
-		// Ignore answers to other questions
+	handleLocal := s.makeHandler(dns.TypeA,
+		func(zone Lookup, r *dns.Msg, q *dns.Question) *dns.Msg {
+			if ip, err := zone.LookupName(q.Name); err == nil {
+				return makeAddressReply(r, q, []net.IP{ip})
+			}
+			return nil
+		})
+
+	handleReverse := s.makeHandler(dns.TypePTR,
+		func(zone Lookup, r *dns.Msg, q *dns.Question) *dns.Msg {
+			if name, err := zone.LookupInaddr(q.Name); err == nil {
+				return makePTRReply(r, q, []string{name})
+			}
+			return nil
+		})
+
+	mux := dns.NewServeMux()
+	mux.HandleFunc(localDomain, handleLocal)
+	mux.HandleFunc(RDNSDomain, handleReverse)
+
+	s.srv = &dns.Server{
+		Listener:   nil,
+		PacketConn: conn,
+		Handler:    mux,
+	}
+	go s.srv.ActivateAndServe()
+	return err
+}
+
+func (s *MDNSServer) Stop() error {
+	return s.srv.Shutdown()
+}
+
+type LookupFunc func(Lookup, *dns.Msg, *dns.Question) *dns.Msg
+
+func (s *MDNSServer) makeHandler(qtype uint16, lookup LookupFunc) dns.HandlerFunc {
+	return func(_ dns.ResponseWriter, r *dns.Msg) {
+		// Handle only questions, ignore answers. We might also ignore
+		// questions that arise locally (i.e., that come from an IP we
+		// think is local), but in the interest of avoiding
+		// complication, and easier testing, this is elided on the
+		// assumption that the client wouldn't ask if it already knew
+		// the answer, and if it does ask, it'll be happy to get an
+		// answer.
 		if len(r.Answer) == 0 && len(r.Question) > 0 {
-			q := r.Question[0]
-			if ip, err := s.zone.LookupLocal(q.Name); err == nil {
-				m := makeAddressReply(r, &q, []net.IP{ip})
-				if err = s.sendResponse(m); err != nil {
-					Warning.Printf("Error writing to %s", w)
+			q := &r.Question[0]
+			if q.Qtype == qtype {
+				if m := lookup(s.zone, r, q); m != nil {
+					Debug.Printf("[mdns msgid %d] Found local answer to mDNS query %s",
+						r.MsgHdr.Id, q.Name)
+					if err := s.sendResponse(m); err != nil {
+						Warning.Printf("[mdns msgid %d] Error writing to %v",
+							r.MsgHdr.Id, s.sendconn)
+					}
+				} else {
+					Debug.Printf("[mdns msgid %d] No local answer for mDNS query %s",
+						r.MsgHdr.Id, q.Name)
 				}
-			} else if s.addrIsLocal(w.RemoteAddr()) {
-				// ignore this - it's our own query received via multicast
-			} else {
-				Debug.Printf("Failed MDNS lookup for %s", q.Name)
 			}
 		}
 	}
-	go dns.ActivateAndServe(nil, conn, dns.HandlerFunc(handleMDNS))
-	return err
 }
 
 func (s *MDNSServer) sendResponse(m *dns.Msg) error {

nameserver/mdns_server_test.go

@@ -2,6 +2,8 @@ package nameserver
 
 import (
 	"github.com/miekg/dns"
+	"github.com/weaveworks/weave/common"
+	wt "github.com/weaveworks/weave/testing"
 	"log"
 	"net"
 	"testing"
@@ -10,7 +12,9 @@ import (
 
 var (
 	containerID = "deadbeef"
-	testAddr1   = "10.0.2.1/24"
+	testName    = "test.weave.local."
+	testAddr1   = "10.2.2.1/24"
+	testInAddr1 = "1.2.2.10.in-addr.arpa."
 )
 
 func sendQuery(name string, querytype uint16) error {
@@ -30,22 +34,40 @@ func sendQuery(name string, querytype uint16) error {
 }
 
 func TestServerSimpleQuery(t *testing.T) {
+	// The ff can be handy for debugging (obvs)
+	common.InitDefaultLogging(true)
+
 	log.Println("TestServerSimpleQuery starting")
 	var zone = new(ZoneDb)
 	ip, _, _ := net.ParseCIDR(testAddr1)
-	zone.AddRecord(containerID, "test.weave.", ip)
-
-	mdnsServer, err := NewMDNSServer(zone)
-	assertNoErr(t, err)
-	err = mdnsServer.Start(nil)
-	assertNoErr(t, err)
+	zone.AddRecord(containerID, testName, ip)
 
 	var receivedAddr net.IP
+	var receivedName string
+	var recvChan chan interface{}
 	receivedCount := 0
 
+	reset := func() {
+		receivedAddr = nil
+		receivedName = ""
+		receivedCount = 0
+		recvChan = make(chan interface{})
+	}
+
+	wait := func() {
+		select {
+		case <-recvChan:
+			return
+		case <-time.After(100 * time.Millisecond):
+			return
+		}
+	}
+
 	// Implement a minimal listener for responses
 	multicast, err := LinkLocalMulticastListener(nil)
-	assertNoErr(t, err)
+	if err != nil {
+		t.Fatalf("Unable to run test server: %s. No default multicast interface?", err)
+	}
 
 	handleMDNS := func(w dns.ResponseWriter, r *dns.Msg) {
 		// Only handle responses here
@@ -55,33 +77,56 @@ func TestServerSimpleQuery(t *testing.T) {
 				case *dns.A:
 					receivedAddr = rr.A
 					receivedCount++
+				case *dns.PTR:
+					receivedName = rr.Ptr
+					receivedCount++
 				}
 			}
+			recvChan <- "ok"
 		}
 	}
 
-	listener := &dns.Server{Unsafe: true, PacketConn: multicast, Handler: dns.HandlerFunc(handleMDNS)}
+	listener := &dns.Server{
+		Unsafe:      true,
+		PacketConn:  multicast,
+		Handler:     dns.HandlerFunc(handleMDNS),
+		ReadTimeout: 100 * time.Millisecond}
 	go listener.ActivateAndServe()
 	defer listener.Shutdown()
 
+	mdnsServer, err := NewMDNSServer(zone)
+	wt.AssertNoErr(t, err)
+	err = mdnsServer.Start(nil, DefaultLocalDomain)
+	wt.AssertNoErr(t, err)
+
 	time.Sleep(100 * time.Millisecond) // Allow for server to get going
 
-	sendQuery("test.weave.", dns.TypeA)
-
-	time.Sleep(time.Second)
+	reset()
+	sendQuery(testName, dns.TypeA)
+	wait()
 
 	if receivedCount != 1 {
-		t.Fatal("Unexpected result count for test.weave", receivedCount)
+		t.Fatalf("Unexpected result count %d for %s", receivedCount, testName)
 	}
 	if !receivedAddr.Equal(ip) {
-		t.Fatal("Unexpected result for test.weave", receivedAddr)
+		t.Fatalf("Unexpected result %s for %s", receivedAddr, testName)
 	}
 
-	receivedCount = 0
-
+	reset()
 	sendQuery("testfail.weave.", dns.TypeA)
+	wait()
 
 	if receivedCount != 0 {
-		t.Fatal("Unexpected result count for testfail.weave", receivedCount)
+		t.Fatalf("Unexpected result count %d for testfail.weave", receivedCount)
+	}
+
+	reset()
+	sendQuery(testInAddr1, dns.TypePTR)
+	wait()
+
+	if receivedCount != 1 {
+		t.Fatalf("Expected an answer to %s, got %d answers", testInAddr1, receivedCount)
+	} else if !(testName == receivedName) {
+		t.Fatalf("Expected answer %s to query for %s, got %s", testName, testInAddr1, receivedName)
 	}
 }

nameserver/server.go

@@ -1,152 +1,365 @@
 package nameserver
 
 import (
+	"bytes"
 	"fmt"
 	"github.com/miekg/dns"
+	. "github.com/weaveworks/weave/common"
 	"net"
+	"sync"
+	"time"
 )
 
 const (
-	LOCAL_DOMAIN = "weave.local."
-	RDNS_DOMAIN  = "in-addr.arpa."
+	DefaultLocalDomain   = "weave.local."     // The default name used for the local domain
+	DefaultServerPort    = 53                 // The default server port
+	DefaultCLICfgFile    = "/etc/resolv.conf" // default "resolv.conf" file to try to load
+	DefaultUDPBuflen     = 4096               // bigger than the default 512
+	DefaultCacheLen      = 8192               // default cache capacity
+	DefaultResolvWorkers = 8                  // default number of resolution workers
+	DefaultTimeout       = 5                  // default timeout for DNS resolutions
 )
 
-// +1 to also exclude a dot
-var rdnsDomainLen = len(RDNS_DOMAIN) + 1
+type DNSServerConfig struct {
+	// (Optional) client config file for resolving upstream servers
+	UpstreamCfgFile string
+	// (Optional) DNS client config for the fallback server(s)
+	UpstreamCfg *dns.ClientConfig
+	// (Optional) port number (for TCP and UDP)
+	Port int
+	// (Optional) local domain (ie, "weave.local.")
+	LocalDomain string
+	// (Optional) cache size
+	CacheLen int
+	// (Optional) timeout for DNS queries
+	Timeout int
+	// (Optional) UDP buffer length
+	UDPBufLen int
+}
 
-func checkFatal(e error) {
-	if e != nil {
-		Error.Fatal(e)
+type dnsProtocol uint8
+
+const (
+	protUDP dnsProtocol = iota // UDP protocol
+	protTCP dnsProtocol = iota // TCP protocol
+)
+
+func (proto dnsProtocol) String() string {
+	switch proto {
+	case protUDP:
+		return "UDP"
+	case protTCP:
+		return "TCP"
 	}
+	return "unknown"
 }
 
-func checkWarn(e error) {
-	if e != nil {
-		Warning.Println(e)
+// get a new dns.Client for a protocol
+func (proto dnsProtocol) GetNewClient(bufsize int) *dns.Client {
+	switch proto {
+	case protTCP:
+		return &dns.Client{Net: "tcp"}
+	case protUDP:
+		return &dns.Client{Net: "udp", UDPSize: uint16(bufsize)}
 	}
+	return nil
 }
 
-func makeDNSFailResponse(r *dns.Msg) *dns.Msg {
-	m := new(dns.Msg)
-	m.SetReply(r)
-	m.RecursionAvailable = true
-	m.Rcode = dns.RcodeNameError
-	return m
+type DNSServer struct {
+	Zone        Zone
+	Iface       *net.Interface
+	Upstream    *dns.ClientConfig
+	udpSrv      *dns.Server
+	tcpSrv      *dns.Server
+	mdnsCli     *MDNSClient
+	mdnsSrv     *MDNSServer
+	cache       *Cache
+	timeout     int
+	udpBuf      int
+	listenersWg *sync.WaitGroup
+
+	Domain     string // the local domain
+	ListenAddr string // the address the server is listening at
 }
 
-func queryHandler(zone Zone, mdnsClient *MDNSClient) dns.HandlerFunc {
-	return func(w dns.ResponseWriter, r *dns.Msg) {
-		q := r.Question[0]
-		Debug.Printf("Local query: %+v", q)
-		if q.Qtype == dns.TypeA {
-			if ip, err := zone.LookupLocal(q.Name); err == nil {
-				m := makeAddressReply(r, &q, []net.IP{ip})
-				w.WriteMsg(m)
-			} else {
-				Debug.Printf("Failed lookup for %s; sending mDNS query", q.Name)
-				// We don't know the answer; see if someone else does
-				channel := make(chan *ResponseA)
-				replies := make([]net.IP, 0)
-				go func() {
-					for resp := range channel {
-						Debug.Printf("Got address response %s to query %s addr %s", resp.Name, q.Name, resp.Addr)
-						replies = append(replies, resp.Addr)
-					}
-					var responseMsg *dns.Msg
-					if len(replies) > 0 {
-						responseMsg = makeAddressReply(r, &q, replies)
-					} else {
-						responseMsg = makeDNSFailResponse(r)
-					}
-					w.WriteMsg(responseMsg)
-				}()
-				mdnsClient.SendQuery(q.Name, dns.TypeA, channel)
-			}
-		} else {
-			Warning.Printf("Local query not handled: %+v", q)
-			m := makeDNSFailResponse(r)
-			w.WriteMsg(m)
+// Creates a new DNS server
+func NewDNSServer(config DNSServerConfig, zone Zone, iface *net.Interface) (s *DNSServer, err error) {
+	s = &DNSServer{
+		Zone:        zone,
+		Iface:       iface,
+		listenersWg: new(sync.WaitGroup),
+
+		Domain:     DefaultLocalDomain,
+		ListenAddr: fmt.Sprintf(":%d", DefaultServerPort),
+	}
+
+	// fill empty parameters with defaults...
+	if config.Port != 0 {
+		s.ListenAddr = fmt.Sprintf(":%d", config.Port)
+	}
+	if len(config.LocalDomain) > 0 {
+		s.Domain = config.LocalDomain
+	}
+	if config.UpstreamCfg != nil {
+		s.Upstream = config.UpstreamCfg
+	} else {
+		cfgFile := DefaultCLICfgFile
+		if len(config.UpstreamCfgFile) > 0 {
+			cfgFile = config.UpstreamCfgFile
 		}
+		if s.Upstream, err = dns.ClientConfigFromFile(cfgFile); err != nil {
+			return nil, err
+		}
+	}
+	if config.Timeout > 0 {
+		s.timeout = config.Timeout
+	}
+	if config.UDPBufLen > 0 {
+		s.udpBuf = config.UDPBufLen
+	}
+	s.mdnsCli, err = NewMDNSClient()
+	if err != nil {
 		return
 	}
+	s.mdnsSrv, err = NewMDNSServer(s.Zone)
+	if err != nil {
+		return
+	}
+	cacheLen := DefaultCacheLen
+	if config.CacheLen > 0 {
+		cacheLen = config.CacheLen
+	}
+	Debug.Printf("[dns] Initializing cache: %d entries", config.CacheLen)
+	s.cache, err = NewCache(cacheLen)
+	if err != nil {
+		return
+	}
+
+	// create two DNS request multiplexerers, depending on the protocol used by clients
+	// (we use the same protocol for asking upstream servers)
+	mux := func(proto dnsProtocol) *dns.ServeMux {
+		m := dns.NewServeMux()
+		m.HandleFunc(s.Domain, s.queryHandler(proto))
+		m.HandleFunc(RDNSDomain, s.rdnsHandler(proto))
+		m.HandleFunc(".", s.notUsHandler(proto))
+		return m
+	}
+
+	s.udpSrv = &dns.Server{Addr: s.ListenAddr, Net: "udp", Handler: mux(protUDP)}
+	s.tcpSrv = &dns.Server{Addr: s.ListenAddr, Net: "tcp", Handler: mux(protTCP)}
+
+	return
 }
 
-func rdnsHandler(zone Zone, mdnsClient *MDNSClient) dns.HandlerFunc {
-	return func(w dns.ResponseWriter, r *dns.Msg) {
-		q := r.Question[0]
-		Debug.Printf("Local rdns query: %+v", q)
-		if q.Qtype == dns.TypePTR {
-			if ip := net.ParseIP(q.Name[:len(q.Name)-rdnsDomainLen]); ip != nil {
-				ip4 := ip.To4()
-				revIP := []byte{ip4[3], ip4[2], ip4[1], ip4[0]}
-				Debug.Printf("Looking for address: %+v", revIP)
-				if name, err := zone.ReverseLookupLocal(revIP); err == nil {
-					Debug.Printf("Found name: %s", name)
-					m := makePTRReply(r, &q, []string{name})
-					w.WriteMsg(m)
-				} else {
-					Debug.Printf("Failed lookup for %s; sending mDNS query", q.Name)
-					// We don't know the answer; see if someone else does
-					// TODO
-				}
-			}
-		}
-	}
+// Start the DNS server
+func (s *DNSServer) Start() error {
+	Info.Printf("Using mDNS on %v", s.Iface)
+	err := s.mdnsCli.Start(s.Iface)
+	CheckFatal(err)
+	err = s.mdnsSrv.Start(s.Iface, s.Domain)
+	CheckFatal(err)
+
+	s.listenersWg.Add(2)
+
+	go func() {
+		defer s.listenersWg.Done()
+
+		Debug.Printf("Listening for DNS on %s (UDP)", s.ListenAddr)
+		err = s.udpSrv.ListenAndServe()
+		CheckFatal(err)
+		Debug.Printf("DNS UDP server exiting...")
+	}()
+
+	go func() {
+		defer s.listenersWg.Done()
+
+		Debug.Printf("Listening for DNS on %s (TCP)", s.ListenAddr)
+		err = s.tcpSrv.ListenAndServe()
+		CheckFatal(err)
+		Debug.Printf("DNS TCP server exiting...")
+	}()
+
+	// Waiting for all goroutines to finish (otherwise they die as main routine dies)
+	s.listenersWg.Wait()
+
+	Info.Printf("WeaveDNS server exiting...")
+	return nil
 }
 
-/* When we receive a request for a name outside of our '.weave' domain, call
-   the underlying lookup mechanism and return the answer(s) it gives.
-   Unfortunately, this means that TTLs from a real DNS server are lost - FIXME.
-*/
-func notUsHandler() dns.HandlerFunc {
-	return func(w dns.ResponseWriter, r *dns.Msg) {
-		q := r.Question[0]
-		Debug.Printf("Non-local query: %+v", q)
-		var responseMsg *dns.Msg
-		if q.Qtype == dns.TypeA || q.Qtype == dns.TypeAAAA {
-			if addrs, err := net.LookupIP(q.Name); err == nil {
-				responseMsg = makeAddressReply(r, &q, addrs)
-			} else {
-				responseMsg = makeDNSFailResponse(r)
-				Debug.Print("Failed fallback lookup ", err)
-			}
-		} else {
-			Warning.Printf("Non-local query not handled: %+v", q)
-			responseMsg = makeDNSFailResponse(r)
-		}
-		w.WriteMsg(responseMsg)
-	}
+// Return status string
+func (s *DNSServer) Status() string {
+	var buf bytes.Buffer
+	fmt.Fprintln(&buf, "Local domain", s.Domain)
+	fmt.Fprintln(&buf, "Listen address", s.ListenAddr)
+	fmt.Fprintln(&buf, "mDNS interface", s.Iface)
+	fmt.Fprintln(&buf, "Fallback DNS config", s.Upstream)
+	fmt.Fprintf(&buf, "Zone database:\n%s", s.Zone)
+	return buf.String()
 }
 
-func StartServer(zone Zone, iface *net.Interface, dnsPort int, httpPort int, wait int) error {
-	go ListenHttp(LOCAL_DOMAIN, zone, httpPort)
-
-	mdnsClient, err := NewMDNSClient()
-	checkFatal(err)
-
-	ifaceName := "default interface"
-	if iface != nil {
-		ifaceName = iface.Name
+// Perform a graceful shutdown
+func (s *DNSServer) Stop() error {
+	// Stop the listeners/handlers
+	if err := s.tcpSrv.Shutdown(); err != nil {
+		return err
 	}
-	Info.Printf("Using mDNS on %s", ifaceName)
-	err = mdnsClient.Start(iface)
-	checkFatal(err)
+	if err := s.udpSrv.Shutdown(); err != nil {
+		return err
+	}
+	s.listenersWg.Wait()
 
-	LocalServeMux := dns.NewServeMux()
-	LocalServeMux.HandleFunc(LOCAL_DOMAIN, queryHandler(zone, mdnsClient))
-	LocalServeMux.HandleFunc(RDNS_DOMAIN, rdnsHandler(zone, mdnsClient))
-	LocalServeMux.HandleFunc(".", notUsHandler())
-
-	mdnsServer, err := NewMDNSServer(zone)
-	checkFatal(err)
-
-	err = mdnsServer.Start(iface)
-	checkFatal(err)
-
-	address := fmt.Sprintf(":%d", dnsPort)
-	Info.Printf("Listening for DNS on %s", address)
-	err = dns.ListenAndServe(address, "udp", LocalServeMux)
-	checkFatal(err)
+	// shutdown the mDNS server
+	s.mdnsSrv.Stop()
 
 	return nil
 }
+
+func (s *DNSServer) queryHandler(proto dnsProtocol) dns.HandlerFunc {
+	return func(w dns.ResponseWriter, r *dns.Msg) {
+		now := time.Now()
+		q := r.Question[0]
+		maxLen := getMaxReplyLen(r, proto)
+		lookups := []Lookup{s.Zone, s.mdnsCli}
+		Debug.Printf("Query: %+v", q)
+
+		reply, err := s.cache.Get(r, maxLen, time.Now())
+		if err != nil {
+			if err == errNoLocalReplies {
+				Debug.Printf("[dns msgid %d] Cached 'no local replies' - skipping local lookup", r.MsgHdr.Id)
+				lookups = []Lookup{s.Zone}
+			} else {
+				Debug.Printf("[dns msgid %d] Error from cache: %s", r.MsgHdr.Id, err)
+				w.WriteMsg(makeDNSFailResponse(r))
+				return
+			}
+		}
+		if reply != nil {
+			Debug.Printf("[dns msgid %d] Returning reply from cache: %s/%d answers",
+				r.MsgHdr.Id, dns.RcodeToString[reply.MsgHdr.Rcode], len(reply.Answer))
+			w.WriteMsg(reply)
+			return
+		}
+
+		// catch unsupported queries
+		if q.Qtype != dns.TypeA {
+			Debug.Printf("[dns msgid %d] Unsuported query type %s", r.MsgHdr.Id, dns.TypeToString[q.Qtype])
+			m := makeDNSNotImplResponse(r)
+			s.cache.Put(r, m, negLocalTTL, 0, now)
+			w.WriteMsg(m)
+			return
+		}
+
+		for _, lookup := range lookups {
+			if ip, err := lookup.LookupName(q.Name); err == nil {
+				m := makeAddressReply(r, &q, []net.IP{ip})
+				m.Authoritative = true
+
+				Debug.Printf("[dns msgid %d] Caching response for %s-query for \"%s\": %s [code:%s]",
+					m.MsgHdr.Id, dns.TypeToString[q.Qtype], q.Name, ip, dns.RcodeToString[m.Rcode])
+				s.cache.Put(r, m, nullTTL, 0, now)
+				w.WriteMsg(m)
+				return
+			}
+			now = time.Now()
+		}
+
+		Info.Printf("[dns msgid %d] No results for type %s query %s [caching no-local]",
+			r.MsgHdr.Id, dns.TypeToString[q.Qtype], q.Name)
+		s.cache.Put(r, nil, negLocalTTL, CacheNoLocalReplies, now)
+		w.WriteMsg(makeDNSFailResponse(r))
+	}
+}
+
+func (s *DNSServer) rdnsHandler(proto dnsProtocol) dns.HandlerFunc {
+	fallback := s.notUsHandler(proto)
+	return func(w dns.ResponseWriter, r *dns.Msg) {
+		now := time.Now()
+		q := r.Question[0]
+		lookups := []Lookup{s.Zone, s.mdnsCli}
+		maxLen := getMaxReplyLen(r, proto)
+
+		Debug.Printf("Reverse query: %+v", q)
+		reply, err := s.cache.Get(r, maxLen, time.Now())
+		if err != nil {
+			if err == errNoLocalReplies {
+				Debug.Printf("[dns msgid %d] Cached 'no local replies' - skipping local lookup", r.MsgHdr.Id)
+				lookups = []Lookup{s.Zone}
+			} else {
+				Debug.Printf("[dns msgid %d] Error from cache: %s", r.MsgHdr.Id, err)
+				w.WriteMsg(makeDNSFailResponse(r))
+				return
+			}
+		}
+		if reply != nil {
+			Debug.Printf("[dns msgid %d] Returning reply from cache: %s/%d answers",
+				r.MsgHdr.Id, dns.RcodeToString[reply.MsgHdr.Rcode], len(reply.Answer))
+			w.WriteMsg(reply)
+			return
+		}
+
+		// catch unsupported queries
+		if q.Qtype != dns.TypePTR {
+			Warning.Printf("[dns msgid %d] Unexpected reverse query type %s: %+v",
+				r.MsgHdr.Id, dns.TypeToString[q.Qtype], q)
+			m := makeDNSNotImplResponse(r)
+			s.cache.Put(r, m, negLocalTTL, 0, now)
+			w.WriteMsg(m)
+			return
+		}
+
+		for _, lookup := range lookups {
+			if name, err := lookup.LookupInaddr(q.Name); err == nil {
+				m := makePTRReply(r, &q, []string{name})
+				m.Authoritative = true
+
+				Debug.Printf("[dns msgid %d] Caching response for %s-query for \"%s\": %s [code:%s]",
+					m.MsgHdr.Id, dns.TypeToString[q.Qtype], q.Name, name, dns.RcodeToString[m.Rcode])
+				s.cache.Put(r, m, nullTTL, 0, now)
+				w.WriteMsg(m)
+				return
+			}
+			now = time.Now()
+		}
+
+		Info.Printf("[dns msgid %d] No results for %s-query about '%s' [caching no-local] -> sending to fallback server",
+			r.MsgHdr.Id, dns.TypeToString[q.Qtype], q.Name)
+		s.cache.Put(r, nil, negLocalTTL, CacheNoLocalReplies, now)
+		fallback(w, r)
+	}
+}
+
+// When we receive a request for a name outside of our '.weave.local.'
+// domain, ask the configured DNS server as a fallback.
+func (s *DNSServer) notUsHandler(proto dnsProtocol) dns.HandlerFunc {
+	dnsClient := proto.GetNewClient(DefaultUDPBuflen)
+
+	return func(w dns.ResponseWriter, r *dns.Msg) {
+		maxLen := getMaxReplyLen(r, proto)
+		q := r.Question[0]
+
+		// create a request where we announce our max payload size
+		rcopy := r
+		rcopy.SetEdns0(uint16(maxLen), false)
+
+		Debug.Printf("[dns msgid %d] Fallback query: %+v [%s, max:%d bytes]", rcopy.MsgHdr.Id, q, proto, maxLen)
+		for _, server := range s.Upstream.Servers {
+			reply, _, err := dnsClient.Exchange(rcopy, fmt.Sprintf("%s:%s", server, s.Upstream.Port))
+			if err != nil {
+				Debug.Printf("[dns msgid %d] Network error trying %s (%s)",
+					r.MsgHdr.Id, server, err)
+				continue
+			}
+			if reply != nil && reply.Rcode != dns.RcodeSuccess {
+				Debug.Printf("[dns msgid %d] Failure reported by %s for query %s",
+					r.MsgHdr.Id, server, q.Name)
+				continue
+			}
+			Debug.Printf("[dns msgid %d] Given answer by %s for query %s",
+				r.MsgHdr.Id, server, q.Name)
+			w.WriteMsg(reply)
+			return
+		}
+		Warning.Printf("[dns msgid %d] Failed lookup for external name %s", r.MsgHdr.Id, q.Name)
+		w.WriteMsg(makeDNSFailResponse(r))
+	}
+}

nameserver/server_test.go

@@ -0,0 +1,300 @@
+package nameserver
+
+import (
+	"encoding/binary"
+	"fmt"
+	"github.com/miekg/dns"
+	. "github.com/weaveworks/weave/common"
+	wt "github.com/weaveworks/weave/testing"
+	"net"
+	"testing"
+	"time"
+)
+
+const (
+	testRDNSsuccess  = "1.2.2.10.in-addr.arpa."
+	testRDNSfail     = "4.3.2.1.in-addr.arpa."
+	testRDNSnonlocal = "8.8.8.8.in-addr.arpa."
+	testUDPBufSize   = 16384
+)
+
+var (
+	testPort = 17625
+)
+
+func setupForTest(t *testing.T) {
+	// fail early if we cannot find a default multicast interface
+	multicast, err := LinkLocalMulticastListener(nil)
+	if err != nil {
+		t.Fatalf("Unable to create multicast listener: %s. No default multicast interface?", err)
+	}
+	multicast.Close()
+
+	testPort++
+}
+
+func TestUDPDNSServer(t *testing.T) {
+	setupForTest(t)
+	const (
+		successTestName = "test1.weave.local."
+		failTestName    = "test2.weave.local."
+		nonLocalName    = "weave.works."
+		testAddr1       = "10.2.2.1"
+	)
+	testCIDR1 := testAddr1 + "/24"
+
+	InitDefaultLogging(true)
+	var zone = new(ZoneDb)
+	ip, _, _ := net.ParseCIDR(testCIDR1)
+	zone.AddRecord(containerID, successTestName, ip)
+
+	fallbackHandler := func(w dns.ResponseWriter, req *dns.Msg) {
+		m := new(dns.Msg)
+		m.SetReply(req)
+		if len(req.Question) == 1 {
+			q := req.Question[0]
+			if q.Name == nonLocalName && q.Qtype == dns.TypeMX {
+				m.Answer = make([]dns.RR, 1)
+				m.Answer[0] = &dns.MX{Hdr: dns.RR_Header{Name: m.Question[0].Name, Rrtype: dns.TypeMX, Class: dns.ClassINET, Ttl: 0}, Mx: "mail." + nonLocalName}
+			} else if q.Name == nonLocalName && q.Qtype == dns.TypeANY {
+				m.Answer = make([]dns.RR, 512/len("mailn."+nonLocalName)+1)
+				for i := range m.Answer {
+					m.Answer[i] = &dns.MX{Hdr: dns.RR_Header{Name: m.Question[0].Name, Rrtype: dns.TypeMX, Class: dns.ClassINET, Ttl: 0}, Mx: fmt.Sprintf("mail%d.%s", i, nonLocalName)}
+				}
+			} else if q.Name == testRDNSnonlocal && q.Qtype == dns.TypePTR {
+				m.Answer = make([]dns.RR, 1)
+				m.Answer[0] = &dns.PTR{Hdr: dns.RR_Header{Name: m.Question[0].Name, Rrtype: dns.TypePTR, Class: dns.ClassINET, Ttl: 0}, Ptr: "ns1.google.com."}
+			} else if q.Name == testRDNSfail && q.Qtype == dns.TypePTR {
+				m.Rcode = dns.RcodeNameError
+			}
+		}
+		w.WriteMsg(m)
+	}
+
+	// Run another DNS server for fallback
+	s, fallbackAddr, err := runLocalUDPServer(t, "127.0.0.1:0", fallbackHandler)
+	wt.AssertNoErr(t, err)
+	defer s.Shutdown()
+
+	_, fallbackPort, err := net.SplitHostPort(fallbackAddr)
+	wt.AssertNoErr(t, err)
+
+	config := &dns.ClientConfig{Servers: []string{"127.0.0.1"}, Port: fallbackPort}
+	srv, err := NewDNSServer(DNSServerConfig{UpstreamCfg: config, Port: testPort}, zone, nil)
+	wt.AssertNoErr(t, err)
+	defer srv.Stop()
+	go srv.Start()
+	time.Sleep(100 * time.Millisecond) // Allow sever goroutine to start
+
+	var r *dns.Msg
+
+	r = assertExchange(t, successTestName, dns.TypeA, 1, 1, 0)
+	wt.AssertType(t, r.Answer[0], (*dns.A)(nil), "DNS record")
+	wt.AssertEqualString(t, r.Answer[0].(*dns.A).A.String(), testAddr1, "IP address")
+
+	assertExchange(t, failTestName, dns.TypeA, 0, 0, dns.RcodeNameError)
+
+	r = assertExchange(t, testRDNSsuccess, dns.TypePTR, 1, 1, 0)
+	wt.AssertType(t, r.Answer[0], (*dns.PTR)(nil), "DNS record")
+	wt.AssertEqualString(t, r.Answer[0].(*dns.PTR).Ptr, successTestName, "IP address")
+
+	assertExchange(t, testRDNSfail, dns.TypePTR, 0, 0, dns.RcodeNameError)
+
+	// This should fail because we don't handle MX records
+	assertExchange(t, successTestName, dns.TypeMX, 0, 0, dns.RcodeNotImplemented)
+
+	// This non-local query for an MX record should succeed by being
+	// passed on to the fallback server
+	assertExchange(t, nonLocalName, dns.TypeMX, 1, -1, 0)
+
+	// Now ask a query that we expect to return a lot of data.
+	assertExchange(t, nonLocalName, dns.TypeANY, 5, -1, 0)
+
+	assertExchange(t, testRDNSnonlocal, dns.TypePTR, 1, -1, 0)
+
+	// Not testing MDNS functionality of server here (yet), since it
+	// needs two servers, each listening on its own address
+}
+
+func TestTCPDNSServer(t *testing.T) {
+	setupForTest(t)
+	const (
+		numAnswers   = 512
+		nonLocalName = "weave.works."
+	)
+	dnsAddr := fmt.Sprintf("localhost:%d", testPort)
+
+	InitDefaultLogging(true)
+	var zone = new(ZoneDb)
+
+	// generate a list of `numAnswers` IP addresses
+	var addrs []net.IP
+	bs := make([]byte, 4)
+	for i := 0; i < numAnswers; i++ {
+		binary.LittleEndian.PutUint32(bs, uint32(i))
+		addrs = append(addrs, net.IPv4(bs[0], bs[1], bs[2], bs[3]))
+	}
+
+	// handler for the fallback server: it will just return a very long response
+	fallbackUDPHandler := func(w dns.ResponseWriter, req *dns.Msg) {
+		maxLen := getMaxReplyLen(req, protUDP)
+
+		t.Logf("Fallback UDP server got asked: returning %d answers", numAnswers)
+		q := req.Question[0]
+		m := makeAddressReply(req, &q, addrs)
+		mLen := m.Len()
+		m.SetEdns0(uint16(maxLen), false)
+
+		if mLen > maxLen {
+			t.Logf("... truncated response (%d > %d)", mLen, maxLen)
+			m.Truncated = true
+		}
+		w.WriteMsg(m)
+	}
+	fallbackTCPHandler := func(w dns.ResponseWriter, req *dns.Msg) {
+		t.Logf("Fallback TCP server got asked: returning %d answers", numAnswers)
+		q := req.Question[0]
+		m := makeAddressReply(req, &q, addrs)
+		w.WriteMsg(m)
+	}
+
+	t.Logf("Running a DNS fallback server with UDP")
+	us, fallbackUDPAddr, err := runLocalUDPServer(t, "127.0.0.1:0", fallbackUDPHandler)
+	wt.AssertNoErr(t, err)
+	defer us.Shutdown()
+
+	_, fallbackPort, err := net.SplitHostPort(fallbackUDPAddr)
+	wt.AssertNoErr(t, err)
+
+	t.Logf("Starting another fallback server, with TCP, on the same port as the UDP server")
+	fallbackTCPAddr := fmt.Sprintf("127.0.0.1:%s", fallbackPort)
+	ts, fallbackTCPAddr, err := runLocalTCPServer(t, fallbackTCPAddr, fallbackTCPHandler)
+	wt.AssertNoErr(t, err)
+	defer ts.Shutdown()
+
+	t.Logf("Creating a WeaveDNS server instance, falling back to 127.0.0.1:%s", fallbackPort)
+	config := &dns.ClientConfig{Servers: []string{"127.0.0.1"}, Port: fallbackPort}
+	srv, err := NewDNSServer(DNSServerConfig{UpstreamCfg: config, Port: testPort}, zone, nil)
+	wt.AssertNoErr(t, err)
+	defer srv.Stop()
+	go srv.Start()
+	time.Sleep(100 * time.Millisecond) // Allow sever goroutine to start
+
+	t.Logf("Creating a UDP and a TCP client")
+	uc := new(dns.Client)
+	uc.UDPSize = minUDPSize
+	tc := new(dns.Client)
+	tc.Net = "tcp"
+
+	t.Logf("Creating DNS query message")
+	m := new(dns.Msg)
+	m.RecursionDesired = true
+	m.SetQuestion(nonLocalName, dns.TypeA)
+
+	t.Logf("Checking the fallback server at %s returns a truncated response with UDP", fallbackUDPAddr)
+	r, _, err := uc.Exchange(m, fallbackUDPAddr)
+	t.Logf("Got response from fallback server (UDP) with %d answers", len(r.Answer))
+	t.Logf("Response:\n%+v\n", r)
+	wt.AssertNoErr(t, err)
+	wt.AssertTrue(t, r.MsgHdr.Truncated, "DNS truncated reponse flag")
+	wt.AssertNotEqualInt(t, len(r.Answer), numAnswers, "number of answers (UDP)")
+
+	t.Logf("Checking the WeaveDNS server at %s returns a truncated reponse with UDP", dnsAddr)
+	r, _, err = uc.Exchange(m, dnsAddr)
+	t.Logf("UDP Response:\n%+v\n", r)
+	wt.AssertNoErr(t, err)
+	wt.AssertNotNil(t, r, "response")
+	t.Logf("%d answers", len(r.Answer))
+	wt.AssertTrue(t, r.MsgHdr.Truncated, "DNS truncated reponse flag")
+	wt.AssertNotEqualInt(t, len(r.Answer), numAnswers, "number of answers (UDP)")
+
+	t.Logf("Checking the WeaveDNS server at %s does not return a truncated reponse with TCP", dnsAddr)
+	r, _, err = tc.Exchange(m, dnsAddr)
+	t.Logf("TCP Response:\n%+v\n", r)
+	wt.AssertNoErr(t, err)
+	wt.AssertNotNil(t, r, "response")
+	t.Logf("%d answers", len(r.Answer))
+	wt.AssertFalse(t, r.MsgHdr.Truncated, "DNS truncated response flag")
+	wt.AssertEqualInt(t, len(r.Answer), numAnswers, "number of answers (TCP)")
+
+	t.Logf("Checking the WeaveDNS server at %s does not return a truncated reponse with UDP with a bigger buffer", dnsAddr)
+	m.SetEdns0(testUDPBufSize, false)
+	r, _, err = uc.Exchange(m, dnsAddr)
+	t.Logf("UDP-large Response:\n%+v\n", r)
+	wt.AssertNoErr(t, err)
+	wt.AssertNotNil(t, r, "response")
+	t.Logf("%d answers", len(r.Answer))
+	wt.AssertNoErr(t, err)
+	wt.AssertFalse(t, r.MsgHdr.Truncated, "DNS truncated response flag")
+	wt.AssertEqualInt(t, len(r.Answer), numAnswers, "number of answers (UDP-long)")
+}
+
+//////////////////////////////////////////////////////////////////////////////
+
+// perform a DNS query and assert the reply code, number or answers, etc
+func assertExchange(t *testing.T, z string, ty uint16, minAnswers int, maxAnswers int, expErr int) *dns.Msg {
+	c := new(dns.Client)
+	c.UDPSize = testUDPBufSize
+	m := new(dns.Msg)
+	m.RecursionDesired = true
+	m.SetQuestion(z, ty)
+	m.SetEdns0(testUDPBufSize, false) // we don't want to play with truncation here...
+
+	r, _, err := c.Exchange(m, fmt.Sprintf("127.0.0.1:%d", testPort))
+	t.Logf("Response:\n%+v\n", r)
+	wt.AssertNoErr(t, err)
+	if minAnswers == 0 && maxAnswers == 0 {
+		wt.AssertStatus(t, r.Rcode, expErr, "DNS response code")
+	} else {
+		wt.AssertStatus(t, r.Rcode, dns.RcodeSuccess, "DNS response code")
+	}
+	answers := len(r.Answer)
+	if minAnswers >= 0 && answers < minAnswers {
+		wt.Fatalf(t, "Number of answers >= %d", minAnswers)
+	}
+	if maxAnswers >= 0 && answers > maxAnswers {
+		wt.Fatalf(t, "Number of answers <= %d", maxAnswers)
+	}
+	return r
+}
+
+// run a UDP fallback server
+func runLocalUDPServer(t *testing.T, laddr string, handler dns.HandlerFunc) (*dns.Server, string, error) {
+	t.Logf("Starting fallback UDP server at %s", laddr)
+	pc, err := net.ListenPacket("udp", laddr)
+	if err != nil {
+		return nil, "", err
+	}
+	server := &dns.Server{PacketConn: pc, Handler: handler, ReadTimeout: 100 * time.Millisecond}
+
+	go func() {
+		server.ActivateAndServe()
+		pc.Close()
+	}()
+
+	t.Logf("Fallback UDP server listening at %s", pc.LocalAddr())
+	return server, pc.LocalAddr().String(), nil
+}
+
+// run a TCP fallback server
+func runLocalTCPServer(t *testing.T, laddr string, handler dns.HandlerFunc) (*dns.Server, string, error) {
+	t.Logf("Starting fallback TCP server at %s", laddr)
+	laddrTCP, err := net.ResolveTCPAddr("tcp", laddr)
+	if err != nil {
+		return nil, "", err
+	}
+
+	l, err := net.ListenTCP("tcp", laddrTCP)
+	if err != nil {
+		return nil, "", err
+	}
+	server := &dns.Server{Listener: l, Handler: handler, ReadTimeout: 100 * time.Millisecond}
+
+	go func() {
+		server.ActivateAndServe()
+		l.Close()
+	}()
+
+	t.Logf("Fallback TCP server listening at %s", l.Addr().String())
+	return server, l.Addr().String(), nil
+}

nameserver/testing_utils.go

@@ -1,38 +0,0 @@
-package nameserver
-
-import (
-	"reflect"
-	"testing"
-)
-
-func assertNoErr(t *testing.T, err error) {
-	if err != nil {
-		t.Fatal(err)
-	}
-}
-
-func assertStatus(t *testing.T, got int, wanted int, desc string) {
-	if got != wanted {
-		t.Fatalf("Expected %s %d but got %d", desc, wanted, got)
-	}
-}
-
-func assertErrorInterface(t *testing.T, got interface{}, wanted interface{}, desc string) {
-	if got == nil {
-		t.Fatalf("Expected %s but got nil (%s)", reflect.TypeOf(wanted).Elem(), desc)
-	}
-	gotT, wantedT := reflect.TypeOf(got), reflect.TypeOf(wanted).Elem()
-	if !gotT.Implements(wantedT) {
-		t.Fatalf("Expected %s but got %s (%s)", wantedT, gotT, desc)
-	}
-}
-
-func assertErrorType(t *testing.T, got interface{}, wanted interface{}, desc string) {
-	if got == nil {
-		t.Fatalf("Expected %s but got nil (%s)", reflect.TypeOf(wanted).Elem(), desc)
-	}
-	gotT, wantedT := reflect.TypeOf(got), reflect.TypeOf(wanted).Elem()
-	if gotT != wantedT {
-		t.Fatalf("Expected %s but got %s (%s)", wantedT, gotT, desc)
-	}
-}

nameserver/updater.go

@@ -1,42 +0,0 @@
-package nameserver
-
-import (
-	"github.com/fsouza/go-dockerclient"
-)
-
-func checkError(err error, apiPath string) {
-	if err != nil {
-		Error.Fatalf("Unable to connect to Docker API on %s: %s", apiPath, err)
-	}
-}
-
-func StartUpdater(apiPath string, zone Zone) error {
-	client, err := docker.NewClient(apiPath)
-	checkError(err, apiPath)
-
-	env, err := client.Version()
-	checkError(err, apiPath)
-
-	events := make(chan *docker.APIEvents)
-	err = client.AddEventListener(events)
-	checkError(err, apiPath)
-
-	Info.Printf("Using Docker API on %s: %v", apiPath, env)
-
-	go func() {
-		for event := range events {
-			handleEvent(zone, event, client)
-		}
-	}()
-	return nil
-}
-
-func handleEvent(zone Zone, event *docker.APIEvents, client *docker.Client) error {
-	switch event.Status {
-	case "die":
-		id := event.ID
-		Info.Printf("Container %s down. Removing records", id)
-		zone.DeleteRecordsFor(id)
-	}
-	return nil
-}

nameserver/zone.go

@@ -1,17 +1,31 @@
 package nameserver
 
 import (
+	"bytes"
+	"fmt"
 	"github.com/miekg/dns"
+	. "github.com/weaveworks/weave/common"
 	"net"
 	"sync"
 )
 
+const (
+	RDNSDomain = "in-addr.arpa."
+)
+
+// +1 to also exclude a dot
+var rdnsDomainLen = len(RDNSDomain) + 1
+
+type Lookup interface {
+	LookupName(name string) (net.IP, error)
+	LookupInaddr(inaddr string) (string, error)
+}
+
 type Zone interface {
 	AddRecord(ident string, name string, ip net.IP) error
 	DeleteRecord(ident string, ip net.IP) error
 	DeleteRecordsFor(ident string) error
-	LookupLocal(name string) (net.IP, error)
-	ReverseLookupLocal(ip net.IP) (string, error)
+	Lookup
 }
 
 type Record struct {
@@ -49,7 +63,17 @@ func (zone *ZoneDb) indexOf(match func(Record) bool) int {
 	return -1
 }
 
-func (zone *ZoneDb) LookupLocal(name string) (net.IP, error) {
+func (zone *ZoneDb) String() string {
+	zone.mx.RLock()
+	defer zone.mx.RUnlock()
+	var buf bytes.Buffer
+	for _, r := range zone.recs {
+		fmt.Fprintf(&buf, "%.12s %s %v\n", r.Ident, r.IP, r.Name)
+	}
+	return buf.String()
+}
+
+func (zone *ZoneDb) LookupName(name string) (net.IP, error) {
 	zone.mx.RLock()
 	defer zone.mx.RUnlock()
 	for _, r := range zone.recs {
@@ -60,25 +84,30 @@ func (zone *ZoneDb) LookupLocal(name string) (net.IP, error) {
 	return nil, LookupError(name)
 }
 
-func (zone *ZoneDb) ReverseLookupLocal(ip net.IP) (string, error) {
-	zone.mx.RLock()
-	defer zone.mx.RUnlock()
-	for _, r := range zone.recs {
-		if r.IP.Equal(ip) {
-			return r.Name, nil
+func (zone *ZoneDb) LookupInaddr(inaddr string) (string, error) {
+	if revIP := net.ParseIP(inaddr[:len(inaddr)-rdnsDomainLen]); revIP != nil {
+		revIP4 := revIP.To4()
+		ip := []byte{revIP4[3], revIP4[2], revIP4[1], revIP4[0]}
+		Debug.Printf("[zonedb] Looking for address: %+v", ip)
+		zone.mx.RLock()
+		defer zone.mx.RUnlock()
+		for _, r := range zone.recs {
+			if r.IP.Equal(ip) {
+				return r.Name, nil
+			}
 		}
+		return "", LookupError(inaddr)
 	}
-	return "", LookupError(ip.String())
+	Warning.Printf("[zonedb] Asked to reverse lookup %s", inaddr)
+	return "", LookupError(inaddr)
 }
 
 func (zone *ZoneDb) AddRecord(ident string, name string, ip net.IP) error {
 	zone.mx.Lock()
 	defer zone.mx.Unlock()
 	fqdn := dns.Fqdn(name)
-	if index := zone.indexOf(
-		func(r Record) bool {
-			return r.Name == fqdn && r.IP.Equal(ip) && r.Ident == ident
-		}); index != -1 {
+	pred := func(r Record) bool { return r.Name == fqdn && r.IP.Equal(ip) && r.Ident == ident }
+	if index := zone.indexOf(pred); index != -1 {
 		return DuplicateError{}
 	}
 	zone.recs = append(zone.recs, Record{ident, fqdn, ip})
@@ -88,15 +117,12 @@ func (zone *ZoneDb) AddRecord(ident string, name string, ip net.IP) error {
 func (zone *ZoneDb) DeleteRecord(ident string, ip net.IP) error {
 	zone.mx.Lock()
 	defer zone.mx.Unlock()
-	if index := zone.indexOf(
-		func(r Record) bool {
-			return r.Ident == ident && r.IP.Equal(ip)
-		}); index == -1 {
-		return LookupError(ident)
-	} else {
+	pred := func(r Record) bool { return r.Ident == ident && r.IP.Equal(ip) }
+	if index := zone.indexOf(pred); index != -1 {
 		zone.recs = append(zone.recs[:index], zone.recs[index+1:]...)
+		return nil
 	}
-	return nil
+	return LookupError(ident)
 }
 
 func (zone *ZoneDb) DeleteRecordsFor(ident string) error {
@@ -113,3 +139,8 @@ func (zone *ZoneDb) DeleteRecordsFor(ident string) error {
 	zone.recs = zone.recs[:w]
 	return nil
 }
+
+func (zone *ZoneDb) ContainerDied(ident string) error {
+	Info.Printf("[zonedb] Container %s down. Removing records", ident)
+	return zone.DeleteRecordsFor(ident)
+}

nameserver/zone_test.go

@@ -1,6 +1,7 @@
 package nameserver
 
 import (
+	wt "github.com/weaveworks/weave/testing"
 	"net"
 	"testing"
 )
@@ -10,78 +11,78 @@ func TestZone(t *testing.T) {
 		containerID      = "deadbeef"
 		otherContainerID = "cowjuice"
 		successTestName  = "test1.weave."
-		testAddr1        = "10.0.2.1/24"
+		testAddr1        = "10.2.2.1/24"
 	)
 
 	var zone = new(ZoneDb)
 
 	ip, _, _ := net.ParseCIDR(testAddr1)
 	err := zone.AddRecord(containerID, successTestName, ip)
-	assertNoErr(t, err)
+	wt.AssertNoErr(t, err)
 
 	// Add a few more records to make the job harder
-	err = zone.AddRecord("abcdef0123", "adummy.weave.", net.ParseIP("10.0.0.1"))
-	assertNoErr(t, err)
-	err = zone.AddRecord("0123abcdef", "zdummy.weave.", net.ParseIP("10.0.0.2"))
-	assertNoErr(t, err)
+	err = zone.AddRecord("abcdef0123", "adummy.weave.", net.ParseIP("10.2.0.1"))
+	wt.AssertNoErr(t, err)
+	err = zone.AddRecord("0123abcdef", "zdummy.weave.", net.ParseIP("10.2.0.2"))
+	wt.AssertNoErr(t, err)
 
 	// Check that the address is now there.
-	foundIP, err := zone.LookupLocal(successTestName)
-	assertNoErr(t, err)
+	foundIP, err := zone.LookupName(successTestName)
+	wt.AssertNoErr(t, err)
 
 	if !foundIP.Equal(ip) {
 		t.Fatal("Unexpected result for", successTestName, foundIP)
 	}
 
 	// See if we can find the address by IP.
-	foundName, err := zone.ReverseLookupLocal(ip)
-	assertNoErr(t, err)
+	foundName, err := zone.LookupInaddr("1.2.2.10.in-addr.arpa.")
+	wt.AssertNoErr(t, err)
 
 	if foundName != successTestName {
 		t.Fatal("Unexpected result for", ip, foundName)
 	}
 
 	err = zone.AddRecord(containerID, successTestName, ip)
-	assertErrorType(t, err, (*DuplicateError)(nil), "duplicate add")
+	wt.AssertErrorType(t, err, (*DuplicateError)(nil), "duplicate add")
 
 	err = zone.AddRecord(otherContainerID, successTestName, ip)
 	// Delete the record for the original container
 	err = zone.DeleteRecord(containerID, ip)
-	assertNoErr(t, err)
+	wt.AssertNoErr(t, err)
 
-	_, err = zone.LookupLocal(successTestName)
-	assertNoErr(t, err)
+	_, err = zone.LookupName(successTestName)
+	wt.AssertNoErr(t, err)
 
 	err = zone.DeleteRecord(otherContainerID, ip)
-	assertNoErr(t, err)
+	wt.AssertNoErr(t, err)
 
 	// Check that the address is not there now.
-	_, err = zone.LookupLocal(successTestName)
-	assertErrorType(t, err, (*LookupError)(nil), "after deleting record")
+	_, err = zone.LookupName(successTestName)
+	wt.AssertErrorType(t, err, (*LookupError)(nil), "after deleting record")
 
 	// Delete a record that isn't there
 	err = zone.DeleteRecord(containerID, net.ParseIP("0.0.0.0"))
-	assertErrorType(t, err, (*LookupError)(nil), "when deleting record that doesn't exist")
+	wt.AssertErrorType(t, err, (*LookupError)(nil), "when deleting record that doesn't exist")
 }
 
 func TestDeleteFor(t *testing.T) {
 	var (
 		id    = "foobar"
 		name  = "foo.weave."
-		addr1 = "10.1.2.3/24"
+		addr1 = "10.2.2.3/24"
 		addr2 = "10.2.7.8/24"
 	)
 	zone := new(ZoneDb)
 	for _, addr := range []string{addr1, addr2} {
 		ip, _, _ := net.ParseCIDR(addr)
 		err := zone.AddRecord(id, name, ip)
-		assertNoErr(t, err)
+		wt.AssertNoErr(t, err)
 	}
 
-	_, err := zone.LookupLocal(name)
-	assertNoErr(t, err)
+	_, err := zone.LookupName(name)
+	wt.AssertNoErr(t, err)
 
 	err = zone.DeleteRecordsFor(id)
-	_, err = zone.LookupLocal(name)
-	assertErrorType(t, err, (*LookupError)(nil), "after deleting records for ident")
+	_, err = zone.LookupName(name)
+	wt.AssertErrorType(t, err, (*LookupError)(nil), "after deleting records for ident")
 }

net/if.go

@@ -10,7 +10,7 @@ func EnsureInterface(ifaceName string, wait int) (iface *net.Interface, err erro
 	if iface, err = findInterface(ifaceName); err == nil || wait == 0 {
 		return
 	}
-	for ; err != nil && wait > 0; wait -= 1 {
+	for ; err != nil && wait > 0; wait-- {
 		time.Sleep(1 * time.Second)
 		iface, err = findInterface(ifaceName)
 	}

router/connection.go

@@ -3,75 +3,134 @@ package router
 import (
 	"encoding/binary"
 	"encoding/gob"
-	"encoding/hex"
 	"fmt"
 	"log"
 	"net"
-	"strconv"
+	"sync"
 	"time"
 )
 
-func NewRemoteConnection(from, to *Peer, tcpAddr string) *RemoteConnection {
+type Connection interface {
+	Local() *Peer
+	Remote() *Peer
+	RemoteTCPAddr() string
+	Outbound() bool
+	Established() bool
+	BreakTie(Connection) ConnectionTieBreak
+	Shutdown(error)
+	Log(args ...interface{})
+}
+
+type ConnectionTieBreak int
+
+const (
+	TieBreakWon ConnectionTieBreak = iota
+	TieBreakLost
+	TieBreakTied
+)
+
+type RemoteConnection struct {
+	local         *Peer
+	remote        *Peer
+	remoteTCPAddr string
+	outbound      bool
+	established   bool
+}
+
+type LocalConnection struct {
+	sync.RWMutex
+	RemoteConnection
+	TCPConn           *net.TCPConn
+	tcpSender         TCPSender
+	remoteUDPAddr     *net.UDPAddr
+	receivedHeartbeat bool
+	stackFrag         bool
+	effectivePMTU     int
+	SessionKey        *[32]byte
+	heartbeatTCP      *time.Ticker
+	heartbeatTimeout  *time.Timer
+	heartbeatFrame    *ForwardedFrame
+	heartbeat         *time.Ticker
+	fragTest          *time.Ticker
+	forwarder         *Forwarder
+	forwarderDF       *ForwarderDF
+	Decryptor         Decryptor
+	Router            *Router
+	uid               uint64
+	actionChan        chan<- ConnectionAction
+	finished          <-chan struct{} // closed to signal that actorLoop has finished
+}
+
+type ConnectionAction func() error
+
+func NewRemoteConnection(from, to *Peer, tcpAddr string, outbound bool, established bool) *RemoteConnection {
 	return &RemoteConnection{
 		local:         from,
 		remote:        to,
-		remoteTCPAddr: tcpAddr}
+		remoteTCPAddr: tcpAddr,
+		outbound:      outbound,
+		established:   established,
+	}
 }
 
-func (conn *RemoteConnection) Local() *Peer {
-	return conn.local
-}
+func (conn *RemoteConnection) Local() *Peer                           { return conn.local }
+func (conn *RemoteConnection) Remote() *Peer                          { return conn.remote }
+func (conn *RemoteConnection) RemoteTCPAddr() string                  { return conn.remoteTCPAddr }
+func (conn *RemoteConnection) Outbound() bool                         { return conn.outbound }
+func (conn *RemoteConnection) Established() bool                      { return conn.established }
+func (conn *RemoteConnection) BreakTie(Connection) ConnectionTieBreak { return TieBreakTied }
+func (conn *RemoteConnection) Shutdown(error)                         {}
 
-func (conn *RemoteConnection) Remote() *Peer {
-	return conn.remote
-}
-
-func (conn *RemoteConnection) RemoteTCPAddr() string {
-	return conn.remoteTCPAddr
-}
-
-func (conn *RemoteConnection) Shutdown() {
-}
-
-func (conn *RemoteConnection) Established() bool {
-	return true
+func (conn *RemoteConnection) Log(args ...interface{}) {
+	log.Println(append(append([]interface{}{}, fmt.Sprintf("->[%s|%s]:", conn.remoteTCPAddr, conn.remote.FullName())), args...)...)
 }
 
 func (conn *RemoteConnection) String() string {
 	from := "<nil>"
 	if conn.local != nil {
-		from = conn.local.Name.String()
+		from = conn.local.FullName()
 	}
 	to := "<nil>"
 	if conn.remote != nil {
-		to = conn.remote.Name.String()
+		to = conn.remote.FullName()
 	}
 	return fmt.Sprint("Connection ", from, "->", to)
 }
 
-// Async. Does not return anything. If the connection is successful,
-// it will end up in the local peer's connections map.
-func NewLocalConnection(connRemote *RemoteConnection, acceptNewPeer bool, tcpConn *net.TCPConn, udpAddr *net.UDPAddr, router *Router) {
+func NewLocalConnection(connRemote *RemoteConnection, tcpConn *net.TCPConn, udpAddr *net.UDPAddr, router *Router) *LocalConnection {
 	if connRemote.local != router.Ourself.Peer {
 		log.Fatal("Attempt to create local connection from a peer which is not ourself")
 	}
-
-	queryChan := make(chan *ConnectionInteraction, ChannelSize)
 	// NB, we're taking a copy of connRemote here.
-	connLocal := &LocalConnection{
+	return &LocalConnection{
 		RemoteConnection: *connRemote,
 		Router:           router,
 		TCPConn:          tcpConn,
 		remoteUDPAddr:    udpAddr,
-		effectivePMTU:    DefaultPMTU,
-		queryChan:        queryChan}
-	go connLocal.queryLoop(queryChan, acceptNewPeer)
+		effectivePMTU:    DefaultPMTU}
 }
 
-func (conn *LocalConnection) Established() bool {
-	conn.RLock()
-	defer conn.RUnlock()
-	return conn.established
+// Async. Does not return anything. If the connection is successful,
+// it will end up in the local peer's connections map.
+func (conn *LocalConnection) Start(acceptNewPeer bool) {
+	actionChan := make(chan ConnectionAction, ChannelSize)
+	conn.actionChan = actionChan
+	finished := make(chan struct{})
+	conn.finished = finished
+	go conn.run(actionChan, finished, acceptNewPeer)
+}
+
+func (conn *LocalConnection) BreakTie(dupConn Connection) ConnectionTieBreak {
+	dupConnLocal := dupConn.(*LocalConnection)
+	// conn.uid is used as the tie breaker here, in the knowledge that
+	// both sides will make the same decision.
+	if conn.uid < dupConnLocal.uid {
+		return TieBreakWon
+	} else if dupConnLocal.uid < conn.uid {
+		return TieBreakLost
+	} else {
+		return TieBreakTied
+	}
 }
 
 // Read by the forwarder processes when in the UDP senders
@@ -81,17 +140,10 @@ func (conn *LocalConnection) RemoteUDPAddr() *net.UDPAddr {
 	return conn.remoteUDPAddr
 }
 
-// Called by the forwarder processes in a few places (including
-// crypto), but the connection TCP receiver process, and by the local
-// peer actor process. Do not call this from the connection's actor
-// process itself.
-func (conn *LocalConnection) CheckFatal(err error) error {
-	if err == nil {
-		return nil
-	}
-	conn.log("error:", err)
-	conn.Shutdown()
-	return err
+func (conn *LocalConnection) Established() bool {
+	conn.RLock()
+	defer conn.RUnlock()
+	return conn.established
 }
 
 // Called by forwarder processes, read in Forward (by sniffer and udp
@@ -101,7 +153,7 @@ func (conn *LocalConnection) setEffectivePMTU(pmtu int) {
 	defer conn.Unlock()
 	if conn.effectivePMTU != pmtu {
 		conn.effectivePMTU = pmtu
-		conn.log("Effective PMTU set to", pmtu)
+		conn.Log("Effective PMTU set to", pmtu)
 	}
 }
 
@@ -114,132 +166,92 @@ func (conn *LocalConnection) setStackFrag(frag bool) {
 	conn.stackFrag = frag
 }
 
-func (conn *LocalConnection) log(args ...interface{}) {
-	v := append([]interface{}{}, fmt.Sprintf("->[%s]:", conn.remote.Name))
-	v = append(v, args...)
-	log.Println(v...)
+// Called by the connection's TCP receiver process.
+func (conn *LocalConnection) pmtuVerified(pmtu int) {
+	conn.RLock()
+	fwd := conn.forwarderDF
+	conn.RUnlock()
+	if fwd != nil {
+		fwd.PMTUVerified(pmtu)
+	}
 }
 
-// ACTOR client API
+// Send directly, not via the Actor.  If it goes via the Actor we can
+// get a deadlock where LocalConnection is blocked talking to
+// LocalPeer and LocalPeer is blocked trying send a ProtocolMsg via
+// LocalConnection, and the channels are full in both directions so
+// nothing can proceed.
+func (conn *LocalConnection) SendProtocolMsg(m ProtocolMsg) {
+	if err := conn.sendProtocolMsg(m); err != nil {
+		conn.Shutdown(err)
+	}
+}
 
-const (
-	CSendTCP          = iota
-	CSetEstablished   = iota
-	CSetRemoteUDPAddr = iota
-	CShutdown         = iota
-)
+// ACTOR methods
+
+// NB: The conn.* fields are only written by the connection actor
+// process, which is the caller of the ConnectionAction funs. Hence we
+// do not need locks for reading, and only need write locks for fields
+// read by other processes.
 
 // Async
-func (conn *LocalConnection) Shutdown() {
-	conn.queryChan <- &ConnectionInteraction{Interaction: Interaction{code: CShutdown}}
+func (conn *LocalConnection) Shutdown(err error) {
+	// Run on its own goroutine in case the channel is backed up
+	go func() {
+		if err == nil {
+			conn.sendAction(nil)
+		} else {
+			conn.sendAction(func() error { return err })
+		}
+	}()
 }
 
 // Async
-func (conn *LocalConnection) SetRemoteUDPAddr(remoteUDPAddr *net.UDPAddr) {
-	if remoteUDPAddr == nil {
+//
+// Heartbeating serves two purposes: a) keeping NAT paths alive, and
+// b) updating a remote peer's knowledge of our address, in the event
+// it changes (e.g. because NAT paths expired).
+func (conn *LocalConnection) ReceivedHeartbeat(remoteUDPAddr *net.UDPAddr, connUID uint64) {
+	if remoteUDPAddr == nil || connUID != conn.uid {
 		return
 	}
-	conn.queryChan <- &ConnectionInteraction{
-		Interaction: Interaction{code: CSetRemoteUDPAddr},
-		payload:     remoteUDPAddr}
+	conn.sendAction(func() error {
+		oldRemoteUDPAddr := conn.remoteUDPAddr
+		old := conn.receivedHeartbeat
+		conn.Lock()
+		conn.remoteUDPAddr = remoteUDPAddr
+		conn.receivedHeartbeat = true
+		conn.Unlock()
+		conn.heartbeatTimeout.Reset(HeartbeatTimeout)
+		if !old {
+			if err := conn.sendSimpleProtocolMsg(ProtocolConnectionEstablished); err != nil {
+				return err
+			}
+		}
+		if oldRemoteUDPAddr == nil {
+			return conn.sendFastHeartbeats()
+		} else if oldRemoteUDPAddr.String() != remoteUDPAddr.String() {
+			log.Println("Peer", conn.remote.FullName(), "moved from", old, "to", remoteUDPAddr)
+		}
+		return nil
+	})
 }
 
 // Async
 func (conn *LocalConnection) SetEstablished() {
-	conn.queryChan <- &ConnectionInteraction{Interaction: Interaction{code: CSetEstablished}}
-}
-
-// Async
-func (conn *LocalConnection) SendTCP(msg []byte) {
-	conn.queryChan <- &ConnectionInteraction{
-		Interaction: Interaction{code: CSendTCP},
-		payload:     msg}
-}
-
-// ACTOR server
-
-func (conn *LocalConnection) queryLoop(queryChan <-chan *ConnectionInteraction, acceptNewPeer bool) {
-	err := conn.handshake(acceptNewPeer)
-	if err != nil {
-		log.Printf("->[%s] encountered error during handshake: %v\n", conn.remoteTCPAddr, err)
-		conn.handleShutdown()
-		return
-	}
-	conn.Router.Ourself.AddConnection(conn)
-	if conn.remoteUDPAddr != nil {
-		if err = conn.ensureForwarders(); err == nil {
-			conn.heartbeat = time.NewTicker(FastHeartbeat)
-			conn.forwardHeartbeatFrame() // avoid initial wait
+	conn.sendAction(func() error {
+		stopTicker(conn.heartbeat)
+		old := conn.established
+		conn.Lock()
+		conn.established = true
+		conn.Unlock()
+		if old {
+			return nil
 		}
-	}
-	terminate := false
-	for !terminate {
-		if err != nil {
-			conn.log("error:", err)
-			break
-		}
-		select {
-		case query, ok := <-queryChan:
-			if !ok {
-				break
-			}
-			switch query.code {
-			case CShutdown:
-				terminate = true
-			case CSetEstablished:
-				err = conn.handleSetEstablished()
-			case CSetRemoteUDPAddr:
-				err = conn.handleSetRemoteUDPAddr(query.payload.(*net.UDPAddr))
-			case CSendTCP:
-				err = conn.handleSendTCP(query.payload.([]byte))
-			}
-		case <-tickerChan(conn.heartbeat):
-			conn.forwardHeartbeatFrame()
-		case <-tickerChan(conn.fetchAll):
-			err = conn.handleSendTCP(ProtocolFetchAllByte)
-		case <-tickerChan(conn.fragTest):
-			conn.setStackFrag(false)
-			err = conn.handleSendTCP(ProtocolStartFragmentationTestByte)
-
-		}
-	}
-	conn.handleShutdown()
-}
-
-func (conn *LocalConnection) handleSetRemoteUDPAddr(remoteUDPAddr *net.UDPAddr) error {
-	if remoteUDPAddr == nil {
-		return nil
-	}
-	conn.Lock()
-	old := conn.remoteUDPAddr
-	conn.remoteUDPAddr = remoteUDPAddr
-	conn.Unlock()
-	if old == nil {
-		if err := conn.handleSendTCP(ProtocolConnectionEstablishedByte); err != nil {
-			return err
-		}
-		return conn.handleSetEstablished()
-	} else if old.String() != remoteUDPAddr.String() {
-		log.Println("Peer", conn.remote.Name, "moved from", old, "to", remoteUDPAddr)
-	}
-	return nil
-}
-
-func (conn *LocalConnection) handleSetEstablished() error {
-	conn.Lock()
-	old := conn.established
-	conn.established = true
-	conn.Unlock()
-	if !old {
 		conn.Router.Ourself.ConnectionEstablished(conn)
 		if err := conn.ensureForwarders(); err != nil {
 			return err
 		}
-		stopTicker(conn.heartbeat)
-		conn.heartbeat = time.NewTicker(SlowHeartbeat)
-		conn.fetchAll = time.NewTicker(FetchAllInterval)
-		conn.fragTest = time.NewTicker(FragTestInterval)
-		conn.forwardHeartbeatFrame() // avoid initial wait
 		// Send a large frame down the DF channel in order to prompt
 		// PMTU discovery to start.
 		conn.Forward(true, &ForwardedFrame{
@@ -247,32 +259,153 @@ func (conn *LocalConnection) handleSetEstablished() error {
 			dstPeer: conn.remote,
 			frame:   PMTUDiscovery},
 			nil)
+		conn.heartbeat = time.NewTicker(SlowHeartbeat)
+		conn.fragTest = time.NewTicker(FragTestInterval)
+		// avoid initial waits for timers to fire
+		conn.Forward(true, conn.heartbeatFrame, nil)
 		conn.setStackFrag(false)
-		return conn.handleSendTCP(ProtocolStartFragmentationTestByte)
-	}
-	return nil
+		if err := conn.sendSimpleProtocolMsg(ProtocolStartFragmentationTest); err != nil {
+			return err
+		}
+		return nil
+	})
 }
 
-func (conn *LocalConnection) handleSendTCP(msg []byte) error {
-	return conn.tcpSender.Send(msg)
+// Send an actor request to the actorLoop, but don't block if
+// actorLoop has exited - see http://blog.golang.org/pipelines for
+// pattern
+func (conn *LocalConnection) sendAction(action ConnectionAction) {
+	select {
+	case conn.actionChan <- action:
+	case <-conn.finished:
+	}
 }
 
-func (conn *LocalConnection) handleShutdown() {
-	if conn.remote != nil {
-		conn.log("connection shutting down")
-	}
+// ACTOR server
 
-	// Whilst some of these elements may have been written to whilst
-	// holding locks, they were only written to by the connection
-	// actor process. handleShutdown is only called by the connection
-	// actor process. So there is no need to take locks to read these
-	// (or write elements which are only read by the same
-	// process). Taking locks is only done for elements which are read
-	// by other processes.
-	if conn.shutdown {
+func (conn *LocalConnection) run(actionChan <-chan ConnectionAction, finished chan<- struct{}, acceptNewPeer bool) {
+	var err error // important to use this var and not create another one with 'err :='
+	defer func() { conn.shutdown(err) }()
+	defer close(finished)
+
+	tcpConn := conn.TCPConn
+	tcpConn.SetLinger(0)
+	enc := gob.NewEncoder(tcpConn)
+	dec := gob.NewDecoder(tcpConn)
+
+	if err = conn.handshake(enc, dec, acceptNewPeer); err != nil {
 		return
 	}
-	conn.shutdown = true
+	conn.Log("completed handshake")
+
+	// The ordering of the following is very important. [1]
+
+	if conn.remoteUDPAddr != nil {
+		if err = conn.ensureForwarders(); err != nil {
+			return
+		}
+	}
+	if err = conn.Router.Ourself.AddConnection(conn); err != nil {
+		return
+	}
+	if err = conn.initHeartbeats(); err != nil {
+		return
+	}
+	go conn.receiveTCP(dec)
+	err = conn.actorLoop(actionChan)
+}
+
+// [1] Ordering constraints:
+//
+// (a) AddConnections must precede initHeartbeats. It is only after
+// the former completes that we know the connection is valid, in
+// particular is not a duplicate connection to the same peer. Sending
+// heartbeats on a duplicate connection can trip up crypto at the
+// other end, since the associated UDP packets may get decoded by the
+// other connection. It is also generally wasteful to engage in any
+// interaction with the remote on a connection that turns out to be
+// invald.
+//
+// (b) AddConnection must precede receiveTCP. In the absence of any
+// indirect connectivity to the remote peer, the first we hear about
+// it (and any peers reachable from it) is through topology gossip it
+// sends us on the connection. We must ensure that the connection has
+// been added to Ourself prior to processing any such gossip,
+// otherwise we risk immediately gc'ing part of that newly received
+// portion of the topology (though not the remote peer itself, since
+// that will have a positive ref count), leaving behind dangling
+// references to peers. Hence we must invoke AddConnection, which is
+// *synchronous*, first.
+//
+// (c) AddConnection must precede actorLoop. More precisely, it must
+// precede shutdown, since that invokes DeleteConnection and is
+// invoked on termination of this entire function. Essentially this
+// boils down to a prohibition on running AddConnection in a separate
+// goroutine, at least not without some synchronisation. Which in turn
+// requires us the launching of the receiveTCP goroutine to precede
+// actorLoop.
+//
+// (d) AddConnection should precede receiveTCP. There is no point
+// starting the latter if the former fails.
+//
+// (e) initHeartbeats should precede actorLoop. The former is setting
+// LocalConnection fields accessed by the latter. Since the latter
+// runs in a separate goroutine, we'd have to add some synchronisation
+// if initHeartbeats isn't run first.
+//
+// (f) ensureForwarders should precede AddConnection. As soon as a
+// connection has been added to LocalPeer by the latter, it becomes
+// visible to the packet routing logic, which will end up dropping
+// packets if the forwarders haven't been created yet. We cannot
+// prevent that completely, since, for example, forwarder can only be
+// created when we know the remote UDP address, but it helps to try.
+
+func (conn *LocalConnection) initHeartbeats() error {
+	conn.heartbeatTCP = time.NewTicker(TCPHeartbeat)
+	conn.heartbeatTimeout = time.NewTimer(HeartbeatTimeout)
+	heartbeatFrameBytes := make([]byte, EthernetOverhead+8)
+	binary.BigEndian.PutUint64(heartbeatFrameBytes[EthernetOverhead:], conn.uid)
+	conn.heartbeatFrame = &ForwardedFrame{
+		srcPeer: conn.local,
+		dstPeer: conn.remote,
+		frame:   heartbeatFrameBytes}
+	if conn.remoteUDPAddr == nil {
+		return nil
+	}
+	return conn.sendFastHeartbeats()
+}
+
+func (conn *LocalConnection) actorLoop(actionChan <-chan ConnectionAction) (err error) {
+	for err == nil {
+		select {
+		case action := <-actionChan:
+			if action == nil {
+				break
+			}
+			err = action()
+		case <-conn.heartbeatTCP.C:
+			err = conn.sendSimpleProtocolMsg(ProtocolHeartbeat)
+		case <-conn.heartbeatTimeout.C:
+			err = fmt.Errorf("timed out waiting for UDP heartbeat")
+		case <-tickerChan(conn.heartbeat):
+			conn.Forward(true, conn.heartbeatFrame, nil)
+		case <-tickerChan(conn.fragTest):
+			conn.setStackFrag(false)
+			err = conn.sendSimpleProtocolMsg(ProtocolStartFragmentationTest)
+		}
+	}
+	return
+}
+
+func (conn *LocalConnection) shutdown(err error) {
+	switch {
+	case err != nil && conn.remote == nil:
+		log.Printf("->[%s] connection shutting down due to error during handshake: %v\n", conn.remoteTCPAddr, err)
+	case err != nil:
+		conn.Log("connection shutting down due to error:", err)
+	default:
+		conn.Log("connection shutting down")
+	}
 
 	if conn.TCPConn != nil {
 		checkWarn(conn.TCPConn.Close())
@@ -283,157 +416,34 @@ func (conn *LocalConnection) handleShutdown() {
 		conn.Router.Ourself.DeleteConnection(conn)
 	}
 
+	if conn.heartbeatTimeout != nil {
+		conn.heartbeatTimeout.Stop()
+	}
+
+	stopTicker(conn.heartbeatTCP)
 	stopTicker(conn.heartbeat)
-	stopTicker(conn.fetchAll)
 	stopTicker(conn.fragTest)
 
 	// blank out the forwardChan so that the router processes don't
 	// try to send any more
 	conn.stopForwarders()
 
-	conn.Router.ConnectionMaker.ConnectionTerminated(conn.remoteTCPAddr)
+	conn.Router.ConnectionMaker.ConnectionTerminated(conn.remoteTCPAddr, err)
 }
 
-func (conn *LocalConnection) handshake(acceptNewPeer bool) error {
-	// We do not need to worry about locking in here as at this point,
-	// the connection is not reachable by any go-routine other than
-	// ourself. Only when we add this connection to the conn.local
-	// peer will it be visible from multiple go-routines.
-	tcpConn := conn.TCPConn
-	tcpConn.SetLinger(0)
-	conn.extendReadDeadline()
+// Helpers
 
-	enc := gob.NewEncoder(tcpConn)
-
-	localConnID := randUint64()
-
-	versionStr := fmt.Sprint(ProtocolVersion)
-	handshakeSend := map[string]string{
-		"Protocol":        Protocol,
-		"ProtocolVersion": versionStr,
-		"PeerNameFlavour": PeerNameFlavour,
-		"Name":            conn.local.Name.String(),
-		"UID":             fmt.Sprint(conn.local.UID),
-		"ConnID":          fmt.Sprint(localConnID)}
-	handshakeRecv := map[string]string{}
-
-	usingPassword := conn.Router.UsingPassword()
-	var public, private *[32]byte
-	var err error
-	if usingPassword {
-		public, private, err = GenerateKeyPair()
-		if err != nil {
-			return err
-		}
-		handshakeSend["PublicKey"] = hex.EncodeToString(public[:])
-	}
-	enc.Encode(handshakeSend)
-
-	dec := gob.NewDecoder(tcpConn)
-	err = dec.Decode(&handshakeRecv)
-	if err != nil {
-		return err
-	}
-	_, err = checkHandshakeStringField("Protocol", Protocol, handshakeRecv)
-	if err != nil {
-		return err
-	}
-	_, err = checkHandshakeStringField("ProtocolVersion", versionStr, handshakeRecv)
-	if err != nil {
-		return err
-	}
-	_, err = checkHandshakeStringField("PeerNameFlavour", PeerNameFlavour, handshakeRecv)
-	if err != nil {
-		return err
-	}
-	nameStr, err := checkHandshakeStringField("Name", "", handshakeRecv)
-	if err != nil {
-		return err
-	}
-	name, err := PeerNameFromString(nameStr)
-	if err != nil {
-		return err
-	}
-	if !acceptNewPeer {
-		if _, found := conn.Router.Peers.Fetch(name); !found {
-			return fmt.Errorf("Found unknown remote name: %s at %s", name, conn.remoteTCPAddr)
-		}
-	}
-	if existingConn, found := conn.local.ConnectionTo(name); found && existingConn.Established() {
-		return fmt.Errorf("Already have connection to %s at %s", name, existingConn.RemoteTCPAddr())
-	}
-
-	uidStr, err := checkHandshakeStringField("UID", "", handshakeRecv)
-	if err != nil {
-		return err
-	}
-	uid, err := strconv.ParseUint(uidStr, 10, 64)
-	if err != nil {
-		return err
-	}
-
-	remoteConnIdStr, err := checkHandshakeStringField("ConnID", "", handshakeRecv)
-	if err != nil {
-		return err
-	}
-	remoteConnID, err := strconv.ParseUint(remoteConnIdStr, 10, 64)
-	if err != nil {
-		return err
-	}
-	conn.UID = localConnID ^ remoteConnID
-
-	if usingPassword {
-		remotePublicStr, rpErr := checkHandshakeStringField("PublicKey", "", handshakeRecv)
-		if rpErr != nil {
-			return rpErr
-		}
-		remotePublicSlice, rpErr := hex.DecodeString(remotePublicStr)
-		if rpErr != nil {
-			return rpErr
-		}
-		remotePublic := [32]byte{}
-		for idx, elem := range remotePublicSlice {
-			remotePublic[idx] = elem
-		}
-		conn.SessionKey = FormSessionKey(&remotePublic, private, conn.Router.Password)
-		conn.tcpSender = NewEncryptedTCPSender(enc, conn)
-		conn.Decryptor = NewNaClDecryptor(conn)
-	} else {
-		if _, found := handshakeRecv["PublicKey"]; found {
-			return fmt.Errorf("Remote network is encrypted. Password required.")
-		}
-		conn.tcpSender = NewSimpleTCPSender(enc)
-		conn.Decryptor = NewNonDecryptor(conn)
-	}
-
-	toPeer := NewPeer(name, uid, 0)
-	toPeer = conn.Router.Peers.FetchWithDefault(toPeer)
-	if toPeer == nil {
-		return fmt.Errorf("Connection appears to be with different version of a peer we already know of")
-	} else if toPeer == conn.local {
-		// have to do assigment here to ensure Shutdown releases ref count
-		conn.remote = toPeer
-		return fmt.Errorf("Cannot connect to ourself")
-	}
-	conn.remote = toPeer
-
-	go conn.receiveTCP(dec, usingPassword)
-	return nil
+func (conn *LocalConnection) sendSimpleProtocolMsg(tag ProtocolTag) error {
+	return conn.sendProtocolMsg(ProtocolMsg{tag: tag})
 }
 
-func checkHandshakeStringField(fieldName string, expectedValue string, handshake map[string]string) (string, error) {
-	val, found := handshake[fieldName]
-	if !found {
-		return "", fmt.Errorf("Field %s is missing", fieldName)
-	}
-	if expectedValue != "" && val != expectedValue {
-		return "", fmt.Errorf("Field %s has wrong value; expected '%s', received '%s'", fieldName, expectedValue, val)
-	}
-	return val, nil
+func (conn *LocalConnection) sendProtocolMsg(m ProtocolMsg) error {
+	return conn.tcpSender.Send(Concat([]byte{byte(m.tag)}, m.msg))
 }
 
-func (conn *LocalConnection) receiveTCP(decoder *gob.Decoder, usingPassword bool) {
+func (conn *LocalConnection) receiveTCP(decoder *gob.Decoder) {
 	defer conn.Decryptor.Shutdown()
+	usingPassword := conn.SessionKey != nil
 	var receiver TCPReceiver
 	if usingPassword {
 		receiver = NewEncryptedTCPReceiver(conn)
@@ -444,94 +454,67 @@ func (conn *LocalConnection) receiveTCP(decoder *gob.Decoder, usingPassword bool
 	for {
 		var msg []byte
 		conn.extendReadDeadline()
-		if conn.CheckFatal(decoder.Decode(&msg)) != nil {
-			return
+		if err = decoder.Decode(&msg); err != nil {
+			break
 		}
 		msg, err = receiver.Decode(msg)
 		if err != nil {
-			checkWarn(err)
-			// remote peer may be using wrong password. Or there could
-			// be some sort of injection attack going on. Just ignore
-			// the traffic rather than shutting down.
+			break
+		}
+		if len(msg) < 1 {
+			conn.Log("ignoring blank msg")
 			continue
 		}
-		if msg[0] == ProtocolConnectionEstablished {
-			// We initiated the connection. We sent fast heartbeats to
-			// the remote side, which has now received at least one of
-			// them and thus has informed us via TCP that it considers
-			// the connection is now up. We now do a fetchAll on it.
-			conn.SetEstablished()
-			conn.SendTCP(ProtocolFetchAllByte)
-		} else if msg[0] == ProtocolStartFragmentationTest {
-			conn.Forward(false, &ForwardedFrame{
-				srcPeer: conn.local,
-				dstPeer: conn.remote,
-				frame:   FragTest},
-				nil)
-		} else if msg[0] == ProtocolFragmentationReceived {
-			conn.setStackFrag(true)
-		} else if usingPassword && msg[0] == ProtocolNonce {
-			conn.Decryptor.ReceiveNonce(msg[1:])
-		} else if msg[0] == ProtocolFetchAll {
-			// There are exactly two messages that relate to topology
-			// updates.
-			//
-			// 1. FetchAll. This carries no payload. The receiver
-			// responds with the entire topology model as the receiver
-			// has it.
-			//
-			// 2. Update. This carries a topology payload. The
-			// receiver merges it with its own topology model. If the
-			// payload is a subset of the receiver's topology, no
-			// further action is taken. Otherwise, the receiver sends
-			// out to all its connections an "improved" update:
-			//  - elements which the original payload added to the
-			//    receiver are included
-			//  - elements which the original payload updated in the
-			//    receiver are included
-			//  - elements which are equal between the receiver and
-			//    the payload are not included
-			//  - elements where the payload was older than the
-			//    receiver's version are updated
-			conn.SendTCP(Concat(ProtocolUpdateByte, conn.Router.Peers.EncodeAllPeers()))
-		} else if msg[0] == ProtocolUpdate {
-			newUpdate, err := conn.Router.Peers.ApplyUpdate(msg[1:])
-			if _, ok := err.(UnknownPeersError); err != nil && ok {
-				// That update contained a peer we didn't know about;
-				// request full update
-				conn.SendTCP(ProtocolFetchAllByte)
-				continue
-			}
-			if conn.CheckFatal(err) != nil {
-				return
-			}
-			if len(newUpdate) != 0 {
-				conn.Router.ConnectionMaker.Refresh()
-				conn.Router.Routes.Recalculate()
-				conn.Router.Ourself.BroadcastTCP(Concat(ProtocolUpdateByte, newUpdate))
-			}
-		} else if msg[0] == ProtocolPMTUVerified {
-			conn.verifyPMTU <- int(binary.BigEndian.Uint16(msg[1:]))
-		} else {
-			conn.log("received unknown msg:\n", msg)
+		if err = conn.handleProtocolMsg(ProtocolTag(msg[0]), msg[1:]); err != nil {
+			break
 		}
 	}
+	conn.Shutdown(err)
+}
+
+func (conn *LocalConnection) handleProtocolMsg(tag ProtocolTag, payload []byte) error {
+	switch tag {
+	case ProtocolHeartbeat:
+	case ProtocolConnectionEstablished:
+		// We sent fast heartbeats to the remote peer, which has now
+		// received at least one of them and told us via this message.
+		// We can now consider the connection as established from our
+		// end.
+		conn.SetEstablished()
+	case ProtocolStartFragmentationTest:
+		conn.Forward(false, &ForwardedFrame{
+			srcPeer: conn.local,
+			dstPeer: conn.remote,
+			frame:   FragTest},
+			nil)
+	case ProtocolFragmentationReceived:
+		conn.setStackFrag(true)
+	case ProtocolNonce:
+		if conn.SessionKey == nil {
+			return fmt.Errorf("unexpected nonce on unencrypted connection")
+		}
+		conn.Decryptor.ReceiveNonce(payload)
+	case ProtocolPMTUVerified:
+		conn.pmtuVerified(int(binary.BigEndian.Uint16(payload)))
+	case ProtocolGossipUnicast, ProtocolGossipBroadcast, ProtocolGossip:
+		return conn.Router.handleGossip(tag, payload)
+	default:
+		conn.Log("ignoring unknown protocol tag:", tag)
+	}
+	return nil
 }
 
 func (conn *LocalConnection) extendReadDeadline() {
-	conn.TCPConn.SetReadDeadline(time.Now().Add(ReadTimeout))
+	conn.TCPConn.SetReadDeadline(time.Now().Add(TCPHeartbeat * 2))
 }
 
-// Heartbeating serves two purposes: a) keeping NAT paths alive, and
-// b) updating a remote peer's knowledge of our address, in the event
-// it changes (e.g. because NAT paths expired).
-// Called only by connection actor process.
-func (conn *LocalConnection) forwardHeartbeatFrame() {
-	heartbeatFrame := &ForwardedFrame{
-		srcPeer: conn.local,
-		dstPeer: conn.remote,
-		frame:   []byte{}}
-	conn.Forward(true, heartbeatFrame, nil)
+func (conn *LocalConnection) sendFastHeartbeats() error {
+	err := conn.ensureForwarders()
+	if err == nil {
+		conn.heartbeat = time.NewTicker(FastHeartbeat)
+		conn.Forward(true, conn.heartbeatFrame, nil) // avoid initial wait
+	}
+	return err
 }
 
 func tickerChan(ticker *time.Ticker) <-chan time.Time {

router/connection_maker.go

@@ -14,96 +14,118 @@ const (
 	MaxInterval     = 10 * time.Minute
 )
 
-const (
-	CMInitiate   = iota
-	CMTerminated = iota
-	CMRefresh    = iota
-	CMStatus     = iota
-)
-
 type ConnectionMaker struct {
-	ourself        *LocalPeer
-	peers          *Peers
-	targets        map[string]*Target
-	cmdLineAddress map[string]bool
-	queryChan      chan<- *ConnectionMakerInteraction
+	ourself      *LocalPeer
+	peers        *Peers
+	port         int
+	targets      map[string]*Target
+	cmdLinePeers map[string]*net.TCPAddr
+	actionChan   chan<- ConnectionMakerAction
 }
 
 // Information about an address where we may find a peer
 type Target struct {
 	attempting  bool          // are we currently attempting to connect there?
+	lastError   error         // reason for disconnection last time
 	tryAfter    time.Time     // next time to try this address
 	tryInterval time.Duration // backoff time on next failure
 }
 
-type ConnectionMakerInteraction struct {
-	Interaction
-	address string
+type ConnectionMakerAction func() bool
+
+func NewConnectionMaker(ourself *LocalPeer, peers *Peers, port int) *ConnectionMaker {
+	return &ConnectionMaker{
+		ourself:      ourself,
+		peers:        peers,
+		port:         port,
+		cmdLinePeers: make(map[string]*net.TCPAddr),
+		targets:      make(map[string]*Target)}
 }
 
-func StartConnectionMaker(ourself *LocalPeer, peers *Peers) *ConnectionMaker {
-	queryChan := make(chan *ConnectionMakerInteraction, ChannelSize)
-	state := &ConnectionMaker{
-		ourself:        ourself,
-		peers:          peers,
-		queryChan:      queryChan,
-		cmdLineAddress: make(map[string]bool),
-		targets:        make(map[string]*Target)}
-	go state.queryLoop(queryChan)
-	return state
+func (cm *ConnectionMaker) Start() {
+	actionChan := make(chan ConnectionMakerAction, ChannelSize)
+	cm.actionChan = actionChan
+	go cm.queryLoop(actionChan)
 }
 
-func (cm *ConnectionMaker) InitiateConnection(address string) {
-	cm.queryChan <- &ConnectionMakerInteraction{
-		Interaction: Interaction{code: CMInitiate},
-		address:     address}
+func (cm *ConnectionMaker) InitiateConnection(peer string) error {
+	host, port, err := net.SplitHostPort(peer)
+	if err != nil {
+		host = peer
+		port = "0" // we use that as an indication that "no port was supplied"
+	}
+	addr, err := net.ResolveTCPAddr("tcp4", fmt.Sprintf("%s:%s", host, port))
+	if err != nil {
+		return err
+	}
+	cm.actionChan <- func() bool {
+		cm.cmdLinePeers[peer] = addr
+		// curtail any existing reconnect interval
+		if target, found := cm.targets[addr.String()]; found {
+			target.tryAfter, target.tryInterval = tryImmediately()
+		}
+		return true
+	}
+	return nil
 }
 
-func (cm *ConnectionMaker) ConnectionTerminated(address string) {
-	cm.queryChan <- &ConnectionMakerInteraction{
-		Interaction: Interaction{code: CMTerminated},
-		address:     address}
+func (cm *ConnectionMaker) ForgetConnection(peer string) {
+	cm.actionChan <- func() bool {
+		delete(cm.cmdLinePeers, peer)
+		return false
+	}
+}
+
+func (cm *ConnectionMaker) ConnectionTerminated(address string, err error) {
+	cm.actionChan <- func() bool {
+		if target, found := cm.targets[address]; found {
+			target.attempting = false
+			target.lastError = err
+			target.tryAfter, target.tryInterval = tryAfter(target.tryInterval)
+		}
+		return true
+	}
 }
 
 func (cm *ConnectionMaker) Refresh() {
-	cm.queryChan <- &ConnectionMakerInteraction{
-		Interaction: Interaction{code: CMRefresh}}
+	cm.actionChan <- func() bool { return true }
 }
 
 func (cm *ConnectionMaker) String() string {
-	resultChan := make(chan interface{}, 0)
-	cm.queryChan <- &ConnectionMakerInteraction{
-		Interaction: Interaction{code: CMStatus, resultChan: resultChan}}
-	result := <-resultChan
-	return result.(string)
+	// We need to Refresh first in order to clear out any 'attempting'
+	// connections from cm.targets that have been established since
+	// the last run of cm.checkStateAndAttemptConnections. These
+	// entries are harmless but do represent stale state that we do
+	// not want to report.
+	cm.Refresh()
+	resultChan := make(chan string, 0)
+	cm.actionChan <- func() bool {
+		var buf bytes.Buffer
+		for address, target := range cm.targets {
+			fmt.Fprintf(&buf, "->[%s]", address)
+			if target.lastError != nil {
+				fmt.Fprintf(&buf, " (%s)", target.lastError)
+			}
+			if target.attempting {
+				fmt.Fprintf(&buf, " trying since %v\n", target.tryAfter)
+			} else {
+				fmt.Fprintf(&buf, " next try at %v\n", target.tryAfter)
+			}
+		}
+		resultChan <- buf.String()
+		return false
+	}
+	return <-resultChan
 }
 
-func (cm *ConnectionMaker) queryLoop(queryChan <-chan *ConnectionMakerInteraction) {
+func (cm *ConnectionMaker) queryLoop(actionChan <-chan ConnectionMakerAction) {
 	timer := time.NewTimer(MaxDuration)
 	run := func() { timer.Reset(cm.checkStateAndAttemptConnections()) }
 	for {
 		select {
-		case query, ok := <-queryChan:
-			if !ok {
-				return
-			}
-			switch query.code {
-			case CMInitiate:
-				cm.cmdLineAddress[NormalisePeerAddr(query.address)] = true
+		case action := <-actionChan:
+			if action() {
 				run()
-			case CMTerminated:
-				if target, found := cm.targets[query.address]; found {
-					target.attempting = false
-					target.tryAfter, target.tryInterval = tryAfter(target.tryInterval)
-				}
-				run()
-			case CMRefresh:
-				run()
-			case CMStatus:
-				run()
-				query.resultChan <- cm.status()
-			default:
-				log.Fatal("Unexpected connection maker query:", query)
 			}
 		case <-timer.C:
 			run()
@@ -112,62 +134,115 @@ func (cm *ConnectionMaker) queryLoop(queryChan <-chan *ConnectionMakerInteractio
 }
 
 func (cm *ConnectionMaker) checkStateAndAttemptConnections() time.Duration {
-	validTarget := make(map[string]bool)
-
-	// copy the set of things we are connected to, so we can access them without locking
-	our_connected_peers := make(map[PeerName]bool)
-	our_connected_targets := make(map[string]bool)
-	cm.ourself.ForEachConnection(func(peer PeerName, conn Connection) {
-		our_connected_peers[peer] = true
-		our_connected_targets[conn.RemoteTCPAddr()] = true
-	})
+	var (
+		validTarget   = make(map[string]struct{})
+		cmdLineTarget = make(map[string]struct{})
+	)
+	// Copy the set of things we are connected to, so we can access
+	// them without locking.  Also clear out any entries in cm.targets
+	// for existing connections.
+	ourConnectedPeers, ourConnectedTargets, ourInboundIPs := cm.ourConnections()
 
 	addTarget := func(address string) {
-		if !our_connected_targets[address] {
-			validTarget[address] = true
-			cm.addTarget(address)
+		if _, connected := ourConnectedTargets[address]; connected {
+			return
 		}
+		validTarget[address] = void
+		if _, found := cm.targets[address]; found {
+			return
+		}
+		target := &Target{}
+		target.tryAfter, target.tryInterval = tryImmediately()
+		cm.targets[address] = target
 	}
 
 	// Add command-line targets that are not connected
-	for address, _ := range cm.cmdLineAddress {
-		addTarget(address)
+	for _, addr := range cm.cmdLinePeers {
+		completeAddr := *addr
+		attempt := true
+		if completeAddr.Port == 0 {
+			completeAddr.Port = cm.port
+			// If a peer was specified w/o a port, then we do not
+			// attempt to connect to it if we have any inbound
+			// connections from that IP.
+			if _, connected := ourInboundIPs[completeAddr.IP.String()]; connected {
+				attempt = false
+			}
+		}
+		address := completeAddr.String()
+		cmdLineTarget[address] = void
+		if attempt {
+			addTarget(address)
+		}
 	}
 
 	// Add targets for peers that someone else is connected to, but we
 	// aren't
-	cm.peers.ForEach(func(name PeerName, peer *Peer) {
-		peer.ForEachConnection(func(otherPeer PeerName, conn Connection) {
-			if otherPeer == cm.ourself.Name || our_connected_peers[otherPeer] {
-				return
+	cm.addPeerTargets(ourConnectedPeers, addTarget)
+
+	return cm.connectToTargets(validTarget, cmdLineTarget)
+}
+
+func (cm *ConnectionMaker) ourConnections() (PeerNameSet, map[string]struct{}, map[string]struct{}) {
+	var (
+		ourConnectedPeers   = make(PeerNameSet)
+		ourConnectedTargets = make(map[string]struct{})
+		ourInboundIPs       = make(map[string]struct{})
+	)
+	for conn := range cm.ourself.Connections() {
+		address := conn.RemoteTCPAddr()
+		delete(cm.targets, address)
+		ourConnectedPeers[conn.Remote().Name] = void
+		ourConnectedTargets[address] = void
+		if conn.Outbound() {
+			continue
+		}
+		if ip, _, err := net.SplitHostPort(address); err == nil { // should always succeed
+			ourInboundIPs[ip] = void
+		}
+	}
+	return ourConnectedPeers, ourConnectedTargets, ourInboundIPs
+}
+
+func (cm *ConnectionMaker) addPeerTargets(ourConnectedPeers PeerNameSet, addTarget func(string)) {
+	cm.peers.ForEach(func(peer *Peer) {
+		for conn := range peer.Connections() {
+			otherPeer := conn.Remote().Name
+			if otherPeer == cm.ourself.Name {
+				continue
+			}
+			if _, connected := ourConnectedPeers[otherPeer]; connected {
+				continue
 			}
 			address := conn.RemoteTCPAddr()
-			// try both portnumber of connection and standard port
-			addTarget(address)
-			if host, _, err := net.SplitHostPort(address); err == nil {
-				addTarget(NormalisePeerAddr(host))
+			// Try both port of connection and standard port.  Don't
+			// use remote side of inbound connection.
+			if conn.Outbound() {
+				addTarget(address)
 			}
-		})
+			if ip, _, err := net.SplitHostPort(address); err == nil {
+				addTarget(fmt.Sprintf("%s:%d", ip, cm.port))
+			}
+		}
 	})
+}
 
+func (cm *ConnectionMaker) connectToTargets(validTarget map[string]struct{}, cmdLineTarget map[string]struct{}) time.Duration {
 	now := time.Now() // make sure we catch items just added
 	after := MaxDuration
 	for address, target := range cm.targets {
-		if our_connected_targets[address] {
-			delete(cm.targets, address)
-			continue
-		}
 		if target.attempting {
 			continue
 		}
-		if !validTarget[address] {
+		if _, valid := validTarget[address]; !valid {
 			delete(cm.targets, address)
 			continue
 		}
 		switch duration := target.tryAfter.Sub(now); {
 		case duration <= 0:
 			target.attempting = true
-			go cm.attemptConnection(address, cm.cmdLineAddress[address])
+			_, isCmdLineTarget := cmdLineTarget[address]
+			go cm.attemptConnection(address, isCmdLineTarget)
 		case duration < after:
 			after = duration
 		}
@@ -175,33 +250,11 @@ func (cm *ConnectionMaker) checkStateAndAttemptConnections() time.Duration {
 	return after
 }
 
-func (cm *ConnectionMaker) addTarget(address string) {
-	if _, found := cm.targets[address]; !found {
-		target := &Target{}
-		target.tryAfter, target.tryInterval = tryImmediately()
-		cm.targets[address] = target
-	}
-}
-
-func (cm *ConnectionMaker) status() string {
-	var buf bytes.Buffer
-	for address, target := range cm.targets {
-		var fmtStr string
-		if target.attempting {
-			fmtStr = "%s (trying since %v)\n"
-		} else {
-			fmtStr = "%s (next try at %v)\n"
-		}
-		buf.WriteString(fmt.Sprintf(fmtStr, address, target.tryAfter))
-	}
-	return buf.String()
-}
-
 func (cm *ConnectionMaker) attemptConnection(address string, acceptNewPeer bool) {
-	log.Println("Attempting connection to", address)
+	log.Printf("->[%s] attempting connection\n", address)
 	if err := cm.ourself.CreateConnection(address, acceptNewPeer); err != nil {
-		log.Println(err)
-		cm.ConnectionTerminated(address)
+		log.Printf("->[%s] error during connection attempt: %v\n", address, err)
+		cm.ConnectionTerminated(address, err)
 	}
 }
 

router/consts.go

@@ -6,46 +6,28 @@ import (
 )
 
 const (
-	Protocol           = "weave"
-	ProtocolVersion    = 8
-	EthernetOverhead   = 14
-	UDPOverhead        = 28 // 20 bytes for IPv4, 8 bytes for UDP
-	Port               = 6783
-	HttpPort           = Port + 1
-	DefaultPMTU        = 65535
-	MaxUDPPacketSize   = 65536
-	ChannelSize        = 16
-	UDPNonceSendAt     = 8192
-	FragTestSize       = 60001
-	PMTUDiscoverySize  = 60000
-	FastHeartbeat      = 500 * time.Millisecond
-	SlowHeartbeat      = 10 * time.Second
-	FetchAllInterval   = 30 * time.Second
-	FragTestInterval   = 5 * time.Minute
-	ReadTimeout        = 2 * FetchAllInterval
-	PMTUVerifyAttempts = 8
-	PMTUVerifyTimeout  = 10 * time.Millisecond // gets doubled with every attempt
-	MaxDuration        = time.Duration(math.MaxInt64)
-)
-
-const (
-	ProtocolConnectionEstablished  = iota
-	ProtocolFragmentationReceived  = iota
-	ProtocolStartFragmentationTest = iota
-	ProtocolNonce                  = iota
-	ProtocolFetchAll               = iota
-	ProtocolUpdate                 = iota
-	ProtocolPMTUVerified           = iota
+	EthernetOverhead    = 14
+	UDPOverhead         = 28 // 20 bytes for IPv4, 8 bytes for UDP
+	Port                = 6783
+	HTTPPort            = Port + 1
+	DefaultPMTU         = 65535
+	MaxUDPPacketSize    = 65536
+	ChannelSize         = 16
+	UDPNonceSendAt      = 8192
+	FragTestSize        = 60001
+	PMTUDiscoverySize   = 60000
+	TCPHeartbeat        = 30 * time.Second
+	FastHeartbeat       = 500 * time.Millisecond
+	SlowHeartbeat       = 10 * time.Second
+	FragTestInterval    = 5 * time.Minute
+	PMTUVerifyAttempts  = 8
+	PMTUVerifyTimeout   = 10 * time.Millisecond // gets doubled with every attempt
+	MaxDuration         = time.Duration(math.MaxInt64)
+	MaxMissedHeartbeats = 3
+	HeartbeatTimeout    = MaxMissedHeartbeats * SlowHeartbeat
 )
 
 var (
-	FragTest                           = make([]byte, FragTestSize)
-	PMTUDiscovery                      = make([]byte, PMTUDiscoverySize)
-	ProtocolConnectionEstablishedByte  = []byte{ProtocolConnectionEstablished}
-	ProtocolFragmentationReceivedByte  = []byte{ProtocolFragmentationReceived}
-	ProtocolStartFragmentationTestByte = []byte{ProtocolStartFragmentationTest}
-	ProtocolNonceByte                  = []byte{ProtocolNonce}
-	ProtocolFetchAllByte               = []byte{ProtocolFetchAll}
-	ProtocolUpdateByte                 = []byte{ProtocolUpdate}
-	ProtocolPMTUVerifiedByte           = []byte{ProtocolPMTUVerified}
+	FragTest      = make([]byte, FragTestSize)
+	PMTUDiscovery = make([]byte, PMTUDiscoverySize)
 )

router/crypto.go

@@ -11,17 +11,18 @@ import (
 	"encoding/gob"
 	"fmt"
 	"log"
+	"sync"
 )
 
 func GenerateKeyPair() (publicKey, privateKey *[32]byte, err error) {
 	return box.GenerateKey(rand.Reader)
 }
 
-func FormSessionKey(remotePublicKey, localPrivateKey *[32]byte, secretKey *[]byte) *[32]byte {
+func FormSessionKey(remotePublicKey, localPrivateKey *[32]byte, secretKey []byte) *[32]byte {
 	var sharedKey [32]byte
 	box.Precompute(&sharedKey, remotePublicKey, localPrivateKey)
 	sharedKeySlice := sharedKey[:]
-	sharedKeySlice = append(sharedKeySlice, *secretKey...)
+	sharedKeySlice = append(sharedKeySlice, secretKey...)
 	sessionKey := sha256.Sum256(sharedKeySlice)
 	return &sessionKey
 }
@@ -83,7 +84,8 @@ func EncodeNonce(df bool) (*[24]byte, []byte, error) {
 		flags = flags | 1
 	}
 	SetNonceLow15Bits(&nonce, flags)
-	return &nonce, Concat(ProtocolNonceByte, nonce[:]), nil
+	// NB: need to make a copy since callers may modify the array
+	return &nonce, Concat(nonce[:]), nil
 }
 
 func DecodeNonce(msg []byte) (bool, *[24]byte) {
@@ -99,6 +101,34 @@ func DecodeNonce(msg []byte) (bool, *[24]byte) {
 
 // Frame Encryptors
 
+type Encryptor interface {
+	FrameOverhead() int
+	PacketOverhead() int
+	IsEmpty() bool
+	Bytes() []byte
+	AppendFrame(src []byte, dst []byte, frame []byte)
+	TotalLen() int
+}
+
+type NonEncryptor struct {
+	buf       []byte
+	bufTail   []byte
+	buffered  int
+	prefixLen int
+}
+
+type NaClEncryptor struct {
+	NonEncryptor
+	buf       []byte
+	offset    uint16
+	nonce     *[24]byte
+	nonceChan chan *[24]byte
+	flags     uint16
+	prefixLen int
+	conn      *LocalConnection
+	df        bool
+}
+
 func NewNonEncryptor(prefix []byte) *NonEncryptor {
 	buf := make([]byte, MaxUDPPacketSize)
 	prefixLen := copy(buf, prefix)
@@ -128,18 +158,17 @@ func (ne *NonEncryptor) Bytes() []byte {
 	return buf
 }
 
-func (ne *NonEncryptor) AppendFrame(frame *ForwardedFrame) {
+func (ne *NonEncryptor) AppendFrame(src []byte, dst []byte, frame []byte) {
 	bufTail := ne.bufTail
-	srcLen := copy(bufTail, frame.srcPeer.NameByte)
+	srcLen := copy(bufTail, src)
 	bufTail = bufTail[srcLen:]
-	dstLen := copy(bufTail, frame.dstPeer.NameByte)
+	dstLen := copy(bufTail, dst)
 	bufTail = bufTail[dstLen:]
-	frameLen := len(frame.frame)
-	binary.BigEndian.PutUint16(bufTail, uint16(frameLen))
+	binary.BigEndian.PutUint16(bufTail, uint16(len(frame)))
 	bufTail = bufTail[2:]
-	copy(bufTail, frame.frame)
-	ne.bufTail = bufTail[frameLen:]
-	ne.buffered += srcLen + dstLen + 2 + frameLen
+	copy(bufTail, frame)
+	ne.bufTail = bufTail[len(frame):]
+	ne.buffered += srcLen + dstLen + 2 + len(frame)
 }
 
 func (ne *NonEncryptor) TotalLen() int {
@@ -173,10 +202,11 @@ func (ne *NaClEncryptor) Bytes() []byte {
 	nonce := ne.nonce
 	if nonce == nil {
 		freshNonce, encodedNonce, err := EncodeNonce(ne.df)
-		if err = ne.conn.CheckFatal(err); err != nil {
+		if err != nil {
+			ne.conn.Shutdown(err)
 			return []byte{}
 		}
-		ne.conn.SendTCP(encodedNonce)
+		ne.conn.SendProtocolMsg(ProtocolMsg{ProtocolNonce, encodedNonce})
 		ne.nonce = freshNonce
 		nonce = freshNonce
 	}
@@ -191,11 +221,12 @@ func (ne *NaClEncryptor) Bytes() []byte {
 		ne.nonce = <-ne.nonceChan
 	} else if offset == 1<<14 { // half way through range, send new nonce
 		nonce, encodedNonce, err := EncodeNonce(ne.df)
-		if err = ne.conn.CheckFatal(err); err != nil {
+		if err != nil {
+			ne.conn.Shutdown(err)
 			return []byte{}
 		}
 		ne.nonceChan <- nonce
-		ne.conn.SendTCP(encodedNonce)
+		ne.conn.SendProtocolMsg(ProtocolMsg{ProtocolNonce, encodedNonce})
 	}
 	ne.offset = offset
 
@@ -212,31 +243,58 @@ func (ne *NaClEncryptor) TotalLen() int {
 
 // Frame Decryptors
 
+type FrameConsumer func(conn *LocalConnection, src []byte, dst []byte, frame []byte)
+
+type Decryptor interface {
+	IterateFrames([]byte, FrameConsumer) error
+	ReceiveNonce([]byte)
+	Shutdown()
+}
+
+type NonDecryptor struct {
+	conn *LocalConnection
+}
+
+type NaClDecryptor struct {
+	NonDecryptor
+	instance   *NaClDecryptorInstance
+	instanceDF *NaClDecryptorInstance
+}
+
+type NaClDecryptorInstance struct {
+	nonce               *[24]byte
+	previousNonce       *[24]byte
+	usedOffsets         *bit.Set
+	previousUsedOffsets *bit.Set
+	highestOffsetSeen   uint16
+	nonceChan           chan *[24]byte
+}
+
+type PacketDecodingError struct {
+	Desc string
+}
+
 func NewNonDecryptor(conn *LocalConnection) *NonDecryptor {
 	return &NonDecryptor{conn: conn}
 }
 
-func (nd *NonDecryptor) IterateFrames(fun FrameConsumer, packet *UDPPacket) error {
-	buf := packet.Packet
-	for len(buf) >= (2 + NameSize + NameSize) {
-		srcNameByte := buf[:NameSize]
-		buf = buf[NameSize:]
-		dstNameByte := buf[:NameSize]
-		buf = buf[NameSize:]
-		length := binary.BigEndian.Uint16(buf[:2])
-		buf = buf[2:]
-		if len(buf) < int(length) {
-			return PacketDecodingError{Desc: fmt.Sprintf("too short; expected frame of length %d, got %d", length, len(buf))}
-		}
-		frame := buf[:length]
-		buf = buf[length:]
-		err := fun(nd.conn, packet.Sender, srcNameByte, dstNameByte, length, frame)
-		if err != nil {
-			return err
+func (nd *NonDecryptor) IterateFrames(packet []byte, consumer FrameConsumer) error {
+	for len(packet) >= (2 + NameSize + NameSize) {
+		srcNameByte := packet[:NameSize]
+		packet = packet[NameSize:]
+		dstNameByte := packet[:NameSize]
+		packet = packet[NameSize:]
+		length := binary.BigEndian.Uint16(packet[:2])
+		packet = packet[2:]
+		if len(packet) < int(length) {
+			return PacketDecodingError{Desc: fmt.Sprintf("too short; expected frame of length %d, got %d", length, len(packet))}
 		}
+		frame := packet[:length]
+		packet = packet[length:]
+		consumer(nd.conn, srcNameByte, dstNameByte, frame)
 	}
-	if len(buf) > 0 {
-		return PacketDecodingError{Desc: fmt.Sprintf("%d octets of trailing garbage", len(buf))}
+	if len(packet) > 0 {
+		return PacketDecodingError{Desc: fmt.Sprintf("%d octets of trailing garbage", len(packet))}
 	}
 	return nil
 }
@@ -249,24 +307,14 @@ func (nd *NonDecryptor) ReceiveNonce(msg []byte) {
 }
 
 func NewNaClDecryptor(conn *LocalConnection) *NaClDecryptor {
-	inst := NaClDecryptorInstance{
-		nonce:               nil,
-		previousNonce:       nil,
-		usedOffsets:         bit.New(),
-		previousUsedOffsets: nil,
-		highestOffsetSeen:   0,
-		nonceChan:           make(chan *[24]byte, ChannelSize)}
-	instDF := NaClDecryptorInstance{
-		nonce:               nil,
-		previousNonce:       nil,
-		usedOffsets:         bit.New(),
-		previousUsedOffsets: nil,
-		highestOffsetSeen:   0,
-		nonceChan:           make(chan *[24]byte, ChannelSize)}
 	return &NaClDecryptor{
 		NonDecryptor: *NewNonDecryptor(conn),
-		instance:     &inst,
-		instanceDF:   &instDF}
+		instance: &NaClDecryptorInstance{
+			usedOffsets: bit.New(),
+			nonceChan:   make(chan *[24]byte, ChannelSize)},
+		instanceDF: &NaClDecryptorInstance{
+			usedOffsets: bit.New(),
+			nonceChan:   make(chan *[24]byte, ChannelSize)}}
 }
 
 func (nd *NaClDecryptor) Shutdown() {
@@ -283,83 +331,27 @@ func (nd *NaClDecryptor) ReceiveNonce(msg []byte) {
 	}
 }
 
-func (nd *NaClDecryptor) IterateFrames(fun FrameConsumer, packet *UDPPacket) error {
-	buf, err := nd.decrypt(packet.Packet)
+func (nd *NaClDecryptor) IterateFrames(packet []byte, consumer FrameConsumer) error {
+	buf, err := nd.decrypt(packet)
 	if err != nil {
-		return PacketDecodingError{Fatal: true, Desc: fmt.Sprint("decryption failed; ", err)}
+		return PacketDecodingError{Desc: fmt.Sprint("decryption failed; ", err)}
 	}
-	packet.Packet = buf
-	return nd.NonDecryptor.IterateFrames(fun, packet)
+	return nd.NonDecryptor.IterateFrames(buf, consumer)
 }
 
 func (nd *NaClDecryptor) decrypt(buf []byte) ([]byte, error) {
 	offset := binary.BigEndian.Uint16(buf[:2])
 	df := (offset & (1 << 15)) != 0
 	offsetNoFlags := offset & ((1 << 15) - 1)
-	var decState *NaClDecryptorInstance
+	var di *NaClDecryptorInstance
 	if df {
-		decState = nd.instanceDF
+		di = nd.instanceDF
 	} else {
-		decState = nd.instance
+		di = nd.instance
 	}
-	var nonce *[24]byte
-	var usedOffsets *bit.Set
-	var ok bool
-	if decState.nonce == nil {
-		if offsetNoFlags > (1 << 13) {
-			// offset is already beyond the first quarter and it's the
-			// first thing we've seen?! I don't think so.
-			return nil, fmt.Errorf("Unexpected offset when decrypting UDP packet")
-		}
-		decState.nonce, ok = <-decState.nonceChan
-		if !ok {
-			return nil, fmt.Errorf("Nonce chan closed")
-		}
-		decState.highestOffsetSeen = offsetNoFlags
-		nonce = decState.nonce
-		usedOffsets = decState.usedOffsets
-	} else {
-		highestOffsetSeen := decState.highestOffsetSeen
-		if offsetNoFlags < (1<<13) && highestOffsetSeen > ((1<<14)+(1<<13)) &&
-			(highestOffsetSeen-offsetNoFlags) > ((1<<14)+(1<<13)) {
-			// offset is in the first quarter, highestOffsetSeen is in
-			// the top quarter and under a quarter behind us. We
-			// interpret this as we need to move to the next nonce
-			decState.previousUsedOffsets = decState.usedOffsets
-			decState.usedOffsets = bit.New()
-			decState.previousNonce = decState.nonce
-			decState.nonce, ok = <-decState.nonceChan
-			if !ok {
-				return nil, fmt.Errorf("Nonce chan closed")
-			}
-			decState.highestOffsetSeen = offsetNoFlags
-			nonce = decState.nonce
-			usedOffsets = decState.usedOffsets
-		} else if offsetNoFlags > highestOffsetSeen &&
-			(offsetNoFlags-highestOffsetSeen) < (1<<13) {
-			// offset is under a quarter above highestOffsetSeen. This
-			// is ok - maybe some packet loss
-			decState.highestOffsetSeen = offsetNoFlags
-			nonce = decState.nonce
-			usedOffsets = decState.usedOffsets
-		} else if offsetNoFlags <= highestOffsetSeen &&
-			(highestOffsetSeen-offsetNoFlags) < (1<<13) {
-			// offset is within a quarter of the highest we've
-			// seen. This is ok - just assuming some out-of-order
-			// delivery.
-			nonce = decState.nonce
-			usedOffsets = decState.usedOffsets
-		} else if highestOffsetSeen < (1<<13) && offsetNoFlags > ((1<<14)+(1<<13)) &&
-			(offsetNoFlags-highestOffsetSeen) > ((1<<14)+(1<<13)) {
-			// offset is in the last quarter, highestOffsetSeen is in
-			// the first quarter, and offset is under a quarter behind
-			// us. This is ok - as above, just some out of order. But
-			// here it means we're dealing with the previous nonce
-			nonce = decState.previousNonce
-			usedOffsets = decState.previousUsedOffsets
-		} else {
-			return nil, fmt.Errorf("Unexpected offset when decrypting UDP packet")
-		}
+	nonce, usedOffsets, err := di.advanceState(offsetNoFlags)
+	if err != nil {
+		return nil, err
 	}
 	offsetNoFlagsInt := int(offsetNoFlags)
 	if usedOffsets.Contains(offsetNoFlagsInt) {
@@ -367,16 +359,90 @@ func (nd *NaClDecryptor) decrypt(buf []byte) ([]byte, error) {
 	}
 	SetNonceLow15Bits(nonce, offsetNoFlags)
 	result, success := secretbox.Open(nil, buf[2:], nonce, nd.conn.SessionKey)
-	if success {
-		usedOffsets.Add(offsetNoFlagsInt)
-		return result, nil
-	} else {
+	if !success {
 		return nil, fmt.Errorf("Unable to decrypt UDP packet")
 	}
+	usedOffsets.Add(offsetNoFlagsInt)
+	return result, nil
+}
+
+func (di *NaClDecryptorInstance) advanceState(offsetNoFlags uint16) (*[24]byte, *bit.Set, error) {
+	var ok bool
+	if di.nonce == nil {
+		if offsetNoFlags > (1 << 13) {
+			// offset is already beyond the first quarter and it's the
+			// first thing we've seen?! I don't think so.
+			return nil, nil, fmt.Errorf("Unexpected offset when decrypting UDP packet")
+		}
+		di.nonce, ok = <-di.nonceChan
+		if !ok {
+			return nil, nil, fmt.Errorf("Nonce chan closed")
+		}
+		di.highestOffsetSeen = offsetNoFlags
+	} else {
+		highestOffsetSeen := di.highestOffsetSeen
+		switch {
+		case offsetNoFlags < (1<<13) && highestOffsetSeen > ((1<<14)+(1<<13)) &&
+			(highestOffsetSeen-offsetNoFlags) > ((1<<14)+(1<<13)):
+			// offset is in the first quarter, highestOffsetSeen is in
+			// the top quarter and under a quarter behind us. We
+			// interpret this as we need to move to the next nonce
+			di.previousUsedOffsets = di.usedOffsets
+			di.usedOffsets = bit.New()
+			di.previousNonce = di.nonce
+			di.nonce, ok = <-di.nonceChan
+			if !ok {
+				return nil, nil, fmt.Errorf("Nonce chan closed")
+			}
+			di.highestOffsetSeen = offsetNoFlags
+		case offsetNoFlags > highestOffsetSeen &&
+			(offsetNoFlags-highestOffsetSeen) < (1<<13):
+			// offset is under a quarter above highestOffsetSeen. This
+			// is ok - maybe some packet loss
+			di.highestOffsetSeen = offsetNoFlags
+		case offsetNoFlags <= highestOffsetSeen &&
+			(highestOffsetSeen-offsetNoFlags) < (1<<13):
+			// offset is within a quarter of the highest we've
+			// seen. This is ok - just assuming some out-of-order
+			// delivery.
+		case highestOffsetSeen < (1<<13) && offsetNoFlags > ((1<<14)+(1<<13)) &&
+			(offsetNoFlags-highestOffsetSeen) > ((1<<14)+(1<<13)):
+			// offset is in the last quarter, highestOffsetSeen is in
+			// the first quarter, and offset is under a quarter behind
+			// us. This is ok - as above, just some out of order. But
+			// here it means we're dealing with the previous nonce
+			return di.previousNonce, di.previousUsedOffsets, nil
+		default:
+			return nil, nil, fmt.Errorf("Unexpected offset when decrypting UDP packet")
+		}
+	}
+	return di.nonce, di.usedOffsets, nil
 }
 
 // TCP Senders
 
+type TCPSender interface {
+	Send([]byte) error
+}
+
+type SimpleTCPSender struct {
+	encoder *gob.Encoder
+}
+
+type EncryptedTCPSender struct {
+	sync.RWMutex
+	outerEncoder *gob.Encoder
+	innerEncoder *gob.Encoder
+	buffer       *bytes.Buffer
+	conn         *LocalConnection
+	msgCount     int
+}
+
+type EncryptedTCPMessage struct {
+	Number int
+	Body   []byte
+}
+
 func NewSimpleTCPSender(encoder *gob.Encoder) *SimpleTCPSender {
 	return &SimpleTCPSender{encoder: encoder}
 }
@@ -416,6 +482,20 @@ func (sender *EncryptedTCPSender) Send(msg []byte) error {
 
 // TCP Receivers
 
+type TCPReceiver interface {
+	Decode([]byte) ([]byte, error)
+}
+
+type SimpleTCPReceiver struct {
+}
+
+type EncryptedTCPReceiver struct {
+	conn     *LocalConnection
+	decoder  *gob.Decoder
+	buffer   *bytes.Buffer
+	msgCount int
+}
+
 func NewSimpleTCPReceiver() *SimpleTCPReceiver {
 	return &SimpleTCPReceiver{}
 }

router/ethernet_decoder.go

@@ -8,6 +8,13 @@ import (
 	"net"
 )
 
+type EthernetDecoder struct {
+	eth     layers.Ethernet
+	ip      layers.IPv4
+	decoded []gopacket.LayerType
+	parser  *gopacket.DecodingLayerParser
+}
+
 func NewEthernetDecoder() *EthernetDecoder {
 	dec := &EthernetDecoder{}
 	dec.parser = gopacket.NewDecodingLayerParser(layers.LayerTypeEthernet, &dec.eth, &dec.ip)
@@ -18,21 +25,7 @@ func (dec *EthernetDecoder) DecodeLayers(data []byte) error {
 	return dec.parser.DecodeLayers(data, &dec.decoded)
 }
 
-func (dec *EthernetDecoder) CheckFrameTooBig(err error, sendFrame func([]byte) error) error {
-	if ftbe, ok := err.(FrameTooBigError); ok {
-		// we know: 1. ip is valid, 2. it was ip and DF was set
-		icmpFrame, err := dec.formICMPMTUPacket(ftbe.EPMTU)
-		if err != nil {
-			return err
-		}
-		log.Printf("Sending ICMP 3,4 (%v -> %v): PMTU= %v\n", dec.ip.DstIP, dec.ip.SrcIP, ftbe.EPMTU)
-		return sendFrame(icmpFrame)
-	} else {
-		return err
-	}
-}
-
-func (dec *EthernetDecoder) formICMPMTUPacket(mtu int) ([]byte, error) {
+func (dec *EthernetDecoder) sendICMPFragNeeded(mtu int, sendFrame func([]byte) error) error {
 	buf := gopacket.NewSerializeBuffer()
 	opts := gopacket.SerializeOptions{
 		FixLengths:       true,
@@ -60,9 +53,11 @@ func (dec *EthernetDecoder) formICMPMTUPacket(mtu int) ([]byte, error) {
 			Seq:      uint16(mtu)},
 		&payload)
 	if err != nil {
-		return []byte{}, err
+		return err
 	}
-	return buf.Bytes(), nil
+
+	log.Printf("Sending ICMP 3,4 (%v -> %v): PMTU= %v\n", dec.ip.DstIP, dec.ip.SrcIP, mtu)
+	return sendFrame(buf.Bytes())
 }
 
 var (
@@ -75,7 +70,7 @@ func (dec *EthernetDecoder) DropFrame() bool {
 	return bytes.Equal(stpMACPrefix, dec.eth.DstMAC[:len(stpMACPrefix)])
 }
 
-func (dec *EthernetDecoder) IsPMTUVerify() bool {
-	return bytes.Equal(zeroMAC, dec.eth.SrcMAC) &&
-		bytes.Equal(zeroMAC, dec.eth.DstMAC)
+func (dec *EthernetDecoder) IsSpecial() bool {
+	return dec.eth.Length == 0 && dec.eth.EthernetType == layers.EthernetTypeLLC &&
+		bytes.Equal(zeroMAC, dec.eth.SrcMAC) && bytes.Equal(zeroMAC, dec.eth.DstMAC)
 }

router/forwarder.go

@@ -3,14 +3,22 @@ package router
 import (
 	"code.google.com/p/gopacket"
 	"code.google.com/p/gopacket/layers"
-	"log"
-	"net"
 	"syscall"
 	"time"
 )
 
+type ForwardedFrame struct {
+	srcPeer *Peer
+	dstPeer *Peer
+	frame   []byte
+}
+
+type FrameTooBigError struct {
+	EPMTU int // effective pmtu, i.e. what we tell packet senders
+}
+
 func (conn *LocalConnection) ensureForwarders() error {
-	if conn.forwardChan != nil || conn.forwardChanDF != nil {
+	if conn.forwarder != nil || conn.forwarderDF != nil {
 		return nil
 	}
 	udpSender := NewSimpleUDPSender(conn)
@@ -29,47 +37,29 @@ func (conn *LocalConnection) ensureForwarders() error {
 		encryptorDF = NewNonEncryptor(conn.local.NameByte)
 	}
 
-	var (
-		forwardChan   = make(chan *ForwardedFrame, ChannelSize)
-		forwardChanDF = make(chan *ForwardedFrame, ChannelSize)
-		stopForward   = make(chan interface{}, 0)
-		stopForwardDF = make(chan interface{}, 0)
-		verifyPMTU    = make(chan int, ChannelSize)
-	)
-	//NB: only forwarderDF can ever encounter EMSGSIZE errors, and
-	//thus perform PMTU verification
-	forwarder := NewForwarder(conn, forwardChan, stopForward, nil, encryptor, udpSender, DefaultPMTU)
-	forwarderDF := NewForwarder(conn, forwardChanDF, stopForwardDF, verifyPMTU, encryptorDF, udpSenderDF, DefaultPMTU)
+	forwarder := NewForwarder(conn, encryptor, udpSender, DefaultPMTU)
+	forwarderDF := NewForwarderDF(conn, encryptorDF, udpSenderDF, DefaultPMTU)
+	effectivePMTU := forwarderDF.unverifiedPMTU
+	forwarder.Start()
+	forwarderDF.Start()
 
 	// Various fields in the conn struct are read by other processes,
 	// so we have to use locks.
 	conn.Lock()
-	conn.forwardChan = forwardChan
-	conn.forwardChanDF = forwardChanDF
-	conn.stopForward = stopForward
-	conn.stopForwardDF = stopForwardDF
-	conn.verifyPMTU = verifyPMTU
-	conn.effectivePMTU = forwarder.unverifiedPMTU
+	conn.forwarder = forwarder
+	conn.forwarderDF = forwarderDF
+	conn.effectivePMTU = effectivePMTU
 	conn.Unlock()
 
-	go forwarder.Run()
-	go forwarderDF.Run()
-
 	return nil
 }
 
 func (conn *LocalConnection) stopForwarders() {
-	conn.Lock()
-	conn.forwardChan = nil
-	conn.forwardChanDF = nil
-	conn.Unlock()
-	// Now signal the forwarder loops to exit. They will drain the
-	// forwarder chans in order to unblock any router processes
-	// blocked on sending.
-	if conn.stopForward != nil {
-		conn.stopForward <- nil
-		conn.stopForwardDF <- nil
+	if conn.forwarder == nil || conn.forwarderDF == nil {
+		return
 	}
+	conn.forwarder.Shutdown()
+	conn.forwarderDF.Shutdown()
 }
 
 // Called from peer.Relay[Broadcast] which is itself invoked from
@@ -79,15 +69,15 @@ func (conn *LocalConnection) stopForwarders() {
 func (conn *LocalConnection) Forward(df bool, frame *ForwardedFrame, dec *EthernetDecoder) error {
 	conn.RLock()
 	var (
-		forwardChan   = conn.forwardChan
-		forwardChanDF = conn.forwardChanDF
+		forwarder     = conn.forwarder
+		forwarderDF   = conn.forwarderDF
 		effectivePMTU = conn.effectivePMTU
 		stackFrag     = conn.stackFrag
 	)
 	conn.RUnlock()
 
-	if forwardChan == nil || forwardChanDF == nil {
-		conn.log("Cannot forward frame yet - awaiting contact")
+	if forwarder == nil || forwarderDF == nil {
+		conn.Log("Cannot forward frame yet - awaiting contact")
 		return nil
 	}
 	// We could use non-blocking channel sends here, i.e. drop frames
@@ -102,29 +92,29 @@ func (conn *LocalConnection) Forward(df bool, frame *ForwardedFrame, dec *Ethern
 	// of our pipeline.
 	if df {
 		if !frameTooBig(frame, effectivePMTU) {
-			forwardChanDF <- frame
+			forwarderDF.Forward(frame)
 			return nil
 		}
 		return FrameTooBigError{EPMTU: effectivePMTU}
-	} else {
-		if stackFrag || dec == nil || len(dec.decoded) < 2 {
-			forwardChan <- frame
-			return nil
-		}
-		// Don't have trustworthy stack, so we're going to have to
-		// send it DF in any case.
-		if !frameTooBig(frame, effectivePMTU) {
-			forwardChanDF <- frame
-			return nil
-		}
-		conn.Router.LogFrame("Fragmenting", frame.frame, &dec.eth)
-		// We can't trust the stack to fragment, we have IP, and we
-		// have a frame that's too big for the MTU, so we have to
-		// fragment it ourself.
-		return fragment(dec.eth, dec.ip, effectivePMTU, frame, func(segFrame *ForwardedFrame) {
-			forwardChanDF <- segFrame
-		})
 	}
+
+	if stackFrag || dec == nil || len(dec.decoded) < 2 {
+		forwarder.Forward(frame)
+		return nil
+	}
+	// Don't have trustworthy stack, so we're going to have to
+	// send it DF in any case.
+	if !frameTooBig(frame, effectivePMTU) {
+		forwarderDF.Forward(frame)
+		return nil
+	}
+	conn.Router.LogFrame("Fragmenting", frame.frame, &dec.eth)
+	// We can't trust the stack to fragment, we have IP, and we
+	// have a frame that's too big for the MTU, so we have to
+	// fragment it ourself.
+	return fragment(dec.eth, dec.ip, effectivePMTU, frame, func(segFrame *ForwardedFrame) {
+		forwarderDF.Forward(segFrame)
+	})
 }
 
 func frameTooBig(frame *ForwardedFrame, effectivePMTU int) bool {
@@ -184,7 +174,7 @@ func fragment(eth layers.Ethernet, ip layers.IPv4, pmtu int, frame *ForwardedFra
 			return err
 		}
 		// make copies of the frame we received
-		var segFrame ForwardedFrame = *frame
+		segFrame := *frame
 		segFrame.frame = buf.Bytes()
 		forward(&segFrame)
 	}
@@ -193,33 +183,164 @@ func fragment(eth layers.Ethernet, ip layers.IPv4, pmtu int, frame *ForwardedFra
 
 // Forwarder
 
-func NewForwarder(conn *LocalConnection, ch <-chan *ForwardedFrame, stop <-chan interface{}, verifyPMTU <-chan int, enc Encryptor, udpSender UDPSender, pmtu int) *Forwarder {
-	fwd := &Forwarder{
-		conn:       conn,
-		ch:         ch,
-		stop:       stop,
-		verifyPMTU: verifyPMTU,
-		enc:        enc,
-		udpSender:  udpSender}
+type Forwarder struct {
+	conn             *LocalConnection
+	ch               chan<- *ForwardedFrame
+	finished         <-chan struct{}
+	enc              Encryptor
+	udpSender        UDPSender
+	maxPayload       int
+	processSendError func(error) error
+}
+
+func NewForwarder(conn *LocalConnection, enc Encryptor, udpSender UDPSender, pmtu int) *Forwarder {
+	return &Forwarder{
+		conn:             conn,
+		enc:              enc,
+		udpSender:        udpSender,
+		maxPayload:       pmtu - UDPOverhead,
+		processSendError: func(err error) error { return err }}
+}
+
+func (fwd *Forwarder) Start() {
+	ch := make(chan *ForwardedFrame, ChannelSize)
+	fwd.ch = ch
+	finished := make(chan struct{})
+	fwd.finished = finished
+	go fwd.run(ch, finished)
+}
+
+func (fwd *Forwarder) Shutdown() {
+	fwd.ch <- nil
+}
+
+func (fwd *Forwarder) Forward(frame *ForwardedFrame) {
+	select {
+	case fwd.ch <- frame:
+	case <-fwd.finished:
+	}
+}
+
+func (fwd *Forwarder) run(ch <-chan *ForwardedFrame, finished chan<- struct{}) {
+	defer fwd.udpSender.Shutdown()
+	for {
+		if !fwd.accumulateAndSendFrames(ch, <-ch) {
+			close(finished)
+			return
+		}
+	}
+}
+
+func (fwd *Forwarder) effectiveOverhead() int {
+	return UDPOverhead + fwd.enc.PacketOverhead() + fwd.enc.FrameOverhead() + EthernetOverhead
+}
+
+// Drain the inbound channel of frames, aggregating them into larger
+// packets for efficient transmission.
+//
+// FIXME Depending on the golang scheduler, and the rate at which
+// franes get sent to the forwarder, we can be going around this loop
+// forever. That is bad since there may be other stuff for us to do,
+// i.e. the other branches in the run loop.
+func (fwd *Forwarder) accumulateAndSendFrames(ch <-chan *ForwardedFrame, frame *ForwardedFrame) bool {
+	if frame == nil {
+		return false
+	}
+	if !fwd.appendFrame(frame) {
+		fwd.logDrop(frame)
+		// [1] The buffer is empty at this point and therefore we must
+		// not flush it. The easiest way to accomplish that is simply
+		// by returning to the surrounding run loop.
+		return true
+	}
+	for {
+		select {
+		case frame = <-ch:
+			if frame == nil {
+				return false
+			}
+			if !fwd.appendFrame(frame) {
+				fwd.flush()
+				if !fwd.appendFrame(frame) {
+					fwd.logDrop(frame)
+					return true // see [1]
+				}
+			}
+		default:
+			fwd.flush()
+			return true
+		}
+	}
+}
+
+func (fwd *Forwarder) logDrop(frame *ForwardedFrame) {
+	fwd.conn.Log("Dropping too big frame during forwarding: frame len:", len(frame.frame), "; effective PMTU:", fwd.maxPayload+UDPOverhead-fwd.effectiveOverhead())
+}
+
+func (fwd *Forwarder) appendFrame(frame *ForwardedFrame) bool {
+	frameLen := len(frame.frame)
+	if fwd.enc.TotalLen()+fwd.enc.FrameOverhead()+frameLen > fwd.maxPayload {
+		return false
+	}
+	fwd.enc.AppendFrame(frame.srcPeer.NameByte, frame.dstPeer.NameByte, frame.frame)
+	return true
+}
+
+func (fwd *Forwarder) flush() {
+	err := fwd.processSendError(fwd.udpSender.Send(fwd.enc.Bytes()))
+	if err != nil && PosixError(err) != syscall.ENOBUFS {
+		fwd.conn.Shutdown(err)
+	}
+}
+
+type ForwarderDF struct {
+	Forwarder
+	verifyPMTUTick  <-chan time.Time
+	verifyPMTU      chan<- int
+	pmtuVerifyCount uint
+	pmtuVerified    bool
+	highestGoodPMTU int
+	unverifiedPMTU  int
+	lowestBadPMTU   int
+}
+
+func NewForwarderDF(conn *LocalConnection, enc Encryptor, udpSender UDPSender, pmtu int) *ForwarderDF {
+	fwd := &ForwarderDF{
+		Forwarder: Forwarder{
+			conn:       conn,
+			enc:        enc,
+			udpSender:  udpSender,
+			maxPayload: pmtu - UDPOverhead}}
+	fwd.Forwarder.processSendError = fwd.processSendError
 	fwd.unverifiedPMTU = pmtu - fwd.effectiveOverhead()
-	fwd.maxPayload = pmtu - UDPOverhead
 	return fwd
 }
 
-func (fwd *Forwarder) Run() {
+func (fwd *ForwarderDF) Start() {
+	ch := make(chan *ForwardedFrame, ChannelSize)
+	fwd.ch = ch
+	finished := make(chan struct{})
+	fwd.finished = finished
+	verifyPMTU := make(chan int, ChannelSize)
+	fwd.verifyPMTU = verifyPMTU
+	go fwd.run(ch, finished, verifyPMTU)
+}
+
+func (fwd *ForwarderDF) PMTUVerified(pmtu int) {
+	select {
+	case fwd.verifyPMTU <- pmtu:
+	case <-fwd.finished:
+	}
+}
+
+func (fwd *ForwarderDF) run(ch <-chan *ForwardedFrame, finished chan<- struct{}, verifyPMTU <-chan int) {
 	defer fwd.udpSender.Shutdown()
-	var flushed, ok bool
-	var frame *ForwardedFrame
 	for {
-		flushed = false
 		select {
-		case <-fwd.stop:
-			fwd.drain()
-			return
 		case <-fwd.verifyPMTUTick:
-			// We only do this case here when we know the buffers are
-			// all empty so that we don't risk appending verify-frames
-			// to other data.
+			// We only do this case here when we know the
+			// buffers are all empty so that we don't risk
+			// appending verify-frames to other data.
 			fwd.verifyPMTUTick = nil
 			if fwd.pmtuVerified {
 				continue
@@ -228,12 +349,12 @@ func (fwd *Forwarder) Run() {
 				fwd.pmtuVerifyCount--
 				fwd.attemptVerifyEffectivePMTU()
 			} else {
-				// we've exceeded the verification attempts of the
-				// unverifiedPMTU
+				// we've exceeded the verification
+				// attempts of the unverifiedPMTU
 				fwd.lowestBadPMTU = fwd.unverifiedPMTU
 				fwd.verifyEffectivePMTU((fwd.highestGoodPMTU + fwd.lowestBadPMTU) / 2)
 			}
-		case epmtu := <-fwd.verifyPMTU:
+		case epmtu := <-verifyPMTU:
 			if fwd.pmtuVerified || epmtu != fwd.unverifiedPMTU {
 				continue
 			}
@@ -244,210 +365,46 @@ func (fwd *Forwarder) Run() {
 				fwd.pmtuVerified = true
 				fwd.maxPayload = epmtu + fwd.effectiveOverhead() - UDPOverhead
 				fwd.conn.setEffectivePMTU(epmtu)
-				fwd.conn.log("Effective PMTU verified at", epmtu)
+				fwd.conn.Log("Effective PMTU verified at", epmtu)
 			}
-		case frame = <-fwd.ch:
-			if !fwd.appendFrame(frame) {
-				fwd.logDrop(frame)
-				continue
-			}
-			for !flushed {
-				select {
-				case frame, ok = <-fwd.ch:
-					if !ok {
-						return
-					}
-					if !fwd.appendFrame(frame) {
-						fwd.flush()
-						if !fwd.appendFrame(frame) {
-							fwd.logDrop(frame)
-							flushed = true
-						}
-					}
-				default:
-					fwd.flush()
-					flushed = true
-				}
+		case frame := <-ch:
+			if !fwd.accumulateAndSendFrames(ch, frame) {
+				close(finished)
+				return
 			}
 		}
 	}
 }
 
-func (fwd *Forwarder) effectiveOverhead() int {
-	return UDPOverhead + fwd.enc.PacketOverhead() + fwd.enc.FrameOverhead() + EthernetOverhead
-}
-
-func (fwd *Forwarder) verifyEffectivePMTU(newUnverifiedPMTU int) {
+func (fwd *ForwarderDF) verifyEffectivePMTU(newUnverifiedPMTU int) {
 	fwd.unverifiedPMTU = newUnverifiedPMTU
 	fwd.pmtuVerifyCount = PMTUVerifyAttempts
 	fwd.attemptVerifyEffectivePMTU()
 }
 
-func (fwd *Forwarder) attemptVerifyEffectivePMTU() {
-	pmtuVerifyFrame := &ForwardedFrame{
-		srcPeer: fwd.conn.local,
-		dstPeer: fwd.conn.remote,
-		frame:   make([]byte, fwd.unverifiedPMTU+EthernetOverhead)}
-	fwd.enc.AppendFrame(pmtuVerifyFrame)
+func (fwd *ForwarderDF) attemptVerifyEffectivePMTU() {
+	fwd.enc.AppendFrame(fwd.conn.local.NameByte, fwd.conn.remote.NameByte,
+		make([]byte, fwd.unverifiedPMTU+EthernetOverhead))
 	fwd.flush()
 	if fwd.verifyPMTUTick == nil {
 		fwd.verifyPMTUTick = time.After(PMTUVerifyTimeout << (PMTUVerifyAttempts - fwd.pmtuVerifyCount))
 	}
 }
 
-func (fwd *Forwarder) appendFrame(frame *ForwardedFrame) bool {
-	frameLen := len(frame.frame)
-	if fwd.enc.TotalLen()+fwd.enc.FrameOverhead()+frameLen > fwd.maxPayload {
-		return false
-	}
-	fwd.enc.AppendFrame(frame)
-	return true
-}
-
-func (fwd *Forwarder) flush() {
-	err := fwd.udpSender.Send(fwd.enc.Bytes())
-	if err != nil {
-		if mtbe, ok := err.(MsgTooBigError); ok {
-			newUnverifiedPMTU := mtbe.PMTU - fwd.effectiveOverhead()
-			if newUnverifiedPMTU >= fwd.unverifiedPMTU {
-				return
-			}
+func (fwd *ForwarderDF) processSendError(err error) error {
+	if mtbe, ok := err.(MsgTooBigError); ok {
+		newUnverifiedPMTU := mtbe.PMTU - fwd.effectiveOverhead()
+		if newUnverifiedPMTU < fwd.unverifiedPMTU {
 			fwd.pmtuVerified = false
 			fwd.maxPayload = mtbe.PMTU - UDPOverhead
 			fwd.highestGoodPMTU = 8
 			fwd.lowestBadPMTU = newUnverifiedPMTU + 1
 			fwd.conn.setEffectivePMTU(newUnverifiedPMTU)
 			fwd.verifyEffectivePMTU(newUnverifiedPMTU)
-		} else if PosixError(err) == syscall.ENOBUFS {
-			// TODO handle this better
-		} else {
-			fwd.conn.CheckFatal(err)
 		}
+
+		return nil
 	}
-}
 
-func (fwd *Forwarder) drain() {
-	// We want to drain before exiting otherwise we could get the
-	// packet sniffer or udp listener blocked on sending to a full
-	// chan
-	for {
-		select {
-		case <-fwd.ch:
-		default:
-			return
-		}
-	}
-}
-
-func (fwd *Forwarder) logDrop(frame *ForwardedFrame) {
-	fwd.conn.log("Dropping too big frame during forwarding: frame len:", len(frame.frame), "; effective PMTU:", fwd.maxPayload+UDPOverhead-fwd.effectiveOverhead())
-}
-
-// UDP Senders
-
-func NewSimpleUDPSender(conn *LocalConnection) *SimpleUDPSender {
-	return &SimpleUDPSender{udpConn: conn.Router.UDPListener, conn: conn}
-}
-
-func (sender *SimpleUDPSender) Send(msg []byte) error {
-	_, err := sender.udpConn.WriteToUDP(msg, sender.conn.RemoteUDPAddr())
 	return err
 }
-
-func (sender *SimpleUDPSender) Shutdown() error {
-	return nil
-}
-
-func NewRawUDPSender(conn *LocalConnection) (*RawUDPSender, error) {
-	ipSocket, err := dialIP(conn)
-	if err != nil {
-		return nil, err
-	}
-	udpHeader := &layers.UDP{SrcPort: layers.UDPPort(Port)}
-	ipBuf := gopacket.NewSerializeBuffer()
-	opts := gopacket.SerializeOptions{
-		FixLengths: true,
-		// UDP header is calculated with a phantom IP
-		// header. Yes, it's totally nuts. Thankfully, for UDP
-		// over IPv4, the checksum is optional. It's not
-		// optional for IPv6, but we'll ignore that for
-		// now. TODO
-		ComputeChecksums: false}
-
-	return &RawUDPSender{
-		ipBuf:     ipBuf,
-		opts:      opts,
-		udpHeader: udpHeader,
-		socket:    ipSocket,
-		conn:      conn}, nil
-}
-
-func (sender *RawUDPSender) Send(msg []byte) error {
-	payload := gopacket.Payload(msg)
-	sender.udpHeader.DstPort = layers.UDPPort(sender.conn.RemoteUDPAddr().Port)
-
-	err := gopacket.SerializeLayers(sender.ipBuf, sender.opts, sender.udpHeader, &payload)
-	if err != nil {
-		return err
-	}
-	packet := sender.ipBuf.Bytes()
-	_, err = sender.socket.Write(packet)
-	if err == nil || PosixError(err) != syscall.EMSGSIZE {
-		return err
-	}
-	f, err := sender.socket.File()
-	if err != nil {
-		return err
-	}
-	defer f.Close()
-	fd := int(f.Fd())
-	log.Println("EMSGSIZE on send, expecting PMTU update (IP packet was",
-		len(packet), "bytes, payload was", len(msg), "bytes)")
-	pmtu, err := syscall.GetsockoptInt(fd, syscall.IPPROTO_IP, syscall.IP_MTU)
-	if err != nil {
-		return err
-	}
-	return MsgTooBigError{PMTU: pmtu}
-}
-
-func (sender *RawUDPSender) Shutdown() error {
-	defer func() { sender.socket = nil }()
-	return sender.socket.Close()
-}
-
-func dialIP(conn *LocalConnection) (*net.IPConn, error) {
-	ipLocalAddr, err := ipAddr(conn.TCPConn.LocalAddr())
-	if err != nil {
-		return nil, err
-	}
-	ipRemoteAddr, err := ipAddr(conn.TCPConn.RemoteAddr())
-	if err != nil {
-		return nil, err
-	}
-	ipSocket, err := net.DialIP("ip4:UDP", ipLocalAddr, ipRemoteAddr)
-	if err != nil {
-		return nil, err
-	}
-	f, err := ipSocket.File()
-	if err != nil {
-		return nil, err
-	}
-	defer f.Close()
-	fd := int(f.Fd())
-	// This Makes sure all packets we send out have DF set on them.
-	err = syscall.SetsockoptInt(fd, syscall.IPPROTO_IP, syscall.IP_MTU_DISCOVER, syscall.IP_PMTUDISC_DO)
-	if err != nil {
-		return nil, err
-	}
-	return ipSocket, nil
-}
-
-func ipAddr(addr net.Addr) (*net.IPAddr, error) {
-	host, _, err := net.SplitHostPort(addr.String())
-	if err != nil {
-		return nil, err
-	}
-	return &net.IPAddr{
-		IP:   net.ParseIP(host),
-		Zone: ""}, nil
-}

router/gossip.go

@@ -0,0 +1,337 @@
+package router
+
+import (
+	"bytes"
+	"encoding/gob"
+	"fmt"
+	"log"
+	"sync"
+	"time"
+)
+
+const GossipInterval = 30 * time.Second
+
+type GossipData interface {
+	Encode() []byte
+	Merge(GossipData)
+}
+
+type Gossip interface {
+	// specific message from one peer to another
+	// intermediate peers relay it using unicast topology.
+	GossipUnicast(dstPeerName PeerName, msg []byte) error
+	// send gossip to every peer, relayed using broadcast topology.
+	GossipBroadcast(update GossipData) error
+}
+
+type Gossiper interface {
+	OnGossipUnicast(sender PeerName, msg []byte) error
+	// merge received data into state and return a representation of
+	// the received data, for further propagation
+	OnGossipBroadcast(update []byte) (GossipData, error)
+	// return state of everything we know; gets called periodically
+	Gossip() GossipData
+	// merge received data into state and return "everything new I've
+	// just learnt", or nil if nothing in the received data was new
+	OnGossip(update []byte) (GossipData, error)
+}
+
+// Accumulates GossipData that needs to be sent to one destination,
+// and sends it when possible.
+type GossipSender struct {
+	send func(GossipData)
+	cell chan GossipData
+}
+
+func NewGossipSender(send func(GossipData)) *GossipSender {
+	return &GossipSender{send: send}
+}
+
+func (sender *GossipSender) Start() {
+	sender.cell = make(chan GossipData, 1)
+	go sender.run()
+}
+
+func (sender *GossipSender) run() {
+	for {
+		if pending := <-sender.cell; pending == nil { // receive zero value when chan is closed
+			break
+		} else {
+			sender.send(pending)
+		}
+	}
+}
+
+func (sender *GossipSender) Send(data GossipData) {
+	// NB: this must not be invoked concurrently
+	select {
+	case pending := <-sender.cell:
+		pending.Merge(data)
+		sender.cell <- pending
+	default:
+		sender.cell <- data
+	}
+}
+
+func (sender *GossipSender) Stop() {
+	close(sender.cell)
+}
+
+type connectionSenders map[Connection]*GossipSender
+type peerSenders map[PeerName]*GossipSender
+
+type GossipChannel struct {
+	sync.Mutex
+	ourself      *LocalPeer
+	routes       *Routes
+	name         string
+	hash         uint32
+	gossiper     Gossiper
+	senders      connectionSenders
+	broadcasters peerSenders
+}
+
+func (router *Router) NewGossip(channelName string, g Gossiper) Gossip {
+	channelHash := hash(channelName)
+	channel := &GossipChannel{
+		ourself:      router.Ourself,
+		routes:       router.Routes,
+		name:         channelName,
+		hash:         channelHash,
+		gossiper:     g,
+		senders:      make(connectionSenders),
+		broadcasters: make(peerSenders)}
+	router.GossipChannels[channelHash] = channel
+	return channel
+}
+
+func (router *Router) SendAllGossip() {
+	for _, channel := range router.GossipChannels {
+		channel.Send(router.Ourself.Name, channel.gossiper.Gossip())
+	}
+}
+
+func (router *Router) SendAllGossipDown(conn Connection) {
+	for _, channel := range router.GossipChannels {
+		channel.SendDown(conn, channel.gossiper.Gossip())
+	}
+}
+
+func (router *Router) handleGossip(tag ProtocolTag, payload []byte) error {
+	decoder := gob.NewDecoder(bytes.NewReader(payload))
+	var channelHash uint32
+	if err := decoder.Decode(&channelHash); err != nil {
+		return err
+	}
+	channel, found := router.GossipChannels[channelHash]
+	if !found {
+		return fmt.Errorf("[gossip] received unknown channel with hash %v", channelHash)
+	}
+	var srcName PeerName
+	if err := decoder.Decode(&srcName); err != nil {
+		return err
+	}
+	switch tag {
+	case ProtocolGossipUnicast:
+		return channel.deliverUnicast(srcName, payload, decoder)
+	case ProtocolGossipBroadcast:
+		return channel.deliverBroadcast(srcName, payload, decoder)
+	case ProtocolGossip:
+		return channel.deliver(srcName, payload, decoder)
+	}
+	return nil
+}
+
+func (c *GossipChannel) deliverUnicast(srcName PeerName, origPayload []byte, dec *gob.Decoder) error {
+	var destName PeerName
+	if err := dec.Decode(&destName); err != nil {
+		return err
+	}
+	if c.ourself.Name != destName {
+		return c.relayUnicast(destName, origPayload)
+	}
+	var payload []byte
+	if err := dec.Decode(&payload); err != nil {
+		return err
+	}
+	return c.gossiper.OnGossipUnicast(srcName, payload)
+}
+
+func (c *GossipChannel) deliverBroadcast(srcName PeerName, _ []byte, dec *gob.Decoder) error {
+	var payload []byte
+	if err := dec.Decode(&payload); err != nil {
+		return err
+	}
+	data, err := c.gossiper.OnGossipBroadcast(payload)
+	if err != nil || data == nil {
+		return err
+	}
+	return c.relayBroadcast(srcName, data)
+}
+
+func (c *GossipChannel) deliver(srcName PeerName, _ []byte, dec *gob.Decoder) error {
+	var payload []byte
+	if err := dec.Decode(&payload); err != nil {
+		return err
+	}
+	if data, err := c.gossiper.OnGossip(payload); err != nil {
+		return err
+	} else if data != nil {
+		c.Send(srcName, data)
+	}
+	return nil
+}
+
+func (c *GossipChannel) Send(srcName PeerName, data GossipData) {
+	// do this outside the lock below so we avoid lock nesting
+	c.routes.EnsureRecalculated()
+	selectedConnections := make(ConnectionSet)
+	for name := range c.routes.RandomNeighbours(srcName) {
+		if conn, found := c.ourself.ConnectionTo(name); found {
+			selectedConnections[conn] = void
+		}
+	}
+	if len(selectedConnections) == 0 {
+		return
+	}
+	connections := c.ourself.Connections()
+	c.Lock()
+	defer c.Unlock()
+	// GC - randomly (courtesy of go's map iterator) pick some
+	// existing entries and stop&remove them if the associated
+	// connection is no longer active.  We stop as soon as we
+	// encounter a valid entry; the idea being that when there is
+	// little or no garbage then this executes close to O(1)[1],
+	// whereas when there is lots of garbage we remove it quickly.
+	//
+	// [1] TODO Unfortunately, due to the desire to avoid nested
+	// locks, instead of simply invoking Peer.ConnectionTo(name)
+	// below, we have that Peer.Connections() invocation above. That
+	// is O(n_our_connections) at best.
+	for conn, sender := range c.senders {
+		if _, found := connections[conn]; !found {
+			delete(c.senders, conn)
+			sender.Stop()
+		} else {
+			break
+		}
+	}
+	for conn := range selectedConnections {
+		c.sendDown(conn, data)
+	}
+}
+
+func (c *GossipChannel) SendDown(conn Connection, data GossipData) {
+	c.Lock()
+	c.sendDown(conn, data)
+	c.Unlock()
+}
+
+func (c *GossipChannel) sendDown(conn Connection, data GossipData) {
+	sender, found := c.senders[conn]
+	if !found {
+		sender = NewGossipSender(func(pending GossipData) {
+			protocolMsg := ProtocolMsg{ProtocolGossip, GobEncode(c.hash, c.ourself.Name, pending.Encode())}
+			conn.(ProtocolSender).SendProtocolMsg(protocolMsg)
+		})
+		c.senders[conn] = sender
+		sender.Start()
+	}
+	sender.Send(data)
+}
+
+func (c *GossipChannel) GossipUnicast(dstPeerName PeerName, msg []byte) error {
+	return c.relayUnicast(dstPeerName, GobEncode(c.hash, c.ourself.Name, dstPeerName, msg))
+}
+
+func (c *GossipChannel) GossipBroadcast(update GossipData) error {
+	return c.relayBroadcast(c.ourself.Name, update)
+}
+
+func (c *GossipChannel) relayUnicast(dstPeerName PeerName, buf []byte) error {
+	if relayPeerName, found := c.routes.UnicastAll(dstPeerName); !found {
+		c.log("unknown relay destination:", dstPeerName)
+	} else if conn, found := c.ourself.ConnectionTo(relayPeerName); !found {
+		c.log("unable to find connection to relay peer", relayPeerName)
+	} else {
+		conn.(ProtocolSender).SendProtocolMsg(ProtocolMsg{ProtocolGossipUnicast, buf})
+	}
+	return nil
+}
+
+func (c *GossipChannel) relayBroadcast(srcName PeerName, update GossipData) error {
+	names := c.routes.PeerNames() // do this outside the lock so they don't nest
+	c.Lock()
+	defer c.Unlock()
+	// GC - randomly (courtesy of go's map iterator) pick some
+	// existing broadcasters and stop&remove them if their source peer
+	// is unknown. We stop as soon as we encounter a valid entry; the
+	// idea being that when there is little or no garbage then this
+	// executes close to O(1)[1], whereas when there is lots of
+	// garbage we remove it quickly.
+	//
+	// [1] TODO Unfortunately, due to the desire to avoid nested
+	// locks, instead of simply invoking Peers.Fetch(name) below, we
+	// have that Peers.Names() invocation above. That is O(n_peers) at
+	// best.
+	for name, broadcaster := range c.broadcasters {
+		if _, found := names[name]; !found {
+			delete(c.broadcasters, name)
+			broadcaster.Stop()
+		} else {
+			break
+		}
+	}
+	broadcaster, found := c.broadcasters[srcName]
+	if !found {
+		broadcaster = NewGossipSender(func(pending GossipData) { c.sendBroadcast(srcName, pending) })
+		c.broadcasters[srcName] = broadcaster
+		broadcaster.Start()
+	}
+	broadcaster.Send(update)
+	return nil
+}
+
+func (c *GossipChannel) sendBroadcast(srcName PeerName, update GossipData) {
+	c.routes.EnsureRecalculated()
+	nextHops := c.routes.BroadcastAll(srcName)
+	if len(nextHops) == 0 {
+		return
+	}
+	protocolMsg := ProtocolMsg{ProtocolGossipBroadcast, GobEncode(c.hash, srcName, update.Encode())}
+	// FIXME a single blocked connection can stall us
+	for _, conn := range c.ourself.ConnectionsTo(nextHops) {
+		conn.(ProtocolSender).SendProtocolMsg(protocolMsg)
+	}
+}
+
+func (c *GossipChannel) log(args ...interface{}) {
+	log.Println(append(append([]interface{}{}, "[gossip "+c.name+"]:"), args...)...)
+}
+
+// for testing
+
+// FIXME this doesn't actually guarantee everything has been sent
+// since a GossipSender may be in the process of sending and there is
+// no easy way for us to know when that has completed.
+func (router *Router) sendPendingGossip() {
+	for _, channel := range router.GossipChannels {
+		for _, sender := range channel.senders {
+			sender.flush()
+		}
+		for _, sender := range channel.broadcasters {
+			sender.flush()
+		}
+	}
+}
+
+func (sender *GossipSender) flush() {
+	for {
+		select {
+		case pending := <-sender.cell:
+			sender.send(pending)
+		default:
+			return
+		}
+	}
+}

router/gossip_test.go

@@ -0,0 +1,149 @@
+package router
+
+import (
+	wt "github.com/weaveworks/weave/testing"
+	"testing"
+	"time"
+)
+
+// TODO test gossip unicast and broadcast; atm we only test topology
+// gossip, which does not employ unicast or broadcast.
+
+type mockChannelConnection struct {
+	RemoteConnection
+	dest *Router
+}
+
+// Construct a "passive" Router, i.e. without any goroutines, except
+// for Routes and GossipSenders.
+func NewTestRouter(name PeerName) *Router {
+	router := NewRouter(RouterConfig{}, name, "")
+	// need to create a dummy channel otherwise tests hang on nil
+	// channels when the Router invoked ConnectionMaker.Refresh
+	router.ConnectionMaker.actionChan = make(chan ConnectionMakerAction, ChannelSize)
+	router.Routes.Start()
+	return router
+}
+
+func (conn *mockChannelConnection) SendProtocolMsg(protocolMsg ProtocolMsg) {
+	if err := conn.dest.handleGossip(protocolMsg.tag, protocolMsg.msg); err != nil {
+		panic(err)
+	}
+	conn.dest.sendPendingGossip()
+}
+
+func (router *Router) AddTestChannelConnection(r *Router) {
+	fromName := router.Ourself.Peer.Name
+	toName := r.Ourself.Peer.Name
+
+	fromPeer := NewPeer(fromName, "", router.Ourself.Peer.UID, 0)
+	toPeer := NewPeer(toName, "", r.Ourself.Peer.UID, 0)
+
+	r.Peers.FetchWithDefault(fromPeer)    // Has side-effect of incrementing refcount
+	router.Peers.FetchWithDefault(toPeer) //
+
+	conn := &mockChannelConnection{RemoteConnection{router.Ourself.Peer, toPeer, "", false, true}, r}
+	router.Ourself.handleAddConnection(conn)
+	router.Ourself.handleConnectionEstablished(conn)
+	router.sendPendingGossip()
+}
+
+func (router *Router) DeleteTestChannelConnection(r *Router) {
+	fromName := router.Ourself.Peer.Name
+	toName := r.Ourself.Peer.Name
+
+	fromPeer, _ := r.Peers.Fetch(fromName)
+	toPeer, _ := router.Peers.Fetch(toName)
+
+	fromPeer.DecrementLocalRefCount()
+	toPeer.DecrementLocalRefCount()
+
+	conn, _ := router.Ourself.ConnectionTo(toName)
+	router.Ourself.handleDeleteConnection(conn)
+	router.sendPendingGossip()
+}
+
+func TestGossipTopology(t *testing.T) {
+	wt.RunWithTimeout(t, 1*time.Second, func() {
+		implTestGossipTopology(t)
+	})
+}
+
+// Create a Peer representing the receiver router, with connections to
+// the routers supplied as arguments, carrying across all UID and
+// version information.
+func (router *Router) tp(routers ...*Router) *Peer {
+	peer := NewPeer(router.Ourself.Peer.Name, "", router.Ourself.Peer.UID, 0)
+	connections := make(map[PeerName]Connection)
+	for _, r := range routers {
+		p := NewPeer(r.Ourself.Peer.Name, "", r.Ourself.Peer.UID, r.Ourself.Peer.version)
+		connections[r.Ourself.Peer.Name] = newMockConnection(peer, p)
+	}
+	peer.SetVersionAndConnections(router.Ourself.Peer.version, connections)
+	return peer
+}
+
+// Check that the topology of router matches the peers and all of their connections
+func checkTopology(t *testing.T, router *Router, wantedPeers ...*Peer) {
+	checkTopologyPeers(t, true, router.Peers.allPeers(), wantedPeers...)
+}
+
+func implTestGossipTopology(t *testing.T) {
+	// Create some peers that will talk to each other
+	peer1Name, _ := PeerNameFromString("01:00:00:01:00:00")
+	peer2Name, _ := PeerNameFromString("02:00:00:02:00:00")
+	peer3Name, _ := PeerNameFromString("03:00:00:03:00:00")
+	r1 := NewTestRouter(peer1Name)
+	r2 := NewTestRouter(peer2Name)
+	r3 := NewTestRouter(peer3Name)
+
+	// Check state when they have no connections
+	checkTopology(t, r1, r1.tp())
+	checkTopology(t, r2, r2.tp())
+
+	// Now try adding some connections
+	r1.AddTestChannelConnection(r2)
+	checkTopology(t, r1, r1.tp(r2), r2.tp())
+	checkTopology(t, r2, r1.tp(r2), r2.tp())
+	r2.AddTestChannelConnection(r1)
+	checkTopology(t, r1, r1.tp(r2), r2.tp(r1))
+	checkTopology(t, r2, r1.tp(r2), r2.tp(r1))
+
+	// Currently, the connection from 2 to 3 is one-way only
+	r2.AddTestChannelConnection(r3)
+	checkTopology(t, r1, r1.tp(r2), r2.tp(r1, r3), r3.tp())
+	checkTopology(t, r2, r1.tp(r2), r2.tp(r1, r3), r3.tp())
+	// When r2 gossiped to r3, 1 was unreachable from r3 so it got removed from the
+	// list of peers, but remains referenced in the connection from 1 to 3.
+	checkTopology(t, r3, r2.tp(r1, r3), r3.tp())
+
+	// Add a connection from 3 to 1 and now r1 is reachable.
+	r3.AddTestChannelConnection(r1)
+	checkTopology(t, r1, r1.tp(r2), r2.tp(r1, r3), r3.tp(r1))
+	checkTopology(t, r2, r1.tp(r2), r2.tp(r1, r3), r3.tp(r1))
+	checkTopology(t, r3, r1.tp(), r2.tp(r1, r3), r3.tp(r1))
+
+	r1.AddTestChannelConnection(r3)
+	checkTopology(t, r1, r1.tp(r2, r3), r2.tp(r1, r3), r3.tp(r1))
+	checkTopology(t, r2, r1.tp(r2, r3), r2.tp(r1, r3), r3.tp(r1))
+	checkTopology(t, r3, r1.tp(r2, r3), r2.tp(r1, r3), r3.tp(r1))
+
+	// Drop the connection from 2 to 3
+	r2.DeleteTestChannelConnection(r3)
+	checkTopology(t, r1, r1.tp(r2, r3), r2.tp(r1), r3.tp(r1))
+	checkTopology(t, r2, r1.tp(r2, r3), r2.tp(r1))
+	checkTopology(t, r3, r1.tp(r2, r3), r2.tp(r1), r3.tp(r1))
+
+	// Drop the connection from 1 to 3
+	r1.DeleteTestChannelConnection(r3)
+	checkTopology(t, r1, r1.tp(r2), r2.tp(r1), r3.tp(r1))
+
+	checkTopology(t, r1, r1.tp(r2), r2.tp(r1), r3.tp(r1))
+	checkTopology(t, r2, r1.tp(r2), r2.tp(r1))
+	// r3 still thinks r1 has a connection to it
+	checkTopology(t, r3, r1.tp(r2, r3), r2.tp(r1), r3.tp(r1))
+
+	// On a timer, r3 will gossip to r1
+	r3.SendAllGossip()
+	checkTopology(t, r1, r1.tp(r2), r2.tp(r1), r3.tp(r1))
+}

router/handshake.go

@@ -0,0 +1,164 @@
+package router
+
+import (
+	"encoding/gob"
+	"encoding/hex"
+	"fmt"
+	"strconv"
+)
+
+type FieldValidator struct {
+	fields map[string]string
+	err    error
+}
+
+func NewFieldValidator(fields map[string]string) *FieldValidator {
+	return &FieldValidator{fields, nil}
+}
+
+func (fv *FieldValidator) Value(fieldName string) (string, error) {
+	if fv.err != nil {
+		return "", fv.err
+	}
+	val, found := fv.fields[fieldName]
+	if !found {
+		fv.err = fmt.Errorf("Field %s is missing", fieldName)
+		return "", fv.err
+	}
+	return val, nil
+}
+
+func (fv *FieldValidator) CheckEqual(fieldName, expectedValue string) error {
+	val, err := fv.Value(fieldName)
+	if err != nil {
+		return err
+	}
+	if val != expectedValue {
+		fv.err = fmt.Errorf("Field %s has wrong value; expected '%s', received '%s'", fieldName, expectedValue, val)
+		return fv.err
+	}
+	return nil
+}
+
+func (fv *FieldValidator) Err() error {
+	return fv.err
+}
+
+func (conn *LocalConnection) handshake(enc *gob.Encoder, dec *gob.Decoder, acceptNewPeer bool) error {
+	// We do not need to worry about locking in here as at this point
+	// the connection is not reachable by any go-routine other than
+	// ourself. Only when we add this connection to the conn.local
+	// peer will it be visible from multiple go-routines.
+
+	conn.extendReadDeadline()
+
+	localConnID := randUint64()
+	usingPassword := conn.Router.UsingPassword()
+	fv, private, err := conn.handshakeSendRecv(localConnID, usingPassword, enc, dec)
+	if err != nil {
+		return err
+	}
+	nameStr, _ := fv.Value("Name")
+	nickNameStr, _ := fv.Value("NickName")
+	uidStr, _ := fv.Value("UID")
+	remoteConnIDStr, _ := fv.Value("ConnID")
+	if err := fv.Err(); err != nil {
+		return err
+	}
+
+	name, err := PeerNameFromString(nameStr)
+	if err != nil {
+		return err
+	}
+	if !acceptNewPeer {
+		if _, found := conn.Router.Peers.Fetch(name); !found {
+			return fmt.Errorf("Found unknown remote name: %s at %s", name, conn.remoteTCPAddr)
+		}
+	}
+	if existingConn, found := conn.local.ConnectionTo(name); found && existingConn.Established() {
+		return fmt.Errorf("Already have connection to %s at %s", existingConn.Remote().FullName(), existingConn.RemoteTCPAddr())
+	}
+	uid, err := strconv.ParseUint(uidStr, 10, 64)
+	if err != nil {
+		return err
+	}
+	remoteConnID, err := strconv.ParseUint(remoteConnIDStr, 10, 64)
+	if err != nil {
+		return err
+	}
+	conn.uid = localConnID ^ remoteConnID
+
+	remotePublicStr, rpErr := fv.Value("PublicKey")
+	if usingPassword {
+		if rpErr != nil {
+			return rpErr
+		}
+		remotePublicSlice, rpErr := hex.DecodeString(remotePublicStr)
+		if rpErr != nil {
+			return rpErr
+		}
+		remotePublic := [32]byte{}
+		for idx, elem := range remotePublicSlice {
+			remotePublic[idx] = elem
+		}
+		conn.SessionKey = FormSessionKey(&remotePublic, private, conn.Router.Password)
+		conn.tcpSender = NewEncryptedTCPSender(enc, conn)
+		conn.Decryptor = NewNaClDecryptor(conn)
+	} else {
+		if rpErr == nil {
+			return fmt.Errorf("Remote network is encrypted. Password required.")
+		}
+		conn.tcpSender = NewSimpleTCPSender(enc)
+		conn.Decryptor = NewNonDecryptor(conn)
+	}
+
+	return conn.setRemote(NewPeer(name, nickNameStr, uid, 0))
+}
+
+func (conn *LocalConnection) handshakeSendRecv(localConnID uint64, usingPassword bool, enc *gob.Encoder, dec *gob.Decoder) (*FieldValidator, *[32]byte, error) {
+	versionStr := fmt.Sprint(ProtocolVersion)
+	handshakeSend := map[string]string{
+		"Protocol":        Protocol,
+		"ProtocolVersion": versionStr,
+		"PeerNameFlavour": PeerNameFlavour,
+		"Name":            conn.local.Name.String(),
+		"NickName":        conn.local.NickName,
+		"UID":             fmt.Sprint(conn.local.UID),
+		"ConnID":          fmt.Sprint(localConnID)}
+	handshakeRecv := map[string]string{}
+
+	var public, private *[32]byte
+	var err error
+	if usingPassword {
+		public, private, err = GenerateKeyPair()
+		if err != nil {
+			return nil, nil, err
+		}
+		handshakeSend["PublicKey"] = hex.EncodeToString(public[:])
+	}
+	enc.Encode(handshakeSend)
+
+	err = dec.Decode(&handshakeRecv)
+	if err != nil {
+		return nil, nil, err
+	}
+	fv := NewFieldValidator(handshakeRecv)
+	fv.CheckEqual("Protocol", Protocol)
+	fv.CheckEqual("ProtocolVersion", versionStr)
+	fv.CheckEqual("PeerNameFlavour", PeerNameFlavour)
+	return fv, private, nil
+}
+
+func (conn *LocalConnection) setRemote(toPeer *Peer) error {
+	toPeer = conn.Router.Peers.FetchWithDefault(toPeer)
+	switch toPeer {
+	case nil:
+		return fmt.Errorf("Connection appears to be with different version of a peer we already know of")
+	case conn.local:
+		conn.remote = toPeer // have to do assigment here to ensure Shutdown releases ref count
+		return fmt.Errorf("Cannot connect to ourself")
+	default:
+		conn.remote = toPeer
+		return nil
+	}
+}

router/handshake_test.go

@@ -0,0 +1,29 @@
+package router
+
+import (
+	wt "github.com/weaveworks/weave/testing"
+	"testing"
+)
+
+func TestFieldValidator(t *testing.T) {
+	testMap := map[string]string{"a": "a"}
+
+	fv := NewFieldValidator(testMap)
+	val, err := fv.Value("a")
+	wt.AssertNoErr(t, err)
+	wt.AssertNoErr(t, fv.Err())
+	wt.AssertEqualString(t, val, "a", "")
+	_, err = fv.Value("x")
+	wt.AssertFalse(t, err == nil || fv.Err() == nil, "Expected error")
+	_, err = fv.Value("a")
+	wt.AssertFalse(t, err == nil || fv.Err() == nil, "Previous error should be retained")
+
+	fv = NewFieldValidator(testMap)
+	err = fv.CheckEqual("a", "a")
+	wt.AssertNoErr(t, err)
+	wt.AssertNoErr(t, fv.Err())
+	err = fv.CheckEqual("a", "b")
+	wt.AssertFalse(t, err == nil || fv.Err() == nil, "Expected error")
+	err = fv.CheckEqual("a", "a")
+	wt.AssertFalse(t, err == nil || fv.Err() == nil, "Previous error should be retained")
+}

router/json.go

@@ -0,0 +1,91 @@
+package router
+
+import (
+	"encoding/json"
+	"fmt"
+	"time"
+)
+
+func (router *Router) GenerateStatusJSON(version, encryption string) ([]byte, error) {
+	return json.Marshal(struct {
+		Version    string
+		Encryption string
+		Name       string
+		NickName   string
+		Interface  string
+		Macs       *MacCache
+		Peers      *Peers
+		Routes     *Routes
+	}{version, encryption, router.Ourself.Name.String(), router.Ourself.NickName, fmt.Sprintf("%v", router.Iface), router.Macs, router.Peers, router.Routes})
+	// leaving out ConectionMaker due to async complexities
+}
+
+func (cache *MacCache) MarshalJSON() ([]byte, error) {
+	type cacheEntry struct {
+		Mac      string
+		Name     string
+		NickName string
+		LastSeen time.Time
+	}
+	var entries []*cacheEntry
+	for key, entry := range cache.table {
+		entries = append(entries, &cacheEntry{intmac(key).String(), entry.peer.Name.String(), entry.peer.NickName, entry.lastSeen})
+	}
+	return json.Marshal(entries)
+}
+
+func (peers *Peers) MarshalJSON() ([]byte, error) {
+	var ps []*Peer
+	peers.ForEach(func(peer *Peer) { ps = append(ps, peer) })
+	return json.Marshal(ps)
+}
+
+func (routes *Routes) MarshalJSON() ([]byte, error) {
+	routes.RLock()
+	defer routes.RUnlock()
+	type uni struct {
+		Dest, Via PeerName
+	}
+	type broad struct {
+		Source PeerName
+		Via    []PeerName
+	}
+	var r struct {
+		Unicast   []*uni
+		Broadcast []*broad
+	}
+	for name, hop := range routes.unicast {
+		r.Unicast = append(r.Unicast, &uni{name, hop})
+	}
+	for name, hops := range routes.broadcast {
+		r.Broadcast = append(r.Broadcast, &broad{name, hops})
+	}
+	return json.Marshal(r)
+}
+
+func (peer *Peer) MarshalJSON() ([]byte, error) {
+	conns := peer.Connections()
+	connections := make([]Connection, 0, len(conns))
+	for conn := range conns {
+		connections = append(connections, conn)
+	}
+	return json.Marshal(struct {
+		Name        string
+		NickName    string
+		UID         uint64
+		Version     uint64
+		Connections []Connection
+	}{peer.Name.String(), peer.NickName, peer.UID, peer.version, connections})
+}
+
+func (conn *RemoteConnection) MarshalJSON() ([]byte, error) {
+	return json.Marshal(struct {
+		Name     string
+		NickName string
+		TCPAddr  string
+	}{conn.Remote().Name.String(), conn.Remote().NickName, conn.RemoteTCPAddr()})
+}
+
+func (name PeerName) MarshalJSON() ([]byte, error) {
+	return json.Marshal(name.String())
+}

router/local_peer.go

@@ -4,41 +4,41 @@ import (
 	"fmt"
 	"log"
 	"net"
+	"time"
 )
 
 type LocalPeer struct {
 	*Peer
-	Router    *Router
-	queryChan chan<- *PeerInteraction
+	router     *Router
+	actionChan chan<- LocalPeerAction
 }
 
-type PeerInteraction struct {
-	Interaction
-	payload interface{}
+type LocalPeerAction func()
+
+func NewLocalPeer(name PeerName, nickName string, router *Router) *LocalPeer {
+	return &LocalPeer{Peer: NewPeer(name, nickName, 0, 0), router: router}
 }
 
-func StartLocalPeer(name PeerName, router *Router) *LocalPeer {
-	peer := &LocalPeer{Peer: NewPeer(name, 0, 0), Router: router}
-	queryChan := make(chan *PeerInteraction, ChannelSize)
-	peer.queryChan = queryChan
-	go peer.queryLoop(queryChan)
-	return peer
+func (peer *LocalPeer) Start() {
+	actionChan := make(chan LocalPeerAction, ChannelSize)
+	peer.actionChan = actionChan
+	go peer.actorLoop(actionChan)
 }
 
 func (peer *LocalPeer) Forward(dstPeer *Peer, df bool, frame []byte, dec *EthernetDecoder) error {
 	return peer.Relay(peer.Peer, dstPeer, df, frame, dec)
 }
 
-func (peer *LocalPeer) Broadcast(df bool, frame []byte, dec *EthernetDecoder) error {
-	return peer.RelayBroadcast(peer.Peer, df, frame, dec)
+func (peer *LocalPeer) Broadcast(df bool, frame []byte, dec *EthernetDecoder) {
+	peer.RelayBroadcast(peer.Peer, df, frame, dec)
 }
 
 func (peer *LocalPeer) Relay(srcPeer, dstPeer *Peer, df bool, frame []byte, dec *EthernetDecoder) error {
-	relayPeerName, found := peer.Router.Routes.Unicast(dstPeer.Name)
+	relayPeerName, found := peer.router.Routes.Unicast(dstPeer.Name)
 	if !found {
 		// Not necessarily an error as there could be a race with the
 		// dst disappearing whilst the frame is in flight
-		log.Println("Received packet for unknown destination:", dstPeer.Name)
+		log.Println("Received packet for unknown destination:", dstPeer.FullName())
 		return nil
 	}
 	conn, found := peer.ConnectionTo(relayPeerName)
@@ -54,46 +54,47 @@ func (peer *LocalPeer) Relay(srcPeer, dstPeer *Peer, df bool, frame []byte, dec
 		dec)
 }
 
-func (peer *LocalPeer) RelayBroadcast(srcPeer *Peer, df bool, frame []byte, dec *EthernetDecoder) error {
-	nextHops := peer.Router.Routes.Broadcast(srcPeer.Name)
+func (peer *LocalPeer) RelayBroadcast(srcPeer *Peer, df bool, frame []byte, dec *EthernetDecoder) {
+	nextHops := peer.router.Routes.Broadcast(srcPeer.Name)
 	if len(nextHops) == 0 {
-		return nil
+		return
 	}
-	// We must not hold a read lock on peer during the conn.Forward
-	// below, since that is a potentially blocking operation (e.g. if
-	// the channel is full).
-	nextConns := make([]*LocalConnection, 0, len(nextHops))
-	peer.RLock()
-	for _, hopName := range nextHops {
-		conn, found := peer.connections[hopName]
-		// Again, !found could just be due to a race.
-		if found {
-			nextConns = append(nextConns, conn.(*LocalConnection))
-		}
-	}
-	peer.RUnlock()
-	var err error
-	for _, conn := range nextConns {
-		err = conn.Forward(df, &ForwardedFrame{
+	for _, conn := range peer.ConnectionsTo(nextHops) {
+		err := conn.(*LocalConnection).Forward(df, &ForwardedFrame{
 			srcPeer: srcPeer,
 			dstPeer: conn.Remote(),
 			frame:   frame},
 			dec)
 		if err != nil {
-			return err
+			if ftbe, ok := err.(FrameTooBigError); ok {
+				log.Printf("dropping too big DF broadcast frame (%v -> %v): PMTU= %v\n", dec.ip.DstIP, dec.ip.SrcIP, ftbe.EPMTU)
+			} else {
+				log.Println(err)
+			}
 		}
 	}
-	return nil
+}
+
+func (peer *LocalPeer) ConnectionsTo(names []PeerName) []Connection {
+	conns := make([]Connection, 0, len(names))
+	peer.RLock()
+	defer peer.RUnlock()
+	for _, name := range names {
+		conn, found := peer.connections[name]
+		// Again, !found could just be due to a race.
+		if found {
+			conns = append(conns, conn)
+		}
+	}
+	return conns
 }
 
 func (peer *LocalPeer) CreateConnection(peerAddr string, acceptNewPeer bool) error {
 	if err := peer.checkConnectionLimit(); err != nil {
 		return err
 	}
-	// We're dialing the remote so that means connections will come from random ports
-	addrStr := NormalisePeerAddr(peerAddr)
-	tcpAddr, tcpErr := net.ResolveTCPAddr("tcp4", addrStr)
-	udpAddr, udpErr := net.ResolveUDPAddr("udp4", addrStr)
+	tcpAddr, tcpErr := net.ResolveTCPAddr("tcp4", peerAddr)
+	udpAddr, udpErr := net.ResolveUDPAddr("udp4", peerAddr)
 	if tcpErr != nil || udpErr != nil {
 		// they really should have the same value, but just in case...
 		if tcpErr == nil {
@@ -105,74 +106,55 @@ func (peer *LocalPeer) CreateConnection(peerAddr string, acceptNewPeer bool) err
 	if err != nil {
 		return err
 	}
-	connRemote := NewRemoteConnection(peer.Peer, nil, tcpConn.RemoteAddr().String())
-	NewLocalConnection(connRemote, acceptNewPeer, tcpConn, udpAddr, peer.Router)
+	connRemote := NewRemoteConnection(peer.Peer, nil, tcpConn.RemoteAddr().String(), true, false)
+	connLocal := NewLocalConnection(connRemote, tcpConn, udpAddr, peer.router)
+	connLocal.Start(acceptNewPeer)
 	return nil
 }
 
 // ACTOR client API
 
-const (
-	PAddConnection         = iota
-	PBroadcastTCP          = iota
-	PDeleteConnection      = iota
-	PConnectionEstablished = iota
-)
+// Sync.
+func (peer *LocalPeer) AddConnection(conn *LocalConnection) error {
+	resultChan := make(chan error)
+	peer.actionChan <- func() {
+		resultChan <- peer.handleAddConnection(conn)
+	}
+	return <-resultChan
+}
 
-// Async: rely on the peer to shut us down if we shouldn't be adding
-// ourselves, so therefore this can be async
-func (peer *LocalPeer) AddConnection(conn *LocalConnection) {
-	peer.queryChan <- &PeerInteraction{
-		Interaction: Interaction{code: PAddConnection},
-		payload:     conn}
+// Async.
+func (peer *LocalPeer) ConnectionEstablished(conn *LocalConnection) {
+	peer.actionChan <- func() {
+		peer.handleConnectionEstablished(conn)
+	}
 }
 
 // Sync.
 func (peer *LocalPeer) DeleteConnection(conn *LocalConnection) {
 	resultChan := make(chan interface{})
-	peer.queryChan <- &PeerInteraction{
-		Interaction: Interaction{code: PDeleteConnection, resultChan: resultChan},
-		payload:     conn}
+	peer.actionChan <- func() {
+		peer.handleDeleteConnection(conn)
+		resultChan <- nil
+	}
 	<-resultChan
 }
 
-// Async.
-func (peer *LocalPeer) ConnectionEstablished(conn *LocalConnection) {
-	peer.queryChan <- &PeerInteraction{
-		Interaction: Interaction{code: PConnectionEstablished},
-		payload:     conn}
-}
-
-// Async.
-func (peer *LocalPeer) BroadcastTCP(msg []byte) {
-	peer.queryChan <- &PeerInteraction{
-		Interaction: Interaction{code: PBroadcastTCP},
-		payload:     msg}
-}
-
 // ACTOR server
 
-func (peer *LocalPeer) queryLoop(queryChan <-chan *PeerInteraction) {
+func (peer *LocalPeer) actorLoop(actionChan <-chan LocalPeerAction) {
+	gossipTimer := time.Tick(GossipInterval)
 	for {
-		query, ok := <-queryChan
-		if !ok {
-			return
-		}
-		switch query.code {
-		case PAddConnection:
-			peer.handleAddConnection(query.payload.(*LocalConnection))
-		case PDeleteConnection:
-			peer.handleDeleteConnection(query.payload.(*LocalConnection))
-			query.resultChan <- nil
-		case PConnectionEstablished:
-			peer.handleConnectionEstablished(query.payload.(*LocalConnection))
-		case PBroadcastTCP:
-			peer.handleBroadcastTCP(query.payload.([]byte))
+		select {
+		case action := <-actionChan:
+			action()
+		case <-gossipTimer:
+			peer.router.SendAllGossip()
 		}
 	}
 }
 
-func (peer *LocalPeer) handleAddConnection(conn *LocalConnection) {
+func (peer *LocalPeer) handleAddConnection(conn Connection) error {
 	if peer.Peer != conn.Local() {
 		log.Fatal("Attempt made to add connection to peer where peer is not the source of connection")
 	}
@@ -180,39 +162,54 @@ func (peer *LocalPeer) handleAddConnection(conn *LocalConnection) {
 		log.Fatal("Attempt made to add connection to peer with unknown remote peer")
 	}
 	toName := conn.Remote().Name
-	dupErr := fmt.Errorf("Multiple connections to %s added to %s", toName, peer.Name)
+	dupErr := fmt.Errorf("Multiple connections to %s added to %s", conn.Remote().FullName(), peer.FullName())
 	// deliberately non symmetrical
 	if dupConn, found := peer.connections[toName]; found {
 		if dupConn == conn {
-			return
+			return nil
 		}
-		// conn.UID is used as the tie breaker here, in the
-		// knowledge that both sides will make the same decision.
-		dupConnLocal := dupConn.(*LocalConnection)
-		if conn.UID == dupConnLocal.UID {
+		switch conn.BreakTie(dupConn) {
+		case TieBreakWon:
+			dupConn.Shutdown(dupErr)
+			peer.handleDeleteConnection(dupConn)
+		case TieBreakLost:
+			return dupErr
+		case TieBreakTied:
 			// oh good grief. Sod it, just kill both of them.
-			conn.CheckFatal(dupErr)
-			dupConnLocal.CheckFatal(dupErr)
-			peer.handleDeleteConnection(dupConnLocal)
-			return
-		} else if conn.UID < dupConnLocal.UID {
-			dupConnLocal.CheckFatal(dupErr)
-			peer.handleDeleteConnection(dupConnLocal)
-		} else {
-			conn.CheckFatal(dupErr)
-			return
+			dupConn.Shutdown(dupErr)
+			peer.handleDeleteConnection(dupConn)
+			return dupErr
 		}
 	}
 	if err := peer.checkConnectionLimit(); err != nil {
-		conn.CheckFatal(err)
-		return
+		return err
 	}
-	peer.Lock()
-	peer.connections[toName] = conn
-	peer.Unlock()
+	_, isConnectedPeer := peer.router.Routes.Unicast(toName)
+	peer.addConnection(conn)
+	if isConnectedPeer {
+		conn.Log("connection added")
+	} else {
+		conn.Log("connection added (new peer)")
+		peer.router.SendAllGossipDown(conn)
+	}
+	peer.broadcastPeerUpdate(conn.Remote())
+	return nil
 }
 
-func (peer *LocalPeer) handleDeleteConnection(conn *LocalConnection) {
+func (peer *LocalPeer) handleConnectionEstablished(conn Connection) {
+	if peer.Peer != conn.Local() {
+		log.Fatal("Peer informed of active connection where peer is not the source of connection")
+	}
+	if dupConn, found := peer.connections[conn.Remote().Name]; !found || conn != dupConn {
+		conn.Shutdown(fmt.Errorf("Cannot set unknown connection active"))
+		return
+	}
+	peer.connectionEstablished(conn)
+	conn.Log("connection fully established")
+	peer.broadcastPeerUpdate()
+}
+
+func (peer *LocalPeer) handleDeleteConnection(conn Connection) {
 	if peer.Peer != conn.Local() {
 		log.Fatal("Attempt made to delete connection from peer where peer is not the source of connection")
 	}
@@ -223,53 +220,23 @@ func (peer *LocalPeer) handleDeleteConnection(conn *LocalConnection) {
 	if connFound, found := peer.connections[toName]; !found || connFound != conn {
 		return
 	}
-	peer.Lock()
-	delete(peer.connections, toName)
-	peer.Unlock()
-	broadcast := false
-	if conn.Established() {
-		peer.Lock()
-		peer.version += 1
-		peer.Unlock()
-		broadcast = true
-	}
+	peer.deleteConnection(conn)
+	conn.Log("connection deleted")
 	// Must do garbage collection first to ensure we don't send out an
 	// update with unreachable peers (can cause looping)
-	peer.Router.Peers.GarbageCollect()
-	if broadcast {
-		peer.broadcastPeerUpdate()
-	}
-}
-
-func (peer *LocalPeer) handleConnectionEstablished(conn *LocalConnection) {
-	if peer.Peer != conn.Local() {
-		log.Fatal("Peer informed of active connection where peer is not the source of connection")
-	}
-	if dupConn, found := peer.connections[conn.Remote().Name]; !found || conn != dupConn {
-		conn.CheckFatal(fmt.Errorf("Cannot set unknown connection active"))
-		return
-	}
-	peer.Lock()
-	peer.version += 1
-	peer.Unlock()
-	log.Println("Peer", peer.Name, "established active connection to remote peer", conn.Remote().Name, "at", conn.RemoteTCPAddr())
-	peer.broadcastPeerUpdate(conn.Remote())
-}
-
-func (peer *LocalPeer) handleBroadcastTCP(msg []byte) {
-	peer.ForEachConnection(func(_ PeerName, conn Connection) {
-		conn.(*LocalConnection).SendTCP(msg)
-	})
+	peer.router.Peers.GarbageCollect()
+	peer.broadcastPeerUpdate()
 }
 
 func (peer *LocalPeer) broadcastPeerUpdate(peers ...*Peer) {
-	peer.Router.Routes.Recalculate()
-	peer.handleBroadcastTCP(Concat(ProtocolUpdateByte, EncodePeers(append(peers, peer.Peer)...)))
+	peer.router.Routes.Recalculate()
+	peer.router.TopologyGossip.GossipBroadcast(NewTopologyGossipData(peer.router.Peers, append(peers, peer.Peer)...))
 }
 
 func (peer *LocalPeer) checkConnectionLimit() error {
-	if 0 != peer.Router.ConnLimit && peer.ConnectionCount() >= peer.Router.ConnLimit {
-		return fmt.Errorf("Connection limit reached (%v)", peer.Router.ConnLimit)
+	limit := peer.router.ConnLimit
+	if 0 != limit && peer.ConnectionCount() >= limit {
+		return fmt.Errorf("Connection limit reached (%v)", limit)
 	}
 	return nil
 }

router/mac_cache.go

@@ -22,12 +22,14 @@ type MacCache struct {
 }
 
 func NewMacCache(maxAge time.Duration, onExpiry func(net.HardwareAddr, *Peer)) *MacCache {
-	res := &MacCache{
+	return &MacCache{
 		table:    make(map[uint64]*MacCacheEntry),
 		maxAge:   maxAge,
 		onExpiry: onExpiry}
-	res.setExpiryTimer()
-	return res
+}
+
+func (cache *MacCache) Start() {
+	cache.setExpiryTimer()
 }
 
 func (cache *MacCache) Enter(mac net.HardwareAddr, peer *Peer) bool {
@@ -38,9 +40,8 @@ func (cache *MacCache) Enter(mac net.HardwareAddr, peer *Peer) bool {
 	if found && entry.peer == peer && now.Before(entry.lastSeen.Add(cache.maxAge/10)) {
 		cache.RUnlock()
 		return false
-	} else {
-		cache.RUnlock()
 	}
+	cache.RUnlock()
 	cache.Lock()
 	defer cache.Unlock()
 	entry, found = cache.table[key]
@@ -88,7 +89,7 @@ func (cache *MacCache) String() string {
 	cache.RLock()
 	defer cache.RUnlock()
 	for key, entry := range cache.table {
-		buf.WriteString(fmt.Sprintf("%v -> %s (%v)\n", intmac(key), entry.peer.Name, entry.lastSeen))
+		fmt.Fprintf(&buf, "%v -> %s (%v)\n", intmac(key), entry.peer.FullName(), entry.lastSeen)
 	}
 	return buf.String()
 }

router/mocks_test.go

@@ -0,0 +1,113 @@
+// No mocks are tested by this file.
+//
+// It supplies some mock implementations to other unit tests, and is
+// named "...test.go" so it is only compiled under `go test`.
+
+package router
+
+import (
+	wt "github.com/weaveworks/weave/testing"
+	"testing"
+)
+
+// Add to peers a connection from peers.ourself to p
+func (peers *Peers) AddTestConnection(p *Peer) {
+	toPeer := NewPeer(p.Name, "", p.UID, 0)
+	peers.FetchWithDefault(toPeer) // Has side-effect of incrementing refcount
+	conn := newMockConnection(peers.ourself, toPeer)
+	peers.ourself.addConnection(conn)
+	peers.ourself.connectionEstablished(conn)
+}
+
+// Add to peers a connection from p1 to p2
+func (peers *Peers) AddTestRemoteConnection(p1, p2 *Peer) {
+	fromName := p1.Name
+	fromPeer := NewPeer(fromName, "", p1.UID, 0)
+	fromPeer = peers.FetchWithDefault(fromPeer)
+	toName := p2.Name
+	toPeer := NewPeer(toName, "", p2.UID, 0)
+	toPeer = peers.FetchWithDefault(toPeer)
+	peers.ourself.addConnection(&RemoteConnection{fromPeer, toPeer, "", false, false})
+}
+
+func (peers *Peers) DeleteTestConnection(p *Peer) {
+	toName := p.Name
+	toPeer, _ := peers.Fetch(toName)
+	toPeer.DecrementLocalRefCount()
+	conn, _ := peers.ourself.ConnectionTo(toName)
+	peers.ourself.deleteConnection(conn)
+}
+
+// mockConnection used in testing is very similar to a
+// RemoteConnection, without the RemoteTCPAddr(). We are making it a
+// separate type in order to distinguish what is created by the test
+// from what is created by the real code.
+func newMockConnection(from, to *Peer) Connection {
+	type mockConnection struct{ RemoteConnection }
+	return &mockConnection{RemoteConnection{from, to, "", false, false}}
+}
+
+func checkEqualConns(t *testing.T, ourName PeerName, got, wanted map[PeerName]Connection) {
+	checkConns := make(PeerNameSet)
+	for _, conn := range wanted {
+		checkConns[conn.Remote().Name] = void
+	}
+	for _, conn := range got {
+		remoteName := conn.Remote().Name
+		if _, found := checkConns[remoteName]; found {
+			delete(checkConns, remoteName)
+		} else {
+			wt.Fatalf(t, "Unexpected connection from %s to %s", ourName, remoteName)
+		}
+	}
+	if len(checkConns) > 0 {
+		wt.Fatalf(t, "Expected connections not found: from %s to %v", ourName, checkConns)
+	}
+}
+
+// Get all the peers from a Peers in a slice
+func (peers *Peers) allPeers() []*Peer {
+	var res []*Peer
+	for _, peer := range peers.table {
+		res = append(res, peer)
+	}
+	return res
+}
+
+func (peers *Peers) allPeersExcept(excludeName PeerName) []*Peer {
+	res := peers.allPeers()
+	for i, peer := range res {
+		if peer.Name == excludeName {
+			return append(res[:i], res[i+1:]...)
+		}
+	}
+	return res
+}
+
+// Check that the peers slice matches the wanted peers
+func checkPeerArray(t *testing.T, peers []*Peer, wantedPeers ...*Peer) {
+	checkTopologyPeers(t, false, peers, wantedPeers...)
+}
+
+// Check that the peers slice matches the wanted peers and optionally
+// all of their connections
+func checkTopologyPeers(t *testing.T, checkConns bool, peers []*Peer, wantedPeers ...*Peer) {
+	check := make(map[PeerName]*Peer)
+	for _, peer := range wantedPeers {
+		check[peer.Name] = peer
+	}
+	for _, peer := range peers {
+		name := peer.Name
+		if wantedPeer, found := check[name]; found {
+			if checkConns {
+				checkEqualConns(t, name, peer.connections, wantedPeer.connections)
+			}
+			delete(check, name)
+		} else {
+			wt.Fatalf(t, "Unexpected peer: %s", name)
+		}
+	}
+	if len(check) > 0 {
+		wt.Fatalf(t, "Expected peers not found: %v", check)
+	}
+}

router/peer.go

@@ -10,19 +10,23 @@ type Peer struct {
 	sync.RWMutex
 	Name          PeerName
 	NameByte      []byte
+	NickName      string
 	UID           uint64
 	version       uint64
 	localRefCount uint64
 	connections   map[PeerName]Connection
 }
 
-func NewPeer(name PeerName, uid uint64, version uint64) *Peer {
+type ConnectionSet map[Connection]struct{}
+
+func NewPeer(name PeerName, nickName string, uid uint64, version uint64) *Peer {
 	if uid == 0 {
 		uid = randUint64()
 	}
 	return &Peer{
 		Name:        name,
 		NameByte:    name.Bin(),
+		NickName:    nickName,
 		UID:         uid,
 		version:     version,
 		connections: make(map[PeerName]Connection)}
@@ -31,7 +35,11 @@ func NewPeer(name PeerName, uid uint64, version uint64) *Peer {
 func (peer *Peer) String() string {
 	peer.RLock()
 	defer peer.RUnlock()
-	return fmt.Sprint("Peer ", peer.Name, " (v", peer.version, ") (UID ", peer.UID, ")")
+	return fmt.Sprint(peer.FullName(), " (v", peer.version, ") (UID ", peer.UID, ")")
+}
+
+func (peer *Peer) FullName() string {
+	return fmt.Sprint(peer.Name, "(", peer.NickName, ")")
 }
 
 func (peer *Peer) Version() uint64 {
@@ -43,13 +51,13 @@ func (peer *Peer) Version() uint64 {
 func (peer *Peer) IncrementLocalRefCount() {
 	peer.Lock()
 	defer peer.Unlock()
-	peer.localRefCount += 1
+	peer.localRefCount++
 }
 
 func (peer *Peer) DecrementLocalRefCount() {
 	peer.Lock()
 	defer peer.Unlock()
-	peer.localRefCount -= 1
+	peer.localRefCount--
 }
 
 func (peer *Peer) IsLocallyReferenced() bool {
@@ -71,12 +79,14 @@ func (peer *Peer) ConnectionTo(name PeerName) (Connection, bool) {
 	return conn, found // yes, you really can't inline that. FFS.
 }
 
-func (peer *Peer) ForEachConnection(fun func(PeerName, Connection)) {
+func (peer *Peer) Connections() ConnectionSet {
+	connections := make(ConnectionSet)
 	peer.RLock()
 	defer peer.RUnlock()
-	for name, conn := range peer.connections {
-		fun(name, conn)
+	for _, conn := range peer.connections {
+		connections[conn] = void
 	}
+	return connections
 }
 
 func (peer *Peer) SetVersionAndConnections(version uint64, connections map[PeerName]Connection) {
@@ -86,6 +96,26 @@ func (peer *Peer) SetVersionAndConnections(version uint64, connections map[PeerN
 	peer.connections = connections
 }
 
+func (peer *Peer) addConnection(conn Connection) {
+	peer.Lock()
+	defer peer.Unlock()
+	peer.connections[conn.Remote().Name] = conn
+	peer.version++
+}
+
+func (peer *Peer) deleteConnection(conn Connection) {
+	peer.Lock()
+	defer peer.Unlock()
+	delete(peer.connections, conn.Remote().Name)
+	peer.version++
+}
+
+func (peer *Peer) connectionEstablished(conn Connection) {
+	peer.Lock()
+	defer peer.Unlock()
+	peer.version++
+}
+
 // Calculate the routing table from this peer to all peers reachable
 // from it, returning a "next hop" map of PeerNameX -> PeerNameY,
 // which says "in order to send a message to X, the peer should send
@@ -99,9 +129,9 @@ func (peer *Peer) SetVersionAndConnections(version uint64, connections map[PeerN
 // same result. This is important since otherwise we risk message loss
 // or routing cycles.
 //
-// When the 'symmetric' flag is set, only symmetric connections are
-// considered, i.e. where both sides indicate they have a connection
-// to the other.
+// When the 'establishedAndSymmetric' flag is set, only connections
+// that are marked as 'established' and are symmetric (i.e. where both
+// sides indicate they have a connection to the other) are considered.
 //
 // When a non-nil stopAt peer is supplied, the widening stops when it
 // reaches that peer. The boolean return indicates whether that has
@@ -111,7 +141,7 @@ func (peer *Peer) SetVersionAndConnections(version uint64, connections map[PeerN
 // traversal. This prevents the connectivity graph from changing
 // underneath us in ways that would invalidate the result. Thus the
 // answer returned may be out of date, but never inconsistent.
-func (peer *Peer) Routes(stopAt *Peer, symmetric bool) (bool, map[PeerName]PeerName) {
+func (peer *Peer) Routes(stopAt *Peer, establishedAndSymmetric bool) (bool, map[PeerName]PeerName) {
 	peer.RLock()
 	defer peer.RUnlock()
 	routes := make(map[PeerName]PeerName)
@@ -127,12 +157,17 @@ func (peer *Peer) Routes(stopAt *Peer, symmetric bool) (bool, map[PeerName]PeerN
 			}
 			curName := curPeer.Name
 			for remoteName, conn := range curPeer.connections {
+				if establishedAndSymmetric && !conn.Established() {
+					continue
+				}
 				if _, found := routes[remoteName]; found {
 					continue
 				}
 				remote := conn.Remote()
 				remote.RLock()
-				if _, found := remote.connections[curName]; !symmetric || found {
+				// DANGER holding rlock on remote, possibly taking
+				// rlock on remoteConn
+				if remoteConn, found := remote.connections[curName]; !establishedAndSymmetric || (found && remoteConn.Established()) {
 					defer remote.RUnlock()
 					nextWorklist = append(nextWorklist, remote)
 					// We now know how to get to remoteName: the same

router/peers.go

@@ -11,15 +11,23 @@ import (
 type Peers struct {
 	sync.RWMutex
 	ourself *Peer
-	macs    *MacCache
 	table   map[PeerName]*Peer
 	onGC    func(*Peer)
 }
 
-func NewPeers(ourself *Peer, macs *MacCache, onGC func(*Peer)) *Peers {
+type UnknownPeerError struct {
+	Name PeerName
+}
+
+type NameCollisionError struct {
+	Name PeerName
+}
+
+type PeerNameSet map[PeerName]struct{}
+
+func NewPeers(ourself *Peer, onGC func(*Peer)) *Peers {
 	return &Peers{
 		ourself: ourself,
-		macs:    macs,
 		table:   make(map[PeerName]*Peer),
 		onGC:    onGC}
 }
@@ -49,21 +57,27 @@ func (peers *Peers) Fetch(name PeerName) (*Peer, bool) {
 	return peer, found // GRRR, why can't I inline this!?
 }
 
-func (peers *Peers) ForEach(fun func(PeerName, *Peer)) {
+func (peers *Peers) ForEach(fun func(*Peer)) {
 	peers.RLock()
 	defer peers.RUnlock()
-	for name, peer := range peers.table {
-		fun(name, peer)
+	for _, peer := range peers.table {
+		fun(peer)
 	}
 }
 
-func (peers *Peers) ApplyUpdate(update []byte) ([]byte, error) {
+// Merge an incoming update with our own topology.
+//
+// We add peers hitherto unknown to us, and update peers for which the
+// update contains a more recent version than known to us. The return
+// value is a) a representation of the received update, and b) an
+// "improved" update containing just these new/updated elements.
+func (peers *Peers) ApplyUpdate(update []byte) (PeerNameSet, PeerNameSet, error) {
 	peers.Lock()
 
 	newPeers, decodedUpdate, decodedConns, err := peers.decodeUpdate(update)
 	if err != nil {
 		peers.Unlock()
-		return nil, err
+		return nil, nil, err
 	}
 
 	// By this point, we know the update doesn't refer to any peers we
@@ -83,20 +97,32 @@ func (peers *Peers) ApplyUpdate(update []byte) ([]byte, error) {
 	// Don't need to hold peers lock any longer
 	peers.Unlock()
 
-	return encodePeersMap(newUpdate), nil
+	updateNames := make(PeerNameSet)
+	for _, peer := range decodedUpdate {
+		updateNames[peer.Name] = void
+	}
+	return updateNames, setFromPeersMap(newUpdate), nil
 }
 
-func (peers *Peers) EncodeAllPeers() []byte {
+func (peers *Peers) Names() PeerNameSet {
 	peers.RLock()
 	defer peers.RUnlock()
-	return encodePeersMap(peers.table)
+	return setFromPeersMap(peers.table)
 }
 
-func EncodePeers(peers ...*Peer) []byte {
+func (peers *Peers) EncodePeers(names PeerNameSet) []byte {
+	peers.RLock()
+	peerList := make([]*Peer, 0, len(names))
+	for name := range names {
+		if peer, found := peers.table[name]; found {
+			peerList = append(peerList, peer)
+		}
+	}
+	peers.RUnlock() // release lock so we don't hold it while encoding
 	buf := new(bytes.Buffer)
 	enc := gob.NewEncoder(buf)
-	for _, peer := range peers {
-		peer.encodePeer(enc)
+	for _, peer := range peerList {
+		peer.encode(enc)
 	}
 	return buf.Bytes()
 }
@@ -109,48 +135,49 @@ func (peers *Peers) GarbageCollect() []*Peer {
 
 func (peers *Peers) String() string {
 	var buf bytes.Buffer
-	peers.ForEach(func(name PeerName, peer *Peer) {
-		buf.WriteString(fmt.Sprint(peer, "\n"))
-		peer.ForEachConnection(func(remoteName PeerName, conn Connection) {
-			buf.WriteString(fmt.Sprintf("   -> %v [%v]\n", remoteName, conn.RemoteTCPAddr()))
-		})
+	peers.ForEach(func(peer *Peer) {
+		fmt.Fprint(&buf, peer, "\n")
+		for conn := range peer.Connections() {
+			established := ""
+			if !conn.Established() {
+				established = " (unestablished)"
+			}
+			fmt.Fprintf(&buf, "   -> %s [%v%s]\n", conn.Remote().FullName(), conn.RemoteTCPAddr(), established)
+		}
 	})
 	return buf.String()
 }
 
 func (peers *Peers) fetchAlias(peer *Peer) (*Peer, bool) {
 	if existingPeer, found := peers.table[peer.Name]; found {
-		if existingPeer.UID == peer.UID {
-			existingPeer.IncrementLocalRefCount()
-			return existingPeer, true
-		} else {
+		if existingPeer.UID != peer.UID {
 			return nil, true
 		}
+		existingPeer.IncrementLocalRefCount()
+		return existingPeer, true
 	}
 	return nil, false
 }
 
 func (peers *Peers) garbageCollect() []*Peer {
 	removed := []*Peer{}
+	_, reached := peers.ourself.Routes(nil, false)
 	for name, peer := range peers.table {
-		found, _ := peers.ourself.Routes(peer, false)
-		if !found && !peer.IsLocallyReferenced() {
-			peers.onGC(peer)
+		if _, found := reached[peer.Name]; !found && !peer.IsLocallyReferenced() {
 			delete(peers.table, name)
-			peers.macs.Delete(peer)
+			peers.onGC(peer)
 			removed = append(removed, peer)
 		}
 	}
 	return removed
 }
 
-func encodePeersMap(peers map[PeerName]*Peer) []byte {
-	buf := new(bytes.Buffer)
-	enc := gob.NewEncoder(buf)
-	for _, peer := range peers {
-		peer.encodePeer(enc)
+func setFromPeersMap(peers map[PeerName]*Peer) PeerNameSet {
+	names := make(PeerNameSet)
+	for name := range peers {
+		names[name] = void
 	}
-	return buf.Bytes()
+	return names
 }
 
 func (peers *Peers) decodeUpdate(update []byte) (newPeers map[PeerName]*Peer, decodedUpdate []*Peer, decodedConns [][]byte, err error) {
@@ -163,7 +190,7 @@ func (peers *Peers) decodeUpdate(update []byte) (newPeers map[PeerName]*Peer, de
 	decoder := gob.NewDecoder(updateBuf)
 
 	for {
-		nameByte, uid, version, connsBuf, decErr := decodePeerNoConns(decoder)
+		nameByte, nickName, uid, version, connsBuf, decErr := decodePeerNoConns(decoder)
 		if decErr == io.EOF {
 			break
 		} else if decErr != nil {
@@ -171,7 +198,7 @@ func (peers *Peers) decodeUpdate(update []byte) (newPeers map[PeerName]*Peer, de
 			return
 		}
 		name := PeerNameFromBin(nameByte)
-		newPeer := NewPeer(name, uid, version)
+		newPeer := NewPeer(name, nickName, uid, version)
 		decodedUpdate = append(decodedUpdate, newPeer)
 		decodedConns = append(decodedConns, connsBuf)
 		existingPeer, found := peers.table[name]
@@ -183,9 +210,8 @@ func (peers *Peers) decodeUpdate(update []byte) (newPeers map[PeerName]*Peer, de
 		}
 	}
 
-	unknownPeers := false
 	for _, connsBuf := range decodedConns {
-		connsIterator(connsBuf, func(remoteNameByte []byte, _ string) {
+		decErr := connsIterator(connsBuf, func(remoteNameByte []byte, _ string, _, _ bool) {
 			remoteName := PeerNameFromBin(remoteNameByte)
 			if _, found := newPeers[remoteName]; found {
 				return
@@ -195,12 +221,14 @@ func (peers *Peers) decodeUpdate(update []byte) (newPeers map[PeerName]*Peer, de
 			}
 			// Update refers to a peer which we have no knowledge
 			// of. Thus we can't apply the update. Abort.
-			unknownPeers = true
+			err = UnknownPeerError{remoteName}
 		})
-	}
-	if unknownPeers {
-		err = UnknownPeersError{}
-		return
+		if decErr != nil && decErr != io.EOF {
+			err = decErr
+		}
+		if err != nil {
+			return
+		}
 	}
 	return
 }
@@ -212,23 +240,12 @@ func (peers *Peers) applyUpdate(decodedUpdate []*Peer, decodedConns [][]byte) ma
 		name := newPeer.Name
 		// guaranteed to find peer in the peers.table
 		peer := peers.table[name]
-		if peer != newPeer {
-			if peer.Version() > newPeer.Version() {
-				// we know more about this one than the update. If
-				// peer is ourself, this is slightly racey (further
-				// changes could occur to ourself in the mean
-				// time). But it doesn't matter as we know that we're
-				// already > newPeer.Version() and that's not going to
-				// change.
-				newUpdate[name] = peer
-				continue
-			} else if peer == peers.ourself {
-				// nobody but us updates us
-				continue
-			} else if peer.Version() == newPeer.Version() {
-				// implication is that connections are equal too
-				continue
-			}
+		if peer != newPeer &&
+			(peer == peers.ourself || peer.Version() >= newPeer.Version()) {
+			// Nobody but us updates us. And if we know more about a
+			// peer than what's in the the update, we ignore the
+			// latter.
+			continue
 		}
 		// If we're here, either it was a new peer, or the update has
 		// more info about the peer than we do. Either case, we need
@@ -247,71 +264,82 @@ func (peers *Peers) applyUpdate(decodedUpdate []*Peer, decodedConns [][]byte) ma
 	return newUpdate
 }
 
-func (peer *Peer) encodePeer(enc *gob.Encoder) {
+func (peer *Peer) encode(enc *gob.Encoder) {
 	peer.RLock()
 	defer peer.RUnlock()
 
 	checkFatal(enc.Encode(peer.NameByte))
+	checkFatal(enc.Encode(peer.NickName))
 	checkFatal(enc.Encode(peer.UID))
 	checkFatal(enc.Encode(peer.version))
 
 	connsBuf := new(bytes.Buffer)
 	connsEnc := gob.NewEncoder(connsBuf)
 	for _, conn := range peer.connections {
-		// DANGER holding rlock on peer, going to take rlock on conn
-		if !conn.Established() {
-			continue
-		}
 		checkFatal(connsEnc.Encode(conn.Remote().NameByte))
 		checkFatal(connsEnc.Encode(conn.RemoteTCPAddr()))
+		checkFatal(connsEnc.Encode(conn.Outbound()))
+		// DANGER holding rlock on peer, going to take rlock on conn
+		checkFatal(connsEnc.Encode(conn.Established()))
 	}
 	checkFatal(enc.Encode(connsBuf.Bytes()))
 }
 
-func decodePeerNoConns(dec *gob.Decoder) (nameByte []byte, uid uint64, version uint64, conns []byte, err error) {
-	err = dec.Decode(&nameByte)
-	if err != nil {
+func decodePeerNoConns(dec *gob.Decoder) (nameByte []byte, nickName string, uid uint64, version uint64, conns []byte, err error) {
+	if err = dec.Decode(&nameByte); err != nil {
 		return
 	}
-	err = dec.Decode(&uid)
-	if err != nil {
+	if err = dec.Decode(&nickName); err != nil {
 		return
 	}
-	err = dec.Decode(&version)
-	if err != nil {
+	if err = dec.Decode(&uid); err != nil {
 		return
 	}
-	err = dec.Decode(&conns)
-	if err == io.EOF {
-		err = nil
+	if err = dec.Decode(&version); err != nil {
+		return
+	}
+	if err = dec.Decode(&conns); err != nil {
+		return
 	}
 	return
 }
 
-func connsIterator(input []byte, fun func([]byte, string)) {
+func connsIterator(input []byte, fun func([]byte, string, bool, bool)) error {
 	buf := new(bytes.Buffer)
 	buf.Write(input)
 	dec := gob.NewDecoder(buf)
 	for {
 		var nameByte []byte
-		err := dec.Decode(&nameByte)
-		if err == io.EOF {
-			return
+		if err := dec.Decode(&nameByte); err != nil {
+			return err
 		}
-		checkFatal(err)
 		var foundAt string
-		checkFatal(dec.Decode(&foundAt))
-		fun(nameByte, string(foundAt))
+		if err := dec.Decode(&foundAt); err != nil {
+			return err
+		}
+		var outbound bool
+		if err := dec.Decode(&outbound); err != nil {
+			return err
+		}
+		var established bool
+		if err := dec.Decode(&established); err != nil {
+			return err
+		}
+		fun(nameByte, foundAt, outbound, established)
 	}
 }
 
 func readConnsMap(peer *Peer, buf []byte, table map[PeerName]*Peer) map[PeerName]Connection {
 	conns := make(map[PeerName]Connection)
-	connsIterator(buf, func(nameByte []byte, remoteTCPAddr string) {
+	if err := connsIterator(buf, func(nameByte []byte, remoteTCPAddr string, outbound bool, established bool) {
 		name := PeerNameFromBin(nameByte)
 		remotePeer := table[name]
-		conn := NewRemoteConnection(peer, remotePeer, remoteTCPAddr)
+		conn := NewRemoteConnection(peer, remotePeer, remoteTCPAddr, outbound, established)
 		conns[name] = conn
-	})
+	}); err != io.EOF {
+		// this should never happen since we've already successfully
+		// decoded the same data in decodeUpdate
+		checkFatal(err)
+	}
 	return conns
 }

router/peers_test.go

@@ -0,0 +1,113 @@
+package router
+
+import (
+	"fmt"
+	wt "github.com/weaveworks/weave/testing"
+	"math/rand"
+	"testing"
+)
+
+// TODO we should also test:
+//
+// - applying an incremental update, including the case where that
+//   leads to an UnknownPeerError
+//
+// - the "improved update" calculation
+//
+// - non-gc of peers that are only referenced locally
+
+func newNode(name PeerName) (*Peer, *Peers) {
+	peer := NewPeer(name, "", 0, 0)
+	peers := NewPeers(peer, func(*Peer) {})
+	peers.FetchWithDefault(peer)
+	return peer, peers
+}
+
+// Check that ApplyUpdate copies the whole topology from peers
+func checkApplyUpdate(t *testing.T, peers *Peers) {
+	dummyName, _ := PeerNameFromString("99:00:00:01:00:00")
+	// We need a new node outside of the network, with a connection
+	// into it.
+	_, testBedPeers := newNode(dummyName)
+	testBedPeers.AddTestConnection(peers.ourself)
+	testBedPeers.ApplyUpdate(peers.EncodePeers(peers.Names()))
+
+	checkTopologyPeers(t, true, testBedPeers.allPeersExcept(dummyName), peers.allPeers()...)
+}
+
+func TestPeersEncoding(t *testing.T) {
+	const numNodes = 20
+	const numIters = 1000
+	var peer [numNodes]*Peer
+	var ps [numNodes]*Peers
+	for i := 0; i < numNodes; i++ {
+		name, _ := PeerNameFromString(fmt.Sprintf("%02d:00:00:01:00:00", i))
+		peer[i], ps[i] = newNode(name)
+	}
+
+	var conns []struct{ from, to int }
+	for i := 0; i < numIters; i++ {
+		oper := rand.Intn(2)
+		switch oper {
+		case 0:
+			from, to := rand.Intn(numNodes), rand.Intn(numNodes)
+			if from != to {
+				if _, found := peer[from].ConnectionTo(peer[to].Name); !found {
+					ps[from].AddTestConnection(peer[to])
+					conns = append(conns, struct{ from, to int }{from, to})
+					checkApplyUpdate(t, ps[from])
+				}
+			}
+		case 1:
+			if len(conns) > 0 {
+				n := rand.Intn(len(conns))
+				c := conns[n]
+				ps[c.from].DeleteTestConnection(peer[c.to])
+				ps[c.from].GarbageCollect()
+				checkApplyUpdate(t, ps[c.from])
+				conns = append(conns[:n], conns[n+1:]...)
+			}
+		}
+	}
+}
+
+func TestPeersGarbageCollection(t *testing.T) {
+	const (
+		peer1NameString = "01:00:00:01:00:00"
+		peer2NameString = "02:00:00:02:00:00"
+		peer3NameString = "03:00:00:03:00:00"
+	)
+	var (
+		peer1Name, _ = PeerNameFromString(peer1NameString)
+		peer2Name, _ = PeerNameFromString(peer2NameString)
+		peer3Name, _ = PeerNameFromString(peer3NameString)
+	)
+
+	// Create some peers with some connections to each other
+	p1, ps1 := newNode(peer1Name)
+	p2, ps2 := newNode(peer2Name)
+	p3, ps3 := newNode(peer3Name)
+	ps1.AddTestConnection(p2)
+	ps2.AddTestRemoteConnection(p1, p2)
+	ps2.AddTestConnection(p1)
+	ps2.AddTestConnection(p3)
+	ps3.AddTestConnection(p1)
+	ps1.AddTestConnection(p3)
+	ps2.AddTestRemoteConnection(p1, p3)
+	ps2.AddTestRemoteConnection(p3, p1)
+
+	// Every peer is referenced, so nothing should be dropped
+	wt.AssertEmpty(t, ps1.GarbageCollect(), "peers removed")
+	wt.AssertEmpty(t, ps2.GarbageCollect(), "peers removed")
+	wt.AssertEmpty(t, ps3.GarbageCollect(), "peers removed")
+
+	// Drop the connection from 2 to 3, and 3 isn't garbage-collected
+	// because 1 has a connection to 3
+	ps2.DeleteTestConnection(p3)
+	wt.AssertEmpty(t, ps2.GarbageCollect(), "peers removed")
+
+	// Drop the connection from 1 to 3, and 3 will get removed by
+	// garbage-collection
+	ps1.DeleteTestConnection(p3)
+	checkPeerArray(t, ps1.GarbageCollect(), p3)
+}

router/protocol.go

@@ -0,0 +1,29 @@
+package router
+
+const (
+	Protocol        = "weave"
+	ProtocolVersion = 15
+)
+
+type ProtocolTag byte
+
+const (
+	ProtocolHeartbeat ProtocolTag = iota
+	ProtocolConnectionEstablished
+	ProtocolFragmentationReceived
+	ProtocolStartFragmentationTest
+	ProtocolNonce
+	ProtocolPMTUVerified
+	ProtocolGossip
+	ProtocolGossipUnicast
+	ProtocolGossipBroadcast
+)
+
+type ProtocolMsg struct {
+	tag ProtocolTag
+	msg []byte
+}
+
+type ProtocolSender interface {
+	SendProtocolMsg(m ProtocolMsg)
+}

router/router.go

@@ -17,55 +17,95 @@ const macMaxAge = 10 * time.Minute // [1]
 // [1] should be greater than typical ARP cache expiries, i.e. > 3/2 *
 // /proc/sys/net/ipv4_neigh/*/base_reachable_time_ms on Linux
 
-func NewRouter(iface *net.Interface, name PeerName, password []byte, connLimit int, bufSz int, logFrame func(string, []byte, *layers.Ethernet)) *Router {
+type LogFrameFunc func(string, []byte, *layers.Ethernet)
+
+type RouterConfig struct {
+	Port      int
+	Iface     *net.Interface
+	Password  []byte
+	ConnLimit int
+	BufSz     int
+	LogFrame  LogFrameFunc
+}
+
+type Router struct {
+	RouterConfig
+	Ourself         *LocalPeer
+	Macs            *MacCache
+	Peers           *Peers
+	Routes          *Routes
+	ConnectionMaker *ConnectionMaker
+	GossipChannels  map[uint32]*GossipChannel
+	TopologyGossip  Gossip
+	UDPListener     *net.UDPConn
+}
+
+type PacketSource interface {
+	ReadPacket() ([]byte, error)
+}
+
+type PacketSink interface {
+	WritePacket([]byte) error
+}
+
+type PacketSourceSink interface {
+	PacketSource
+	PacketSink
+}
+
+func NewRouter(config RouterConfig, name PeerName, nickName string) *Router {
+	router := &Router{RouterConfig: config, GossipChannels: make(map[uint32]*GossipChannel)}
 	onMacExpiry := func(mac net.HardwareAddr, peer *Peer) {
-		log.Println("Expired MAC", mac, "at", peer.Name)
+		log.Println("Expired MAC", mac, "at", peer.FullName())
 	}
 	onPeerGC := func(peer *Peer) {
-		log.Println("Removing unreachable", peer)
+		router.Macs.Delete(peer)
+		log.Println("Removed unreachable peer", peer.FullName())
 	}
-	router := &Router{
-		Iface:     iface,
-		Macs:      NewMacCache(macMaxAge, onMacExpiry),
-		ConnLimit: connLimit,
-		BufSz:     bufSz,
-		LogFrame:  logFrame}
-	if len(password) > 0 {
-		router.Password = &password
-	}
-	router.Ourself = StartLocalPeer(name, router)
-	router.Peers = NewPeers(router.Ourself.Peer, router.Macs, onPeerGC)
+	router.Ourself = NewLocalPeer(name, nickName, router)
+	router.Macs = NewMacCache(macMaxAge, onMacExpiry)
+	router.Peers = NewPeers(router.Ourself.Peer, onPeerGC)
 	router.Peers.FetchWithDefault(router.Ourself.Peer)
-	log.Println("Our name is", router.Ourself.Name)
-
+	router.Routes = NewRoutes(router.Ourself.Peer, router.Peers)
+	router.ConnectionMaker = NewConnectionMaker(router.Ourself, router.Peers, router.Port)
+	router.TopologyGossip = router.NewGossip("topology", router)
 	return router
 }
 
+func (router *Router) Start() {
+	// we need two pcap handles since they aren't thread-safe
+	var pio PacketSourceSink
+	var po PacketSink
+	var err error
+	if router.Iface != nil {
+		pio, err = NewPcapIO(router.Iface.Name, router.BufSz)
+		checkFatal(err)
+		po, err = NewPcapO(router.Iface.Name)
+		checkFatal(err)
+	}
+	router.Ourself.Start()
+	router.Macs.Start()
+	router.Routes.Start()
+	router.ConnectionMaker.Start()
+	router.UDPListener = router.listenUDP(router.Port, po)
+	router.listenTCP(router.Port)
+	if pio != nil {
+		router.sniff(pio)
+	}
+}
+
 func (router *Router) UsingPassword() bool {
 	return router.Password != nil
 }
 
-func (router *Router) Start() {
-	// we need two pcap handles since they aren't thread-safe
-	pio, err := NewPcapIO(router.Iface.Name, router.BufSz)
-	checkFatal(err)
-	po, err := NewPcapO(router.Iface.Name)
-	checkFatal(err)
-	router.Routes = StartRoutes(router.Ourself.Peer, router.Peers)
-	router.ConnectionMaker = StartConnectionMaker(router.Ourself, router.Peers)
-	router.UDPListener = router.listenUDP(Port, po)
-	router.listenTCP(Port)
-	router.sniff(pio)
-}
-
 func (router *Router) Status() string {
 	var buf bytes.Buffer
-	buf.WriteString(fmt.Sprintln("Our name is", router.Ourself.Name))
-	buf.WriteString(fmt.Sprintln("Sniffing traffic on", router.Iface))
-	buf.WriteString(fmt.Sprintf("MACs:\n%s", router.Macs))
-	buf.WriteString(fmt.Sprintf("Peers:\n%s", router.Peers))
-	buf.WriteString(fmt.Sprintf("Routes:\n%s", router.Routes))
-	buf.WriteString(fmt.Sprintf("Reconnects:\n%s", router.ConnectionMaker))
+	fmt.Fprintln(&buf, "Our name is", router.Ourself.FullName())
+	fmt.Fprintln(&buf, "Sniffing traffic on", router.Iface)
+	fmt.Fprintf(&buf, "MACs:\n%s", router.Macs)
+	fmt.Fprintf(&buf, "Peers:\n%s", router.Peers)
+	fmt.Fprintf(&buf, "Routes:\n%s", router.Routes)
+	fmt.Fprintf(&buf, "Reconnects:\n%s", router.ConnectionMaker)
 	return buf.String()
 }
 
@@ -73,8 +113,6 @@ func (router *Router) sniff(pio PacketSourceSink) {
 	log.Println("Sniffing traffic on", router.Iface)
 
 	dec := NewEthernetDecoder()
-	injectFrame := func(frame []byte) error { return pio.WritePacket(frame) }
-	checkFrameTooBig := func(err error) error { return dec.CheckFrameTooBig(err, injectFrame) }
 	mac := router.Iface.HardwareAddr
 	if router.Macs.Enter(mac, router.Ourself.Peer) {
 		log.Println("Discovered our MAC", mac)
@@ -84,16 +122,16 @@ func (router *Router) sniff(pio PacketSourceSink) {
 			pkt, err := pio.ReadPacket()
 			checkFatal(err)
 			router.LogFrame("Sniffed", pkt, nil)
-			checkWarn(router.handleCapturedPacket(pkt, dec, checkFrameTooBig))
+			router.handleCapturedPacket(pkt, dec, pio)
 		}
 	}()
 }
 
-func (router *Router) handleCapturedPacket(frameData []byte, dec *EthernetDecoder, checkFrameTooBig func(error) error) error {
+func (router *Router) handleCapturedPacket(frameData []byte, dec *EthernetDecoder, po PacketSink) {
 	dec.DecodeLayers(frameData)
 	decodedLen := len(dec.decoded)
 	if decodedLen == 0 {
-		return nil
+		return
 	}
 	srcMac := dec.eth.SrcMAC
 	srcPeer, found := router.Macs.Lookup(srcMac)
@@ -101,18 +139,18 @@ func (router *Router) handleCapturedPacket(frameData []byte, dec *EthernetDecode
 	// frames, the srcMAC will have been recorded as associated with a
 	// different peer.
 	if found && srcPeer != router.Ourself.Peer {
-		return nil
+		return
 	}
 	if router.Macs.Enter(srcMac, router.Ourself.Peer) {
 		log.Println("Discovered local MAC", srcMac)
 	}
 	if dec.DropFrame() {
-		return nil
+		return
 	}
 	dstMac := dec.eth.DstMAC
 	dstPeer, found := router.Macs.Lookup(dstMac)
 	if found && dstPeer == router.Ourself.Peer {
-		return nil
+		return
 	}
 	df := decodedLen == 2 && (dec.ip.Flags&layers.IPv4DontFragment != 0)
 	if df {
@@ -127,11 +165,18 @@ func (router *Router) handleCapturedPacket(frameData []byte, dec *EthernetDecode
 	frameCopy := make([]byte, frameLen, frameLen)
 	copy(frameCopy, frameData)
 
+	// If we don't know which peer corresponds to the dest MAC,
+	// broadcast it.
 	if !found {
-		return checkFrameTooBig(router.Ourself.Broadcast(df, frameCopy, dec))
-	} else {
-		return checkFrameTooBig(router.Ourself.Forward(dstPeer, df, frameCopy, dec))
+		router.Ourself.Broadcast(df, frameCopy, dec)
+		return
 	}
+
+	err := router.Ourself.Forward(dstPeer, df, frameCopy, dec)
+	if ftbe, ok := err.(FrameTooBigError); ok {
+		err = dec.sendICMPFragNeeded(ftbe.EPMTU, po.WritePacket)
+	}
+	checkWarn(err)
 }
 
 func (router *Router) listenTCP(localPort int) {
@@ -154,10 +199,13 @@ func (router *Router) listenTCP(localPort int) {
 
 func (router *Router) acceptTCP(tcpConn *net.TCPConn) {
 	// someone else is dialing us, so our udp sender is the conn
-	// on Port and we wait for them to send us something on UDP to
+	// on router.Port and we wait for them to send us something on UDP to
 	// start.
-	connRemote := NewRemoteConnection(router.Ourself.Peer, nil, tcpConn.RemoteAddr().String())
-	NewLocalConnection(connRemote, true, tcpConn, nil, router)
+	remoteAddrStr := tcpConn.RemoteAddr().String()
+	log.Printf("->[%s] connection accepted\n", remoteAddrStr)
+	connRemote := NewRemoteConnection(router.Ourself.Peer, nil, remoteAddrStr, false, false)
+	connLocal := NewLocalConnection(connRemote, tcpConn, nil, router)
+	connLocal.Start(true)
 }
 
 func (router *Router) listenUDP(localPort int, po PacketSink) *net.UDPConn {
@@ -179,7 +227,6 @@ func (router *Router) listenUDP(localPort int, po PacketSink) *net.UDPConn {
 func (router *Router) udpReader(conn *net.UDPConn, po PacketSink) {
 	defer conn.Close()
 	dec := NewEthernetDecoder()
-	handleUDPPacket := router.handleUDPPacketFunc(dec, po)
 	buf := make([]byte, MaxUDPPacketSize)
 	for {
 		n, sender, err := conn.ReadFromUDP(buf)
@@ -195,10 +242,6 @@ func (router *Router) udpReader(conn *net.UDPConn, po PacketSink) {
 		name := PeerNameFromBin(buf[:NameSize])
 		packet := make([]byte, n-NameSize)
 		copy(packet, buf[NameSize:n])
-		udpPacket := &UDPPacket{
-			Name:   name,
-			Packet: packet,
-			Sender: sender}
 		peerConn, found := router.Ourself.ConnectionTo(name)
 		if !found {
 			continue
@@ -207,94 +250,174 @@ func (router *Router) udpReader(conn *net.UDPConn, po PacketSink) {
 		if !ok {
 			continue
 		}
-		err = relayConn.Decryptor.IterateFrames(handleUDPPacket, udpPacket)
-		if pde, ok := err.(PacketDecodingError); ok {
-			if pde.Fatal {
-				relayConn.CheckFatal(pde)
-			} else {
-				relayConn.log(pde.Error())
-			}
-		} else {
-			checkWarn(err)
+		if err := relayConn.Decryptor.IterateFrames(packet, router.handleUDPPacketFunc(dec, sender, po)); err != nil {
+			// Errors during UDP packet decoding / processing are
+			// non-fatal. One common cause is that we receive and
+			// attempt to decrypt a "stray" packet. This can actually
+			// happen quite easily if there is some connection churn
+			// between two peers. After all, UDP isn't a
+			// connection-oriented protocol, yet we pretend it is.
+			//
+			// If anything really is seriously, unrecoverably amiss
+			// with a connection, that will typically result in missed
+			// heartbeats and the connection getting shut down because
+			// of that.
+			relayConn.Log(err)
 		}
 	}
 }
 
-func (router *Router) handleUDPPacketFunc(dec *EthernetDecoder, po PacketSink) FrameConsumer {
-	checkFrameTooBig := func(err error, srcPeer *Peer) error {
-		if err == nil { // optimisation: avoid closure creation in common case
-			return nil
-		}
-		return dec.CheckFrameTooBig(err,
-			func(icmpFrame []byte) error {
-				return router.Ourself.Forward(srcPeer, false, icmpFrame, nil)
-			})
-	}
-
-	return func(relayConn *LocalConnection, sender *net.UDPAddr, srcNameByte, dstNameByte []byte, frameLen uint16, frame []byte) error {
-		srcName := PeerNameFromBin(srcNameByte)
-		dstName := PeerNameFromBin(dstNameByte)
-		srcPeer, found := router.Peers.Fetch(srcName)
+func (router *Router) handleUDPPacketFunc(dec *EthernetDecoder, sender *net.UDPAddr, po PacketSink) FrameConsumer {
+	return func(relayConn *LocalConnection, srcNameByte, dstNameByte []byte, frame []byte) {
+		srcPeer, found := router.Peers.Fetch(PeerNameFromBin(srcNameByte))
 		if !found {
-			return nil
+			return
 		}
-		dstPeer, found := router.Peers.Fetch(dstName)
+		dstPeer, found := router.Peers.Fetch(PeerNameFromBin(dstNameByte))
 		if !found {
-			return nil
+			return
 		}
 
 		dec.DecodeLayers(frame)
 		decodedLen := len(dec.decoded)
+		if decodedLen == 0 {
+			return
+		}
+		// Handle special frames produced internally (rather than
+		// captured/forwarded) by the remote router.
+		//
+		// We really shouldn't be decoding these above, since they are
+		// not genuine Ethernet frames. However, it is actually more
+		// efficient to do so, as we want to optimise for the common
+		// (i.e. non-special) frames. These always need decoding, and
+		// detecting special frames is cheaper post decoding than pre.
+		if decodedLen == 1 && dec.IsSpecial() {
+			if srcPeer == relayConn.Remote() && dstPeer == router.Ourself.Peer {
+				handleSpecialFrame(relayConn, sender, frame)
+			}
+			return
+		}
+
 		df := decodedLen == 2 && (dec.ip.Flags&layers.IPv4DontFragment != 0)
-		srcMac := dec.eth.SrcMAC
 
 		if dstPeer != router.Ourself.Peer {
 			// it's not for us, we're just relaying it
-			if decodedLen == 0 {
-				return nil
-			}
 			if df {
 				router.LogFrame("Relaying DF", frame, &dec.eth)
 			} else {
 				router.LogFrame("Relaying", frame, &dec.eth)
 			}
-			return checkFrameTooBig(router.Ourself.Relay(srcPeer, dstPeer, df, frame, dec), srcPeer)
-		}
 
-		if relayConn.Remote().Name == srcPeer.Name {
-			if frameLen == 0 {
-				relayConn.SetRemoteUDPAddr(sender)
-				return nil
-			} else if frameLen == FragTestSize && bytes.Equal(frame, FragTest) {
-				relayConn.SendTCP(ProtocolFragmentationReceivedByte)
-				return nil
-			} else if frameLen == PMTUDiscoverySize && bytes.Equal(frame, PMTUDiscovery) {
-				return nil
+			err := router.Ourself.Relay(srcPeer, dstPeer, df, frame, dec)
+			if ftbe, ok := err.(FrameTooBigError); ok {
+				err = dec.sendICMPFragNeeded(ftbe.EPMTU, func(icmpFrame []byte) error {
+					return router.Ourself.Forward(srcPeer, false, icmpFrame, nil)
+				})
 			}
+
+			checkWarn(err)
+			return
 		}
 
-		if decodedLen == 0 {
-			return nil
-		}
-
-		if dec.IsPMTUVerify() && relayConn.Remote().Name == srcPeer.Name {
-			frameLenBytes := []byte{0, 0}
-			binary.BigEndian.PutUint16(frameLenBytes, uint16(frameLen-EthernetOverhead))
-			relayConn.SendTCP(Concat(ProtocolPMTUVerifiedByte, frameLenBytes))
-			return nil
-		}
+		srcMac := dec.eth.SrcMAC
+		dstMac := dec.eth.DstMAC
 
 		if router.Macs.Enter(srcMac, srcPeer) {
-			log.Println("Discovered remote MAC", srcMac, "at", srcPeer.Name)
+			log.Println("Discovered remote MAC", srcMac, "at", srcPeer.FullName())
+		}
+		if po != nil {
+			router.LogFrame("Injecting", frame, &dec.eth)
+			checkWarn(po.WritePacket(frame))
 		}
-		router.LogFrame("Injecting", frame, &dec.eth)
-		checkWarn(po.WritePacket(frame))
 
-		dstPeer, found = router.Macs.Lookup(dec.eth.DstMAC)
+		dstPeer, found = router.Macs.Lookup(dstMac)
 		if !found || dstPeer != router.Ourself.Peer {
-			return checkFrameTooBig(router.Ourself.RelayBroadcast(srcPeer, df, frame, dec), srcPeer)
+			router.LogFrame("Relaying broadcast", frame, &dec.eth)
+			router.Ourself.RelayBroadcast(srcPeer, df, frame, dec)
 		}
-
-		return nil
 	}
 }
+
+func handleSpecialFrame(relayConn *LocalConnection, sender *net.UDPAddr, frame []byte) {
+	frameLen := len(frame)
+	switch {
+	case frameLen == EthernetOverhead+8:
+		relayConn.ReceivedHeartbeat(sender, binary.BigEndian.Uint64(frame[EthernetOverhead:]))
+	case frameLen == FragTestSize && bytes.Equal(frame, FragTest):
+		relayConn.SendProtocolMsg(ProtocolMsg{ProtocolFragmentationReceived, nil})
+	case frameLen == PMTUDiscoverySize && bytes.Equal(frame, PMTUDiscovery):
+	default:
+		frameLenBytes := []byte{0, 0}
+		binary.BigEndian.PutUint16(frameLenBytes, uint16(frameLen-EthernetOverhead))
+		relayConn.SendProtocolMsg(ProtocolMsg{ProtocolPMTUVerified, frameLenBytes})
+	}
+}
+
+// Gossiper methods - the Router is the topology Gossiper
+
+type TopologyGossipData struct {
+	peers  *Peers
+	update PeerNameSet
+}
+
+func NewTopologyGossipData(peers *Peers, update ...*Peer) *TopologyGossipData {
+	names := make(PeerNameSet)
+	for _, p := range update {
+		names[p.Name] = void
+	}
+	return &TopologyGossipData{peers: peers, update: names}
+}
+
+func (d *TopologyGossipData) Merge(other GossipData) {
+	for name := range other.(*TopologyGossipData).update {
+		d.update[name] = void
+	}
+}
+
+func (d *TopologyGossipData) Encode() []byte {
+	return d.peers.EncodePeers(d.update)
+}
+
+func (router *Router) OnGossipUnicast(sender PeerName, msg []byte) error {
+	return fmt.Errorf("unexpected topology gossip unicast: %v", msg)
+}
+
+func (router *Router) OnGossipBroadcast(update []byte) (GossipData, error) {
+	origUpdate, _, err := router.applyTopologyUpdate(update)
+	if err != nil || len(origUpdate) == 0 {
+		return nil, err
+	}
+	return &TopologyGossipData{peers: router.Peers, update: origUpdate}, nil
+}
+
+func (router *Router) Gossip() GossipData {
+	return &TopologyGossipData{peers: router.Peers, update: router.Peers.Names()}
+}
+
+func (router *Router) OnGossip(update []byte) (GossipData, error) {
+	_, newUpdate, err := router.applyTopologyUpdate(update)
+	if err != nil || len(newUpdate) == 0 {
+		return nil, err
+	}
+	return &TopologyGossipData{peers: router.Peers, update: newUpdate}, nil
+}
+
+func (router *Router) applyTopologyUpdate(update []byte) (PeerNameSet, PeerNameSet, error) {
+	origUpdate, newUpdate, err := router.Peers.ApplyUpdate(update)
+	if _, ok := err.(UnknownPeerError); err != nil && ok {
+		// That update contained a reference to a peer which wasn't
+		// itself included in the update, and we didn't know about
+		// already. We ignore this; eventually we should receive an
+		// update containing a complete topology.
+		log.Println("Topology gossip:", err)
+		return nil, nil, nil
+	}
+	if err != nil {
+		return nil, nil, err
+	}
+	if len(newUpdate) > 0 {
+		router.ConnectionMaker.Refresh()
+		router.Routes.Recalculate()
+	}
+	return origUpdate, newUpdate, nil
+}

router/routes.go

@@ -3,17 +3,49 @@ package router
 import (
 	"bytes"
 	"fmt"
-	"log"
+	"math"
 	"sync"
 )
 
 type Routes struct {
 	sync.RWMutex
-	ourself   *Peer
-	peers     *Peers
-	unicast   map[PeerName]PeerName
-	broadcast map[PeerName][]PeerName
-	queryChan chan<- *Interaction
+	ourself      *Peer
+	peers        *Peers
+	unicast      map[PeerName]PeerName
+	unicastAll   map[PeerName]PeerName // [1]
+	broadcast    map[PeerName][]PeerName
+	broadcastAll map[PeerName][]PeerName // [1]
+	recalculate  chan<- *struct{}
+	wait         chan<- chan struct{}
+	// [1] based on *all* connections, not just established &
+	// symmetric ones
+}
+
+func NewRoutes(ourself *Peer, peers *Peers) *Routes {
+	routes := &Routes{
+		ourself:      ourself,
+		peers:        peers,
+		unicast:      make(map[PeerName]PeerName),
+		unicastAll:   make(map[PeerName]PeerName),
+		broadcast:    make(map[PeerName][]PeerName),
+		broadcastAll: make(map[PeerName][]PeerName)}
+	routes.unicast[ourself.Name] = UnknownPeerName
+	routes.unicastAll[ourself.Name] = UnknownPeerName
+	routes.broadcast[ourself.Name] = []PeerName{}
+	routes.broadcastAll[ourself.Name] = []PeerName{}
+	return routes
+}
+
+func (routes *Routes) Start() {
+	recalculate := make(chan *struct{}, 1)
+	wait := make(chan chan struct{})
+	routes.recalculate = recalculate
+	routes.wait = wait
+	go routes.run(recalculate, wait)
+}
+
+func (routes *Routes) PeerNames() PeerNameSet {
+	return routes.peers.Names()
 }
 
 func (routes *Routes) Unicast(name PeerName) (PeerName, bool) {
@@ -23,6 +55,13 @@ func (routes *Routes) Unicast(name PeerName) (PeerName, bool) {
 	return hop, found
 }
 
+func (routes *Routes) UnicastAll(name PeerName) (PeerName, bool) {
+	routes.RLock()
+	defer routes.RUnlock()
+	hop, found := routes.unicastAll[name]
+	return hop, found
+}
+
 func (routes *Routes) Broadcast(name PeerName) []PeerName {
 	routes.RLock()
 	defer routes.RUnlock()
@@ -33,68 +72,114 @@ func (routes *Routes) Broadcast(name PeerName) []PeerName {
 	return hops
 }
 
+func (routes *Routes) BroadcastAll(name PeerName) []PeerName {
+	routes.RLock()
+	defer routes.RUnlock()
+	hops, found := routes.broadcastAll[name]
+	if !found {
+		return []PeerName{}
+	}
+	return hops
+}
+
+// Choose min(log2(n_peers), n_neighbouring_peers) neighbours, with a
+// random distribution that is topology-sensitive, favouring
+// neighbours at the end of "bottleneck links". We determine the
+// latter based on the unicast routing table. If a neighbour appears
+// as the value more frequently than others - meaning that we reach a
+// higher proportion of peers via that neighbour than other neighbours
+// - then it is chosen with a higher probability.
+//
+// Note that we choose log2(n_peers) *neighbours*, not
+// peers. Consequently, on sparsely connected peers this function
+// returns a higher proportion of neighbours than elsewhere. In
+// extremis, on peers with fewer than log2(n_peers) neighbours, all
+// neighbours are returned.
+func (routes *Routes) RandomNeighbours(except PeerName) PeerNameSet {
+	res := make(PeerNameSet)
+	routes.RLock()
+	defer routes.RUnlock()
+	count := int(math.Log2(float64(len(routes.unicastAll))))
+	// depends on go's random map iteration
+	for _, dst := range routes.unicastAll {
+		if dst != UnknownPeerName && dst != except {
+			res[dst] = void
+			if len(res) >= count {
+				break
+			}
+		}
+	}
+	return res
+}
+
 func (routes *Routes) String() string {
 	var buf bytes.Buffer
 	routes.RLock()
 	defer routes.RUnlock()
-	buf.WriteString(fmt.Sprintln("unicast:"))
+	fmt.Fprintln(&buf, "unicast:")
 	for name, hop := range routes.unicast {
-		buf.WriteString(fmt.Sprintf("%s -> %s\n", name, hop))
+		fmt.Fprintf(&buf, "%s -> %s\n", name, hop)
 	}
-	buf.WriteString(fmt.Sprintln("broadcast:"))
+	fmt.Fprintln(&buf, "broadcast:")
 	for name, hops := range routes.broadcast {
-		buf.WriteString(fmt.Sprintf("%s -> %v\n", name, hops))
+		fmt.Fprintf(&buf, "%s -> %v\n", name, hops)
 	}
+	// We don't include the 'all' routes here since they are of
+	// limited utility in troubleshooting
 	return buf.String()
 }
 
-func StartRoutes(ourself *Peer, peers *Peers) *Routes {
-	queryChan := make(chan *Interaction, ChannelSize)
-	state := &Routes{
-		ourself:   ourself,
-		peers:     peers,
-		unicast:   make(map[PeerName]PeerName),
-		broadcast: make(map[PeerName][]PeerName),
-		queryChan: queryChan}
-	state.unicast[ourself.Name] = UnknownPeerName
-	state.broadcast[ourself.Name] = []PeerName{}
-	go state.queryLoop(queryChan)
-	return state
-}
-
-// ACTOR client API
-
-const (
-	RRecalculate = iota
-)
-
-// Async.
+// Request recalculation of the routing table. This is async but can
+// effectively be made synchronous with a subsequent call to
+// EnsureRecalculated.
 func (routes *Routes) Recalculate() {
-	routes.queryChan <- &Interaction{code: RRecalculate}
+	// The use of a 1-capacity channel in combination with the
+	// non-blocking send is an optimisation that results in multiple
+	// requests being coalesced.
+	select {
+	case routes.recalculate <- nil:
+	default:
+	}
 }
 
-// ACTOR server
+// Wait for any preceding Recalculate requests to be processed.
+func (routes *Routes) EnsureRecalculated() {
+	done := make(chan struct{})
+	routes.wait <- done
+	<-done
+}
 
-func (routes *Routes) queryLoop(queryChan <-chan *Interaction) {
+func (routes *Routes) run(recalculate <-chan *struct{}, wait <-chan chan struct{}) {
 	for {
-		query, ok := <-queryChan
-		if !ok {
-			return
-		}
-		switch query.code {
-		case RRecalculate:
-			unicast := routes.calculateUnicast()
-			broadcast := routes.calculateBroadcast()
-			routes.Lock()
-			routes.unicast = unicast
-			routes.broadcast = broadcast
-			routes.Unlock()
-		default:
-			log.Fatal("Unexpected routes query:", query)
+		select {
+		case <-recalculate:
+			routes.calculate()
+		case done := <-wait:
+			select {
+			case <-recalculate:
+				routes.calculate()
+			default:
+			}
+			close(done)
 		}
 	}
 }
 
+func (routes *Routes) calculate() {
+	var (
+		unicast      = routes.calculateUnicast(true)
+		unicastAll   = routes.calculateUnicast(false)
+		broadcast    = routes.calculateBroadcast(true)
+		broadcastAll = routes.calculateBroadcast(false)
+	)
+	routes.Lock()
+	routes.unicast = unicast
+	routes.unicastAll = unicastAll
+	routes.broadcast = broadcast
+	routes.broadcastAll = broadcastAll
+	routes.Unlock()
+}
+
 // Calculate all the routes for the question: if *we* want to send a
 // packet to Peer X, what is the next hop?
 //
@@ -104,8 +189,8 @@ func (routes *Routes) queryLoop(queryChan <-chan *Interaction) {
 // any knowledge of the MAC address at all. Thus there's no need
 // to exchange knowledge of MAC addresses, nor any constraints on
 // the routes that we construct.
-func (routes *Routes) calculateUnicast() map[PeerName]PeerName {
-	_, unicast := routes.ourself.Routes(nil, true)
+func (routes *Routes) calculateUnicast(establishedAndSymmetric bool) map[PeerName]PeerName {
+	_, unicast := routes.ourself.Routes(nil, establishedAndSymmetric)
 	return unicast
 }
 
@@ -127,23 +212,31 @@ func (routes *Routes) calculateUnicast() map[PeerName]PeerName {
 //     Y =/= Z /\ X.Routes(Y) <= X.Routes(Z) =>
 //     X.Routes(Y) u [P | Y.HasSymmetricConnectionTo(P)] <= X.Routes(Z)
 // where <= is the subset relationship on keys of the returned map.
-func (routes *Routes) calculateBroadcast() map[PeerName][]PeerName {
+func (routes *Routes) calculateBroadcast(establishedAndSymmetric bool) map[PeerName][]PeerName {
 	broadcast := make(map[PeerName][]PeerName)
 	ourself := routes.ourself
+	ourConnections := ourself.Connections()
 
-	routes.peers.ForEach(func(name PeerName, peer *Peer) {
+	routes.peers.ForEach(func(peer *Peer) {
 		hops := []PeerName{}
-		if found, reached := peer.Routes(ourself, true); found {
-			ourself.ForEachConnection(func(remoteName PeerName, conn Connection) {
-				if _, found := reached[remoteName]; found {
-					return
+		if found, reached := peer.Routes(ourself, establishedAndSymmetric); found {
+			// This is rather similar to the inner loop on
+			// peer.Routes(...); the main difference is in the
+			// locking.
+			for conn := range ourConnections {
+				if establishedAndSymmetric && !conn.Established() {
+					continue
 				}
-				if _, found := conn.Remote().ConnectionTo(ourself.Name); found {
+				remoteName := conn.Remote().Name
+				if _, found := reached[remoteName]; found {
+					continue
+				}
+				if remoteConn, found := conn.Remote().ConnectionTo(ourself.Name); !establishedAndSymmetric || (found && remoteConn.Established()) {
 					hops = append(hops, remoteName)
 				}
-			})
+			}
 		}
-		broadcast[name] = hops
+		broadcast[peer.Name] = hops
 	})
 	return broadcast
 }

router/types.go

@@ -1,269 +0,0 @@
-package router
-
-import (
-	"bytes"
-	"code.google.com/p/go-bit/bit"
-	"code.google.com/p/gopacket"
-	"code.google.com/p/gopacket/layers"
-	"encoding/gob"
-	"net"
-	"sync"
-	"time"
-)
-
-type Router struct {
-	Iface           *net.Interface
-	Ourself         *LocalPeer
-	Macs            *MacCache
-	Peers           *Peers
-	Routes          *Routes
-	ConnectionMaker *ConnectionMaker
-	UDPListener     *net.UDPConn
-	Password        *[]byte
-	ConnLimit       int
-	BufSz           int
-	LogFrame        func(string, []byte, *layers.Ethernet)
-}
-
-type Connection interface {
-	Local() *Peer
-	Remote() *Peer
-	RemoteTCPAddr() string
-	Established() bool
-	Shutdown()
-}
-
-type RemoteConnection struct {
-	local         *Peer
-	remote        *Peer
-	remoteTCPAddr string
-}
-
-type LocalConnection struct {
-	sync.RWMutex
-	RemoteConnection
-	TCPConn       *net.TCPConn
-	tcpSender     TCPSender
-	remoteUDPAddr *net.UDPAddr
-	established   bool
-	stackFrag     bool
-	effectivePMTU int
-	SessionKey    *[32]byte
-	heartbeat     *time.Ticker
-	fetchAll      *time.Ticker
-	fragTest      *time.Ticker
-	forwardChan   chan<- *ForwardedFrame
-	forwardChanDF chan<- *ForwardedFrame
-	stopForward   chan<- interface{}
-	stopForwardDF chan<- interface{}
-	verifyPMTU    chan<- int
-	Decryptor     Decryptor
-	Router        *Router
-	UID           uint64
-	shutdown      bool
-	queryChan     chan<- *ConnectionInteraction
-}
-
-type ConnectionInteraction struct {
-	Interaction
-	payload interface{}
-}
-
-type Forwarder struct {
-	conn            *LocalConnection
-	ch              <-chan *ForwardedFrame
-	stop            <-chan interface{}
-	verifyPMTUTick  <-chan time.Time
-	verifyPMTU      <-chan int
-	pmtuVerifyCount uint
-	enc             Encryptor
-	udpSender       UDPSender
-	maxPayload      int
-	pmtuVerified    bool
-	highestGoodPMTU int
-	unverifiedPMTU  int
-	lowestBadPMTU   int
-}
-
-type UDPPacket struct {
-	Name   PeerName
-	Packet []byte
-	Sender *net.UDPAddr
-}
-
-type EthernetDecoder struct {
-	eth     layers.Ethernet
-	ip      layers.IPv4
-	decoded []gopacket.LayerType
-	parser  *gopacket.DecodingLayerParser
-}
-
-type MsgTooBigError struct {
-	PMTU int // actual pmtu, i.e. what the kernel told us
-}
-
-type FrameTooBigError struct {
-	EPMTU int // effective pmtu, i.e. what we tell packet senders
-}
-
-type UnknownPeersError struct {
-}
-
-type NameCollisionError struct {
-	Name PeerName
-}
-
-type PacketDecodingError struct {
-	Fatal bool
-	Desc  string
-}
-
-type LocalAddress struct {
-	ip      net.IP
-	network *net.IPNet
-}
-
-type ForwardedFrame struct {
-	srcPeer *Peer
-	dstPeer *Peer
-	frame   []byte
-}
-
-type Interaction struct {
-	code       int
-	resultChan chan<- interface{}
-}
-
-// UDPSender interface and implementations
-
-type UDPSender interface {
-	Send([]byte) error
-	Shutdown() error
-}
-
-type SimpleUDPSender struct {
-	conn    *LocalConnection
-	udpConn *net.UDPConn
-}
-
-type RawUDPSender struct {
-	ipBuf     gopacket.SerializeBuffer
-	opts      gopacket.SerializeOptions
-	udpHeader *layers.UDP
-	socket    *net.IPConn
-	conn      *LocalConnection
-}
-
-// TCPSender interface and implementations
-
-type TCPSender interface {
-	Send([]byte) error
-}
-
-type SimpleTCPSender struct {
-	encoder *gob.Encoder
-}
-
-type EncryptedTCPSender struct {
-	sync.RWMutex
-	outerEncoder *gob.Encoder
-	innerEncoder *gob.Encoder
-	buffer       *bytes.Buffer
-	conn         *LocalConnection
-	msgCount     int
-}
-
-type EncryptedTCPMessage struct {
-	Number int
-	Body   []byte
-}
-
-// TCPReceiver interface and implementations
-
-type TCPReceiver interface {
-	Decode([]byte) ([]byte, error)
-}
-
-type SimpleTCPReceiver struct {
-}
-
-type EncryptedTCPReceiver struct {
-	conn     *LocalConnection
-	decoder  *gob.Decoder
-	buffer   *bytes.Buffer
-	msgCount int
-}
-
-// Encryptor interface and implementations
-
-type Encryptor interface {
-	FrameOverhead() int
-	PacketOverhead() int
-	IsEmpty() bool
-	Bytes() []byte
-	AppendFrame(*ForwardedFrame)
-	TotalLen() int
-}
-
-type NonEncryptor struct {
-	buf       []byte
-	bufTail   []byte
-	buffered  int
-	prefixLen int
-}
-
-type NaClEncryptor struct {
-	NonEncryptor
-	buf       []byte
-	offset    uint16
-	nonce     *[24]byte
-	nonceChan chan *[24]byte
-	flags     uint16
-	prefixLen int
-	conn      *LocalConnection
-	df        bool
-}
-
-// Decryptor interface and implementations
-
-type FrameConsumer func(*LocalConnection, *net.UDPAddr, []byte, []byte, uint16, []byte) error
-
-type Decryptor interface {
-	IterateFrames(FrameConsumer, *UDPPacket) error
-	ReceiveNonce([]byte)
-	Shutdown()
-}
-
-type NonDecryptor struct {
-	conn *LocalConnection
-}
-
-type NaClDecryptor struct {
-	NonDecryptor
-	instance   *NaClDecryptorInstance
-	instanceDF *NaClDecryptorInstance
-}
-
-type NaClDecryptorInstance struct {
-	nonce               *[24]byte
-	previousNonce       *[24]byte
-	usedOffsets         *bit.Set
-	previousUsedOffsets *bit.Set
-	highestOffsetSeen   uint16
-	nonceChan           chan *[24]byte
-}
-
-// Packet capture/inject interfaces
-
-type PacketSource interface {
-	ReadPacket() ([]byte, error)
-}
-
-type PacketSink interface {
-	WritePacket([]byte) error
-}
-
-type PacketSourceSink interface {
-	PacketSource
-	PacketSink
-}

router/udp_sender.go

@@ -0,0 +1,138 @@
+package router
+
+import (
+	"code.google.com/p/gopacket"
+	"code.google.com/p/gopacket/layers"
+	"log"
+	"net"
+	"syscall"
+)
+
+type UDPSender interface {
+	Send([]byte) error
+	Shutdown() error
+}
+
+type SimpleUDPSender struct {
+	conn    *LocalConnection
+	udpConn *net.UDPConn
+}
+
+type RawUDPSender struct {
+	ipBuf     gopacket.SerializeBuffer
+	opts      gopacket.SerializeOptions
+	udpHeader *layers.UDP
+	socket    *net.IPConn
+	conn      *LocalConnection
+}
+
+type MsgTooBigError struct {
+	PMTU int // actual pmtu, i.e. what the kernel told us
+}
+
+func NewSimpleUDPSender(conn *LocalConnection) *SimpleUDPSender {
+	return &SimpleUDPSender{udpConn: conn.Router.UDPListener, conn: conn}
+}
+
+func (sender *SimpleUDPSender) Send(msg []byte) error {
+	_, err := sender.udpConn.WriteToUDP(msg, sender.conn.RemoteUDPAddr())
+	return err
+}
+
+func (sender *SimpleUDPSender) Shutdown() error {
+	return nil
+}
+
+func NewRawUDPSender(conn *LocalConnection) (*RawUDPSender, error) {
+	ipSocket, err := dialIP(conn)
+	if err != nil {
+		return nil, err
+	}
+	udpHeader := &layers.UDP{SrcPort: layers.UDPPort(conn.Router.Port)}
+	ipBuf := gopacket.NewSerializeBuffer()
+	opts := gopacket.SerializeOptions{
+		FixLengths: true,
+		// UDP header is calculated with a phantom IP
+		// header. Yes, it's totally nuts. Thankfully, for UDP
+		// over IPv4, the checksum is optional. It's not
+		// optional for IPv6, but we'll ignore that for
+		// now. TODO
+		ComputeChecksums: false}
+
+	return &RawUDPSender{
+		ipBuf:     ipBuf,
+		opts:      opts,
+		udpHeader: udpHeader,
+		socket:    ipSocket,
+		conn:      conn}, nil
+}
+
+func (sender *RawUDPSender) Send(msg []byte) error {
+	payload := gopacket.Payload(msg)
+	sender.udpHeader.DstPort = layers.UDPPort(sender.conn.RemoteUDPAddr().Port)
+
+	err := gopacket.SerializeLayers(sender.ipBuf, sender.opts, sender.udpHeader, &payload)
+	if err != nil {
+		return err
+	}
+	packet := sender.ipBuf.Bytes()
+	_, err = sender.socket.Write(packet)
+	if err == nil || PosixError(err) != syscall.EMSGSIZE {
+		return err
+	}
+	f, err := sender.socket.File()
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	fd := int(f.Fd())
+	log.Println("EMSGSIZE on send, expecting PMTU update (IP packet was",
+		len(packet), "bytes, payload was", len(msg), "bytes)")
+	pmtu, err := syscall.GetsockoptInt(fd, syscall.IPPROTO_IP, syscall.IP_MTU)
+	if err != nil {
+		return err
+	}
+	return MsgTooBigError{PMTU: pmtu}
+}
+
+func (sender *RawUDPSender) Shutdown() error {
+	defer func() { sender.socket = nil }()
+	return sender.socket.Close()
+}
+
+func dialIP(conn *LocalConnection) (*net.IPConn, error) {
+	ipLocalAddr, err := ipAddr(conn.TCPConn.LocalAddr())
+	if err != nil {
+		return nil, err
+	}
+	ipRemoteAddr, err := ipAddr(conn.TCPConn.RemoteAddr())
+	if err != nil {
+		return nil, err
+	}
+	ipSocket, err := net.DialIP("ip4:UDP", ipLocalAddr, ipRemoteAddr)
+	if err != nil {
+		return nil, err
+	}
+	f, err := ipSocket.File()
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	fd := int(f.Fd())
+	// This Makes sure all packets we send out have DF set on them.
+	err = syscall.SetsockoptInt(fd, syscall.IPPROTO_IP, syscall.IP_MTU_DISCOVER, syscall.IP_PMTUDISC_DO)
+	if err != nil {
+		return nil, err
+	}
+	return ipSocket, nil
+}
+
+func ipAddr(addr net.Addr) (*net.IPAddr, error) {
+	host, _, err := net.SplitHostPort(addr.String())
+	if err != nil {
+		return nil, err
+	}
+	return &net.IPAddr{
+		IP:   net.ParseIP(host),
+		Zone: ""}, nil
+}

router/utils.go

@@ -1,12 +1,17 @@
 package router
 
 import (
+	"bytes"
 	"crypto/rand"
+	"encoding/gob"
 	"fmt"
+	"hash/fnv"
 	"log"
 	"net"
 )
 
+var void = struct{}{}
+
 func checkFatal(e error) {
 	if e != nil {
 		log.Fatal(e)
@@ -38,8 +43,8 @@ func (ftbe FrameTooBigError) Error() string {
 	return fmt.Sprint("Frame too big error. Effective PMTU is ", ftbe.EPMTU)
 }
 
-func (upe UnknownPeersError) Error() string {
-	return fmt.Sprint("Reference to unknown peers")
+func (upe UnknownPeerError) Error() string {
+	return fmt.Sprint("Reference to unknown peer ", upe.Name)
 }
 
 func (nce NameCollisionError) Error() string {
@@ -50,10 +55,6 @@ func (pde PacketDecodingError) Error() string {
 	return fmt.Sprint("Failed to decode packet: ", pde.Desc)
 }
 
-func (packet UDPPacket) String() string {
-	return fmt.Sprintf("UDP Packet\n name: %s\n sender: %v\n payload: % X", packet.Name, packet.Sender, packet.Packet)
-}
-
 func Concat(elems ...[]byte) []byte {
 	res := []byte{}
 	for _, e := range elems {
@@ -73,6 +74,21 @@ func randUint64() (r uint64) {
 	return
 }
 
+func hash(s string) uint32 {
+	h := fnv.New32a()
+	h.Write([]byte(s))
+	return h.Sum32()
+}
+
+func GobEncode(items ...interface{}) []byte {
+	buf := new(bytes.Buffer)
+	enc := gob.NewEncoder(buf)
+	for _, i := range items {
+		checkFatal(enc.Encode(i))
+	}
+	return buf.Bytes()
+}
+
 func macint(mac net.HardwareAddr) (r uint64) {
 	for _, b := range mac {
 		r <<= 8
@@ -101,14 +117,3 @@ func (lop ListOfPeers) Swap(i, j int) {
 func (lop ListOfPeers) Less(i, j int) bool {
 	return lop[i].Name < lop[j].Name
 }
-
-// given an address like '1.2.3.4:567', return the address if it has a port,
-// otherwise return the address with weave's standard port number
-func NormalisePeerAddr(peerAddr string) string {
-	_, _, err := net.SplitHostPort(peerAddr)
-	if err == nil {
-		return peerAddr
-	} else {
-		return fmt.Sprintf("%s:%d", peerAddr, Port)
-	}
-}

sigproxy/main.go

@@ -0,0 +1,50 @@
+package main
+
+import (
+	"log"
+	"os"
+	"os/exec"
+	"os/signal"
+	"syscall"
+)
+
+func main() {
+	if len(os.Args) == 1 {
+		log.Fatal("USAGE: sigproxy <command> [arguments ...]")
+	}
+
+	// Install signal handler as soon as possible - channel is buffered so
+	// we'll catch signals that arrive whilst child process is starting
+	sc := make(chan os.Signal, 1)
+	signal.Notify(sc, syscall.SIGINT, syscall.SIGTERM)
+
+	cmd := exec.Command(os.Args[1], os.Args[2:]...)
+
+	// These default to /dev/null, so set them explicitly to ours
+	cmd.Stdin = os.Stdin
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+
+	if err := cmd.Start(); err != nil {
+		log.Fatal(err)
+	}
+
+	// Only begin delivering signals after the child has started
+	go func() {
+		for {
+			// Signalling PID 0 delivers to our process group
+			syscall.Kill(0, (<-sc).(syscall.Signal))
+		}
+	}()
+
+	if err := cmd.Wait(); err != nil {
+		// Exit status is platform specific so not directly accessible - casts
+		// required to access system-dependent exit information
+		if exitErr, ok := err.(*exec.ExitError); ok {
+			waitStatus := exitErr.Sys().(syscall.WaitStatus)
+			os.Exit(waitStatus.ExitStatus())
+		}
+		os.Exit(1)
+	}
+	os.Exit(0)
+}

site/_layouts/default.html

@@ -15,7 +15,7 @@
     <!-- HEADER -->
     <div id="header_wrap" class="outer">
         <header class="inner">
-          <a id="forkme_banner" href="https://github.com/zettio/weave">View on GitHub</a>
+          <a id="forkme_banner" href="https://github.com/weaveworks/weave">View on GitHub</a>
 
           <h1 id="project_title">weave</h1>
           <h2 id="project_tagline">The Docker Network</h2>

site/boot2docker.md

@@ -23,7 +23,7 @@ example above, the 'launch' command would be run like this:
 
 and the 'run' command like this:
 
-    host1# C=$(boot2docker ssh "sudo ./weave run 10.0.1.1/24 -t -i ubuntu")
+    host1# C=$(boot2docker ssh "sudo ./weave run 10.2.1.1/24 -t -i ubuntu")
 
 For more information about how to access services running in
 Boot2Docker from the host or other machines (i.e. outside of the weave

site/building.md

@@ -3,35 +3,110 @@ title: Building Weave
 layout: default
 ---
 
-**NB** This is only necessary if you want to work on the weave code.
+You only need to build weave if you want to work on the weave codebase
+(or you just enjoy building software).
 
-## Building directly on Linux
+Apart from the `weave` shell script, weave is delivered as a set of
+container images.  There is no distribution-specific packaging, so in
+principle it shouldn't matter which Linux distribution you build
+under.  But naturally, Docker is a prerequisite (version 1.3.0 or
+later).  And it is difficult to build under Fedora because [Fedora
+does not include static
+libraries](http://fedoraproject.org/wiki/Packaging:Guidelines#Packaging_Static_Libraries).
+So we recommend building under Ubuntu.
 
-You can work on weave without using a VM if you are running the docker
-daemon outside a VM. (These instructions have only been tested on
-Ubuntu.)
+You can also build in a container under any system that supports
+Docker.  And you can run Ubuntu in a VM and build there.  These
+options are described below.
 
-To build weave you need `libpcap-dev` and `docker` (version 1.3.0 or
-later) installed. And `go` (and `git` and `hg` to fetch dependencies).
+## Building directly on Ubuntu
 
-The package name is `github.com/zettio/weave`, so assuming `$GOPATH`
-is set:
+The weave git repository should be cloned into
+`$GOPATH/src/github.com/weaveworks/weave`, in accordance with [the go
+workspace conventions](https://golang.org/doc/code.html#Workspaces):
 
 ```bash
-$ cd $GOPATH
-$ WEAVE=github.com/zettio/weave
-$ git clone https://$WEAVE src/$WEAVE
-$ cd src/$WEAVE
+$ WEAVE=github.com/weaveworks/weave
+$ git clone https://$WEAVE $GOPATH/src/$WEAVE
+$ cd $GOPATH/src/$WEAVE
 ```
 
-Then simply run
+Some prerequisites are needed to build weave:
+
+* Install Docker if you haven't already, by following the instructions
+  [on the Docker
+  site](https://docs.docker.com/installation/ubuntulinux/).  Note that
+  we recommend using the Docker-maintained `lxc-docker` package, rather
+  than the `docker.io` package which contains a very old version.  Then
+  install the other prerequisites for building with:
+* Building weave requires Go 1.4.2, and the `golang` package in Ubuntu
+  14.04 LTS is too old.  So you may need to [install the tarball from
+  golang.org](http://golang.org/doc/install).
+* A few other packages are also needed:
+```bash
+$ sudo apt-get install build-essential git mercurial libpcap-dev
+```
+
+Then to actually build, simply do:
 
 ```bash
 $ make
 ```
 
-This will build the weave router, produce a docker image
-`zettio/weave` and export that image to `/var/tmp/weave.tar`.
+This will build the weave components and package them into three
+Docker images (`weaveworks/weave`, `weaveworks/weavedns`, and
+`weaveworks/weaveexec`).  These are then exported (as `weave.tar`,
+`weavedns.tar` and `weaveexec.tar`).
+
+## Building in a Docker container
+
+As a preliminary step, we create a container image based on Ubuntu
+that has all the prerequisites.  This avoids the need to download and
+install them for each build.  In the `weave` directory, do:
+
+```bash
+$ sudo docker build -t weaveworks/weave-build build
+```
+
+Next we run a container based on that image. That container requires
+access to the Docker daemon on the host, via
+`/var/run/docker.sock`. If you are building under a Fedora or RHEL
+Docker host (or another distribution that uses SELinux), and you have
+SELinux set to enforcing mode, it will block attempts to access
+`/var/run/docker.sock` inside the container.  See
+[dpw/selinux-dockersock](https://github.com/dpw/selinux-dockersock)
+for a way to work around this problem.
+
+To perform a build, run:
+
+```bash
+$ sudo docker run -v /var/run/docker.sock:/var/run/docker.sock weaveworks/weave-build https://github.com/weaveworks/weave.git
+```
+
+This will clone the weave git repository, then do the build.
+
+When the build completes, the resulting images are stored in docker on
+the host, as when building directly under Ubuntu.
+
+The container arguments are passed to `git clone`, so for example, you
+can build from a forked repository and a specific branch with:
+
+```bash
+$ sudo docker run -v /var/run/docker.sock:/var/run/docker.sock weaveworks/weave-build -b <branch name> <repo URI>
+```
+
+Alternatively, you might want to build from a weave source tree
+already present on the host.  You can do this by using the `-v` option
+to bind the bind your go workspace containing the weave repository to
+`/home/go` inside the container.  No container arguments should be
+passed in this case:
+
+```bash
+$ sudo docker run -v /var/run/docker.sock:/var/run/docker.sock -v <host gopath>:/home/go weaveworks/weave-build
+```
+
+This will leave the intermediate build artifacts on the host, so that
+you can modify the weave source code and rebuild quickly.
 
 ## Building using Vagrant
 
@@ -45,7 +120,7 @@ Vagrant to run VMs in.
 First, check out the code:
 
 ```bash
-$ git clone https://github.com/zettio/weave
+$ git clone https://github.com/weaveworks/weave
 $ cd weave
 ```
 
@@ -66,7 +141,7 @@ $ vagrant up
 
 and wait for a while (don't worry, the long download and package
 installation is done just once). The working directory is sync'ed with
-`~/src/github.com/zettio/weave` on the VM, so you can edit files and
+`~/src/github.com/weaveworks/weave` on the VM, so you can edit files and
 use git and so on in the regular filesystem.
 
 To build and run the code, you need to use the VM. To log in and build
@@ -74,7 +149,7 @@ the weave image, do
 
 ```bash
 $ vagrant ssh
-vm$ cd src/github.com/zettio/weave
+vm$ cd src/github.com/weaveworks/weave
 vm$ make
 ```
 
@@ -82,7 +157,7 @@ The docker daemon is also running in this VM, so you can then do
 
 ```bash
 vm$ sudo ./weave launch
-vm$ docker ps
+vm$ sudo docker ps
 ```
 
 and so on.

site/features.md

@@ -6,7 +6,7 @@ layout: default
 # Weave features
 
 Weave has a few more features beyond those illustrated by the [basic
-example](https://github.com/zettio/weave#example):
+example](https://github.com/weaveworks/weave#example):
 
  * [Virtual ethernet switch](#virtual-ethernet-switch)
  * [Application isolation](#application-isolation)
@@ -37,14 +37,14 @@ container on `$HOST1` we can start a netcat "service" with
 
 and then connect to it from the container on `$HOST2` with
 
-    root@f76829496120:/# echo 'Hello, world.' | nc 10.0.1.1 4422
+    root@f76829496120:/# echo 'Hello, world.' | nc 10.2.1.1 4422
 
 Note that *any* protocol is supported. Doesn't even have to be over
 TCP/IP, e.g. a netcat UDP service would be run with
 
     root@28841bd02eff:/# nc -lu -p 5533
 
-    root@f76829496120:/# echo 'Hello, world.' | nc -u 10.0.1.1 5533
+    root@f76829496120:/# echo 'Hello, world.' | nc -u 10.2.1.1 5533
 
 We can deploy the entire arsenal of standard network tools and
 applications, developed over decades, to configure, secure, monitor,
@@ -63,39 +63,46 @@ To accomplish that, we assign each application a different subnet. So,
 in the above example, if we wanted to add another application similar
 to, but isolated from, our first, we'd launch the containers with...
 
-    host1# D=$(weave run 10.0.2.1/24 -t -i ubuntu)
-    host2# D=$(weave run 10.0.2.2/24 -t -i ubuntu)
+    host1# D=$(weave run 10.2.2.1/24 -t -i ubuntu)
+    host2# D=$(weave run 10.2.2.2/24 -t -i ubuntu)
 
 A quick 'ping' test in the containers confirms that they can talk to
 each other but not the containers of our first application...
 
     host1# docker attach $D
     
-    root@da50502598d5:/# ping -c 1 -q 10.0.2.2
-    PING 10.0.2.2 (10.0.2.2): 48 data bytes
-    --- 10.0.2.2 ping statistics ---
+    root@da50502598d5:/# ping -c 1 -q 10.2.2.2
+    PING 10.2.2.2 (10.2.2.2): 48 data bytes
+    --- 10.2.2.2 ping statistics ---
     1 packets transmitted, 1 packets received, 0% packet loss
     round-trip min/avg/max/stddev = 0.562/0.562/0.562/0.000 ms
     
-    root@da50502598d5:/# ping -c 1 -q 10.0.1.1
-    PING 10.0.1.1 (10.0.1.1) 56(84) bytes of data.
-    --- 10.0.1.1 ping statistics ---
+    root@da50502598d5:/# ping -c 1 -q 10.2.1.1
+    PING 10.2.1.1 (10.2.1.1) 56(84) bytes of data.
+    --- 10.2.1.1 ping statistics ---
     1 packets transmitted, 0 received, 100% packet loss, time 0ms
     
-    root@da50502598d5:/# ping -c 1 -q 10.0.1.2
-    PING 10.0.1.2 (10.0.1.2) 56(84) bytes of data.
-    --- 10.0.1.2 ping statistics ---
+    root@da50502598d5:/# ping -c 1 -q 10.2.1.2
+    PING 10.2.1.2 (10.2.1.2) 56(84) bytes of data.
+    --- 10.2.1.2 ping statistics ---
     1 packets transmitted, 0 received, 100% packet loss, time 0ms
 
 This isolation-through-subnets scheme is an example of carrying over a
 well-known technique from the 'on metal' days to containers.
 
+If desired, a container can be attached to multiple subnets when it is
+started:
+
+    host1# weave run 10.2.2.1/24 10.2.3.1/24 -t -i ubuntu
+
 NB: By default docker permits communication between containers on the
 same host, via their docker-assigned IP addresses. For complete
 isolation between application containers, that feature needs to be
 disabled by
 [setting `--icc=false`](https://docs.docker.com/articles/networking/#between-containers)
-in the docker daemon configuration.
+in the docker daemon configuration. Furthermore, containers should be
+prevented from capturing and injecting raw network packets - this can
+be accomplished by starting them with the `--cap-drop net_raw` option.
 
 ### <a name="dynamic-network-attachment"></a>Dynamic network attachment
 
@@ -108,44 +115,50 @@ to the weave network. To illustrate, we can achieve the same effect as
 the first example with
 
     host1# C=$(docker run -d -t -i ubuntu)
-    host1# weave attach 10.0.1.1/24 $C
+    host1# weave attach 10.2.1.1/24 $C
 
 There is a matching `weave detach` command:
 
-    host1# weave detach 10.0.1.1/24 $C
+    host1# weave detach 10.2.1.1/24 $C
 
 You can detach a container from one application network and attach it
 to another:
 
-    host1# weave detach 10.0.1.1/24 $C
-    host1# weave attach 10.0.2.1/24 $C
+    host1# weave detach 10.2.1.1/24 $C
+    host1# weave attach 10.2.2.1/24 $C
 
 or attach a container to multiple application networks, effectively
 sharing it between applications:
 
-    host1# weave attach 10.0.1.1/24 $C
-    host1# weave attach 10.0.2.1/24 $C
+    host1# weave attach 10.2.1.1/24 $C
+    host1# weave attach 10.2.2.1/24 $C
+
+Finally, multiple addresses can be attached or detached with a single
+invocation:
+
+    host1# weave attach 10.2.1.1/24 10.2.2.1/24 10.2.3.1/24 $C
+    host1# weave detach 10.2.1.1/24 10.2.2.1/24 10.2.3.1/24 $C
 
 ### <a name="security"></a>Security
 
 In order to connect containers across untrusted networks, weave peers
-can be told to encrypt traffic by supplying a `-password` or
-`-passwordfile` option when launching weave, e.g.
+can be told to encrypt traffic by supplying a `-password` option or
+`WEAVE_PASSWORD` environment variable when launching weave, e.g.
 
     host1# weave launch -password wEaVe
 
 or
 
-    host1# weave launch -passwordfile /path/to/file
+    host1# export WEAVE_PASSWORD=wEaVe
+    host1# weave launch
 
-where the file must contain a single line like this:
-
-    WEAVE_PASSWORD=wEaVe
+_NOTE: The command line option takes precedence over the environment
+variable._
 
 The same password must be specified for all weave peers; it is a
 component in the creation of ephemeral session keys for connections
-between peers. See the
-[crypto documentation](how-it-works.html#crypto) for more details.
+between peers. See the [crypto documentation](how-it-works.html#crypto)
+for more details.
 
 ### <a name="host-network-integration"></a>Host network integration
 
@@ -156,21 +169,32 @@ anywhere.
 Let's say that in our example we want `$HOST2` to have access to the
 application containers. On `$HOST2` we run
 
-    host2# weave expose 10.0.1.102/24
+    host2# weave expose 10.2.1.102/24
 
 choosing an unused IP address in the application subnet. (There is a
 corresponding 'hide' command to revert this step.)
 
 Now
 
-    host2# ping 10.0.1.2
+    host2# ping 10.2.1.2
 
 will work. And, more interestingly,
 
-    host2# ping 10.0.1.1
+    host2# ping 10.2.1.1
 
 will work too, which is talking to a container that resides on `$HOST1`.
 
+Multiple subnet addresses can be exposed or hidden with a single
+invocation:
+
+    host2# weave expose 10.2.1.102/24 10.2.2.102/24
+    host2# weave hide 10.2.1.102/24 10.2.2.102/24
+
+Finally, exposed addresses can be added to weaveDNS by supplying a
+hostname:
+
+    host2# weave expose 10.2.1.102/24 -h exposed.weave.local
+
 ### <a name="service-export"></a>Service export
 
 Services running in containers on a weave network can be made
@@ -184,18 +208,18 @@ a container on `$HOST1`, accessible to the outside world via `$HOST2`.
 First we need to expose the application network to `$HOST2`, as
 explained [above](#host-network-integration), i.e.
 
-    host2# weave expose 10.0.1.102/24
+    host2# weave expose 10.2.1.102/24
 
 Then we add a NAT rule to route from the outside world to the
 destination container service.
 
     host2# iptables -t nat -A PREROUTING -p tcp -i eth0 --dport 2211 \
-           -j DNAT --to-destination 10.0.1.1:4422
+           -j DNAT --to-destination 10.2.1.1:4422
 
 Here we are assuming that the "outside world" is connecting to `$HOST2`
 via 'eth0'. We want TCP traffic to port 2211 on the external IPs to be
 routed to our 'nc' service, which is running on port 4422 in the
-container with IP 10.0.1.1.
+container with IP 10.2.1.1.
 
 With the above in place, we can connect to our 'nc' service from
 anywhere with
@@ -224,23 +248,23 @@ First we need to expose the application network to the host, as
 explained [above](#host-network-integration), this time on `$HOST1`,
 i.e.
 
-    host1# weave expose 10.0.1.101/24
+    host1# weave expose 10.2.1.101/24
 
 Then we add a NAT rule to route from the above IP to the destination
 service.
 
-    host1# iptables -t nat -A PREROUTING -p tcp -d 10.0.1.101 --dport 3322 \
+    host1# iptables -t nat -A PREROUTING -p tcp -d 10.2.1.101 --dport 3322 \
            -j DNAT --to-destination $HOST3:2211
 
 This allows any application container to reach the service by
-connecting to 10.0.1.101:3322. So if `$HOST3` is indeed running a netcat
+connecting to 10.2.1.101:3322. So if `$HOST3` is indeed running a netcat
 service on port 2211, e.g.
 
     host3# nc -lk -p 2211
 
 then we can connect to it from our application container on `$HOST2` with
 
-    root@f76829496120:/# echo 'Hello, world.' | nc 10.0.1.101 3322
+    root@f76829496120:/# echo 'Hello, world.' | nc 10.2.1.101 3322
 
 The same command will work from any application container.
 
@@ -249,7 +273,7 @@ The same command will work from any application container.
 Importing a service provides a degree of indirection that allows late
 and dynamic binding, similar to what can be achieved with a proxy. In
 our example, application containers are unaware that the service they
-are accessing at `10.0.1.101:3322` is in fact residing on
+are accessing at `10.2.1.101:3322` is in fact residing on
 `$HOST3:2211`. We can point application containers at another service
 location by changing the above NAT rule, without having to alter the
 applications themselves.
@@ -271,14 +295,14 @@ In our example above, the netcat service on `$HOST3` is imported into
 weave via `$HOST1`. We can export it on `$HOST2` by first exposing the
 application network with
 
-    host2# weave expose 10.0.1.102/24
+    host2# weave expose 10.2.1.102/24
 
 and then adding a NAT rule which routes traffic from the `$HOST2`
 network (i.e. anything which can connect to `$HOST2`) to the service
 endpoint in the weave network
 
     host2# iptables -t nat -A PREROUTING -p tcp -i eth0 --dport 4433 \
-           -j DNAT --to-destination 10.0.1.101:3322
+           -j DNAT --to-destination 10.2.1.101:3322
 
 Now any host on the same network as `$HOST2` can access the service with
 
@@ -287,7 +311,7 @@ Now any host on the same network as `$HOST2` can access the service with
 Furthermore, as explained in [service-binding](#service-binding), we
 can dynamically alter the service locations without having to touch
 the applications that access them, e.g. we could move the example
-netcat service to `$HOST4:2211` while retaining its 10.0.1.101:3322
+netcat service to `$HOST4:2211` while retaining its 10.2.1.101:3322
 endpoint in the weave network.
 
 ### <a name="multi-cloud-networking"></a>Multi-cloud networking
@@ -297,7 +321,10 @@ data centres. So, for example, one could run an application consisting
 of containers on GCE, EC2 and in local data centres.
 
 To enable this, the network must be configured to permit TCP and UDP
-connections to port 6783 of the docker hosts.
+connections to the weave port of the docker hosts. The weave port
+defaults to 6783. This can be overriden by setting `WEAVE_PORT`, but
+it is highly recommended that all peers in a weave network are given
+the same port setting.
 
 ### <a name="multi-hop-routing"></a>Multi-hop routing
 
@@ -329,7 +356,15 @@ then on one of the existing hosts runs
     host# weave connect $NEW_HOST
 
 Other hosts in the weave network will automatically attempt to
-establish connections to the new host too.
+establish connections to the new host too. Conversely, it is possible
+to instruct a peer to forget a particular host that was specified
+to it via `weave launch` or `weave connect`:
+
+    host# weave forget $DECOMMISSIONED_HOST
+
+This will prevent the peer from trying to reconnect to that host once
+connectivity to it is lost, and thus can be used to administratively
+remove decommissioned peers from the network.
 
 ### <a name="container-mobility"></a>Container mobility
 

site/how-it-works.md

@@ -7,14 +7,15 @@ layout: default
 
 A weave network consists of a number of 'peers' - weave routers
 residing on different hosts. Each peer has a name, which tends to
-remain the same over restarts, and a unique identifier (UID) which is
+remain the same over restarts, a human friendly nickname for use in
+status and logging output and a unique identifier (UID) which is
 different each time it is run.  These are opaque identifiers as far as
 the router is concerned, although the name defaults to a MAC address.
 
 Weave routers establish TCP connections to each other, over which they
-perform a protocol handshake and subsequently exchange topology
-information (see below). These connections are encrypted if so
-configured. Peers also establish UDP "connections", possibly
+perform a protocol handshake and subsequently exchange
+[topology](#topology) information. These connections are encrypted if
+so configured. Peers also establish UDP "connections", possibly
 encrypted, which carry encapsulated network packets. These
 "connections" are duplex and can traverse firewalls.
 
@@ -35,9 +36,11 @@ packet to peers.
 Weave routers learn which peer host a particular MAC address resides
 on. They combine this knowledge with topology information in order to
 make routing decisions and thus avoid forwarding every packet to every
-peer. The topology information captures which peers are connected to
-which other peers; weave can route packets in partially connected
-networks with changing topology.
+peer. Weave can route packets in partially connected networks with
+changing topology. For example, in this network, peer 1 is connected
+directly to 2 and 3, but if 1 needs to send a packet to 4 or 5 it must
+first send it to peer 3:
+![Partially connected Weave Network](images/top-diag1.png "Partially connected Weave Network")
 
 ### <a name="encapsulation"></a>Encapsulation
 
@@ -78,7 +81,7 @@ The name of the sending peer enables the receiving peer to identify
 who sent this UDP packet. This is followed by the meta data and
 payload for one or more captured frames. The router performs batching:
 if it captures several frames very quickly that all need forwarding to
-the same peer, it will fit as many of them as possible into a single
+the same peer, it fits as many of them as possible into a single
 UDP packet.
 
 The meta data for each frame contains the names of the capturing and
@@ -88,13 +91,133 @@ receiving peers to build up their mappings of which client MAC
 addresses are local to which peers. The destination peer name enables
 the receiving peer to identify whether this frame is destined for
 itself or whether it should be forwarded on to some other peer,
-accomodating multi-hop routing. This will work even when the receiving
+accommodating multi-hop routing. This works even when the receiving
 intermediate peer has no knowledge of the destination MAC: only the
 original capturing peer needs to determine the destination peer from
 the MAC. This way weave peers never need to exchange the MAC addresses
 of clients and need not take any special action for ARP traffic and
 MAC discovery.
 
+### <a name="topology"></a>Topology
+
+The topology information captures which peers are connected to which
+other peers. Weave peers communicate their knowledge of the topology
+(and changes to it) to their neighbours, who then pass this
+information on to their own neighbours, and so on, until the entire
+network knows about any change.
+
+Topology is communicated over the TCP links between peers, using a
+Gossip mechanism.  Topology messages are sent by a peer...
+
+- when a connection has been added; the entire topology is sent to the
+  remote peer, and an incremental update, containing information on
+  just the two peers at the ends of the connection, is sent to all
+  neighbours,
+- when a connection has been marked as 'established', indicating that
+  the remote peer can receive UDP traffic from the peer; an update
+  containing just information about the local peer is sent to all
+  neighbours,
+- when a connection has been torn down; an update containing just
+  information about the local peer is sent to all neighbours,
+- periodically, on a timer, in case someone has missed an update.
+
+The receiver of a topology update merges that update with its own
+topology model, adding peers hitherto unknown to it, and updating
+peers for which the update contains a more recent version than known
+to it. If there were any such new/updated peers, then an improved
+update containing them is sent out on all connections.
+
+If the update mentions a peer that the receiver does not know, then
+the entire update is ignored.
+
+#### Message details
+Every gossip message is structured as follows:
+
+    +-----------------------------------+
+    | 1-byte message type - Gossip      |
+    +-----------------------------------+
+    | 4-byte Gossip channel - Topology  |
+    +-----------------------------------+
+    | Peer Name of source               |
+    +-----------------------------------+
+    | Gossip payload (topology update)  |
+    +-----------------------------------+
+
+The topology update payload is laid out like this:
+
+    +-----------------------------------+
+    | Peer 1: Name                      |
+    +-----------------------------------+
+    | Peer 1: NickName                  |
+    +-----------------------------------+
+    | Peer 1: UID                       |
+    +-----------------------------------+
+    | Peer 1: Version number            |
+    +-----------------------------------+
+    | Peer 1: List of connections       |
+    +-----------------------------------+
+    |                ...                |
+    +-----------------------------------+
+    | Peer N: Name                      |
+    +-----------------------------------+
+    | Peer N: NickName                  |
+    +-----------------------------------+
+    | Peer N: UID                       |
+    +-----------------------------------+
+    | Peer N: Version number            |
+    +-----------------------------------+
+    | Peer N: List of connections       |
+    +-----------------------------------+
+
+Each List of connections is encapsulated as a byte buffer, within
+which the structure is:
+
+    +-----------------------------------+
+    | Connection 1: Remote Peer Name    |
+    +-----------------------------------+
+    | Connection 1: Remote IP address   |
+    +-----------------------------------+
+    | Connection 1: Outbound            |
+    +-----------------------------------+
+    | Connection 1: Established         |
+    +-----------------------------------+
+    | Connection 2: Remote Peer Name    |
+    +-----------------------------------+
+    | Connection 2: Remote IP address   |
+    +-----------------------------------+
+    | Connection 2: Outbound            |
+    +-----------------------------------+
+    | Connection 2: Established         |
+    +-----------------------------------+
+    |                ...                |
+    +-----------------------------------+
+    | Connection N: Remote Peer Name    |
+    +-----------------------------------+
+    | Connection N: Remote IP address   |
+    +-----------------------------------+
+    | Connection N: Outbound            |
+    +-----------------------------------+
+    | Connection N: Established         |
+    +-----------------------------------+
+
+#### Removal of peers
+If a peer, after receiving a topology update, sees that another peer
+no longer has any connections within the network, it drops all
+knowledge of that second peer.
+
+#### Out-of-date topology
+The peer-to-peer gossiping of updates is not instantaneous, so it is
+very possible for a node elsewhere in the network to have an
+out-of-date view.
+
+If the destination peer for a packet is still reachable, then
+out-of-date topology can result in it taking a less efficient route.
+
+If the out-of-date topology makes it look as if the destination peer
+is not reachable, then the packet is dropped.  For most protocols
+(e.g. TCP), the transmission will be retried a short time later, by
+which time the topology should have updated.
+
 ### <a name="crypto"></a>Crypto
 
 Weave can be configured to encrypt both the data passing over the TCP
@@ -210,7 +333,7 @@ This is very similar to the [non-crypto encapsulation](#encapsulation).
 All of the frames are encrypted with the same ephemeral session key
 and all must be decrypted by the receiving peer. Frames which are to
 be forwarded on to some further peer will be re-encrypted with the
-relevant ephemeral session keys for the onwards connections. Thus all
+relevant ephemeral session keys for the onward connections. Thus all
 traffic is fully decrypted on every peer it passes through.
 Encryption is again done with the NaCl `secretbox.Seal` function.
 
@@ -274,4 +397,4 @@ further.
 
 ### Further reading
 More details on the inner workings of weave can be found in the
-[architecture documentation](https://github.com/zettio/weave/blob/master/docs/architecture.txt).
+[architecture documentation](https://github.com/weaveworks/weave/blob/master/docs/architecture.txt).

site/index.md

@@ -5,10 +5,12 @@ layout: default
 
 # weave documentation
 
- * [Introduction and example (in README)](https://github.com/zettio/weave#readme)
+ * [Introduction and example (in README)](https://github.com/weaveworks/weave#readme)
  * [Features](features.html)
  * [Troubleshooting](troubleshooting.html)
  * [Building](building.html)
- * [Using with Boot2Docker](boot2docker.html)
  * [How it works](how-it-works.html)
- * [Contact information](https://github.com/zettio/weave#contact-us)
+ * [Contact information](https://github.com/weaveworks/weave#contact-us)
+ * *Installation Notes*
+   * [Using with boot2docker](boot2docker.html)
+   * [Using with systemd](systemd.html)

site/systemd.md

@@ -0,0 +1,70 @@
+---
+title: Using Weave with systemd
+layout: default
+---
+
+Having installed `weave` as per [readme][], you might wish to configure the
+init daemon to start it on boot. Most recent Linux distribution releases are
+shipping with [systemd][]. The information below should provide you with some
+initial guidance on getting a weave service configured on systemd-based OS.
+
+## Weave Service Unit and Configuration
+
+A regular service unit definition for weave is shown below and you should
+normally place it in `/etc/systemd/system/weave.service`.
+
+    [Unit]
+    Description=Weave Network
+    Documentation=http://weaveworks.github.io/weave/
+    Requires=docker.service
+    After=docker.service
+    [Service]
+    EnvironmentFile=-/etc/sysconfig/weave
+    ExecStartPre=/usr/local/bin/weave launch $PEERS
+    ExecStart=/usr/bin/docker logs -f weave
+    SuccessExitStatus=2
+    ExecStop=/usr/local/bin/weave stop
+    [Install]
+    WantedBy=multi-user.target
+
+
+To specify the addresses or names of other weave hosts to join the network
+you can create the `/etc/sysconfig/weave` environment file which would be of
+the following format:
+
+    PEERS="HOST1 HOST2 .. HOSTn"
+
+You can also use the [connect][] command to add participating hosts dynamically.
+
+Additionally, if you want to enable [encryption][] you can specify a password with
+`WEAVE_PASSWORD="MakeSureThisIsSecure"` in the `/etc/sysconfig/weave` environment
+file, and it will get picked up by weave on launch.
+
+You now should be able to launch weave with
+
+    sudo systemctl start weave
+
+To ensure weave launches after reboot, you need run
+
+    sudo systemctl enable weave
+
+For more information on systemd, please refer to the documentation supplied
+by your distribution of Linux.
+
+## SELinux Tweaks
+
+If your OS has SELinux enabled and you wish to run weave as a systemd unit,
+then you should follow the instructions below. These instructions apply to
+CentOS and RHEL as of 7.0. On Fedora 21, there is no need to do this.
+
+Once you have installed `weave` in `/usr/local/bin`, set its execution
+context with the commands shown below. You will need to have the
+`policycoreutils-python` package installed.
+
+    sudo semanage fcontext -a -t unconfined_exec_t -f f /usr/local/bin/weave
+    sudo restorecon /usr/local/bin/weave
+
+[readme]: https://github.com/weaveworks/weave/blob/master/README.md#installation
+[connect]: http://weaveworks.github.io/weave/features.html#dynamic-topologies
+[systemd]: http://www.freedesktop.org/wiki/Software/systemd/
+[encryption]: http://weaveworks.github.io/weave/how-it-works.html#crypto

site/top-diag1.dot

@@ -0,0 +1,9 @@
+graph weave {
+rankdir=LR;
+"peer 1"  -- "peer 2" ;
+"peer 1"  -- "peer 3" ;
+"peer 2"  -- "peer 3" ;
+"peer 3"  -- "peer 4" ;
+"peer 4"  -- "peer 5" ;
+"peer 3"  -- "peer 5" ;
+}

site/troubleshooting.md

@@ -5,10 +5,14 @@ layout: default
 
 ## Troubleshooting
 
-Make sure you are running the latest version - you can download it
-with
+Check what version of weave you are running with
 
-    docker pull zettio/weave:latest
+    weave version
+
+If it is not the latest version, as shown in the list of
+[releases](https://github.com/weaveworks/weave/releases), then it is
+highly recommended that you upgrade by following the
+[installation instructions](https://github.com/weaveworks/weave#installation).
 
 Check the weave container logs with
 
@@ -51,25 +55,26 @@ The command
 
     weave status
 
-reports on the current status of the weave router.
+reports on the current status of the weave router and DNS.
 
 This produces output like:
 
 ````
 weave router 0.7.0
-Our name is 7a:f4:56:87:76:3b
+Encryption off
+Our name is 7a:f4:56:87:76:3b(weave01)
 Sniffing traffic on &{39 65535 ethwe ae:e3:07:9c:8c:d4 up|broadcast|multicast}
 MACs:
-ba:8c:b9:dc:e1:c9 -> 7a:f4:56:87:76:3b (2014-10-23 16:39:19.482338935 +0000 UTC)
-ce:15:34:a9:b5:6d -> 7a:f4:56:87:76:3b (2014-10-23 16:39:28.257103595 +0000 UTC)
-7a:61:a2:49:4b:91 -> 7a:f4:56:87:76:3b (2014-10-23 16:39:27.482970752 +0000 UTC)
-9e:95:0c:54:8e:39 -> 7a:16:dd:5b:83:de (2014-10-23 16:39:28.795601325 +0000 UTC)
-72:5f:a4:60:e5:ce -> 7a:16:dd:5b:83:de (2014-10-23 16:39:29.575995255 +0000 UTC)
+ba:8c:b9:dc:e1:c9 -> 7a:f4:56:87:76:3b(weave01) (2014-10-23 16:39:19.482338935 +0000 UTC)
+ce:15:34:a9:b5:6d -> 7a:f4:56:87:76:3b(weave01) (2014-10-23 16:39:28.257103595 +0000 UTC)
+7a:61:a2:49:4b:91 -> 7a:f4:56:87:76:3b(weave01) (2014-10-23 16:39:27.482970752 +0000 UTC)
+9e:95:0c:54:8e:39 -> 7a:16:dd:5b:83:de(weave02) (2014-10-23 16:39:28.795601325 +0000 UTC)
+72:5f:a4:60:e5:ce -> 7a:16:dd:5b:83:de(weave02) (2014-10-23 16:39:29.575995255 +0000 UTC)
 Peers:
-Peer 7a:16:dd:5b:83:de (v31) (UID 13151318985609435078)
-   -> 7a:f4:56:87:76:3b [37.157.33.76:7195]
-Peer 7a:f4:56:87:76:3b (v1) (UID 6913268221365110570)
-   -> 7a:16:dd:5b:83:de [191.235.147.190:6783]
+7a:16:dd:5b:83:de(weave02) (v31) (UID 13151318985609435078)
+   -> 7a:f4:56:87:76:3b(weave01) [37.157.33.76:7195]
+7a:f4:56:87:76:3b(weave01) (v1) (UID 6913268221365110570)
+   -> 7a:16:dd:5b:83:de(weave02) [191.235.147.190:6783]
 Routes:
 unicast:
 7a:f4:56:87:76:3b -> 00:00:00:00:00:00
@@ -78,14 +83,17 @@ broadcast:
 7a:f4:56:87:76:3b -> [7a:16:dd:5b:83:de]
 7a:16:dd:5b:83:de -> []
 Reconnects:
-192.168.32.1:6783 (next try at 2014-10-23 16:39:50.585932102 +0000 UTC)
+->[192.168.32.1:6783] (dial tcp4 192.168.32.1:6783: connection timed out) next try at 2014-10-23 16:39:50.585932102 +0000 UTC
 ````
 
 The terms used here are explained further at
 [how it works](how-it-works.html).
 
 The 'Our name' line identifies the local weave router as a peer in the
-weave network.
+weave network, displaying the peer name followed by the peer's nickname
+in parenthesis. The nickname defaults to the name of the host on which
+the weave container was launched; if desired it can be overriden by
+supplying the `-nickname` argument to `weave launch`.
 
 The 'Sniffing traffic' line shows details of the virtual ethernet
 interface that weave is using to receive packets on the local
@@ -99,23 +107,24 @@ router last saw some traffic from that MAC. The router forgets
 addresses which are inactive for longer than 10 minutes.
 
 The 'Peers' section lists all peers known to this router, including
-itself.  Each peer is shown with its name, version number (incremented
-on each reconnect) and the UID.  Then each line beginning `->` shows
-another peer that it is connected to, with the IP address and port
-number of the connection. In the above example, the local router has
-connected to its peer using address 191.235.147.190:6783, and its peer
-sees the same connection as coming from 37.157.33.76:7195.
+itself.  Each peer is shown with its name, nickname, version number
+(incremented on each reconnect) and the UID.  Then each line
+beginning `->` shows another peer that it is connected to, with the
+IP address and port number of the connection. In the above example,
+the local router has connected to its peer using address
+191.235.147.190:6783, and its peer sees the same connection as coming
+from 37.157.33.76:7195.
 
 The 'Routes' section summarised the information for deciding how to
 route packets between peers, which is mostly of interest when the
 weave network is not fully connected.  See the
-[architecture documentation](https://raw.githubusercontent.com/zettio/weave/master/docs/architecture.txt)
+[architecture documentation](https://raw.githubusercontent.com/weaveworks/weave/master/docs/architecture.txt)
 for a full explanation.
 
-Finally, 'Reconnects' lists peers that this router is aware of, but is
+The 'Reconnects' section lists peers that this router is aware of, but is
 not currently connected to.  Each line contains some information about
-whether it is attempting to connect or is waiting for a while before
-connecting again.
+what went wrong the last time; whether it is attempting to connect or
+is waiting for a while before connecting again.
 
 ### <a name="list-attached-containers"></a>List attached containers
 
@@ -124,15 +133,24 @@ connecting again.
 Produces a list of all the containers running on this host that are
 connected to the weave network, like this:
 
+    weave:expose 7a:c4:8b:a1:e6:ad 10.2.5.2/24
     b07565b06c53 ae:e3:07:9c:8c:d4
-    5245643870f1 ce:15:34:a9:b5:6d 10.0.5.1/24
-    e32a7d37a93a 7a:61:a2:49:4b:91 10.0.8.3/24
-    caa1d4ee2570 ba:8c:b9:dc:e1:c9 10.0.1.1/24 10.0.2.1/24
+    5245643870f1 ce:15:34:a9:b5:6d 10.2.5.1/24
+    e32a7d37a93a 7a:61:a2:49:4b:91 10.2.8.3/24
+    caa1d4ee2570 ba:8c:b9:dc:e1:c9 10.2.1.1/24 10.2.2.1/24
 
 On each line are the container ID, its MAC address, then the list of
 IP address/routing prefix length ([CIDR
 notation](http://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing))
-assigned on the weave network.
+assigned on the weave network. The special container name `weave:expose`
+displays the weave bridge MAC and any IP addresses added to it via the
+`weave expose` command.
+
+You can also supply a list of container IDs/names to `weave ps`, like this:
+
+    $ sudo weave ps able baker
+    able ce:15:34:a9:b5:6d 10.2.5.1/24
+    baker 7a:61:a2:49:4b:91 10.2.8.3/24
 
 ### Reboots
 
@@ -148,3 +166,17 @@ For a more permanent solution,
 [disable Docker's auto-restart feature](https://docs.docker.com/articles/host_integration/)
 and create appropriate startup scripts to launch weave and run
 application containers from your favourite process manager.
+
+### <a name="snapshots"></a>Snapshot Releases
+
+We sometimes publish snapshot releases, to provide previews of new
+features, assist in validation of bug fixes, etc. One can install the
+latest snapshot release with
+
+    sudo wget -O /usr/local/bin/weave \
+      https://raw.githubusercontent.com/weaveworks/weave/master/weave
+    sudo chmod a+x /usr/local/bin/weave
+    sudo weave setup
+
+Snapshot releases report the script version as "(unreleased version)",
+and the container image versions as git hashes.

test/.gitignore

@@ -1,3 +1,4 @@
 insecure_private_key
 assert.sh
 .vagrant
+.ssh_known_hosts

test/100_cross_hosts_test.sh

@@ -2,23 +2,23 @@
 
 . ./config.sh
 
-C1=10.1.1.4
-C2=10.1.1.7
+C1=10.2.1.4
+C2=10.2.1.7
 
 start_suite "Ping over cross-host weave network"
 
 for HOST in $HOST1 $HOST2; do
-    run_on $HOST sudo $WEAVE stop || true
-    run_on $HOST sudo $WEAVE stop-dns || true
+    weave_on $HOST stop || true
+    weave_on $HOST stop-dns || true
     docker_on $HOST rm -f c1 c2 || true
 done
 
-run_on $HOST1 sudo $WEAVE launch
-run_on $HOST2 sudo $WEAVE launch $HOST1
+weave_on $HOST1 launch
+weave_on $HOST2 launch $HOST1
 
-run_on $HOST2 sudo $WEAVE run $C2/24 -t --name=c2 ubuntu
-run_on $HOST1 sudo $WEAVE run $C1/24 -t --name=c1 ubuntu
-ok=$(docker -H tcp://$HOST1:2375 exec -i c1 sh -c "ping -q -c 4 $C2 >&2 && echo ok")
+weave_on $HOST2 run $C2/24 -t --name=c2 ubuntu
+weave_on $HOST1 run $C1/24 -t --name=c1 ubuntu
+ok=$(docker -H tcp://$HOST1:2375 exec c1 sh -c "ping -q -c 4 $C2 >&2 && echo ok")
 assert "echo $ok" "ok"
 
 end_suite

test/110_encryption_test.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+. ./config.sh
+
+C1=10.2.1.4
+C2=10.2.1.7
+
+start_suite "Ping over encrypted cross-host weave network"
+
+for HOST in $HOST1 $HOST2; do
+    weave_on $HOST stop || true
+    weave_on $HOST stop-dns || true
+    docker_on $HOST rm -f c1 c2 || true
+done
+
+weave_on $HOST1 launch -password notverysecure
+weave_on $HOST2 launch -password notverysecure $HOST1
+
+weave_on $HOST2 run $C2/24 -t --name=c2 ubuntu
+weave_on $HOST1 run $C1/24 -t --name=c1 ubuntu
+ok=$(docker -H tcp://$HOST1:2375 exec c1 sh -c "ping -q -c 4 $C2 >&2 && echo ok")
+assert "echo $ok" "ok"
+
+end_suite

test/200_dns_test.sh

@@ -5,18 +5,21 @@
 C1=10.2.0.78
 C2=10.2.0.34
 
-start_suite "Resolve names"
+start_suite "Resolve names on a single host"
 
-run_on $HOST1 sudo $WEAVE stop || true
-run_on $HOST1 sudo $WEAVE stop-dns || true
-run_on $HOST1 sudo $WEAVE launch-dns 10.0.0.2/8
+weave_on $HOST1 stop || true
+weave_on $HOST1 stop-dns || true
+weave_on $HOST1 launch-dns 10.2.254.1/24
 
 docker_on $HOST1 rm -f c1 c2 || true
 
-run_on $HOST1 sudo $WEAVE run $C2/24 -t --name=c2 -h seetwo.weave.local ubuntu
-run_on $HOST1 sudo $WEAVE run --with-dns $C1/24 -t --name=c1 ubuntu
+weave_on $HOST1 run $C2/24 -t --name=c2 -h seetwo.weave.local ubuntu
+weave_on $HOST1 run --with-dns $C1/24 -t --name=c1 aanand/docker-dnsutils /bin/sh
 
-ok=$(docker -H tcp://$HOST1:2375 exec -i c1 sh -c "ping -q -c4 seetwo.weave.local >&2 && echo ok")
-assert "echo $ok" "ok"
+ok=$(docker -H tcp://$HOST1:2375 exec c1 sh -c "dig +short seetwo.weave.local")
+assert "echo $ok" "$C2"
+
+ok=$(docker -H tcp://$HOST1:2375 exec c1 sh -c "dig +short -x $C2")
+assert "echo $ok" "seetwo.weave.local."
 
 end_suite

test/210_dns_multicast_test.sh

@@ -0,0 +1,34 @@
+#! /bin/bash
+
+. ./config.sh
+
+C1=10.2.3.78
+C2=10.2.3.34
+
+start_suite "Resolve names across hosts"
+
+for host in $HOST1 $HOST2; do
+    weave_on $host stop || true
+    weave_on $host stop-dns || true
+    docker_on $host rm -f c1 c2 || true
+done
+
+weave_on $HOST1 launch
+weave_on $HOST2 launch $HOST1
+
+weave_on $HOST1 launch-dns 10.2.254.1/24 -debug
+weave_on $HOST2 launch-dns 10.2.254.2/24 -debug
+
+weave_on $HOST2 run $C2/24 -t --name=c2 -h seetwo.weave.local ubuntu
+weave_on $HOST1 run --with-dns $C1/24 --name=c1 -t aanand/docker-dnsutils /bin/sh
+
+ok=$(docker -H tcp://$HOST1:2375 exec c1 dig +short seetwo.weave.local)
+assert "echo $ok" "$C2"
+
+ok=$(docker -H tcp://$HOST1:2375 exec c1 dig +short -x $C2)
+assert "echo $ok" "seetwo.weave.local."
+
+ok=$(docker -H tcp://$HOST1:2375 exec c1 dig +short -x 8.8.8.8)
+assert "test -n \"$ok\" && echo pass" "pass"
+
+end_suite

test/220_dns_custom_test.sh

@@ -0,0 +1,25 @@
+#! /bin/bash
+
+. ./config.sh
+
+C1=10.2.56.34
+C2=10.2.54.91
+
+start_suite "Resolve names in custom domain"
+
+weave_on $HOST1 stop || true
+weave_on $HOST1 stop-dns || true
+weave_on $HOST1 launch-dns 10.2.254.1/24 --domain foo.bar.
+
+docker_on $HOST1 rm -f c1 c2 || true
+
+weave_on $HOST1 run $C2/24 -t --name=c2 -h seetwo.foo.bar ubuntu
+weave_on $HOST1 run --with-dns $C1/24 -t --name=c1 aanand/docker-dnsutils /bin/sh
+
+ok=$(docker -H tcp://$HOST1:2375 exec c1 sh -c "dig +short seetwo.foo.bar.")
+assert "echo $ok" "$C2"
+
+ok=$(docker -H tcp://$HOST1:2375 exec c1 sh -c "dig +short -x $C2")
+assert "echo $ok" "seetwo.foo.bar."
+
+end_suite

test/300_fallback_dns_test.sh

@@ -0,0 +1,19 @@
+#! /bin/bash
+
+. ./config.sh
+
+start_suite "Resolve a non-weave address"
+
+weave_on $HOST1 stop-dns || true
+weave_on $HOST1 launch-dns 10.2.254.1/24 -debug
+docker_on $HOST1 rm -f c1 || true
+
+weave_on $HOST1 run --with-dns 10.2.1.5/24 --name=c1 -t aanand/docker-dnsutils /bin/sh
+
+ok=$(docker -H tcp://$HOST1:2375 exec c1 dig +short -t MX weave.works)
+assert "test -n \"$ok\" && echo pass" "pass"
+
+ok=$(docker -H tcp://$HOST1:2375 exec c1 dig +short -x 8.8.8.8)
+assert "test -n \"$ok\" && echo pass" "pass"
+
+end_suite

test/400_weave_local_test.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+. ./config.sh
+
+start_suite "Run weave with --local"
+
+run_on $HOST1 sudo ./weave --local reset
+
+run_on $HOST1 sudo ./weave --local launch
+assert_raises "docker_on $HOST1 ps | grep weave" 0
+
+docker_on $HOST1 rm -f c1 || true
+run_on $HOST1 sudo ./weave --local run 10.2.6.5/24 -ti --name=c1 ubuntu
+ok=$(docker -H tcp://$HOST1:2375 exec c1 sh -c "ifconfig | grep ethwe")
+assert "test -n \"$ok\" && echo pass" "pass"
+
+docker_on $HOST1 rm -f c1 || true
+
+end_suite

test/500_weave_multi_cidr_test.sh

@@ -0,0 +1,94 @@
+#! /bin/bash
+
+. ./config.sh
+
+start_suite "Weave run/start/attach/detach with multiple cidr arguments"
+
+# Cleanup from previous run
+weave_on $HOST1 stop || true
+weave_on $HOST1 stop-dns || true
+docker_on $HOST1 rm -f multicidr || true
+
+weave_on $HOST1 launch
+weave_on $HOST1 launch-dns 10.254.254.254/24
+
+# weave_status_on <host>
+weave_status_on() {
+    HOST=$1; shift
+
+    DOCKER_HOST=tcp://$HOST:2375 $WEAVE status
+}
+
+# weave_ps_on <host>
+weave_ps_on() {
+    HOST=$1; shift
+
+    DOCKER_HOST=tcp://$HOST:2375 $WEAVE ps
+}
+
+# assert_container_cidrs <host> <cid> <cidr> [<cidr> ...]
+assert_container_cidrs() {
+    HOST=$1; shift
+    CID=$1; shift
+    CIDRS=$@
+
+    # Assert container has attached CIDRs
+    assert_raises "weave_ps_on $HOST | grep \"^$CID [^ ]* $CIDRS$\""
+}
+
+# assert_zone_records <host> <cid> <fqdn> <ip> [<ip> ...]
+assert_zone_records() {
+    HOST=$1; shift
+    CID=$1; shift
+    FQDN=$1; shift
+
+    # Assert correct number of records exist
+    assert "weave_status_on $HOST | grep \"^$CID\" | wc -l" $#
+
+    # Assert correct records exist
+    for IP in $@; do
+        assert_raises "weave_status_on $HOST | grep \"^$CID $IP $FQDN$\""
+    done
+}
+
+# assert_bridge_cidrs <host> <dev> <cidr> [<cidr> ...]
+assert_bridge_cidrs() {
+    HOST=$1; shift
+    DEV=$1; shift
+    CIDRS=$@
+
+    BRIDGE_CIDRS=$($SSH $HOST ip addr show dev $DEV | grep -o 'inet .*' | cut -d ' ' -f 2)
+
+    assert "echo $BRIDGE_CIDRS" "$CIDRS"
+}
+
+# Run container with three cidrs
+CID=$(DOCKER_HOST=tcp://$HOST1:2375 $WEAVE run 10.2.1.1/24 10.2.2.1/24 10.2.3.1/24 -t --name multicidr -h multicidr.weave.local ubuntu | cut -b 1-12)
+assert_container_cidrs $HOST1 $CID 10.2.1.1/24 10.2.2.1/24 10.2.3.1/24
+assert_zone_records $HOST1 $CID multicidr.weave.local. 10.2.1.1 10.2.2.1 10.2.3.1
+
+# Remove two of them
+weave_on $HOST1 detach 10.2.1.1/24 10.2.3.1/24 $CID
+assert_container_cidrs $HOST1 $CID 10.2.2.1/24
+assert_zone_records $HOST1 $CID multicidr.weave.local. 10.2.2.1
+
+# Put them both back
+weave_on $HOST1 attach 10.2.1.1/24 10.2.3.1/24 $CID
+assert_container_cidrs $HOST1 $CID 10.2.2.1/24 10.2.1.1/24 10.2.3.1/24
+assert_zone_records $HOST1 $CID multicidr.weave.local. 10.2.2.1 10.2.1.1 10.2.3.1
+
+# Stop the container, restart with three IPs
+docker_on $HOST1 stop $CID
+weave_on $HOST1 start 10.2.1.1/24 10.2.2.1/24 10.2.3.1/24 $CID
+assert_container_cidrs $HOST1 $CID 10.2.1.1/24 10.2.2.1/24 10.2.3.1/24
+assert_zone_records $HOST1 $CID multicidr.weave.local. 10.2.1.1 10.2.2.1 10.2.3.1
+
+# Expose some cidrs
+weave_on $HOST1 expose 10.2.1.2/24 10.2.2.2/24 10.2.3.2/24
+assert_bridge_cidrs $HOST1 weave 10.2.1.2/24 10.2.2.2/24 10.2.3.2/24
+
+# Hide some cidrs
+weave_on $HOST1 hide 10.2.1.2/24 10.2.3.2/24
+assert_bridge_cidrs $HOST1 weave 10.2.2.2/24
+
+end_suite

test/README.md

@@ -33,8 +33,8 @@ to run everything named `*_test.sh`.
 
 By default the tests assume the Vagrant VMs are used.
 
-To use other VMs, set the environment variables <var>HOST1</var> and
-<var>HOST2</var> to the IP addresses of two docker hosts, and set the
+To use other VMs, set the environment variable <var>HOSTS</var> to the
+space-separated list of IP addresses of the docker hosts, and set the
 environment variable <var>SSH</var> to a command that will log into
 either (which may just be `ssh`).
 

test/Vagrantfile

@@ -1,17 +1,24 @@
 VAGRANTFILE_API_VERSION = "2"
 
-host1_ip = "192.168.48.19" # arbitrary private IP
-host2_ip = "192.168.48.15" # arbitrary private IP
+# Necessary because the switch disabling unwanted insecure key
+# replacement in Vagrant 1.7+ is not accepted by earlier versions
+Vagrant.require_version ">= 1.7.0"
 
-def configure_docker(host, ip)
-  pkgs = "lxc-docker ethtool conntrack"
+# these ought to match what is in config.sh
+n_machines = 2
+ip_prefix = "192.168.48"
+ip_suffix_base = 10
+
+def configure_docker(host, hostname, ip)
+  pkgs = "lxc-docker ethtool"
 
   host.vm.box = "phusion/ubuntu-14.04-amd64"
   host.vm.box_url = "https://oss-binaries.phusionpassenger.com/vagrant/boxes/latest/ubuntu-14.04-amd64-vbox.box"
 
+  host.vm.hostname = hostname
   host.vm.network "private_network", ip: ip
 
-  # host.vm.synced_folder "./", "/home/vagrant/src/github.com/zettio/weave"
+  host.vm.synced_folder ".", "/vagrant", disabled: true
 
   host.vm.provision :shell, :inline => "sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 36A1D7869245C8950F966E92D8576A8BA88D21E9"
   host.vm.provision :shell, :inline => "echo deb https://get.docker.io/ubuntu docker main > /etc/apt/sources.list.d/docker.list"
@@ -30,12 +37,24 @@ end
 
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
 
-  config.vm.define "host1" do |host1|
-    configure_docker(host1, host1_ip)
+  config.vm.provider :virtualbox do |vb|
+    vb.customize ["modifyvm", :id, "--natdnshostresolver1", "off"]
+    vb.customize ["modifyvm", :id, "--natdnsproxy1", "off"]
   end
 
-  config.vm.define "host2" do |host2|
-    configure_docker(host2, host2_ip)
+  # Disable Vagrant 1.7 per host insecure key replacement
+  config.ssh.insert_key = false
+
+  (1..n_machines).each do |i|
+    config.vm.define "host#{i}" do |host|
+      ip_suffix = ip_suffix_base + i
+      configure_docker(host, "host#{i}", "#{ip_prefix}.#{ip_suffix}")
+    end
   end
 
 end
+
+begin
+  load 'Vagrantfile.local'
+rescue LoadError
+end

test/config.sh

@@ -2,11 +2,25 @@
 
 set -e
 
+# these ought to match what is in Vagrantfile
+N_MACHINES=${N_MACHINES:-2}
+IP_PREFIX=${IP_PREFIX:-192.168.48}
+IP_SUFFIX_BASE=${IP_SUFFIX_BASE:-10}
+
+if [ -z "$HOSTS" ] ; then
+    for i in $(seq 1 $N_MACHINES); do
+        IP="${IP_PREFIX}.$((${IP_SUFFIX_BASE}+$i))"
+        HOSTS="$HOSTS $IP"
+    done
+fi
+
+# these are used by the tests
+HOST1=$(echo $HOSTS | cut -f 1 -d ' ')
+HOST2=$(echo $HOSTS | cut -f 2 -d ' ')
+
 . ./assert.sh
 
-SSH=${SSH:-ssh -l vagrant -i ./insecure_private_key}
-HOST1=${HOST1:-192.168.48.19}
-HOST2=${HOST2:-192.168.48.15}
+SSH=${SSH:-ssh -l vagrant -i ./insecure_private_key -o UserKnownHostsFile=./.ssh_known_hosts -o CheckHostIP=no -o StrictHostKeyChecking=no}
 
 remote() {
     rem=$1
@@ -36,6 +50,13 @@ docker_on() {
     docker -H tcp://$host:2375 $@
 }
 
+weave_on() {
+    host=$1
+    shift 1
+    greyly echo "Weave on $host: $@"
+    DOCKER_HOST=tcp://$host:2375 $WEAVE $@
+}
+
 start_suite() {
     whitely echo $@
 }
@@ -44,5 +65,5 @@ end_suite() {
     whitely assert_end
 }
 
-WEAVE=./bin/weave
+WEAVE=../weave
 DOCKER_NS=./bin/docker-ns

test/setup.sh

@@ -2,19 +2,21 @@
 
 set -e
 
-echo Fetching assert script
-curl -sS https://raw.githubusercontent.com/lehmannro/assert.sh/master/assert.sh > ./assert.sh
+if ! [ -f "./assert.sh" ]; then
+    echo Fetching assert script
+    curl -sS https://raw.githubusercontent.com/lehmannro/assert.sh/master/assert.sh > ./assert.sh
+fi
 
 . ./config.sh
 
 echo Copying weave images and script to hosts
-for HOST in $HOST1 $HOST2; do
-    docker_on $HOST load -i /var/tmp/weave.tar
-    docker_on $HOST load -i /var/tmp/weavedns.tar
-    run_on $HOST mkdir -p `dirname $WEAVE`
-    cat ../weave         | run_on $HOST sh -c "cat > $WEAVE"
+for HOST in $HOSTS; do
+    docker_on $HOST load -i ../weave.tar
+    docker_on $HOST load -i ../weavedns.tar
+    docker_on $HOST load -i ../weaveexec.tar
+    run_on $HOST mkdir -p bin
     cat ../bin/docker-ns | run_on $HOST sh -c "cat > $DOCKER_NS"
-    run_on $HOST chmod a+x $WEAVE
-    run_on $HOST chmod a+x $DOCKER_NS
+    cat ../weave | run_on $HOST sh -c "cat > ./weave"
+    run_on $HOST chmod a+x $DOCKER_NS ./weave
     run_on $HOST sudo service docker restart
 done

testing/util.go

@@ -0,0 +1,129 @@
+package testing
+
+import (
+	"reflect"
+	"runtime"
+	"testing"
+	"time"
+)
+
+func AssertTrue(t *testing.T, cond bool, desc string) {
+	if !cond {
+		Fatalf(t, "Expected %s", desc)
+	}
+}
+
+func AssertFalse(t *testing.T, cond bool, desc string) {
+	if cond {
+		Fatalf(t, "Unexpected %s", desc)
+	}
+}
+
+func AssertNil(t *testing.T, p interface{}, desc string) {
+	val := reflect.ValueOf(p)
+	if val.IsValid() && !val.IsNil() {
+		Fatalf(t, "Expected nil pointer for %s but got a \"%s\": \"%s\"",
+			desc, reflect.TypeOf(p), val.Interface())
+	}
+}
+
+func AssertNotNil(t *testing.T, p interface{}, desc string) {
+	val := reflect.ValueOf(p)
+	if !val.IsValid() || val.IsNil() {
+		Fatalf(t, "Unexpected nil pointer for %s", desc)
+	}
+}
+
+func AssertNoErr(t *testing.T, err error) {
+	if err != nil {
+		Fatalf(t, "Unexpected error: %s", err)
+	}
+}
+
+func AssertEqualuint64(t *testing.T, got, wanted uint64, desc string) {
+	if got != wanted {
+		Fatalf(t, "Expected %s %d but got %d", desc, wanted, got)
+	}
+}
+
+func AssertEqualInt(t *testing.T, got, wanted int, desc string) {
+	if got != wanted {
+		Fatalf(t, "Expected %s %d but got %d", desc, wanted, got)
+	}
+}
+
+func AssertNotEqualInt(t *testing.T, got, wanted int, desc string) {
+	if got == wanted {
+		Fatalf(t, "Expected %s %d to be different to %d", desc, wanted, got)
+	}
+}
+
+func AssertEqualString(t *testing.T, got, wanted string, desc string) {
+	if got != wanted {
+		Fatalf(t, "Expected %s '%s' but got '%s'", desc, wanted, got)
+	}
+}
+
+func AssertStatus(t *testing.T, got int, wanted int, desc string) {
+	if got != wanted {
+		Fatalf(t, "Expected %s %d but got %d", desc, wanted, got)
+	}
+}
+
+func AssertErrorInterface(t *testing.T, got interface{}, wanted interface{}, desc string) {
+	gotT, wantedT := reflect.TypeOf(got), reflect.TypeOf(wanted).Elem()
+	if !gotT.Implements(wantedT) {
+		Fatalf(t, "Expected %s but got %s (%s)", wantedT.String(), gotT.String(), desc)
+	}
+}
+
+func AssertErrorType(t *testing.T, got interface{}, wanted interface{}, desc string) {
+	gotT, wantedT := reflect.TypeOf(got), reflect.TypeOf(wanted).Elem()
+	if gotT != wantedT {
+		Fatalf(t, "Expected %s but got %s (%s)", wantedT.String(), gotT.String(), desc)
+	}
+}
+
+func AssertType(t *testing.T, got interface{}, wanted interface{}, desc string) {
+	gotT, wantedT := reflect.TypeOf(got), reflect.TypeOf(wanted)
+	if gotT != wantedT {
+		Fatalf(t, "Expected %s but got %s (%s)", wantedT.String(), gotT.String(), desc)
+	}
+}
+
+func AssertEmpty(t *testing.T, array interface{}, desc string) {
+	if reflect.ValueOf(array).Len() != 0 {
+		Fatalf(t, "Expected empty %s but got %s", desc, array)
+	}
+}
+
+// Like testing.Fatalf, but adds the stack trace of the current call
+func Fatalf(t *testing.T, format string, args ...interface{}) {
+	t.Fatalf(format+"\n%s", append(args, StackTrace())...)
+}
+
+func StackTrace() string {
+	return stackTrace(false)
+}
+
+func stackTrace(all bool) string {
+	buf := make([]byte, 1<<20)
+	stacklen := runtime.Stack(buf, all)
+	return string(buf[:stacklen])
+}
+
+// Borrowed from net/http tests:
+// goTimeout runs f, failing t if f takes more than d to complete.
+func RunWithTimeout(t *testing.T, d time.Duration, f func()) {
+	ch := make(chan bool, 2)
+	timer := time.AfterFunc(d, func() {
+		t.Errorf("Timeout expired after %v: stacks:\n%s", d, stackTrace(true))
+		ch <- true
+	})
+	defer timer.Stop()
+	go func() {
+		defer func() { ch <- true }()
+		f()
+	}()
+	<-ch
+}

weave

@@ -2,19 +2,34 @@
 set -e
 
 SCRIPT_VERSION="(unreleased version)"
+if [ "$SCRIPT_VERSION" = "(unreleased version)" ] ; then
+    IMAGE_VERSION=latest
+else
+    IMAGE_VERSION=$SCRIPT_VERSION
+fi
+IMAGE_VERSION=${VERSION:-$IMAGE_VERSION}
+
+# These are potentially used for remote execs, and have defaults, so
+# we introduce them here
+DOCKERHUB_USER=weaveworks
+BASE_EXEC_IMAGE=$DOCKERHUB_USER/weaveexec
+EXEC_IMAGE=$BASE_EXEC_IMAGE:$IMAGE_VERSION
+PROCFS=${PROCFS:-/proc}
 
 usage() {
     echo "Usage:"
-    echo "weave launch     [-password <password> | -passwordfile <path>] <peer> ..."
+    echo "weave setup"
+    echo "weave launch     [-password <password>] [-nickname <nickname>] <peer> ..."
     echo "weave launch-dns <cidr>"
     echo "weave connect    <peer>"
-    echo "weave run        [--with-dns] <cidr> <docker run args> ..."
-    echo "weave start      <cidr> <container_id>"
-    echo "weave attach     <cidr> <container_id>"
-    echo "weave detach     <cidr> <container_id>"
-    echo "weave expose     <cidr>"
-    echo "weave hide       <cidr>"
-    echo "weave ps"
+    echo "weave forget     <peer>"
+    echo "weave run        [--with-dns] <cidr> [<cidr> ...] <docker run args> ..."
+    echo "weave start      <cidr> [<cidr> ...] <container_id>"
+    echo "weave attach     <cidr> [<cidr> ...] <container_id>"
+    echo "weave detach     <cidr> [<cidr> ...] <container_id>"
+    echo "weave expose     <cidr> [<cidr> ...] [-h <hostname>]"
+    echo "weave hide       <cidr> [<cidr> ...]"
+    echo "weave ps         [<container_id> ...]"
     echo "weave status"
     echo "weave version"
     echo "weave stop"
@@ -26,39 +41,31 @@ usage() {
     exit 1
 }
 
-[ $(id -u) = 0 ] || {
-    echo "weave must be run as 'root'" >&2
-    exit 1
+exec_remote() {
+    docker run --rm \
+        -v /var/run/docker.sock:/var/run/docker.sock \
+        -v "$PROCFS":/hostproc \
+        -e VERSION \
+        -e WEAVE_DOCKER_ARGS \
+        -e WEAVEDNS_DOCKER_ARGS \
+        -e WEAVE_PASSWORD \
+        -e WEAVE_PORT \
+        -e WEAVE_CONTAINER_NAME \
+        -e DOCKER_BRIDGE \
+        -e PROCFS=/hostproc --privileged --net=host \
+        $EXEC_IMAGE --local "$@"
 }
 
-[ $# -gt 0 ] || usage
+check_docker_version() {
+    if ! DOCKER_VERSION=$(docker -v | sed -n -e 's|^Docker version \([0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*\).*|\1|p') || [ -z "$DOCKER_VERSION" ] ; then
+        echo "ERROR: Unable to parse docker version" >&2
+        exit 1
+    fi
 
-IMAGE=zettio/weave
-DNS_IMAGE=zettio/weavedns
-CONTAINER_NAME=weave
-DNS_CONTAINER_NAME=weavedns
-BRIDGE=weave
-CONTAINER_IFNAME=ethwe
-MTU=65535
-PORT=6783
-HTTP_PORT=6784
-DNS_HTTP_PORT=6785
-DOCKER_BRIDGE=${DOCKER_BRIDGE:-docker0}
-
-COMMAND=$1
-
-shift 1
-
-# utility function to check whether a command can be executed by the shell
-# see http://stackoverflow.com/questions/592620/how-to-check-if-a-program-exists-from-a-bash-script
-command_exists () {
-    command -v $1 >/dev/null 2>&1
-}
-
-command_exists_or_warn () {
-    if ! command_exists $1 ; then
-        echo "Cannot find $1; please install it. Continuing without it." >&2
-        return 1
+    # guard against https://github.com/docker/docker/issues/8632
+    if [ "$DOCKER_VERSION" = "1.3.0" ] ; then
+        echo "You are running Docker version $DOCKER_VERSION, which contains a bug that prevents weave from working properly. Please upgrade." >&2
+        exit 1
     fi
 }
 
@@ -70,11 +77,73 @@ is_cidr() {
 validate_cidr() {
     if ! is_cidr "$1" ; then
         echo "Invalid CIDR: $1" >&2
-        echo "CIDR must of be of form <ip_address>/<routing_prefix_length>, e.g. 10.0.1.1/24" >&2
+        echo "CIDR must of be of form <ip_address>/<routing_prefix_length>, e.g. 10.2.1.1/24" >&2
         exit 1
     fi
 }
 
+collect_cidr_args() {
+    CIDR_ARGS=""
+    CIDR_COUNT=0
+    while is_cidr "$1"; do
+        CIDR_ARGS="$CIDR_ARGS $1"
+        CIDR_COUNT=$((CIDR_COUNT + 1))
+        shift 1
+    done
+    if [ $CIDR_COUNT -eq 0 ]; then
+        [ $# -gt 0 ] && validate_cidr "$1" || usage
+    fi
+}
+
+# "run" is a special case because we want to use docker directly,
+# rather than the docker in $EXEC_IMAGE remotely. That's because we
+# are passing arbitrary arguments on to docker run, and we can't rely
+# on our baked-in docker to support those arguments.
+if [ "$1" = "run" ]; then
+    shift 1
+    check_docker_version
+    if [ "$1" = "--with-dns" ]; then
+        shift 1
+        DNS_ARGS=$(exec_remote dns-args "$@")
+    fi
+    collect_cidr_args "$@"
+    shift $CIDR_COUNT
+    CONTAINER=$(docker run $DNS_ARGS -d "$@")
+    exec_remote attach $CIDR_ARGS $CONTAINER
+    echo $CONTAINER
+    exit 0
+elif [ "$1" != "--local" ]; then
+    exec_remote "$@"
+    exit $?
+fi
+
+shift 1
+
+BASE_IMAGE=$DOCKERHUB_USER/weave
+BASE_DNS_IMAGE=$DOCKERHUB_USER/weavedns
+IMAGE=$BASE_IMAGE:$IMAGE_VERSION
+DNS_IMAGE=$BASE_DNS_IMAGE:$IMAGE_VERSION
+
+DOCKER_BRIDGE=${DOCKER_BRIDGE:-docker0}
+CONTAINER_NAME=${WEAVE_CONTAINER_NAME:-weave}
+DNS_CONTAINER_NAME=weavedns
+BRIDGE=weave
+CONTAINER_IFNAME=ethwe
+MTU=65535
+PORT=${WEAVE_PORT:-6783}
+HTTP_PORT=6784
+DNS_HTTP_PORT=6785
+
+######################################################################
+# general helpers; independent of docker and weave
+######################################################################
+
+# utility function to check whether a command can be executed by the shell
+# see http://stackoverflow.com/questions/592620/how-to-check-if-a-program-exists-from-a-bash-script
+command_exists () {
+    command -v $1 >/dev/null 2>&1
+}
+
 run_iptables() {
     # -w is recent addition to iptables
     if [ -z "$CHECKED_IPTABLES_W" ] ; then
@@ -107,10 +176,49 @@ delete_iptables_rule() {
     fi
 }
 
+# Configure the ARP cache parameters for the given interface.  This
+# makes containers react more quickly to a change in the MAC address
+# associated with an IP address.
+configure_arp_cache() {
+    $2 sh -c "echo 5 >/proc/sys/net/ipv4/neigh/$1/base_reachable_time &&
+              echo 2 >/proc/sys/net/ipv4/neigh/$1/delay_first_probe_time &&
+              echo 1 >/proc/sys/net/ipv4/neigh/$1/ucast_solicit"
+}
+
+# Send out an unsolicated ARP response to update ARP cache entries
+# accross the weave network.  We do this in addition to
+# configure_arp_cache because a) with those ARP cache settings it
+# still takes a few seconds to correct a stale ARP mapping, and b)
+# there is a kernel bug that means that the base_reachable_time
+# setting is not promptly obeyed
+# (<https://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/commit/?id=4bf6980dd0328530783fd657c776e3719b421d30>>).
+arp_update() {
+    # It's not the end of the world if this doesn't run - we configure
+    # ARP caches so that stale entries will be noticed quickly.
+    if command_exists arping ; then
+        $3 arping -U -q -I $1 -c 1 ${2%/*}
+    fi
+}
+
+# Generate a random MAC value
+random_mac() {
+    # In the first byte of the MAC, the 'multicast' bit should be
+    # clear and 'locally administered' bit should be set.  All other
+    # bits should be random.
+    od -txC -An -N6 /dev/urandom | sed 's|^ ||;s| |:|;s|[0-3]:|2:|;s|[4-7]:|6:|;s|[89ab]:|a:|;s|[c-f]:|e:|;s| |:|g'
+}
+
+######################################################################
+# weave and docker specific helpers
+######################################################################
+
 create_bridge() {
     [ ! -d /sys/class/net/$BRIDGE ] && {
         ip link add name $BRIDGE type bridge
-        ip link set dev $BRIDGE address 7a:$(od -txC -An -N5 /dev/urandom | tr \  : | tail -c+2)
+        # Set a random MAC address on the bridge.  Current Linux
+        # kernels already do this when creating a bridge, but there
+        # are rumours it was not always so.
+        ip link set dev $BRIDGE address $(random_mac)
         # Attempting to set the bridge MTU to a high value directly
         # fails. Bridges take the lowest MTU of their interfaces. So
         # instead we create a temporary interface with the desired
@@ -118,8 +226,6 @@ create_bridge() {
         ip link add name v${CONTAINER_IFNAME}du mtu $MTU type dummy
         ip link set dev v${CONTAINER_IFNAME}du master $BRIDGE
         ip link del dev v${CONTAINER_IFNAME}du
-        # disable offloading
-        if command_exists_or_warn ethtool; then ethtool -K $BRIDGE tx off >/dev/null; fi
         # Work around the situation where there are no rules allowing traffic
         # across our bridge. E.g. ufw
         add_iptables_rule filter FORWARD -i $BRIDGE -o $BRIDGE -j ACCEPT
@@ -127,7 +233,15 @@ create_bridge() {
         run_iptables -t nat -N WEAVE >/dev/null 2>&1 || true
         add_iptables_rule nat POSTROUTING -j WEAVE
     }
+    if [ ! "$1" = "--without-ethtool" ] ; then
+        ethtool -K $BRIDGE tx off >/dev/null
+    fi
+
     ip link set dev $BRIDGE up
+
+    # Configure the ARP cache parameters on the bridge interface for
+    # the sake of 'weave expose'
+    configure_arp_cache $BRIDGE
 }
 
 destroy_bridge() {
@@ -153,7 +267,7 @@ docker_bridge_ip() {
 # If an error is caused by container dying, swallow output from error.
 with_container_netns () {
     CONTAINER="$1"
-    CONTAINER_PID=$(docker inspect --format='{{ .State.Pid }}' $CONTAINER)
+    CONTAINER_PID=$(docker inspect --format='{{.State.Pid}}' $CONTAINER)
 
     if [ "$CONTAINER_PID" = 0 ] ; then
         echo "Container $CONTAINER not running." >&2
@@ -168,7 +282,7 @@ with_container_netns () {
     NETNS=$CONTAINER_PID
     [ ! -d /var/run/netns ] && mkdir -p /var/run/netns
     rm -f /var/run/netns/$NETNS
-    ln -s /proc/$CONTAINER_PID/ns/net /var/run/netns/$NETNS
+    ln -s $PROCFS/$CONTAINER_PID/ns/net /var/run/netns/$NETNS
 
     LOCAL_IFNAME="v${CONTAINER_IFNAME}pl${CONTAINER_PID}"
     GUEST_IFNAME="v${CONTAINER_IFNAME}pg${CONTAINER_PID}"
@@ -181,7 +295,7 @@ with_container_netns () {
     shift 1
     if ! "$@" >$IP_TMPOUT 2>$IP_TMPERR ; then
         STATUS=1
-        if [ "$(docker inspect --format='{{ .State.Pid }}' $CONTAINER)" != "$CONTAINER_PID" ] ; then
+        if ! kill -0 "$CONTAINER_PID" 2>/dev/null ; then
             echo "Container $CONTAINER died" >&2
         else
             echo "Failure during network configuration for container $CONTAINER:" >&2
@@ -196,7 +310,7 @@ with_container_netns () {
 }
 
 connect_container_to_bridge() {
-    if [ "$(readlink /proc/$CONTAINER_PID/ns/net)" = "$(readlink /proc/$$/ns/net)" ] ; then
+    if [ -h "$PROCFS/$CONTAINER_PID/ns/net" -a -h "$PROCFS/$$/ns/net" -a "$(readlink $PROCFS/$CONTAINER_PID/ns/net)" = "$(readlink $PROCFS/$$/ns/net)" ] ; then
         echo "Container is running in the host network namespace, and therefore cannot be" >&2
         echo "connected to weave. Perhaps the container was started with --net=host." >&2
         return 1
@@ -205,48 +319,78 @@ connect_container_to_bridge() {
         return 1
     fi
 
-    if { if command_exists_or_warn ethtool; then ethtool -K $GUEST_IFNAME tx off >/dev/null; fi } &&
-        ip link set $LOCAL_IFNAME master $BRIDGE &&
-        ip link set $LOCAL_IFNAME up &&
-        ip link set $GUEST_IFNAME netns $NETNS ; then
-        ip netns exec $NETNS ip link set $GUEST_IFNAME name $CONTAINER_IFNAME
-    else
+    if ! ethtool -K $GUEST_IFNAME tx off >/dev/null ||
+        ! ip link set $LOCAL_IFNAME master $BRIDGE ||
+        ! ip link set $LOCAL_IFNAME up ||
+        ! ip link set $GUEST_IFNAME netns $NETNS ; then
         # failed before we assigned the veth to the container's
-        # namespace - delete it
+        # namespace
         ip link del $LOCAL_IFNAME type veth || true
         return 1
     fi
+
+    if ! ip netns exec $NETNS ip link set $GUEST_IFNAME name $CONTAINER_IFNAME ||
+       ! configure_arp_cache $CONTAINER_IFNAME "ip netns exec $NETNS" ; then
+        return 1
+    fi
 }
 
+ask_version() {
+    if ! DOCKERIMAGE=$(docker inspect --format='{{.Image}}' $1 2>/dev/null) ; then
+            if ! DOCKERIMAGE=$(docker inspect --format='{{.Id}}' $2 2>/dev/null) ; then
+                echo "Unable to find $2 image." >&2
+            fi
+    fi
+    [ -n "$DOCKERIMAGE" ] && docker run --rm $DOCKERIMAGE --version
+}
+
+######################################################################
+# functions invoked through with_container_netns
+######################################################################
+
 launch() {
     if ! ip netns exec $NETNS ip link show eth0 >/dev/null ; then
         echo "Perhaps you are running the docker daemon with container networking disabled (-b=none)." >&2
         return 1
     fi
     connect_container_to_bridge &&
-        { if command_exists_or_warn ethtool ; then ip netns exec $NETNS ethtool -K eth0 tx off >/dev/null; fi } &&
+        ip netns exec $NETNS ethtool -K eth0 tx off >/dev/null &&
         ip netns exec $NETNS ip link set $CONTAINER_IFNAME up
 }
 
+test_launch() {
+    ip netns exec $NETNS ethtool -K eth0 tx off >/dev/null
+}
+
 attach() {
     if ip netns exec $NETNS ip link show $CONTAINER_IFNAME >/dev/null 2>&1 ; then
-        # container already has the expected network interface, so assume we set it up already;
-        # just add the IP address.
-        if ip netns exec $NETNS ip addr show dev $CONTAINER_IFNAME | grep -F $1 >/dev/null ; then
-            # address was there already
-            return 0
-        fi
+        for ADDR; do
+            # container already has the expected network interface, so assume we set it up already;
+            # just add the IP address.
+            if ip netns exec $NETNS ip addr show dev $CONTAINER_IFNAME | grep -F $ADDR >/dev/null ; then
+                # address was there already
+                continue
+            fi
 
-        if ! ip netns exec $NETNS ip addr add $1 dev $CONTAINER_IFNAME ; then
-            return 1
-        fi
+            if ! ip netns exec $NETNS ip addr add $ADDR dev $CONTAINER_IFNAME ; then
+                return 1
+            fi
+        done
 
         return 0
     fi
 
-    if ! connect_container_to_bridge ||
-        ! ip netns exec $NETNS ip addr add $1 dev $CONTAINER_IFNAME ||
-        ! ip netns exec $NETNS ip link set $CONTAINER_IFNAME up ; then
+    if ! connect_container_to_bridge; then
+        return 1
+    fi
+
+    for ADDR; do
+        if ! ip netns exec $NETNS ip addr add $ADDR dev $CONTAINER_IFNAME; then
+            return 1
+        fi
+    done
+
+    if ! ip netns exec $NETNS ip link set $CONTAINER_IFNAME up ; then
         return 1
     fi
 
@@ -254,17 +398,21 @@ attach() {
     if ! ip netns exec $NETNS ip route show | grep '^224\.0\.0\.0/4' >/dev/null ; then
         ip netns exec $NETNS ip route add 224.0.0.0/4 dev $CONTAINER_IFNAME
     fi
+
+    arp_update $CONTAINER_IFNAME $1 "ip netns exec $NETNS"
 }
 
 detach() {
-    if ! ip netns exec $NETNS ip addr show dev $CONTAINER_IFNAME | grep -F $1 >/dev/null ; then
-        # address is not there, leave the device alone
-        return 0
-    fi
+    for ADDR; do
+        if ! ip netns exec $NETNS ip addr show dev $CONTAINER_IFNAME | grep -F $ADDR >/dev/null ; then
+            # address is not there, leave the device alone
+            continue
+        fi
 
-    if ! ip netns exec $NETNS ip addr del $1 dev $CONTAINER_IFNAME ; then
-        return 1
-    fi
+        if ! ip netns exec $NETNS ip addr del $ADDR dev $CONTAINER_IFNAME ; then
+            return 1
+        fi
+    done
 
     if [ -n "$(ip netns exec $NETNS ip -f inet addr show dev $CONTAINER_IFNAME)" ] ; then
         # other addresses are left, leave the device alone
@@ -275,9 +423,54 @@ detach() {
     ip netns exec $NETNS ip link del $CONTAINER_IFNAME type veth
 }
 
+container_weave_addrs() {
+    ip netns exec $NETNS ip addr show dev $CONTAINER_IFNAME
+}
+
+######################################################################
+# functions for interacting with containers
+######################################################################
+
+# Check that a container named $1 with image $2 is not running
+check_not_running() {
+    case $(docker inspect --format='{{.State.Running}} {{.Config.Image}}' $1 2>/dev/null) in
+        "true $2")
+            echo "$1 is already running." >&2
+            exit 1
+            ;;
+        "true $2:"*)
+            echo "$1 is already running." >&2
+            exit 1
+            ;;
+        "false $2")
+            docker rm $1 >/dev/null
+            ;;
+        "false $2:"*)
+            docker rm $1 >/dev/null
+            ;;
+        true*)
+            echo "Found another running container named '$1'. Aborting." >&2
+            exit 1
+            ;;
+        false*)
+            echo "Found another container named '$1'. Aborting." >&2
+            exit 1
+            ;;
+    esac
+}
+
+http_call_ip() {
+    ip="$1"
+    port="$2"
+    http_verb="$3"
+    url="$4"
+    shift 4
+    curl --connect-timeout 3 -s -X $http_verb "$@" http://$ip:$port$url
+}
+
 # Call url $4 with http verb $3 on container $1 at port $2
 http_call() {
-    if ! status=$(docker inspect --format='{{ .State.Running }} {{ .NetworkSettings.IPAddress }}' $1 2>/dev/null); then
+    if ! status=$(docker inspect --format='{{.State.Running}} {{.NetworkSettings.IPAddress}}' $1 2>/dev/null); then
         echo "$1 container is not present; have you launched it?" >&2
         return 1
     fi
@@ -298,88 +491,97 @@ http_call() {
     http_call_ip $ip "$@"
 }
 
-http_call_ip() {
-    ip="$1"
-    port="$2"
-    http_verb="$3"
-    url="$4"
-    shift 4
-    if command_exists curl ; then
-        curl -s -X $http_verb "$@" http://$ip:$port$url
-    else
-        echo "Cannot find curl, which is required for this operation." >&2
+# Wait until container $1 on port $2 is alive enough to respond to its status call
+wait_for_status() {
+    TARGET_IP=$(docker inspect --format='{{.NetworkSettings.IPAddress}}' $1)
+    WAIT_TIME=1
+    while ! http_call_ip $TARGET_IP $2 GET /status >/dev/null && [ $WAIT_TIME -lt 4 ]; do
+        if ! status=$(docker inspect --format='{{.State.Running}}' $1 2>/dev/null) || [ "$status" != "true" ] ; then
+            echo "Container $1 died" >&2
+            return 1
+        fi
+        sleep $WAIT_TIME
+        WAIT_TIME=$((WAIT_TIME+1))
+    done
+    if [ $WAIT_TIME = 4 ]; then
+        echo "Timed out waiting for $1 container to start." >&2
+        return 1
+    fi
+    return 0
+}
+
+######################################################################
+# weaveDNS helpers
+######################################################################
+
+dns_args() {
+    if [ \( "$DOCKER_VERSION_MAJOR" -lt 1 \) -o \
+        \( "$DOCKER_VERSION_MAJOR" -eq 1 -a \
+        "$DOCKER_VERSION_MINOR" -lt 2 \) ] ; then
+        echo "ERROR: The '--with-dns' option requires Docker 1.2.0 or later; you are running $DOCKER_VERSION" >&2
         exit 1
     fi
+    docker_bridge_ip
+    DNS_ARG="--dns $DOCKER_BRIDGE_IP"
+    DNS_SEARCH_ARG="--dns-search=."
+    for arg; do
+        case $arg in
+            --dns-search=*)
+                DNS_SEARCH_ARG=""
+                ;;
+            *)
+                ;;
+        esac;
+    done
+    DNS_ARGS="$DNS_ARG $DNS_SEARCH_ARG"
 }
 
 # Perform operation $1 on container ID $2 to local DNS database at address $3
 # This function is only called where we know $2 is a valid container name
 tell_dns() {
-    if ! status=$(docker inspect --format='{{ .State.Running }}' $DNS_CONTAINER_NAME 2>/dev/null) || [ "$status" != "true" ] ; then
+    METHOD="$1"
+    CONTAINER_ID="$2"
+    shift 2
+
+    if ! status=$(docker inspect --format='{{.State.Running}}' $DNS_CONTAINER_NAME 2>/dev/null) || [ "$status" != "true" ] ; then
         # weavedns not running - silently return
         return
     fi
     # get the long form of the container ID
-    CONTAINER=$(docker inspect --format='{{ .Id }}' $2 2>/dev/null)
-    # extract IP address and routing prefix from CIDR
-    WEAVE_ADDR_IP=$(echo $3 | sed -e 's/\([^/]*\)\/\(.*\)/\1/')
-    MORE_ARGS=$(docker inspect --format='--data-urlencode fqdn={{ .Config.Hostname }}.{{ .Config.Domainname }}.' $CONTAINER 2>/dev/null) && true
-    http_call $DNS_CONTAINER_NAME $DNS_HTTP_PORT $1 /name/$CONTAINER/$WEAVE_ADDR_IP $MORE_ARGS || true
+    CONTAINER=$(docker inspect --format='{{.Id}}' $CONTAINER_ID 2>/dev/null)
+    MORE_ARGS=$(docker inspect --format='--data-urlencode fqdn={{.Config.Hostname}}.{{.Config.Domainname}}.' $CONTAINER 2>/dev/null) && true
+    for ADDR; do
+        http_call $DNS_CONTAINER_NAME $DNS_HTTP_PORT $METHOD /name/$CONTAINER/${ADDR%/*} $MORE_ARGS || true
+    done
 }
 
 # Tell the newly-started weaveDNS about existing weave IPs
 populate_dns() {
-    DNS_IP=$(docker inspect --format='{{ .NetworkSettings.IPAddress }}' $DNS_CONTAINER_NAME)
-    WAIT_TIME=1
-    while ! http_call_ip $DNS_IP $DNS_HTTP_PORT GET /status >/dev/null && [ $WAIT_TIME -lt 4 ]; do
-        sleep $WAIT_TIME
-        WAIT_TIME=$((WAIT_TIME+1))
-    done
-    if [ $WAIT_TIME = 4 ]; then
-        echo "Timed out waiting for weaveDNS container to start." >&2
+    if ! wait_for_status $DNS_CONTAINER_NAME $DNS_HTTP_PORT ; then
         echo "If running, it will not be pre-populated." >&2
+        return 0
     fi
-
     for CONTAINER in $(docker ps -q --no-trunc); do
-        MORE_ARGS=$(docker inspect --format='--data-urlencode fqdn={{ .Config.Hostname }}.{{ .Config.Domainname }}.' $CONTAINER 2>/dev/null) && true
-        if CONTAINER_ADDRS=$(with_container_netns $CONTAINER container_weave_addrs 2>&1) ; then
-            CONTAINER_IPS=$(echo "$CONTAINER_ADDRS" | grep -o 'inet .*' | sed -e 's/inet \([^/]*\)\/\(.*\)/\1/')
+        MORE_ARGS=$(docker inspect --format='--data-urlencode fqdn={{.Config.Hostname}}.{{.Config.Domainname}}.' $CONTAINER 2>/dev/null) && true
+        if CONTAINER_IPS=$(with_container_netns $CONTAINER container_weave_addrs 2>&1 | sed -n -e 's/inet \([^/]*\)\/\(.*\)/\1/p') ; then
             for IP in $CONTAINER_IPS; do
-                http_call_ip $DNS_IP $DNS_HTTP_PORT PUT /name/$CONTAINER/$IP $MORE_ARGS
+                http_call_ip $TARGET_IP $DNS_HTTP_PORT PUT /name/$CONTAINER/$IP $MORE_ARGS
             done
         fi
     done
 }
 
-# Check that a container named $1 with image $2 is not running
-check_not_running() {
-    case $(docker inspect --format='{{ .State.Running }} {{ .Config.Image }}' $1 2>/dev/null) in
-        "true $2")
-            echo "$1 is already running." >&2
-            exit 1
-            ;;
-        "false $2")
-            docker rm $1 >/dev/null
-            ;;
-        true*)
-            echo "Found another running container named '$1'. Aborting." >&2
-            exit 1
-            ;;
-        false*)
-            echo "Found another container named '$1'. Aborting." >&2
-            exit 1
-            ;;
-    esac
+##########################################################################################
+
+[ $(id -u) = 0 ] || {
+    echo "weave must be run as 'root' when run locally" >&2
+    exit 1
 }
 
-container_weave_addrs() {
-    ip netns exec $NETNS ip addr show dev $CONTAINER_IFNAME
-}
-
-uname -s -r | sed -n 's|^\([^ ]*\) \([0-9]\+\)\.\([0-9]\+\).*|\1 \2 \3|p' | {
+uname -s -r | sed -n -e 's|^\([^ ]*\) \([0-9][0-9]*\)\.\([0-9][0-9]*\).*|\1 \2 \3|p' | {
     if ! read sys maj min ; then
-      echo "ERROR: Unable to parse operating system version $(uname -s -r)" >&2
-      exit 1
+        echo "ERROR: Unable to parse operating system version $(uname -s -r)" >&2
+        exit 1
     fi
 
     if [ "$sys" != 'Linux' ] ; then
@@ -392,15 +594,6 @@ uname -s -r | sed -n 's|^\([^ ]*\) \([0-9]\+\)\.\([0-9]\+\).*|\1 \2 \3|p' | {
     fi
 }
 
-ask_version() {
-    if ! DOCKERIMAGE=$(docker inspect --format='{{ .Image }}' $1 2>/dev/null) ; then
-            if ! DOCKERIMAGE=$(docker inspect --format='{{ .Id }}' $2 2>/dev/null) ; then
-                echo "Unable to find $2 image." >&2
-            fi
-    fi
-    [ -n "$DOCKERIMAGE" ] && docker run --rm $DOCKERIMAGE --version
-}
-
 if ! command_exists ip ; then
     echo "ERROR: ip utility is missing. Please install it." >&2
     exit 1
@@ -412,24 +605,28 @@ if ! ip netns list >/dev/null 2>&1 ; then
     exit 1
 fi
 
-if ! DOCKER_VERSION=$(docker -v | sed -n 's|^Docker version \([0-9]\+\.[0-9]\+.[0-9]\+\).*|\1|p') || [ -z "$DOCKER_VERSION" ] ; then
-    echo "ERROR: Unable to parse docker version" >&2
-    exit 1
-fi
-
-# guard against https://github.com/docker/docker/issues/8632
-if [ "$DOCKER_VERSION" = "1.3.0" ] ; then
-    echo "You are running Docker version $DOCKER_VERSION, which contains a bug that prevents weave from working properly. Please upgrade." >&2
-    exit 1
-fi
-
+check_docker_version
 DOCKER_VERSION_MAJOR=$(echo "$DOCKER_VERSION" | cut -d. -f 1)
 DOCKER_VERSION_MINOR=$(echo "$DOCKER_VERSION" | cut -d. -f 2)
 DOCKER_VERSION_PATCH=$(echo "$DOCKER_VERSION" | cut -d. -f 3)
 
+[ $# -gt 0 ] || usage
+COMMAND=$1
+shift 1
+
 case "$COMMAND" in
+    setup)
+        for img in $IMAGE $DNS_IMAGE $EXEC_IMAGE ; do
+            docker pull $img
+        done
+        ;;
+    # intentionally undocumented since it assumes knowledge of weave
+    # internals
+    create-bridge)
+        create_bridge --without-ethtool
+        ;;
     launch)
-        check_not_running $CONTAINER_NAME $IMAGE
+        check_not_running $CONTAINER_NAME $BASE_IMAGE
         create_bridge
         # We set the router name to the bridge mac since that is
         # stable across re-creations of the containers.
@@ -443,35 +640,72 @@ case "$COMMAND" in
             echo "WARNING: $1 parameter ignored; 'weave launch' no longer takes a CIDR as the first parameter" >&2
             shift 1
         fi
-        if [ "$1" = "-password" ] ; then
-            [ $# -gt 1 ] || usage
-            PASSWORD=$2
-            shift 2
-        elif [ "$1" = "-passwordfile" ] ; then
-            [ $# -gt 1 ] || usage
-            PASSWORDFILE=$2
-            shift 2
-        fi
+        NICKNAME=$(hostname)
+        CONTAINER_PORT=$PORT
+        while true; do
+            case "$1" in
+                -password)
+                    [ $# -gt 1 ] || usage
+                    WEAVE_PASSWORD="$2"
+                    export WEAVE_PASSWORD
+                    shift 2
+                    ;;
+                -nickname)
+                    [ $# -gt 1 ] || usage
+                    NICKNAME="$2"
+                    shift 2
+                    ;;
+                -port)
+                    [ $# -gt 1 ] || usage
+                    CONTAINER_PORT="$2"
+                    shift 2
+                    ;;
+                *)
+                    break
+                    ;;
+            esac
+        done
         # Set WEAVE_DOCKER_ARGS in the environment in order to supply
         # additional parameters, such as resource limits, to docker
         # when launching the weave container.
         CONTAINER=$(docker run --privileged -d --name=$CONTAINER_NAME \
-            -p $PORT:$PORT/tcp -p $PORT:$PORT/udp \
-            ${PASSWORD:+-e WEAVE_PASSWORD="$PASSWORD"} ${PASSWORDFILE:+--env-file="$PASSWORDFILE"} \
-            $WEAVE_DOCKER_ARGS $IMAGE -name $MACADDR -iface $CONTAINER_IFNAME "$@")
+            -p $PORT:$CONTAINER_PORT/tcp -p $PORT:$CONTAINER_PORT/udp -e WEAVE_PASSWORD \
+            $WEAVE_DOCKER_ARGS $IMAGE -iface $CONTAINER_IFNAME -port $CONTAINER_PORT -name $MACADDR -nickname "$NICKNAME" "$@")
         with_container_netns $CONTAINER launch >/dev/null
+        wait_for_status $CONTAINER_NAME $HTTP_PORT
+        echo $CONTAINER
+        ;;
+    test-launch)
+        # For testing purposes. Allows launching of several weave
+        # routers on the same machine, by invoking this multiple times
+        # with different WEAVE_CONTAINER_NAMEs
+        #
+        # Note that the routers are not connected to the $BRIDGE and
+        # do not perform any capture/injection. So this is mainly
+        # intended for testing of administrative and control plane
+        # functionality, rather than routing.
+        check_not_running $CONTAINER_NAME $BASE_IMAGE
+        MACADDR=$(random_mac)
+        CONTAINER=$(docker run --privileged -d --name=$CONTAINER_NAME -e WEAVE_PASSWORD \
+            $WEAVE_DOCKER_ARGS $IMAGE -name $MACADDR -nickname "$CONTAINER_NAME" "$@")
+        with_container_netns $CONTAINER test_launch >/dev/null
+        wait_for_status $CONTAINER $HTTP_PORT
         echo $CONTAINER
         ;;
     launch-dns)
         [ $# -gt 0 ] || usage
         CIDR=$1
+        validate_cidr $CIDR
         shift 1
-        check_not_running $DNS_CONTAINER_NAME $DNS_IMAGE
+        check_not_running $DNS_CONTAINER_NAME $BASE_DNS_IMAGE
         create_bridge
         docker_bridge_ip
+        # Set WEAVEDNS_DOCKER_ARGS in the environment in order to supply
+        # additional parameters, such as resource limits, to docker
+        # when launching the weave container.
         DNS_CONTAINER=$(docker run --privileged -d --name=$DNS_CONTAINER_NAME \
             -p $DOCKER_BRIDGE_IP:53:53/udp -v /var/run/docker.sock:/var/run/docker.sock \
-            $DNS_IMAGE -iface $CONTAINER_IFNAME "$@")
+            $WEAVEDNS_DOCKER_ARGS $DNS_IMAGE -iface $CONTAINER_IFNAME "$@")
         with_container_netns $DNS_CONTAINER attach $CIDR >/dev/null
         populate_dns
         echo $DNS_CONTAINER
@@ -480,13 +714,24 @@ case "$COMMAND" in
         [ $# -eq 1 ] || usage
         http_call $CONTAINER_NAME $HTTP_PORT POST /connect -d "peer=$1"
         ;;
+    forget)
+        [ $# -eq 1 ] || usage
+        http_call $CONTAINER_NAME $HTTP_PORT POST /forget -d "peer=$1"
+        ;;
     status)
-        http_call $CONTAINER_NAME $HTTP_PORT GET /status
+        http_call $CONTAINER_NAME $HTTP_PORT GET /status || true
+        echo
+        http_call $DNS_CONTAINER_NAME $DNS_HTTP_PORT GET /status 2>/dev/null || true
         ;;
     ps)
-        [ $# -eq 0 ] || usage
-        for CONTAINER_ID in $(docker ps -q) ; do
-            if CONTAINER_ADDRS=$(with_container_netns $CONTAINER_ID container_weave_addrs 2>&1) ; then
+        [ $# -eq 0 ] && CONTAINERS="weave:expose $(docker ps -q)" || CONTAINERS="$@"
+        for CONTAINER_ID in $CONTAINERS ; do
+            if [ "$CONTAINER_ID" = "weave:expose" ]; then
+                ADDRS_CMD="ip addr show dev $BRIDGE"
+            else
+                ADDRS_CMD="with_container_netns $CONTAINER_ID container_weave_addrs"
+            fi
+            if CONTAINER_ADDRS=$($ADDRS_CMD 2>&1) ; then
                 CONTAINER_MAC=$(echo "$CONTAINER_ADDRS" | grep -o 'link/ether .*' | cut -d ' ' -f 2)
                 CONTAINER_IPS=$(echo "$CONTAINER_ADDRS" | grep -o 'inet .*'       | cut -d ' ' -f 2)
                 echo $CONTAINER_ID $CONTAINER_MAC $CONTAINER_IPS
@@ -498,109 +743,123 @@ case "$COMMAND" in
         echo weave script $SCRIPT_VERSION
         ask_version $CONTAINER_NAME $IMAGE
         ask_version $DNS_CONTAINER_NAME $DNS_IMAGE
+        if ! EXEC_IMAGE_ID=$(docker inspect --format='{{.Id}}' $EXEC_IMAGE 2>/dev/null) ; then
+            echo "Unable to find $EXEC_IMAGE image." >&2
+        else
+            EXEC_VERSION=$(docker images --no-trunc | grep $EXEC_IMAGE_ID | grep -v latest | tr -s ' ' | cut -d ' ' -f 2 | tr "\n" ' ')
+            if [ -n "$EXEC_VERSION" ] ; then
+                echo "weave exec $EXEC_VERSION"
+            else
+                echo "weave exec (unreleased version)"
+            fi
+        fi
         ;;
     run)
         [ $# -gt 0 ] || usage
         if [ "$1" = "--with-dns" ] ; then
             shift 1
-            if [ \( "$DOCKER_VERSION_MAJOR" -lt 1 \) -o \
-                 \( "$DOCKER_VERSION_MAJOR" -eq 1 -a \
-                    "$DOCKER_VERSION_MINOR" -lt 2 \) ] ; then
-                echo "ERROR: The '--with-dns' option requires Docker 1.2.0 or later; you are running $DOCKER_VERSION" >&2
-                exit 1
-            fi
-            docker_bridge_ip
-            DNS_ARG="--dns $DOCKER_BRIDGE_IP"
-            DNS_SEARCH_ARG="--dns-search=."
-            for arg in "$@"; do
-                case $arg in
-                    --dns-search=*)
-                        DNS_SEARCH_ARG=""
-                        ;;
-                    *)
-                        ;;
-                esac;
-            done
+            dns_args "$@"
         fi
-        validate_cidr $1
-        CIDR=$1
-        shift 1
+        collect_cidr_args "$@"
+        shift $CIDR_COUNT
+        CONTAINER=$(docker run $DNS_ARGS -d "$@")
         create_bridge
-        CONTAINER=$(docker run $DNS_ARG $DNS_SEARCH_ARG -d "$@")
-        with_container_netns $CONTAINER attach $CIDR >/dev/null
-        tell_dns PUT $CONTAINER $CIDR
+        with_container_netns $CONTAINER attach $CIDR_ARGS >/dev/null
+        tell_dns PUT $CONTAINER $CIDR_ARGS
         echo $CONTAINER
         ;;
+    dns-args)
+        dns_args "$@"
+        echo -n $DNS_ARGS
+        ;;
     start)
-        [ $# -eq 2 ] || usage
-        validate_cidr $1
+        collect_cidr_args "$@"
+        shift $CIDR_COUNT
+        [ $# -eq 1 ] || usage
+        CONTAINER_ID="$1"
         create_bridge
-        CONTAINER=$(docker start $2)
-        with_container_netns $CONTAINER attach $1 >/dev/null
-        tell_dns PUT $CONTAINER $1
+        CONTAINER=$(docker start $CONTAINER_ID)
+        with_container_netns $CONTAINER_ID attach $CIDR_ARGS >/dev/null
+        tell_dns PUT $CONTAINER_ID $CIDR_ARGS
         echo $CONTAINER
         ;;
     attach)
-        [ $# -eq 2 ] || usage
-        validate_cidr $1
+        collect_cidr_args "$@"
+        shift $CIDR_COUNT
+        [ $# -eq 1 ] || usage
+        CONTAINER_ID="$1"
         create_bridge
-        with_container_netns $2 attach $1 >/dev/null
-        tell_dns PUT $2 $1
+        with_container_netns $CONTAINER_ID attach $CIDR_ARGS >/dev/null
+        tell_dns PUT $CONTAINER_ID $CIDR_ARGS
         ;;
     detach)
-        [ $# -eq 2 ] || usage
-        validate_cidr $1
-        with_container_netns $2 detach $1 >/dev/null
-        tell_dns DELETE $2 $1
+        collect_cidr_args "$@"
+        shift $CIDR_COUNT
+        [ $# -eq 1 ] || usage
+        CONTAINER_ID="$1"
+        with_container_netns $CONTAINER_ID detach $CIDR_ARGS >/dev/null
+        tell_dns DELETE $CONTAINER_ID $CIDR_ARGS
         ;;
     expose)
-        [ $# -eq 1 ] || usage
-        validate_cidr $1
-        create_bridge
-        if ! ip addr show dev $BRIDGE | grep -qF $1
-        then
-            ip addr add dev $BRIDGE $1
-            add_iptables_rule nat WEAVE -o $BRIDGE ! -s $1 -j MASQUERADE
-            add_iptables_rule nat WEAVE -s $1 ! -o $BRIDGE -j MASQUERADE
+        collect_cidr_args "$@"
+        shift $CIDR_COUNT
+        if [ $# -eq 0 ]; then
+            FQDN=""
+        else
+            [ $# -eq 2 -a "$1" = "-h" ] || usage
+            FQDN="$2"
         fi
+        create_bridge --without-ethtool
+        for CIDR in $CIDR_ARGS; do
+            if ! ip addr show dev $BRIDGE | grep -qF $CIDR
+            then
+                ip addr add dev $BRIDGE $CIDR
+                arp_update $BRIDGE $CIDR
+                add_iptables_rule nat WEAVE -o $BRIDGE ! -s $CIDR -j MASQUERADE
+                add_iptables_rule nat WEAVE -s $CIDR ! -o $BRIDGE -j MASQUERADE
+                if [ "$FQDN" ]; then
+                    http_call $DNS_CONTAINER_NAME $DNS_HTTP_PORT PUT /name/weave:expose/${CIDR%/*} --data-urlencode "fqdn=$FQDN" 2>/dev/null || true
+                fi
+            fi
+        done
         ;;
     hide)
-        [ $# -eq 1 ] || usage
-        validate_cidr $1
-        create_bridge
-        if ip addr show dev $BRIDGE | grep -qF $1
-        then
-            ip addr del dev $BRIDGE $1
-            delete_iptables_rule nat WEAVE -o $BRIDGE ! -s $1 -j MASQUERADE
-            delete_iptables_rule nat WEAVE -s $1 ! -o $BRIDGE -j MASQUERADE
-        fi
+        collect_cidr_args "$@"
+        shift $CIDR_COUNT
+        [ $# -eq 0 ] || usage
+        create_bridge --without-ethtool
+        for CIDR in $CIDR_ARGS; do
+            if ip addr show dev $BRIDGE | grep -qF $CIDR
+            then
+                ip addr del dev $BRIDGE $CIDR
+                delete_iptables_rule nat WEAVE -o $BRIDGE ! -s $CIDR -j MASQUERADE
+                delete_iptables_rule nat WEAVE -s $CIDR ! -o $BRIDGE -j MASQUERADE
+                http_call $DNS_CONTAINER_NAME $DNS_HTTP_PORT DELETE /name/weave:expose/${CIDR%/*} 2>/dev/null || true
+            fi
+        done
         ;;
     stop)
         [ $# -eq 0 ] || usage
-        if ! docker kill $CONTAINER_NAME >/dev/null 2>&1 ; then
+        if ! docker stop $CONTAINER_NAME >/dev/null 2>&1 ; then
             echo "Weave is not running." >&2
         fi
         docker rm -f $CONTAINER_NAME >/dev/null 2>&1 || true
-        if command_exists_or_warn conntrack; then
-            conntrack -D -p udp --dport $PORT >/dev/null 2>&1 || true
-        fi
+        conntrack -D -p udp --dport $PORT >/dev/null 2>&1 || true
         ;;
     stop-dns)
         [ $# -eq 0 ] || usage
-        if ! docker kill $DNS_CONTAINER_NAME >/dev/null 2>&1 ; then
+        if ! docker stop $DNS_CONTAINER_NAME >/dev/null 2>&1 ; then
             echo "WeaveDNS is not running." >&2
         fi
         docker rm -f $DNS_CONTAINER_NAME >/dev/null 2>&1 || true
         ;;
     reset)
         [ $# -eq 0 ] || usage
-        docker kill  $CONTAINER_NAME     >/dev/null 2>&1 || true
-        docker kill  $DNS_CONTAINER_NAME >/dev/null 2>&1 || true
+        docker stop  $CONTAINER_NAME     >/dev/null 2>&1 || true
+        docker stop  $DNS_CONTAINER_NAME >/dev/null 2>&1 || true
         docker rm -f $CONTAINER_NAME     >/dev/null 2>&1 || true
         docker rm -f $DNS_CONTAINER_NAME >/dev/null 2>&1 || true
-        if command_exists_or_warn conntrack; then
-            conntrack -D -p udp --dport $PORT >/dev/null 2>&1 || true
-        fi
+        conntrack -D -p udp --dport $PORT >/dev/null 2>&1 || true
         destroy_bridge
         for LOCAL_IFNAME in $(ip link show | grep v${CONTAINER_IFNAME}pl | cut -d ' ' -f 2 | tr -d ':') ; do
             ip link del $LOCAL_IFNAME

weavedns/Dockerfile

@@ -3,4 +3,4 @@ MAINTAINER Weaveworks Inc <help@weave.works>
 WORKDIR /home/weave
 ADD ./weavedns /home/weave/
 EXPOSE 53/udp
-ENTRYPOINT ["/home/weave/weavedns", "-wait", "5"]
+ENTRYPOINT ["/home/weave/weavedns", "-wait", "20"]

weavedns/README.md

@@ -1,52 +1,73 @@
 # Weave DNS server
 
-The Weave DNS server answers name queries in a Weave network. It is
-run per-host, to be supplied as the nameserver for containers on that
-host. It is then told about hostnames for the local containers. For
-other names it will ask the other weave hosts, or fall back to using
-the host's configured name server.
+The Weave DNS server answers name queries in a Weave network. This
+provides a simple way for containers to find each other: just give
+them hostnames and tell other containers to connect to those names.
+Unlike Docker 'links', this requires no code changes and works across
+hosts.
 
 ## Using weaveDNS
 
-The weave script command `launch-dns` starts the DNS container, and
-then you use the `--with-dns` option on containers you wish to use it.
-Subsquently, giving any container a hostname in the domain
-`.weave.local` will register it in DNS. For example:
+WeaveDNS is deployed as a set of containers that communicate with each
+other over the weave network. One such container needs to be started
+on every weave host, by invoking the weave script command
+`launch-dns`. Application containers are then instructed to use
+WeaveDNS as their nameserver by supplying the `--with-dns` option when
+starting them. Giving any container a hostname in the `.weave.local`
+domain registers it in weaveDNS.  For example:
 
 ```bash
 $ weave launch
-$ weave launch-dns 10.1.254.1/24
-$ weave run 10.1.1.25/24 -ti -h pingme.weave.local ubuntu
-$ shell1=$(weave run --with-dns 10.1.1.26/24 -ti -h ubuntu.weave.local ubuntu)
+$ weave launch-dns 10.2.254.1/24
+$ weave run 10.2.1.25/24 -ti -h pingme.weave.local ubuntu
+$ shell1=$(weave run --with-dns 10.2.1.26/24 -ti -h ubuntu.weave.local ubuntu)
 $ docker attach $shell1
 
 # ping pingme
 ...
 ```
 
-The weave IP address supplied to `weave launch-dns` must not be used
-by any other container, and the supplied network must be the same for
-all DNS containers, and be disjoint from all application networks.
+Each weaveDNS container started with `launch-dns` needs to be given
+its own, unique, IP address, in a subnet that is a) common to all
+weaveDNS containers, b) disjoint from the application subnets, and c)
+not in use on any of the hosts. In our example the weaveDNS address is
+in subnet 10.2.254.0/24 and the application containers are in
+subnet 10.2.1.0/24.
 
-The DNS container can be stopped with `stop-dns`.
+WeaveDNS containers can be stopped with `stop-dns`.
 
+## How it works
+
+The weaveDNS container running on every host acts as the nameserver
+for containers on that host. It is told about hostnames for local
+containers by the `weave run` command. If a hostname is in the
+`.weave.local` domain then weaveDNS records the association of that
+name with the container's weave IP address(es).
+
+When weaveDNS is queried for a name in the `.weave.local` domain, it
+first checks its own records. If the name is not found there, it asks
+the weaveDNS servers on the other hosts in the weave network.
+
+When weaveDNS is queried for a name in a domain other than
+`.weave.local`, it queries the host's configured nameserver,
+which is the standard behaviour for Docker containers.
 
 ## Domain search paths
 
 If you don't supply a domain search path (with `--dns-search=`),
-`weave run ...` will tell a container to look for "bare" hostnames,
-like `pingme`, in its own domain. That's why you can just say `ping
+`weave run ...` tells a container to look for "bare" hostnames, like
+`pingme`, in its own domain. That's why you can just invoke `ping
 pingme` above -- since the hostname is `ubuntu.weave.local`, it will
 look for `pingme.weave.local`.
 
 If you want to supply other entries for the domain search path,
 e.g. if you want containers in different sub-domains to resolve
-hostnames across all sub-domains plus some external domains, you will
-need *also* to supply the `weave.local` domain to retain the above
+hostnames across all sub-domains plus some external domains, you need
+*also* to supply the `weave.local` domain to retain the above
 behaviour.
 
 ```bash
-weave run --with-dns 10.1.1.4/24 -ti \
+weave run --with-dns 10.2.1.4/24 -ti \
   --dns-search=zone1.weave.local --dns-search=zone2.weave.local \
   --dns-search=corp1.com --dns-search=corp2.com \
   --dns-search=weave.local ubuntu
@@ -54,10 +75,10 @@ weave run --with-dns 10.1.1.4/24 -ti \
 
 ## Doing things more manually
 
-If you use the `--with-dns` option, `weave run` will automatically
-supply the DNS server address to the new container. And both
-`weave run` and `weave attach` will register the hostname of the given
-container against the given weave network IP address.
+If you use the `--with-dns` option, `weave run` automatically supplies
+the DNS server address to the new container. And both `weave run` and
+`weave attach` register the hostname of the given container against
+the given weave network IP address.
 
 In some circumstances, you may not want to use the `weave`
 command. You can still take advantage of a running weaveDNS, with some
@@ -69,12 +90,12 @@ So that containers can connect to a stable and always routable IP
 address, weaveDNS publishes its port 53 to the Docker bridge device,
 which is assumed to be `docker0`.
 
-Some configurations will use a different Docker bridge device. To
+Some configurations may use a different Docker bridge device. To
 supply a different bridge device, use the environment variable
 `DOCKER_BRIDGE`, e.g.,
 
 ```bash
-$ sudo DOCKER_BRIDGE=someother weave launch-dns 10.1.254.1/24
+$ sudo DOCKER_BRIDGE=someother weave launch-dns 10.2.254.1/24
 ```
 
 ### Supplying the DNS server
@@ -86,7 +107,7 @@ make it use weaveDNS:
 ```bash
 $ docker_ip=$(docker inspect --format='{{ .NetworkSettings.Gateway }}' weavedns)
 $ shell2=$(docker run --dns=$docker_ip -ti ubuntu)
-$ weave attach 10.1.1.27/24 $shell2
+$ weave attach 10.2.1.27/24 $shell2
 ```
 
 This isn't very useful unless the container is also attached to the
@@ -102,10 +123,10 @@ $ docker_ip=$(ip -4 addr show dev docker0 | grep -o 'inet [0-9.]*' | cut -d ' '
 
 ### Supplying the domain search path
 
-By default, Docker will provide containers with a `/etc/resolv.conf`
-that matches that for the host. In some circumstances, this may
-include a DNS search path, which will break the nice "bare names
-resolve" property above.
+By default, Docker provides containers with a `/etc/resolv.conf` that
+matches that for the host. In some circumstances, this may include a
+DNS search path, which will break the nice "bare names resolve"
+property above.
 
 Therefore, when starting containers with `docker run` instead of
 `weave run`, you will usually want to supply a domain search path so
@@ -113,6 +134,21 @@ that you can use unqualified hostnames. Use `--dns-search=.` to make
 the resolver use the container's domain, or e.g.,
 `--dns-search=weave.local` to make it look in `weave.local`.
 
+### Using a different local domain
+
+By default, WeaveDNS uses `weave.local.` as the domain for names on the
+Weave network. In general users do not need to change this domain, but you
+can force WeaveDNS to use a different domain by launching it
+with the `--domain` argument. For example,
+
+```bash
+$ weave launch-dns 10.2.254.1/24 --domain="mycompany.local."
+```
+
+The local domain should end with `local.`, since these names are
+link-local as per [RFC6762](https://tools.ietf.org/html/rfc6762),
+(though this is not strictly neccessary).
+
 ### Adding containers to DNS
 
 If DNS is started after you've attached a container to the weave
@@ -121,26 +157,28 @@ its hostname, you can register it using the HTTP API:
 
 ```bash
 $ docker start $shell2
-$ shell2_ip=$(docker inspect --format='{{ .NetworkSettings.IPAddress }}' $shell2)
-$ curl -X PUT "http://$dns_ip:6785/name/$shell2/10.1.1.27" -d local_ip=$shell2_ip -d fqdn=shell2.weave.local
+$ curl -X PUT "http://$dns_ip:6785/name/$shell2/10.2.1.27" -d fqdn=shell2.weave.local
 ```
 
 ### Registering multiple containers with the same name
 
-This is supported; in the initial implementation weaveDNS will pick one address to return when you ask for the name.  Since weave-dns will remove any container that dies, this is a simple way to implement redundancy.  In the current implementation it does not attempt to do load-balancing.
+This is supported; weaveDNS picks one address to return when you ask
+for the name. Since weaveDNS removes any container that dies, this is
+a simple way to implement redundancy.  In the current implementation
+it does not attempt to do load-balancing.
 
 ### Replacing one container with another at the same name
 
-If you would like to deploy a new version of a service, keep the old one running because it has active connections but make all new requests go to the new version, then you can simply start the new server container and then [unregister](https://github.com/zettio/weave/tree/master/weavedns#unregistering) the old one from DNS. And finally, when all connections to the old server have terminated, stop the container as normal.
+If you would like to deploy a new version of a service, keep the old one running because it has active connections but make all new requests go to the new version, then you can simply start the new server container and then [unregister](https://github.com/weaveworks/weave/tree/master/weavedns#unregistering) the old one from DNS. And finally, when all connections to the old server have terminated, stop the container as normal.
 
 ### Not watching docker events
 
-By default, the server will watch docker events and remove entries for
-any containers that die. You can tell it not to, by adding
-`--watch=false` to the container args:
+By default, weaveDNS watches docker events and removes entries for any
+containers that die. You can tell it not to, by adding `--watch=false`
+to the container args:
 
 ```bash
-$ weave launch-dns 10.1.254.1/24 --watch=false
+$ weave launch-dns 10.2.254.1/24 --watch=false
 ```
 
 ### Unregistering
@@ -151,9 +189,44 @@ API with e.g., `curl`:
 ```bash
 $ docker stop $shell2
 $ dns_ip=$(docker inspect --format='{{ .NetworkSettings.IPAddress }}' weavedns)
-$ curl -X DELETE "http://$dns_ip:6785/name/$shell2/10.1.1.27"
+$ curl -X DELETE "http://$dns_ip:6785/name/$shell2/10.2.1.27"
 ```
 
+## Troubleshooting
+
+The command
+
+    weave status
+
+reports on the current status of the weave router and DNS:
+
+````
+weave router git-8f675f15c0b5
+...
+
+weave DNS git-8f675f15c0b5
+Local domain weave.local.
+Listen address :53
+mDNS interface &{26 65535 ethwe fa:b6:b1:85:ac:9b up|broadcast|multicast}
+Fallback DNS config &{[66.28.0.45 8.8.8.8] [] 53 1 5 2}
+Zone database:
+710978857a88 10.2.1.26 wiff.weave.local.
+e8d85b1dcdb1 10.2.1.27 waff.weave.local.
+26bd05f9a0cb 10.2.1.28 ping.weave.local.
+1ab0e8b17c39 10.2.1.29 pong.weave.local.
+````
+
+The first section covers the router; see the troubleshooting guide in
+the main documentation for more detail.
+
+The second section is pertinent to weaveDNS, and includes:
+
+* The local domain suffix which is being served
+* The address on which the DNS server is listening
+* The interface being used for multicast DNS
+* The fallback DNS which will be used to resolve non local names
+* The names known to the local weaveDNS server. Each entry comprises the container ID, IP address and its fully qualified domain name
+
 ## Present limitations
 
  * The server will not know about restarted containers, but if you

weavedns/main.go

@@ -3,12 +3,15 @@ package main
 import (
 	"flag"
 	"fmt"
-	weavedns "github.com/zettio/weave/nameserver"
-	weavenet "github.com/zettio/weave/net"
-	"io"
-	"io/ioutil"
+	. "github.com/weaveworks/weave/common"
+	"github.com/weaveworks/weave/common/updater"
+	weavedns "github.com/weaveworks/weave/nameserver"
+	weavenet "github.com/weaveworks/weave/net"
 	"net"
 	"os"
+	"os/signal"
+	"runtime"
+	"syscall"
 )
 
 var version = "(unreleased version)"
@@ -18,57 +21,99 @@ func main() {
 		justVersion bool
 		ifaceName   string
 		apiPath     string
+		domain      string
 		dnsPort     int
 		httpPort    int
 		wait        int
+		timeout     int
+		udpbuf      int
+		cacheLen    int
 		watch       bool
 		debug       bool
+		err         error
 	)
 
 	flag.BoolVar(&justVersion, "version", false, "print version and exit")
 	flag.StringVar(&ifaceName, "iface", "", "name of interface to use for multicast")
-	flag.StringVar(&apiPath, "api", "unix:///var/run/docker.sock", "Path to Docker API socket")
+	flag.StringVar(&apiPath, "api", "unix:///var/run/docker.sock", "path to Docker API socket")
+	flag.StringVar(&domain, "domain", weavedns.DefaultLocalDomain, "local domain (ie, 'weave.local.')")
 	flag.IntVar(&wait, "wait", 0, "number of seconds to wait for interface to be created and come up")
-	flag.IntVar(&dnsPort, "dnsport", 53, "port to listen to dns requests")
+	flag.IntVar(&dnsPort, "dnsport", weavedns.DefaultServerPort, "port to listen to DNS requests")
 	flag.IntVar(&httpPort, "httpport", 6785, "port to listen to HTTP requests")
+	flag.IntVar(&timeout, "timeout", weavedns.DefaultTimeout, "timeout for resolutions")
+	flag.IntVar(&udpbuf, "udpbuf", weavedns.DefaultUDPBuflen, "UDP buffer length")
+	flag.IntVar(&cacheLen, "cache", weavedns.DefaultCacheLen, "cache length")
 	flag.BoolVar(&watch, "watch", true, "watch the docker socket for container events")
 	flag.BoolVar(&debug, "debug", false, "output debugging info to stderr")
 	flag.Parse()
 
 	if justVersion {
-		io.WriteString(os.Stdout, fmt.Sprintf("weave DNS %s\n", version))
+		fmt.Printf("weave DNS %s\n", version)
 		os.Exit(0)
 	}
 
-	debugOut := ioutil.Discard
-	if debug {
-		debugOut = os.Stderr
-	}
-	weavedns.InitLogging(debugOut, os.Stdout, os.Stdout, os.Stderr)
+	InitDefaultLogging(debug)
+	Info.Printf("WeaveDNS version %s\n", version) // first thing in log: the version
 
 	var zone = new(weavedns.ZoneDb)
 
 	if watch {
-		err := weavedns.StartUpdater(apiPath, zone)
+		err := updater.Start(apiPath, zone)
 		if err != nil {
-			weavedns.Error.Fatal("Unable to start watcher", err)
+			Error.Fatal("Unable to start watcher", err)
 		}
 	}
 
-	var iface *net.Interface = nil
+	var iface *net.Interface
 	if ifaceName != "" {
 		var err error
-		weavedns.Info.Println("Waiting for interface", ifaceName, "to come up")
+		Info.Println("Waiting for interface", ifaceName, "to come up")
 		iface, err = weavenet.EnsureInterface(ifaceName, wait)
 		if err != nil {
-			weavedns.Error.Fatal(err)
+			Error.Fatal(err)
 		} else {
-			weavedns.Info.Println("Interface", ifaceName, "is up")
+			Info.Println("Interface", ifaceName, "is up")
 		}
 	}
 
-	err := weavedns.StartServer(zone, iface, dnsPort, httpPort, wait)
+	srvConfig := weavedns.DNSServerConfig{
+		Port:        dnsPort,
+		CacheLen:    cacheLen,
+		LocalDomain: domain,
+		Timeout:     timeout,
+		UDPBufLen:   udpbuf,
+	}
+
+	srv, err := weavedns.NewDNSServer(srvConfig, zone, iface)
 	if err != nil {
-		weavedns.Error.Fatal("Failed to start server", err)
+		Error.Fatal("Failed to initialize the WeaveDNS server", err)
+	}
+	Info.Println("Upstream", srv.Upstream)
+
+	Debug.Printf("Starting the signals handler")
+	go handleSignals(srv)
+
+	go weavedns.ListenHTTP(version, srv, domain, zone, httpPort)
+	err = srv.Start()
+	if err != nil {
+		Error.Fatal("Failed to start the WeaveDNS server", err)
+	}
+}
+
+func handleSignals(s *weavedns.DNSServer) {
+	sigs := make(chan os.Signal, 1)
+	signal.Notify(sigs, syscall.SIGINT, syscall.SIGQUIT)
+	buf := make([]byte, 1<<20)
+	for {
+		sig := <-sigs
+		switch sig {
+		case syscall.SIGINT:
+			Info.Printf("=== received SIGINT ===\n*** exiting\n")
+			s.Stop()
+			os.Exit(0)
+		case syscall.SIGQUIT:
+			stacklen := runtime.Stack(buf, true)
+			Info.Printf("=== received SIGQUIT ===\n*** goroutine dump...\n%s\n*** end\n", buf[:stacklen])
+		}
 	}
 }

weaveexec/Dockerfile

@@ -0,0 +1,9 @@
+FROM gliderlabs/alpine
+MAINTAINER Weaveworks Inc <help@weave.works>
+WORKDIR /home/weave
+RUN ["apk", "add", "--update", "ethtool", "conntrack-tools", "curl", "iptables", "iproute2"]
+RUN ["sh", "-c", "rm -rf /var/cache/apk/*"]
+ADD ./weave /home/weave/
+ADD ./sigproxy /home/weave/
+ADD ./docker.tgz /
+ENTRYPOINT ["/home/weave/sigproxy", "/home/weave/weave"]

weaver/.gitignore

@@ -1 +0,0 @@
-weaver

weaver/Dockerfile

@@ -2,4 +2,4 @@ FROM scratch
 MAINTAINER Weaveworks Inc <help@weave.works>
 WORKDIR /home/weave
 ADD ./weaver /home/weave/
-ENTRYPOINT ["/home/weave/weaver", "-wait", "5"]
+ENTRYPOINT ["/home/weave/weaver", "-wait", "20"]

weaver/main.go

@@ -6,15 +6,16 @@ import (
 	"flag"
 	"fmt"
 	"github.com/davecheney/profile"
-	weavenet "github.com/zettio/weave/net"
-	weave "github.com/zettio/weave/router"
-	"io"
+	"github.com/gorilla/mux"
+	weavenet "github.com/weaveworks/weave/net"
+	weave "github.com/weaveworks/weave/router"
 	"log"
 	"net"
 	"net/http"
 	"os"
 	"os/signal"
 	"runtime"
+	"strings"
 	"syscall"
 )
 
@@ -34,123 +35,178 @@ func main() {
 	runtime.GOMAXPROCS(procs)
 
 	var (
+		config      weave.RouterConfig
 		justVersion bool
 		ifaceName   string
 		routerName  string
+		nickName    string
 		password    string
 		wait        int
 		debug       bool
 		prof        string
 		peers       []string
-		connLimit   int
-		bufSz       int
+		bufSzMB     int
+		httpAddr    string
 	)
 
 	flag.BoolVar(&justVersion, "version", false, "print version and exit")
-	flag.StringVar(&ifaceName, "iface", "", "name of interface to read from")
-	flag.StringVar(&routerName, "name", "", "name of router (defaults to MAC)")
+	flag.IntVar(&config.Port, "port", weave.Port, "router port")
+	flag.StringVar(&ifaceName, "iface", "", "name of interface to capture/inject from (disabled if blank)")
+	flag.StringVar(&routerName, "name", "", "name of router (defaults to MAC of interface)")
+	flag.StringVar(&nickName, "nickname", "", "nickname of peer (defaults to hostname)")
 	flag.StringVar(&password, "password", "", "network password")
-	flag.IntVar(&wait, "wait", 0, "number of seconds to wait for interface to be created and come up (defaults to 0, i.e. don't wait)")
+	flag.IntVar(&wait, "wait", 0, "number of seconds to wait for interface to be created and come up (0 = don't wait)")
 	flag.BoolVar(&debug, "debug", false, "enable debug logging")
 	flag.StringVar(&prof, "profile", "", "enable profiling and write profiles to given path")
-	flag.IntVar(&connLimit, "connlimit", 10, "connection limit (defaults to 10, set to 0 for unlimited)")
-	flag.IntVar(&bufSz, "bufsz", 8, "capture buffer size in MB (defaults to 8MB)")
+	flag.IntVar(&config.ConnLimit, "connlimit", 30, "connection limit (0 for unlimited)")
+	flag.IntVar(&bufSzMB, "bufsz", 8, "capture buffer size in MB")
+	flag.StringVar(&httpAddr, "httpaddr", fmt.Sprintf(":%d", weave.HTTPPort), "address to bind HTTP interface to (disabled if blank, absolute path indicates unix domain socket)")
 	flag.Parse()
 	peers = flag.Args()
 
 	if justVersion {
-		io.WriteString(os.Stdout, fmt.Sprintf("weave router %s\n", version))
+		fmt.Printf("weave router %s\n", version)
 		os.Exit(0)
 	}
 
-	options := make(map[string]string)
-	flag.Visit(func(f *flag.Flag) {
-		value := f.Value.String()
-		if f.Name == "password" {
-			value = "<elided>"
-		}
-		options[f.Name] = value
-	})
-	log.Println("Command line options:", options)
+	log.Println("Command line options:", options())
 	log.Println("Command line peers:", peers)
 
-	if ifaceName == "" {
-		fmt.Println("Missing required parameter 'iface'")
-		os.Exit(1)
-	}
-	iface, err := weavenet.EnsureInterface(ifaceName, wait)
-	if err != nil {
-		log.Fatal(err)
+	var err error
+
+	if ifaceName != "" {
+		config.Iface, err = weavenet.EnsureInterface(ifaceName, wait)
+		if err != nil {
+			log.Fatal(err)
+		}
 	}
 
 	if routerName == "" {
-		routerName = iface.HardwareAddr.String()
+		if config.Iface == nil {
+			log.Fatal("Either an interface must be specified with -iface or a name with -name")
+		}
+		routerName = config.Iface.HardwareAddr.String()
 	}
-
-	ourName, err := weave.PeerNameFromUserInput(routerName)
+	name, err := weave.PeerNameFromUserInput(routerName)
 	if err != nil {
 		log.Fatal(err)
 	}
 
+	if nickName == "" {
+		nickName, err = os.Hostname()
+		if err != nil {
+			log.Fatal(err)
+		}
+	}
+
 	if password == "" {
 		password = os.Getenv("WEAVE_PASSWORD")
 	}
+
 	if password == "" {
 		log.Println("Communication between peers is unencrypted.")
 	} else {
+		config.Password = []byte(password)
 		log.Println("Communication between peers is encrypted.")
 	}
 
-	var logFrame func(string, []byte, *layers.Ethernet)
-	if debug {
-		logFrame = func(prefix string, frame []byte, eth *layers.Ethernet) {
-			h := fmt.Sprintf("%x", sha256.Sum256(frame))
-			if eth == nil {
-				log.Println(prefix, len(frame), "bytes (", h, ")")
-			} else {
-				log.Println(prefix, len(frame), "bytes (", h, "):", eth.SrcMAC, "->", eth.DstMAC)
-			}
-		}
-	} else {
-		logFrame = func(prefix string, frame []byte, eth *layers.Ethernet) {}
-	}
-
 	if prof != "" {
 		p := *profile.CPUProfile
 		p.ProfilePath = prof
 		defer profile.Start(&p).Stop()
 	}
 
-	router := weave.NewRouter(iface, ourName, []byte(password), connLimit, bufSz*1024*1024, logFrame)
+	config.BufSz = bufSzMB * 1024 * 1024
+	config.LogFrame = logFrameFunc(debug)
+
+	router := weave.NewRouter(config, name, nickName)
+	log.Println("Our name is", router.Ourself.FullName())
 	router.Start()
-	for _, peer := range peers {
-		if addr, err := net.ResolveTCPAddr("tcp4", weave.NormalisePeerAddr(peer)); err == nil {
-			router.ConnectionMaker.InitiateConnection(addr.String())
-		} else {
-			log.Fatal(err)
-		}
+	initiateConnections(router, peers)
+	if httpAddr != "" {
+		go handleHTTP(router, httpAddr)
 	}
-	go handleHttp(router)
 	handleSignals(router)
 }
 
-func handleHttp(router *weave.Router) {
-	http.HandleFunc("/status", func(w http.ResponseWriter, r *http.Request) {
-		io.WriteString(w, fmt.Sprintln("weave router", version))
-		io.WriteString(w, router.Status())
+func options() map[string]string {
+	options := make(map[string]string)
+	flag.Visit(func(f *flag.Flag) {
+		value := f.Value.String()
+		if f.Name == "password" {
+			value = "<elided>"
+		}
+		options[f.Name] = value
 	})
-	http.HandleFunc("/connect", func(w http.ResponseWriter, r *http.Request) {
-		peer := r.FormValue("peer")
-		if addr, err := net.ResolveTCPAddr("tcp4", weave.NormalisePeerAddr(peer)); err == nil {
-			router.ConnectionMaker.InitiateConnection(addr.String())
+	return options
+}
+
+func logFrameFunc(debug bool) weave.LogFrameFunc {
+	if !debug {
+		return func(prefix string, frame []byte, eth *layers.Ethernet) {}
+	}
+	return func(prefix string, frame []byte, eth *layers.Ethernet) {
+		h := fmt.Sprintf("%x", sha256.Sum256(frame))
+		if eth == nil {
+			log.Println(prefix, len(frame), "bytes (", h, ")")
 		} else {
+			log.Println(prefix, len(frame), "bytes (", h, "):", eth.SrcMAC, "->", eth.DstMAC)
+		}
+	}
+}
+
+func initiateConnections(router *weave.Router, peers []string) {
+	for _, peer := range peers {
+		if err := router.ConnectionMaker.InitiateConnection(peer); err != nil {
+			log.Fatal(err)
+		}
+	}
+}
+
+func handleHTTP(router *weave.Router, httpAddr string) {
+	encryption := "off"
+	if router.UsingPassword() {
+		encryption = "on"
+	}
+
+	muxRouter := mux.NewRouter()
+
+	muxRouter.Methods("GET").Path("/status").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintln(w, "weave router", version)
+		fmt.Fprintln(w, "Encryption", encryption)
+		fmt.Fprintln(w, router.Status())
+	})
+
+	muxRouter.Methods("GET").Path("/status-json").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		json, _ := router.GenerateStatusJSON(version, encryption)
+		w.Write(json)
+	})
+
+	muxRouter.Methods("POST").Path("/connect").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if err := router.ConnectionMaker.InitiateConnection(r.FormValue("peer")); err != nil {
 			http.Error(w, fmt.Sprint("invalid peer address: ", err), http.StatusBadRequest)
 		}
 	})
-	address := fmt.Sprintf(":%d", weave.HttpPort)
-	err := http.ListenAndServe(address, nil)
+
+	muxRouter.Methods("POST").Path("/forget").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		router.ConnectionMaker.ForgetConnection(r.FormValue("peer"))
+	})
+
+	http.Handle("/", muxRouter)
+
+	protocol := "tcp"
+	if strings.HasPrefix(httpAddr, "/") {
+		os.Remove(httpAddr) // in case it's there from last time
+		protocol = "unix"
+	}
+	l, err := net.Listen(protocol, httpAddr)
 	if err != nil {
-		log.Fatal("Unable to create http listener: ", err)
+		log.Fatal("Unable to create http listener socket: ", err)
+	}
+
+	err = http.Serve(l, nil)
+	if err != nil {
+		log.Fatal("Unable to create http server", err)
 	}
 }
 
@@ -162,8 +218,8 @@ func handleSignals(router *weave.Router) {
 		sig := <-sigs
 		switch sig {
 		case syscall.SIGQUIT:
-			runtime.Stack(buf, true)
-			log.Printf("=== received SIGQUIT ===\n*** goroutine dump...\n%s\n*** end\n", buf)
+			stacklen := runtime.Stack(buf, true)
+			log.Printf("=== received SIGQUIT ===\n*** goroutine dump...\n%s\n*** end\n", buf[:stacklen])
 		case syscall.SIGUSR1:
 			log.Printf("=== received SIGUSR1 ===\n*** status...\n%s\n*** end\n", router.Status())
 		}