Add release v1.3.1

Raise minimum Docker version to 1.6

.gitignore

@@ -39,22 +39,24 @@ prog/weavehosts/weavehosts
 prog/weaveproxy/weaveproxy
 prog/sigproxy/sigproxy
 prog/weavewait/weavewait
+prog/weavewait/weavewait_noop
 prog/netcheck/netcheck
+prog/docker_tls_args/docker_tls_args
 testing/cover/cover
 testing/runner/runner
-tools/bin
-tools/build
 releases
 .weaver.uptodate
 .weavedns.uptodate
 .weaveexec.uptodate
-weave.tar
+weave.tar.gz
 prog/weaveexec/weave
 prog/weaveexec/weavehosts
 prog/weaveexec/weaveproxy
 prog/weaveexec/weavewait
+prog/weaveexec/weavewait_noop
 prog/weaveexec/sigproxy
 prog/weaveexec/netcheck
+prog/weaveexec/docker_tls_args
 prog/weaveexec/docker*.tgz
 Vagrantfile.local
 test/tls/*.pem

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "tools"]
+	path = tools
+	url = https://github.com/weaveworks/build-tools

.travis.yml

@@ -21,7 +21,6 @@ install:
   - go get ./...
 
 script:
-  - bin/lint .
   - make tests
 
 after_success:

CHANGELOG.md

@@ -1,3 +1,58 @@
+## Release 1.3.1
+
+**Highlights**
+
+* The minimum Docker version has been increased to 1.6 due to the
+  [upcoming deprecation of Dockerhub access](https://blog.docker.com/2015/10/docker-hub-deprecation-1-5/)
+  for old clients. From December 7th onwards previous versions of the
+  `weave` script will fail to pull down images from the hub; if you
+  are unable to upgrade to 1.3.1 immediately you can work around this
+  by running `weave --local setup` in conjunction with a compatible
+  Docker client installation
+* Docker networking plugin now works with older kernels and allows you
+  to configure the MTU
+
+More details in the
+[change log](https://github.com/weaveworks/weave/issues?q=milestone%3A1.3.1).
+
+## Release 1.3.0
+
+**Highlights**
+
+This release includes a Docker Plugin, so you have the option to use
+Weave Net that way. More details in the
+[change log](https://github.com/weaveworks/weave/issues?q=milestone%3A1.3.0).
+
+## Release 1.2.1
+
+This release contains a number of bug fixes and minor
+enhancements. More details in the
+[change log](https://github.com/weaveworks/weave/issues?q=milestone%3A1.2.1).
+
+The release is fully compatible with 1.2.0 versions, so existing
+clusters can be upgraded incrementally.
+
+## Release 1.2.0
+
+**Highlights**
+
+This release introduces the 
+[Fast Data Path](http://docs.weave.works/weave/master/head/features.html#fast-data-path),
+which allows Weave networks to operate at near wire-level speeds. This
+new feature is enabled by default.
+
+Other highlights:
+
+- auto-configuration of TLS for the Weave Docker API proxy, making it
+  easier to run Weave on Macs and in conjunction with tools like
+  Docker Swarm
+- support for restart policies on application containers and weave
+  infrastructure containers
+- better compatibility with recent and future Docker versions
+
+More details in the 
+[change log](https://github.com/weaveworks/weave/issues?q=milestone%3A1.2.0).
+
 ## Release 1.1.2
 
 This release contains a small number of bug fixes. More details in the

Makefile

@@ -1,7 +1,7 @@
 PUBLISH=publish_weave publish_weaveexec
 
 .DEFAULT: all
-.PHONY: all update tests publish $(PUBLISH) clean prerequisites build travis run-smoketests
+.PHONY: all update tests lint publish $(PUBLISH) clean clean-bin prerequisites build travis run-smoketests
 
 # If you can use docker without being root, you can do "make SUDO="
 SUDO=sudo
@@ -13,12 +13,12 @@ WEAVER_EXE=prog/weaver/weaver
 WEAVEPROXY_EXE=prog/weaveproxy/weaveproxy
 SIGPROXY_EXE=prog/sigproxy/sigproxy
 WEAVEWAIT_EXE=prog/weavewait/weavewait
-WEAVEHOSTS_EXE=prog/weavehosts/weavehosts
+WEAVEWAIT_NOOP_EXE=prog/weavewait/weavewait_noop
 NETCHECK_EXE=prog/netcheck/netcheck
-COVER_EXE=testing/cover/cover
-RUNNER_EXE=testing/runner/runner
+DOCKERTLSARGS_EXE=prog/docker_tls_args/docker_tls_args
+RUNNER_EXE=tools/runner/runner
 
-EXES=$(WEAVER_EXE) $(SIGPROXY_EXE) $(WEAVEPROXY_EXE) $(WEAVEWAIT_EXE) $(WEAVEHOSTS_EXE) $(NETCHECK_EXE) $(COVER_EXE) $(RUNNER_EXE)
+EXES=$(WEAVER_EXE) $(SIGPROXY_EXE) $(WEAVEPROXY_EXE) $(WEAVEWAIT_EXE) $(WEAVEWAIT_NOOP_EXE) $(NETCHECK_EXE) $(DOCKERTLSARGS_EXE)
 
 WEAVER_UPTODATE=.weaver.uptodate
 WEAVEEXEC_UPTODATE=.weaveexec.uptodate
@@ -30,9 +30,9 @@ WEAVEEXEC_IMAGE=$(DOCKERHUB_USER)/weaveexec
 
 IMAGES=$(WEAVER_IMAGE) $(WEAVEEXEC_IMAGE)
 
-WEAVE_EXPORT=weave.tar
+WEAVE_EXPORT=weave.tar.gz
 
-WEAVEEXEC_DOCKER_VERSION=1.3.1
+WEAVEEXEC_DOCKER_VERSION=1.6.2
 DOCKER_DISTRIB=prog/weaveexec/docker-$(WEAVEEXEC_DOCKER_VERSION).tgz
 DOCKER_DISTRIB_URL=https://get.docker.com/builds/Linux/x86_64/docker-$(WEAVEEXEC_DOCKER_VERSION).tgz
 NETGO_CHECK=@strings $@ | grep cgo_stub\\\.go >/dev/null || { \
@@ -45,7 +45,9 @@ NETGO_CHECK=@strings $@ | grep cgo_stub\\\.go >/dev/null || { \
 }
 BUILD_FLAGS=-ldflags "-extldflags \"-static\" -X main.version $(WEAVE_VERSION)" -tags netgo
 
-all: $(WEAVE_EXPORT) $(COVER_EXE) $(RUNNER_EXE)
+PACKAGE_BASE=$(shell go list -e ./)
+
+all: $(WEAVE_EXPORT) $(RUNNER_EXE)
 
 travis: $(EXES)
 
@@ -54,7 +56,7 @@ update:
 
 $(WEAVER_EXE) $(WEAVEPROXY_EXE): common/*.go common/*/*.go net/*.go
 ifeq ($(COVERAGE),true)
-	$(eval COVERAGE_MODULES := $(shell (go list ./$(@D); go list -f '{{join .Deps "\n"}}' ./$(@D) | grep "weaveworks") | paste -s -d,))
+	$(eval COVERAGE_MODULES := $(shell (go list ./$(@D); go list -f '{{join .Deps "\n"}}' ./$(@D) | grep "^$(PACKAGE_BASE)/") | paste -s -d,))
 	go get -t -tags netgo ./$(@D)
 	go test -c -o ./$@ $(BUILD_FLAGS) -v -covermode=atomic -coverpkg $(COVERAGE_MODULES) ./$(@D)/
 else
@@ -68,45 +70,57 @@ $(NETCHECK_EXE): common/*.go common/*/*.go net/*.go
 	go build $(BUILD_FLAGS) -o $@ ./$(@D)
 	$(NETGO_CHECK)
 
-$(WEAVER_EXE): router/*.go ipam/*.go ipam/*/*.go nameserver/*.go prog/weaver/*.go
+$(WEAVER_EXE): router/*.go mesh/*.go ipam/*.go ipam/*/*.go nameserver/*.go prog/weaver/*.go
 $(WEAVEPROXY_EXE): proxy/*.go prog/weaveproxy/main.go
 $(NETCHECK_EXE): prog/netcheck/netcheck.go
 
 # Sigproxy and weavewait need separate rules as they fail the netgo check in
 # the main build stanza due to not importing net package
 $(SIGPROXY_EXE): prog/sigproxy/main.go
-$(WEAVEWAIT_EXE): prog/weavewait/main.go net/*.go
-$(WEAVEHOSTS_EXE): prog/weavehosts/weavehosts.go
-$(COVER_EXE): testing/cover/cover.go
-$(RUNNER_EXE): testing/runner/runner.go
+$(WEAVEWAIT_EXE): prog/weavewait/*.go net/*.go
+$(DOCKERTLSARGS_EXE): prog/docker_tls_args/*.go
 
-$(WEAVEWAIT_EXE) $(SIGPROXY_EXE) $(WEAVEHOSTS_EXE) $(COVER_EXE) $(RUNNER_EXE):
+$(WEAVEWAIT_EXE) $(SIGPROXY_EXE) $(DOCKERTLSARGS_EXE):
 	go get -tags netgo ./$(@D)
 	go build $(BUILD_FLAGS) -o $@ ./$(@D)
 
+$(WEAVEWAIT_NOOP_EXE): prog/weavewait/*.go
+	go get -tags netgo ./$(@D)
+	go build $(BUILD_FLAGS) -tags noop -o $@ ./$(@D)
+
 $(WEAVER_UPTODATE): prog/weaver/Dockerfile $(WEAVER_EXE)
 	$(SUDO) docker build -t $(WEAVER_IMAGE) prog/weaver
 	touch $@
 
-$(WEAVEEXEC_UPTODATE): prog/weaveexec/Dockerfile $(DOCKER_DISTRIB) weave $(SIGPROXY_EXE) $(WEAVEPROXY_EXE) $(WEAVEWAIT_EXE) $(WEAVEHOSTS_EXE) $(NETCHECK_EXE)
+$(WEAVEEXEC_UPTODATE): prog/weaveexec/Dockerfile $(DOCKER_DISTRIB) weave $(SIGPROXY_EXE) $(WEAVEPROXY_EXE) $(WEAVEWAIT_EXE) $(WEAVEWAIT_NOOP_EXE) $(NETCHECK_EXE) $(DOCKERTLSARGS_EXE)
 	cp weave prog/weaveexec/weave
 	cp $(SIGPROXY_EXE) prog/weaveexec/sigproxy
 	cp $(WEAVEPROXY_EXE) prog/weaveexec/weaveproxy
 	cp $(WEAVEWAIT_EXE) prog/weaveexec/weavewait
-	cp $(WEAVEHOSTS_EXE) prog/weaveexec/weavehosts
+	cp $(WEAVEWAIT_NOOP_EXE) prog/weaveexec/weavewait_noop
 	cp $(NETCHECK_EXE) prog/weaveexec/netcheck
+	cp $(DOCKERTLSARGS_EXE) prog/weaveexec/docker_tls_args
 	cp $(DOCKER_DISTRIB) prog/weaveexec/docker.tgz
 	$(SUDO) docker build -t $(WEAVEEXEC_IMAGE) prog/weaveexec
 	touch $@
 
 $(WEAVE_EXPORT): $(IMAGES_UPTODATE)
-	$(SUDO) docker save $(addsuffix :latest,$(IMAGES)) > $@
+	$(SUDO) docker save $(addsuffix :latest,$(IMAGES)) | gzip > $@
 
 $(DOCKER_DISTRIB):
 	curl -o $(DOCKER_DISTRIB) $(DOCKER_DISTRIB_URL)
 
-tests: $(COVER_EXE)
-	@test/units.sh
+tests: tools/.git
+	tools/test
+
+lint: tools/.git
+	tools/lint -nocomment -notestpackage .
+
+tools/.git:
+	git submodule update --init
+
+$(RUNNER_EXE): tools/.git
+	make -C tools/runner
 
 $(PUBLISH): publish_%: $(IMAGES_UPTODATE)
 	$(SUDO) docker tag -f $(DOCKERHUB_USER)/$* $(DOCKERHUB_USER)/$*:$(WEAVE_VERSION)
@@ -117,10 +131,12 @@ endif
 
 publish: $(PUBLISH)
 
-clean:
+clean-bin:
 	-$(SUDO) docker rmi $(IMAGES)
-	go clean -r ./...
+	go clean -r $(addprefix ./,$(dir $(EXES)))
 	rm -f $(EXES) $(IMAGES_UPTODATE) $(WEAVE_EXPORT)
+
+clean: clean-bin
 	rm -rf test/tls/*.pem test/coverage.* test/coverage
 
 build:

README.md

@@ -7,11 +7,9 @@
 [Weaveworks](http://weave.works) is the company that develops Weave -
 the most productive way for developers to connect, observe and control
 Docker containers. To learn about our products, including getting
-started tutorials, visit our [website](http://weave.works), read about
-the
-[1.0 release of Weave](http://blog.weave.works/2015/06/18/weave-1-0-is-out-and-it-is-good/),
-visit our [documentation](http://docs.weave.works) or continue to read
-about some of the more technical aspects of Weave in this readme.
+started tutorials, visit our [website](http://weave.works) and
+[documentation](http://docs.weave.works) or continue to read about
+some of the more technical aspects of Weave in this readme.
 
 # Weave
 
@@ -43,19 +41,29 @@ capabilities, so these can continue to be used by containers.
 ## Installation
 
 Ensure you are running Linux (kernel 3.8 or later) and have Docker
-(version 1.3.1 or later) installed. Then install weave with
+(version 1.6.0 or later) installed. Then install weave with
 
     sudo curl -L git.io/weave -o /usr/local/bin/weave
     sudo chmod a+x /usr/local/bin/weave
 
+For usage on OSX (with Docker Machine) you first need to make sure
+that a VM is running and configured, as shown in [the Docker Machine
+documentation](https://docs.docker.com/installation/mac/#from-your-shell).
+Then you can launch weave directly from the OSX host.
+
 CoreOS users see [here](https://github.com/fintanr/weave-gs/blob/master/coreos-simple/user-data) for an example of installing weave using cloud-config.
 
+Amazon ECS users see
+[here](https://github.com/weaveworks/guides/blob/master/aws-ecs/LATESTAMIs.md)
+for the latest Weave AMIs and
+[here](http://weave.works/guides/service-discovery-with-weave-aws-ecs.html) to get started with Weave on ECS.
+
 Weave respects the environment variable `DOCKER_HOST`, so you can run
 it locally to control a weave network on a remote host.
 
 ## Quick Start Screencast
 
-<a href="https://youtu.be/BF9YJSPQUcY" alt="Click to watch the screencast" target="_blank">
+<a href="https://youtu.be/kihQCCT1ykE" alt="Click to watch the screencast" target="_blank">
   <img src="/docs/hello-screencast.png" />
 </a>
 
@@ -99,9 +107,8 @@ The only difference, apart from the name of the application container,
 is that we tell our weave that it should peer with the weave on
 `$HOST1` (specified as the IP address or hostname, and optional
 `:port`, by which `$HOST2` can reach it). NB: if there is a firewall
-between `$HOST1` and `$HOST2`, you must open the weave port (6783 by
-default; this can be overriden by setting `WEAVE_PORT`) for TCP and
-UDP.
+between `$HOST1` and `$HOST2`, you must permit traffic to the weave
+control port (TCP 6783) and data ports (UDP 6783/6784).
 
 Note that we could instead have told the weave on `$HOST1` to connect to
 `$HOST2`, or told both about each other. Order does not matter here;
@@ -148,12 +155,16 @@ to each other.
 
 ## Contact Us
 
-Found a bug, want to suggest a feature, or have a question?
-[File an issue](https://github.com/weaveworks/weave/issues), or email
-help@weave.works. When reporting a bug, please include which version of
-weave you are running, as shown by `weave version`.
+Found a bug, want to suggest a feature, or have a question?  You can
+[File an issue](https://github.com/weaveworks/weave/issues).  And
+there is a [Weave Users Google
+Group](https://groups.google.com/a/weave.works/forum/#!forum/weave-users),
+which you can post to by email at weave-users@weave.works.  Or you can
+contact Weaveworks directly at help@weave.works. When reporting a
+bug, please include which version of weave you are running, as shown
+by `weave version`.
 
-Follow weave on Twitter:
+Follow on Twitter:
 [@weaveworks](https://twitter.com/weaveworks).
 
 Read the Weave blog:

bin/lint

@@ -1,86 +0,0 @@
-#!/bin/bash
-# This scipt lints go files for common errors.
-#
-# Its runs gofmt and go vet, and optionally golint and
-# gocyclo, if they are installed.
-#
-# With no arguments, it lints the current files staged
-# for git commit.  Or you can pass it explicit filenames
-# (or directories) and it will lint them.
-#
-# To use this script automatically, run:
-#   ln -s ../../bin/lint .git/hooks/pre-commit
-
-set -eu
-
-function lint_go {
-	filename="$1"
-	result=0
-
-	if [ -n "$(gofmt -l "${filename}")" ]; then
-		result=1
-		echo "${filename}: run gofmt -w ${filename}!"
-	fi
-
-	go tool vet "${filename}" || result=$?
-
-	# golint is completely optional.  If you don't like it
-	# don't have it installed.
-	if type golint >/dev/null 2>&1; then
-		# golint doesn't set an exit code it seems
-		lintoutput=$(golint "${filename}" | grep -vE 'comment|dot imports')
-		if [ "$lintoutput" != "" ]; then
-			result=1
-			echo "$lintoutput"
-		fi
-	fi
-
-	# gocyclo is completely optional.  If you don't like it
-	# don't have it installed.  Also never blocks a commit,
-	# it just warns.
-	if type gocyclo >/dev/null 2>&1; then
-		gocyclo -over 25 "${filename}" | while read line; do
-			echo "${filename}": higher than 25 cyclomatic complexity - "${line}"
-		done
-	fi
-
-	return $result
-}
-
-function lint {
-	filename="$1"
-	ext="${filename##*\.}"
-
-	# Don't lint this script
-	if [ "${filename}" = "bin/lint" ]; then
-		return
-	fi
-
-	# Don't lint files that don't exist
-	if ! [ -a "${filename}" ]; then
-		return
-	fi
-
-	case "$ext" in
-		go) lint_go "${filename}"
-		;;
-	esac
-}
-
-function lint_files {
-	lint_result=0
-	while read filename; do
-		lint "${filename}" || lint_result=1
-	done
-	exit $lint_result
-}
-
-function list_files {
-	if [ $# -gt 0 ]; then
-		find "$@" -type f
-	else
-		git diff --cached --name-only
-	fi
-}
-
-list_files "$@" | lint_files

build/Dockerfile

@@ -3,7 +3,7 @@ FROM ubuntu
 RUN apt-get -y update && apt-get -y install --no-install-recommends ca-certificates apt-transport-https
 RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 36A1D7869245C8950F966E92D8576A8BA88D21E9
 RUN echo deb https://get.docker.io/ubuntu docker main > /etc/apt/sources.list.d/docker.list
-RUN apt-get -y update && apt-get -y install --no-install-recommends build-essential git lxc-docker-1.3.1 mercurial libpcap-dev curl make pkg-config gcc bison flex python-requests
+RUN apt-get -y update && apt-get -y install --no-install-recommends build-essential git lxc-docker-1.6.2 mercurial libpcap-dev curl make pkg-config gcc bison flex python-requests
 
 # When doing a build in a container, "apt-get update" happens twice,
 # which can be a very significant overhead for incremental builds.

build/build.sh

@@ -36,11 +36,7 @@ EOF
     echo "weave:*:::::::" >>/etc/shadow
     echo "weave	ALL=(ALL)	NOPASSWD: ALL" >>/etc/sudoers
 
-    if [ "$1" = "tests" ] ; then
-        su weave -c "PATH=$PATH make -C $WEAVE_SRC tests"
-    else
-        su weave -c "PATH=$PATH make -C $WEAVE_SRC build"
-    fi
+    su weave -c "PATH=$PATH make -C $WEAVE_SRC build"
 else
     # There are arguments to pass to git-clone
     mkdir -p ${WEAVE_SRC%/*}

circle.yml

@@ -12,41 +12,45 @@ machine:
     SRCDIR: /home/ubuntu/src/github.com/weaveworks/weave
     PATH: $PATH:$HOME/.local/bin:$HOME/google-cloud-sdk/bin/
     CLOUDSDK_CORE_DISABLE_PROMPTS: 1
-    WEAVE_BUILD: $HOME/docker/weave-build.tar
+    WEAVE_TEST: $HOME/docker/weave-test.tar
 
 dependencies:
   cache_directories:
     - "~/docker"
   override:
-    - if [[ -e "$WEAVE_BUILD" ]]; then
-        docker load -i $WEAVE_BUILD;
+    - if [[ -e "$WEAVE_TEST" ]]; then
+        docker load -i $WEAVE_TEST;
       else
         docker build -t weaveworks/weave-build build;
-        mkdir -p $(dirname "$WEAVE_BUILD");
-        docker save weaveworks/weave-build >$WEAVE_BUILD;
+        docker build -t weaveworks/weave-test test/image;
+        mkdir -p $(dirname "$WEAVE_TEST");
+        docker save weaveworks/weave-test >$WEAVE_TEST;
       fi
   post:
-    - sudo apt-get update
-    - sudo apt-get install bc jq
-    - pip install requests
+    - sudo apt-get update && sudo apt-get install bc jq && pip install requests
     - curl https://sdk.cloud.google.com | bash
     - bin/setup-circleci-secrets "$SECRET_PASSWORD"
-    - mkdir -p $(dirname $SRCDIR)
-    - cp -r $(pwd)/ $SRCDIR
+    - mkdir -p $(dirname $SRCDIR) && cp -r $(pwd)/ $SRCDIR
+    - cd $SRCDIR/test; ./gce.sh make_template:
+        parallel: false
 
 test:
   override:
-    - docker run -v /var/run/docker.sock:/run/docker.sock -v /home/ubuntu:/home/go -e COVERDIR=test/coverage -e SLOW=true -e CIRCLECI -e CIRCLE_BUILD_NUM -e CIRCLE_NODE_TOTAL -e CIRCLE_NODE_INDEX weaveworks/weave-build tests:
+    - cd $SRCDIR; make lint:
+        parallel: true
+    - docker run -v /home/ubuntu:/home/go -e COVERDIR=test/coverage -e SLOW=true -e CIRCLECI -e CIRCLE_BUILD_NUM -e CIRCLE_NODE_TOTAL -e CIRCLE_NODE_INDEX weaveworks/weave-test:
         parallel: true
     - docker run -v /var/run/docker.sock:/run/docker.sock -v /home/ubuntu:/home/go -e COVERAGE=true weaveworks/weave-build:
         parallel: true
-    - cd $SRCDIR/test; ./gce.sh setup:
-        parallel: true
-    - cd $SRCDIR/test; eval $(./gce.sh hosts); ./setup.sh:
+    - cd $SRCDIR/test; ./gce.sh setup && eval $(./gce.sh hosts) && ./setup.sh:
         parallel: true
     - cd $SRCDIR/test; eval $(./gce.sh hosts); export COVERAGE=true; ./run_all.sh:
         parallel: true
         timeout: 300
+    - cd $SRCDIR; make clean-bin:
+        parallel: true
+    - docker run -v /var/run/docker.sock:/run/docker.sock -v /home/ubuntu:/home/go weaveworks/weave-build:
+        parallel: true
   post:
     - cd $SRCDIR/test; ./gce.sh destroy:
         parallel: true
@@ -59,3 +63,11 @@ teardown:
         parallel: true
     - test "$CIRCLE_NODE_INDEX" != "0" || (cd $SRCDIR/test; cp coverage.* $CIRCLE_ARTIFACTS):
         parallel: true
+
+deployment:
+  hub:
+    branch: master
+    owner: weaveworks
+    commands:
+      - docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS
+      - cd $SRCDIR; make publish

common/docker/client.go

@@ -7,6 +7,7 @@ import (
 
 // An observer for container events
 type ContainerObserver interface {
+	ContainerStarted(ident string)
 	ContainerDied(ident string)
 }
 
@@ -22,12 +23,26 @@ func NewClient(apiPath string) (*Client, error) {
 	}
 	client := &Client{dc}
 
-	env, err := client.Version()
+	return client, client.checkWorking(apiPath)
+}
+
+func NewVersionedClient(apiPath string, apiVersionString string) (*Client, error) {
+	dc, err := docker.NewVersionedClient(apiPath, apiVersionString)
 	if err != nil {
 		return nil, err
 	}
+	client := &Client{dc}
+
+	return client, client.checkWorking(apiPath)
+}
+
+func (c *Client) checkWorking(apiPath string) error {
+	env, err := c.Version()
+	if err != nil {
+		return err
+	}
 	Log.Infof("[docker] Using Docker API on %s: %v", apiPath, env)
-	return client, nil
+	return nil
 }
 
 // AddObserver adds an observer for docker events
@@ -41,6 +56,9 @@ func (c *Client) AddObserver(ob ContainerObserver) error {
 	go func() {
 		for event := range events {
 			switch event.Status {
+			case "start":
+				id := event.ID
+				ob.ContainerStarted(id)
 			case "die":
 				id := event.ID
 				ob.ContainerDied(id)

common/logging.go

@@ -52,3 +52,15 @@ func SetLogLevel(levelname string) {
 	}
 	Log.Level = level
 }
+
+func CheckFatal(e error) {
+	if e != nil {
+		Log.Fatal(e)
+	}
+}
+
+func CheckWarn(e error) {
+	if e != nil {
+		Log.Warnln(e)
+	}
+}

common/odp/odp.go

@@ -0,0 +1,93 @@
+package odp
+
+import (
+	"fmt"
+	"net"
+	"syscall"
+
+	"github.com/weaveworks/go-odp/odp"
+)
+
+// ODP admin functionality
+
+func CreateDatapath(dpname string) (err error, supported bool) {
+	dpif, err := odp.NewDpif()
+	if err != nil {
+		if odp.IsKernelLacksODPError(err) {
+			return nil, false
+		}
+
+		return err, true
+	}
+
+	defer dpif.Close()
+
+	dp, err := dpif.CreateDatapath(dpname)
+	if err != nil && !odp.IsDatapathNameAlreadyExistsError(err) {
+		return err, true
+	}
+
+	// Pick an ephemeral port number to use in probing for vxlan
+	// support.
+	udpconn, err := net.ListenUDP("udp4", nil)
+	if err != nil {
+		return err, true
+	}
+
+	// we leave the UDP socket open, so creating a vxlan vport on
+	// the same port number should fail.  But that's fine: It's
+	// still sufficient to probe for support.
+	portno := uint16(udpconn.LocalAddr().(*net.UDPAddr).Port)
+	vpid, err := dp.CreateVport(odp.NewVxlanVportSpec(
+		fmt.Sprintf("vxlan-%d", portno), portno))
+	if nlerr, ok := err.(odp.NetlinkError); ok {
+		if syscall.Errno(nlerr) == syscall.EAFNOSUPPORT {
+			dp.Delete()
+			return fmt.Errorf("kernel does not have Open vSwitch VXLAN support"), false
+		}
+	}
+
+	if err == nil {
+		dp.DeleteVport(vpid)
+	}
+
+	udpconn.Close()
+	return nil, true
+}
+
+func DeleteDatapath(dpname string) error {
+	dpif, err := odp.NewDpif()
+	if err != nil {
+		return err
+	}
+
+	defer dpif.Close()
+
+	dp, err := dpif.LookupDatapath(dpname)
+	if err != nil {
+		if odp.IsNoSuchDatapathError(err) {
+			return nil
+		}
+
+		return err
+	}
+
+	return dp.Delete()
+}
+
+func AddDatapathInterface(dpname string, ifname string) error {
+	dpif, err := odp.NewDpif()
+	if err != nil {
+		return err
+	}
+
+	defer dpif.Close()
+
+	dp, err := dpif.LookupDatapath(dpname)
+	if err != nil {
+		return err
+	}
+
+	_, err = dp.CreateVport(odp.NewNetdevVportSpec(ifname))
+	return err
+}

common/signals.go

@@ -14,7 +14,7 @@ type SignalReceiver interface {
 
 func SignalHandlerLoop(ss ...SignalReceiver) {
 	sigs := make(chan os.Signal, 1)
-	signal.Notify(sigs, syscall.SIGINT, syscall.SIGQUIT, syscall.SIGUSR1, syscall.SIGTERM)
+	signal.Notify(sigs, syscall.SIGINT, syscall.SIGQUIT, syscall.SIGTERM)
 	buf := make([]byte, 1<<20)
 	for {
 		switch <-sigs {

docs/architecture.txt

@@ -66,23 +66,23 @@ Router:
     simply responds to traffic received from the remote peer via TCP
   5. We register this connection with the local peer
   6a. If we initiated this connection then we now start sending fast
-    heartbeats to the remote peer so that the remote peer can determine what
-    address/port it should use to send UDP back to us. To do this, we
-    spawn off two further threads which are the forwarder
-    loops. These receive frames which are to be sent to the remote
-    peer. One of them sends frames without the DF flag set. To do this
-    it just sends the packets out of the UDP Listener socket. The
-    other needs to send frames with the DF flag set and needs its own
-    socket so that it can do PMTU discovery easily. To do this, it
-    uses a Raw IP socket (IP has no ports, so there's no collision
-    issue with the UDP Listener socket) and so it must add UDP headers
-    itself.
-  6b. If we did not initiate this connection then the UDP Listener
+    heartbeats to the remote peer so that the remote peer can
+    determine what address/port it should use to send UDP back to
+    us. To do this, we spawn off a "forwarder" thread to send
+    heartbeats, monitor incoming heartbeats, and some other auxiliary
+    duties.  It also consumes frames to be encapsulated and send via
+    UDP from two channels, for DF and non-DF cases.  In the non-DF
+    case, it can just send the packets out of the UDP Listener
+    socket. In the DF case, it needs its own socket so that it can do
+    PMTU discovery easily. To do this, it uses a Raw IP socket (IP has
+    no ports, so there's no collision issue with the UDP Listener
+    socket) and so it must add UDP headers itself.
+ 6b. If we did not initiate this connection then the UDP Listener
     should start receiving fast heartbeats from the remote peer. From
     those it should be able to identify the local connection via the
     local peer. It will tell the local connection (communicating to
     the actor thread) about the UDP address of the remote peer. The
-    local connection will then start its forwarder threads as
+    local connection will then start its forwarder thread as
     described in 6a, and start sending fast heartbeats. We send to the
     remote peer via TCP a ConnectionEstablished message. The remote
     peer receives this (on the TCP receiver process), tells the

ipam/allocator.go

@@ -11,8 +11,8 @@ import (
 	"github.com/weaveworks/weave/ipam/paxos"
 	"github.com/weaveworks/weave/ipam/ring"
 	"github.com/weaveworks/weave/ipam/space"
+	"github.com/weaveworks/weave/mesh"
 	"github.com/weaveworks/weave/net/address"
-	"github.com/weaveworks/weave/router"
 )
 
 // Kinds of message we can unicast to other peers
@@ -43,37 +43,39 @@ type operation interface {
 // are used around data structures.
 type Allocator struct {
 	actionChan       chan<- func()
-	ourName          router.PeerName
+	ourName          mesh.PeerName
 	universe         address.Range                // superset of all ranges
 	ring             *ring.Ring                   // information on ranges owned by all peers
 	space            space.Space                  // more detail on ranges owned by us
 	owned            map[string][]address.Address // who owns what addresses, indexed by container-ID
-	nicknames        map[router.PeerName]string   // so we can map nicknames for rmpeer
+	nicknames        map[mesh.PeerName]string     // so we can map nicknames for rmpeer
 	pendingAllocates []operation                  // held until we get some free space
 	pendingClaims    []operation                  // held until we know who owns the space
-	gossip           router.Gossip                // our link to the outside world for sending messages
+	gossip           mesh.Gossip                  // our link to the outside world for sending messages
 	paxos            *paxos.Node
 	paxosTicker      *time.Ticker
 	shuttingDown     bool // to avoid doing any requests while trying to shut down
+	isKnownPeer      func(mesh.PeerName) bool
 	now              func() time.Time
 }
 
 // NewAllocator creates and initialises a new Allocator
-func NewAllocator(ourName router.PeerName, ourUID router.PeerUID, ourNickname string, universe address.Range, quorum uint) *Allocator {
+func NewAllocator(ourName mesh.PeerName, ourUID mesh.PeerUID, ourNickname string, universe address.Range, quorum uint, isKnownPeer func(name mesh.PeerName) bool) *Allocator {
 	return &Allocator{
-		ourName:   ourName,
-		universe:  universe,
-		ring:      ring.New(universe.Start, universe.End, ourName),
-		owned:     make(map[string][]address.Address),
-		paxos:     paxos.NewNode(ourName, ourUID, quorum),
-		nicknames: map[router.PeerName]string{ourName: ourNickname},
-		now:       time.Now,
+		ourName:     ourName,
+		universe:    universe,
+		ring:        ring.New(universe.Start, universe.End, ourName),
+		owned:       make(map[string][]address.Address),
+		paxos:       paxos.NewNode(ourName, ourUID, quorum),
+		nicknames:   map[mesh.PeerName]string{ourName: ourNickname},
+		isKnownPeer: isKnownPeer,
+		now:         time.Now,
 	}
 }
 
 // Start runs the allocator goroutine
 func (alloc *Allocator) Start() {
-	actionChan := make(chan func(), router.ChannelSize)
+	actionChan := make(chan func(), mesh.ChannelSize)
 	alloc.actionChan = actionChan
 	go alloc.actorLoop(actionChan)
 }
@@ -163,7 +165,7 @@ func (alloc *Allocator) tryPendingOps() {
 	}
 }
 
-func (alloc *Allocator) spaceRequestDenied(sender router.PeerName, r address.Range) {
+func (alloc *Allocator) spaceRequestDenied(sender mesh.PeerName, r address.Range) {
 	for i := 0; i < len(alloc.pendingClaims); {
 		claim := alloc.pendingClaims[i].(*claim)
 		if r.Contains(claim.addr) {
@@ -175,17 +177,6 @@ func (alloc *Allocator) spaceRequestDenied(sender router.PeerName, r address.Ran
 	}
 }
 
-func hasBeenCancelled(cancelChan <-chan bool) func() bool {
-	return func() bool {
-		select {
-		case <-cancelChan:
-			return true
-		default:
-			return false
-		}
-	}
-}
-
 type errorCancelled struct {
 	kind  string
 	ident string
@@ -199,10 +190,9 @@ func (e *errorCancelled) Error() string {
 
 // Allocate (Sync) - get new IP address for container with given name in range
 // if there isn't any space in that range we block indefinitely
-func (alloc *Allocator) Allocate(ident string, r address.Range, cancelChan <-chan bool) (address.Address, error) {
+func (alloc *Allocator) Allocate(ident string, r address.Range, hasBeenCancelled func() bool) (address.Address, error) {
 	resultChan := make(chan allocateResult)
-	op := &allocate{resultChan: resultChan, ident: ident, r: r,
-		hasBeenCancelled: hasBeenCancelled(cancelChan)}
+	op := &allocate{resultChan: resultChan, ident: ident, r: r, hasBeenCancelled: hasBeenCancelled}
 	alloc.doOperation(op, &alloc.pendingAllocates)
 	result := <-resultChan
 	return result.addr, result.err
@@ -237,6 +227,8 @@ func (alloc *Allocator) ContainerDied(ident string) {
 	}
 }
 
+func (alloc *Allocator) ContainerStarted(ident string) {}
+
 // Delete (Sync) - release all IP addresses for container with given name
 func (alloc *Allocator) Delete(ident string) error {
 	errChan := make(chan error)
@@ -284,6 +276,33 @@ func (alloc *Allocator) Free(ident string, addrToFree address.Address) error {
 	return <-errChan
 }
 
+func (alloc *Allocator) pickPeerFromNicknames(isValid func(mesh.PeerName) bool) mesh.PeerName {
+	for name := range alloc.nicknames {
+		if name != alloc.ourName && isValid(name) {
+			return name
+		}
+	}
+	return mesh.UnknownPeerName
+}
+
+func (alloc *Allocator) pickPeerForTransfer() mesh.PeerName {
+	// first try alive peers that actively participate in IPAM (i.e. have entries)
+	if heir := alloc.ring.PickPeerForTransfer(alloc.isKnownPeer); heir != mesh.UnknownPeerName {
+		return heir
+	}
+	// next try alive peers that have IPAM enabled but have no entries
+	if heir := alloc.pickPeerFromNicknames(alloc.isKnownPeer); heir != mesh.UnknownPeerName {
+		return heir
+	}
+	// next try disappeared peers that still have entries
+	t := func(mesh.PeerName) bool { return true }
+	if heir := alloc.ring.PickPeerForTransfer(t); heir != mesh.UnknownPeerName {
+		return heir
+	}
+	// finally, disappeared peers that that passively participated in IPAM
+	return alloc.pickPeerFromNicknames(t)
+}
+
 // Shutdown (Sync)
 func (alloc *Allocator) Shutdown() {
 	alloc.infof("Shutdown")
@@ -292,7 +311,7 @@ func (alloc *Allocator) Shutdown() {
 		alloc.shuttingDown = true
 		alloc.cancelOps(&alloc.pendingClaims)
 		alloc.cancelOps(&alloc.pendingAllocates)
-		if heir := alloc.ring.PickPeerForTransfer(); heir != router.UnknownPeerName {
+		if heir := alloc.pickPeerForTransfer(); heir != mesh.UnknownPeerName {
 			alloc.ring.Transfer(alloc.ourName, heir)
 			alloc.space.Clear()
 			alloc.gossip.GossipBroadcast(alloc.Gossip())
@@ -320,7 +339,6 @@ func (alloc *Allocator) AdminTakeoverRanges(peerNameOrNickname string) error {
 			return
 		}
 
-		delete(alloc.nicknames, peername)
 		newRanges, err := alloc.ring.Transfer(peername, alloc.ourName)
 		alloc.space.AddRanges(newRanges)
 		resultChan <- err
@@ -332,33 +350,45 @@ func (alloc *Allocator) AdminTakeoverRanges(peerNameOrNickname string) error {
 // call into the router for this because we are interested in peers
 // that have gone away but are still in the ring, which is why we
 // maintain our own nicknames map.
-func (alloc *Allocator) lookupPeername(name string) (router.PeerName, error) {
+func (alloc *Allocator) lookupPeername(name string) (mesh.PeerName, error) {
 	for peername, nickname := range alloc.nicknames {
 		if nickname == name {
 			return peername, nil
 		}
 	}
 
-	return router.PeerNameFromString(name)
+	return mesh.PeerNameFromString(name)
 }
 
-// Restrict the peers in "nicknames" to those in the ring and our own
+// Restrict the peers in "nicknames" to those in the ring plus peers known to the router
 func (alloc *Allocator) pruneNicknames() {
 	ringPeers := alloc.ring.PeerNames()
 	for name := range alloc.nicknames {
-		if _, ok := ringPeers[name]; !ok && name != alloc.ourName {
+		if _, ok := ringPeers[name]; !ok && !alloc.isKnownPeer(name) {
 			delete(alloc.nicknames, name)
 		}
 	}
 }
 
+func (alloc *Allocator) annotatePeernames(names []mesh.PeerName) []string {
+	var res []string
+	for _, name := range names {
+		if nickname, found := alloc.nicknames[name]; found {
+			res = append(res, fmt.Sprint(name, "(", nickname, ")"))
+		} else {
+			res = append(res, name.String())
+		}
+	}
+	return res
+}
+
 func decodeRange(msg []byte) (r address.Range, err error) {
 	decoder := gob.NewDecoder(bytes.NewReader(msg))
 	return r, decoder.Decode(&r)
 }
 
 // OnGossipUnicast (Sync)
-func (alloc *Allocator) OnGossipUnicast(sender router.PeerName, msg []byte) error {
+func (alloc *Allocator) OnGossipUnicast(sender mesh.PeerName, msg []byte) error {
 	alloc.debugln("OnGossipUnicast from", sender, ": ", len(msg), "bytes")
 	resultChan := make(chan error)
 	alloc.actionChan <- func() {
@@ -384,7 +414,7 @@ func (alloc *Allocator) OnGossipUnicast(sender router.PeerName, msg []byte) erro
 }
 
 // OnGossipBroadcast (Sync)
-func (alloc *Allocator) OnGossipBroadcast(sender router.PeerName, msg []byte) (router.GossipData, error) {
+func (alloc *Allocator) OnGossipBroadcast(sender mesh.PeerName, msg []byte) (mesh.GossipData, error) {
 	alloc.debugln("OnGossipBroadcast from", sender, ":", len(msg), "bytes")
 	resultChan := make(chan error)
 	alloc.actionChan <- func() {
@@ -397,7 +427,7 @@ type gossipState struct {
 	// We send a timstamp along with the information to be
 	// gossipped in order to detect skewed clocks
 	Now       int64
-	Nicknames map[router.PeerName]string
+	Nicknames map[mesh.PeerName]string
 
 	Paxos paxos.GossipState
 	Ring  *ring.Ring
@@ -433,11 +463,11 @@ func (alloc *Allocator) Encode() []byte {
 }
 
 // OnGossip (Sync)
-func (alloc *Allocator) OnGossip(msg []byte) (router.GossipData, error) {
+func (alloc *Allocator) OnGossip(msg []byte) (mesh.GossipData, error) {
 	alloc.debugln("Allocator.OnGossip:", len(msg), "bytes")
 	resultChan := make(chan error)
 	alloc.actionChan <- func() {
-		resultChan <- alloc.update(router.UnknownPeerName, msg)
+		resultChan <- alloc.update(mesh.UnknownPeerName, msg)
 	}
 	return nil, <-resultChan // for now, we never propagate updates. TBD
 }
@@ -448,8 +478,8 @@ type ipamGossipData struct {
 	alloc *Allocator
 }
 
-func (d *ipamGossipData) Merge(other router.GossipData) {
-	// no-op
+func (d *ipamGossipData) Merge(other mesh.GossipData) mesh.GossipData {
+	return d // no-op
 }
 
 func (d *ipamGossipData) Encode() [][]byte {
@@ -458,12 +488,12 @@ func (d *ipamGossipData) Encode() [][]byte {
 
 // Gossip returns a GossipData implementation, which in this case always
 // returns the latest ring state (and does nothing on merge)
-func (alloc *Allocator) Gossip() router.GossipData {
+func (alloc *Allocator) Gossip() mesh.GossipData {
 	return &ipamGossipData{alloc}
 }
 
 // SetInterfaces gives the allocator two interfaces for talking to the outside world
-func (alloc *Allocator) SetInterfaces(gossip router.Gossip) {
+func (alloc *Allocator) SetInterfaces(gossip mesh.Gossip) {
 	alloc.gossip = gossip
 }
 
@@ -510,7 +540,7 @@ func (alloc *Allocator) establishRing() {
 	}
 }
 
-func (alloc *Allocator) createRing(peers []router.PeerName) {
+func (alloc *Allocator) createRing(peers []mesh.PeerName) {
 	alloc.debugln("Paxos consensus:", peers)
 	alloc.ring.ClaimForPeers(normalizeConsensus(peers))
 	alloc.gossip.GossipBroadcast(alloc.Gossip())
@@ -533,7 +563,7 @@ func (alloc *Allocator) ringUpdated() {
 }
 
 // For compatibility with sort.Interface
-type peerNames []router.PeerName
+type peerNames []mesh.PeerName
 
 func (a peerNames) Len() int           { return len(a) }
 func (a peerNames) Less(i, j int) bool { return a[i] < a[j] }
@@ -542,7 +572,7 @@ func (a peerNames) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
 // When we get a consensus from Paxos, the peer names are not in a
 // defined order and may contain duplicates.  This function sorts them
 // and de-dupes.
-func normalizeConsensus(consensus []router.PeerName) []router.PeerName {
+func normalizeConsensus(consensus []mesh.PeerName) []mesh.PeerName {
 	if len(consensus) == 0 {
 		return nil
 	}
@@ -577,22 +607,22 @@ func encodeRange(r address.Range) []byte {
 	return buf.Bytes()
 }
 
-func (alloc *Allocator) sendSpaceRequest(dest router.PeerName, r address.Range) error {
-	msg := router.Concat([]byte{msgSpaceRequest}, encodeRange(r))
+func (alloc *Allocator) sendSpaceRequest(dest mesh.PeerName, r address.Range) error {
+	msg := append([]byte{msgSpaceRequest}, encodeRange(r)...)
 	return alloc.gossip.GossipUnicast(dest, msg)
 }
 
-func (alloc *Allocator) sendSpaceRequestDenied(dest router.PeerName, r address.Range) error {
-	msg := router.Concat([]byte{msgSpaceRequestDenied}, encodeRange(r))
+func (alloc *Allocator) sendSpaceRequestDenied(dest mesh.PeerName, r address.Range) error {
+	msg := append([]byte{msgSpaceRequestDenied}, encodeRange(r)...)
 	return alloc.gossip.GossipUnicast(dest, msg)
 }
 
-func (alloc *Allocator) sendRingUpdate(dest router.PeerName) {
-	msg := router.Concat([]byte{msgRingUpdate}, alloc.encode())
+func (alloc *Allocator) sendRingUpdate(dest mesh.PeerName) {
+	msg := append([]byte{msgRingUpdate}, alloc.encode()...)
 	alloc.gossip.GossipUnicast(dest, msg)
 }
 
-func (alloc *Allocator) update(sender router.PeerName, msg []byte) error {
+func (alloc *Allocator) update(sender mesh.PeerName, msg []byte) error {
 	reader := bytes.NewReader(msg)
 	decoder := gob.NewDecoder(reader)
 	var data gossipState
@@ -616,16 +646,20 @@ func (alloc *Allocator) update(sender router.PeerName, msg []byte) error {
 	// shouldn't get updates for a empty Ring. But tolerate
 	// them just in case.
 	if data.Ring != nil {
-		if data.Ring.Range() != alloc.universe {
-			return fmt.Errorf("Incompatible IP allocation range %s; ours is %s",
-				data.Ring.Range().AsCIDRString(), alloc.universe.AsCIDRString())
+		switch err = alloc.ring.Merge(*data.Ring); err {
+		case ring.ErrDifferentSeeds:
+			return fmt.Errorf("IP allocation was seeded by different peers (received: %v, ours: %v)",
+				alloc.annotatePeernames(data.Ring.Seeds), alloc.annotatePeernames(alloc.ring.Seeds))
+		case ring.ErrDifferentRange:
+			return fmt.Errorf("Incompatible IP allocation ranges (received: %s, ours: %s)",
+				data.Ring.Range().AsCIDRString(), alloc.ring.Range().AsCIDRString())
+		default:
+			if err == nil && !alloc.ring.Empty() {
+				alloc.pruneNicknames()
+				alloc.ringUpdated()
+			}
+			return err
 		}
-		err = alloc.ring.Merge(*data.Ring)
-		if !alloc.ring.Empty() {
-			alloc.pruneNicknames()
-			alloc.ringUpdated()
-		}
-		return err
 	}
 
 	if data.Paxos != nil {
@@ -640,7 +674,7 @@ func (alloc *Allocator) update(sender router.PeerName, msg []byte) error {
 					alloc.createRing(cons.Value)
 				}
 			}
-		} else if sender != router.UnknownPeerName {
+		} else if sender != mesh.UnknownPeerName {
 			// Sender is trying to initialize a ring, but we have one
 			// already - send it straight back
 			alloc.sendRingUpdate(sender)
@@ -650,7 +684,7 @@ func (alloc *Allocator) update(sender router.PeerName, msg []byte) error {
 	return nil
 }
 
-func (alloc *Allocator) donateSpace(r address.Range, to router.PeerName) {
+func (alloc *Allocator) donateSpace(r address.Range, to mesh.PeerName) {
 	// No matter what we do, we'll send a unicast gossip
 	// of our ring back to tha chap who asked for space.
 	// This serves to both tell him of any space we might

ipam/allocator_test.go

@@ -20,6 +20,8 @@ const (
 	testStart3 = "10.0.3.0"
 )
 
+func returnFalse() bool { return false }
+
 func TestAllocFree(t *testing.T) {
 	const (
 		container1 = "abcdef"
@@ -39,31 +41,31 @@ func TestAllocFree(t *testing.T) {
 	_, cidr2, _ := address.ParseCIDR(subnet2)
 
 	alloc.claimRingForTesting()
-	addr1, err := alloc.Allocate(container1, cidr1.HostRange(), nil)
+	addr1, err := alloc.Allocate(container1, cidr1.HostRange(), returnFalse)
 	require.NoError(t, err)
 	require.Equal(t, testAddr1, addr1.String(), "address")
 
-	addr2, err := alloc.Allocate(container1, cidr2.HostRange(), nil)
+	addr2, err := alloc.Allocate(container1, cidr2.HostRange(), returnFalse)
 	require.NoError(t, err)
 	require.Equal(t, testAddr2, addr2.String(), "address")
 
 	// Ask for another address for a different container and check it's different
-	addr1b, _ := alloc.Allocate(container2, cidr1.HostRange(), nil)
+	addr1b, _ := alloc.Allocate(container2, cidr1.HostRange(), returnFalse)
 	if addr1b.String() == testAddr1 {
 		t.Fatalf("Expected different address but got %s", addr1b.String())
 	}
 
 	// Ask for the first container again and we should get the same addresses again
-	addr1a, _ := alloc.Allocate(container1, cidr1.HostRange(), nil)
+	addr1a, _ := alloc.Allocate(container1, cidr1.HostRange(), returnFalse)
 	require.Equal(t, testAddr1, addr1a.String(), "address")
-	addr2a, _ := alloc.Allocate(container1, cidr2.HostRange(), nil)
+	addr2a, _ := alloc.Allocate(container1, cidr2.HostRange(), returnFalse)
 	require.Equal(t, testAddr2, addr2a.String(), "address")
 
 	// Now delete the first container, and we should get its addresses back
 	require.NoError(t, alloc.Delete(container1))
-	addr3, _ := alloc.Allocate(container3, cidr1.HostRange(), nil)
+	addr3, _ := alloc.Allocate(container3, cidr1.HostRange(), returnFalse)
 	require.Equal(t, testAddr1, addr3.String(), "address")
-	addr4, _ := alloc.Allocate(container3, cidr2.HostRange(), nil)
+	addr4, _ := alloc.Allocate(container3, cidr2.HostRange(), returnFalse)
 	require.Equal(t, testAddr2, addr4.String(), "address")
 
 	alloc.ContainerDied(container2)
@@ -93,7 +95,7 @@ func TestBootstrap(t *testing.T) {
 	ExpectBroadcastMessage(alloc1, nil) // alloc1 will try to form consensus
 	done := make(chan bool)
 	go func() {
-		alloc1.Allocate("somecontainer", subnet, nil)
+		alloc1.Allocate("somecontainer", subnet, returnFalse)
 		done <- true
 	}()
 	time.Sleep(100 * time.Millisecond)
@@ -147,15 +149,15 @@ func TestAllocatorClaim(t *testing.T) {
 	require.NoError(t, err)
 
 	// Do one allocate to ensure paxos is all done
-	alloc.Allocate("unused", subnet, nil)
+	alloc.Allocate("unused", subnet, returnFalse)
 	// Do an allocate on the other peer, which we will try to claim later
-	addrx, err := allocs[0].Allocate(container1, subnet, nil)
+	addrx, err := allocs[0].Allocate(container1, subnet, returnFalse)
 
 	// Now try the claim again
 	err = alloc.Claim(container3, addr1, true)
 	require.NoError(t, err)
 	// Check we get this address back if we try an allocate
-	addr3, _ := alloc.Allocate(container3, subnet, nil)
+	addr3, _ := alloc.Allocate(container3, subnet, returnFalse)
 	require.Equal(t, testAddr1, addr3.String(), "address")
 	// one more claim should still work
 	err = alloc.Claim(container3, addr1, true)
@@ -206,9 +208,9 @@ func TestCancel(t *testing.T) {
 	alloc1.OnGossipBroadcast(alloc2.ourName, alloc2.Encode())
 
 	// Get some IPs, so each allocator has some space
-	res1, _ := alloc1.Allocate("foo", subnet, nil)
+	res1, _ := alloc1.Allocate("foo", subnet, returnFalse)
 	common.Log.Debugf("res1 = %s", res1.String())
-	res2, _ := alloc2.Allocate("bar", subnet, nil)
+	res2, _ := alloc2.Allocate("bar", subnet, returnFalse)
 	common.Log.Debugf("res2 = %s", res2.String())
 	if res1 == res2 {
 		require.FailNow(t, "Error: got same ips!")
@@ -220,12 +222,20 @@ func TestCancel(t *testing.T) {
 
 	// Use up all the IPs that alloc1 owns, so the allocation after this will prompt a request to alloc2
 	for i := 0; alloc1.NumFreeAddresses(subnet) > 0; i++ {
-		alloc1.Allocate(fmt.Sprintf("tmp%d", i), subnet, nil)
+		alloc1.Allocate(fmt.Sprintf("tmp%d", i), subnet, returnFalse)
 	}
 	cancelChan := make(chan bool, 1)
 	doneChan := make(chan bool)
 	go func() {
-		_, ok := alloc1.Allocate("baz", subnet, cancelChan)
+		_, ok := alloc1.Allocate("baz", subnet,
+			func() bool {
+				select {
+				case <-cancelChan:
+					return true
+				default:
+					return false
+				}
+			})
 		doneChan <- ok == nil
 	}()
 
@@ -252,7 +262,7 @@ func TestCancelOnDied(t *testing.T) {
 
 	doneChan := make(chan bool)
 	f := func() {
-		_, ok := alloc1.Allocate(container1, subnet, nil)
+		_, ok := alloc1.Allocate(container1, subnet, returnFalse)
 		doneChan <- ok == nil
 	}
 
@@ -283,11 +293,11 @@ func TestGossipShutdown(t *testing.T) {
 	defer alloc.Stop()
 
 	alloc.claimRingForTesting()
-	alloc.Allocate(container1, subnet, nil)
+	alloc.Allocate(container1, subnet, returnFalse)
 
 	alloc.Shutdown()
 
-	_, err := alloc.Allocate(container2, subnet, nil) // trying to allocate after shutdown should fail
+	_, err := alloc.Allocate(container2, subnet, returnFalse) // trying to allocate after shutdown should fail
 	require.False(t, err == nil, "no address")
 
 	CheckAllExpectedMessagesSent(alloc)
@@ -302,10 +312,10 @@ func TestTransfer(t *testing.T) {
 	alloc2 := allocs[1]
 	alloc3 := allocs[2] // This will be 'master' and get the first range
 
-	_, err := alloc2.Allocate("foo", subnet, nil)
+	_, err := alloc2.Allocate("foo", subnet, returnFalse)
 	require.True(t, err == nil, "Failed to get address")
 
-	_, err = alloc3.Allocate("bar", subnet, nil)
+	_, err = alloc3.Allocate("bar", subnet, returnFalse)
 	require.True(t, err == nil, "Failed to get address")
 
 	alloc2.gossip.GossipBroadcast(alloc2.Gossip())
@@ -323,7 +333,7 @@ func TestTransfer(t *testing.T) {
 
 	require.Equal(t, address.Offset(1022), alloc1.NumFreeAddresses(subnet))
 
-	_, err = alloc1.Allocate("foo", subnet, nil)
+	_, err = alloc1.Allocate("foo", subnet, returnFalse)
 	require.True(t, err == nil, "Failed to get address")
 	alloc1.Stop()
 }
@@ -338,7 +348,7 @@ func TestFakeRouterSimple(t *testing.T) {
 	alloc1 := allocs[0]
 	//alloc2 := allocs[1]
 
-	_, err := alloc1.Allocate("foo", subnet, nil)
+	_, err := alloc1.Allocate("foo", subnet, returnFalse)
 	require.NoError(t, err, "Failed to get address")
 }
 
@@ -419,7 +429,7 @@ func TestAllocatorFuzz(t *testing.T) {
 		allocIndex := rand.Int31n(nodes)
 		alloc := allocs[allocIndex]
 		//common.Log.Infof("Allocate: asking allocator %d", allocIndex)
-		addr, err := alloc.Allocate(name, subnet, nil)
+		addr, err := alloc.Allocate(name, subnet, returnFalse)
 
 		if err != nil {
 			panic(fmt.Sprintf("Could not allocate addr"))
@@ -476,7 +486,7 @@ func TestAllocatorFuzz(t *testing.T) {
 
 		//common.Log.Infof("Asking for %s (%s) on allocator %d again", res.name, addr, res.alloc)
 
-		newAddr, _ := alloc.Allocate(res.name, subnet, nil)
+		newAddr, _ := alloc.Allocate(res.name, subnet, returnFalse)
 		oldAddr, _ := address.ParseIP(addr)
 		if newAddr != oldAddr {
 			panic(fmt.Sprintf("Got different address for repeat request for %s: %s != %s", res.name, newAddr, oldAddr))

ipam/claim.go

@@ -4,8 +4,8 @@ import (
 	"fmt"
 
 	"github.com/weaveworks/weave/common"
+	"github.com/weaveworks/weave/mesh"
 	"github.com/weaveworks/weave/net/address"
-	"github.com/weaveworks/weave/router"
 )
 
 type claim struct {
@@ -43,7 +43,7 @@ func (c *claim) Try(alloc *Allocator) bool {
 	switch owner := alloc.ring.Owner(c.addr); owner {
 	case alloc.ourName:
 		// success
-	case router.UnknownPeerName:
+	case mesh.UnknownPeerName:
 		// If our ring doesn't know, it must be empty.
 		if c.noErrorOnUnknown {
 			alloc.infof("Claim %s for %s: address allocator still initializing; will try later.", c.addr, c.ident)
@@ -86,7 +86,7 @@ func (c *claim) Try(alloc *Allocator) bool {
 	return true
 }
 
-func (c *claim) deniedBy(alloc *Allocator, owner router.PeerName) {
+func (c *claim) deniedBy(alloc *Allocator, owner mesh.PeerName) {
 	name, found := alloc.nicknames[owner]
 	if found {
 		name = " (" + name + ")"

ipam/http.go

@@ -31,7 +31,17 @@ func parseCIDR(w http.ResponseWriter, cidrStr string) (address.CIDR, bool) {
 
 func (alloc *Allocator) handleHTTPAllocate(dockerCli *docker.Client, w http.ResponseWriter, ident string, checkAlive bool, subnet address.CIDR) {
 	closedChan := w.(http.CloseNotifier).CloseNotify()
-	addr, err := alloc.Allocate(ident, subnet.HostRange(), closedChan)
+	addr, err := alloc.Allocate(ident, subnet.HostRange(),
+		func() bool {
+			select {
+			case <-closedChan:
+				return true
+			default:
+				res := checkAlive && dockerCli != nil && dockerCli.IsContainerNotRunning(ident)
+				checkAlive = false // we check only once; if the container dies later we learn about that through events
+				return res
+			}
+		})
 	if err != nil {
 		if _, ok := err.(*errorCancelled); ok { // cancellation is not really an error
 			common.Log.Infoln("[allocator]:", err.Error())
@@ -41,12 +51,6 @@ func (alloc *Allocator) handleHTTPAllocate(dockerCli *docker.Client, w http.Resp
 		badRequest(w, err)
 		return
 	}
-	if checkAlive && dockerCli != nil && dockerCli.IsContainerNotRunning(ident) {
-		common.Log.Infof("[allocator] '%s' is not running: freeing %s", ident, addr)
-		alloc.Free(ident, addr)
-		fmt.Fprint(w, "cancelled")
-		return
-	}
 
 	fmt.Fprintf(w, "%s/%d", addr, subnet.PrefixLen)
 }

ipam/paxos/paxos.go

@@ -1,7 +1,7 @@
 package paxos
 
 import (
-	"github.com/weaveworks/weave/router"
+	"github.com/weaveworks/weave/mesh"
 )
 
 // The node identifier.  The use of the UID here is important: Paxos
@@ -9,8 +9,8 @@ import (
 // node does not restart and lose its Paxos state but claim to have
 // the same ID.
 type NodeID struct {
-	Name router.PeerName
-	UID  router.PeerUID
+	Name mesh.PeerName
+	UID  mesh.PeerUID
 }
 
 // note all fields exported in structs so we can Gob them
@@ -39,7 +39,7 @@ func (a ProposalID) valid() bool {
 }
 
 // For seeding IPAM, the value we want consensus on is a set of peer names
-type Value []router.PeerName
+type Value []mesh.PeerName
 
 // An AcceptedValue is a Value plus the proposal which originated that
 // Value.  The origin is not essential, but makes comparing
@@ -72,7 +72,7 @@ type Node struct {
 	knows  GossipState
 }
 
-func NewNode(name router.PeerName, uid router.PeerUID, quorum uint) *Node {
+func NewNode(name mesh.PeerName, uid mesh.PeerUID, quorum uint) *Node {
 	return &Node{
 		id:     NodeID{name, uid},
 		quorum: quorum,
@@ -223,7 +223,7 @@ func (node *Node) Think() bool {
 // about.  This is not necessarily all peer names, but it is at least
 // a quorum, and so good enough for seeding the ring.
 func (node *Node) pickValue() Value {
-	val := make([]router.PeerName, len(node.knows))
+	val := make([]mesh.PeerName, len(node.knows))
 	i := 0
 	for id := range node.knows {
 		val[i] = id.Name

ipam/paxos/paxos_test.go

@@ -6,7 +6,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/weaveworks/weave/router"
+	"github.com/weaveworks/weave/mesh"
 )
 
 type TestNode struct {
@@ -97,8 +97,8 @@ func makeRandomModel(params *TestParams, r *rand.Rand, t *testing.T) *Model {
 	}
 
 	for i := range m.nodes {
-		m.nodes[i].Node = NewNode(router.PeerName(i/2+1),
-			router.PeerUID(r.Int63()), m.quorum)
+		m.nodes[i].Node = NewNode(mesh.PeerName(i/2+1),
+			mesh.PeerUID(r.Int63()), m.quorum)
 		m.nodes[i].Propose()
 	}
 
@@ -172,8 +172,8 @@ func (m *Model) isolateNode(node *TestNode) {
 
 // Restart a node
 func (m *Model) restart(node *TestNode) {
-	node.Node = NewNode(router.PeerName(m.nextID),
-		router.PeerUID(m.r.Int63()), m.quorum)
+	node.Node = NewNode(mesh.PeerName(m.nextID),
+		mesh.PeerUID(m.r.Int63()), m.quorum)
 	m.nextID++
 	node.Propose()
 

ipam/ring/entry.go

@@ -4,14 +4,14 @@ import (
 	"sort"
 
 	"github.com/weaveworks/weave/common"
+	"github.com/weaveworks/weave/mesh"
 	"github.com/weaveworks/weave/net/address"
-	"github.com/weaveworks/weave/router"
 )
 
 // Entry represents entries around the ring
 type entry struct {
 	Token   address.Address // The start of this range
-	Peer    router.PeerName // Who owns this range
+	Peer    mesh.PeerName   // Who owns this range
 	Version uint32          // Version of this range
 	Free    address.Offset  // Number of free IPs in this range
 }
@@ -21,7 +21,7 @@ func (e *entry) Equal(e2 *entry) bool {
 		e.Version == e2.Version
 }
 
-func (e *entry) update(peername router.PeerName, free address.Offset) {
+func (e *entry) update(peername mesh.PeerName, free address.Offset) {
 	e.Peer = peername
 	e.Version++
 	e.Free = free

ipam/ring/ring.go

@@ -13,15 +13,16 @@ import (
 	"time"
 
 	"github.com/weaveworks/weave/common"
+	"github.com/weaveworks/weave/mesh"
 	"github.com/weaveworks/weave/net/address"
-	"github.com/weaveworks/weave/router"
 )
 
 // Ring represents the ring itself
 type Ring struct {
 	Start, End address.Address // [min, max) tokens in this ring.  Due to wrapping, min == max (effectively)
-	Peer       router.PeerName // name of peer owning this ring instance
+	Peer       mesh.PeerName   // name of peer owning this ring instance
 	Entries    entries         // list of entries sorted by token
+	Seeds      []mesh.PeerName // peers with which the ring was seeded
 }
 
 func (r *Ring) assertInvariants() {
@@ -36,6 +37,7 @@ var (
 	ErrNotSorted       = errors.New("Ring not sorted")
 	ErrTokenRepeated   = errors.New("Token appears twice in ring")
 	ErrTokenOutOfRange = errors.New("Token is out of range")
+	ErrDifferentSeeds  = errors.New("Received ring was seeded differently from ours!")
 	ErrDifferentRange  = errors.New("Received range differs from ours!")
 	ErrNewerVersion    = errors.New("Received new version for entry I own!")
 	ErrInvalidEntry    = errors.New("Received invalid state update!")
@@ -82,7 +84,7 @@ func (r *Ring) checkInvariants() error {
 }
 
 // New creates an empty ring belonging to peer.
-func New(start, end address.Address, peer router.PeerName) *Ring {
+func New(start, end address.Address, peer mesh.PeerName) *Ring {
 	common.Assert(start < end)
 
 	ring := &Ring{Start: start, End: end, Peer: peer, Entries: make([]*entry, 0)}
@@ -109,7 +111,7 @@ func (r *Ring) distance(start, end address.Address) address.Offset {
 // Preconditions:
 // - start < end
 // - [start, end) must be owned by the calling peer
-func (r *Ring) GrantRangeToHost(start, end address.Address, peer router.PeerName) {
+func (r *Ring) GrantRangeToHost(start, end address.Address, peer mesh.PeerName) {
 	//fmt.Printf("%s GrantRangeToHost [%v,%v) -> %s\n", r.Peer, start, end, peer)
 
 	r.assertInvariants()
@@ -188,6 +190,17 @@ func (r *Ring) Merge(gossip Ring) error {
 		return err
 	}
 
+	if len(gossip.Seeds) > 0 && len(r.Seeds) > 0 {
+		if len(gossip.Seeds) != len(r.Seeds) {
+			return ErrDifferentSeeds
+		}
+		for i, seed := range gossip.Seeds {
+			if seed != r.Seeds[i] {
+				return ErrDifferentSeeds
+			}
+		}
+	}
+
 	if r.Start != gossip.Start || r.End != gossip.End {
 		return ErrDifferentRange
 	}
@@ -197,7 +210,7 @@ func (r *Ring) Merge(gossip Ring) error {
 	addToResult := func(e entry) { result = append(result, &e) }
 
 	var mine, theirs *entry
-	var previousOwner *router.PeerName
+	var previousOwner *mesh.PeerName
 	// i is index into r.Entries; j is index into gossip.Entries
 	var i, j int
 	for i < len(r.Entries) && j < len(gossip.Entries) {
@@ -254,6 +267,9 @@ func (r *Ring) Merge(gossip Ring) error {
 		previousOwner = nil
 	}
 
+	if len(r.Seeds) == 0 {
+		r.Seeds = gossip.Seeds
+	}
 	r.Entries = result
 	return nil
 }
@@ -302,7 +318,7 @@ func (r *Ring) OwnedRanges() (result []address.Range) {
 
 // ClaimForPeers claims the entire ring for the array of peers passed
 // in.  Only works for empty rings.
-func (r *Ring) ClaimForPeers(peers []router.PeerName) {
+func (r *Ring) ClaimForPeers(peers []mesh.PeerName) {
 	common.Assert(r.Empty())
 	defer r.assertInvariants()
 	defer r.updateExportedVariables()
@@ -330,9 +346,11 @@ func (r *Ring) ClaimForPeers(peers []router.PeerName) {
 	}
 
 	common.Assert(pos == r.End)
+
+	r.Seeds = peers
 }
 
-func (r *Ring) FprintWithNicknames(w io.Writer, m map[router.PeerName]string) {
+func (r *Ring) FprintWithNicknames(w io.Writer, m map[mesh.PeerName]string) {
 	for _, entry := range r.Entries {
 		nickname, found := m[entry.Peer]
 		if found {
@@ -347,7 +365,7 @@ func (r *Ring) FprintWithNicknames(w io.Writer, m map[router.PeerName]string) {
 func (r *Ring) String() string {
 	var buffer bytes.Buffer
 	fmt.Fprintf(&buffer, "Ring [%s, %s)", r.Start, r.End)
-	r.FprintWithNicknames(&buffer, make(map[router.PeerName]string))
+	r.FprintWithNicknames(&buffer, make(map[mesh.PeerName]string))
 	return buffer.String()
 }
 
@@ -397,7 +415,7 @@ func (r *Ring) ReportFree(freespace map[address.Address]address.Offset) {
 
 type weightedPeer struct {
 	weight   float64
-	peername router.PeerName
+	peername mesh.PeerName
 }
 type weightedPeers []weightedPeer
 
@@ -408,10 +426,10 @@ func (ws weightedPeers) Swap(i, j int)      { ws[i], ws[j] = ws[j], ws[i] }
 
 // ChoosePeersToAskForSpace returns all peers we can ask for space in
 // the range [start, end), in weighted-random order.  Assumes start<end.
-func (r *Ring) ChoosePeersToAskForSpace(start, end address.Address) []router.PeerName {
+func (r *Ring) ChoosePeersToAskForSpace(start, end address.Address) []mesh.PeerName {
 	var (
 		sum               address.Offset
-		totalSpacePerPeer = make(map[router.PeerName]address.Offset) // Compute total free space per peer
+		totalSpacePerPeer = make(map[mesh.PeerName]address.Offset) // Compute total free space per peer
 	)
 
 	// iterate through tokens
@@ -445,25 +463,25 @@ func (r *Ring) ChoosePeersToAskForSpace(start, end address.Address) []router.Pee
 		ws = append(ws, weightedPeer{weight: float64(space) * rand.Float64(), peername: peername})
 	}
 	sort.Sort(ws)
-	result := make([]router.PeerName, len(ws))
+	result := make([]mesh.PeerName, len(ws))
 	for i, wp := range ws {
 		result[i] = wp.peername
 	}
 	return result
 }
 
-func (r *Ring) PickPeerForTransfer() router.PeerName {
+func (r *Ring) PickPeerForTransfer(isValid func(mesh.PeerName) bool) mesh.PeerName {
 	for _, entry := range r.Entries {
-		if entry.Peer != r.Peer {
+		if entry.Peer != r.Peer && isValid(entry.Peer) {
 			return entry.Peer
 		}
 	}
-	return router.UnknownPeerName
+	return mesh.UnknownPeerName
 }
 
 // Transfer will mark all entries associated with 'from' peer as owned by 'to' peer
 // and return ranges indicating the new space we picked up
-func (r *Ring) Transfer(from, to router.PeerName) ([]address.Range, error) {
+func (r *Ring) Transfer(from, to mesh.PeerName) ([]address.Range, error) {
 	r.assertInvariants()
 	defer r.assertInvariants()
 	defer r.updateExportedVariables()
@@ -493,13 +511,13 @@ func (r *Ring) Contains(addr address.Address) bool {
 }
 
 // Owner returns the peername which owns the range containing addr
-func (r *Ring) Owner(token address.Address) router.PeerName {
+func (r *Ring) Owner(token address.Address) mesh.PeerName {
 	common.Assert(r.Start <= token && token < r.End)
 
 	r.assertInvariants()
 	// There can be no owners on an empty ring
 	if r.Empty() {
-		return router.UnknownPeerName
+		return mesh.UnknownPeerName
 	}
 
 	// Look for the right-most entry, less than or equal to token
@@ -512,8 +530,8 @@ func (r *Ring) Owner(token address.Address) router.PeerName {
 }
 
 // Get the set of PeerNames mentioned in the ring
-func (r *Ring) PeerNames() map[router.PeerName]struct{} {
-	res := make(map[router.PeerName]struct{})
+func (r *Ring) PeerNames() map[mesh.PeerName]struct{} {
+	res := make(map[mesh.PeerName]struct{})
 
 	for _, entry := range r.Entries {
 		res[entry.Peer] = struct{}{}

ipam/ring/ring_test.go

@@ -9,14 +9,14 @@ import (
 
 	"github.com/stretchr/testify/require"
 	"github.com/weaveworks/weave/common"
+	"github.com/weaveworks/weave/mesh"
 	"github.com/weaveworks/weave/net/address"
-	"github.com/weaveworks/weave/router"
 )
 
 var (
-	peer1name, _ = router.PeerNameFromString("01:00:00:00:02:00")
-	peer2name, _ = router.PeerNameFromString("02:00:00:00:02:00")
-	peer3name, _ = router.PeerNameFromString("03:00:00:00:02:00")
+	peer1name, _ = mesh.PeerNameFromString("01:00:00:00:02:00")
+	peer2name, _ = mesh.PeerNameFromString("02:00:00:00:02:00")
+	peer3name, _ = mesh.PeerNameFromString("03:00:00:00:02:00")
 
 	start, end    = ParseIP("10.0.0.0"), ParseIP("10.0.0.255")
 	dot10, dot245 = ParseIP("10.0.0.10"), ParseIP("10.0.0.245")
@@ -351,7 +351,7 @@ func TestGossip(t *testing.T) {
 	assertRing(ring2, []*entry{{Token: start, Peer: peer1name, Free: 255}})
 }
 
-func assertPeersWithSpace(t *testing.T, ring *Ring, start, end address.Address, expected int) []router.PeerName {
+func assertPeersWithSpace(t *testing.T, ring *Ring, start, end address.Address, expected int) []mesh.PeerName {
 	peers := ring.ChoosePeersToAskForSpace(start, end)
 	require.Equal(t, expected, len(peers))
 	return peers
@@ -375,7 +375,7 @@ func TestFindFree(t *testing.T) {
 	ring1.assertInvariants()
 
 	// We should return others
-	var peers []router.PeerName
+	var peers []mesh.PeerName
 
 	ring1.Entries = []*entry{{Token: start, Peer: peer2name, Free: 1}}
 	peers = assertPeersWithSpace(t, ring1, start, end, 1)
@@ -493,7 +493,7 @@ func TestOwner(t *testing.T) {
 	require.True(t, ring1.Contains(start), "start should be in ring")
 	require.False(t, ring1.Contains(end), "end should not be in ring")
 
-	require.Equal(t, router.UnknownPeerName, ring1.Owner(start))
+	require.Equal(t, mesh.UnknownPeerName, ring1.Owner(start))
 
 	ring1.ClaimItAll()
 	ring1.GrantRangeToHost(middle, end, peer2name)
@@ -517,9 +517,9 @@ func TestFuzzRing(t *testing.T) {
 		iterations = 1000
 	)
 
-	peers := make([]router.PeerName, numPeers)
+	peers := make([]mesh.PeerName, numPeers)
 	for i := 0; i < numPeers; i++ {
-		peer, _ := router.PeerNameFromString(fmt.Sprintf("%02d:00:00:00:02:00", i))
+		peer, _ := mesh.PeerNameFromString(fmt.Sprintf("%02d:00:00:00:02:00", i))
 		peers[i] = peer
 	}
 
@@ -601,13 +601,13 @@ func TestFuzzRingHard(t *testing.T) {
 	var (
 		numPeers   = 100
 		iterations = 3000
-		peers      []router.PeerName
+		peers      []mesh.PeerName
 		rings      []*Ring
 		nextPeerID = 0
 	)
 
 	addPeer := func() {
-		peer, _ := router.PeerNameFromString(fmt.Sprintf("%02d:%02d:00:00:00:00", nextPeerID/10, nextPeerID%10))
+		peer, _ := mesh.PeerNameFromString(fmt.Sprintf("%02d:%02d:00:00:00:00", nextPeerID/10, nextPeerID%10))
 		common.Log.Debugf("%s: Adding peer", peer)
 		nextPeerID++
 		peers = append(peers, peer)
@@ -620,7 +620,7 @@ func TestFuzzRingHard(t *testing.T) {
 
 	rings[0].ClaimItAll()
 
-	randomPeer := func(exclude int) (int, router.PeerName, *Ring) {
+	randomPeer := func(exclude int) (int, mesh.PeerName, *Ring) {
 		var peerIndex int
 		if exclude >= 0 {
 			peerIndex = rand.Intn(len(peers) - 1)
@@ -729,7 +729,7 @@ func TestFuzzRingHard(t *testing.T) {
 }
 
 func (r *Ring) ClaimItAll() {
-	r.ClaimForPeers([]router.PeerName{r.Peer})
+	r.ClaimForPeers([]mesh.PeerName{r.Peer})
 }
 
 func (es entries) String() string {

ipam/testutils_test.go

@@ -8,13 +8,13 @@ import (
 	"time"
 
 	"github.com/stretchr/testify/require"
+	"github.com/weaveworks/weave/mesh"
 	"github.com/weaveworks/weave/net/address"
-	"github.com/weaveworks/weave/router"
 	"github.com/weaveworks/weave/testing/gossip"
 )
 
 type mockMessage struct {
-	dst     router.PeerName
+	dst     mesh.PeerName
 	msgType byte
 	buf     []byte
 }
@@ -48,13 +48,13 @@ func (m *mockGossipComms) String() string {
 // that the contents of messages are never re-ordered.  Which, for instance,
 // requires they are not based off iterating through a map.
 
-func (m *mockGossipComms) GossipBroadcast(update router.GossipData) error {
+func (m *mockGossipComms) GossipBroadcast(update mesh.GossipData) error {
 	m.Lock()
 	defer m.Unlock()
 	buf := []byte{}
 	if len(m.messages) == 0 {
 		require.FailNow(m, fmt.Sprintf("%s: Gossip broadcast message unexpected: \n%x", m.name, buf))
-	} else if msg := m.messages[0]; msg.dst != router.UnknownPeerName {
+	} else if msg := m.messages[0]; msg.dst != mesh.UnknownPeerName {
 		require.FailNow(m, fmt.Sprintf("%s: Expected Gossip message to %s but got broadcast", m.name, msg.dst))
 	} else if msg.buf != nil && !equalByteBuffer(msg.buf, buf) {
 		require.FailNow(m, fmt.Sprintf("%s: Gossip message not sent as expected: \nwant: %x\ngot : %x", m.name, msg.buf, buf))
@@ -77,12 +77,12 @@ func equalByteBuffer(a, b []byte) bool {
 	return true
 }
 
-func (m *mockGossipComms) GossipUnicast(dstPeerName router.PeerName, buf []byte) error {
+func (m *mockGossipComms) GossipUnicast(dstPeerName mesh.PeerName, buf []byte) error {
 	m.Lock()
 	defer m.Unlock()
 	if len(m.messages) == 0 {
 		require.FailNow(m, fmt.Sprintf("%s: Gossip message to %s unexpected: \n%s", m.name, dstPeerName, buf))
-	} else if msg := m.messages[0]; msg.dst == router.UnknownPeerName {
+	} else if msg := m.messages[0]; msg.dst == mesh.UnknownPeerName {
 		require.FailNow(m, fmt.Sprintf("%s: Expected Gossip broadcast message but got dest %s", m.name, dstPeerName))
 	} else if msg.dst != dstPeerName {
 		require.FailNow(m, fmt.Sprintf("%s: Expected Gossip message to %s but got dest %s", m.name, msg.dst, dstPeerName))
@@ -99,7 +99,7 @@ func (m *mockGossipComms) GossipUnicast(dstPeerName router.PeerName, buf []byte)
 
 func ExpectMessage(alloc *Allocator, dst string, msgType byte, buf []byte) {
 	m := alloc.gossip.(*mockGossipComms)
-	dstPeerName, _ := router.PeerNameFromString(dst)
+	dstPeerName, _ := mesh.PeerNameFromString(dst)
 	m.Lock()
 	m.messages = append(m.messages, mockMessage{dstPeerName, msgType, buf})
 	m.Unlock()
@@ -108,7 +108,7 @@ func ExpectMessage(alloc *Allocator, dst string, msgType byte, buf []byte) {
 func ExpectBroadcastMessage(alloc *Allocator, buf []byte) {
 	m := alloc.gossip.(*mockGossipComms)
 	m.Lock()
-	m.messages = append(m.messages, mockMessage{router.UnknownPeerName, 0, buf})
+	m.messages = append(m.messages, mockMessage{mesh.UnknownPeerName, 0, buf})
 	m.Unlock()
 }
 
@@ -124,7 +124,7 @@ func CheckAllExpectedMessagesSent(allocs ...*Allocator) {
 }
 
 func makeAllocator(name string, cidrStr string, quorum uint) (*Allocator, address.Range) {
-	peername, err := router.PeerNameFromString(name)
+	peername, err := mesh.PeerNameFromString(name)
 	if err != nil {
 		panic(err)
 	}
@@ -134,8 +134,8 @@ func makeAllocator(name string, cidrStr string, quorum uint) (*Allocator, addres
 		panic(err)
 	}
 
-	alloc := NewAllocator(peername, router.PeerUID(rand.Int63()),
-		"nick-"+name, cidr.Range(), quorum)
+	alloc := NewAllocator(peername, mesh.PeerUID(rand.Int63()),
+		"nick-"+name, cidr.Range(), quorum, func(mesh.PeerName) bool { return true })
 
 	return alloc, cidr.HostRange()
 }
@@ -149,7 +149,7 @@ func makeAllocatorWithMockGossip(t *testing.T, name string, universeCIDR string,
 }
 
 func (alloc *Allocator) claimRingForTesting(allocs ...*Allocator) {
-	peers := []router.PeerName{alloc.ourName}
+	peers := []mesh.PeerName{alloc.ourName}
 	for _, alloc2 := range allocs {
 		peers = append(peers, alloc2.ourName)
 	}

mesh/connection.go

@@ -1,8 +1,6 @@
-package router
+package mesh
 
 import (
-	"encoding/binary"
-	"errors"
 	"fmt"
 	"net"
 	"strconv"
@@ -29,7 +27,7 @@ const (
 	TieBreakTied
 )
 
-var ErrConnectToSelf = errors.New("Cannot connect to ourself")
+var ErrConnectToSelf = fmt.Errorf("Cannot connect to ourself")
 
 type RemoteConnection struct {
 	local         *Peer
@@ -42,26 +40,16 @@ type RemoteConnection struct {
 type LocalConnection struct {
 	sync.RWMutex
 	RemoteConnection
-	TCPConn           *net.TCPConn
-	version           byte
-	tcpSender         TCPSender
-	remoteUDPAddr     *net.UDPAddr
-	receivedHeartbeat bool
-	stackFrag         bool
-	effectivePMTU     int
-	SessionKey        *[32]byte
-	heartbeatTCP      *time.Ticker
-	heartbeatTimeout  *time.Timer
-	heartbeatFrame    []byte
-	heartbeat         *time.Ticker
-	fragTest          *time.Ticker
-	forwarder         *Forwarder
-	forwarderDF       *ForwarderDF
-	Decryptor         Decryptor
-	Router            *Router
-	uid               uint64
-	actionChan        chan<- ConnectionAction
-	finished          <-chan struct{} // closed to signal that actorLoop has finished
+	TCPConn      *net.TCPConn
+	version      byte
+	tcpSender    TCPSender
+	SessionKey   *[32]byte
+	heartbeatTCP *time.Ticker
+	Router       *Router
+	uid          uint64
+	actionChan   chan<- ConnectionAction
+	finished     <-chan struct{} // closed to signal that actorLoop has finished
+	OverlayConn  OverlayConnection
 }
 
 type ConnectionAction func() error
@@ -94,7 +82,7 @@ func (conn *RemoteConnection) ErrorLog(args ...interface{}) {
 
 // Does not return anything. If the connection is successful, it will
 // end up in the local peer's connections map.
-func StartLocalConnection(connRemote *RemoteConnection, tcpConn *net.TCPConn, udpAddr *net.UDPAddr, router *Router, acceptNewPeer bool) {
+func StartLocalConnection(connRemote *RemoteConnection, tcpConn *net.TCPConn, router *Router, acceptNewPeer bool) {
 	if connRemote.local != router.Ourself.Peer {
 		log.Fatal("Attempt to create local connection from a peer which is not ourself")
 	}
@@ -105,8 +93,6 @@ func StartLocalConnection(connRemote *RemoteConnection, tcpConn *net.TCPConn, ud
 		RemoteConnection: *connRemote,
 		Router:           router,
 		TCPConn:          tcpConn,
-		remoteUDPAddr:    udpAddr,
-		effectivePMTU:    DefaultPMTU,
 		uid:              randUint64(),
 		actionChan:       actionChan,
 		finished:         finished}
@@ -126,48 +112,10 @@ func (conn *LocalConnection) BreakTie(dupConn Connection) ConnectionTieBreak {
 	}
 }
 
-// Read by the forwarder processes when in the UDP senders
-func (conn *LocalConnection) RemoteUDPAddr() *net.UDPAddr {
-	conn.RLock()
-	defer conn.RUnlock()
-	return conn.remoteUDPAddr
-}
-
 func (conn *LocalConnection) Established() bool {
-	conn.RLock()
-	defer conn.RUnlock()
 	return conn.established
 }
 
-// Called by forwarder processes, read in Forward (by sniffer and udp
-// listener process in router).
-func (conn *LocalConnection) setEffectivePMTU(pmtu int) {
-	conn.Lock()
-	defer conn.Unlock()
-	if conn.effectivePMTU != pmtu {
-		conn.effectivePMTU = pmtu
-		conn.Log("Effective PMTU set to", pmtu)
-	}
-}
-
-// Called by the connection's actor process, and by the connection's
-// TCP receiver process. StackFrag is read in conn.forward
-func (conn *LocalConnection) setStackFrag(frag bool) {
-	conn.Lock()
-	defer conn.Unlock()
-	conn.stackFrag = frag
-}
-
-// Called by the connection's TCP receiver process.
-func (conn *LocalConnection) pmtuVerified(pmtu int) {
-	conn.RLock()
-	fwd := conn.forwarderDF
-	conn.RUnlock()
-	if fwd != nil {
-		fwd.PMTUVerified(pmtu)
-	}
-}
-
 // Send directly, not via the Actor.  If it goes via the Actor we can
 // get a deadlock where LocalConnection is blocked talking to
 // LocalPeer and LocalPeer is blocked trying send a ProtocolMsg via
@@ -197,64 +145,6 @@ func (conn *LocalConnection) Shutdown(err error) {
 	go func() { conn.sendAction(func() error { return err }) }()
 }
 
-// Async
-//
-// Heartbeating serves two purposes: a) keeping NAT paths alive, and
-// b) updating a remote peer's knowledge of our address, in the event
-// it changes (e.g. because NAT paths expired).
-func (conn *LocalConnection) ReceivedHeartbeat(remoteUDPAddr *net.UDPAddr, connUID uint64) {
-	if remoteUDPAddr == nil || connUID != conn.uid {
-		return
-	}
-	conn.sendAction(func() error {
-		oldRemoteUDPAddr := conn.remoteUDPAddr
-		old := conn.receivedHeartbeat
-		conn.Lock()
-		conn.remoteUDPAddr = remoteUDPAddr
-		conn.receivedHeartbeat = true
-		conn.Unlock()
-		conn.heartbeatTimeout.Reset(HeartbeatTimeout)
-		if !old {
-			if err := conn.sendSimpleProtocolMsg(ProtocolConnectionEstablished); err != nil {
-				return err
-			}
-		}
-		if oldRemoteUDPAddr == nil {
-			return conn.sendFastHeartbeats()
-		} else if oldRemoteUDPAddr.String() != remoteUDPAddr.String() {
-			log.Println("Peer", conn.remote, "moved from", oldRemoteUDPAddr, "to", remoteUDPAddr)
-		}
-		return nil
-	})
-}
-
-// Async
-func (conn *LocalConnection) SetEstablished() {
-	conn.sendAction(func() error {
-		stopTicker(conn.heartbeat)
-		old := conn.established
-		conn.Lock()
-		conn.established = true
-		conn.Unlock()
-		if old {
-			return nil
-		}
-		conn.Router.Ourself.ConnectionEstablished(conn)
-		if err := conn.ensureForwarders(); err != nil {
-			return err
-		}
-		// Send a large frame down the DF channel in order to prompt
-		// PMTU discovery to start.
-		conn.Send(true, PMTUDiscovery)
-		conn.heartbeat = time.NewTicker(SlowHeartbeat)
-		conn.fragTest = time.NewTicker(FragTestInterval)
-		// avoid initial waits for timers to fire
-		conn.Send(true, conn.heartbeatFrame)
-		conn.performFragTest()
-		return nil
-	})
-}
-
 // Send an actor request to the actorLoop, but don't block if
 // actorLoop has exited - see http://blog.golang.org/pipelines for
 // pattern
@@ -298,85 +188,75 @@ func (conn *LocalConnection) run(actionChan <-chan ConnectionAction, finished ch
 		return
 	}
 
-	if conn.SessionKey == nil {
-		conn.Decryptor = NewNonDecryptor()
-	} else {
-		conn.Decryptor = NewNaClDecryptor(conn.SessionKey, conn.outbound)
-	}
-
 	conn.Log("connection ready; using protocol version", conn.version)
 
-	// The ordering of the following is very important. [1]
-
-	if conn.remoteUDPAddr != nil {
-		if err = conn.ensureForwarders(); err != nil {
-			return
-		}
+	params := OverlayConnectionParams{
+		RemotePeer:         conn.remote,
+		LocalAddr:          conn.TCPConn.LocalAddr().(*net.TCPAddr),
+		RemoteAddr:         conn.TCPConn.RemoteAddr().(*net.TCPAddr),
+		Outbound:           conn.outbound,
+		ConnUID:            conn.uid,
+		SessionKey:         conn.SessionKey,
+		SendControlMessage: conn.sendOverlayControlMessage,
+		Features:           intro.Features,
 	}
+	if conn.OverlayConn, err = conn.Router.Overlay.PrepareConnection(params); err != nil {
+		return
+	}
+
+	// As soon as we do AddConnection, the new connection becomes
+	// visible to the packet routing logic.  So AddConnection must
+	// come after PrepareConnection
 	if err = conn.Router.Ourself.AddConnection(conn); err != nil {
 		return
 	}
-	if err = conn.initHeartbeats(); err != nil {
-		return
-	}
+	conn.Router.ConnectionMaker.ConnectionCreated(conn)
 
+	// OverlayConnection confirmation comes after AddConnection,
+	// because only after that completes do we know the connection is
+	// valid: in particular that it is not a duplicate connection to
+	// the same peer. Overlay communication on a duplicate connection
+	// can cause problems such as tripping up overlay crypto at the
+	// other end due to data being decoded by the other connection. It
+	// is also generally wasteful to engage in any interaction with
+	// the remote on a connection that turns out to be invalid.
+	conn.OverlayConn.Confirm()
+
+	// receiveTCP must follow also AddConnection. In the absence
+	// of any indirect connectivity to the remote peer, the first
+	// we hear about it (and any peers reachable from it) is
+	// through topology gossip it sends us on the connection. We
+	// must ensure that the connection has been added to Ourself
+	// prior to processing any such gossip, otherwise we risk
+	// immediately gc'ing part of that newly received portion of
+	// the topology (though not the remote peer itself, since that
+	// will have a positive ref count), leaving behind dangling
+	// references to peers. Hence we must invoke AddConnection,
+	// which is *synchronous*, first.
+	conn.heartbeatTCP = time.NewTicker(TCPHeartbeat)
 	go conn.receiveTCP(intro.Receiver)
+
+	// AddConnection must precede actorLoop. More precisely, it
+	// must precede shutdown, since that invokes DeleteConnection
+	// and is invoked on termination of this entire
+	// function. Essentially this boils down to a prohibition on
+	// running AddConnection in a separate goroutine, at least not
+	// without some synchronisation. Which in turn requires the
+	// launching of the receiveTCP goroutine to precede actorLoop.
 	err = conn.actorLoop(actionChan)
 }
 
-// [1] Ordering constraints:
-//
-// (a) AddConnections must precede initHeartbeats. It is only after
-// the former completes that we know the connection is valid, in
-// particular is not a duplicate connection to the same peer. Sending
-// heartbeats on a duplicate connection can trip up crypto at the
-// other end, since the associated UDP packets may get decoded by the
-// other connection. It is also generally wasteful to engage in any
-// interaction with the remote on a connection that turns out to be
-// invald.
-//
-// (b) AddConnection must precede receiveTCP. In the absence of any
-// indirect connectivity to the remote peer, the first we hear about
-// it (and any peers reachable from it) is through topology gossip it
-// sends us on the connection. We must ensure that the connection has
-// been added to Ourself prior to processing any such gossip,
-// otherwise we risk immediately gc'ing part of that newly received
-// portion of the topology (though not the remote peer itself, since
-// that will have a positive ref count), leaving behind dangling
-// references to peers. Hence we must invoke AddConnection, which is
-// *synchronous*, first.
-//
-// (c) AddConnection must precede actorLoop. More precisely, it must
-// precede shutdown, since that invokes DeleteConnection and is
-// invoked on termination of this entire function. Essentially this
-// boils down to a prohibition on running AddConnection in a separate
-// goroutine, at least not without some synchronisation. Which in turn
-// requires us the launching of the receiveTCP goroutine to precede
-// actorLoop.
-//
-// (d) AddConnection should precede receiveTCP. There is no point
-// starting the latter if the former fails.
-//
-// (e) initHeartbeats should precede actorLoop. The former is setting
-// LocalConnection fields accessed by the latter. Since the latter
-// runs in a separate goroutine, we'd have to add some synchronisation
-// if initHeartbeats isn't run first.
-//
-// (f) ensureForwarders should precede AddConnection. As soon as a
-// connection has been added to LocalPeer by the latter, it becomes
-// visible to the packet routing logic, which will end up dropping
-// packets if the forwarders haven't been created yet. We cannot
-// prevent that completely, since, for example, forwarder can only be
-// created when we know the remote UDP address, but it helps to try.
-
 func (conn *LocalConnection) makeFeatures() map[string]string {
-	return map[string]string{
+	features := map[string]string{
 		"PeerNameFlavour": PeerNameFlavour,
 		"Name":            conn.local.Name.String(),
 		"NickName":        conn.local.NickName,
+		"ShortID":         fmt.Sprint(conn.local.ShortID),
 		"UID":             fmt.Sprint(conn.local.UID),
 		"ConnID":          fmt.Sprint(conn.uid),
 	}
+	conn.Router.Overlay.AddFeaturesTo(features)
+	return features
 }
 
 type features map[string]string
@@ -411,6 +291,17 @@ func (conn *LocalConnection) parseFeatures(features features) (*Peer, error) {
 
 	nickName := features.Get("NickName")
 
+	var shortID uint64
+	var hasShortID bool
+	if shortIDStr, present := features["ShortID"]; present {
+		hasShortID = true
+		shortID, err = strconv.ParseUint(shortIDStr, 10,
+			PeerShortIDBits)
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	uid, err := ParsePeerUID(features.Get("UID"))
 	if err != nil {
 		return nil, err
@@ -422,7 +313,9 @@ func (conn *LocalConnection) parseFeatures(features features) (*Peer, error) {
 	}
 
 	conn.uid ^= remoteConnID
-	return NewPeer(name, nickName, uid, 0), nil
+	peer := NewPeer(name, nickName, uid, 0, PeerShortID(shortID))
+	peer.HasShortID = hasShortID
+	return peer, nil
 }
 
 func (conn *LocalConnection) registerRemote(remote *Peer, acceptNewPeer bool) error {
@@ -435,10 +328,6 @@ func (conn *LocalConnection) registerRemote(remote *Peer, acceptNewPeer bool) er
 		}
 	}
 
-	if conn.remote.UID != remote.UID {
-		return fmt.Errorf("Connection appears to be with different version of a peer we already know of")
-	}
-
 	if conn.remote == conn.local {
 		return ErrConnectToSelf
 	}
@@ -446,30 +335,24 @@ func (conn *LocalConnection) registerRemote(remote *Peer, acceptNewPeer bool) er
 	return nil
 }
 
-func (conn *LocalConnection) initHeartbeats() error {
-	conn.heartbeatTCP = time.NewTicker(TCPHeartbeat)
-	conn.heartbeatTimeout = time.NewTimer(HeartbeatTimeout)
-	conn.heartbeatFrame = make([]byte, EthernetOverhead+8)
-	binary.BigEndian.PutUint64(conn.heartbeatFrame[EthernetOverhead:], conn.uid)
-	if conn.remoteUDPAddr == nil {
-		return nil
-	}
-	return conn.sendFastHeartbeats()
-}
-
 func (conn *LocalConnection) actorLoop(actionChan <-chan ConnectionAction) (err error) {
+	fwdErrorChan := conn.OverlayConn.ErrorChannel()
+	fwdEstablishedChan := conn.OverlayConn.EstablishedChannel()
+
 	for err == nil {
 		select {
 		case action := <-actionChan:
 			err = action()
+
 		case <-conn.heartbeatTCP.C:
 			err = conn.sendSimpleProtocolMsg(ProtocolHeartbeat)
-		case <-conn.heartbeatTimeout.C:
-			err = fmt.Errorf("timed out waiting for UDP heartbeat")
-		case <-tickerChan(conn.heartbeat):
-			conn.Send(true, conn.heartbeatFrame)
-		case <-tickerChan(conn.fragTest):
-			conn.performFragTest()
+
+		case <-fwdEstablishedChan:
+			conn.established = true
+			fwdEstablishedChan = nil
+			conn.Router.Ourself.ConnectionEstablished(conn)
+
+		case err = <-fwdErrorChan:
 		}
 	}
 	return
@@ -491,19 +374,19 @@ func (conn *LocalConnection) shutdown(err error) {
 		conn.Router.Ourself.DeleteConnection(conn)
 	}
 
-	if conn.heartbeatTimeout != nil {
-		conn.heartbeatTimeout.Stop()
+	if conn.heartbeatTCP != nil {
+		conn.heartbeatTCP.Stop()
 	}
 
-	stopTicker(conn.heartbeatTCP)
-	stopTicker(conn.heartbeat)
-	stopTicker(conn.fragTest)
+	if conn.OverlayConn != nil {
+		conn.OverlayConn.Stop()
+	}
 
-	// blank out the forwardChan so that the router processes don't
-	// try to send any more
-	conn.stopForwarders()
+	conn.Router.ConnectionMaker.ConnectionTerminated(conn, err)
+}
 
-	conn.Router.ConnectionMaker.ConnectionTerminated(conn.remoteTCPAddr, err)
+func (conn *LocalConnection) sendOverlayControlMessage(tag byte, msg []byte) error {
+	return conn.sendProtocolMsg(ProtocolMsg{ProtocolTag(tag), msg})
 }
 
 // Helpers
@@ -513,7 +396,7 @@ func (conn *LocalConnection) sendSimpleProtocolMsg(tag ProtocolTag) error {
 }
 
 func (conn *LocalConnection) sendProtocolMsg(m ProtocolMsg) error {
-	return conn.tcpSender.Send(Concat([]byte{byte(m.tag)}, m.msg))
+	return conn.tcpSender.Send(append([]byte{byte(m.tag)}, m.msg...))
 }
 
 func (conn *LocalConnection) receiveTCP(receiver TCPReceiver) {
@@ -539,16 +422,8 @@ func (conn *LocalConnection) receiveTCP(receiver TCPReceiver) {
 func (conn *LocalConnection) handleProtocolMsg(tag ProtocolTag, payload []byte) error {
 	switch tag {
 	case ProtocolHeartbeat:
-	case ProtocolConnectionEstablished:
-		// We sent fast heartbeats to the remote peer, which has now
-		// received at least one of them and told us via this message.
-		// We can now consider the connection as established from our
-		// end.
-		conn.SetEstablished()
-	case ProtocolFragmentationReceived:
-		conn.setStackFrag(true)
-	case ProtocolPMTUVerified:
-		conn.pmtuVerified(int(binary.BigEndian.Uint16(payload)))
+	case ProtocolReserved1, ProtocolReserved2, ProtocolReserved3, ProtocolOverlayControlMsg:
+		conn.OverlayConn.ControlMessage(byte(tag), payload)
 	case ProtocolGossipUnicast, ProtocolGossipBroadcast, ProtocolGossip:
 		return conn.Router.handleGossip(tag, payload)
 	default:
@@ -560,30 +435,3 @@ func (conn *LocalConnection) handleProtocolMsg(tag ProtocolTag, payload []byte)
 func (conn *LocalConnection) extendReadDeadline() {
 	conn.TCPConn.SetReadDeadline(time.Now().Add(TCPHeartbeat * 2))
 }
-
-func (conn *LocalConnection) sendFastHeartbeats() error {
-	err := conn.ensureForwarders()
-	if err == nil {
-		conn.heartbeat = time.NewTicker(FastHeartbeat)
-		conn.Send(true, conn.heartbeatFrame) // avoid initial wait
-	}
-	return err
-}
-
-func (conn *LocalConnection) performFragTest() {
-	conn.setStackFrag(false)
-	conn.Send(false, FragTest)
-}
-
-func tickerChan(ticker *time.Ticker) <-chan time.Time {
-	if ticker != nil {
-		return ticker.C
-	}
-	return nil
-}
-
-func stopTicker(ticker *time.Ticker) {
-	if ticker != nil {
-		ticker.Stop()
-	}
-}

mesh/connection_maker.go

@@ -1,4 +1,4 @@
-package router
+package mesh
 
 import (
 	"fmt"
@@ -10,6 +10,7 @@ import (
 const (
 	InitialInterval = 2 * time.Second
 	MaxInterval     = 6 * time.Minute
+	ResetAfter      = 1 * time.Minute
 )
 
 type peerAddrs map[string]*net.TCPAddr
@@ -20,13 +21,22 @@ type ConnectionMaker struct {
 	port        int
 	discovery   bool
 	targets     map[string]*Target
+	connections map[Connection]struct{}
 	directPeers peerAddrs
 	actionChan  chan<- ConnectionMakerAction
 }
 
+type TargetState int
+
+const (
+	TargetWaiting    TargetState = iota // we are waiting to connect there
+	TargetAttempting                    // we are attempting to connect there
+	TargetConnected                     // we are connected to there
+)
+
 // Information about an address where we may find a peer
 type Target struct {
-	attempting  bool          // are we currently attempting to connect there?
+	state       TargetState
 	lastError   error         // reason for disconnection last time
 	tryAfter    time.Time     // next time to try this address
 	tryInterval time.Duration // retry delay on next failure
@@ -43,6 +53,7 @@ func NewConnectionMaker(ourself *LocalPeer, peers *Peers, port int, discovery bo
 		discovery:   discovery,
 		directPeers: peerAddrs{},
 		targets:     make(map[string]*Target),
+		connections: make(map[Connection]struct{}),
 		actionChan:  actionChan}
 	go cm.queryLoop(actionChan)
 	return cm
@@ -71,7 +82,7 @@ func (cm *ConnectionMaker) InitiateConnections(peers []string, replace bool) []e
 			cm.directPeers[peer] = addr
 			// curtail any existing reconnect interval
 			if target, found := cm.targets[addr.String()]; found {
-				target.tryNow()
+				target.nextTryNow()
 			}
 		}
 		return true
@@ -88,12 +99,42 @@ func (cm *ConnectionMaker) ForgetConnections(peers []string) {
 	}
 }
 
-func (cm *ConnectionMaker) ConnectionTerminated(address string, err error) {
+func (cm *ConnectionMaker) ConnectionAborted(address string, err error) {
 	cm.actionChan <- func() bool {
-		if target, found := cm.targets[address]; found {
-			target.attempting = false
+		target := cm.targets[address]
+		target.state = TargetWaiting
+		target.lastError = err
+		target.nextTryLater()
+		return true
+	}
+}
+
+func (cm *ConnectionMaker) ConnectionCreated(conn Connection) {
+	cm.actionChan <- func() bool {
+		cm.connections[conn] = void
+		if conn.Outbound() {
+			target := cm.targets[conn.RemoteTCPAddr()]
+			target.state = TargetConnected
+		}
+		return false
+	}
+}
+
+func (cm *ConnectionMaker) ConnectionTerminated(conn Connection, err error) {
+	cm.actionChan <- func() bool {
+		delete(cm.connections, conn)
+		if conn.Outbound() {
+			target := cm.targets[conn.RemoteTCPAddr()]
+			target.state = TargetWaiting
 			target.lastError = err
-			target.retry()
+			switch {
+			case err == ErrConnectToSelf:
+				target.nextTryNever()
+			case time.Now().After(target.tryAfter.Add(ResetAfter)):
+				target.nextTryNow()
+			default:
+				target.nextTryLater()
+			}
 		}
 		return true
 	}
@@ -123,9 +164,6 @@ func (cm *ConnectionMaker) checkStateAndAttemptConnections() time.Duration {
 		validTarget  = make(map[string]struct{})
 		directTarget = make(map[string]struct{})
 	)
-	// Copy the set of things we are connected to, so we can access
-	// them without locking.  Also clear out any entries in cm.targets
-	// for existing connections.
 	ourConnectedPeers, ourConnectedTargets, ourInboundIPs := cm.ourConnections()
 
 	addTarget := func(address string) {
@@ -136,8 +174,8 @@ func (cm *ConnectionMaker) checkStateAndAttemptConnections() time.Duration {
 		if _, found := cm.targets[address]; found {
 			return
 		}
-		target := &Target{}
-		target.tryNow()
+		target := &Target{state: TargetWaiting}
+		target.nextTryNow()
 		cm.targets[address] = target
 	}
 
@@ -176,9 +214,8 @@ func (cm *ConnectionMaker) ourConnections() (PeerNameSet, map[string]struct{}, m
 		ourConnectedTargets = make(map[string]struct{})
 		ourInboundIPs       = make(map[string]struct{})
 	)
-	for conn := range cm.ourself.Connections() {
+	for conn := range cm.connections {
 		address := conn.RemoteTCPAddr()
-		delete(cm.targets, address)
 		ourConnectedPeers[conn.Remote().Name] = void
 		ourConnectedTargets[address] = void
 		if conn.Outbound() {
@@ -212,7 +249,7 @@ func (cm *ConnectionMaker) addPeerTargets(ourConnectedPeers PeerNameSet, addTarg
 			} else if ip, _, err := net.SplitHostPort(address); err == nil {
 				// There is no point connecting to the (likely
 				// ephemeral) remote port of an inbound connection
-				// that some peer has. Let's try to connect to on the
+				// that some peer has. Let's try to connect on the
 				// weave port instead.
 				addTarget(fmt.Sprintf("%s:%d", ip, cm.port))
 			}
@@ -224,7 +261,7 @@ func (cm *ConnectionMaker) connectToTargets(validTarget map[string]struct{}, dir
 	now := time.Now() // make sure we catch items just added
 	after := MaxDuration
 	for address, target := range cm.targets {
-		if target.attempting {
+		if target.state != TargetWaiting {
 			continue
 		}
 		if _, valid := validTarget[address]; !valid {
@@ -236,7 +273,7 @@ func (cm *ConnectionMaker) connectToTargets(validTarget map[string]struct{}, dir
 		}
 		switch duration := target.tryAfter.Sub(now); {
 		case duration <= 0:
-			target.attempting = true
+			target.state = TargetAttempting
 			_, isCmdLineTarget := directTarget[address]
 			go cm.attemptConnection(address, isCmdLineTarget)
 		case duration < after:
@@ -250,23 +287,23 @@ func (cm *ConnectionMaker) attemptConnection(address string, acceptNewPeer bool)
 	log.Printf("->[%s] attempting connection", address)
 	if err := cm.ourself.CreateConnection(address, acceptNewPeer); err != nil {
 		log.Errorf("->[%s] error during connection attempt: %v", address, err)
-		cm.ConnectionTerminated(address, err)
+		cm.ConnectionAborted(address, err)
 	}
 }
 
-func (t *Target) tryNow() {
+func (t *Target) nextTryNever() {
+	t.tryAfter = time.Time{}
+	t.tryInterval = MaxInterval
+}
+
+func (t *Target) nextTryNow() {
 	t.tryAfter = time.Now()
 	t.tryInterval = InitialInterval
 }
 
 // The delay at the nth retry is a random value in the range
 // [i-i/2,i+i/2], where i = InitialInterval * 1.5^(n-1).
-func (t *Target) retry() {
-	if t.lastError == ErrConnectToSelf {
-		t.tryAfter = time.Time{}
-		t.tryInterval = MaxInterval
-		return
-	}
+func (t *Target) nextTryLater() {
 	t.tryAfter = time.Now().Add(t.tryInterval/2 + time.Duration(rand.Int63n(int64(t.tryInterval))))
 	t.tryInterval = t.tryInterval * 3 / 2
 	if t.tryInterval > MaxInterval {

mesh/gossip.go

@@ -1,4 +1,4 @@
-package router
+package mesh
 
 import (
 	"fmt"
@@ -6,7 +6,7 @@ import (
 
 type GossipData interface {
 	Encode() [][]byte
-	Merge(GossipData)
+	Merge(GossipData) GossipData
 }
 
 type Gossip interface {
@@ -79,8 +79,7 @@ func (sender *GossipSender) Send(data GossipData) {
 	// NB: this must not be invoked concurrently
 	select {
 	case pending := <-sender.cell:
-		pending.Merge(data)
-		sender.cell <- pending
+		sender.cell <- pending.Merge(data)
 	default:
 		sender.cell <- data
 	}
@@ -142,7 +141,7 @@ func (router *Router) SendAllGossip() {
 func (router *Router) SendAllGossipDown(conn Connection) {
 	for channel := range router.gossipChannelSet() {
 		if gossip := channel.gossiper.Gossip(); gossip != nil {
-			channel.SendDown(conn, channel.gossiper.Gossip())
+			channel.SendDown(conn, gossip)
 		}
 	}
 }

mesh/gossip_channel.go

@@ -1,4 +1,4 @@
-package router
+package mesh
 
 import (
 	"bytes"
@@ -239,3 +239,12 @@ func (c *GossipChannel) sendBroadcast(srcName PeerName, update GossipData) {
 func (c *GossipChannel) log(args ...interface{}) {
 	log.Println(append(append([]interface{}{}, "[gossip "+c.name+"]:"), args...)...)
 }
+
+func GobEncode(items ...interface{}) []byte {
+	buf := new(bytes.Buffer)
+	enc := gob.NewEncoder(buf)
+	for _, i := range items {
+		checkFatal(enc.Encode(i))
+	}
+	return buf.Bytes()
+}

mesh/gossip_test.go

@@ -1,4 +1,4 @@
-package router
+package mesh
 
 import (
 	"fmt"
@@ -20,7 +20,7 @@ type mockChannelConnection struct {
 
 func NewTestRouter(name string) *Router {
 	peerName, _ := PeerNameFromString(name)
-	router := NewRouter(Config{}, peerName, "nick")
+	router := NewRouter(Config{}, peerName, "nick", nil)
 	router.Start()
 	return router
 }
@@ -42,11 +42,8 @@ func sendPendingGossip(routers ...*Router) {
 }
 
 func (router *Router) AddTestChannelConnection(r *Router) {
-	fromName := router.Ourself.Peer.Name
-	toName := r.Ourself.Peer.Name
-
-	fromPeer := NewPeer(fromName, "", router.Ourself.Peer.UID, 0)
-	toPeer := NewPeer(toName, "", r.Ourself.Peer.UID, 0)
+	fromPeer := NewPeerFrom(router.Ourself.Peer)
+	toPeer := NewPeerFrom(r.Ourself.Peer)
 
 	r.Peers.FetchWithDefault(fromPeer)    // Has side-effect of incrementing refcount
 	router.Peers.FetchWithDefault(toPeer) //
@@ -80,13 +77,13 @@ func TestGossipTopology(t *testing.T) {
 // the routers supplied as arguments, carrying across all UID and
 // version information.
 func (router *Router) tp(routers ...*Router) *Peer {
-	peer := NewPeer(router.Ourself.Peer.Name, "", router.Ourself.Peer.UID, 0)
+	peer := NewPeerFrom(router.Ourself.Peer)
 	connections := make(map[PeerName]Connection)
 	for _, r := range routers {
-		p := NewPeer(r.Ourself.Peer.Name, "", r.Ourself.Peer.UID, r.Ourself.Peer.version)
+		p := NewPeerFrom(r.Ourself.Peer)
 		connections[r.Ourself.Peer.Name] = newMockConnection(peer, p)
 	}
-	peer.version = router.Ourself.Peer.version
+	peer.Version = router.Ourself.Peer.Version
 	peer.connections = connections
 	return peer
 }

mesh/local_peer.go

@@ -1,4 +1,4 @@
-package router
+package mesh
 
 import (
 	"fmt"
@@ -19,7 +19,7 @@ type LocalPeerAction func()
 func NewLocalPeer(name PeerName, nickName string, router *Router) *LocalPeer {
 	actionChan := make(chan LocalPeerAction, ChannelSize)
 	peer := &LocalPeer{
-		Peer:       NewPeer(name, nickName, 0, 0),
+		Peer:       NewPeer(name, nickName, randomPeerUID(), 0, randomPeerShortID()),
 		router:     router,
 		actionChan: actionChan,
 	}
@@ -27,56 +27,6 @@ func NewLocalPeer(name PeerName, nickName string, router *Router) *LocalPeer {
 	return peer
 }
 
-func (peer *LocalPeer) Forward(dstPeer *Peer, frame []byte, dec *EthernetDecoder) error {
-	return peer.Relay(peer.Peer, dstPeer, frame, dec)
-}
-
-func (peer *LocalPeer) Broadcast(frame []byte, dec *EthernetDecoder) {
-	peer.RelayBroadcast(peer.Peer, frame, dec)
-}
-
-func (peer *LocalPeer) Relay(srcPeer, dstPeer *Peer, frame []byte, dec *EthernetDecoder) error {
-	relayPeerName, found := peer.router.Routes.Unicast(dstPeer.Name)
-	if !found {
-		// Not necessarily an error as there could be a race with the
-		// dst disappearing whilst the frame is in flight
-		log.Println("Received packet for unknown destination:", dstPeer)
-		return nil
-	}
-	conn, found := peer.ConnectionTo(relayPeerName)
-	if !found {
-		// Again, could just be a race, not necessarily an error
-		log.Println("Unable to find connection to relay peer", relayPeerName)
-		return nil
-	}
-	return conn.(*LocalConnection).Forward(&ForwardedFrame{
-		srcPeer: srcPeer,
-		dstPeer: dstPeer,
-		frame:   frame},
-		dec)
-}
-
-func (peer *LocalPeer) RelayBroadcast(srcPeer *Peer, frame []byte, dec *EthernetDecoder) {
-	nextHops := peer.router.Routes.Broadcast(srcPeer.Name)
-	if len(nextHops) == 0 {
-		return
-	}
-	for _, conn := range peer.ConnectionsTo(nextHops) {
-		err := conn.(*LocalConnection).Forward(&ForwardedFrame{
-			srcPeer: srcPeer,
-			dstPeer: conn.Remote(),
-			frame:   frame},
-			dec)
-		if err != nil {
-			if ftbe, ok := err.(FrameTooBigError); ok {
-				log.Warningf("dropping too big DF broadcast frame (%v -> %v): PMTU=%v", dec.IP.SrcIP, dec.IP.DstIP, ftbe.EPMTU)
-			} else {
-				log.Errorln(err)
-			}
-		}
-	}
-}
-
 func (peer *LocalPeer) Connections() ConnectionSet {
 	connections := make(ConnectionSet)
 	peer.RLock()
@@ -112,21 +62,16 @@ func (peer *LocalPeer) CreateConnection(peerAddr string, acceptNewPeer bool) err
 	if err := peer.checkConnectionLimit(); err != nil {
 		return err
 	}
-	tcpAddr, tcpErr := net.ResolveTCPAddr("tcp4", peerAddr)
-	udpAddr, udpErr := net.ResolveUDPAddr("udp4", peerAddr)
-	if tcpErr != nil || udpErr != nil {
-		// they really should have the same value, but just in case...
-		if tcpErr == nil {
-			return udpErr
-		}
-		return tcpErr
+	tcpAddr, err := net.ResolveTCPAddr("tcp4", peerAddr)
+	if err != nil {
+		return err
 	}
 	tcpConn, err := net.DialTCP("tcp4", nil, tcpAddr)
 	if err != nil {
 		return err
 	}
 	connRemote := NewRemoteConnection(peer.Peer, nil, tcpConn.RemoteAddr().String(), true, false)
-	StartLocalConnection(connRemote, tcpConn, udpAddr, peer.router, acceptNewPeer)
+	StartLocalConnection(connRemote, tcpConn, peer.router, acceptNewPeer)
 	return nil
 }
 
@@ -210,7 +155,10 @@ func (peer *LocalPeer) handleAddConnection(conn Connection) error {
 		conn.Log("connection added (new peer)")
 		peer.router.SendAllGossipDown(conn)
 	}
+
+	peer.router.Routes.Recalculate()
 	peer.broadcastPeerUpdate(conn.Remote())
+
 	return nil
 }
 
@@ -224,6 +172,8 @@ func (peer *LocalPeer) handleConnectionEstablished(conn Connection) {
 	}
 	peer.connectionEstablished(conn)
 	conn.Log("connection fully established")
+
+	peer.router.Routes.Recalculate()
 	peer.broadcastPeerUpdate()
 }
 
@@ -243,14 +193,20 @@ func (peer *LocalPeer) handleDeleteConnection(conn Connection) {
 	// Must do garbage collection first to ensure we don't send out an
 	// update with unreachable peers (can cause looping)
 	peer.router.Peers.GarbageCollect()
+	peer.router.Routes.Recalculate()
 	peer.broadcastPeerUpdate()
 }
 
 // helpers
 
 func (peer *LocalPeer) broadcastPeerUpdate(peers ...*Peer) {
-	peer.router.Routes.Recalculate()
-	peer.router.TopologyGossip.GossipBroadcast(NewTopologyGossipData(peer.router.Peers, append(peers, peer.Peer)...))
+	// Some tests run without a router.  This should be fixed so
+	// that the relevant part of Router can be easily run in the
+	// context of a test, but that will involve significant
+	// reworking of tests.
+	if peer.router != nil {
+		peer.router.BroadcastTopologyUpdate(append(peers, peer.Peer))
+	}
 }
 
 func (peer *LocalPeer) checkConnectionLimit() error {
@@ -265,20 +221,20 @@ func (peer *LocalPeer) addConnection(conn Connection) {
 	peer.Lock()
 	defer peer.Unlock()
 	peer.connections[conn.Remote().Name] = conn
-	peer.version++
+	peer.Version++
 }
 
 func (peer *LocalPeer) deleteConnection(conn Connection) {
 	peer.Lock()
 	defer peer.Unlock()
 	delete(peer.connections, conn.Remote().Name)
-	peer.version++
+	peer.Version++
 }
 
 func (peer *LocalPeer) connectionEstablished(conn Connection) {
 	peer.Lock()
 	defer peer.Unlock()
-	peer.version++
+	peer.Version++
 }
 
 func (peer *LocalPeer) connectionCount() int {
@@ -286,3 +242,20 @@ func (peer *LocalPeer) connectionCount() int {
 	defer peer.RUnlock()
 	return len(peer.connections)
 }
+
+func (peer *LocalPeer) setShortID(shortID PeerShortID) {
+	peer.Lock()
+	defer peer.Unlock()
+	peer.ShortID = shortID
+	peer.Version++
+}
+
+func (peer *LocalPeer) setVersionBeyond(version uint64) bool {
+	peer.Lock()
+	defer peer.Unlock()
+	if version >= peer.Version {
+		peer.Version = version + 1
+		return true
+	}
+	return false
+}

mesh/mocks_test.go

@@ -3,7 +3,7 @@
 // It supplies some mock implementations to other unit tests, and is
 // named "...test.go" so it is only compiled under `go test`.
 
-package router
+package mesh
 
 import (
 	"fmt"
@@ -14,7 +14,9 @@ import (
 
 // Add to peers a connection from peers.ourself to p
 func (peers *Peers) AddTestConnection(p *Peer) {
-	toPeer := NewPeer(p.Name, "", p.UID, 0)
+	summary := p.PeerSummary
+	summary.Version = 0
+	toPeer := NewPeerFromSummary(summary)
 	peers.FetchWithDefault(toPeer) // Has side-effect of incrementing refcount
 	conn := newMockConnection(peers.ourself.Peer, toPeer)
 	peers.ourself.addConnection(conn)
@@ -23,11 +25,9 @@ func (peers *Peers) AddTestConnection(p *Peer) {
 
 // Add to peers a connection from p1 to p2
 func (peers *Peers) AddTestRemoteConnection(p1, p2 *Peer) {
-	fromName := p1.Name
-	fromPeer := NewPeer(fromName, "", p1.UID, 0)
+	fromPeer := NewPeerFrom(p1)
 	fromPeer = peers.FetchWithDefault(fromPeer)
-	toName := p2.Name
-	toPeer := NewPeer(toName, "", p2.UID, 0)
+	toPeer := NewPeerFrom(p2)
 	toPeer = peers.FetchWithDefault(toPeer)
 	peers.ourself.addConnection(&RemoteConnection{fromPeer, toPeer, "", false, false})
 }
@@ -70,7 +70,7 @@ func checkEqualConns(t *testing.T, ourName PeerName, got, wanted map[PeerName]Co
 // Get all the peers from a Peers in a slice
 func (peers *Peers) allPeers() []*Peer {
 	var res []*Peer
-	for _, peer := range peers.table {
+	for _, peer := range peers.byName {
 		res = append(res, peer)
 	}
 	return res

mesh/overlay.go

@@ -0,0 +1,116 @@
+package mesh
+
+import (
+	"net"
+)
+
+type Overlay interface {
+	// Enhance a features map with overlay-related features
+	AddFeaturesTo(map[string]string)
+
+	// Prepare on overlay connection. The connection should remain
+	// passive until it has been Confirm()ed.
+	PrepareConnection(OverlayConnectionParams) (OverlayConnection, error)
+
+	// Obtain diagnostic information specific to the overlay
+	Diagnostics() interface{}
+}
+
+type OverlayConnectionParams struct {
+	RemotePeer *Peer
+
+	// The local address of the corresponding TCP connection. Used to
+	// derive the local IP address for sending. May differ for
+	// different overlay connections.
+	LocalAddr *net.TCPAddr
+
+	// The remote address of the corresponding TCP connection. Used to
+	// determine the address to send to, but only if the TCP
+	// connection is outbound. Otherwise the Overlay needs to discover
+	// it (e.g. from incoming datagrams).
+	RemoteAddr *net.TCPAddr
+
+	// Is the corresponding TCP connection outbound?
+	Outbound bool
+
+	// Unique identifier for this connection
+	ConnUID uint64
+
+	// Session key, if connection is encrypted; nil otherwise.
+	//
+	// NB: overlay connections must take care not to use nonces which
+	// may collide with those of the main connection. These nonces are
+	// 192 bits, with the top most bit unspecified, the next bit set
+	// to 1, followed by 126 zero bits, and a message sequence number
+	// in the lowest 64 bits.
+	SessionKey *[32]byte
+
+	// Function to send a control message to the counterpart
+	// overlay connection.
+	SendControlMessage func(tag byte, msg []byte) error
+
+	// Features passed at connection initiation
+	Features map[string]string
+}
+
+// All of the machinery to manage overlay connectivity to a particular
+// peer
+type OverlayConnection interface {
+	// Confirm that the connection is really wanted, and so the
+	// Overlay should begin heartbeats etc. to verify the operation of
+	// the overlay connection.
+	Confirm()
+
+	// A channel indicating that the overlay connection is
+	// established, i.e. its operation has been confirmed.
+	EstablishedChannel() <-chan struct{}
+
+	// A channel indicating an error from the overlay connection.  The
+	// overlay connection is not expected to be operational after the
+	// first error, so the channel only needs to buffer a single
+	// error.
+	ErrorChannel() <-chan error
+
+	Stop()
+
+	// Handle a message from the peer.  'tag' exists for
+	// compatibility, and should always be
+	// ProtocolOverlayControlMessage for non-sleeve overlays.
+	ControlMessage(tag byte, msg []byte)
+
+	// User facing overlay name
+	DisplayName() string
+}
+
+type NullOverlay struct{}
+
+func (NullOverlay) AddFeaturesTo(map[string]string) {
+}
+
+func (NullOverlay) PrepareConnection(OverlayConnectionParams) (OverlayConnection, error) {
+	return NullOverlay{}, nil
+}
+
+func (NullOverlay) Diagnostics() interface{} {
+	return nil
+}
+func (NullOverlay) Confirm() {
+}
+
+func (NullOverlay) EstablishedChannel() <-chan struct{} {
+	return nil
+}
+
+func (NullOverlay) ErrorChannel() <-chan error {
+	return nil
+}
+
+func (NullOverlay) Stop() {
+}
+
+func (NullOverlay) ControlMessage(byte, []byte) {
+}
+
+func (NullOverlay) DisplayName() string {
+	return "null"
+}

mesh/peer.go

@@ -1,41 +1,102 @@
-package router
+package mesh
 
 import (
+	"crypto/rand"
+	"encoding/binary"
 	"fmt"
 	"sort"
 	"strconv"
 )
 
+func randBytes(n int) []byte {
+	buf := make([]byte, n)
+	_, err := rand.Read(buf)
+	checkFatal(err)
+	return buf
+}
+
+func randUint64() (r uint64) {
+	return binary.LittleEndian.Uint64(randBytes(8))
+}
+
+func randUint16() (r uint16) {
+	return binary.LittleEndian.Uint16(randBytes(2))
+}
+
 type PeerUID uint64
 
+func randomPeerUID() PeerUID {
+	return PeerUID(randUint64())
+}
+
 func ParsePeerUID(s string) (PeerUID, error) {
 	uid, err := strconv.ParseUint(s, 10, 64)
 	return PeerUID(uid), err
 }
 
+// Short IDs exist for the sake of fast datapath.  They are 12 bits,
+// randomly assigned, but we detect and recover from collisions.  This
+// does limit us to 4096 peers, but that should be sufficient for a
+// while.
+type PeerShortID uint16
+
+const PeerShortIDBits = 12
+
+func randomPeerShortID() PeerShortID {
+	return PeerShortID(randUint16() & (1<<PeerShortIDBits - 1))
+}
+
+type PeerSummary struct {
+	NameByte   []byte
+	NickName   string
+	UID        PeerUID
+	Version    uint64
+	ShortID    PeerShortID
+	HasShortID bool
+}
+
 type Peer struct {
-	Name          PeerName
-	NameByte      []byte
-	NickName      string
-	UID           PeerUID
-	version       uint64
+	Name PeerName
+	PeerSummary
 	localRefCount uint64 // maintained by Peers
 	connections   map[PeerName]Connection
 }
 
+type ListOfPeers []*Peer
+
+func (lop ListOfPeers) Len() int {
+	return len(lop)
+}
+func (lop ListOfPeers) Swap(i, j int) {
+	lop[i], lop[j] = lop[j], lop[i]
+}
+func (lop ListOfPeers) Less(i, j int) bool {
+	return lop[i].Name < lop[j].Name
+}
+
 type ConnectionSet map[Connection]struct{}
 
-func NewPeer(name PeerName, nickName string, uid PeerUID, version uint64) *Peer {
-	if uid == 0 {
-		uid = PeerUID(randUint64())
-	}
+func NewPeerFromSummary(summary PeerSummary) *Peer {
 	return &Peer{
-		Name:        name,
-		NameByte:    name.Bin(),
-		NickName:    nickName,
-		UID:         uid,
-		version:     version,
-		connections: make(map[PeerName]Connection)}
+		Name:        PeerNameFromBin(summary.NameByte),
+		PeerSummary: summary,
+		connections: make(map[PeerName]Connection),
+	}
+}
+
+func NewPeer(name PeerName, nickName string, uid PeerUID, version uint64, shortID PeerShortID) *Peer {
+	return NewPeerFromSummary(PeerSummary{
+		NameByte:   name.Bin(),
+		NickName:   nickName,
+		UID:        uid,
+		Version:    version,
+		ShortID:    shortID,
+		HasShortID: true,
+	})
+}
+
+func NewPeerFrom(peer *Peer) *Peer {
+	return NewPeerFromSummary(peer.PeerSummary)
 }
 
 func (peer *Peer) String() string {
@@ -66,7 +127,7 @@ func (peer *Peer) String() string {
 // NB: This function should generally be invoked while holding a read
 // lock on Peers and LocalPeer.
 func (peer *Peer) Routes(stopAt *Peer, establishedAndSymmetric bool) (bool, map[PeerName]PeerName) {
-	routes := make(map[PeerName]PeerName)
+	routes := make(unicastRoutes)
 	routes[peer.Name] = UnknownPeerName
 	nextWorklist := []*Peer{peer}
 	for len(nextWorklist) > 0 {

mesh/peer_name_hash.go

@@ -3,7 +3,7 @@
 // Let peer names be sha256 hashes...of anything (as long as it's
 // unique).
 
-package router
+package mesh
 
 import (
 	"crypto/sha256"

mesh/peer_name_mac.go

@@ -15,7 +15,7 @@
 // name. In particular it doesn't actually have to be the MAC of, say,
 // the network interface the peer is sniffing on.
 
-package router
+package mesh
 
 import (
 	"net"
@@ -56,3 +56,20 @@ func (name PeerName) Bin() []byte {
 func (name PeerName) String() string {
 	return intmac(uint64(name)).String()
 }
+
+func macint(mac net.HardwareAddr) (r uint64) {
+	for _, b := range mac {
+		r <<= 8
+		r |= uint64(b)
+	}
+	return
+}
+
+func intmac(key uint64) (r net.HardwareAddr) {
+	r = make([]byte, 6)
+	for i := 5; i >= 0; i-- {
+		r[i] = byte(key)
+		key >>= 8
+	}
+	return
+}

mesh/peers.go

@@ -0,0 +1,538 @@
+package mesh
+
+import (
+	"bytes"
+	"encoding/gob"
+	"fmt"
+	"io"
+	"math/rand"
+	"sync"
+)
+
+var void = struct{}{}
+
+type Peers struct {
+	sync.RWMutex
+	ourself   *LocalPeer
+	byName    map[PeerName]*Peer
+	byShortID map[PeerShortID]ShortIDPeers
+	onGC      []func(*Peer)
+
+	// Called when the mapping from short ids to peers changes
+	onInvalidateShortIDs []func()
+}
+
+type ShortIDPeers struct {
+	// If we know about a single peer with the short id, this is
+	// that peer.  If there is a collision, this is the peer with
+	// the lowest Name.
+	peer *Peer
+
+	// In case of a collision, this holds the other peers.
+	others []*Peer
+}
+
+type UnknownPeerError struct {
+	Name PeerName
+}
+
+func (upe UnknownPeerError) Error() string {
+	return fmt.Sprint("Reference to unknown peer ", upe.Name)
+}
+
+type PeerNameSet map[PeerName]struct{}
+
+type ConnectionSummary struct {
+	NameByte      []byte
+	RemoteTCPAddr string
+	Outbound      bool
+	Established   bool
+}
+
+// Pending notifications due to changes to Peers that need to be sent
+// out once the Peers is unlocked.
+type PeersPendingNotifications struct {
+	// Peers that have been GCed
+	removed []*Peer
+
+	// The mapping from shorts ids to peers changed
+	invalidateShortIDs bool
+
+	// The local short ID needs reassigning due to a collision
+	reassignLocalShortID bool
+
+	// The local peer was modified
+	localPeerModified bool
+}
+
+func NewPeers(ourself *LocalPeer) *Peers {
+	peers := &Peers{
+		ourself:   ourself,
+		byName:    make(map[PeerName]*Peer),
+		byShortID: make(map[PeerShortID]ShortIDPeers),
+	}
+	peers.FetchWithDefault(ourself.Peer)
+	return peers
+}
+
+func (peers *Peers) OnGC(callback func(*Peer)) {
+	peers.Lock()
+	defer peers.Unlock()
+
+	// Although the array underlying peers.onGC might be accessed
+	// without holding the lock in unlockAndNotify, we don't
+	// support removing callbacks, so a simple append here is
+	// safe.
+	peers.onGC = append(peers.onGC, callback)
+}
+
+func (peers *Peers) OnInvalidateShortIDs(callback func()) {
+	peers.Lock()
+	defer peers.Unlock()
+
+	// Safe, as in OnGC
+	peers.onInvalidateShortIDs = append(peers.onInvalidateShortIDs, callback)
+}
+
+func (peers *Peers) unlockAndNotify(pending *PeersPendingNotifications) {
+	broadcastLocalPeer := (pending.reassignLocalShortID && peers.reassignLocalShortID(pending)) ||
+		pending.localPeerModified
+	onGC := peers.onGC
+	onInvalidateShortIDs := peers.onInvalidateShortIDs
+	peers.Unlock()
+
+	if pending.removed != nil {
+		for _, callback := range onGC {
+			for _, peer := range pending.removed {
+				callback(peer)
+			}
+		}
+	}
+
+	if pending.invalidateShortIDs {
+		for _, callback := range onInvalidateShortIDs {
+			callback()
+		}
+	}
+
+	if broadcastLocalPeer {
+		peers.ourself.broadcastPeerUpdate()
+	}
+}
+
+func (peers *Peers) addByShortID(peer *Peer, pending *PeersPendingNotifications) {
+	if !peer.HasShortID {
+		return
+	}
+
+	entry, ok := peers.byShortID[peer.ShortID]
+	if !ok {
+		entry = ShortIDPeers{peer: peer}
+	} else if entry.peer == nil {
+		// This short ID is free, but was used in the past.
+		// Because we are reusing it, it's an invalidation
+		// event.
+		entry.peer = peer
+		pending.invalidateShortIDs = true
+	} else if peer.Name < entry.peer.Name {
+		// Short ID collision, this peer becomes the principal
+		// peer for the short ID, bumping the previous one
+		// into others.
+
+		if entry.peer == peers.ourself.Peer {
+			// The bumped peer is peers.ourself, so we
+			// need to look for a new short id
+			pending.reassignLocalShortID = true
+		}
+
+		entry.others = append(entry.others, entry.peer)
+		entry.peer = peer
+		pending.invalidateShortIDs = true
+	} else {
+		// Short ID collision, this peer is secondary
+		entry.others = append(entry.others, peer)
+	}
+
+	peers.byShortID[peer.ShortID] = entry
+}
+
+func (peers *Peers) deleteByShortID(peer *Peer, pending *PeersPendingNotifications) {
+	if !peer.HasShortID {
+		return
+	}
+
+	entry := peers.byShortID[peer.ShortID]
+	var otherIndex int
+
+	if peer != entry.peer {
+		// peer is secondary, find its index in others
+		otherIndex = -1
+
+		for i, other := range entry.others {
+			if peer == other {
+				otherIndex = i
+				break
+			}
+		}
+
+		if otherIndex < 0 {
+			return
+		}
+	} else if len(entry.others) != 0 {
+		// need to find the peer with the lowest name to
+		// become the new principal one
+		otherIndex = 0
+		minName := entry.others[0].Name
+
+		for i := 1; i < len(entry.others); i++ {
+			otherName := entry.others[i].Name
+			if otherName < minName {
+				minName = otherName
+				otherIndex = i
+			}
+		}
+
+		entry.peer = entry.others[otherIndex]
+		pending.invalidateShortIDs = true
+	} else {
+		// This is the last peer with the short id.  We clear
+		// the entry, don't delete it, in order to detect when
+		// it gets re-used.
+		peers.byShortID[peer.ShortID] = ShortIDPeers{}
+		return
+	}
+
+	entry.others[otherIndex] = entry.others[len(entry.others)-1]
+	entry.others = entry.others[:len(entry.others)-1]
+	peers.byShortID[peer.ShortID] = entry
+}
+
+func (peers *Peers) reassignLocalShortID(pending *PeersPendingNotifications) bool {
+	newShortID, ok := peers.chooseShortID()
+	if ok {
+		peers.setLocalShortID(newShortID, pending)
+		return true
+	}
+
+	// Otherwise we'll try again later on in garbageColleect
+	return false
+}
+
+func (peers *Peers) setLocalShortID(newShortID PeerShortID, pending *PeersPendingNotifications) {
+	peers.deleteByShortID(peers.ourself.Peer, pending)
+	peers.ourself.setShortID(newShortID)
+	peers.addByShortID(peers.ourself.Peer, pending)
+}
+
+// Choose an available short id at random
+func (peers *Peers) chooseShortID() (PeerShortID, bool) {
+	rng := rand.New(rand.NewSource(int64(randUint64())))
+
+	// First, just try picking some short ids at random, and
+	// seeing if they are available:
+	for i := 0; i < 10; i++ {
+		shortID := PeerShortID(rng.Intn(1 << PeerShortIDBits))
+		if peers.byShortID[shortID].peer == nil {
+			return shortID, true
+		}
+	}
+
+	// Looks like most short ids are used.  So count the number of
+	// unused ones, and pick one at random.
+	available := int(1 << PeerShortIDBits)
+	for _, entry := range peers.byShortID {
+		if entry.peer != nil {
+			available--
+		}
+	}
+
+	if available == 0 {
+		// All short ids are used.
+		return 0, false
+	}
+
+	n := rng.Intn(available)
+	var i PeerShortID
+	for {
+		if peers.byShortID[i].peer == nil {
+			if n == 0 {
+				return PeerShortID(i), true
+			}
+
+			n--
+		}
+
+		i++
+	}
+}
+
+func (peers *Peers) FetchWithDefault(peer *Peer) *Peer {
+	peers.Lock()
+	var pending PeersPendingNotifications
+	defer peers.unlockAndNotify(&pending)
+
+	if existingPeer, found := peers.byName[peer.Name]; found {
+		existingPeer.localRefCount++
+		return existingPeer
+	}
+
+	peers.byName[peer.Name] = peer
+	peers.addByShortID(peer, &pending)
+	peer.localRefCount++
+	return peer
+}
+
+func (peers *Peers) Fetch(name PeerName) *Peer {
+	peers.RLock()
+	defer peers.RUnlock()
+	return peers.byName[name]
+}
+
+func (peers *Peers) FetchAndAddRef(name PeerName) *Peer {
+	peers.Lock()
+	defer peers.Unlock()
+	peer := peers.byName[name]
+	if peer != nil {
+		peer.localRefCount++
+	}
+	return peer
+}
+
+func (peers *Peers) FetchByShortID(shortID PeerShortID) *Peer {
+	peers.RLock()
+	defer peers.RUnlock()
+	return peers.byShortID[shortID].peer
+}
+
+func (peers *Peers) Dereference(peer *Peer) {
+	peers.Lock()
+	defer peers.Unlock()
+	peer.localRefCount--
+}
+
+func (peers *Peers) ForEach(fun func(*Peer)) {
+	peers.RLock()
+	defer peers.RUnlock()
+	for _, peer := range peers.byName {
+		fun(peer)
+	}
+}
+
+// Merge an incoming update with our own topology.
+//
+// We add peers hitherto unknown to us, and update peers for which the
+// update contains a more recent version than known to us. The return
+// value is a) a representation of the received update, and b) an
+// "improved" update containing just these new/updated elements.
+func (peers *Peers) ApplyUpdate(update []byte) (PeerNameSet, PeerNameSet, error) {
+	peers.Lock()
+	var pending PeersPendingNotifications
+	defer peers.unlockAndNotify(&pending)
+
+	newPeers, decodedUpdate, decodedConns, err := peers.decodeUpdate(update)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	// By this point, we know the update doesn't refer to any peers we
+	// have no knowledge of. We can now apply the update. Start by
+	// adding in any new peers.
+	for name, newPeer := range newPeers {
+		peers.byName[name] = newPeer
+		peers.addByShortID(newPeer, &pending)
+	}
+
+	// Now apply the updates
+	newUpdate := peers.applyUpdate(decodedUpdate, decodedConns, &pending)
+	peers.garbageCollect(&pending)
+	for _, peerRemoved := range pending.removed {
+		delete(newUpdate, peerRemoved.Name)
+	}
+
+	updateNames := make(PeerNameSet)
+	for _, peer := range decodedUpdate {
+		updateNames[peer.Name] = void
+	}
+
+	return updateNames, newUpdate, nil
+}
+
+func (peers *Peers) Names() PeerNameSet {
+	peers.RLock()
+	defer peers.RUnlock()
+
+	names := make(PeerNameSet)
+	for name := range peers.byName {
+		names[name] = void
+	}
+	return names
+}
+
+func (peers *Peers) EncodePeers(names PeerNameSet) []byte {
+	buf := new(bytes.Buffer)
+	enc := gob.NewEncoder(buf)
+	peers.RLock()
+	defer peers.RUnlock()
+	for name := range names {
+		if peer, found := peers.byName[name]; found {
+			if peer == peers.ourself.Peer {
+				peers.ourself.Encode(enc)
+			} else {
+				peer.Encode(enc)
+			}
+		}
+	}
+	return buf.Bytes()
+}
+
+func (peers *Peers) GarbageCollect() {
+	peers.Lock()
+	var pending PeersPendingNotifications
+	defer peers.unlockAndNotify(&pending)
+
+	peers.garbageCollect(&pending)
+}
+
+func (peers *Peers) garbageCollect(pending *PeersPendingNotifications) {
+	peers.ourself.RLock()
+	_, reached := peers.ourself.Routes(nil, false)
+	peers.ourself.RUnlock()
+
+	for name, peer := range peers.byName {
+		if _, found := reached[peer.Name]; !found && peer.localRefCount == 0 {
+			delete(peers.byName, name)
+			peers.deleteByShortID(peer, pending)
+			pending.removed = append(pending.removed, peer)
+		}
+	}
+
+	if peers.byShortID[peers.ourself.ShortID].peer != peers.ourself.Peer {
+		// The local peer doesn't own its short id.  Garbage
+		// collection might have freed some up, so try to
+		// reassign.
+		pending.reassignLocalShortID = true
+	}
+}
+
+func (peers *Peers) decodeUpdate(update []byte) (newPeers map[PeerName]*Peer, decodedUpdate []*Peer, decodedConns [][]ConnectionSummary, err error) {
+	newPeers = make(map[PeerName]*Peer)
+	decodedUpdate = []*Peer{}
+	decodedConns = [][]ConnectionSummary{}
+
+	decoder := gob.NewDecoder(bytes.NewReader(update))
+
+	for {
+		peerSummary, connSummaries, decErr := decodePeer(decoder)
+		if decErr == io.EOF {
+			break
+		} else if decErr != nil {
+			err = decErr
+			return
+		}
+		newPeer := NewPeerFromSummary(peerSummary)
+		decodedUpdate = append(decodedUpdate, newPeer)
+		decodedConns = append(decodedConns, connSummaries)
+		if _, found := peers.byName[newPeer.Name]; !found {
+			newPeers[newPeer.Name] = newPeer
+		}
+	}
+
+	for _, connSummaries := range decodedConns {
+		for _, connSummary := range connSummaries {
+			remoteName := PeerNameFromBin(connSummary.NameByte)
+			if _, found := newPeers[remoteName]; found {
+				continue
+			}
+			if _, found := peers.byName[remoteName]; found {
+				continue
+			}
+			// Update refers to a peer which we have no knowledge
+			// of. Thus we can't apply the update. Abort.
+			err = UnknownPeerError{remoteName}
+			return
+		}
+	}
+	return
+}
+
+func (peers *Peers) applyUpdate(decodedUpdate []*Peer, decodedConns [][]ConnectionSummary, pending *PeersPendingNotifications) PeerNameSet {
+	newUpdate := make(PeerNameSet)
+	for idx, newPeer := range decodedUpdate {
+		connSummaries := decodedConns[idx]
+		name := newPeer.Name
+		// guaranteed to find peer in the peers.byName
+		switch peer := peers.byName[name]; peer {
+		case peers.ourself.Peer:
+			if newPeer.UID != peer.UID {
+				// The update contains information about an old
+				// incarnation of ourselves. We increase our version
+				// number beyond that which we received, so our
+				// information supersedes the old one when it is
+				// received by other peers.
+				pending.localPeerModified = peers.ourself.setVersionBeyond(newPeer.Version)
+			}
+		case newPeer:
+			peer.connections = makeConnsMap(peer, connSummaries, peers.byName)
+			newUpdate[name] = void
+		default: // existing peer
+			if newPeer.Version < peer.Version ||
+				(newPeer.Version == peer.Version && newPeer.UID <= peer.UID) {
+				continue
+			}
+			peer.Version = newPeer.Version
+			peer.UID = newPeer.UID
+			peer.connections = makeConnsMap(peer, connSummaries, peers.byName)
+
+			if newPeer.ShortID != peer.ShortID {
+				peers.deleteByShortID(peer, pending)
+				peer.ShortID = newPeer.ShortID
+				peers.addByShortID(peer, pending)
+			}
+			newUpdate[name] = void
+		}
+	}
+	return newUpdate
+}
+
+func (peer *Peer) Encode(enc *gob.Encoder) {
+	checkFatal(enc.Encode(peer.PeerSummary))
+
+	connSummaries := []ConnectionSummary{}
+	for _, conn := range peer.connections {
+		connSummaries = append(connSummaries, ConnectionSummary{
+			conn.Remote().NameByte,
+			conn.RemoteTCPAddr(),
+			conn.Outbound(),
+			conn.Established(),
+		})
+	}
+
+	checkFatal(enc.Encode(connSummaries))
+}
+
+func (peer *LocalPeer) Encode(enc *gob.Encoder) {
+	peer.RLock()
+	defer peer.RUnlock()
+	peer.Peer.Encode(enc)
+}
+
+func decodePeer(dec *gob.Decoder) (peerSummary PeerSummary, connSummaries []ConnectionSummary, err error) {
+	if err = dec.Decode(&peerSummary); err != nil {
+		return
+	}
+	if err = dec.Decode(&connSummaries); err != nil {
+		return
+	}
+	return
+}
+
+func makeConnsMap(peer *Peer, connSummaries []ConnectionSummary, byName map[PeerName]*Peer) map[PeerName]Connection {
+	conns := make(map[PeerName]Connection)
+	for _, connSummary := range connSummaries {
+		name := PeerNameFromBin(connSummary.NameByte)
+		remotePeer := byName[name]
+		conn := NewRemoteConnection(peer, remotePeer, connSummary.RemoteTCPAddr, connSummary.Outbound, connSummary.Established)
+		conns[name] = conn
+	}
+	return conns
+}

mesh/peers_test.go

@@ -0,0 +1,376 @@
+package mesh
+
+import (
+	"fmt"
+	"math/rand"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+// TODO we should also test:
+//
+// - applying an incremental update, including the case where that
+//   leads to an UnknownPeerError
+//
+// - the "improved update" calculation
+//
+// - non-gc of peers that are only referenced locally
+
+func newNode(name PeerName) (*Peer, *Peers) {
+	peer := NewLocalPeer(name, "", nil)
+	peers := NewPeers(peer)
+	return peer.Peer, peers
+}
+
+// Check that ApplyUpdate copies the whole topology from peers
+func checkApplyUpdate(t *testing.T, peers *Peers) {
+	dummyName, _ := PeerNameFromString("99:00:00:01:00:00")
+	// We need a new node outside of the network, with a connection
+	// into it.
+	_, testBedPeers := newNode(dummyName)
+	testBedPeers.AddTestConnection(peers.ourself.Peer)
+	testBedPeers.ApplyUpdate(peers.EncodePeers(peers.Names()))
+
+	checkTopologyPeers(t, true, testBedPeers.allPeersExcept(dummyName), peers.allPeers()...)
+}
+
+func TestPeersEncoding(t *testing.T) {
+	const numNodes = 20
+	const numIters = 1000
+	var peer [numNodes]*Peer
+	var ps [numNodes]*Peers
+	for i := 0; i < numNodes; i++ {
+		name, _ := PeerNameFromString(fmt.Sprintf("%02d:00:00:01:00:00", i))
+		peer[i], ps[i] = newNode(name)
+	}
+
+	var conns []struct{ from, to int }
+	for i := 0; i < numIters; i++ {
+		oper := rand.Intn(2)
+		switch oper {
+		case 0:
+			from, to := rand.Intn(numNodes), rand.Intn(numNodes)
+			if from != to {
+				if _, found := peer[from].connections[peer[to].Name]; !found {
+					ps[from].AddTestConnection(peer[to])
+					conns = append(conns, struct{ from, to int }{from, to})
+					checkApplyUpdate(t, ps[from])
+				}
+			}
+		case 1:
+			if len(conns) > 0 {
+				n := rand.Intn(len(conns))
+				c := conns[n]
+				ps[c.from].DeleteTestConnection(peer[c.to])
+				ps[c.from].GarbageCollect()
+				checkApplyUpdate(t, ps[c.from])
+				conns = append(conns[:n], conns[n+1:]...)
+			}
+		}
+	}
+}
+
+func garbageCollect(peers *Peers) []*Peer {
+	var removed []*Peer
+	peers.OnGC(func(peer *Peer) { removed = append(removed, peer) })
+	peers.GarbageCollect()
+	return removed
+}
+
+func TestPeersGarbageCollection(t *testing.T) {
+	const (
+		peer1NameString = "01:00:00:01:00:00"
+		peer2NameString = "02:00:00:02:00:00"
+		peer3NameString = "03:00:00:03:00:00"
+	)
+	var (
+		peer1Name, _ = PeerNameFromString(peer1NameString)
+		peer2Name, _ = PeerNameFromString(peer2NameString)
+		peer3Name, _ = PeerNameFromString(peer3NameString)
+	)
+
+	// Create some peers with some connections to each other
+	p1, ps1 := newNode(peer1Name)
+	p2, ps2 := newNode(peer2Name)
+	p3, ps3 := newNode(peer3Name)
+	ps1.AddTestConnection(p2)
+	ps2.AddTestRemoteConnection(p1, p2)
+	ps2.AddTestConnection(p1)
+	ps2.AddTestConnection(p3)
+	ps3.AddTestConnection(p1)
+	ps1.AddTestConnection(p3)
+	ps2.AddTestRemoteConnection(p1, p3)
+	ps2.AddTestRemoteConnection(p3, p1)
+
+	// Every peer is referenced, so nothing should be dropped
+	require.Empty(t, garbageCollect(ps1), "peers removed")
+	require.Empty(t, garbageCollect(ps2), "peers removed")
+	require.Empty(t, garbageCollect(ps3), "peers removed")
+
+	// Drop the connection from 2 to 3, and 3 isn't garbage-collected
+	// because 1 has a connection to 3
+	ps2.DeleteTestConnection(p3)
+	require.Empty(t, garbageCollect(ps2), "peers removed")
+
+	// Drop the connection from 1 to 3, and 3 will get removed by
+	// garbage-collection
+	ps1.DeleteTestConnection(p3)
+	checkPeerArray(t, garbageCollect(ps1), p3)
+}
+
+func TestShortIDCollisions(t *testing.T) {
+	rng := rand.New(rand.NewSource(time.Now().UnixNano()))
+	_, peers := newNode(PeerName(1 << PeerShortIDBits))
+
+	// Make enough peers that short id collisions are
+	// overwhelmingly likely
+	ps := make([]*Peer, 1<<PeerShortIDBits)
+	for i := 0; i < 1<<PeerShortIDBits; i++ {
+		ps[i] = NewPeer(PeerName(i), "", PeerUID(i), 0,
+			PeerShortID(rng.Intn(1<<PeerShortIDBits)))
+	}
+
+	shuffle := func() {
+		for i := range ps {
+			j := rng.Intn(i + 1)
+			ps[i], ps[j] = ps[j], ps[i]
+		}
+	}
+
+	// Fill peers
+	shuffle()
+	var pending PeersPendingNotifications
+	for _, p := range ps {
+		peers.addByShortID(p, &pending)
+	}
+
+	// Check invariants
+	counts := make([]int, 1<<PeerShortIDBits)
+	saw := func(p *Peer) {
+		if p != peers.ourself.Peer {
+			counts[p.UID]++
+		}
+	}
+
+	for shortID, entry := range peers.byShortID {
+		if entry.peer == nil {
+			// no principal peer for this short id, so
+			// others must be empty
+			require.Empty(t, entry.others)
+			continue
+		}
+
+		require.Equal(t, shortID, entry.peer.ShortID)
+		saw(entry.peer)
+
+		for _, p := range entry.others {
+			saw(p)
+			require.Equal(t, shortID, p.ShortID)
+
+			// the principal peer should have the lowest name
+			require.True(t, p.Name > entry.peer.Name)
+		}
+	}
+
+	// Check that every peer was seen
+	for _, n := range counts {
+		require.Equal(t, 1, n)
+	}
+
+	// Delete all the peers
+	shuffle()
+	for _, p := range ps {
+		peers.deleteByShortID(p, &pending)
+	}
+
+	for _, entry := range peers.byShortID {
+		if entry.peer != peers.ourself.Peer {
+			require.Nil(t, entry.peer)
+		}
+
+		require.Empty(t, entry.others)
+	}
+}
+
+// Test the easy case of short id reassignment, when few short ids are taken
+func TestShortIDReassignmentEasy(t *testing.T) {
+	rng := rand.New(rand.NewSource(time.Now().UnixNano()))
+	_, peers := newNode(PeerName(0))
+
+	for i := 1; i <= 10; i++ {
+		peers.FetchWithDefault(NewPeer(PeerName(i), "", PeerUID(i), 0,
+			PeerShortID(rng.Intn(1<<PeerShortIDBits))))
+	}
+
+	checkShortIDReassignment(t, peers)
+}
+
+// Test the hard case of short id reassignment, when most short ids are taken
+func TestShortIDReassignmentHard(t *testing.T) {
+	rng := rand.New(rand.NewSource(time.Now().UnixNano()))
+	_, peers := newNode(PeerName(1 << PeerShortIDBits))
+
+	// Take all short ids
+	ps := make([]*Peer, 1<<PeerShortIDBits)
+	var pending PeersPendingNotifications
+	for i := 0; i < 1<<PeerShortIDBits; i++ {
+		ps[i] = NewPeer(PeerName(i), "", PeerUID(i), 0,
+			PeerShortID(i))
+		peers.addByShortID(ps[i], &pending)
+	}
+
+	// As all short ids are taken, an attempted reassigment won't
+	// do anything
+	oldShortID := peers.ourself.ShortID
+	require.False(t, peers.reassignLocalShortID(&pending))
+	require.Equal(t, oldShortID, peers.ourself.ShortID)
+
+	// Free up a few ids
+	for i := 0; i < 10; i++ {
+		x := rng.Intn(len(ps))
+		if ps[x] != nil {
+			peers.deleteByShortID(ps[x], &pending)
+			ps[x] = nil
+		}
+	}
+
+	checkShortIDReassignment(t, peers)
+}
+
+func checkShortIDReassignment(t *testing.T, peers *Peers) {
+	oldShortID := peers.ourself.ShortID
+	peers.reassignLocalShortID(&PeersPendingNotifications{})
+	require.NotEqual(t, oldShortID, peers.ourself.ShortID)
+	require.Equal(t, peers.ourself.Peer, peers.byShortID[peers.ourself.ShortID].peer)
+}
+
+func TestShortIDInvalidation(t *testing.T) {
+	_, peers := newNode(PeerName(1 << PeerShortIDBits))
+
+	// need to use a short id that is not the local peer's
+	shortID := peers.ourself.ShortID + 1
+
+	var pending PeersPendingNotifications
+
+	requireInvalidateShortIDs := func(expect bool) {
+		require.Equal(t, expect, pending.invalidateShortIDs)
+		pending.invalidateShortIDs = false
+	}
+
+	// The use of a fresh short id does not cause invalidation
+	a := NewPeer(PeerName(1), "", PeerUID(1), 0, shortID)
+	peers.addByShortID(a, &pending)
+	requireInvalidateShortIDs(false)
+
+	// An addition which does not change the mapping
+	b := NewPeer(PeerName(2), "", PeerUID(2), 0, shortID)
+	peers.addByShortID(b, &pending)
+	requireInvalidateShortIDs(false)
+
+	// An addition which does change the mapping
+	c := NewPeer(PeerName(0), "", PeerUID(0), 0, shortID)
+	peers.addByShortID(c, &pending)
+	requireInvalidateShortIDs(true)
+
+	// A deletion which does not change the mapping
+	peers.deleteByShortID(b, &pending)
+	requireInvalidateShortIDs(false)
+
+	// A deletion which does change the mapping
+	peers.deleteByShortID(c, &pending)
+	requireInvalidateShortIDs(true)
+
+	// Deleting the last peer with a short id does not cause invalidation
+	peers.deleteByShortID(a, &pending)
+	requireInvalidateShortIDs(false)
+
+	// .. but subsequent reuse of that short id does cause invalidation
+	peers.addByShortID(a, &pending)
+	requireInvalidateShortIDs(true)
+}
+
+func TestShortIDPropagation(t *testing.T) {
+	_, peers1 := newNode(PeerName(1))
+	_, peers2 := newNode(PeerName(2))
+
+	peers1.AddTestConnection(peers2.ourself.Peer)
+	peers1.ApplyUpdate(peers2.EncodePeers(peers2.Names()))
+	peers12 := peers1.Fetch(PeerName(2))
+	old := peers12.PeerSummary
+
+	require.True(t,
+		peers2.reassignLocalShortID(&PeersPendingNotifications{}))
+	peers1.ApplyUpdate(peers2.EncodePeers(peers2.Names()))
+	require.NotEqual(t, old.Version, peers12.Version)
+	require.NotEqual(t, old.ShortID, peers12.ShortID)
+}
+
+func TestShortIDCollision(t *testing.T) {
+	// Create 3 peers
+	_, peers1 := newNode(PeerName(1))
+	_, peers2 := newNode(PeerName(2))
+	_, peers3 := newNode(PeerName(3))
+
+	var pending PeersPendingNotifications
+	peers1.setLocalShortID(1, &pending)
+	peers2.setLocalShortID(2, &pending)
+	peers3.setLocalShortID(3, &pending)
+
+	peers2.AddTestConnection(peers1.ourself.Peer)
+	peers3.AddTestConnection(peers2.ourself.Peer)
+
+	// Propogate from 1 to 2 to 3
+	peers2.ApplyUpdate(peers1.EncodePeers(peers1.Names()))
+	peers3.ApplyUpdate(peers2.EncodePeers(peers2.Names()))
+
+	// Force the short id of peer 1 to collide with peer 2.  Peer
+	// 1 has the lowest name, so it gets to keep the short id
+	peers1.setLocalShortID(2, &pending)
+
+	oldShortID := peers2.ourself.ShortID
+	_, updated, _ := peers2.ApplyUpdate(peers1.EncodePeers(peers1.Names()))
+
+	// peer 2 should have noticed the collision and resolved it
+	require.NotEqual(t, oldShortID, peers2.ourself.ShortID)
+
+	// The Peers do not have a Router, so broadcastPeerUpdate does
+	// nothing in the context of this test.  So we fake what it
+	// would do.
+	updated[PeerName(2)] = void
+
+	// the update from peer 2 should include its short id change
+	peers3.ApplyUpdate(peers2.EncodePeers(updated))
+	require.Equal(t, peers2.ourself.ShortID,
+		peers3.Fetch(PeerName(2)).ShortID)
+}
+
+// Test the case where all short ids are taken, but then some peers go
+// away, so the local peer reassigns
+func TestDeferredShortIDReassignment(t *testing.T) {
+	rng := rand.New(rand.NewSource(time.Now().UnixNano()))
+	_, us := newNode(PeerName(1 << PeerShortIDBits))
+
+	// Connect us to other peers occupying all short ids
+	others := make([]*Peers, 1<<PeerShortIDBits)
+	var pending PeersPendingNotifications
+	for i := range others {
+		_, others[i] = newNode(PeerName(i))
+		others[i].setLocalShortID(PeerShortID(i), &pending)
+		us.AddTestConnection(others[i].ourself.Peer)
+	}
+
+	// Check that, as expected, the local peer does not own its
+	// short id
+	require.NotEqual(t, us.ourself.Peer,
+		us.byShortID[us.ourself.ShortID].peer)
+
+	// Disconnect one peer, and we should now be able to claim its
+	// short id
+	other := others[rng.Intn(1<<PeerShortIDBits)]
+	us.DeleteTestConnection(other.ourself.Peer)
+	us.GarbageCollect()
+
+	require.Equal(t, us.ourself.Peer, us.byShortID[us.ourself.ShortID].peer)
+}

mesh/protocol.go

@@ -1,4 +1,4 @@
-package router
+package mesh
 
 import (
 	"bytes"
@@ -328,13 +328,14 @@ func (res *ProtocolIntroResults) setupCrypto(params ProtocolIntroParams, remoteP
 type ProtocolTag byte
 
 const (
-	ProtocolHeartbeat ProtocolTag = iota
-	ProtocolConnectionEstablished
-	ProtocolFragmentationReceived
-	ProtocolPMTUVerified
+	ProtocolHeartbeat = iota
+	ProtocolReserved1 // legacy overlay control messages
+	ProtocolReserved2 //
+	ProtocolReserved3 //
 	ProtocolGossip
 	ProtocolGossipUnicast
 	ProtocolGossipBroadcast
+	ProtocolOverlayControlMsg
 )
 
 type ProtocolMsg struct {

mesh/protocol_crypto.go

@@ -0,0 +1,180 @@
+package mesh
+
+import (
+	"crypto/rand"
+	"crypto/sha256"
+	"encoding/binary"
+	"encoding/gob"
+	"fmt"
+	"io"
+	"sync"
+
+	"golang.org/x/crypto/nacl/box"
+	"golang.org/x/crypto/nacl/secretbox"
+)
+
+const MaxTCPMsgSize = 10 * 1024 * 1024
+
+func GenerateKeyPair() (publicKey, privateKey *[32]byte, err error) {
+	return box.GenerateKey(rand.Reader)
+}
+
+func FormSessionKey(remotePublicKey, localPrivateKey *[32]byte, secretKey []byte) *[32]byte {
+	var sharedKey [32]byte
+	box.Precompute(&sharedKey, remotePublicKey, localPrivateKey)
+	sharedKeySlice := sharedKey[:]
+	sharedKeySlice = append(sharedKeySlice, secretKey...)
+	sessionKey := sha256.Sum256(sharedKeySlice)
+	return &sessionKey
+}
+
+// TCP Senders/Receivers
+
+// The lowest 64 bits of the nonce contain the message sequence
+// number. The top most bit indicates the connection polarity at the
+// sender - '1' for outbound; the next indicates protocol type - '1'
+// for TCP. The remaining 126 bits are zero. The polarity is needed so
+// that the two ends of a connection do not use the same nonces; the
+// protocol type so that the TCP connection nonces are distinct from
+// nonces used by overlay connections, if they share the session key.
+// This is a requirement of the NaCl Security Model; see
+// http://nacl.cr.yp.to/box.html.
+type TCPCryptoState struct {
+	sessionKey *[32]byte
+	nonce      [24]byte
+	seqNo      uint64
+}
+
+func NewTCPCryptoState(sessionKey *[32]byte, outbound bool) *TCPCryptoState {
+	s := &TCPCryptoState{sessionKey: sessionKey}
+	if outbound {
+		s.nonce[0] |= (1 << 7)
+	}
+	s.nonce[0] |= (1 << 6)
+	return s
+}
+
+func (s *TCPCryptoState) advance() {
+	s.seqNo++
+	binary.BigEndian.PutUint64(s.nonce[16:24], s.seqNo)
+}
+
+type TCPSender interface {
+	Send([]byte) error
+}
+
+type GobTCPSender struct {
+	encoder *gob.Encoder
+}
+
+type LengthPrefixTCPSender struct {
+	writer io.Writer
+}
+
+type EncryptedTCPSender struct {
+	sync.RWMutex
+	sender TCPSender
+	state  *TCPCryptoState
+}
+
+func NewGobTCPSender(encoder *gob.Encoder) *GobTCPSender {
+	return &GobTCPSender{encoder: encoder}
+}
+
+func (sender *GobTCPSender) Send(msg []byte) error {
+	return sender.encoder.Encode(msg)
+}
+
+func NewLengthPrefixTCPSender(writer io.Writer) *LengthPrefixTCPSender {
+	return &LengthPrefixTCPSender{writer: writer}
+}
+
+func (sender *LengthPrefixTCPSender) Send(msg []byte) error {
+	l := len(msg)
+	if l > MaxTCPMsgSize {
+		return fmt.Errorf("outgoing message exceeds maximum size: %d > %d", l, MaxTCPMsgSize)
+	}
+	// We copy the message so we can send it in a single Write
+	// operation, thus making this thread-safe without locking.
+	prefixedMsg := make([]byte, 4+l)
+	binary.BigEndian.PutUint32(prefixedMsg, uint32(l))
+	copy(prefixedMsg[4:], msg)
+	_, err := sender.writer.Write(prefixedMsg)
+	return err
+}
+
+func NewEncryptedTCPSender(sender TCPSender, sessionKey *[32]byte, outbound bool) *EncryptedTCPSender {
+	return &EncryptedTCPSender{sender: sender, state: NewTCPCryptoState(sessionKey, outbound)}
+}
+
+func (sender *EncryptedTCPSender) Send(msg []byte) error {
+	sender.Lock()
+	defer sender.Unlock()
+	encodedMsg := secretbox.Seal(nil, msg, &sender.state.nonce, sender.state.sessionKey)
+	sender.state.advance()
+	return sender.sender.Send(encodedMsg)
+}
+
+type TCPReceiver interface {
+	Receive() ([]byte, error)
+}
+
+type GobTCPReceiver struct {
+	decoder *gob.Decoder
+}
+
+type LengthPrefixTCPReceiver struct {
+	reader io.Reader
+}
+
+type EncryptedTCPReceiver struct {
+	receiver TCPReceiver
+	state    *TCPCryptoState
+}
+
+func NewGobTCPReceiver(decoder *gob.Decoder) *GobTCPReceiver {
+	return &GobTCPReceiver{decoder: decoder}
+}
+
+func (receiver *GobTCPReceiver) Receive() ([]byte, error) {
+	var msg []byte
+	err := receiver.decoder.Decode(&msg)
+	return msg, err
+}
+
+func NewLengthPrefixTCPReceiver(reader io.Reader) *LengthPrefixTCPReceiver {
+	return &LengthPrefixTCPReceiver{reader: reader}
+}
+
+func (receiver *LengthPrefixTCPReceiver) Receive() ([]byte, error) {
+	lenPrefix := make([]byte, 4)
+	if _, err := io.ReadFull(receiver.reader, lenPrefix); err != nil {
+		return nil, err
+	}
+	l := binary.BigEndian.Uint32(lenPrefix)
+	if l > MaxTCPMsgSize {
+		return nil, fmt.Errorf("incoming message exceeds maximum size: %d > %d", l, MaxTCPMsgSize)
+	}
+	msg := make([]byte, l)
+	_, err := io.ReadFull(receiver.reader, msg)
+	return msg, err
+}
+
+func NewEncryptedTCPReceiver(receiver TCPReceiver, sessionKey *[32]byte, outbound bool) *EncryptedTCPReceiver {
+	return &EncryptedTCPReceiver{receiver: receiver, state: NewTCPCryptoState(sessionKey, !outbound)}
+}
+
+func (receiver *EncryptedTCPReceiver) Receive() ([]byte, error) {
+	msg, err := receiver.receiver.Receive()
+	if err != nil {
+		return nil, err
+	}
+
+	decodedMsg, success := secretbox.Open(nil, msg, &receiver.state.nonce, receiver.state.sessionKey)
+	if !success {
+		return nil, fmt.Errorf("Unable to decrypt TCP msg")
+	}
+
+	receiver.state.advance()
+	return decodedMsg, nil
+}

mesh/protocol_test.go

@@ -1,4 +1,4 @@
-package router
+package mesh
 
 import (
 	"io"

mesh/router.go

@@ -0,0 +1,191 @@
+package mesh
+
+import (
+	"fmt"
+	"math"
+	"net"
+	"sync"
+	"time"
+
+	"github.com/weaveworks/weave/common"
+)
+
+const (
+	Port           = 6783
+	ChannelSize    = 16
+	TCPHeartbeat   = 30 * time.Second
+	GossipInterval = 30 * time.Second
+	MaxDuration    = time.Duration(math.MaxInt64)
+
+	acceptMaxTokens  = 100                    // [1]
+	acceptTokenDelay = 100 * time.Millisecond // [2]
+)
+
+// [1] capacity of token bucket for rate limiting accepts
+
+// [2] control rate at which new tokens are added to the bucket
+
+var (
+	log        = common.Log
+	checkFatal = common.CheckFatal
+	checkWarn  = common.CheckWarn
+)
+
+type Config struct {
+	Port               int
+	ProtocolMinVersion byte
+	Password           []byte
+	ConnLimit          int
+	PeerDiscovery      bool
+}
+
+type Router struct {
+	Config
+	Overlay         Overlay
+	Ourself         *LocalPeer
+	Peers           *Peers
+	Routes          *Routes
+	ConnectionMaker *ConnectionMaker
+	gossipLock      sync.RWMutex
+	gossipChannels  GossipChannels
+	TopologyGossip  Gossip
+	acceptLimiter   *TokenBucket
+}
+
+func NewRouter(config Config, name PeerName, nickName string, overlay Overlay) *Router {
+	router := &Router{Config: config, gossipChannels: make(GossipChannels)}
+
+	if overlay == nil {
+		overlay = NullOverlay{}
+	}
+
+	router.Overlay = overlay
+	router.Ourself = NewLocalPeer(name, nickName, router)
+	router.Peers = NewPeers(router.Ourself)
+	router.Peers.OnGC(func(peer *Peer) {
+		log.Println("Removed unreachable peer", peer)
+	})
+	router.Routes = NewRoutes(router.Ourself, router.Peers)
+	router.ConnectionMaker = NewConnectionMaker(router.Ourself, router.Peers, router.Port, router.PeerDiscovery)
+	router.TopologyGossip = router.NewGossip("topology", router)
+	router.acceptLimiter = NewTokenBucket(acceptMaxTokens, acceptTokenDelay)
+
+	return router
+}
+
+// Start listening for TCP connections. This is separate from
+// NewRouter so that gossipers can register before we start forming
+// connections.
+func (router *Router) Start() {
+	router.listenTCP(router.Port)
+}
+
+func (router *Router) Stop() error {
+	// TODO: perform graceful shutdown...
+	return nil
+}
+
+func (router *Router) UsingPassword() bool {
+	return router.Password != nil
+}
+
+func (router *Router) listenTCP(localPort int) {
+	localAddr, err := net.ResolveTCPAddr("tcp4", fmt.Sprint(":", localPort))
+	checkFatal(err)
+	ln, err := net.ListenTCP("tcp4", localAddr)
+	checkFatal(err)
+	go func() {
+		defer ln.Close()
+		for {
+			tcpConn, err := ln.AcceptTCP()
+			if err != nil {
+				log.Errorln(err)
+				continue
+			}
+			router.acceptTCP(tcpConn)
+			router.acceptLimiter.Wait()
+		}
+	}()
+}
+
+func (router *Router) acceptTCP(tcpConn *net.TCPConn) {
+	remoteAddrStr := tcpConn.RemoteAddr().String()
+	log.Printf("->[%s] connection accepted", remoteAddrStr)
+	connRemote := NewRemoteConnection(router.Ourself.Peer, nil, remoteAddrStr, false, false)
+	StartLocalConnection(connRemote, tcpConn, router, true)
+}
+
+// Gossiper methods - the Router is the topology Gossiper
+
+type TopologyGossipData struct {
+	peers  *Peers
+	update PeerNameSet
+}
+
+func (router *Router) BroadcastTopologyUpdate(update []*Peer) {
+	names := make(PeerNameSet)
+	for _, p := range update {
+		names[p.Name] = void
+	}
+	router.TopologyGossip.GossipBroadcast(
+		&TopologyGossipData{peers: router.Peers, update: names})
+}
+
+func (d *TopologyGossipData) Merge(other GossipData) GossipData {
+	names := make(PeerNameSet)
+	for name := range d.update {
+		names[name] = void
+	}
+	for name := range other.(*TopologyGossipData).update {
+		names[name] = void
+	}
+	return &TopologyGossipData{peers: d.peers, update: names}
+}
+
+func (d *TopologyGossipData) Encode() [][]byte {
+	return [][]byte{d.peers.EncodePeers(d.update)}
+}
+
+func (router *Router) OnGossipUnicast(sender PeerName, msg []byte) error {
+	return fmt.Errorf("unexpected topology gossip unicast: %v", msg)
+}
+
+func (router *Router) OnGossipBroadcast(_ PeerName, update []byte) (GossipData, error) {
+	origUpdate, _, err := router.applyTopologyUpdate(update)
+	if err != nil || len(origUpdate) == 0 {
+		return nil, err
+	}
+	return &TopologyGossipData{peers: router.Peers, update: origUpdate}, nil
+}
+
+func (router *Router) Gossip() GossipData {
+	return &TopologyGossipData{peers: router.Peers, update: router.Peers.Names()}
+}
+
+func (router *Router) OnGossip(update []byte) (GossipData, error) {
+	_, newUpdate, err := router.applyTopologyUpdate(update)
+	if err != nil || len(newUpdate) == 0 {
+		return nil, err
+	}
+	return &TopologyGossipData{peers: router.Peers, update: newUpdate}, nil
+}
+
+func (router *Router) applyTopologyUpdate(update []byte) (PeerNameSet, PeerNameSet, error) {
+	origUpdate, newUpdate, err := router.Peers.ApplyUpdate(update)
+	if _, ok := err.(UnknownPeerError); err != nil && ok {
+		// That update contained a reference to a peer which wasn't
+		// itself included in the update, and we didn't know about
+		// already. We ignore this; eventually we should receive an
+		// update containing a complete topology.
+		log.Println("Topology gossip:", err)
+		return nil, nil, nil
+	}
+	if err != nil {
+		return nil, nil, err
+	}
+	if len(newUpdate) > 0 {
+		router.ConnectionMaker.Refresh()
+		router.Routes.Recalculate()
+	}
+	return origUpdate, newUpdate, nil
+}

mesh/routes.go

@@ -1,18 +1,22 @@
-package router
+package mesh
 
 import (
 	"math"
 	"sync"
 )
 
+type unicastRoutes map[PeerName]PeerName
+type broadcastRoutes map[PeerName][]PeerName
+
 type Routes struct {
 	sync.RWMutex
 	ourself      *LocalPeer
 	peers        *Peers
-	unicast      map[PeerName]PeerName
-	unicastAll   map[PeerName]PeerName // [1]
-	broadcast    map[PeerName][]PeerName
-	broadcastAll map[PeerName][]PeerName // [1]
+	onChange     []func()
+	unicast      unicastRoutes
+	unicastAll   unicastRoutes // [1]
+	broadcast    broadcastRoutes
+	broadcastAll broadcastRoutes // [1]
 	recalculate  chan<- *struct{}
 	wait         chan<- chan struct{}
 	// [1] based on *all* connections, not just established &
@@ -25,10 +29,10 @@ func NewRoutes(ourself *LocalPeer, peers *Peers) *Routes {
 	routes := &Routes{
 		ourself:      ourself,
 		peers:        peers,
-		unicast:      make(map[PeerName]PeerName),
-		unicastAll:   make(map[PeerName]PeerName),
-		broadcast:    make(map[PeerName][]PeerName),
-		broadcastAll: make(map[PeerName][]PeerName),
+		unicast:      make(unicastRoutes),
+		unicastAll:   make(unicastRoutes),
+		broadcast:    make(broadcastRoutes),
+		broadcastAll: make(broadcastRoutes),
 		recalculate:  recalculate,
 		wait:         wait}
 	routes.unicast[ourself.Name] = UnknownPeerName
@@ -39,6 +43,12 @@ func NewRoutes(ourself *LocalPeer, peers *Peers) *Routes {
 	return routes
 }
 
+func (routes *Routes) OnChange(callback func()) {
+	routes.Lock()
+	defer routes.Unlock()
+	routes.onChange = append(routes.onChange, callback)
+}
+
 func (routes *Routes) PeerNames() PeerNameSet {
 	return routes.peers.Names()
 }
@@ -60,21 +70,13 @@ func (routes *Routes) UnicastAll(name PeerName) (PeerName, bool) {
 func (routes *Routes) Broadcast(name PeerName) []PeerName {
 	routes.RLock()
 	defer routes.RUnlock()
-	hops, found := routes.broadcast[name]
-	if !found {
-		return []PeerName{}
-	}
-	return hops
+	return routes.broadcast[name]
 }
 
 func (routes *Routes) BroadcastAll(name PeerName) []PeerName {
 	routes.RLock()
 	defer routes.RUnlock()
-	hops, found := routes.broadcastAll[name]
-	if !found {
-		return []PeerName{}
-	}
-	return hops
+	return routes.broadcastAll[name]
 }
 
 // Choose min(log2(n_peers), n_neighbouring_peers) neighbours, with a
@@ -147,6 +149,8 @@ func (routes *Routes) calculate() {
 	routes.peers.RLock()
 	routes.ourself.RLock()
 	var (
+		oldUnicast   = routes.unicast
+		oldBroadcast = routes.broadcast
 		unicast      = routes.calculateUnicast(true)
 		unicastAll   = routes.calculateUnicast(false)
 		broadcast    = routes.calculateBroadcast(true)
@@ -160,7 +164,14 @@ func (routes *Routes) calculate() {
 	routes.unicastAll = unicastAll
 	routes.broadcast = broadcast
 	routes.broadcastAll = broadcastAll
+	onChange := routes.onChange
 	routes.Unlock()
+
+	if !unicast.equals(oldUnicast) || !broadcast.equals(oldBroadcast) {
+		for _, callback := range onChange {
+			callback()
+		}
+	}
 }
 
 // Calculate all the routes for the question: if *we* want to send a
@@ -172,7 +183,7 @@ func (routes *Routes) calculate() {
 // any knowledge of the MAC address at all. Thus there's no need
 // to exchange knowledge of MAC addresses, nor any constraints on
 // the routes that we construct.
-func (routes *Routes) calculateUnicast(establishedAndSymmetric bool) map[PeerName]PeerName {
+func (routes *Routes) calculateUnicast(establishedAndSymmetric bool) unicastRoutes {
 	_, unicast := routes.ourself.Routes(nil, establishedAndSymmetric)
 	return unicast
 }
@@ -195,9 +206,9 @@ func (routes *Routes) calculateUnicast(establishedAndSymmetric bool) map[PeerNam
 //     Y =/= Z /\ X.Routes(Y) <= X.Routes(Z) =>
 //     X.Routes(Y) u [P | Y.HasSymmetricConnectionTo(P)] <= X.Routes(Z)
 // where <= is the subset relationship on keys of the returned map.
-func (routes *Routes) calculateBroadcast(establishedAndSymmetric bool) map[PeerName][]PeerName {
-	broadcast := make(map[PeerName][]PeerName)
-	for _, peer := range routes.peers.table {
+func (routes *Routes) calculateBroadcast(establishedAndSymmetric bool) broadcastRoutes {
+	broadcast := make(broadcastRoutes)
+	for _, peer := range routes.peers.byName {
 		hops := []PeerName{}
 		if found, reached := peer.Routes(routes.ourself.Peer, establishedAndSymmetric); found {
 			routes.ourself.ForEachConnectedPeer(establishedAndSymmetric, reached,
@@ -207,3 +218,55 @@ func (routes *Routes) calculateBroadcast(establishedAndSymmetric bool) map[PeerN
 	}
 	return broadcast
 }
+
+func (a unicastRoutes) equals(b unicastRoutes) bool {
+	for key, aval := range a {
+		if bval, ok := b[key]; !ok || bval != aval {
+			return false
+		}
+	}
+
+	for key := range b {
+		if _, ok := a[key]; !ok {
+			return false
+		}
+	}
+
+	return true
+}
+
+func (a broadcastRoutes) equals(b broadcastRoutes) bool {
+	set := make(map[PeerName]struct{})
+
+	for key, aval := range a {
+		bval, ok := b[key]
+		if !ok {
+			return false
+		}
+
+		for _, peer := range aval {
+			set[peer] = struct{}{}
+		}
+
+		for _, peer := range bval {
+			if _, ok := set[peer]; !ok {
+				return false
+			}
+
+			delete(set, peer)
+		}
+
+		if len(set) != 0 {
+			return false
+		}
+
+	}
+
+	for key := range b {
+		if _, ok := a[key]; !ok {
+			return false
+		}
+	}
+
+	return true
+}

mesh/status.go

@@ -1,7 +1,7 @@
-package router
+package mesh
 
 import (
-	"time"
+	"fmt"
 )
 
 type Status struct {
@@ -13,27 +13,19 @@ type Status struct {
 	Name               string
 	NickName           string
 	Port               int
-	Interface          string
-	CaptureStats       map[string]int
-	MACs               []MACStatus
 	Peers              []PeerStatus
 	UnicastRoutes      []UnicastRouteStatus
 	BroadcastRoutes    []BroadcastRouteStatus
 	Connections        []LocalConnectionStatus
 	Targets            []string
-}
-
-type MACStatus struct {
-	Mac      string
-	Name     string
-	NickName string
-	LastSeen time.Time
+	OverlayDiagnostics interface{}
 }
 
 type PeerStatus struct {
 	Name        string
 	NickName    string
 	UID         PeerUID
+	ShortID     PeerShortID
 	Version     uint64
 	Connections []ConnectionStatus
 }
@@ -63,14 +55,6 @@ type LocalConnectionStatus struct {
 }
 
 func NewStatus(router *Router) *Status {
-	var ifaceName string
-	if router.Iface != nil {
-		ifaceName = router.Iface.Name
-	}
-	var captureStats map[string]int
-	if router.PacketSource != nil {
-		captureStats = router.PacketSource.Stats()
-	}
 	return &Status{
 		Protocol,
 		ProtocolMinVersion,
@@ -80,30 +64,12 @@ func NewStatus(router *Router) *Status {
 		router.Ourself.Name.String(),
 		router.Ourself.NickName,
 		router.Port,
-		ifaceName,
-		captureStats,
-		NewMACStatusSlice(router.Macs),
 		NewPeerStatusSlice(router.Peers),
 		NewUnicastRouteStatusSlice(router.Routes),
 		NewBroadcastRouteStatusSlice(router.Routes),
 		NewLocalConnectionStatusSlice(router.ConnectionMaker),
-		NewTargetSlice(router.ConnectionMaker)}
-}
-
-func NewMACStatusSlice(cache *MacCache) []MACStatus {
-	cache.RLock()
-	defer cache.RUnlock()
-
-	var slice []MACStatus
-	for key, entry := range cache.table {
-		slice = append(slice, MACStatus{
-			intmac(key).String(),
-			entry.peer.Name.String(),
-			entry.peer.NickName,
-			entry.lastSeen})
-	}
-
-	return slice
+		NewTargetSlice(router.ConnectionMaker),
+		router.Overlay.Diagnostics()}
 }
 
 func NewPeerStatusSlice(peers *Peers) []PeerStatus {
@@ -127,7 +93,8 @@ func NewPeerStatusSlice(peers *Peers) []PeerStatus {
 			peer.Name.String(),
 			peer.NickName,
 			peer.UID,
-			peer.version,
+			peer.ShortID,
+			peer.Version,
 			connections})
 	})
 
@@ -170,40 +137,41 @@ func NewBroadcastRouteStatusSlice(routes *Routes) []BroadcastRouteStatus {
 }
 
 func NewLocalConnectionStatusSlice(cm *ConnectionMaker) []LocalConnectionStatus {
-	// We need to Refresh first in order to clear out any 'attempting'
-	// connections from cm.targets that have been established since
-	// the last run of cm.checkStateAndAttemptConnections. These
-	// entries are harmless but do represent stale state that we do
-	// not want to report.
-	cm.Refresh()
 	resultChan := make(chan []LocalConnectionStatus, 0)
 	cm.actionChan <- func() bool {
 		var slice []LocalConnectionStatus
-		for conn := range cm.ourself.Connections() {
+		for conn := range cm.connections {
 			state := "pending"
 			if conn.Established() {
 				state = "established"
 			}
-			slice = append(slice, LocalConnectionStatus{conn.RemoteTCPAddr(), conn.Outbound(), state, conn.Remote().String()})
+			lc, _ := conn.(*LocalConnection)
+			info := fmt.Sprintf("%-6v %v", lc.OverlayConn.DisplayName(), conn.Remote())
+			slice = append(slice, LocalConnectionStatus{conn.RemoteTCPAddr(), conn.Outbound(), state, info})
 		}
 		for address, target := range cm.targets {
-			var state, info string
-			switch {
-			case target.lastError == nil:
-				state = "connecting"
-				info = ""
-			case target.attempting:
-				state = "retrying"
-				info = target.lastError.Error()
-			default:
-				state = "failed"
-				retry := "never"
+			add := func(state, info string) {
+				slice = append(slice, LocalConnectionStatus{address, true, state, info})
+			}
+			switch target.state {
+			case TargetWaiting:
+				until := "never"
 				if !target.tryAfter.IsZero() {
-					retry = target.tryAfter.String()
+					until = target.tryAfter.String()
 				}
-				info = target.lastError.Error() + ", retry: " + retry
+				if target.lastError == nil { // shouldn't happen
+					add("waiting", "until: "+until)
+				} else {
+					add("failed", target.lastError.Error()+", retry: "+until)
+				}
+			case TargetAttempting:
+				if target.lastError == nil {
+					add("connecting", "")
+				} else {
+					add("retrying", target.lastError.Error())
+				}
+			case TargetConnected:
 			}
-			slice = append(slice, LocalConnectionStatus{address, true, state, info})
 		}
 		resultChan <- slice
 		return false

mesh/surrogate_gossiper.go

@@ -1,4 +1,4 @@
-package router
+package mesh
 
 type SurrogateGossipData struct {
 	messages [][]byte
@@ -12,8 +12,12 @@ func (d *SurrogateGossipData) Encode() [][]byte {
 	return d.messages
 }
 
-func (d *SurrogateGossipData) Merge(other GossipData) {
-	d.messages = append(d.messages, other.(*SurrogateGossipData).messages...)
+func (d *SurrogateGossipData) Merge(other GossipData) GossipData {
+	o := other.(*SurrogateGossipData)
+	messages := make([][]byte, 0, len(d.messages)+len(o.messages))
+	messages = append(messages, d.messages...)
+	messages = append(messages, o.messages...)
+	return &SurrogateGossipData{messages: messages}
 }
 
 // SurrogateGossiper ignores unicasts and relays broadcasts and gossips.

mesh/token_bucket.go

@@ -1,4 +1,4 @@
-package router
+package mesh
 
 import (
 	"time"

nameserver/dns.go

@@ -216,7 +216,7 @@ func (h *handler) handleRecursive(w dns.ResponseWriter, req *dns.Msg) {
 		reqCopy := req.Copy()
 		reqCopy.Id = dns.Id()
 		response, _, err := h.client.Exchange(reqCopy, fmt.Sprintf("%s:%s", server, h.upstream.Port))
-		if err != nil || response == nil {
+		if (err != nil && err != dns.ErrTruncated) || response == nil {
 			h.ns.debugf("error trying %s: %v", server, err)
 			continue
 		}

nameserver/dns_test.go

@@ -11,14 +11,14 @@ import (
 	"github.com/miekg/dns"
 	"github.com/stretchr/testify/require"
 
+	"github.com/weaveworks/weave/mesh"
 	"github.com/weaveworks/weave/net/address"
-	"github.com/weaveworks/weave/router"
 )
 
 func startServer(t *testing.T, upstream *dns.ClientConfig) (*DNSServer, *Nameserver, int, int) {
-	peername, err := router.PeerNameFromString("00:00:00:02:00:00")
+	peername, err := mesh.PeerNameFromString("00:00:00:02:00:00")
 	require.Nil(t, err)
-	nameserver := New(peername, nil, nil, "")
+	nameserver := New(peername, "", func(mesh.PeerName) bool { return true })
 	dnsserver, err := NewDNSServer(nameserver, "weave.local.", "0.0.0.0:0", "", 30, 5*time.Second)
 	require.Nil(t, err)
 	udpPort := dnsserver.servers[0].PacketConn.LocalAddr().(*net.UDPAddr).Port
@@ -39,20 +39,20 @@ func TestTruncation(t *testing.T) {
 	addrs := []address.Address{}
 	for i := address.Address(0); i < 100; i++ {
 		addrs = append(addrs, i)
-		nameserver.AddEntry("foo.weave.local.", "", router.UnknownPeerName, i)
+		nameserver.AddEntry("foo.weave.local.", "", mesh.UnknownPeerName, i)
 	}
 
-	doRequest := func(client *dns.Client, request *dns.Msg, port int) *dns.Msg {
+	doRequest := func(client *dns.Client, request *dns.Msg, port int, expectedErr error) *dns.Msg {
 		request.SetQuestion("foo.weave.local.", dns.TypeA)
 		response, _, err := client.Exchange(request, fmt.Sprintf("127.0.0.1:%d", port))
-		require.Nil(t, err)
+		require.Equal(t, expectedErr, err)
 		return response
 	}
 
 	// do a udp query, ensure we get a truncated response
 	{
 		udpClient := dns.Client{Net: "udp", UDPSize: minUDPSize}
-		response := doRequest(&udpClient, &dns.Msg{}, udpPort)
+		response := doRequest(&udpClient, &dns.Msg{}, udpPort, dns.ErrTruncated)
 		require.True(t, response.MsgHdr.Truncated)
 		require.True(t, len(response.Answer) < 100)
 	}
@@ -62,7 +62,7 @@ func TestTruncation(t *testing.T) {
 		udpClient := dns.Client{Net: "udp", UDPSize: 65535}
 		request := &dns.Msg{}
 		request.SetEdns0(65535, false)
-		response := doRequest(&udpClient, request, udpPort)
+		response := doRequest(&udpClient, request, udpPort, nil)
 		require.False(t, response.MsgHdr.Truncated)
 		require.Equal(t, len(response.Answer), 100)
 	}
@@ -70,7 +70,7 @@ func TestTruncation(t *testing.T) {
 	// do a tcp query, ensure we don't get a truncated response
 	{
 		tcpClient := dns.Client{Net: "tcp"}
-		response := doRequest(&tcpClient, &dns.Msg{}, tcpPort)
+		response := doRequest(&tcpClient, &dns.Msg{}, tcpPort, nil)
 		require.False(t, response.MsgHdr.Truncated)
 		require.Equal(t, len(response.Answer), 100)
 	}

nameserver/entry.go

@@ -5,26 +5,46 @@ import (
 	"encoding/gob"
 	"fmt"
 	"sort"
+	"strings"
 	"time"
 
+	"github.com/weaveworks/weave/mesh"
 	"github.com/weaveworks/weave/net/address"
-	"github.com/weaveworks/weave/router"
 )
 
 var now = func() int64 { return time.Now().Unix() }
 
 type Entry struct {
 	ContainerID string
-	Origin      router.PeerName
+	Origin      mesh.PeerName
 	Addr        address.Address
-	Hostname    string
+	Hostname    string // as supplied
+	lHostname   string // lowercased (not exported, so not encoded by gob)
 	Version     int
 	Tombstone   int64 // timestamp of when it was deleted
 }
 
 type Entries []Entry
+type CaseSensitive Entries
+type CaseInsensitive Entries
+type SortableEntries interface {
+	sort.Interface
+	Get(i int) Entry
+}
 
-func (e1 *Entry) equal(e2 *Entry) bool {
+// Gossip messages are sorted in a case sensitive order...
+func (es CaseSensitive) Len() int           { return len(es) }
+func (es CaseSensitive) Swap(i, j int)      { es[i], es[j] = es[j], es[i] }
+func (es CaseSensitive) Get(i int) Entry    { return es[i] }
+func (es CaseSensitive) Less(i, j int) bool { return es[i].less(&es[j]) }
+
+// ... but we store entries in a case insensitive order.
+func (es CaseInsensitive) Len() int           { return len(es) }
+func (es CaseInsensitive) Swap(i, j int)      { es[i], es[j] = es[j], es[i] }
+func (es CaseInsensitive) Get(i int) Entry    { return es[i] }
+func (es CaseInsensitive) Less(i, j int) bool { return es[i].insensitiveLess(&es[j]) }
+
+func (e1 Entry) equal(e2 Entry) bool {
 	return e1.ContainerID == e2.ContainerID &&
 		e1.Origin == e2.Origin &&
 		e1.Addr == e2.Addr &&
@@ -48,6 +68,24 @@ func (e1 *Entry) less(e2 *Entry) bool {
 	}
 }
 
+func (e1 *Entry) insensitiveLess(e2 *Entry) bool {
+	// Entries are kept sorted by Hostname, Origin, ContainerID then address
+	e1Hostname, e2Hostname := e1.lHostname, e2.lHostname
+	switch {
+	case e1Hostname != e2Hostname:
+		return e1Hostname < e2Hostname
+
+	case e1.Origin != e2.Origin:
+		return e1.Origin < e2.Origin
+
+	case e1.ContainerID != e2.ContainerID:
+		return e1.ContainerID < e2.ContainerID
+
+	default:
+		return e1.Addr < e2.Addr
+	}
+}
+
 // returns true to indicate a change
 func (e1 *Entry) merge(e2 *Entry) bool {
 	// we know container id, origin, add and hostname are equal
@@ -63,37 +101,42 @@ func (e1 *Entry) String() string {
 	return fmt.Sprintf("%s -> %s", e1.Hostname, e1.Addr.String())
 }
 
-func (es Entries) Len() int           { return len(es) }
-func (es Entries) Swap(i, j int)      { panic("Swap") }
-func (es Entries) Less(i, j int) bool { return es[i].less(&es[j]) }
+func (e1 *Entry) addLowercase() {
+	e1.lHostname = strings.ToLower(e1.Hostname)
+}
 
-func (es Entries) check() error {
+func check(es SortableEntries) error {
 	if !sort.IsSorted(es) {
 		return fmt.Errorf("Not sorted!")
 	}
-	for i := 1; i < len(es); i++ {
-		if es[i].equal(&es[i-1]) {
-			return fmt.Errorf("Duplicate entry: %d:%v and %d:%v", i-1, es[i-1], i, es[i])
+	for i := 1; i < es.Len(); i++ {
+		if es.Get(i).equal(es.Get(i - 1)) {
+			return fmt.Errorf("Duplicate entry: %d:%v and %d:%v", i-1, es.Get(i-1), i, es.Get(i))
 		}
 	}
 	return nil
 }
 
-func (es *Entries) checkAndPanic() {
-	if err := es.check(); err != nil {
+func checkAndPanic(es SortableEntries) {
+	if err := check(es); err != nil {
 		panic(err)
 	}
 }
 
-func (es *Entries) add(hostname, containerid string, origin router.PeerName, addr address.Address) Entry {
-	es.checkAndPanic()
-	defer es.checkAndPanic()
+func (es *Entries) checkAndPanic() *Entries {
+	checkAndPanic(CaseInsensitive(*es))
+	return es
+}
 
-	entry := Entry{Hostname: hostname, Origin: origin, ContainerID: containerid, Addr: addr}
+func (es *Entries) add(hostname, containerid string, origin mesh.PeerName, addr address.Address) Entry {
+	defer es.checkAndPanic().checkAndPanic()
+
+	entry := Entry{Hostname: hostname, lHostname: strings.ToLower(hostname),
+		Origin: origin, ContainerID: containerid, Addr: addr}
 	i := sort.Search(len(*es), func(i int) bool {
-		return !(*es)[i].less(&entry)
+		return !(*es)[i].insensitiveLess(&entry)
 	})
-	if i < len(*es) && (*es)[i].equal(&entry) {
+	if i < len(*es) && (*es)[i].equal(entry) {
 		if (*es)[i].Tombstone > 0 {
 			(*es)[i].Tombstone = 0
 			(*es)[i].Version++
@@ -107,16 +150,17 @@ func (es *Entries) add(hostname, containerid string, origin router.PeerName, add
 }
 
 func (es *Entries) merge(incoming Entries) Entries {
-	es.checkAndPanic()
-	defer es.checkAndPanic()
+	defer es.checkAndPanic().checkAndPanic()
+	incoming.checkAndPanic()
+
 	newEntries := Entries{}
 	i := 0
 
 	for _, entry := range incoming {
-		for i < len(*es) && (*es)[i].less(&entry) {
+		for i < len(*es) && (*es)[i].insensitiveLess(&entry) {
 			i++
 		}
-		if i < len(*es) && (*es)[i].equal(&entry) {
+		if i < len(*es) && (*es)[i].equal(entry) {
 			if (*es)[i].merge(&entry) {
 				newEntries = append(newEntries, entry)
 			}
@@ -132,9 +176,9 @@ func (es *Entries) merge(incoming Entries) Entries {
 }
 
 // f returning true means keep the entry.
-func (es *Entries) tombstone(ourname router.PeerName, f func(*Entry) bool) Entries {
-	es.checkAndPanic()
-	defer es.checkAndPanic()
+func (es *Entries) tombstone(ourname mesh.PeerName, f func(*Entry) bool) Entries {
+	defer es.checkAndPanic().checkAndPanic()
+
 	tombstoned := Entries{}
 	for i, e := range *es {
 		if f(&e) && e.Origin == ourname {
@@ -148,8 +192,8 @@ func (es *Entries) tombstone(ourname router.PeerName, f func(*Entry) bool) Entri
 }
 
 func (es *Entries) filter(f func(*Entry) bool) {
-	es.checkAndPanic()
-	defer es.checkAndPanic()
+	defer es.checkAndPanic().checkAndPanic()
+
 	i := 0
 	for _, e := range *es {
 		if !f(&e) {
@@ -163,23 +207,26 @@ func (es *Entries) filter(f func(*Entry) bool) {
 
 func (es Entries) lookup(hostname string) Entries {
 	es.checkAndPanic()
+
+	lowerHostname := strings.ToLower(hostname)
 	i := sort.Search(len(es), func(i int) bool {
-		return es[i].Hostname >= hostname
+		return es[i].lHostname >= lowerHostname
 	})
-	if i >= len(es) || es[i].Hostname != hostname {
+	if i >= len(es) || es[i].lHostname != lowerHostname {
 		return Entries{}
 	}
 
 	j := sort.Search(len(es)-i, func(j int) bool {
-		return es[i+j].Hostname > hostname
+		return es[i+j].lHostname > lowerHostname
 	})
 
 	return es[i : i+j]
 }
 
-func (es *Entries) first(f func(*Entry) bool) (*Entry, error) {
+func (es Entries) first(f func(*Entry) bool) (*Entry, error) {
 	es.checkAndPanic()
-	for _, e := range *es {
+
+	for _, e := range es {
 		if f(&e) {
 			return &e, nil
 		}
@@ -187,23 +234,49 @@ func (es *Entries) first(f func(*Entry) bool) (*Entry, error) {
 	return nil, fmt.Errorf("Not found")
 }
 
+func (es Entries) addLowercase() {
+	for i := range es {
+		es[i].addLowercase()
+	}
+}
+
 type GossipData struct {
 	Timestamp int64
 	Entries
 }
 
-func (g *GossipData) Merge(o router.GossipData) {
+func (g *GossipData) Merge(o mesh.GossipData) mesh.GossipData {
 	other := o.(*GossipData)
-	g.Entries.merge(other.Entries)
-	if g.Timestamp < other.Timestamp {
-		g.Timestamp = other.Timestamp
+	gossip := g.copy()
+	gossip.Entries.merge(other.Entries)
+	if gossip.Timestamp < other.Timestamp {
+		gossip.Timestamp = other.Timestamp
 	}
+	return gossip
+}
+
+func (g *GossipData) Decode(msg []byte) error {
+	if err := gob.NewDecoder(bytes.NewReader(msg)).Decode(g); err != nil {
+		return err
+	}
+
+	g.Entries.addLowercase() // lowercase strings not sent on the wire
+	sort.Sort(CaseInsensitive(g.Entries))
+	return nil
 }
 
 func (g *GossipData) Encode() [][]byte {
+	g2 := g.copy()
+	sort.Sort(CaseSensitive(g2.Entries))
 	buf := &bytes.Buffer{}
-	if err := gob.NewEncoder(buf).Encode(g); err != nil {
+	if err := gob.NewEncoder(buf).Encode(g2); err != nil {
 		panic(err)
 	}
 	return [][]byte{buf.Bytes()}
 }
+
+func (g *GossipData) copy() *GossipData {
+	g2 := &GossipData{Timestamp: g.Timestamp, Entries: make(Entries, len(g.Entries))}
+	copy(g2.Entries, g.Entries)
+	return g2
+}

nameserver/entry_test.go

@@ -5,79 +5,70 @@ import (
 
 	"github.com/stretchr/testify/require"
 
+	"github.com/weaveworks/weave/mesh"
 	"github.com/weaveworks/weave/net/address"
-	"github.com/weaveworks/weave/router"
 )
 
+func l(es Entries) Entries {
+	es.addLowercase()
+	return es
+}
+
+func makeEntries(values string) Entries {
+	entries := make(Entries, len(values))
+	for i, c := range values {
+		entries[i] = Entry{Hostname: string(c)}
+	}
+	return l(entries)
+}
+
 func TestAdd(t *testing.T) {
 	oldNow := now
 	defer func() { now = oldNow }()
 	now = func() int64 { return 1234 }
 
 	entries := Entries{}
-	entries.add("A", "", router.UnknownPeerName, address.Address(0))
-	expected := Entries{
-		Entry{Hostname: "A", Origin: router.UnknownPeerName, Addr: address.Address(0)},
-	}
+	entries.add("A", "", mesh.UnknownPeerName, address.Address(0))
+	expected := l(Entries{
+		Entry{Hostname: "A", Origin: mesh.UnknownPeerName, Addr: address.Address(0)},
+	})
 	require.Equal(t, entries, expected)
 
-	entries.tombstone(router.UnknownPeerName, func(e *Entry) bool { return e.Hostname == "A" })
-	expected = Entries{
-		Entry{Hostname: "A", Origin: router.UnknownPeerName, Addr: address.Address(0), Version: 1, Tombstone: 1234},
-	}
+	entries.tombstone(mesh.UnknownPeerName, func(e *Entry) bool { return e.Hostname == "A" })
+	expected = l(Entries{
+		Entry{Hostname: "A", Origin: mesh.UnknownPeerName, Addr: address.Address(0), Version: 1, Tombstone: 1234},
+	})
 	require.Equal(t, entries, expected)
 
-	entries.add("A", "", router.UnknownPeerName, address.Address(0))
-	expected = Entries{
-		Entry{Hostname: "A", Origin: router.UnknownPeerName, Addr: address.Address(0), Version: 2},
-	}
+	entries.add("A", "", mesh.UnknownPeerName, address.Address(0))
+	expected = l(Entries{
+		Entry{Hostname: "A", Origin: mesh.UnknownPeerName, Addr: address.Address(0), Version: 2},
+	})
 	require.Equal(t, entries, expected)
 }
 
 func TestMerge(t *testing.T) {
-	e1 := Entries{
-		Entry{Hostname: "A"},
-		Entry{Hostname: "C"},
-		Entry{Hostname: "D"},
-		Entry{Hostname: "F"},
-	}
-
-	e2 := Entries{
-		Entry{Hostname: "B"},
-		Entry{Hostname: "E"},
-		Entry{Hostname: "F"},
-	}
+	e1 := makeEntries("ACDF")
+	e2 := makeEntries("BEF")
 
 	diff := e1.merge(e2)
-	expectedDiff := Entries{
-		Entry{Hostname: "B"},
-		Entry{Hostname: "E"},
-	}
-	require.Equal(t, expectedDiff, diff)
 
-	expected := Entries{
-		Entry{Hostname: "A"},
-		Entry{Hostname: "B"},
-		Entry{Hostname: "C"},
-		Entry{Hostname: "D"},
-		Entry{Hostname: "E"},
-		Entry{Hostname: "F"},
-	}
-	require.Equal(t, expected, e1)
+	require.Equal(t, makeEntries("BE"), diff)
+	require.Equal(t, makeEntries("ABCDEF"), e1)
 
 	diff = e1.merge(e1)
 	require.Equal(t, Entries{}, diff)
 }
 
 func TestOldMerge(t *testing.T) {
-	e1 := Entries{Entry{Hostname: "A", Version: 0}}
-	diff := e1.merge(Entries{Entry{Hostname: "A", Version: 1}})
-	require.Equal(t, Entries{Entry{Hostname: "A", Version: 1}}, diff)
-	require.Equal(t, Entries{Entry{Hostname: "A", Version: 1}}, e1)
+	e1 := l(Entries{Entry{Hostname: "A", Version: 0}})
+	diff := e1.merge(l(Entries{Entry{Hostname: "A", Version: 1}}))
+	require.Equal(t, l(Entries{Entry{Hostname: "A", Version: 1}}), diff)
+	require.Equal(t, l(Entries{Entry{Hostname: "A", Version: 1}}), e1)
 
-	diff = e1.merge(Entries{Entry{Hostname: "A", Version: 0}})
+	diff = e1.merge(l(Entries{Entry{Hostname: "A", Version: 0}}))
 	require.Equal(t, Entries{}, diff)
-	require.Equal(t, Entries{Entry{Hostname: "A", Version: 1}}, e1)
+	require.Equal(t, l(Entries{Entry{Hostname: "A", Version: 1}}), e1)
 }
 
 func TestTombstone(t *testing.T) {
@@ -85,48 +76,48 @@ func TestTombstone(t *testing.T) {
 	defer func() { now = oldNow }()
 	now = func() int64 { return 1234 }
 
-	es := Entries{
-		Entry{Hostname: "A"},
-		Entry{Hostname: "B"},
-	}
+	es := makeEntries("AB")
 
-	es.tombstone(router.UnknownPeerName, func(e *Entry) bool {
+	es.tombstone(mesh.UnknownPeerName, func(e *Entry) bool {
 		return e.Hostname == "B"
 	})
-	expected := Entries{
+	expected := l(Entries{
 		Entry{Hostname: "A"},
 		Entry{Hostname: "B", Version: 1, Tombstone: 1234},
-	}
+	})
 	require.Equal(t, expected, es)
 }
 
 func TestDelete(t *testing.T) {
-	es := Entries{
-		Entry{Hostname: "A"},
-		Entry{Hostname: "B"},
-	}
+	es := makeEntries("AB")
 
 	es.filter(func(e *Entry) bool {
 		return e.Hostname != "A"
 	})
-	expected := Entries{
-		Entry{Hostname: "B"},
-	}
-	require.Equal(t, expected, es)
+	require.Equal(t, makeEntries("B"), es)
 }
 
 func TestLookup(t *testing.T) {
-	es := Entries{
+	es := l(Entries{
 		Entry{Hostname: "A"},
 		Entry{Hostname: "B", ContainerID: "bar"},
 		Entry{Hostname: "B", ContainerID: "foo"},
 		Entry{Hostname: "C"},
-	}
+	})
 
 	have := es.lookup("B")
-	want := Entries{
+	want := l(Entries{
 		Entry{Hostname: "B", ContainerID: "bar"},
 		Entry{Hostname: "B", ContainerID: "foo"},
-	}
+	})
 	require.Equal(t, have, want)
 }
+
+func TestGossipDataMerge(t *testing.T) {
+	g1 := GossipData{Entries: makeEntries("AcDf")}
+	g2 := GossipData{Entries: makeEntries("BEf")}
+
+	g3 := g1.Merge(&g2).(*GossipData)
+
+	require.Equal(t, GossipData{Entries: makeEntries("ABcDEf")}, *g3)
+}

nameserver/http.go

@@ -8,6 +8,7 @@ import (
 	"github.com/gorilla/mux"
 	"github.com/miekg/dns"
 
+	"github.com/weaveworks/weave/common/docker"
 	"github.com/weaveworks/weave/net/address"
 )
 
@@ -16,7 +17,7 @@ func (n *Nameserver) badRequest(w http.ResponseWriter, err error) {
 	n.infof("%v", err)
 }
 
-func (n *Nameserver) HandleHTTP(router *mux.Router) {
+func (n *Nameserver) HandleHTTP(router *mux.Router, dockerCli *docker.Client) {
 	router.Methods("GET").Path("/domain").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		fmt.Fprint(w, n.domain)
 	})
@@ -34,12 +35,17 @@ func (n *Nameserver) HandleHTTP(router *mux.Router) {
 			return
 		}
 
+		if !dns.IsSubDomain(n.domain, hostname) {
+			n.infof("Ignoring registration %s %s %s (not a subdomain of %s)", hostname, ipStr, container, n.domain)
+			return
+		}
+
 		if err := n.AddEntry(hostname, container, n.ourName, ip); err != nil {
 			n.badRequest(w, fmt.Errorf("Unable to add entry: %v", err))
 			return
 		}
 
-		if r.FormValue("check-alive") == "true" && n.docker.IsContainerNotRunning(container) {
+		if r.FormValue("check-alive") == "true" && dockerCli != nil && dockerCli.IsContainerNotRunning(container) {
 			n.infof("container '%s' is not running: removing", container)
 			if err := n.Delete(hostname, container, ipStr, ip); err != nil {
 				n.infof("failed to remove: %v", err)

nameserver/nameserver.go

@@ -1,8 +1,6 @@
 package nameserver
 
 import (
-	"bytes"
-	"encoding/gob"
 	"fmt"
 	"sync"
 	"time"
@@ -10,9 +8,8 @@ import (
 	"github.com/miekg/dns"
 
 	. "github.com/weaveworks/weave/common"
-	"github.com/weaveworks/weave/common/docker"
+	"github.com/weaveworks/weave/mesh"
 	"github.com/weaveworks/weave/net/address"
-	"github.com/weaveworks/weave/router"
 )
 
 const (
@@ -37,30 +34,24 @@ const (
 // - Update is O(n) for now
 type Nameserver struct {
 	sync.RWMutex
-	ourName router.PeerName
-	domain  string
-	gossip  router.Gossip
-	docker  *docker.Client
-	entries Entries
-	peers   *router.Peers
-	quit    chan struct{}
+	ourName     mesh.PeerName
+	domain      string
+	gossip      mesh.Gossip
+	entries     Entries
+	isKnownPeer func(mesh.PeerName) bool
+	quit        chan struct{}
 }
 
-func New(ourName router.PeerName, peers *router.Peers, docker *docker.Client, domain string) *Nameserver {
-	ns := &Nameserver{
-		ourName: ourName,
-		domain:  dns.Fqdn(domain),
-		peers:   peers,
-		docker:  docker,
-		quit:    make(chan struct{}),
+func New(ourName mesh.PeerName, domain string, isKnownPeer func(mesh.PeerName) bool) *Nameserver {
+	return &Nameserver{
+		ourName:     ourName,
+		domain:      dns.Fqdn(domain),
+		isKnownPeer: isKnownPeer,
+		quit:        make(chan struct{}),
 	}
-	if peers != nil {
-		peers.OnGC(ns.PeerGone)
-	}
-	return ns
 }
 
-func (n *Nameserver) SetGossip(gossip router.Gossip) {
+func (n *Nameserver) SetGossip(gossip mesh.Gossip) {
 	n.gossip = gossip
 }
 
@@ -92,7 +83,7 @@ func (n *Nameserver) broadcastEntries(es ...Entry) error {
 	return nil
 }
 
-func (n *Nameserver) AddEntry(hostname, containerid string, origin router.PeerName, addr address.Address) error {
+func (n *Nameserver) AddEntry(hostname, containerid string, origin mesh.PeerName, addr address.Address) error {
 	n.infof("adding entry %s -> %s", hostname, addr.String())
 	n.Lock()
 	entry := n.entries.add(hostname, containerid, origin, addr)
@@ -130,6 +121,8 @@ func (n *Nameserver) ReverseLookup(ip address.Address) (string, error) {
 	return match.Hostname, nil
 }
 
+func (n *Nameserver) ContainerStarted(ident string) {}
+
 func (n *Nameserver) ContainerDied(ident string) {
 	n.Lock()
 	entries := n.entries.tombstone(n.ourName, func(e *Entry) bool {
@@ -147,12 +140,12 @@ func (n *Nameserver) ContainerDied(ident string) {
 	}
 }
 
-func (n *Nameserver) PeerGone(peer *router.Peer) {
+func (n *Nameserver) PeerGone(peer mesh.PeerName) {
 	n.infof("peer %s gone", peer.String())
 	n.Lock()
 	defer n.Unlock()
 	n.entries.filter(func(e *Entry) bool {
-		return e.Origin != peer.Name
+		return e.Origin != peer
 	})
 }
 
@@ -188,7 +181,7 @@ func (n *Nameserver) deleteTombstones() {
 	})
 }
 
-func (n *Nameserver) Gossip() router.GossipData {
+func (n *Nameserver) Gossip() mesh.GossipData {
 	n.RLock()
 	defer n.RUnlock()
 	gossip := &GossipData{
@@ -199,32 +192,25 @@ func (n *Nameserver) Gossip() router.GossipData {
 	return gossip
 }
 
-func (n *Nameserver) OnGossipUnicast(sender router.PeerName, msg []byte) error {
+func (n *Nameserver) OnGossipUnicast(sender mesh.PeerName, msg []byte) error {
 	return nil
 }
 
-func (n *Nameserver) receiveGossip(msg []byte) (router.GossipData, router.GossipData, error) {
+func (n *Nameserver) receiveGossip(msg []byte) (mesh.GossipData, mesh.GossipData, error) {
 	var gossip GossipData
-	if err := gob.NewDecoder(bytes.NewReader(msg)).Decode(&gossip); err != nil {
+	if err := gossip.Decode(msg); err != nil {
 		return nil, nil, err
 	}
-
 	if delta := gossip.Timestamp - now(); delta > gossipWindow || delta < -gossipWindow {
 		return nil, nil, fmt.Errorf("clock skew of %d detected", delta)
 	}
 
-	if err := gossip.Entries.check(); err != nil {
-		return nil, nil, err
-	}
-
 	n.Lock()
 	defer n.Unlock()
 
-	if n.peers != nil {
-		gossip.Entries.filter(func(e *Entry) bool {
-			return n.peers.Fetch(e.Origin) != nil
-		})
-	}
+	gossip.Entries.filter(func(e *Entry) bool {
+		return n.isKnownPeer(e.Origin)
+	})
 
 	newEntries := n.entries.merge(gossip.Entries)
 	if len(newEntries) > 0 {
@@ -235,14 +221,14 @@ func (n *Nameserver) receiveGossip(msg []byte) (router.GossipData, router.Gossip
 
 // merge received data into state and return "everything new I've
 // just learnt", or nil if nothing in the received data was new
-func (n *Nameserver) OnGossip(msg []byte) (router.GossipData, error) {
+func (n *Nameserver) OnGossip(msg []byte) (mesh.GossipData, error) {
 	newEntries, _, err := n.receiveGossip(msg)
 	return newEntries, err
 }
 
 // merge received data into state and return a representation of
 // the received data, for further propagation
-func (n *Nameserver) OnGossipBroadcast(_ router.PeerName, msg []byte) (router.GossipData, error) {
+func (n *Nameserver) OnGossipBroadcast(_ mesh.PeerName, msg []byte) (mesh.GossipData, error) {
 	_, entries, err := n.receiveGossip(msg)
 	return entries, err
 }

nameserver/nameserver_test.go

@@ -11,19 +11,23 @@ import (
 
 	"github.com/stretchr/testify/require"
 
+	"github.com/weaveworks/weave/mesh"
 	"github.com/weaveworks/weave/net/address"
-	"github.com/weaveworks/weave/router"
 	wt "github.com/weaveworks/weave/testing"
 	"github.com/weaveworks/weave/testing/gossip"
 )
 
+func makeNameserver(name mesh.PeerName) *Nameserver {
+	return New(name, "", func(mesh.PeerName) bool { return true })
+}
+
 func makeNetwork(size int) ([]*Nameserver, *gossip.TestRouter) {
 	gossipRouter := gossip.NewTestRouter(0.0)
 	nameservers := make([]*Nameserver, size)
 
 	for i := 0; i < size; i++ {
-		name, _ := router.PeerNameFromString(fmt.Sprintf("%02d:00:00:02:00:00", i))
-		nameserver := New(name, nil, nil, "")
+		name, _ := mesh.PeerNameFromString(fmt.Sprintf("%02d:00:00:02:00:00", i))
+		nameserver := makeNameserver(name)
 		nameserver.SetGossip(gossipRouter.Connect(nameserver.ourName, nameserver))
 		nameserver.Start()
 		nameservers[i] = nameserver
@@ -40,7 +44,7 @@ func stopNetwork(nameservers []*Nameserver, grouter *gossip.TestRouter) {
 }
 
 type pair struct {
-	origin router.PeerName
+	origin mesh.PeerName
 	addr   address.Address
 }
 
@@ -72,7 +76,7 @@ func (m mapping) Addrs() []address.Address {
 }
 
 func TestNameservers(t *testing.T) {
-	wt.RunWithTimeout(t, time.Minute, func() {
+	wt.RunWithTimeout(t, 2*time.Minute, func() {
 		testNameservers(t)
 	})
 }
@@ -81,9 +85,13 @@ func testNameservers(t *testing.T) {
 	//common.SetLogLevel("debug")
 
 	lookupTimeout := 10 // ms
-	nameservers, grouter := makeNetwork(50)
+	nameservers, grouter := makeNetwork(30)
 	defer stopNetwork(nameservers, grouter)
-	nameserversByName := map[router.PeerName]*Nameserver{}
+	// This subset will sometimes lose touch with some of the others
+	badNameservers := nameservers[25:]
+	// This subset will remain well-connected, and we will deal mainly with them
+	nameservers = nameservers[:25]
+	nameserversByName := map[mesh.PeerName]*Nameserver{}
 	for _, n := range nameservers {
 		nameserversByName[n.ourName] = n
 	}
@@ -106,7 +114,13 @@ func testNameservers(t *testing.T) {
 	addMapping := func() {
 		nameserver := nameservers[rand.Intn(len(nameservers))]
 		addr := address.Address(rand.Int31())
-		hostname := fmt.Sprintf("hostname%d", rand.Int63())
+		// Create a hostname which has some upper and lowercase letters,
+		// and a unique number so we don't have to check if we allocated it already
+		randomBits := rand.Int63()
+		firstLetter := 'H' + (randomBits&1)*32
+		secondLetter := 'O' + (randomBits&2)*16
+		randomBits = randomBits >> 2
+		hostname := fmt.Sprintf("%c%cstname%d", firstLetter, secondLetter, randomBits)
 		mapping := mapping{hostname, []pair{{nameserver.ourName, addr}}}
 		mappings = append(mappings, mapping)
 
@@ -129,6 +143,12 @@ func testNameservers(t *testing.T) {
 		check(nameserver, mapping)
 	}
 
+	loseConnection := func() {
+		nameserver1 := badNameservers[rand.Intn(len(badNameservers))]
+		nameserver2 := nameservers[rand.Intn(len(nameservers))]
+		nameserver1.PeerGone(nameserver2.ourName)
+	}
+
 	deleteMapping := func() {
 		if len(mappings) <= 0 {
 			return
@@ -179,7 +199,7 @@ func testNameservers(t *testing.T) {
 		require.Equal(t, mapping.hostname, hostname)
 	}
 
-	for i := 0; i < 1000; i++ {
+	for i := 0; i < 800; i++ {
 		r := rand.Float32()
 		switch {
 		case r < 0.1:
@@ -191,7 +211,10 @@ func testNameservers(t *testing.T) {
 		case 0.2 <= r && r < 0.3:
 			deleteMapping()
 
-		case 0.3 <= r && r < 0.9:
+		case 0.3 <= r && r < 0.35:
+			loseConnection()
+
+		case 0.35 <= r && r < 0.9:
 			doLookup()
 
 		case 0.9 <= r:
@@ -203,9 +226,9 @@ func testNameservers(t *testing.T) {
 }
 
 func TestContainerAndPeerDeath(t *testing.T) {
-	peername, err := router.PeerNameFromString("00:00:00:02:00:00")
+	peername, err := mesh.PeerNameFromString("00:00:00:02:00:00")
 	require.Nil(t, err)
-	nameserver := New(peername, nil, nil, "")
+	nameserver := makeNameserver(peername)
 
 	err = nameserver.AddEntry("hostname", "containerid", peername, address.Address(0))
 	require.Nil(t, err)
@@ -218,7 +241,7 @@ func TestContainerAndPeerDeath(t *testing.T) {
 	require.Nil(t, err)
 	require.Equal(t, []address.Address{0}, nameserver.Lookup("hostname"))
 
-	nameserver.PeerGone(&router.Peer{Name: peername})
+	nameserver.PeerGone(peername)
 	require.Equal(t, []address.Address{}, nameserver.Lookup("hostname"))
 }
 
@@ -227,9 +250,9 @@ func TestTombstoneDeletion(t *testing.T) {
 	defer func() { now = oldNow }()
 	now = func() int64 { return 1234 }
 
-	peername, err := router.PeerNameFromString("00:00:00:02:00:00")
+	peername, err := mesh.PeerNameFromString("00:00:00:02:00:00")
 	require.Nil(t, err)
-	nameserver := New(peername, nil, nil, "")
+	nameserver := makeNameserver(peername)
 
 	err = nameserver.AddEntry("hostname", "containerid", peername, address.Address(0))
 	require.Nil(t, err)
@@ -241,14 +264,14 @@ func TestTombstoneDeletion(t *testing.T) {
 	err = nameserver.Delete("hostname", "containerid", "", address.Address(0))
 	require.Nil(t, err)
 	require.Equal(t, []address.Address{}, nameserver.Lookup("hostname"))
-	require.Equal(t, Entries{{
+	require.Equal(t, l(Entries{Entry{
 		ContainerID: "containerid",
 		Origin:      peername,
 		Addr:        address.Address(0),
 		Hostname:    "hostname",
 		Version:     1,
 		Tombstone:   1234,
-	}}, nameserver.entries)
+	}}), nameserver.entries)
 
 	now = func() int64 { return 1234 + int64(tombstoneTimeout/time.Second) + 1 }
 	nameserver.deleteTombstones()

net/address/address.go

@@ -19,7 +19,7 @@ func NewRange(start Address, size Offset) Range {
 	return Range{Start: start, End: Add(start, size)}
 }
 func (r Range) Size() Offset               { return Subtract(r.End, r.Start) }
-func (r Range) String() string             { return fmt.Sprintf("[%s-%s)", r.Start, r.End) }
+func (r Range) String() string             { return fmt.Sprintf("%s-%s", r.Start, r.End-1) }
 func (r Range) Overlaps(or Range) bool     { return !(r.Start >= or.End || r.End <= or.Start) }
 func (r Range) Contains(addr Address) bool { return addr >= r.Start && addr < r.End }
 

prog/docker_tls_args/main.go

@@ -0,0 +1,77 @@
+// docker_tls_args: find the docker daemon's tls args
+// This reimplements pgrep, because pgrep will only look in /proc, but not in
+// $PROCFS.
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+)
+
+func main() {
+	procRoot := os.Getenv("PROCFS")
+	if procRoot == "" {
+		procRoot = "/proc"
+	}
+
+	dirEntries, err := ioutil.ReadDir(procRoot)
+	checkErr(err)
+
+	for _, dirEntry := range dirEntries {
+		dirName := dirEntry.Name()
+		if _, err := strconv.Atoi(dirName); err != nil {
+			continue
+		}
+
+		if comm, err := ioutil.ReadFile(filepath.Join(procRoot, dirName, "comm")); err != nil || string(comm) != "docker\n" {
+			continue
+		}
+
+		cmdline, err := ioutil.ReadFile(filepath.Join(procRoot, dirName, "cmdline"))
+		if err != nil {
+			continue
+		}
+
+		isDaemon := false
+		tlsArgs := []string{}
+		args := bytes.Split(cmdline, []byte{'\000'})
+		for i := 0; i < len(args); i++ {
+			arg := string(args[i])
+			switch {
+			case arg == "-d" || arg == "daemon":
+				isDaemon = true
+				break
+			case arg == "--tls", arg == "--tlsverify":
+				tlsArgs = append(tlsArgs, arg)
+			case strings.HasPrefix(arg, "--tls"):
+				tlsArgs = append(tlsArgs, arg)
+				if len(args) > i+1 &&
+					!strings.Contains(arg, "=") &&
+					!strings.HasPrefix(string(args[i+1]), "-") {
+					tlsArgs = append(tlsArgs, string(args[i+1]))
+					i++
+				}
+			}
+		}
+		if !isDaemon {
+			continue
+		}
+
+		fmt.Println(strings.Join(tlsArgs, " "))
+		os.Exit(0)
+	}
+
+	os.Exit(1)
+}
+
+func checkErr(err error) {
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+}

prog/weaveexec/Dockerfile

@@ -1,10 +1,11 @@
-FROM gliderlabs/alpine
+FROM alpine
 
 MAINTAINER Weaveworks Inc <help@weave.works>
 LABEL works.weave.role=system
 
 WORKDIR /home/weave
 VOLUME /w
+VOLUME /w-noop
 ENTRYPOINT ["/home/weave/sigproxy", "/home/weave/weave"]
 
 RUN apk add --update \
@@ -18,6 +19,7 @@ RUN apk add --update \
   && rm -rf /var/cache/apk/*
 
 ADD ./weave ./sigproxy ./weaveproxy /home/weave/
-ADD ./netcheck ./weavehosts /usr/bin/
+ADD ./netcheck ./docker_tls_args /usr/bin/
 ADD ./weavewait /w/w
+ADD ./weavewait_noop /w-noop/w
 ADD ./docker.tgz /

prog/weavehosts/weavehosts.go

@@ -1,120 +0,0 @@
-/* weavehosts: rewrite /etc/hosts as necessary to put the weave address(es) at the top */
-package main
-
-import (
-	"bufio"
-	"bytes"
-	"fmt"
-	"io/ioutil"
-	"net"
-	"os"
-	"sort"
-	"strings"
-)
-
-func main() {
-	args := os.Args
-	if len(args) < 4 {
-		fmt.Fprintln(os.Stderr, "usage: weavehosts hostpath hostname cidr [cidr ...]")
-		os.Exit(1)
-	}
-
-	checkErr(updateHosts(args[1], args[2], args[3:]))
-
-	os.Exit(0)
-}
-
-func checkErr(err error) {
-	if err != nil {
-		fmt.Fprintln(os.Stderr, err)
-		os.Exit(1)
-	}
-}
-
-func updateHosts(path, hostname string, cidrs []string) error {
-	hosts, err := parseHosts(path)
-	if err != nil {
-		return err
-	}
-
-	// Remove existing ips pointing to our hostname
-	toRemove := []string{}
-	for ip, addrs := range hosts {
-		for _, addr := range addrs {
-			if addr == hostname {
-				toRemove = append(toRemove, ip)
-				break
-			}
-		}
-	}
-	for _, ip := range toRemove {
-		delete(hosts, ip)
-	}
-
-	// Add the weave ip(s)
-	for _, cidr := range cidrs {
-		ip, _, err := net.ParseCIDR(cidr)
-		checkErr(err)
-		ipStr := ip.String()
-		hosts[ipStr] = append(hosts[ipStr], hostname)
-	}
-
-	return writeHosts(path, hosts)
-}
-
-func parseHosts(path string) (map[string][]string, error) {
-	f, err := os.Open(path)
-	if err != nil {
-		return nil, err
-	}
-	defer f.Close()
-	ips := map[string][]string{}
-	scanner := bufio.NewScanner(f)
-	for scanner.Scan() {
-		line := scanner.Text()
-
-		// Remove any comments
-		if i := strings.IndexByte(line, '#'); i != -1 {
-			line = line[:i]
-		}
-
-		fields := strings.Fields(line)
-		if len(fields) > 0 {
-			ips[fields[0]] = append(ips[fields[0]], fields[1:]...)
-		}
-	}
-	if scanner.Err() != nil {
-		return nil, scanner.Err()
-	}
-	return ips, nil
-}
-
-func writeHosts(path string, contents map[string][]string) error {
-	ips := []string{}
-	for ip := range contents {
-		ips = append(ips, ip)
-	}
-	sort.Strings(ips)
-
-	buf := &bytes.Buffer{}
-	fmt.Fprintln(buf, "# modified by weave")
-	for _, ip := range ips {
-		if addrs := contents[ip]; len(addrs) > 0 {
-			fmt.Fprintf(buf, "%s\t%s\n", ip, strings.Join(uniqueStrs(addrs), " "))
-		}
-	}
-	return ioutil.WriteFile(path, buf.Bytes(), 644)
-}
-
-func uniqueStrs(s []string) []string {
-	m := map[string]struct{}{}
-	result := []string{}
-	for _, str := range s {
-		if _, ok := m[str]; !ok {
-			m[str] = struct{}{}
-			result = append(result, str)
-		}
-	}
-	sort.Strings(result)
-	return result
-}

prog/weaveproxy/main.go

@@ -35,7 +35,7 @@ func main() {
 	mflag.BoolVar(&c.NoRewriteHosts, []string{"-no-rewrite-hosts"}, false, "do not automatically rewrite /etc/hosts. Use if you need the docker IP to remain in /etc/hosts")
 	mflag.StringVar(&c.TLSConfig.CACert, []string{"#tlscacert", "-tlscacert"}, "", "Trust certs signed only by this CA")
 	mflag.StringVar(&c.TLSConfig.Cert, []string{"#tlscert", "-tlscert"}, "", "Path to TLS certificate file")
-	mflag.BoolVar(&c.TLSConfig.Enabled, []string{"#tls", "-tls"}, false, "Use TLS; implied by --tls-verify")
+	mflag.BoolVar(&c.TLSConfig.Enabled, []string{"#tls", "-tls"}, false, "Use TLS; implied by --tlsverify")
 	mflag.StringVar(&c.TLSConfig.Key, []string{"#tlskey", "-tlskey"}, "", "Path to TLS key file")
 	mflag.BoolVar(&c.TLSConfig.Verify, []string{"#tlsverify", "-tlsverify"}, false, "Use TLS and verify the remote")
 	mflag.BoolVar(&c.WithDNS, []string{"-with-dns", "w"}, false, "instruct created containers to always use weaveDNS as their nameserver")
@@ -43,7 +43,7 @@ func main() {
 	mflag.Parse()
 
 	if justVersion {
-		fmt.Printf("weave proxy %s\n", version)
+		fmt.Printf("weave proxy  %s\n", version)
 		os.Exit(0)
 	}
 
@@ -61,6 +61,9 @@ func main() {
 		Log.Fatalf("Could not start proxy: %s", err)
 	}
 
-	go p.ListenAndServe()
+	listeners := p.Listen()
+	p.AttachExistingContainers()
+	go p.Serve(listeners)
+	go p.ListenAndServeStatus("/home/weave/status.sock")
 	SignalHandlerLoop()
 }

prog/weaver/http.go

@@ -6,6 +6,7 @@ import (
 	"github.com/gorilla/mux"
 	. "github.com/weaveworks/weave/common"
 	"github.com/weaveworks/weave/ipam"
+	"github.com/weaveworks/weave/mesh"
 	"github.com/weaveworks/weave/nameserver"
 	"github.com/weaveworks/weave/net/address"
 	weave "github.com/weaveworks/weave/router"
@@ -24,14 +25,14 @@ var rootTemplate = template.New("root").Funcs(map[string]interface{}{
 		}
 		return count
 	},
-	"printConnectionCounts": func(conns []weave.LocalConnectionStatus) string {
+	"printConnectionCounts": func(conns []mesh.LocalConnectionStatus) string {
 		counts := make(map[string]int)
 		for _, conn := range conns {
 			counts[conn.State]++
 		}
 		return printCounts(counts, []string{"established", "pending", "retrying", "failed", "connecting"})
 	},
-	"printPeerConnectionCounts": func(peers []weave.PeerStatus) string {
+	"printPeerConnectionCounts": func(peers []mesh.PeerStatus) string {
 		counts := make(map[string]int)
 		for _, peer := range peers {
 			for _, conn := range peer.Connections {
@@ -89,7 +90,7 @@ var statusTemplate = defTemplate("status", `\
  PeerDiscovery: {{printState .Router.PeerDiscovery}}
        Targets: {{len .Router.Targets}}
    Connections: {{len .Router.Connections}}{{with printConnectionCounts .Router.Connections}} ({{.}}){{end}}
-         Peers: {{len .Router.Peers}}{{with printPeerConnectionCounts .Router.Peers}} (with {{.}} connections between them){{end}}
+         Peers: {{len .Router.Peers}}{{with printPeerConnectionCounts .Router.Peers}} (with {{.}} connections){{end}}
 {{if .IPAM}}\
 
        Service: ipam
@@ -146,22 +147,27 @@ var dnsEntriesTemplate = defTemplate("dnsEntries", `\
 
 type WeaveStatus struct {
 	Version string
-	Router  *weave.Status      `json:"Router,omitempty"`
-	IPAM    *ipam.Status       `json:"IPAM,omitempty"`
-	DNS     *nameserver.Status `json:"DNS,omitempty"`
+	Router  *weave.NetworkRouterStatus `json:"Router,omitempty"`
+	IPAM    *ipam.Status               `json:"IPAM,omitempty"`
+	DNS     *nameserver.Status         `json:"DNS,omitempty"`
 }
 
-func HandleHTTP(muxRouter *mux.Router, version string, router *weave.Router, allocator *ipam.Allocator, defaultSubnet address.CIDR, ns *nameserver.Nameserver, dnsserver *nameserver.DNSServer) {
+func HandleHTTP(muxRouter *mux.Router, version string, router *weave.NetworkRouter, allocator *ipam.Allocator, defaultSubnet address.CIDR, ns *nameserver.Nameserver, dnsserver *nameserver.DNSServer) {
 	status := func() WeaveStatus {
 		return WeaveStatus{
 			version,
-			weave.NewStatus(router),
+			weave.NewNetworkRouterStatus(router),
 			ipam.NewStatus(allocator, defaultSubnet),
 			nameserver.NewStatus(ns, dnsserver)}
 	}
 	muxRouter.Methods("GET").Path("/report").Headers("Accept", "application/json").HandlerFunc(
 		func(w http.ResponseWriter, r *http.Request) {
-			json, _ := json.MarshalIndent(status(), "", "    ")
+			json, err := json.MarshalIndent(status(), "", "    ")
+			if err != nil {
+				http.Error(w, err.Error(), http.StatusInternalServerError)
+				Log.Error("Error during report marshalling: ", err)
+				return
+			}
 			w.Header().Set("Content-Type", "application/json")
 			w.Write(json)
 		})

prog/weaver/main.go

@@ -1,7 +1,6 @@
 package main
 
 import (
-	"crypto/sha256"
 	"fmt"
 	"net"
 	"net/http"
@@ -17,7 +16,9 @@ import (
 
 	. "github.com/weaveworks/weave/common"
 	"github.com/weaveworks/weave/common/docker"
+	"github.com/weaveworks/weave/common/odp"
 	"github.com/weaveworks/weave/ipam"
+	"github.com/weaveworks/weave/mesh"
 	"github.com/weaveworks/weave/nameserver"
 	weavenet "github.com/weaveworks/weave/net"
 	"github.com/weaveworks/weave/net/address"
@@ -36,36 +37,47 @@ func main() {
 	runtime.GOMAXPROCS(procs)
 
 	var (
-		config             weave.Config
-		justVersion        bool
-		protocolMinVersion int
-		ifaceName          string
-		routerName         string
-		nickName           string
-		password           string
-		pktdebug           bool
-		logLevel           string
-		prof               string
-		bufSzMB            int
-		noDiscovery        bool
-		httpAddr           string
-		iprangeCIDR        string
-		ipsubnetCIDR       string
-		peerCount          int
-		apiPath            string
-		peers              []string
+		// flags that cause immediate exit
+		justVersion          bool
+		createDatapath       bool
+		deleteDatapath       bool
+		addDatapathInterface string
 
+		config                    mesh.Config
+		networkConfig             weave.NetworkConfig
+		protocolMinVersion        int
+		ifaceName                 string
+		routerName                string
+		nickName                  string
+		password                  string
+		pktdebug                  bool
+		logLevel                  string
+		prof                      string
+		bufSzMB                   int
+		noDiscovery               bool
+		httpAddr                  string
+		iprangeCIDR               string
+		ipsubnetCIDR              string
+		peerCount                 int
+		dockerAPI                 string
+		peers                     []string
 		noDNS                     bool
 		dnsDomain                 string
 		dnsListenAddress          string
 		dnsTTL                    int
 		dnsClientTimeout          time.Duration
 		dnsEffectiveListenAddress string
+		iface                     *net.Interface
+		datapathName              string
 	)
 
 	mflag.BoolVar(&justVersion, []string{"#version", "-version"}, false, "print version and exit")
-	mflag.IntVar(&config.Port, []string{"#port", "-port"}, weave.Port, "router port")
-	mflag.IntVar(&protocolMinVersion, []string{"-min-protocol-version"}, weave.ProtocolMinVersion, "minimum weave protocol version")
+	mflag.BoolVar(&createDatapath, []string{"-create-datapath"}, false, "create ODP datapath and exit")
+	mflag.BoolVar(&deleteDatapath, []string{"-delete-datapath"}, false, "delete ODP datapath and exit")
+	mflag.StringVar(&addDatapathInterface, []string{"-add-datapath-iface"}, "", "add a network interface to the ODP datapath and exit")
+
+	mflag.IntVar(&config.Port, []string{"#port", "-port"}, mesh.Port, "router port")
+	mflag.IntVar(&protocolMinVersion, []string{"-min-protocol-version"}, mesh.ProtocolMinVersion, "minimum weave protocol version")
 	mflag.StringVar(&ifaceName, []string{"#iface", "-iface"}, "", "name of interface to capture/inject from (disabled if blank)")
 	mflag.StringVar(&routerName, []string{"#name", "-name"}, "", "name of router (defaults to MAC of interface)")
 	mflag.StringVar(&nickName, []string{"#nickname", "-nickname"}, "", "nickname of peer (defaults to hostname)")
@@ -76,17 +88,18 @@ func main() {
 	mflag.IntVar(&config.ConnLimit, []string{"#connlimit", "#-connlimit", "-conn-limit"}, 30, "connection limit (0 for unlimited)")
 	mflag.BoolVar(&noDiscovery, []string{"#nodiscovery", "#-nodiscovery", "-no-discovery"}, false, "disable peer discovery")
 	mflag.IntVar(&bufSzMB, []string{"#bufsz", "-bufsz"}, 8, "capture buffer size in MB")
-	mflag.StringVar(&httpAddr, []string{"#httpaddr", "#-httpaddr", "-http-addr"}, fmt.Sprintf(":%d", weave.HTTPPort), "address to bind HTTP interface to (disabled if blank, absolute path indicates unix domain socket)")
+	mflag.StringVar(&httpAddr, []string{"#httpaddr", "#-httpaddr", "-http-addr"}, "", "address to bind HTTP interface to (disabled if blank, absolute path indicates unix domain socket)")
 	mflag.StringVar(&iprangeCIDR, []string{"#iprange", "#-iprange", "-ipalloc-range"}, "", "IP address range reserved for automatic allocation, in CIDR notation")
 	mflag.StringVar(&ipsubnetCIDR, []string{"#ipsubnet", "#-ipsubnet", "-ipalloc-default-subnet"}, "", "subnet to allocate within by default, in CIDR notation")
 	mflag.IntVar(&peerCount, []string{"#initpeercount", "#-initpeercount", "-init-peer-count"}, 0, "number of peers in network (for IP address allocation)")
-	mflag.StringVar(&apiPath, []string{"#api", "-api"}, "unix:///var/run/docker.sock", "Path to Docker API socket")
+	mflag.StringVar(&dockerAPI, []string{"#api", "#-api", "-docker-api"}, "", "Docker API endpoint, e.g. unix:///var/run/docker.sock")
 	mflag.BoolVar(&noDNS, []string{"-no-dns"}, false, "disable DNS server")
 	mflag.StringVar(&dnsDomain, []string{"-dns-domain"}, nameserver.DefaultDomain, "local domain to server requests for")
 	mflag.StringVar(&dnsListenAddress, []string{"-dns-listen-address"}, nameserver.DefaultListenAddress, "address to listen on for DNS requests")
 	mflag.IntVar(&dnsTTL, []string{"-dns-ttl"}, nameserver.DefaultTTL, "TTL for DNS request from our domain")
 	mflag.DurationVar(&dnsClientTimeout, []string{"-dns-fallback-timeout"}, nameserver.DefaultClientTimeout, "timeout for fallback DNS requests")
 	mflag.StringVar(&dnsEffectiveListenAddress, []string{"-dns-effective-listen-address"}, "", "address DNS will actually be listening, after Docker port mapping")
+	mflag.StringVar(&datapathName, []string{"-datapath"}, "", "ODP datapath name")
 
 	// crude way of detecting that we probably have been started in a
 	// container, with `weave launch` --> suppress misleading paths in
@@ -95,59 +108,118 @@ func main() {
 		os.Args[0] = "weave"
 		mflag.CommandLine.Init("weave", mflag.ExitOnError)
 	}
+
 	mflag.Parse()
 
 	peers = mflag.Args()
 
 	SetLogLevel(logLevel)
-	if justVersion {
+
+	switch {
+	case justVersion:
 		fmt.Printf("weave router %s\n", version)
 		os.Exit(0)
+
+	case createDatapath:
+		err, odp_supported := odp.CreateDatapath(datapathName)
+		if !odp_supported {
+			if err != nil {
+				Log.Error(err)
+			}
+
+			// When the kernel lacks ODP support, exit
+			// with a special status to distinguish it for
+			// the weave script.
+			os.Exit(17)
+		}
+
+		checkFatal(err)
+		os.Exit(0)
+
+	case deleteDatapath:
+		checkFatal(odp.DeleteDatapath(datapathName))
+		os.Exit(0)
+
+	case addDatapathInterface != "":
+		checkFatal(odp.AddDatapathInterface(datapathName, addDatapathInterface))
+		os.Exit(0)
 	}
 
 	Log.Println("Command line options:", options())
 	Log.Println("Command line peers:", peers)
 
-	if protocolMinVersion < weave.ProtocolMinVersion || protocolMinVersion > weave.ProtocolMaxVersion {
-		Log.Fatalf("--min-protocol-version must be in range [%d,%d]", weave.ProtocolMinVersion, weave.ProtocolMaxVersion)
+	if protocolMinVersion < mesh.ProtocolMinVersion || protocolMinVersion > mesh.ProtocolMaxVersion {
+		Log.Fatalf("--min-protocol-version must be in range [%d,%d]", mesh.ProtocolMinVersion, mesh.ProtocolMaxVersion)
 	}
 	config.ProtocolMinVersion = byte(protocolMinVersion)
 
-	var err error
+	var fastDPOverlay weave.NetworkOverlay
+	if datapathName != "" {
+		// A datapath name implies that "Bridge" and "Overlay"
+		// packet handling use fast datapath, although other
+		// options can override that below.  Even if both
+		// things are overridden, we might need bridging on
+		// the datapath.
+		fastdp, err := weave.NewFastDatapath(weave.FastDatapathConfig{
+			DatapathName: datapathName,
+			Port:         config.Port,
+		})
+
+		checkFatal(err)
+		networkConfig.Bridge = fastdp.Bridge()
+		fastDPOverlay = fastdp.Overlay()
+	}
 
 	if ifaceName != "" {
-		config.Iface, err = weavenet.EnsureInterface(ifaceName)
-		if err != nil {
-			Log.Fatal(err)
-		}
-	}
+		// -iface can coexist with -datapath, because
+		// pcap-based packet capture is a bit more efficient
+		// than capture via ODP misses, even when using an
+		// ODP-based bridge.  So when using weave encyption,
+		// it's preferable to use -iface.
+		var err error
+		iface, err = weavenet.EnsureInterface(ifaceName)
+		checkFatal(err)
 
-	if routerName == "" {
-		if config.Iface == nil {
-			Log.Fatal("Either an interface must be specified with --iface or a name with -name")
-		}
-		routerName = config.Iface.HardwareAddr.String()
-	}
-	name, err := weave.PeerNameFromUserInput(routerName)
-	if err != nil {
-		Log.Fatal(err)
-	}
-
-	if nickName == "" {
-		nickName, err = os.Hostname()
-		if err != nil {
-			Log.Fatal(err)
-		}
+		// bufsz flag is in MB
+		networkConfig.Bridge, err = weave.NewPcap(iface, bufSzMB*1024*1024)
+		checkFatal(err)
 	}
 
 	if password == "" {
 		password = os.Getenv("WEAVE_PASSWORD")
 	}
+
 	if password == "" {
 		Log.Println("Communication between peers is unencrypted.")
 	} else {
 		config.Password = []byte(password)
 		Log.Println("Communication between peers is encrypted.")
+
+		// fastdp doesn't support encryption
+		fastDPOverlay = nil
+	}
+
+	overlays := weave.NewOverlaySwitch()
+	if fastDPOverlay != nil {
+		overlays.Add("fastdp", fastDPOverlay)
+	}
+	sleeve := weave.NewSleeveOverlay(config.Port)
+	overlays.Add("sleeve", sleeve)
+	overlays.SetCompatOverlay(sleeve)
+
+	if routerName == "" {
+		if iface == nil {
+			Log.Fatal("Either an interface must be specified with --iface or a name with -name")
+		}
+		routerName = iface.HardwareAddr.String()
+	}
+
+	name, err := mesh.PeerNameFromUserInput(routerName)
+	checkFatal(err)
+
+	if nickName == "" {
+		nickName, err = os.Hostname()
+		checkFatal(err)
 	}
 
 	if prof != "" {
@@ -157,25 +229,40 @@ func main() {
 		defer profile.Start(&p).Stop()
 	}
 
-	config.BufSz = bufSzMB * 1024 * 1024
-	config.LogFrame = logFrameFunc(pktdebug)
 	config.PeerDiscovery = !noDiscovery
 
-	router := weave.NewRouter(config, name, nickName)
-	Log.Println("Our name is", router.Ourself)
-
-	dockerCli, err := docker.NewClient(apiPath)
-	if err != nil {
-		Log.Fatal("Unable to start docker client: ", err)
+	if pktdebug {
+		networkConfig.PacketLogging = packetLogging{}
+	} else {
+		networkConfig.PacketLogging = nopPacketLogging{}
 	}
 
+	router := weave.NewNetworkRouter(config, networkConfig, name, nickName, overlays)
+	Log.Println("Our name is", router.Ourself)
+
+	var dockerCli *docker.Client
+	if dockerAPI != "" {
+		dc, err := docker.NewClient(dockerAPI)
+		if err != nil {
+			Log.Fatal("Unable to start docker client: ", err)
+		}
+		dockerCli = dc
+	}
+	observeContainers := func(o docker.ContainerObserver) {
+		if dockerCli != nil {
+			if err = dockerCli.AddObserver(o); err != nil {
+				Log.Fatal("Unable to start watcher", err)
+			}
+		}
+	}
+	isKnownPeer := func(name mesh.PeerName) bool {
+		return router.Peers.Fetch(name) != nil
+	}
 	var allocator *ipam.Allocator
 	var defaultSubnet address.CIDR
 	if iprangeCIDR != "" {
-		allocator, defaultSubnet = createAllocator(router, iprangeCIDR, ipsubnetCIDR, determineQuorum(peerCount, peers))
-		if err = dockerCli.AddObserver(allocator); err != nil {
-			Log.Fatal("Unable to start watcher", err)
-		}
+		allocator, defaultSubnet = createAllocator(router.Router, iprangeCIDR, ipsubnetCIDR, determineQuorum(peerCount, peers), isKnownPeer)
+		observeContainers(allocator)
 	} else if peerCount > 0 {
 		Log.Fatal("--init-peer-count flag specified without --ipalloc-range")
 	}
@@ -185,11 +272,10 @@ func main() {
 		dnsserver *nameserver.DNSServer
 	)
 	if !noDNS {
-		ns = nameserver.New(router.Ourself.Peer.Name, router.Peers, dockerCli, dnsDomain)
+		ns = nameserver.New(router.Ourself.Peer.Name, dnsDomain, isKnownPeer)
+		router.Peers.OnGC(func(peer *mesh.Peer) { ns.PeerGone(peer.Name) })
 		ns.SetGossip(router.NewGossip("nameserver", ns))
-		if err = dockerCli.AddObserver(ns); err != nil {
-			Log.Fatal("Unable to start watcher", err)
-		}
+		observeContainers(ns)
 		ns.Start()
 		defer ns.Stop()
 		dnsserver, err = nameserver.NewDNSServer(ns, dnsDomain, dnsListenAddress,
@@ -197,6 +283,11 @@ func main() {
 		if err != nil {
 			Log.Fatal("Unable to start dns server: ", err)
 		}
+		listenAddr := dnsListenAddress
+		if dnsEffectiveListenAddress != "" {
+			listenAddr = dnsEffectiveListenAddress
+		}
+		Log.Println("Listening for DNS queries on", listenAddr)
 		dnsserver.ActivateAndServe()
 		defer dnsserver.Stop()
 	}
@@ -215,11 +306,12 @@ func main() {
 			allocator.HandleHTTP(muxRouter, defaultSubnet, dockerCli)
 		}
 		if ns != nil {
-			ns.HandleHTTP(muxRouter)
+			ns.HandleHTTP(muxRouter, dockerCli)
 		}
 		router.HandleHTTP(muxRouter)
 		HandleHTTP(muxRouter, version, router, allocator, defaultSubnet, ns, dnsserver)
 		http.Handle("/", muxRouter)
+		Log.Println("Listening for HTTP control messages on", httpAddr)
 		go listenAndServeHTTP(httpAddr, muxRouter)
 	}
 
@@ -248,38 +340,35 @@ func canonicalName(f *mflag.Flag) string {
 	return ""
 }
 
-func logFrameFunc(debug bool) weave.LogFrameFunc {
-	if !debug {
-		return func(prefix string, frame []byte, dec *weave.EthernetDecoder) {}
-	}
-	return func(prefix string, frame []byte, dec *weave.EthernetDecoder) {
-		h := fmt.Sprintf("%x", sha256.Sum256(frame))
-		parts := []interface{}{prefix, len(frame), "bytes (", h, ")"}
+type packetLogging struct{}
 
-		if dec != nil {
-			parts = append(parts, dec.Eth.SrcMAC, "->", dec.Eth.DstMAC)
+func (packetLogging) LogPacket(msg string, key weave.PacketKey) {
+	Log.Println(msg, key.SrcMAC, "->", key.DstMAC)
+}
 
-			if dec.DF() {
-				parts = append(parts, "(DF)")
-			}
-		}
+func (packetLogging) LogForwardPacket(msg string, key weave.ForwardPacketKey) {
+	Log.Println(msg, key.SrcPeer, key.SrcMAC, "->", key.DstPeer, key.DstMAC)
+}
 
-		Log.Println(parts...)
-	}
+type nopPacketLogging struct{}
+
+func (nopPacketLogging) LogPacket(string, weave.PacketKey) {
+}
+
+func (nopPacketLogging) LogForwardPacket(string, weave.ForwardPacketKey) {
 }
 
 func parseAndCheckCIDR(cidrStr string) address.CIDR {
 	_, cidr, err := address.ParseCIDR(cidrStr)
-	if err != nil {
-		Log.Fatal(err)
-	}
+	checkFatal(err)
+
 	if cidr.Size() < ipam.MinSubnetSize {
 		Log.Fatalf("Allocation range smaller than minimum size %d: %s", ipam.MinSubnetSize, cidrStr)
 	}
 	return cidr
 }
 
-func createAllocator(router *weave.Router, ipRangeStr string, defaultSubnetStr string, quorum uint) (*ipam.Allocator, address.CIDR) {
+func createAllocator(router *mesh.Router, ipRangeStr string, defaultSubnetStr string, quorum uint, isKnownPeer func(mesh.PeerName) bool) (*ipam.Allocator, address.CIDR) {
 	ipRange := parseAndCheckCIDR(ipRangeStr)
 	defaultSubnet := ipRange
 	if defaultSubnetStr != "" {
@@ -288,7 +377,7 @@ func createAllocator(router *weave.Router, ipRangeStr string, defaultSubnetStr s
 			Log.Fatalf("IP address allocation default subnet %s does not overlap with allocation range %s", defaultSubnet, ipRange)
 		}
 	}
-	allocator := ipam.NewAllocator(router.Ourself.Peer.Name, router.Ourself.Peer.UID, router.Ourself.Peer.NickName, ipRange.Range(), quorum)
+	allocator := ipam.NewAllocator(router.Ourself.Peer.Name, router.Ourself.Peer.UID, router.Ourself.Peer.NickName, ipRange.Range(), quorum, isKnownPeer)
 
 	allocator.SetInterfaces(router.NewGossip("IPallocation", allocator))
 	allocator.Start()
@@ -331,3 +420,9 @@ func listenAndServeHTTP(httpAddr string, muxRouter *mux.Router) {
 		Log.Fatal("Unable to create http server", err)
 	}
 }
+
+func checkFatal(e error) {
+	if e != nil {
+		Log.Fatal(e)
+	}
+}

prog/weavewait/check_network.go

@@ -0,0 +1,12 @@
+// +build !noop
+
+package main
+
+import (
+	weavenet "github.com/weaveworks/weave/net"
+)
+
+func checkNetwork() error {
+	_, err := weavenet.EnsureInterfaceAndMcastRoute("ethwe")
+	return err
+}

prog/weavewait/main.go

@@ -6,8 +6,6 @@ import (
 	"os"
 	"os/exec"
 	"syscall"
-
-	weavenet "github.com/weaveworks/weave/net"
 )
 
 var (
@@ -19,8 +17,7 @@ func main() {
 		args = os.Args[1:]
 	)
 
-	_, err := weavenet.EnsureInterfaceAndMcastRoute("ethwe")
-	checkErr(err)
+	checkErr(checkNetwork())
 
 	if len(args) == 0 {
 		checkErr(ErrNoCommandSpecified)

prog/weavewait/noop.go

@@ -0,0 +1,7 @@
+// +build noop
+
+package main
+
+func checkNetwork() error {
+	return nil
+}

proxy/common.go

@@ -20,19 +20,34 @@ import (
 var (
 	containerIDRegexp   = regexp.MustCompile("^(/v[0-9\\.]*)?/containers/([^/]*)/.*")
 	weaveWaitEntrypoint = []string{"/w/w"}
+	weaveEntrypoint     = "/home/weave/weaver"
+	weaveContainerName  = "/weave"
 )
 
 func callWeave(args ...string) ([]byte, []byte, error) {
 	args = append([]string{"--local"}, args...)
-	Log.Debug("Calling weave", args)
 	cmd := exec.Command("./weave", args...)
 	cmd.Env = []string{
 		"PATH=/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
 		"PROCFS=/hostproc",
 	}
-	if bridge := os.Getenv("DOCKER_BRIDGE"); bridge != "" {
-		cmd.Env = append(cmd.Env, fmt.Sprintf("DOCKER_BRIDGE=%s", bridge))
+
+	propagateEnv := func(key string) {
+		if val := os.Getenv(key); val != "" {
+			cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", key, val))
+		}
 	}
+
+	propagateEnv("DOCKER_BRIDGE")
+
+	// Propogage WEAVE_DEBUG, to make debugging easier.
+	propagateEnv("WEAVE_DEBUG")
+
+	// This prevents the code coverage contortions in our
+	// integration test suite breaking things.
+	propagateEnv("COVERAGE")
+
+	Log.Debug("Calling weave args: ", args, "env: ", cmd.Env)
 	var stdout, stderr bytes.Buffer
 	cmd.Stdout = &stdout
 	cmd.Stderr = &stderr
@@ -45,6 +60,7 @@ func unmarshalRequestBody(r *http.Request, target interface{}) error {
 	if err != nil {
 		return err
 	}
+	Log.Debugf("->requestBody: %s", body)
 	if err := r.Body.Close(); err != nil {
 		return err
 	}
@@ -60,6 +76,7 @@ func marshalRequestBody(r *http.Request, body interface{}) error {
 	if err != nil {
 		return err
 	}
+	Log.Debugf("<-requestBody: %s", newBody)
 	r.Body = ioutil.NopCloser(bytes.NewReader(newBody))
 	r.ContentLength = int64(len(newBody))
 	return nil
@@ -70,6 +87,7 @@ func unmarshalResponseBody(r *http.Response, target interface{}) error {
 	if err != nil {
 		return err
 	}
+	Log.Debugf("->responseBody: %s", body)
 	if err := r.Body.Close(); err != nil {
 		return err
 	}
@@ -85,12 +103,12 @@ func marshalResponseBody(r *http.Response, body interface{}) error {
 	if err != nil {
 		return err
 	}
+	Log.Debugf("<-responseBody: %s", newBody)
 	r.Body = ioutil.NopCloser(bytes.NewReader(newBody))
 	r.ContentLength = int64(len(newBody))
 	// Stop it being chunked, because that hangs
 	r.TransferEncoding = nil
 	return nil
-
 }
 
 func inspectContainerInPath(client *docker.Client, path string) (*docker.Container, error) {
@@ -137,3 +155,25 @@ func weaveContainerIPs(containerID string) (mac string, ips []net.IP, nets []*ne
 	}
 	return
 }
+
+func addVolume(hostConfig jsonObject, source, target, mode string) error {
+	configBinds, err := hostConfig.StringArray("Binds")
+	if err != nil {
+		return err
+	}
+
+	var binds []string
+	for _, bind := range configBinds {
+		s := strings.Split(bind, ":")
+		if len(s) >= 2 && s[1] == target {
+			continue
+		}
+		binds = append(binds, bind)
+	}
+	bind := source + ":" + target
+	if mode != "" {
+		bind += ":" + mode
+	}
+	hostConfig["Binds"] = append(binds, bind)
+	return nil
+}

proxy/create_container_interceptor.go

@@ -2,16 +2,11 @@ package proxy
 
 import (
 	"errors"
-	"fmt"
-	"io/ioutil"
 	"net/http"
 	"strings"
 
 	"github.com/fsouza/go-dockerclient"
-
 	. "github.com/weaveworks/weave/common"
-	"github.com/weaveworks/weave/nameserver"
-	"github.com/weaveworks/weave/router"
 )
 
 const MaxDockerHostname = 64
@@ -22,12 +17,6 @@ var (
 
 type createContainerInterceptor struct{ proxy *Proxy }
 
-type createContainerRequestBody struct {
-	*docker.Config
-	HostConfig *docker.HostConfig `json:"HostConfig,omitempty" yaml:"HostConfig,omitempty"`
-	MacAddress string             `json:"MacAddress,omitempty" yaml:"MacAddress,omitempty"`
-}
-
 // ErrNoSuchImage replaces docker.NoSuchImage, which does not contain the image
 // name, which in turn breaks docker clients post 1.7.0 since they expect the
 // image name to be present in errors.
@@ -40,34 +29,47 @@ func (err *ErrNoSuchImage) Error() string {
 }
 
 func (i *createContainerInterceptor) InterceptRequest(r *http.Request) error {
-	container := createContainerRequestBody{}
+	container := jsonObject{}
 	if err := unmarshalRequestBody(r, &container); err != nil {
 		return err
 	}
 
-	if cidrs, err := i.proxy.weaveCIDRsFromConfig(container.Config, container.HostConfig); err != nil {
+	hostConfig, err := container.Object("HostConfig")
+	if err != nil {
+		return err
+	}
+
+	networkMode, err := hostConfig.String("NetworkMode")
+	if err != nil {
+		return err
+	}
+
+	env, err := container.StringArray("Env")
+	if err != nil {
+		return err
+	}
+
+	if cidrs, err := i.proxy.weaveCIDRs(networkMode, env); err != nil {
 		Log.Infof("Leaving container alone because %s", err)
 	} else {
 		Log.Infof("Creating container with WEAVE_CIDR \"%s\"", strings.Join(cidrs, " "))
-		if container.HostConfig == nil {
-			container.HostConfig = &docker.HostConfig{}
-		}
-		if container.Config == nil {
-			container.Config = &docker.Config{}
-		}
-		i.addWeaveWaitVolume(container.HostConfig)
-		if err := i.setWeaveWaitEntrypoint(container.Config); err != nil {
+		if err := addVolume(hostConfig, i.proxy.weaveWaitVolume, "/w", "ro"); err != nil {
 			return err
 		}
-		hostname := r.URL.Query().Get("name")
-		if i.proxy.Config.HostnameFromLabel != "" {
-			if labelValue, ok := container.Labels[i.proxy.Config.HostnameFromLabel]; ok {
-				hostname = labelValue
+		if err := i.setWeaveWaitEntrypoint(container); err != nil {
+			return err
+		}
+		hostname, err := i.containerHostname(r, container)
+		if err != nil {
+			return err
+		}
+		if dnsDomain := i.proxy.getDNSDomain(); dnsDomain != "" {
+			if err := i.setHostname(container, hostname, dnsDomain); err != nil {
+				return err
+			}
+			if err := i.proxy.setWeaveDNS(hostConfig, hostname, dnsDomain); err != nil {
+				return err
 			}
-		}
-		hostname = i.proxy.hostnameMatchRegexp.ReplaceAllString(hostname, i.proxy.HostnameReplacement)
-		if err := i.setWeaveDNS(&container, hostname); err != nil {
-			return err
 		}
 
 		return marshalRequestBody(r, container)
@@ -76,105 +78,97 @@ func (i *createContainerInterceptor) InterceptRequest(r *http.Request) error {
 	return nil
 }
 
-func (i *createContainerInterceptor) addWeaveWaitVolume(hostConfig *docker.HostConfig) {
-	var binds []string
-	for _, bind := range hostConfig.Binds {
-		s := strings.Split(bind, ":")
-		if len(s) >= 2 && s[1] == "/w" {
-			continue
-		}
-		binds = append(binds, bind)
+func (i *createContainerInterceptor) setWeaveWaitEntrypoint(container jsonObject) error {
+	var entrypoint []string
+	entrypoint, err := container.StringArray("Entrypoint")
+	if err != nil {
+		return err
 	}
-	hostConfig.Binds = append(binds, fmt.Sprintf("%s:/w:ro", i.proxy.weaveWaitVolume))
-}
 
-func (i *createContainerInterceptor) setWeaveWaitEntrypoint(container *docker.Config) error {
-	if len(container.Entrypoint) == 0 {
-		image, err := i.proxy.client.InspectImage(container.Image)
+	cmd, err := container.StringArray("Cmd")
+	if err != nil {
+		return err
+	}
+
+	if len(entrypoint) == 0 {
+		containerImage, err := container.String("Image")
+		if err != nil {
+			return err
+		}
+
+		image, err := i.proxy.client.InspectImage(containerImage)
 		if err == docker.ErrNoSuchImage {
-			return &ErrNoSuchImage{container.Image}
+			return &ErrNoSuchImage{containerImage}
 		} else if err != nil {
 			return err
 		}
 
-		if len(container.Cmd) == 0 {
-			container.Cmd = image.Config.Cmd
+		if len(cmd) == 0 {
+			cmd = image.Config.Cmd
+			container["Cmd"] = cmd
 		}
 
-		if container.Entrypoint == nil {
-			container.Entrypoint = image.Config.Entrypoint
+		if entrypoint == nil {
+			entrypoint = image.Config.Entrypoint
+			container["Entrypoint"] = entrypoint
 		}
 	}
 
-	if len(container.Entrypoint) == 0 && len(container.Cmd) == 0 {
+	if len(entrypoint) == 0 && len(cmd) == 0 {
 		return ErrNoCommandSpecified
 	}
 
-	if len(container.Entrypoint) == 0 || container.Entrypoint[0] != weaveWaitEntrypoint[0] {
-		container.Entrypoint = append(weaveWaitEntrypoint, container.Entrypoint...)
+	if len(entrypoint) == 0 || entrypoint[0] != weaveWaitEntrypoint[0] {
+		container["Entrypoint"] = append(weaveWaitEntrypoint, entrypoint...)
 	}
 
 	return nil
 }
 
-func (i *createContainerInterceptor) setWeaveDNS(container *createContainerRequestBody, name string) error {
-	if i.proxy.WithoutDNS {
-		return nil
+func (i *createContainerInterceptor) setHostname(container jsonObject, name, dnsDomain string) error {
+	hostname, err := container.String("Hostname")
+	if err != nil {
+		return err
 	}
-
-	dnsDomain, dnsRunning := i.getDNSDomain()
-	if !(dnsRunning || i.proxy.WithDNS) {
-		return nil
-	}
-
-	container.HostConfig.DNS = append(container.HostConfig.DNS, i.proxy.dockerBridgeIP)
-
-	if container.Hostname == "" && name != "" {
+	if hostname == "" && name != "" {
 		// Strip trailing period because it's unusual to see it used on the end of a host name
 		trimmedDNSDomain := strings.TrimSuffix(dnsDomain, ".")
 		if len(name)+1+len(trimmedDNSDomain) > MaxDockerHostname {
 			Log.Warningf("Container name [%s] too long to be used as hostname", name)
 		} else {
-			container.Hostname = name
-			container.Domainname = trimmedDNSDomain
-		}
-	}
-
-	if len(container.HostConfig.DNSSearch) == 0 {
-		if container.Hostname == "" {
-			container.HostConfig.DNSSearch = []string{dnsDomain}
-		} else {
-			container.HostConfig.DNSSearch = []string{"."}
+			container["Hostname"] = name
+			container["Domainname"] = trimmedDNSDomain
 		}
 	}
 
 	return nil
 }
 
-func (i *createContainerInterceptor) getDNSDomain() (domain string, running bool) {
-	domain = nameserver.DefaultDomain
-	weaveContainer, err := i.proxy.client.InspectContainer("weave")
-	if err != nil ||
-		weaveContainer.NetworkSettings == nil ||
-		weaveContainer.NetworkSettings.IPAddress == "" {
-		return
-	}
-
-	url := fmt.Sprintf("http://%s:%d/domain", weaveContainer.NetworkSettings.IPAddress, router.HTTPPort)
-	resp, err := http.Get(url)
-	if err != nil || resp.StatusCode != http.StatusOK {
-		return
-	}
-	defer resp.Body.Close()
-
-	b, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return
-	}
-
-	return string(b), true
-}
-
 func (i *createContainerInterceptor) InterceptResponse(r *http.Response) error {
 	return nil
 }
+
+func (i *createContainerInterceptor) containerHostname(r *http.Request, container jsonObject) (hostname string, err error) {
+	hostname = r.URL.Query().Get("name")
+	if i.proxy.Config.HostnameFromLabel != "" {
+		hostname, err = i.hostnameFromLabel(hostname, container)
+	}
+	hostname = i.proxy.hostnameMatchRegexp.ReplaceAllString(hostname, i.proxy.HostnameReplacement)
+	return
+}
+
+func (i *createContainerInterceptor) hostnameFromLabel(hostname string, container jsonObject) (string, error) {
+	labels, err := container.Object("Labels")
+	if err != nil {
+		return "", err
+	}
+	label, err := labels.String(i.proxy.Config.HostnameFromLabel)
+	if err != nil {
+		return "", err
+	}
+	if label == "" {
+		return hostname, nil
+	}
+
+	return label, nil
+}

proxy/create_exec_interceptor.go

@@ -4,14 +4,13 @@ import (
 	"net/http"
 	"strings"
 
-	"github.com/fsouza/go-dockerclient"
 	. "github.com/weaveworks/weave/common"
 )
 
 type createExecInterceptor struct{ proxy *Proxy }
 
 func (i *createExecInterceptor) InterceptRequest(r *http.Request) error {
-	options := docker.CreateExecOptions{}
+	options := jsonObject{}
 	if err := unmarshalRequestBody(r, &options); err != nil {
 		return err
 	}
@@ -25,14 +24,19 @@ func (i *createExecInterceptor) InterceptRequest(r *http.Request) error {
 		return nil
 	}
 
-	cidrs, err := i.proxy.weaveCIDRsFromConfig(container.Config, container.HostConfig)
+	cidrs, err := i.proxy.weaveCIDRs(container.HostConfig.NetworkMode, container.Config.Env)
 	if err != nil {
 		Log.Infof("Leaving container %s alone because %s", container.ID, err)
 		return nil
 	}
 
+	cmd, err := options.StringArray("Cmd")
+	if err != nil {
+		return err
+	}
+
 	Log.Infof("Exec in container %s with WEAVE_CIDR \"%s\"", container.ID, strings.Join(cidrs, " "))
-	options.Cmd = append(weaveWaitEntrypoint, options.Cmd...)
+	options["Cmd"] = append(weaveWaitEntrypoint, cmd...)
 
 	return marshalRequestBody(r, options)
 }

proxy/inspect_container_interceptor.go

@@ -17,35 +17,14 @@ func (i *inspectContainerInterceptor) InterceptResponse(r *http.Response) error
 		return nil
 	}
 
-	container := map[string]interface{}{}
+	container := jsonObject{}
 	if err := unmarshalResponseBody(r, &container); err != nil {
 		return err
 	}
 
-	if err := updateContainerNetworkSettings(container); err != nil {
+	if err := i.proxy.updateContainerNetworkSettings(container); err != nil {
 		Log.Warningf("Inspecting container %s failed: %s", container["Id"], err)
 	}
 
 	return marshalResponseBody(r, container)
 }
-
-func updateContainerNetworkSettings(container map[string]interface{}) error {
-	containerID, err := lookupString(container, "Id")
-	if err != nil {
-		return err
-	}
-
-	mac, ips, nets, err := weaveContainerIPs(containerID)
-	if err != nil || len(ips) == 0 {
-		return err
-	}
-
-	networkSettings, err := lookupObject(container, "NetworkSettings")
-	if err != nil {
-		return err
-	}
-	networkSettings["MacAddress"] = mac
-	networkSettings["IPAddress"] = ips[0].String()
-	networkSettings["IPPrefixLen"], _ = nets[0].Mask.Size()
-	return nil
-}

proxy/inspect_exec_interceptor.go

@@ -17,17 +17,17 @@ func (i *inspectExecInterceptor) InterceptResponse(r *http.Response) error {
 		return nil
 	}
 
-	exec := map[string]interface{}{}
+	exec := jsonObject{}
 	if err := unmarshalResponseBody(r, &exec); err != nil {
 		return err
 	}
 
-	container, err := lookupObject(exec, "Container")
+	container, err := exec.Object("Container")
 	if err != nil {
 		return err
 	}
 
-	if err := updateContainerNetworkSettings(container); err != nil {
+	if err := i.proxy.updateContainerNetworkSettings(container); err != nil {
 		Log.Warningf("Inspecting exec %s failed: %s", exec["Id"], err)
 	}
 

proxy/json.go

@@ -13,12 +13,13 @@ func (e *UnmarshalWrongTypeError) Error() string {
 	return fmt.Sprintf("Wrong type for %s field, expected %s, but got %T", e.Field, e.Expected, e.Got)
 }
 
-// For parsing json objects. Look up (or create) a child object in the given map.
-func lookupObject(m map[string]interface{}, key string) (map[string]interface{}, error) {
-	iface, ok := m[key]
+type jsonObject map[string]interface{}
+
+func (j jsonObject) Object(key string) (jsonObject, error) {
+	iface, ok := j[key]
 	if !ok || iface == nil {
-		result := map[string]interface{}{}
-		m[key] = result
+		result := jsonObject{}
+		j[key] = result
 		return result, nil
 	}
 
@@ -27,12 +28,11 @@ func lookupObject(m map[string]interface{}, key string) (map[string]interface{},
 		return nil, &UnmarshalWrongTypeError{key, "object", iface}
 	}
 
-	return result, nil
+	return jsonObject(result), nil
 }
 
-// For parsing json objects. Look up (or create) a string in the given map.
-func lookupString(m map[string]interface{}, key string) (string, error) {
-	iface, ok := m[key]
+func (j jsonObject) String(key string) (string, error) {
+	iface, ok := j[key]
 	if !ok || iface == nil {
 		return "", nil
 	}
@@ -44,3 +44,29 @@ func lookupString(m map[string]interface{}, key string) (string, error) {
 
 	return result, nil
 }
+
+func (j jsonObject) StringArray(key string) ([]string, error) {
+	iface, ok := j[key]
+	if !ok || iface == nil {
+		return nil, nil
+	}
+
+	switch o := iface.(type) {
+	case string:
+		return []string{o}, nil
+	case []string:
+		return o, nil
+	case []interface{}:
+		result := []string{}
+		for _, s := range o {
+			if s, ok := s.(string); ok {
+				result = append(result, s)
+			} else {
+				return nil, &UnmarshalWrongTypeError{key, "string or array of strings", iface}
+			}
+		}
+		return result, nil
+	}
+
+	return nil, &UnmarshalWrongTypeError{key, "string or array of strings", iface}
+}

proxy/json_test.go

@@ -0,0 +1,116 @@
+package proxy
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestLookupObject(t *testing.T) {
+	tests := []struct {
+		root   jsonObject
+		key    string
+		result jsonObject
+		err    error
+	}{
+		{
+			jsonObject{},
+			"a",
+			jsonObject{},
+			nil,
+		},
+		{
+			jsonObject{"a": map[string]interface{}{"b": int(1)}},
+			"a",
+			jsonObject{"b": int(1)},
+			nil,
+		},
+		{
+			jsonObject{"nonObject": int(1)},
+			"nonObject",
+			nil,
+			&UnmarshalWrongTypeError{Field: "nonObject", Expected: "object", Got: 1},
+		},
+	}
+	for _, test := range tests {
+		gotResult, gotErr := test.root.Object(test.key)
+		msg := fmt.Sprintf("%q.Object(%q) => %q, %q", test.root, test.key, gotResult, gotErr)
+		assert.Equal(t, test.result, gotResult, msg)
+		assert.Equal(t, test.err, gotErr, msg)
+	}
+}
+
+func TestLookupString(t *testing.T) {
+	tests := []struct {
+		root   jsonObject
+		key    string
+		result string
+		err    error
+	}{
+		{
+			jsonObject{},
+			"a",
+			"",
+			nil,
+		},
+		{
+			jsonObject{"nonString": int(1)},
+			"nonString",
+			"",
+			&UnmarshalWrongTypeError{Field: "nonString", Expected: "string", Got: 1},
+		},
+	}
+	for _, test := range tests {
+		gotResult, gotErr := test.root.String(test.key)
+		msg := fmt.Sprintf("%q.String(%q) => %q, %q", test.root, test.key, gotResult, gotErr)
+		assert.Equal(t, test.result, gotResult, msg)
+		assert.Equal(t, test.err, gotErr, msg)
+	}
+}
+
+func TestLookupStringArray(t *testing.T) {
+	tests := []struct {
+		root   jsonObject
+		key    string
+		result []string
+		err    error
+	}{
+		{
+			jsonObject{},
+			"a",
+			nil,
+			nil,
+		},
+		{
+			jsonObject{"a": []string{"foo"}},
+			"a",
+			[]string{"foo"},
+			nil,
+		},
+		{
+			jsonObject{"a": []string{}},
+			"a",
+			[]string{},
+			nil,
+		},
+		{
+			jsonObject{"a": "foo"},
+			"a",
+			[]string{"foo"},
+			nil,
+		},
+		{
+			jsonObject{"int": 5},
+			"int",
+			nil,
+			&UnmarshalWrongTypeError{Field: "int", Expected: "string or array of strings", Got: 5},
+		},
+	}
+	for _, test := range tests {
+		gotResult, gotErr := test.root.StringArray(test.key)
+		msg := fmt.Sprintf("%q.String(%q) => %q, %q", test.root, test.key, gotResult, gotErr)
+		assert.Equal(t, test.result, gotResult, msg)
+		assert.Equal(t, test.err, gotErr, msg)
+	}
+}

proxy/proxy.go

@@ -4,15 +4,19 @@ import (
 	"crypto/tls"
 	"errors"
 	"fmt"
+	"io/ioutil"
 	"net"
 	"net/http"
 	"os"
 	"regexp"
 	"strings"
+	"sync"
 	"syscall"
 
-	"github.com/fsouza/go-dockerclient"
+	docker "github.com/fsouza/go-dockerclient"
 	. "github.com/weaveworks/weave/common"
+	weavedocker "github.com/weaveworks/weave/common/docker"
+	"github.com/weaveworks/weave/nameserver"
 )
 
 const (
@@ -52,31 +56,41 @@ type Config struct {
 	WithoutDNS          bool
 }
 
+type wait struct {
+	ident string
+	ch    chan error
+	done  bool
+}
+
 type Proxy struct {
+	sync.Mutex
 	Config
 	client              *docker.Client
 	dockerBridgeIP      string
 	hostnameMatchRegexp *regexp.Regexp
 	weaveWaitVolume     string
+	weaveWaitNoopVolume string
+	normalisedAddrs     []string
+	waiters             map[*http.Request]*wait
 }
 
 func NewProxy(c Config) (*Proxy, error) {
-	p := &Proxy{Config: c}
+	p := &Proxy{Config: c, waiters: make(map[*http.Request]*wait)}
 
-	if err := p.TLSConfig.loadCerts(); err != nil {
+	if err := p.TLSConfig.LoadCerts(); err != nil {
 		Log.Fatalf("Could not configure tls for proxy: %s", err)
 	}
 
-	// We pin the protocol version to 1.15 (which corresponds to
-	// Docker 1.3.x; the earliest version supported by weave) in order
+	// We pin the protocol version to 1.18 (which corresponds to
+	// Docker 1.6.x; the earliest version supported by weave) in order
 	// to insulate ourselves from breaking changes to the API, as
 	// happened in 1.20 (Docker 1.8.0) when the presentation of
 	// volumes changed in `inspect`.
-	client, err := docker.NewVersionedClient(dockerSockUnix, "1.15")
+	client, err := weavedocker.NewVersionedClient(dockerSockUnix, "1.18")
 	if err != nil {
 		return nil, err
 	}
-	p.client = client
+	p.client = client.Client
 
 	if !p.WithoutDNS {
 		dockerBridgeIP, stderr, err := callWeave("docker-bridge-ip")
@@ -92,34 +106,60 @@ func NewProxy(c Config) (*Proxy, error) {
 		return nil, err
 	}
 
-	if err = p.findWeaveWaitVolume(); err != nil {
+	if err = p.findWeaveWaitVolumes(); err != nil {
 		return nil, err
 	}
 
+	client.AddObserver(p)
+
 	return p, nil
 }
 
+func (proxy *Proxy) AttachExistingContainers() {
+	containers, _ := proxy.client.ListContainers(docker.ListContainersOptions{})
+	for _, c := range containers {
+		container, err := proxy.client.InspectContainer(c.ID)
+		if err != nil {
+			if _, ok := err.(*docker.NoSuchContainer); !ok {
+				Log.Warningf("unable to attach existing container %s since inspecting it failed: %v", c.ID, err)
+			}
+			continue
+		}
+		if containerShouldAttach(container) && (container.State.Running || container.State.Paused) {
+			proxy.attach(container, false)
+		}
+	}
+}
+
 func (proxy *Proxy) Dial() (net.Conn, error) {
 	return net.Dial("unix", dockerSock)
 }
 
-func (proxy *Proxy) findWeaveWaitVolume() error {
+func (proxy *Proxy) findWeaveWaitVolumes() error {
+	var err error
+	if proxy.weaveWaitVolume, err = proxy.findVolume("/w"); err != nil {
+		return err
+	}
+	proxy.weaveWaitNoopVolume, err = proxy.findVolume("/w-noop")
+	return err
+}
+
+func (proxy *Proxy) findVolume(v string) (string, error) {
 	container, err := proxy.client.InspectContainer("weaveproxy")
 	if err != nil {
-		return fmt.Errorf("Could not find the weavewait volume: %s", err)
+		return "", fmt.Errorf("Could not find the weavewait volume: %s", err)
 	}
 
 	if container.Volumes == nil {
-		return fmt.Errorf("Could not find the weavewait volume")
+		return "", fmt.Errorf("Could not find the weavewait volume")
 	}
 
-	volume, ok := container.Volumes["/w"]
+	volume, ok := container.Volumes[v]
 	if !ok {
-		return fmt.Errorf("Could not find the weavewait volume")
+		return "", fmt.Errorf("Could not find the weavewait volume")
 	}
 
-	proxy.weaveWaitVolume = volume
-	return nil
+	return volume, nil
 }
 
 func (proxy *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
@@ -143,22 +183,25 @@ func (proxy *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	proxy.Intercept(i, w, r)
 }
 
-func (proxy *Proxy) ListenAndServe() {
+func (proxy *Proxy) Listen() []net.Listener {
 	listeners := []net.Listener{}
-	addrs := []string{}
+	proxy.normalisedAddrs = []string{}
 	for _, addr := range proxy.ListenAddrs {
 		listener, normalisedAddr, err := proxy.listen(addr)
 		if err != nil {
 			Log.Fatalf("Cannot listen on %s: %s", addr, err)
 		}
 		listeners = append(listeners, listener)
-		addrs = append(addrs, normalisedAddr)
+		proxy.normalisedAddrs = append(proxy.normalisedAddrs, normalisedAddr)
 	}
 
-	for _, addr := range addrs {
+	for _, addr := range proxy.normalisedAddrs {
 		Log.Infoln("proxy listening on", addr)
 	}
+	return listeners
+}
 
+func (proxy *Proxy) Serve(listeners []net.Listener) {
 	errs := make(chan error)
 	for _, listener := range listeners {
 		go func(listener net.Listener) {
@@ -173,6 +216,26 @@ func (proxy *Proxy) ListenAndServe() {
 	}
 }
 
+func (proxy *Proxy) ListenAndServeStatus(socket string) {
+	if err := os.Remove(socket); err != nil && !os.IsNotExist(err) {
+		Log.Fatalf("Error removing existing status socket: %s", err)
+	}
+	listener, err := net.Listen("unix", socket)
+	if err != nil {
+		Log.Fatalf("ListenAndServeStatus failed: %s", err)
+	}
+	handler := http.HandlerFunc(proxy.StatusHTTP)
+	if err := (&http.Server{Handler: handler}).Serve(listener); err != nil {
+		Log.Fatalf("ListenAndServeStatus failed: %s", err)
+	}
+}
+
+func (proxy *Proxy) StatusHTTP(w http.ResponseWriter, r *http.Request) {
+	for _, addr := range proxy.normalisedAddrs {
+		fmt.Fprintln(w, addr)
+	}
+}
+
 func copyOwnerAndPermissions(from, to string) error {
 	stat, err := os.Stat(from)
 	if err != nil {
@@ -215,7 +278,7 @@ func (proxy *Proxy) listen(protoAndAddr string) (net.Listener, string, error) {
 		if err != nil {
 			return nil, "", err
 		}
-		if proxy.TLSConfig.enabled() {
+		if proxy.TLSConfig.IsEnabled() {
 			listener = tls.NewListener(listener, proxy.TLSConfig.Config)
 		}
 
@@ -239,15 +302,140 @@ func (proxy *Proxy) listen(protoAndAddr string) (net.Listener, string, error) {
 	return listener, fmt.Sprintf("%s://%s", proto, addr), nil
 }
 
-func (proxy *Proxy) weaveCIDRsFromConfig(config *docker.Config, hostConfig *docker.HostConfig) ([]string, error) {
-	netMode := ""
-	if hostConfig != nil {
-		netMode = hostConfig.NetworkMode
+// weavedocker.ContainerObserver interface
+func (proxy *Proxy) ContainerStarted(ident string) {
+	container, err := proxy.client.InspectContainer(ident)
+	if err == nil {
+		if containerShouldAttach(container) {
+			err = proxy.attach(container, true)
+		} else if containerIsWeaveRouter(container) {
+			err = proxy.attachRouter(container)
+		}
 	}
-	if netMode == "host" || strings.HasPrefix(netMode, "container:") {
-		return nil, fmt.Errorf("the container was created with the '--net=%s'", netMode)
+	if _, ok := err.(*docker.NoSuchContainer); err != nil && !ok {
+		Log.Warningf("unable to attach new container %s since inspecting it failed: %v", ident, err)
 	}
-	for _, e := range config.Env {
+	proxy.notifyWaiters(ident, err)
+}
+
+func containerShouldAttach(container *docker.Container) bool {
+	return len(container.Config.Entrypoint) > 0 && container.Config.Entrypoint[0] == weaveWaitEntrypoint[0]
+}
+
+func containerIsWeaveRouter(container *docker.Container) bool {
+	return container.Name == weaveContainerName &&
+		len(container.Config.Entrypoint) > 0 && container.Config.Entrypoint[0] == weaveEntrypoint
+}
+
+func (proxy *Proxy) createWait(r *http.Request, ident string) {
+	proxy.Lock()
+	proxy.waiters[r] = &wait{ident: ident, ch: make(chan error, 1)}
+	proxy.Unlock()
+}
+
+func (proxy *Proxy) removeWait(r *http.Request) {
+	proxy.Lock()
+	delete(proxy.waiters, r)
+	proxy.Unlock()
+}
+
+func (proxy *Proxy) notifyWaiters(ident string, err error) {
+	proxy.Lock()
+	for _, wait := range proxy.waiters {
+		if ident == wait.ident && !wait.done {
+			wait.ch <- err
+			close(wait.ch)
+			wait.done = true
+		}
+	}
+	proxy.Unlock()
+}
+
+func (proxy *Proxy) waitForStart(r *http.Request) error {
+	var ch chan error
+	proxy.Lock()
+	wait, found := proxy.waiters[r]
+	if found {
+		ch = wait.ch
+	}
+	proxy.Unlock()
+	if ch != nil {
+		Log.Debugf("Wait for start of container %s", wait.ident)
+		return <-ch
+	}
+	return nil
+}
+
+// If some other operation is waiting for a container to start, join in the wait
+func (proxy *Proxy) waitForStartByIdent(ident string) error {
+	var ch chan error
+	proxy.Lock()
+	for _, wait := range proxy.waiters {
+		if ident == wait.ident && !wait.done {
+			ch = wait.ch
+			break
+		}
+	}
+	proxy.Unlock()
+	if ch != nil {
+		Log.Debugf("Wait for start of container %s", ident)
+		return <-ch
+	}
+	return nil
+}
+
+func (proxy *Proxy) ContainerDied(ident string) {
+}
+
+func (proxy *Proxy) attach(container *docker.Container, orDie bool) error {
+	cidrs, err := proxy.weaveCIDRs(container.HostConfig.NetworkMode, container.Config.Env)
+	if err != nil {
+		Log.Infof("Leaving container %s alone because %s", container.ID, err)
+		return nil
+	}
+	Log.Infof("Attaching container %s with WEAVE_CIDR \"%s\" to weave network", container.ID, strings.Join(cidrs, " "))
+	args := []string{"attach"}
+	args = append(args, cidrs...)
+	if !proxy.NoRewriteHosts {
+		args = append(args, "--rewrite-hosts")
+
+		if container.HostConfig != nil {
+			for _, eh := range container.HostConfig.ExtraHosts {
+				args = append(args, fmt.Sprintf("--add-host=%s", eh))
+			}
+		}
+	}
+	if orDie {
+		args = append(args, "--or-die")
+	}
+	args = append(args, container.ID)
+	if _, stderr, err := callWeave(args...); err != nil {
+		Log.Warningf("Attaching container %s to weave network failed: %s", container.ID, string(stderr))
+		return errors.New(string(stderr))
+	} else if len(stderr) > 0 {
+		Log.Warningf("Attaching container %s to weave network: %s", container.ID, string(stderr))
+	}
+
+	return nil
+}
+
+func (proxy *Proxy) attachRouter(container *docker.Container) error {
+	Log.Infof("Attaching weave router container: %s", container.ID)
+	args := []string{"attach-router"}
+	if _, stderr, err := callWeave(args...); err != nil {
+		Log.Warningf("Attaching container %s to weave network failed: %s", container.ID, string(stderr))
+		return errors.New(string(stderr))
+	} else if len(stderr) > 0 {
+		Log.Warningf("Attaching container %s to weave network: %s", container.ID, string(stderr))
+	}
+	return nil
+}
+
+func (proxy *Proxy) weaveCIDRs(networkMode string, env []string) ([]string, error) {
+	if networkMode == "host" || strings.HasPrefix(networkMode, "container:") {
+		return nil, fmt.Errorf("the container has '--net=%s'", networkMode)
+	}
+	for _, e := range env {
 		if strings.HasPrefix(e, "WEAVE_CIDR=") {
 			if e[11:] == "none" {
 				return nil, ErrWeaveCIDRNone
@@ -260,3 +448,82 @@ func (proxy *Proxy) weaveCIDRsFromConfig(config *docker.Config, hostConfig *dock
 	}
 	return nil, nil
 }
+
+func (proxy *Proxy) setWeaveDNS(hostConfig jsonObject, hostname, dnsDomain string) error {
+	dns, err := hostConfig.StringArray("Dns")
+	if err != nil {
+		return err
+	}
+	hostConfig["Dns"] = append(dns, proxy.dockerBridgeIP)
+
+	dnsSearch, err := hostConfig.StringArray("DnsSearch")
+	if err != nil {
+		return err
+	}
+	if len(dnsSearch) == 0 {
+		if hostname == "" {
+			hostConfig["DnsSearch"] = []string{dnsDomain}
+		} else {
+			hostConfig["DnsSearch"] = []string{"."}
+		}
+	}
+
+	return nil
+}
+
+func (proxy *Proxy) getDNSDomain() (domain string) {
+	if proxy.WithoutDNS {
+		return ""
+	}
+
+	if proxy.WithDNS {
+		domain = nameserver.DefaultDomain
+	}
+
+	weaveContainer, err := proxy.client.InspectContainer("weave")
+	var weaveIP string
+	if err == nil && weaveContainer.NetworkSettings != nil {
+		weaveIP = weaveContainer.NetworkSettings.IPAddress
+	}
+	if weaveIP == "" {
+		weaveIP = "127.0.0.1"
+	}
+
+	url := fmt.Sprintf("http://%s:6784/domain", weaveIP)
+	resp, err := http.Get(url)
+	if err != nil || resp.StatusCode != http.StatusOK {
+		return
+	}
+	defer resp.Body.Close()
+
+	b, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return
+	}
+
+	return string(b)
+}
+
+func (proxy *Proxy) updateContainerNetworkSettings(container jsonObject) error {
+	containerID, err := container.String("Id")
+	if err != nil {
+		return err
+	}
+
+	if err := proxy.waitForStartByIdent(containerID); err != nil {
+		return err
+	}
+	mac, ips, nets, err := weaveContainerIPs(containerID)
+	if err != nil || len(ips) == 0 {
+		return err
+	}
+
+	networkSettings, err := container.Object("NetworkSettings")
+	if err != nil {
+		return err
+	}
+	networkSettings["MacAddress"] = mac
+	networkSettings["IPAddress"] = ips[0].String()
+	networkSettings["IPPrefixLen"], _ = nets[0].Mask.Size()
+	return nil
+}

proxy/start_container_interceptor.go

@@ -1,47 +1,70 @@
 package proxy
 
 import (
-	"errors"
 	"net/http"
 	"strings"
-
-	. "github.com/weaveworks/weave/common"
 )
 
 type startContainerInterceptor struct{ proxy *Proxy }
 
 func (i *startContainerInterceptor) InterceptRequest(r *http.Request) error {
-	return nil
-}
-
-func (i *startContainerInterceptor) InterceptResponse(r *http.Response) error {
-	if r.StatusCode < 200 || r.StatusCode >= 300 { // Docker didn't do the start
-		return nil
-	}
-
-	container, err := inspectContainerInPath(i.proxy.client, r.Request.URL.Path)
+	container, err := inspectContainerInPath(i.proxy.client, r.URL.Path)
 	if err != nil {
 		return err
 	}
 
-	cidrs, err := i.proxy.weaveCIDRsFromConfig(container.Config, container.HostConfig)
-	if err != nil {
-		Log.Infof("Leaving container %s alone because %s", container.ID, err)
-		return nil
-	}
-	Log.Infof("Attaching container %s with WEAVE_CIDR \"%s\" to weave network", container.ID, strings.Join(cidrs, " "))
-	args := []string{"attach"}
-	args = append(args, cidrs...)
-	if !i.proxy.NoRewriteHosts {
-		args = append(args, "--rewrite-hosts")
-	}
-	args = append(args, "--or-die", container.ID)
-	if _, stderr, err := callWeave(args...); err != nil {
-		Log.Warningf("Attaching container %s to weave network failed: %s", container.ID, string(stderr))
-		return errors.New(string(stderr))
-	} else if len(stderr) > 0 {
-		Log.Warningf("Attaching container %s to weave network: %s", container.ID, string(stderr))
-	}
+	// If the client has sent some JSON which might be a HostConfig, add our
+	// parameters back into it, otherwise Docker will consider them overwritten
+	if containerShouldAttach(container) && r.Header.Get("Content-Type") == "application/json" {
+		params := map[string]interface{}{}
+		if err := unmarshalRequestBody(r, &params); err != nil {
+			return err
+		}
+		// HostConfig can be sent either as a struct named HostConfig, or unnamed at top level
+		var hostConfig map[string]interface{}
+		if subParam, found := params["HostConfig"]; found {
+			if typecast, ok := subParam.(map[string]interface{}); ok {
+				hostConfig = typecast
+			}
+			// We can't reliably detect what Docker will see as a top-level HostConfig,
+			// so just assume any parameter at top level indicates there is one.
+		} else if len(params) > 0 {
+			hostConfig = params
+		}
+		if hostConfig != nil {
+			networkMode, err := jsonObject(hostConfig).String("NetworkMode")
+			if err != nil {
+				return err
+			}
+			if strings.HasPrefix(networkMode, "container:") || networkMode == "host" {
+				if err := addVolume(hostConfig, i.proxy.weaveWaitNoopVolume, "/w", "ro"); err != nil {
+					return err
+				}
+			} else {
+				if err := addVolume(hostConfig, i.proxy.weaveWaitVolume, "/w", "ro"); err != nil {
+					return err
+				}
+				if dnsDomain := i.proxy.getDNSDomain(); dnsDomain != "" {
+					if err := i.proxy.setWeaveDNS(hostConfig, container.Config.Hostname, dnsDomain); err != nil {
+						return err
+					}
+				}
+			}
 
+			// Note we marshal the original top-level dictionary to avoid disturbing anything else
+			if err := marshalRequestBody(r, params); err != nil {
+				return err
+			}
+		}
+	}
+	i.proxy.createWait(r, container.ID)
 	return nil
 }
+
+func (i *startContainerInterceptor) InterceptResponse(r *http.Response) error {
+	defer i.proxy.removeWait(r.Request)
+	if r.StatusCode < 200 || r.StatusCode >= 300 { // Docker didn't do the start
+		return nil
+	}
+	return i.proxy.waitForStart(r.Request)
+}

proxy/tls.go

@@ -17,15 +17,17 @@ type TLSConfig struct {
 	*tls.Config
 }
 
-func (c *TLSConfig) enabled() bool {
+// IsEnabled returns true if TLS is enable, according to the config.
+func (c *TLSConfig) IsEnabled() bool {
 	if c == nil {
 		return false
 	}
 	return c.Enabled || c.Verify
 }
 
-func (c *TLSConfig) loadCerts() error {
-	if !c.enabled() {
+// LoadCerts loads the certificates into c.Config, if TLS is enabled.
+func (c *TLSConfig) LoadCerts() error {
+	if !c.IsEnabled() {
 		return nil
 	}
 

router/bridge.go

@@ -0,0 +1,35 @@
+package router
+
+// Interface to packet handling on the local virtual bridge
+type Bridge interface {
+	// Inject a packet to be delivered locally
+	InjectPacket(PacketKey) FlowOp
+
+	// Start consuming packets from the bridge.  Injected packets
+	// should not be included.
+	StartConsumingPackets(BridgeConsumer) error
+
+	String() string
+	Stats() map[string]int
+}
+
+// A function that determines how to handle locally captured packets.
+type BridgeConsumer func(PacketKey) FlowOp
+
+type NullBridge struct{}
+
+func (NullBridge) InjectPacket(PacketKey) FlowOp {
+	return nil
+}
+
+func (NullBridge) StartConsumingPackets(BridgeConsumer) error {
+	return nil
+}
+
+func (NullBridge) String() string {
+	return "<no bridge networking>"
+}
+
+func (NullBridge) Stats() map[string]int {
+	return nil
+}

router/consts.go

@@ -1,34 +0,0 @@
-package router
-
-import (
-	"math"
-	"time"
-)
-
-const (
-	EthernetOverhead    = 14
-	UDPOverhead         = 28 // 20 bytes for IPv4, 8 bytes for UDP
-	Port                = 6783
-	HTTPPort            = Port + 1
-	DefaultPMTU         = 65535
-	MaxUDPPacketSize    = 65535
-	ChannelSize         = 16
-	FragTestSize        = 60001
-	PMTUDiscoverySize   = 60000
-	TCPHeartbeat        = 30 * time.Second
-	FastHeartbeat       = 500 * time.Millisecond
-	SlowHeartbeat       = 10 * time.Second
-	FragTestInterval    = 5 * time.Minute
-	GossipInterval      = 30 * time.Second
-	PMTUVerifyAttempts  = 8
-	PMTUVerifyTimeout   = 10 * time.Millisecond // gets doubled with every attempt
-	MaxDuration         = time.Duration(math.MaxInt64)
-	MaxMissedHeartbeats = 6
-	HeartbeatTimeout    = MaxMissedHeartbeats * SlowHeartbeat
-	MaxTCPMsgSize       = 10 * 1024 * 1024
-)
-
-var (
-	FragTest      = make([]byte, FragTestSize)
-	PMTUDiscovery = make([]byte, PMTUDiscoverySize)
-)

router/ethernet_decoder.go

@@ -29,7 +29,13 @@ func (dec *EthernetDecoder) DecodeLayers(data []byte) {
 	dec.parser.DecodeLayers(data, &dec.decoded)
 }
 
-func (dec *EthernetDecoder) sendICMPFragNeeded(mtu int, sendFrame func([]byte) error) error {
+func (dec *EthernetDecoder) PacketKey() (key PacketKey) {
+	copy(key.SrcMAC[:], dec.Eth.SrcMAC)
+	copy(key.DstMAC[:], dec.Eth.DstMAC)
+	return
+}
+
+func (dec *EthernetDecoder) makeICMPFragNeeded(mtu int) ([]byte, error) {
 	buf := gopacket.NewSerializeBuffer()
 	opts := gopacket.SerializeOptions{
 		FixLengths:       true,
@@ -57,23 +63,17 @@ func (dec *EthernetDecoder) sendICMPFragNeeded(mtu int, sendFrame func([]byte) e
 			Seq:      uint16(mtu)},
 		&payload)
 	if err != nil {
-		return err
+		return nil, err
 	}
 
 	log.Printf("Sending ICMP 3,4 (%v -> %v): PMTU=%v", dec.IP.DstIP, dec.IP.SrcIP, mtu)
-	return sendFrame(buf.Bytes())
+	return buf.Bytes(), nil
 }
 
 var (
-	// see http://en.wikipedia.org/wiki/Multicast_address#Ethernet
-	stpMACPrefix = []byte{0x01, 0x80, 0xC2, 0x00, 0x00}
-	zeroMAC, _   = net.ParseMAC("00:00:00:00:00:00")
+	zeroMAC, _ = net.ParseMAC("00:00:00:00:00:00")
 )
 
-func (dec *EthernetDecoder) DropFrame() bool {
-	return bytes.Equal(stpMACPrefix, dec.Eth.DstMAC[:len(stpMACPrefix)])
-}
-
 func (dec *EthernetDecoder) IsSpecial() bool {
 	return dec.Eth.Length == 0 && dec.Eth.EthernetType == layers.EthernetTypeLLC &&
 		bytes.Equal(zeroMAC, dec.Eth.SrcMAC) && bytes.Equal(zeroMAC, dec.Eth.DstMAC)

router/flow.go

@@ -0,0 +1,105 @@
+package router
+
+import (
+	"net"
+
+	"github.com/weaveworks/weave/mesh"
+)
+
+// Just enough flow machinery for the weave router
+
+type MAC [6]byte
+
+func (mac MAC) String() string {
+	return net.HardwareAddr(mac[:]).String()
+}
+
+type PacketKey struct {
+	SrcMAC MAC
+	DstMAC MAC
+}
+
+type ForwardPacketKey struct {
+	SrcPeer *mesh.Peer
+	DstPeer *mesh.Peer
+	PacketKey
+}
+
+type FlowOp interface {
+	// The caller must supply an EthernetDecoder specific to this
+	// thread, which has already been used to decode the frame.
+	// The broadcast parameter is a hint whether the packet is
+	// being broadcast.
+	Process(frame []byte, dec *EthernetDecoder, broadcast bool)
+
+	// Does the FlowOp discard the packet?
+	Discards() bool
+}
+
+type DiscardingFlowOp struct{}
+
+func (DiscardingFlowOp) Process([]byte, *EthernetDecoder, bool) {
+}
+
+func (DiscardingFlowOp) Discards() bool {
+	return true
+}
+
+type NonDiscardingFlowOp struct{}
+
+func (NonDiscardingFlowOp) Discards() bool {
+	return false
+}
+
+type MultiFlowOp struct {
+	broadcast bool
+	ops       []FlowOp
+}
+
+func NewMultiFlowOp(broadcast bool, ops ...FlowOp) *MultiFlowOp {
+	mfop := &MultiFlowOp{broadcast: broadcast}
+	for _, op := range ops {
+		mfop.Add(op)
+	}
+	return mfop
+}
+
+func (mfop *MultiFlowOp) Add(op FlowOp) {
+	mfop.ops = append(mfop.ops, op)
+}
+
+func (mfop *MultiFlowOp) Process(frame []byte, dec *EthernetDecoder, broadcast bool) {
+	for _, op := range mfop.ops {
+		op.Process(frame, dec, mfop.broadcast)
+	}
+}
+
+func (mfop *MultiFlowOp) Discards() bool {
+	for _, op := range mfop.ops {
+		if !op.Discards() {
+			return false
+		}
+	}
+
+	return true
+}
+
+func FlattenFlowOp(fop FlowOp) []FlowOp {
+	return collectFlowOps(nil, fop)
+}
+
+func collectFlowOps(into []FlowOp, fop FlowOp) []FlowOp {
+	if fop == nil {
+		return into
+	}
+
+	if mfop, ok := fop.(*MultiFlowOp); ok {
+		for _, op := range mfop.ops {
+			into = collectFlowOps(into, op)
+		}
+
+		return into
+	}
+
+	return append(into, fop)
+}

router/forwarder.go

@@ -1,419 +0,0 @@
-package router
-
-import (
-	"syscall"
-	"time"
-
-	"github.com/google/gopacket"
-	"github.com/google/gopacket/layers"
-)
-
-type ForwardedFrame struct {
-	srcPeer *Peer
-	dstPeer *Peer
-	frame   []byte
-}
-
-type FrameTooBigError struct {
-	EPMTU int // effective pmtu, i.e. what we tell packet senders
-}
-
-func (conn *LocalConnection) ensureForwarders() error {
-	if conn.forwarder != nil || conn.forwarderDF != nil {
-		return nil
-	}
-	udpSender := NewSimpleUDPSender(conn)
-	udpSenderDF, err := NewRawUDPSender(conn) // only thing that can error, so do it early
-	if err != nil {
-		return err
-	}
-
-	usingPassword := conn.SessionKey != nil
-	var encryptor, encryptorDF Encryptor
-	if usingPassword {
-		encryptor = NewNaClEncryptor(conn.local.NameByte, conn.SessionKey, conn.outbound, false)
-		encryptorDF = NewNaClEncryptor(conn.local.NameByte, conn.SessionKey, conn.outbound, true)
-	} else {
-		encryptor = NewNonEncryptor(conn.local.NameByte)
-		encryptorDF = NewNonEncryptor(conn.local.NameByte)
-	}
-
-	forwarder := NewForwarder(conn, encryptor, udpSender, DefaultPMTU)
-	forwarderDF := NewForwarderDF(conn, encryptorDF, udpSenderDF, DefaultPMTU)
-
-	// Various fields in the conn struct are read by other processes,
-	// so we have to use locks.
-	conn.Lock()
-	conn.forwarder = forwarder
-	conn.forwarderDF = forwarderDF
-	conn.Unlock()
-
-	return nil
-}
-
-func (conn *LocalConnection) stopForwarders() {
-	if conn.forwarder == nil || conn.forwarderDF == nil {
-		return
-	}
-	conn.forwarder.Shutdown()
-	conn.forwarderDF.Shutdown()
-}
-
-// Called from connection's actor process, and from the connection's
-// TCP receiver process.
-func (conn *LocalConnection) Send(df bool, frameBytes []byte) error {
-	frame := &ForwardedFrame{
-		srcPeer: conn.local,
-		dstPeer: conn.remote,
-		frame:   frameBytes}
-	return conn.forward(frame, nil, df)
-}
-
-// Called from LocalPeer.Relay[Broadcast] which is itself invoked from
-// router (both UDP listener process and sniffer process).
-func (conn *LocalConnection) Forward(frame *ForwardedFrame, dec *EthernetDecoder) error {
-	return conn.forward(frame, dec, dec != nil && dec.DF())
-}
-
-func (conn *LocalConnection) forward(frame *ForwardedFrame, dec *EthernetDecoder, df bool) error {
-	conn.RLock()
-	var (
-		forwarder     = conn.forwarder
-		forwarderDF   = conn.forwarderDF
-		effectivePMTU = conn.effectivePMTU
-		stackFrag     = conn.stackFrag
-	)
-	conn.RUnlock()
-
-	if forwarder == nil || forwarderDF == nil {
-		conn.Log("Cannot forward frame yet - awaiting contact")
-		return nil
-	}
-
-	// We could use non-blocking channel sends here, i.e. drop frames
-	// on the floor when the forwarder is busy. This would allow our
-	// caller - the capturing loop in the router - to read frames more
-	// quickly when under load, i.e. we'd drop fewer frames on the
-	// floor during capture. And we could maximise CPU utilisation
-	// since we aren't stalling a thread. However, a lot of work has
-	// already been done by the time we get here. Since any packet we
-	// drop will likely get re-transmitted we end up paying that cost
-	// multiple times. So it's better to drop things at the beginning
-	// of our pipeline.
-	if df {
-		if !frameTooBig(frame, effectivePMTU) {
-			forwarderDF.Forward(frame)
-			return nil
-		}
-		return FrameTooBigError{EPMTU: effectivePMTU}
-	}
-
-	if stackFrag || dec == nil || len(dec.decoded) < 2 {
-		forwarder.Forward(frame)
-		return nil
-	}
-	// Don't have trustworthy stack, so we're going to have to
-	// send it DF in any case.
-	if !frameTooBig(frame, effectivePMTU) {
-		forwarderDF.Forward(frame)
-		return nil
-	}
-	// We can't trust the stack to fragment, we have IP, and we
-	// have a frame that's too big for the MTU, so we have to
-	// fragment it ourself.
-	return fragment(dec.Eth, dec.IP, effectivePMTU, frame, func(segFrame *ForwardedFrame) {
-		forwarderDF.Forward(segFrame)
-	})
-}
-
-func frameTooBig(frame *ForwardedFrame, effectivePMTU int) bool {
-	// We capture/forward complete ethernet frames. Therefore the
-	// frame length includes the ethernet header. However, MTUs
-	// operate at the IP layer and thus do not include the ethernet
-	// header. To put it another way, when a sender that was told an
-	// MTU of M sends an IP packet of exactly that length, we will
-	// capture/forward M + EthernetOverhead bytes of data.
-	return len(frame.frame) > effectivePMTU+EthernetOverhead
-}
-
-func fragment(eth layers.Ethernet, ip layers.IPv4, pmtu int, frame *ForwardedFrame, forward func(*ForwardedFrame)) error {
-	// We are not doing any sort of NAT, so we don't need to worry
-	// about checksums of IP payload (eg UDP checksum).
-	headerSize := int(ip.IHL) * 4
-	// &^ is bit clear (AND NOT). So here we're clearing the lowest 3
-	// bits.
-	maxSegmentSize := (pmtu - headerSize) &^ 7
-	opts := gopacket.SerializeOptions{
-		FixLengths:       false,
-		ComputeChecksums: true}
-	payloadSize := int(ip.Length) - headerSize
-	payload := ip.BaseLayer.Payload[:payloadSize]
-	offsetBase := int(ip.FragOffset) << 3
-	origFlags := ip.Flags
-	ip.Flags = ip.Flags | layers.IPv4MoreFragments
-	ip.Length = uint16(headerSize + maxSegmentSize)
-	if eth.EthernetType == layers.EthernetTypeLLC {
-		// using LLC, so must set eth length correctly. eth length
-		// is just the length of the payload
-		eth.Length = ip.Length
-	} else {
-		eth.Length = 0
-	}
-	for offset := 0; offset < payloadSize; offset += maxSegmentSize {
-		var segmentPayload []byte
-		if len(payload) <= maxSegmentSize {
-			// last one
-			segmentPayload = payload
-			ip.Length = uint16(len(payload) + headerSize)
-			ip.Flags = origFlags
-			if eth.EthernetType == layers.EthernetTypeLLC {
-				eth.Length = ip.Length
-			} else {
-				eth.Length = 0
-			}
-		} else {
-			segmentPayload = payload[:maxSegmentSize]
-			payload = payload[maxSegmentSize:]
-		}
-		ip.FragOffset = uint16((offset + offsetBase) >> 3)
-		buf := gopacket.NewSerializeBuffer()
-		segPayload := gopacket.Payload(segmentPayload)
-		err := gopacket.SerializeLayers(buf, opts, &eth, &ip, &segPayload)
-		if err != nil {
-			return err
-		}
-		// make copies of the frame we received
-		segFrame := *frame
-		segFrame.frame = buf.Bytes()
-		forward(&segFrame)
-	}
-	return nil
-}
-
-// Forwarder
-
-type Forwarder struct {
-	conn             *LocalConnection
-	ch               chan<- *ForwardedFrame
-	finished         <-chan struct{}
-	enc              Encryptor
-	udpSender        UDPSender
-	maxPayload       int
-	processSendError func(error) error
-}
-
-func NewForwarder(conn *LocalConnection, enc Encryptor, udpSender UDPSender, pmtu int) *Forwarder {
-	ch := make(chan *ForwardedFrame, ChannelSize)
-	finished := make(chan struct{})
-	fwd := &Forwarder{
-		conn:             conn,
-		ch:               ch,
-		finished:         finished,
-		enc:              enc,
-		udpSender:        udpSender,
-		maxPayload:       pmtu - UDPOverhead,
-		processSendError: func(err error) error { return err }}
-	go fwd.run(ch, finished)
-	return fwd
-}
-
-func (fwd *Forwarder) Shutdown() {
-	fwd.ch <- nil
-}
-
-func (fwd *Forwarder) Forward(frame *ForwardedFrame) {
-	select {
-	case fwd.ch <- frame:
-	case <-fwd.finished:
-	}
-}
-
-func (fwd *Forwarder) run(ch <-chan *ForwardedFrame, finished chan<- struct{}) {
-	defer fwd.udpSender.Shutdown()
-	for {
-		if !fwd.accumulateAndSendFrames(ch, <-ch) {
-			close(finished)
-			return
-		}
-	}
-}
-
-func (fwd *Forwarder) effectiveOverhead() int {
-	return UDPOverhead + fwd.enc.PacketOverhead() + fwd.enc.FrameOverhead() + EthernetOverhead
-}
-
-// Drain the inbound channel of frames, aggregating them into larger
-// packets for efficient transmission.
-//
-// FIXME Depending on the golang scheduler, and the rate at which
-// franes get sent to the forwarder, we can be going around this loop
-// forever. That is bad since there may be other stuff for us to do,
-// i.e. the other branches in the run loop.
-func (fwd *Forwarder) accumulateAndSendFrames(ch <-chan *ForwardedFrame, frame *ForwardedFrame) bool {
-	if frame == nil {
-		return false
-	}
-	if !fwd.appendFrame(frame) {
-		fwd.logDrop(frame)
-		// [1] The buffer is empty at this point and therefore we must
-		// not flush it. The easiest way to accomplish that is simply
-		// by returning to the surrounding run loop.
-		return true
-	}
-	for {
-		select {
-		case frame = <-ch:
-			if frame == nil {
-				return false
-			}
-			if !fwd.appendFrame(frame) {
-				fwd.flush()
-				if !fwd.appendFrame(frame) {
-					fwd.logDrop(frame)
-					return true // see [1]
-				}
-			}
-		default:
-			fwd.flush()
-			return true
-		}
-	}
-}
-
-func (fwd *Forwarder) logDrop(frame *ForwardedFrame) {
-	fwd.conn.ErrorLog("Dropping too big frame during forwarding: frame len:", len(frame.frame), "; effective PMTU:", fwd.maxPayload+UDPOverhead-fwd.effectiveOverhead())
-}
-
-func (fwd *Forwarder) appendFrame(frame *ForwardedFrame) bool {
-	frameLen := len(frame.frame)
-	if fwd.enc.TotalLen()+fwd.enc.FrameOverhead()+frameLen > fwd.maxPayload {
-		return false
-	}
-	fwd.enc.AppendFrame(frame.srcPeer.NameByte, frame.dstPeer.NameByte, frame.frame)
-	return true
-}
-
-func (fwd *Forwarder) flush() {
-	msg, err := fwd.enc.Bytes()
-	if err != nil {
-		fwd.conn.Shutdown(err)
-	}
-	err = fwd.processSendError(fwd.udpSender.Send(msg))
-	if err != nil && PosixError(err) != syscall.ENOBUFS {
-		fwd.conn.Shutdown(err)
-	}
-}
-
-type ForwarderDF struct {
-	Forwarder
-	verifyPMTUTick  <-chan time.Time
-	verifyPMTU      chan<- int
-	pmtuVerifyCount uint
-	pmtuVerified    bool
-	highestGoodPMTU int
-	unverifiedPMTU  int
-	lowestBadPMTU   int
-}
-
-func NewForwarderDF(conn *LocalConnection, enc Encryptor, udpSender UDPSender, pmtu int) *ForwarderDF {
-	ch := make(chan *ForwardedFrame, ChannelSize)
-	finished := make(chan struct{})
-	verifyPMTU := make(chan int, ChannelSize)
-	fwd := &ForwarderDF{
-		Forwarder: Forwarder{
-			conn:       conn,
-			ch:         ch,
-			finished:   finished,
-			enc:        enc,
-			udpSender:  udpSender,
-			maxPayload: pmtu - UDPOverhead},
-		verifyPMTU: verifyPMTU}
-	fwd.Forwarder.processSendError = fwd.processSendError
-	fwd.unverifiedPMTU = pmtu - fwd.effectiveOverhead()
-	conn.setEffectivePMTU(fwd.unverifiedPMTU)
-	go fwd.run(ch, finished, verifyPMTU)
-	return fwd
-}
-
-func (fwd *ForwarderDF) PMTUVerified(pmtu int) {
-	select {
-	case fwd.verifyPMTU <- pmtu:
-	case <-fwd.finished:
-	}
-}
-
-func (fwd *ForwarderDF) run(ch <-chan *ForwardedFrame, finished chan<- struct{}, verifyPMTU <-chan int) {
-	defer fwd.udpSender.Shutdown()
-	for {
-		select {
-		case <-fwd.verifyPMTUTick:
-			// We only do this case here when we know the
-			// buffers are all empty so that we don't risk
-			// appending verify-frames to other data.
-			fwd.verifyPMTUTick = nil
-			if fwd.pmtuVerified {
-				continue
-			}
-			if fwd.pmtuVerifyCount > 0 {
-				fwd.pmtuVerifyCount--
-				fwd.attemptVerifyEffectivePMTU()
-			} else {
-				// we've exceeded the verification
-				// attempts of the unverifiedPMTU
-				fwd.lowestBadPMTU = fwd.unverifiedPMTU
-				fwd.verifyEffectivePMTU((fwd.highestGoodPMTU + fwd.lowestBadPMTU) / 2)
-			}
-		case epmtu := <-verifyPMTU:
-			if fwd.pmtuVerified || epmtu != fwd.unverifiedPMTU {
-				continue
-			}
-			if epmtu+1 < fwd.lowestBadPMTU {
-				fwd.highestGoodPMTU = fwd.unverifiedPMTU // = epmtu
-				fwd.verifyEffectivePMTU((fwd.highestGoodPMTU + fwd.lowestBadPMTU) / 2)
-			} else {
-				fwd.pmtuVerified = true
-				fwd.maxPayload = epmtu + fwd.effectiveOverhead() - UDPOverhead
-				fwd.conn.setEffectivePMTU(epmtu)
-				fwd.conn.Log("Effective PMTU verified at", epmtu)
-			}
-		case frame := <-ch:
-			if !fwd.accumulateAndSendFrames(ch, frame) {
-				close(finished)
-				return
-			}
-		}
-	}
-}
-
-func (fwd *ForwarderDF) verifyEffectivePMTU(newUnverifiedPMTU int) {
-	fwd.unverifiedPMTU = newUnverifiedPMTU
-	fwd.pmtuVerifyCount = PMTUVerifyAttempts
-	fwd.attemptVerifyEffectivePMTU()
-}
-
-func (fwd *ForwarderDF) attemptVerifyEffectivePMTU() {
-	fwd.enc.AppendFrame(fwd.conn.local.NameByte, fwd.conn.remote.NameByte,
-		make([]byte, fwd.unverifiedPMTU+EthernetOverhead))
-	fwd.flush()
-	if fwd.verifyPMTUTick == nil {
-		fwd.verifyPMTUTick = time.After(PMTUVerifyTimeout << (PMTUVerifyAttempts - fwd.pmtuVerifyCount))
-	}
-}
-
-func (fwd *ForwarderDF) processSendError(err error) error {
-	if mtbe, ok := err.(MsgTooBigError); ok {
-		newUnverifiedPMTU := mtbe.PMTU - fwd.effectiveOverhead()
-		if newUnverifiedPMTU < fwd.unverifiedPMTU {
-			fwd.pmtuVerified = false
-			fwd.maxPayload = mtbe.PMTU - UDPOverhead
-			fwd.highestGoodPMTU = 8
-			fwd.lowestBadPMTU = newUnverifiedPMTU + 1
-			fwd.conn.setEffectivePMTU(newUnverifiedPMTU)
-			fwd.verifyEffectivePMTU(newUnverifiedPMTU)
-		}
-
-		return nil
-	}
-
-	return err
-}

router/frame_crypto.go

@@ -1,32 +1,13 @@
 package router
 
 import (
-	"crypto/rand"
-	"crypto/sha256"
 	"encoding/binary"
-	"encoding/gob"
 	"fmt"
-	"io"
-	"sync"
 
 	"github.com/andybalholm/go-bit"
-	"golang.org/x/crypto/nacl/box"
 	"golang.org/x/crypto/nacl/secretbox"
 )
 
-func GenerateKeyPair() (publicKey, privateKey *[32]byte, err error) {
-	return box.GenerateKey(rand.Reader)
-}
-
-func FormSessionKey(remotePublicKey, localPrivateKey *[32]byte, secretKey []byte) *[32]byte {
-	var sharedKey [32]byte
-	box.Precompute(&sharedKey, remotePublicKey, localPrivateKey)
-	sharedKeySlice := sharedKey[:]
-	sharedKeySlice = append(sharedKeySlice, secretKey...)
-	sessionKey := sha256.Sum256(sharedKeySlice)
-	return &sessionKey
-}
-
 // Frame Encryptors
 
 type Encryptor interface {
@@ -180,6 +161,10 @@ type PacketDecodingError struct {
 	Desc string
 }
 
+func (pde PacketDecodingError) Error() string {
+	return fmt.Sprint("Failed to decode packet: ", pde.Desc)
+}
+
 func NewNonDecryptor() *NonDecryptor {
 	return &NonDecryptor{}
 }
@@ -297,154 +282,3 @@ func (di *NaClDecryptorInstance) advanceState(seqNo uint64) (int, *bit.Set) {
 		return offset, di.usedOffsets
 	}
 }
-
-// TCP Senders/Receivers
-
-// The lowest 64 bits of the nonce contain the message sequence
-// number. The top most bit indicates the connection polarity at the
-// sender - '1' for outbound; the next indicates protocol type - '1'
-// for TCP. The remaining 126 bits are zero. The polarity is needed so
-// that the two ends of a connection do not use the same nonces; the
-// protocol type so that the TCP and UDP sender nonces are disjoint.
-// This is a requirement of the NaCl Security Model; see
-// http://nacl.cr.yp.to/box.html.
-type TCPCryptoState struct {
-	sessionKey *[32]byte
-	nonce      [24]byte
-	seqNo      uint64
-}
-
-func NewTCPCryptoState(sessionKey *[32]byte, outbound bool) *TCPCryptoState {
-	s := &TCPCryptoState{sessionKey: sessionKey}
-	if outbound {
-		s.nonce[0] |= (1 << 7)
-	}
-	// Ensure that TCP and UDP sequences are disjoint
-	s.nonce[0] |= (1 << 6)
-	return s
-}
-
-func (s *TCPCryptoState) advance() {
-	s.seqNo++
-	binary.BigEndian.PutUint64(s.nonce[16:24], s.seqNo)
-}
-
-type TCPSender interface {
-	Send([]byte) error
-}
-
-type GobTCPSender struct {
-	encoder *gob.Encoder
-}
-
-type LengthPrefixTCPSender struct {
-	writer io.Writer
-}
-
-type EncryptedTCPSender struct {
-	sync.RWMutex
-	sender TCPSender
-	state  *TCPCryptoState
-}
-
-func NewGobTCPSender(encoder *gob.Encoder) *GobTCPSender {
-	return &GobTCPSender{encoder: encoder}
-}
-
-func (sender *GobTCPSender) Send(msg []byte) error {
-	return sender.encoder.Encode(msg)
-}
-
-func NewLengthPrefixTCPSender(writer io.Writer) *LengthPrefixTCPSender {
-	return &LengthPrefixTCPSender{writer: writer}
-}
-
-func (sender *LengthPrefixTCPSender) Send(msg []byte) error {
-	l := len(msg)
-	if l > MaxTCPMsgSize {
-		return fmt.Errorf("outgoing message exceeds maximum size: %d > %d", l, MaxTCPMsgSize)
-	}
-	// We copy the message so we can send it in a single Write
-	// operation, thus making this thread-safe without locking.
-	prefixedMsg := make([]byte, 4+l)
-	binary.BigEndian.PutUint32(prefixedMsg, uint32(l))
-	copy(prefixedMsg[4:], msg)
-	_, err := sender.writer.Write(prefixedMsg)
-	return err
-}
-
-func NewEncryptedTCPSender(sender TCPSender, sessionKey *[32]byte, outbound bool) *EncryptedTCPSender {
-	return &EncryptedTCPSender{sender: sender, state: NewTCPCryptoState(sessionKey, outbound)}
-}
-
-func (sender *EncryptedTCPSender) Send(msg []byte) error {
-	sender.Lock()
-	defer sender.Unlock()
-	encodedMsg := secretbox.Seal(nil, msg, &sender.state.nonce, sender.state.sessionKey)
-	sender.state.advance()
-	return sender.sender.Send(encodedMsg)
-}
-
-type TCPReceiver interface {
-	Receive() ([]byte, error)
-}
-
-type GobTCPReceiver struct {
-	decoder *gob.Decoder
-}
-
-type LengthPrefixTCPReceiver struct {
-	reader io.Reader
-}
-
-type EncryptedTCPReceiver struct {
-	receiver TCPReceiver
-	state    *TCPCryptoState
-}
-
-func NewGobTCPReceiver(decoder *gob.Decoder) *GobTCPReceiver {
-	return &GobTCPReceiver{decoder: decoder}
-}
-
-func (receiver *GobTCPReceiver) Receive() ([]byte, error) {
-	var msg []byte
-	err := receiver.decoder.Decode(&msg)
-	return msg, err
-}
-
-func NewLengthPrefixTCPReceiver(reader io.Reader) *LengthPrefixTCPReceiver {
-	return &LengthPrefixTCPReceiver{reader: reader}
-}
-
-func (receiver *LengthPrefixTCPReceiver) Receive() ([]byte, error) {
-	lenPrefix := make([]byte, 4)
-	if _, err := io.ReadFull(receiver.reader, lenPrefix); err != nil {
-		return nil, err
-	}
-	l := binary.BigEndian.Uint32(lenPrefix)
-	if l > MaxTCPMsgSize {
-		return nil, fmt.Errorf("incoming message exceeds maximum size: %d > %d", l, MaxTCPMsgSize)
-	}
-	msg := make([]byte, l)
-	_, err := io.ReadFull(receiver.reader, msg)
-	return msg, err
-}
-
-func NewEncryptedTCPReceiver(receiver TCPReceiver, sessionKey *[32]byte, outbound bool) *EncryptedTCPReceiver {
-	return &EncryptedTCPReceiver{receiver: receiver, state: NewTCPCryptoState(sessionKey, !outbound)}
-}
-
-func (receiver *EncryptedTCPReceiver) Receive() ([]byte, error) {
-	msg, err := receiver.receiver.Receive()
-	if err != nil {
-		return nil, err
-	}
-
-	decodedMsg, success := secretbox.Open(nil, msg, &receiver.state.nonce, receiver.state.sessionKey)
-	if !success {
-		return nil, fmt.Errorf("Unable to decrypt TCP msg")
-	}
-
-	receiver.state.advance()
-	return decodedMsg, nil
-}

router/http.go

@@ -2,12 +2,13 @@ package router
 
 import (
 	"fmt"
+	"net/http"
+
 	"github.com/gorilla/mux"
 	"github.com/weaveworks/weave/common"
-	"net/http"
 )
 
-func (router *Router) HandleHTTP(muxRouter *mux.Router) {
+func (router *NetworkRouter) HandleHTTP(muxRouter *mux.Router) {
 
 	muxRouter.Methods("POST").Path("/connect").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		if err := r.ParseForm(); err != nil {

router/mac_cache.go

@@ -4,11 +4,13 @@ import (
 	"net"
 	"sync"
 	"time"
+
+	"github.com/weaveworks/weave/mesh"
 )
 
 type MacCacheEntry struct {
 	lastSeen time.Time
-	peer     *Peer
+	peer     *mesh.Peer
 }
 
 type MacCache struct {
@@ -16,10 +18,10 @@ type MacCache struct {
 	table       map[uint64]*MacCacheEntry
 	maxAge      time.Duration
 	expiryTimer *time.Timer
-	onExpiry    func(net.HardwareAddr, *Peer)
+	onExpiry    func(net.HardwareAddr, *mesh.Peer)
 }
 
-func NewMacCache(maxAge time.Duration, onExpiry func(net.HardwareAddr, *Peer)) *MacCache {
+func NewMacCache(maxAge time.Duration, onExpiry func(net.HardwareAddr, *mesh.Peer)) *MacCache {
 	cache := &MacCache{
 		table:    make(map[uint64]*MacCacheEntry),
 		maxAge:   maxAge,
@@ -28,46 +30,62 @@ func NewMacCache(maxAge time.Duration, onExpiry func(net.HardwareAddr, *Peer)) *
 	return cache
 }
 
-func (cache *MacCache) Enter(mac net.HardwareAddr, peer *Peer) bool {
+func (cache *MacCache) add(mac net.HardwareAddr, peer *mesh.Peer, force bool) (bool, *mesh.Peer) {
 	key := macint(mac)
 	now := time.Now()
+
 	cache.RLock()
 	entry, found := cache.table[key]
 	if found && entry.peer == peer && now.Before(entry.lastSeen.Add(cache.maxAge/10)) {
 		cache.RUnlock()
-		return false
+		return false, nil
 	}
 	cache.RUnlock()
+
 	cache.Lock()
 	defer cache.Unlock()
+
 	entry, found = cache.table[key]
 	if !found {
 		cache.table[key] = &MacCacheEntry{lastSeen: now, peer: peer}
-		return true
+		return true, nil
 	}
+
 	if entry.peer != peer {
-		entry.lastSeen = now
+		if !force {
+			return false, entry.peer
+		}
+
 		entry.peer = peer
-		return true
 	}
+
 	if now.After(entry.lastSeen.Add(cache.maxAge / 10)) {
 		entry.lastSeen = now
 	}
-	return false
+
+	return false, nil
 }
 
-func (cache *MacCache) Lookup(mac net.HardwareAddr) (*Peer, bool) {
+func (cache *MacCache) Add(mac net.HardwareAddr, peer *mesh.Peer) (bool, *mesh.Peer) {
+	return cache.add(mac, peer, false)
+}
+
+func (cache *MacCache) AddForced(mac net.HardwareAddr, peer *mesh.Peer) (bool, *mesh.Peer) {
+	return cache.add(mac, peer, true)
+}
+
+func (cache *MacCache) Lookup(mac net.HardwareAddr) *mesh.Peer {
 	key := macint(mac)
 	cache.RLock()
 	defer cache.RUnlock()
 	entry, found := cache.table[key]
 	if !found {
-		return nil, false
+		return nil
 	}
-	return entry.peer, true
+	return entry.peer
 }
 
-func (cache *MacCache) Delete(peer *Peer) bool {
+func (cache *MacCache) Delete(peer *mesh.Peer) bool {
 	found := false
 	cache.Lock()
 	defer cache.Unlock()
@@ -96,3 +114,20 @@ func (cache *MacCache) expire() {
 	}
 	cache.setExpiryTimer()
 }
+
+func macint(mac net.HardwareAddr) (r uint64) {
+	for _, b := range mac {
+		r <<= 8
+		r |= uint64(b)
+	}
+	return
+}
+
+func intmac(key uint64) (r net.HardwareAddr) {
+	r = make([]byte, 6)
+	for i := 5; i >= 0; i-- {
+		r[i] = byte(key)
+		key >>= 8
+	}
+	return
+}

router/network_overlay.go

@@ -0,0 +1,50 @@
+package router
+
+import (
+	"github.com/weaveworks/weave/mesh"
+)
+
+// Interface to overlay network packet handling
+type NetworkOverlay interface {
+	mesh.Overlay
+
+	// The routes have changed, so any cached information should
+	// be discarded.
+	InvalidateRoutes()
+
+	// A mapping of a short id to a peer has changed
+	InvalidateShortIDs()
+
+	// Start consuming forwarded packets.
+	StartConsumingPackets(*mesh.Peer, *mesh.Peers, OverlayConsumer) error
+}
+
+// When a consumer is called, the decoder will already have been used
+// to decode the frame.
+type OverlayConsumer func(ForwardPacketKey) FlowOp
+
+// All of the machinery to forward packets to a particular peer
+type OverlayForwarder interface {
+	mesh.OverlayConnection
+	// Forward a packet across the connection.  May be called as soon
+	// as the overlay connection is created, in particular before
+	// Confirm().  The return value nil means the key could not be
+	// handled by this forwarder.
+	Forward(ForwardPacketKey) FlowOp
+}
+
+type NullNetworkOverlay struct{ mesh.NullOverlay }
+
+func (NullNetworkOverlay) InvalidateRoutes() {
+}
+
+func (NullNetworkOverlay) InvalidateShortIDs() {
+}
+
+func (NullNetworkOverlay) StartConsumingPackets(*mesh.Peer, *mesh.Peers, OverlayConsumer) error {
+	return nil
+}
+
+func (NullNetworkOverlay) Forward(ForwardPacketKey) FlowOp {
+	return DiscardingFlowOp{}
+}

router/network_router.go

@@ -0,0 +1,209 @@
+package router
+
+import (
+	"math"
+	"net"
+	"time"
+
+	"github.com/weaveworks/weave/common"
+	"github.com/weaveworks/weave/mesh"
+)
+
+const (
+	ChannelSize         = 16
+	MaxUDPPacketSize    = 65535
+	FastHeartbeat       = 500 * time.Millisecond
+	SlowHeartbeat       = 10 * time.Second
+	MaxMissedHeartbeats = 6
+	HeartbeatTimeout    = MaxMissedHeartbeats * SlowHeartbeat
+	MaxDuration         = time.Duration(math.MaxInt64)
+	NameSize            = mesh.NameSize
+	// should be greater than typical ARP cache expiries, i.e. > 3/2 *
+	// /proc/sys/net/ipv4_neigh/*/base_reachable_time_ms on Linux
+	macMaxAge = 10 * time.Minute
+)
+
+var (
+	log        = common.Log
+	checkFatal = common.CheckFatal
+	checkWarn  = common.CheckWarn
+)
+
+type NetworkConfig struct {
+	BufSz         int
+	PacketLogging PacketLogging
+	Bridge        Bridge
+}
+
+type PacketLogging interface {
+	LogPacket(string, PacketKey)
+	LogForwardPacket(string, ForwardPacketKey)
+}
+
+type NetworkRouter struct {
+	*mesh.Router
+	NetworkConfig
+	Macs *MacCache
+}
+
+func NewNetworkRouter(config mesh.Config, networkConfig NetworkConfig, name mesh.PeerName, nickName string, overlay NetworkOverlay) *NetworkRouter {
+	if overlay == nil {
+		overlay = NullNetworkOverlay{}
+	}
+	if networkConfig.Bridge == nil {
+		networkConfig.Bridge = NullBridge{}
+	}
+
+	router := &NetworkRouter{Router: mesh.NewRouter(config, name, nickName, overlay), NetworkConfig: networkConfig}
+	router.Peers.OnInvalidateShortIDs(overlay.InvalidateShortIDs)
+	router.Routes.OnChange(overlay.InvalidateRoutes)
+	router.Macs = NewMacCache(macMaxAge,
+		func(mac net.HardwareAddr, peer *mesh.Peer) {
+			log.Println("Expired MAC", mac, "at", peer)
+		})
+	router.Peers.OnGC(func(peer *mesh.Peer) { router.Macs.Delete(peer) })
+	return router
+}
+
+// Start listening for TCP connections, locally captured packets, and
+// forwarded packets.
+func (router *NetworkRouter) Start() {
+	log.Println("Sniffing traffic on", router.Bridge)
+	checkFatal(router.Bridge.StartConsumingPackets(router.handleCapturedPacket))
+	checkFatal(router.Overlay.(NetworkOverlay).StartConsumingPackets(router.Ourself.Peer, router.Peers, router.handleForwardedPacket))
+	router.Router.Start()
+}
+
+func (router *NetworkRouter) handleCapturedPacket(key PacketKey) FlowOp {
+	router.PacketLogging.LogPacket("Captured", key)
+	srcMac := net.HardwareAddr(key.SrcMAC[:])
+
+	switch newSrcMac, conflictPeer := router.Macs.Add(srcMac, router.Ourself.Peer); {
+	case newSrcMac:
+		log.Println("Discovered local MAC", srcMac)
+
+	case conflictPeer != nil:
+		// The MAC cache has an entry for the source MAC
+		// associated with another peer.  This probably means
+		// we are seeing a frame we injected ourself.  That
+		// shouldn't happen, but discard it just in case.
+		log.Error("Captured frame from MAC (", srcMac, ") associated with another peer ", conflictPeer)
+		return DiscardingFlowOp{}
+	}
+
+	// Discard STP broadcasts
+	if key.DstMAC == [...]byte{0x01, 0x80, 0xC2, 0x00, 0x00, 0x00} {
+		return DiscardingFlowOp{}
+	}
+
+	dstMac := net.HardwareAddr(key.DstMAC[:])
+	switch dstPeer := router.Macs.Lookup(dstMac); dstPeer {
+	case router.Ourself.Peer:
+		// The packet is destined for a local MAC.  The bridge
+		// won't normally send us such packets, and if it does
+		// it's likely to be broadcasting the packet to all
+		// ports.  So if it happens, just drop the packet to
+		// avoid warnings if we try to forward it.
+		return DiscardingFlowOp{}
+	case nil:
+		// If we don't know which peer corresponds to the dest
+		// MAC, broadcast it.
+		router.PacketLogging.LogPacket("Broadcasting", key)
+		return router.relayBroadcast(router.Ourself.Peer, key)
+	default:
+		router.PacketLogging.LogPacket("Forwarding", key)
+		return router.relay(ForwardPacketKey{
+			PacketKey: key,
+			SrcPeer:   router.Ourself.Peer,
+			DstPeer:   dstPeer})
+	}
+}
+
+func (router *NetworkRouter) handleForwardedPacket(key ForwardPacketKey) FlowOp {
+	if key.DstPeer != router.Ourself.Peer {
+		// it's not for us, we're just relaying it
+		router.PacketLogging.LogForwardPacket("Relaying", key)
+		return router.relay(key)
+	}
+
+	// At this point, it's either unicast to us, or a broadcast
+	// (because the DstPeer on a forwarded broadcast packet is
+	// always set to the peer being forwarded to)
+
+	srcMac := net.HardwareAddr(key.SrcMAC[:])
+	dstMac := net.HardwareAddr(key.DstMAC[:])
+
+	switch newSrcMac, conflictPeer := router.Macs.AddForced(srcMac, key.SrcPeer); {
+	case newSrcMac:
+		log.Print("Discovered remote MAC ", srcMac, " at ", key.SrcPeer)
+
+	case conflictPeer != nil:
+		log.Print("Discovered remote MAC ", srcMac, " at ", key.SrcPeer, " (was at ", conflictPeer, ")")
+
+		// We need to clear out any flows destined to the MAC
+		// that forward to the old peer.
+		router.Overlay.(NetworkOverlay).InvalidateRoutes()
+	}
+
+	router.PacketLogging.LogForwardPacket("Injecting", key)
+	injectFop := router.Bridge.InjectPacket(key.PacketKey)
+	dstPeer := router.Macs.Lookup(dstMac)
+	if dstPeer == router.Ourself.Peer {
+		return injectFop
+	}
+
+	router.PacketLogging.LogForwardPacket("Relaying broadcast", key)
+	relayFop := router.relayBroadcast(key.SrcPeer, key.PacketKey)
+	switch {
+	case injectFop == nil:
+		return relayFop
+
+	case relayFop == nil:
+		return injectFop
+
+	default:
+		mfop := NewMultiFlowOp(false)
+		mfop.Add(injectFop)
+		mfop.Add(relayFop)
+		return mfop
+	}
+}
+
+// Routing
+
+func (router *NetworkRouter) relay(key ForwardPacketKey) FlowOp {
+	relayPeerName, found := router.Routes.Unicast(key.DstPeer.Name)
+	if !found {
+		// Not necessarily an error as there could be a race with the
+		// dst disappearing whilst the frame is in flight
+		log.Println("Received packet for unknown destination:", key.DstPeer)
+		return DiscardingFlowOp{}
+	}
+
+	conn, found := router.Ourself.ConnectionTo(relayPeerName)
+	if !found {
+		// Again, could just be a race, not necessarily an error
+		log.Println("Unable to find connection to relay peer", relayPeerName)
+		return DiscardingFlowOp{}
+	}
+
+	return conn.(*mesh.LocalConnection).OverlayConn.(OverlayForwarder).Forward(key)
+}
+
+func (router *NetworkRouter) relayBroadcast(srcPeer *mesh.Peer, key PacketKey) FlowOp {
+	nextHops := router.Routes.Broadcast(srcPeer.Name)
+	if len(nextHops) == 0 {
+		return DiscardingFlowOp{}
+	}
+
+	op := NewMultiFlowOp(true)
+
+	for _, conn := range router.Ourself.ConnectionsTo(nextHops) {
+		op.Add(conn.(*mesh.LocalConnection).OverlayConn.(OverlayForwarder).Forward(ForwardPacketKey{
+			PacketKey: key,
+			SrcPeer:   srcPeer,
+			DstPeer:   conn.Remote()}))
+	}
+
+	return op
+}

router/network_router_status.go

@@ -0,0 +1,45 @@
+package router
+
+import (
+	"time"
+
+	"github.com/weaveworks/weave/mesh"
+)
+
+type NetworkRouterStatus struct {
+	*mesh.Status
+	Interface    string
+	CaptureStats map[string]int
+	MACs         []MACStatus
+}
+
+type MACStatus struct {
+	Mac      string
+	Name     string
+	NickName string
+	LastSeen time.Time
+}
+
+func NewNetworkRouterStatus(router *NetworkRouter) *NetworkRouterStatus {
+	return &NetworkRouterStatus{
+		mesh.NewStatus(router.Router),
+		router.Bridge.String(),
+		router.Bridge.Stats(),
+		NewMACStatusSlice(router.Macs)}
+}
+
+func NewMACStatusSlice(cache *MacCache) []MACStatus {
+	cache.RLock()
+	defer cache.RUnlock()
+
+	var slice []MACStatus
+	for key, entry := range cache.table {
+		slice = append(slice, MACStatus{
+			intmac(key).String(),
+			entry.peer.Name.String(),
+			entry.peer.NickName,
+			entry.lastSeen})
+	}
+
+	return slice
+}

router/overlay_switch.go

@@ -0,0 +1,433 @@
+package router
+
+import (
+	"fmt"
+	"strings"
+	"sync"
+
+	"github.com/weaveworks/weave/mesh"
+)
+
+// OverlaySwitch selects which overlay to use, from a set of
+// subsidiary overlays.  First, it passes a list of supported overlays
+// in the connection features, and uses that to determine which
+// overlays are in common.  Then it tries those common overlays, and
+// uses the best one that seems to be working.
+
+type OverlaySwitch struct {
+	overlays      map[string]NetworkOverlay
+	overlayNames  []string
+	compatOverlay NetworkOverlay
+}
+
+func NewOverlaySwitch() *OverlaySwitch {
+	return &OverlaySwitch{overlays: make(map[string]NetworkOverlay)}
+}
+
+func (osw *OverlaySwitch) Add(name string, overlay NetworkOverlay) {
+	// check for repeated names
+	if _, present := osw.overlays[name]; present {
+		log.Fatal("OverlaySwitch: repeated overlay name")
+	}
+
+	osw.overlays[name] = overlay
+	osw.overlayNames = append(osw.overlayNames, name)
+}
+
+func (osw *OverlaySwitch) SetCompatOverlay(overlay NetworkOverlay) {
+	osw.compatOverlay = overlay
+}
+
+func (osw *OverlaySwitch) AddFeaturesTo(features map[string]string) {
+	features["Overlays"] = strings.Join(osw.overlayNames, " ")
+}
+
+func (osw *OverlaySwitch) Diagnostics() interface{} {
+	diagnostics := make(map[string]interface{})
+	for name, overlay := range osw.overlays {
+		diagnostics[name] = overlay.Diagnostics()
+	}
+	return diagnostics
+}
+
+func (osw *OverlaySwitch) InvalidateRoutes() {
+	for _, overlay := range osw.overlays {
+		overlay.InvalidateRoutes()
+	}
+}
+
+func (osw *OverlaySwitch) InvalidateShortIDs() {
+	for _, overlay := range osw.overlays {
+		overlay.InvalidateShortIDs()
+	}
+}
+
+func (osw *OverlaySwitch) StartConsumingPackets(localPeer *mesh.Peer, peers *mesh.Peers, consumer OverlayConsumer) error {
+	for _, overlay := range osw.overlays {
+		if err := overlay.StartConsumingPackets(localPeer, peers, consumer); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+type namedOverlay struct {
+	NetworkOverlay
+	name string
+}
+
+// Find the common set of overlays supported by both sides, with the
+// ordering being the same on both sides too.
+func (osw *OverlaySwitch) commonOverlays(params mesh.OverlayConnectionParams) ([]namedOverlay, error) {
+	var peerOverlays []string
+	if overlaysFeature, present := params.Features["Overlays"]; present {
+		peerOverlays = strings.Split(overlaysFeature, " ")
+	}
+
+	common := make(map[string]NetworkOverlay)
+	for _, name := range peerOverlays {
+		if overlay := osw.overlays[name]; overlay != nil {
+			common[name] = overlay
+		}
+	}
+
+	if len(common) == 0 {
+		return nil, fmt.Errorf("no overlays in common with peer")
+	}
+
+	// we order them according to the connecting node
+	ordering := osw.overlayNames
+	if params.RemoteAddr == nil {
+		// we are the connectee
+		ordering = peerOverlays
+	}
+
+	res := make([]namedOverlay, 0, len(common))
+	for _, name := range ordering {
+		overlay := common[name]
+		if overlay != nil {
+			res = append(res, namedOverlay{overlay, name})
+		}
+	}
+
+	// we use bytes to represent forwarder indices in control
+	// messages, so just in case:
+	if len(res) > 256 {
+		res = res[:256]
+	}
+
+	return res, nil
+}
+
+type overlaySwitchForwarder struct {
+	remotePeer *mesh.Peer
+
+	lock sync.Mutex
+
+	// the index of the forwarder to send on
+	best int
+
+	// the subsidiary forwarders
+	forwarders []subForwarder
+
+	// closed to tell the main goroutine to stop
+	stopChan chan<- struct{}
+
+	alreadyEstablished bool
+	establishedChan    chan struct{}
+	errorChan          chan error
+}
+
+// A subsidiary forwarder
+type subForwarder struct {
+	fwd         OverlayForwarder
+	overlayName string
+
+	// Has the forwarder signalled that it is established?
+	established bool
+
+	// closed to tell the forwarder monitor goroutine to stop
+	stopChan chan<- struct{}
+}
+
+// An event from a subsidiary forwarder
+type subForwarderEvent struct {
+	// the index of the forwarder
+	index int
+
+	// is this an "established" event?
+	established bool
+
+	// is this an error event?
+	err error
+}
+
+func (osw *OverlaySwitch) PrepareConnection(params mesh.OverlayConnectionParams) (mesh.OverlayConnection, error) {
+	if _, present := params.Features["Overlays"]; !present && osw.compatOverlay != nil {
+		return osw.compatOverlay.PrepareConnection(params)
+	}
+
+	overlays, err := osw.commonOverlays(params)
+	if err != nil {
+		return nil, err
+	}
+
+	// channel to carry events from the subforwarder monitors to
+	// the main goroutine
+	eventsChan := make(chan subForwarderEvent)
+
+	// channel to stop the main goroutine
+	stopChan := make(chan struct{})
+
+	fwd := &overlaySwitchForwarder{
+		remotePeer: params.RemotePeer,
+
+		best:       -1,
+		forwarders: make([]subForwarder, len(overlays)),
+		stopChan:   stopChan,
+
+		establishedChan: make(chan struct{}),
+		errorChan:       make(chan error, 1),
+	}
+
+	origSendControlMessage := params.SendControlMessage
+	for i, overlay := range overlays {
+		// Prefix control messages to indicate the relevant forwarder
+		index := i
+		params.SendControlMessage = func(tag byte, msg []byte) error {
+			xmsg := make([]byte, len(msg)+2)
+			xmsg[0] = byte(index)
+			xmsg[1] = tag
+			copy(xmsg[2:], msg)
+			return origSendControlMessage(mesh.ProtocolOverlayControlMsg, xmsg)
+		}
+
+		subConn, err := overlay.PrepareConnection(params)
+		if err != nil {
+			fwd.stopFrom(0)
+			return nil, err
+		}
+		subFwd := subConn.(OverlayForwarder)
+
+		subStopChan := make(chan struct{})
+		go monitorForwarder(i, eventsChan, subStopChan, subFwd)
+		fwd.forwarders[i] = subForwarder{
+			fwd:         subFwd,
+			overlayName: overlay.name,
+			stopChan:    subStopChan,
+		}
+	}
+
+	fwd.chooseBest()
+	go fwd.run(eventsChan, stopChan)
+	return fwd, nil
+}
+
+func monitorForwarder(index int, eventsChan chan<- subForwarderEvent, stopChan <-chan struct{}, fwd OverlayForwarder) {
+	establishedChan := fwd.EstablishedChannel()
+loop:
+	for {
+		e := subForwarderEvent{index: index}
+
+		select {
+		case <-establishedChan:
+			e.established = true
+			establishedChan = nil
+
+		case err := <-fwd.ErrorChannel():
+			e.err = err
+
+		case <-stopChan:
+			break loop
+		}
+
+		select {
+		case eventsChan <- e:
+		case <-stopChan:
+			break loop
+		}
+
+		if e.err != nil {
+			break loop
+		}
+	}
+
+	fwd.Stop()
+}
+
+func (fwd *overlaySwitchForwarder) run(eventsChan <-chan subForwarderEvent, stopChan <-chan struct{}) {
+loop:
+	for {
+		select {
+		case <-stopChan:
+			break loop
+
+		case e := <-eventsChan:
+			switch {
+			case e.established:
+				fwd.established(e.index)
+			case e.err != nil:
+				fwd.error(e.index, e.err)
+			}
+		}
+	}
+
+	fwd.lock.Lock()
+	defer fwd.lock.Unlock()
+	fwd.stopFrom(0)
+}
+
+func (fwd *overlaySwitchForwarder) established(index int) {
+	fwd.lock.Lock()
+	defer fwd.lock.Unlock()
+
+	fwd.forwarders[index].established = true
+
+	if !fwd.alreadyEstablished {
+		fwd.alreadyEstablished = true
+		close(fwd.establishedChan)
+	}
+
+	fwd.chooseBest()
+}
+
+func (fwd *overlaySwitchForwarder) logPrefix() string {
+	return fmt.Sprintf("overlay_switch ->[%s] ", fwd.remotePeer)
+}
+
+func (fwd *overlaySwitchForwarder) error(index int, err error) {
+	fwd.lock.Lock()
+	defer fwd.lock.Unlock()
+
+	log.Info(fwd.logPrefix(), fwd.forwarders[index].overlayName, " ", err)
+	fwd.forwarders[index].fwd = nil
+	fwd.chooseBest()
+}
+
+func (fwd *overlaySwitchForwarder) stopFrom(index int) {
+	for index < len(fwd.forwarders) {
+		subFwd := &fwd.forwarders[index]
+		if subFwd.fwd != nil {
+			subFwd.fwd = nil
+			close(subFwd.stopChan)
+		}
+		index++
+	}
+}
+
+func (fwd *overlaySwitchForwarder) chooseBest() {
+	// the most preferred established forwarder is the best
+	// otherwise, the most preferred working forwarder is the best
+	bestEstablished := -1
+	bestWorking := -1
+
+	for i := range fwd.forwarders {
+		subFwd := &fwd.forwarders[i]
+		if subFwd.fwd == nil {
+			continue
+		}
+
+		if bestWorking < 0 {
+			bestWorking = i
+		}
+
+		if bestEstablished < 0 && subFwd.established {
+			bestEstablished = i
+		}
+	}
+
+	best := bestEstablished
+	if best < 0 {
+		if bestWorking < 0 {
+			select {
+			case fwd.errorChan <- fmt.Errorf("no working forwarders to %s", fwd.remotePeer):
+			default:
+			}
+
+			return
+		}
+
+		best = bestWorking
+	}
+
+	if fwd.best != best {
+		fwd.best = best
+		log.Info(fwd.logPrefix(), "using ", fwd.forwarders[best].overlayName)
+	}
+}
+
+func (fwd *overlaySwitchForwarder) Confirm() {
+	var forwarders []OverlayForwarder
+
+	fwd.lock.Lock()
+	for _, subFwd := range fwd.forwarders {
+		if subFwd.fwd != nil {
+			forwarders = append(forwarders, subFwd.fwd)
+		}
+	}
+	fwd.lock.Unlock()
+
+	for _, subFwd := range forwarders {
+		subFwd.Confirm()
+	}
+}
+
+func (fwd *overlaySwitchForwarder) Forward(pk ForwardPacketKey) FlowOp {
+	fwd.lock.Lock()
+
+	if fwd.best >= 0 {
+		for i := fwd.best; i < len(fwd.forwarders); i++ {
+			best := fwd.forwarders[i].fwd
+			if best != nil {
+				fwd.lock.Unlock()
+				if op := best.Forward(pk); op != nil {
+					return op
+				}
+				fwd.lock.Lock()
+			}
+		}
+	}
+
+	fwd.lock.Unlock()
+
+	return DiscardingFlowOp{}
+}
+
+func (fwd *overlaySwitchForwarder) EstablishedChannel() <-chan struct{} {
+	return fwd.establishedChan
+}
+
+func (fwd *overlaySwitchForwarder) ErrorChannel() <-chan error {
+	return fwd.errorChan
+}
+
+func (fwd *overlaySwitchForwarder) Stop() {
+	fwd.lock.Lock()
+	defer fwd.lock.Unlock()
+	fwd.stopFrom(0)
+}
+
+func (fwd *overlaySwitchForwarder) ControlMessage(tag byte, msg []byte) {
+	fwd.lock.Lock()
+	subFwd := fwd.forwarders[msg[0]].fwd
+	fwd.lock.Unlock()
+	if subFwd != nil {
+		subFwd.ControlMessage(msg[1], msg[2:])
+	}
+}
+
+func (fwd *overlaySwitchForwarder) DisplayName() string {
+	var best OverlayForwarder
+
+	fwd.lock.Lock()
+	if fwd.best >= 0 {
+		best = fwd.forwarders[fwd.best].fwd
+	}
+	fwd.lock.Unlock()
+
+	if best != nil {
+		return best.DisplayName()
+	}
+
+	return "none"
+}

router/pcap.go

@@ -2,18 +2,44 @@ package router
 
 import (
 	"fmt"
+	"net"
+	"sync"
 
 	"github.com/google/gopacket/pcap"
 )
 
-type PcapIO struct {
-	handle *pcap.Handle
+type Pcap struct {
+	NonDiscardingFlowOp
+
+	iface *net.Interface
+	bufSz int
+
+	// The libpcap handle for writing packets. It's possible that a
+	// single handle could be used for reading and writing, but
+	// we'd have to examine the performance implications.
+	writeHandle *pcap.Handle
+
+	// pcap handles are single-threaded, so we need to lock around
+	// uses of writeHandle.
+	mutex sync.Mutex
+
+	// The libpcap handle for reading packets
+	readHandle *pcap.Handle
 }
 
-func NewPcapIO(ifName string, bufSz int) (PacketSourceSink, error) {
-	pio, err := newPcapIO(ifName, true, 65535, bufSz)
+func NewPcap(iface *net.Interface, bufSz int) (Bridge, error) {
+	wh, err := newPcapHandle(iface.Name, false, 0, 0)
 	if err != nil {
-		return pio, err
+		return nil, err
+	}
+
+	return &Pcap{iface: iface, bufSz: bufSz, writeHandle: wh}, nil
+}
+
+func (p *Pcap) StartConsumingPackets(consumer BridgeConsumer) error {
+	rh, err := newPcapHandle(p.iface.Name, true, 65535, p.bufSz)
+	if err != nil {
+		return err
 	}
 
 	// Under Linux, libpcap implements the SetDirection filtering
@@ -21,16 +47,25 @@ func NewPcapIO(ifName string, bufSz int) (PacketSourceSink, error) {
 	// packets inside the kernel.  We do this here rather than in
 	// newPcapIO because libpcap doesn't like this in combination
 	// with a 0 snaplen.
-	err = pio.handle.SetBPFFilter("inbound")
-	return pio, err
+	err = rh.SetBPFFilter("inbound")
+	if err != nil {
+		rh.Close()
+		return err
+	}
+
+	// readHandle is just for the benefit of Stats.
+	p.mutex.Lock()
+	defer p.mutex.Unlock()
+	if p.readHandle != nil {
+		panic("already consuming")
+	}
+
+	p.readHandle = rh
+	go p.sniff(rh, consumer)
+	return nil
 }
 
-func NewPcapO(ifName string) (po PacketSink, err error) {
-	po, err = newPcapIO(ifName, false, 0, 0)
-	return
-}
-
-func newPcapIO(ifName string, promisc bool, snaplen int, bufSz int) (handle *PcapIO, err error) {
+func newPcapHandle(ifName string, promisc bool, snaplen int, bufSz int) (handle *pcap.Handle, err error) {
 	inactive, err := pcap.NewInactiveHandle(ifName)
 	if err != nil {
 		return
@@ -61,38 +96,72 @@ func newPcapIO(ifName string, promisc bool, snaplen int, bufSz int) (handle *Pca
 	if err = inactive.SetBufferSize(bufSz); err != nil {
 		return
 	}
-	active, err := inactive.Activate()
+	handle, err = inactive.Activate()
 	if err != nil {
 		return
 	}
-	if err = active.SetDirection(pcap.DirectionIn); err != nil {
-		return
-	}
-	return &PcapIO{handle: active}, nil
-}
-
-func (pio *PcapIO) ReadPacket() (data []byte, err error) {
-	for {
-		data, _, err = pio.handle.ZeroCopyReadPacketData()
-		if err == nil || err != pcap.NextErrorTimeoutExpired {
-			break
-		}
-	}
+	err = handle.SetDirection(pcap.DirectionIn)
 	return
 }
 
-func (pio *PcapIO) WritePacket(data []byte) error {
-	return pio.handle.WritePacketData(data)
+func (p *Pcap) String() string {
+	return fmt.Sprint(p.iface.Name, " (via pcap)")
 }
 
-func (pio *PcapIO) Stats() map[string]int {
-	stats, err := pio.handle.Stats()
-	if err != nil {
-		return nil
+func (p *Pcap) InjectPacket(PacketKey) FlowOp {
+	return p
+}
+
+func (p *Pcap) Process(frame []byte, dec *EthernetDecoder, broadcast bool) {
+	p.mutex.Lock()
+	defer p.mutex.Unlock()
+	checkWarn(p.writeHandle.WritePacketData(frame))
+}
+
+func (p *Pcap) sniff(readHandle *pcap.Handle, consumer BridgeConsumer) {
+	dec := NewEthernetDecoder()
+
+	for {
+		pkt, _, err := readHandle.ZeroCopyReadPacketData()
+		if err == pcap.NextErrorTimeoutExpired {
+			continue
+		}
+
+		checkFatal(err)
+		dec.DecodeLayers(pkt)
+		if len(dec.decoded) == 0 {
+			continue
+		}
+
+		if fop := consumer(dec.PacketKey()); !fop.Discards() {
+			// We are handing over the frame to
+			// forwarders, so we need to make a copy of it
+			// in order to prevent the next capture from
+			// overwriting the data
+			pktLen := len(pkt)
+			pktCopy := make([]byte, pktLen, pktLen)
+			copy(pktCopy, pkt)
+
+			fop.Process(pktCopy, dec, false)
+		}
 	}
-	res := make(map[string]int)
-	res["PacketsReceived"] = stats.PacketsReceived
-	res["PacketsDropped"] = stats.PacketsDropped
-	res["PacketsIfDropped"] = stats.PacketsIfDropped
-	return res
+}
+
+func (p *Pcap) Stats() map[string]int {
+	p.mutex.Lock()
+	rh := p.readHandle
+	p.mutex.Unlock()
+
+	if rh != nil {
+		stats, err := rh.Stats()
+		if err == nil {
+			return map[string]int{
+				"PacketsReceived":  stats.PacketsReceived,
+				"PacketsDropped":   stats.PacketsDropped,
+				"PacketsIfDropped": stats.PacketsIfDropped,
+			}
+		}
+	}
+
+	return nil
 }

router/peers.go

@@ -1,316 +0,0 @@
-package router
-
-import (
-	"bytes"
-	"encoding/gob"
-	"io"
-	"sync"
-)
-
-type Peers struct {
-	sync.RWMutex
-	ourself *LocalPeer
-	table   map[PeerName]*Peer
-	onGC    []func(*Peer)
-}
-
-type UnknownPeerError struct {
-	Name PeerName
-}
-
-type NameCollisionError struct {
-	Name PeerName
-}
-
-type PeerNameSet map[PeerName]struct{}
-
-type PeerSummary struct {
-	NameByte []byte
-	NickName string
-	UID      PeerUID
-	Version  uint64
-}
-
-type ConnectionSummary struct {
-	NameByte      []byte
-	RemoteTCPAddr string
-	Outbound      bool
-	Established   bool
-}
-
-func NewPeers(ourself *LocalPeer) *Peers {
-	return &Peers{ourself: ourself, table: make(map[PeerName]*Peer)}
-}
-
-func (peers *Peers) OnGC(callback func(*Peer)) {
-	peers.Lock()
-	defer peers.Unlock()
-	peers.onGC = append(peers.onGC, callback)
-}
-
-func (peers *Peers) invokeOnGCCallbacks(removed []*Peer) {
-	for _, callback := range peers.onGC {
-		for _, peer := range removed {
-			callback(peer)
-		}
-	}
-}
-
-func (peers *Peers) FetchWithDefault(peer *Peer) *Peer {
-	peers.Lock()
-	defer peers.Unlock()
-	if existingPeer, found := peers.table[peer.Name]; found {
-		existingPeer.localRefCount++
-		return existingPeer
-	}
-	peers.table[peer.Name] = peer
-	peer.localRefCount++
-	return peer
-}
-
-func (peers *Peers) Fetch(name PeerName) *Peer {
-	peers.RLock()
-	defer peers.RUnlock()
-	return peers.table[name]
-}
-
-func (peers *Peers) FetchAndAddRef(name PeerName) *Peer {
-	peers.Lock()
-	defer peers.Unlock()
-	peer := peers.table[name]
-	if peer != nil {
-		peer.localRefCount++
-	}
-	return peer
-}
-
-func (peers *Peers) Dereference(peer *Peer) {
-	peers.Lock()
-	defer peers.Unlock()
-	peer.localRefCount--
-}
-
-func (peers *Peers) ForEach(fun func(*Peer)) {
-	peers.RLock()
-	defer peers.RUnlock()
-	for _, peer := range peers.table {
-		fun(peer)
-	}
-}
-
-// Merge an incoming update with our own topology.
-//
-// We add peers hitherto unknown to us, and update peers for which the
-// update contains a more recent version than known to us. The return
-// value is a) a representation of the received update, and b) an
-// "improved" update containing just these new/updated elements.
-func (peers *Peers) ApplyUpdate(update []byte) (PeerNameSet, PeerNameSet, error) {
-	peers.Lock()
-
-	newPeers, decodedUpdate, decodedConns, err := peers.decodeUpdate(update)
-	if err != nil {
-		peers.Unlock()
-		return nil, nil, err
-	}
-
-	// By this point, we know the update doesn't refer to any peers we
-	// have no knowledge of. We can now apply the update. Start by
-	// adding in any new peers into the cache.
-	for name, newPeer := range newPeers {
-		peers.table[name] = newPeer
-	}
-
-	// Now apply the updates
-	newUpdate := peers.applyUpdate(decodedUpdate, decodedConns)
-	removed := peers.garbageCollect()
-	for _, peerRemoved := range removed {
-		delete(newUpdate, peerRemoved.Name)
-	}
-
-	// Don't need to hold peers lock any longer
-	peers.Unlock()
-	peers.invokeOnGCCallbacks(removed)
-
-	updateNames := make(PeerNameSet)
-	for _, peer := range decodedUpdate {
-		updateNames[peer.Name] = void
-	}
-	return updateNames, setFromPeersMap(newUpdate), nil
-}
-
-func (peers *Peers) Names() PeerNameSet {
-	peers.RLock()
-	defer peers.RUnlock()
-	return setFromPeersMap(peers.table)
-}
-
-func (peers *Peers) EncodePeers(names PeerNameSet) []byte {
-	buf := new(bytes.Buffer)
-	enc := gob.NewEncoder(buf)
-	peers.RLock()
-	defer peers.RUnlock()
-	for name := range names {
-		if peer, found := peers.table[name]; found {
-			if peer == peers.ourself.Peer {
-				peers.ourself.Encode(enc)
-			} else {
-				peer.Encode(enc)
-			}
-		}
-	}
-	return buf.Bytes()
-}
-
-func (peers *Peers) GarbageCollect() []*Peer {
-	peers.Lock()
-	removed := peers.garbageCollect()
-	peers.Unlock()
-	peers.invokeOnGCCallbacks(removed)
-	return removed
-}
-
-func (peers *Peers) garbageCollect() []*Peer {
-	removed := []*Peer{}
-	peers.ourself.RLock()
-	_, reached := peers.ourself.Routes(nil, false)
-	peers.ourself.RUnlock()
-	for name, peer := range peers.table {
-		if _, found := reached[peer.Name]; !found && peer.localRefCount == 0 {
-			delete(peers.table, name)
-			removed = append(removed, peer)
-		}
-	}
-	return removed
-}
-
-func setFromPeersMap(peers map[PeerName]*Peer) PeerNameSet {
-	names := make(PeerNameSet)
-	for name := range peers {
-		names[name] = void
-	}
-	return names
-}
-
-func (peers *Peers) decodeUpdate(update []byte) (newPeers map[PeerName]*Peer, decodedUpdate []*Peer, decodedConns [][]ConnectionSummary, err error) {
-	newPeers = make(map[PeerName]*Peer)
-	decodedUpdate = []*Peer{}
-	decodedConns = [][]ConnectionSummary{}
-
-	decoder := gob.NewDecoder(bytes.NewReader(update))
-
-	for {
-		peerSummary, connSummaries, decErr := decodePeer(decoder)
-		if decErr == io.EOF {
-			break
-		} else if decErr != nil {
-			err = decErr
-			return
-		}
-		name := PeerNameFromBin(peerSummary.NameByte)
-		newPeer := NewPeer(name, peerSummary.NickName, peerSummary.UID, peerSummary.Version)
-		decodedUpdate = append(decodedUpdate, newPeer)
-		decodedConns = append(decodedConns, connSummaries)
-		existingPeer, found := peers.table[name]
-		if !found {
-			newPeers[name] = newPeer
-		} else if existingPeer.UID != newPeer.UID {
-			err = NameCollisionError{Name: newPeer.Name}
-			return
-		}
-	}
-
-	for _, connSummaries := range decodedConns {
-		for _, connSummary := range connSummaries {
-			remoteName := PeerNameFromBin(connSummary.NameByte)
-			if _, found := newPeers[remoteName]; found {
-				continue
-			}
-			if _, found := peers.table[remoteName]; found {
-				continue
-			}
-			// Update refers to a peer which we have no knowledge
-			// of. Thus we can't apply the update. Abort.
-			err = UnknownPeerError{remoteName}
-			return
-		}
-	}
-	return
-}
-
-func (peers *Peers) applyUpdate(decodedUpdate []*Peer, decodedConns [][]ConnectionSummary) map[PeerName]*Peer {
-	newUpdate := make(map[PeerName]*Peer)
-	for idx, newPeer := range decodedUpdate {
-		connSummaries := decodedConns[idx]
-		name := newPeer.Name
-		// guaranteed to find peer in the peers.table
-		peer := peers.table[name]
-		if peer != newPeer &&
-			(peer == peers.ourself.Peer || peer.version >= newPeer.version) {
-			// Nobody but us updates us. And if we know more about a
-			// peer than what's in the the update, we ignore the
-			// latter.
-			continue
-		}
-		// If we're here, either it was a new peer, or the update has
-		// more info about the peer than we do. Either case, we need
-		// to set version and conns and include the updated peer in
-		// the outgoing update.
-
-		// Can peer have been updated by anyone else in the mean time?
-		// No - we know that peer is not ourself, so the only prospect
-		// for an update would be someone else calling
-		// router.Peers.ApplyUpdate. But ApplyUpdate takes the Lock on
-		// the router.Peers, so there can be no race here.
-		peer.version = newPeer.version
-		peer.connections = makeConnsMap(peer, connSummaries, peers.table)
-		newUpdate[name] = peer
-	}
-	return newUpdate
-}
-
-func (peer *Peer) Encode(enc *gob.Encoder) {
-	checkFatal(enc.Encode(PeerSummary{
-		peer.NameByte,
-		peer.NickName,
-		peer.UID,
-		peer.version}))
-
-	connSummaries := []ConnectionSummary{}
-	for _, conn := range peer.connections {
-		connSummaries = append(connSummaries, ConnectionSummary{
-			conn.Remote().NameByte,
-			conn.RemoteTCPAddr(),
-			conn.Outbound(),
-			conn.Established(),
-		})
-	}
-
-	checkFatal(enc.Encode(connSummaries))
-}
-
-func (peer *LocalPeer) Encode(enc *gob.Encoder) {
-	peer.RLock()
-	defer peer.RUnlock()
-	peer.Peer.Encode(enc)
-}
-
-func decodePeer(dec *gob.Decoder) (peerSummary PeerSummary, connSummaries []ConnectionSummary, err error) {
-	if err = dec.Decode(&peerSummary); err != nil {
-		return
-	}
-	if err = dec.Decode(&connSummaries); err != nil {
-		return
-	}
-	return
-}
-
-func makeConnsMap(peer *Peer, connSummaries []ConnectionSummary, table map[PeerName]*Peer) map[PeerName]Connection {
-	conns := make(map[PeerName]Connection)
-	for _, connSummary := range connSummaries {
-		name := PeerNameFromBin(connSummary.NameByte)
-		remotePeer := table[name]
-		conn := NewRemoteConnection(peer, remotePeer, connSummary.RemoteTCPAddr, connSummary.Outbound, connSummary.Established)
-		conns[name] = conn
-	}
-	return conns
-}

router/peers_test.go

@@ -1,114 +0,0 @@
-package router
-
-import (
-	"fmt"
-	"math/rand"
-	"testing"
-
-	"github.com/stretchr/testify/require"
-)
-
-// TODO we should also test:
-//
-// - applying an incremental update, including the case where that
-//   leads to an UnknownPeerError
-//
-// - the "improved update" calculation
-//
-// - non-gc of peers that are only referenced locally
-
-func newNode(name PeerName) (*Peer, *Peers) {
-	peer := NewLocalPeer(name, "", nil)
-	peers := NewPeers(peer)
-	peers.FetchWithDefault(peer.Peer)
-	return peer.Peer, peers
-}
-
-// Check that ApplyUpdate copies the whole topology from peers
-func checkApplyUpdate(t *testing.T, peers *Peers) {
-	dummyName, _ := PeerNameFromString("99:00:00:01:00:00")
-	// We need a new node outside of the network, with a connection
-	// into it.
-	_, testBedPeers := newNode(dummyName)
-	testBedPeers.AddTestConnection(peers.ourself.Peer)
-	testBedPeers.ApplyUpdate(peers.EncodePeers(peers.Names()))
-
-	checkTopologyPeers(t, true, testBedPeers.allPeersExcept(dummyName), peers.allPeers()...)
-}
-
-func TestPeersEncoding(t *testing.T) {
-	const numNodes = 20
-	const numIters = 1000
-	var peer [numNodes]*Peer
-	var ps [numNodes]*Peers
-	for i := 0; i < numNodes; i++ {
-		name, _ := PeerNameFromString(fmt.Sprintf("%02d:00:00:01:00:00", i))
-		peer[i], ps[i] = newNode(name)
-	}
-
-	var conns []struct{ from, to int }
-	for i := 0; i < numIters; i++ {
-		oper := rand.Intn(2)
-		switch oper {
-		case 0:
-			from, to := rand.Intn(numNodes), rand.Intn(numNodes)
-			if from != to {
-				if _, found := peer[from].connections[peer[to].Name]; !found {
-					ps[from].AddTestConnection(peer[to])
-					conns = append(conns, struct{ from, to int }{from, to})
-					checkApplyUpdate(t, ps[from])
-				}
-			}
-		case 1:
-			if len(conns) > 0 {
-				n := rand.Intn(len(conns))
-				c := conns[n]
-				ps[c.from].DeleteTestConnection(peer[c.to])
-				ps[c.from].GarbageCollect()
-				checkApplyUpdate(t, ps[c.from])
-				conns = append(conns[:n], conns[n+1:]...)
-			}
-		}
-	}
-}
-
-func TestPeersGarbageCollection(t *testing.T) {
-	const (
-		peer1NameString = "01:00:00:01:00:00"
-		peer2NameString = "02:00:00:02:00:00"
-		peer3NameString = "03:00:00:03:00:00"
-	)
-	var (
-		peer1Name, _ = PeerNameFromString(peer1NameString)
-		peer2Name, _ = PeerNameFromString(peer2NameString)
-		peer3Name, _ = PeerNameFromString(peer3NameString)
-	)
-
-	// Create some peers with some connections to each other
-	p1, ps1 := newNode(peer1Name)
-	p2, ps2 := newNode(peer2Name)
-	p3, ps3 := newNode(peer3Name)
-	ps1.AddTestConnection(p2)
-	ps2.AddTestRemoteConnection(p1, p2)
-	ps2.AddTestConnection(p1)
-	ps2.AddTestConnection(p3)
-	ps3.AddTestConnection(p1)
-	ps1.AddTestConnection(p3)
-	ps2.AddTestRemoteConnection(p1, p3)
-	ps2.AddTestRemoteConnection(p3, p1)
-
-	// Every peer is referenced, so nothing should be dropped
-	require.Empty(t, ps1.GarbageCollect(), "peers removed")
-	require.Empty(t, ps2.GarbageCollect(), "peers removed")
-	require.Empty(t, ps3.GarbageCollect(), "peers removed")
-
-	// Drop the connection from 2 to 3, and 3 isn't garbage-collected
-	// because 1 has a connection to 3
-	ps2.DeleteTestConnection(p3)
-	require.Empty(t, ps2.GarbageCollect(), "peers removed")
-
-	// Drop the connection from 1 to 3, and 3 will get removed by
-	// garbage-collection
-	ps1.DeleteTestConnection(p3)
-	checkPeerArray(t, ps1.GarbageCollect(), p3)
-}

router/router.go

@@ -1,418 +0,0 @@
-package router
-
-import (
-	"bytes"
-	"encoding/binary"
-	"fmt"
-	"io"
-	"net"
-	"sync"
-	"syscall"
-	"time"
-)
-
-const (
-	macMaxAge        = 10 * time.Minute       // [1]
-	acceptMaxTokens  = 100                    // [2]
-	acceptTokenDelay = 100 * time.Millisecond // [3]
-)
-
-// [1] should be greater than typical ARP cache expiries, i.e. > 3/2 *
-// /proc/sys/net/ipv4_neigh/*/base_reachable_time_ms on Linux
-
-// [2] capacity of token bucket for rate limiting accepts
-
-// [3] control rate at which new tokens are added to the bucket
-
-type LogFrameFunc func(string, []byte, *EthernetDecoder)
-
-type Config struct {
-	Port               int
-	ProtocolMinVersion byte
-	Iface              *net.Interface
-	Password           []byte
-	ConnLimit          int
-	PeerDiscovery      bool
-	BufSz              int
-	LogFrame           LogFrameFunc
-}
-
-type Router struct {
-	Config
-	Ourself         *LocalPeer
-	Macs            *MacCache
-	Peers           *Peers
-	Routes          *Routes
-	ConnectionMaker *ConnectionMaker
-	gossipLock      sync.RWMutex
-	gossipChannels  GossipChannels
-	TopologyGossip  Gossip
-	PacketSource    PacketSource
-	UDPListener     *net.UDPConn
-	acceptLimiter   *TokenBucket
-}
-
-type PacketSource interface {
-	ReadPacket() ([]byte, error)
-	Stats() map[string]int
-}
-
-type PacketSink interface {
-	WritePacket([]byte) error
-}
-
-type PacketSourceSink interface {
-	PacketSource
-	PacketSink
-}
-
-func NewRouter(config Config, name PeerName, nickName string) *Router {
-	router := &Router{Config: config, gossipChannels: make(GossipChannels)}
-	onMacExpiry := func(mac net.HardwareAddr, peer *Peer) {
-		log.Println("Expired MAC", mac, "at", peer)
-	}
-	onPeerGC := func(peer *Peer) {
-		router.Macs.Delete(peer)
-		log.Println("Removed unreachable peer", peer)
-	}
-	router.Ourself = NewLocalPeer(name, nickName, router)
-	router.Macs = NewMacCache(macMaxAge, onMacExpiry)
-	router.Peers = NewPeers(router.Ourself)
-	router.Peers.OnGC(onPeerGC)
-	router.Peers.FetchWithDefault(router.Ourself.Peer)
-	router.Routes = NewRoutes(router.Ourself, router.Peers)
-	router.ConnectionMaker = NewConnectionMaker(router.Ourself, router.Peers, router.Port, router.PeerDiscovery)
-	router.TopologyGossip = router.NewGossip("topology", router)
-	router.acceptLimiter = NewTokenBucket(acceptMaxTokens, acceptTokenDelay)
-	return router
-}
-
-// Start listening for TCP connections, locally captured packets, and
-// packets forwarded over UDP.  This is separate from NewRouter so
-// that gossipers can register before we start forming connections.
-func (router *Router) Start() {
-	// we need two pcap handles since they aren't thread-safe
-	var pio PacketSourceSink
-	var po PacketSink
-	var err error
-	if router.Iface != nil {
-		pio, err = NewPcapIO(router.Iface.Name, router.BufSz)
-		checkFatal(err)
-		po, err = NewPcapO(router.Iface.Name)
-		checkFatal(err)
-	}
-	router.UDPListener = router.listenUDP(router.Port, po)
-	router.listenTCP(router.Port)
-	if pio != nil {
-		router.PacketSource = pio
-		router.sniff(pio)
-	}
-}
-
-func (router *Router) Stop() error {
-	// TODO: perform graceful shutdown...
-	return nil
-}
-
-func (router *Router) UsingPassword() bool {
-	return router.Password != nil
-}
-
-func (router *Router) sniff(pio PacketSourceSink) {
-	log.Println("Sniffing traffic on", router.Iface)
-
-	dec := NewEthernetDecoder()
-	mac := router.Iface.HardwareAddr
-	if router.Macs.Enter(mac, router.Ourself.Peer) {
-		log.Println("Discovered our MAC", mac)
-	}
-	go func() {
-		for {
-			pkt, err := pio.ReadPacket()
-			checkFatal(err)
-			router.LogFrame("Sniffed", pkt, nil)
-			router.handleCapturedPacket(pkt, dec, pio)
-		}
-	}()
-}
-
-func (router *Router) handleCapturedPacket(frameData []byte, dec *EthernetDecoder, po PacketSink) {
-	dec.DecodeLayers(frameData)
-	decodedLen := len(dec.decoded)
-	if decodedLen == 0 {
-		return
-	}
-	srcMac := dec.Eth.SrcMAC
-	srcPeer, found := router.Macs.Lookup(srcMac)
-	// We need to filter out frames we injected ourselves. For such
-	// frames, the srcMAC will have been recorded as associated with a
-	// different peer.
-	if found && srcPeer != router.Ourself.Peer {
-		return
-	}
-	if router.Macs.Enter(srcMac, router.Ourself.Peer) {
-		log.Println("Discovered local MAC", srcMac)
-	}
-	if dec.DropFrame() {
-		return
-	}
-	dstMac := dec.Eth.DstMAC
-	dstPeer, found := router.Macs.Lookup(dstMac)
-	if found && dstPeer == router.Ourself.Peer {
-		return
-	}
-	router.LogFrame("Forwarding", frameData, dec)
-
-	// at this point we are handing over the frame to forwarders, so
-	// we need to make a copy of it in order to prevent the next
-	// capture from overwriting the data
-	frameLen := len(frameData)
-	frameCopy := make([]byte, frameLen, frameLen)
-	copy(frameCopy, frameData)
-
-	// If we don't know which peer corresponds to the dest MAC,
-	// broadcast it.
-	if !found {
-		router.Ourself.Broadcast(frameCopy, dec)
-		return
-	}
-
-	err := router.Ourself.Forward(dstPeer, frameCopy, dec)
-	if ftbe, ok := err.(FrameTooBigError); ok {
-		err = dec.sendICMPFragNeeded(ftbe.EPMTU, po.WritePacket)
-	}
-	checkWarn(err)
-}
-
-func (router *Router) listenTCP(localPort int) {
-	localAddr, err := net.ResolveTCPAddr("tcp4", fmt.Sprint(":", localPort))
-	checkFatal(err)
-	ln, err := net.ListenTCP("tcp4", localAddr)
-	checkFatal(err)
-	go func() {
-		defer ln.Close()
-		for {
-			tcpConn, err := ln.AcceptTCP()
-			if err != nil {
-				log.Errorln(err)
-				continue
-			}
-			router.acceptTCP(tcpConn)
-			router.acceptLimiter.Wait()
-		}
-	}()
-}
-
-func (router *Router) acceptTCP(tcpConn *net.TCPConn) {
-	// someone else is dialing us, so our udp sender is the conn
-	// on router.Port and we wait for them to send us something on UDP to
-	// start.
-	remoteAddrStr := tcpConn.RemoteAddr().String()
-	log.Printf("->[%s] connection accepted", remoteAddrStr)
-	connRemote := NewRemoteConnection(router.Ourself.Peer, nil, remoteAddrStr, false, false)
-	StartLocalConnection(connRemote, tcpConn, nil, router, true)
-}
-
-func (router *Router) listenUDP(localPort int, po PacketSink) *net.UDPConn {
-	localAddr, err := net.ResolveUDPAddr("udp4", fmt.Sprint(":", localPort))
-	checkFatal(err)
-	conn, err := net.ListenUDP("udp4", localAddr)
-	checkFatal(err)
-	f, err := conn.File()
-	defer f.Close()
-	checkFatal(err)
-	fd := int(f.Fd())
-	// This one makes sure all packets we send out do not have DF set on them.
-	err = syscall.SetsockoptInt(fd, syscall.IPPROTO_IP, syscall.IP_MTU_DISCOVER, syscall.IP_PMTUDISC_DONT)
-	checkFatal(err)
-	go router.udpReader(conn, po)
-	return conn
-}
-
-func (router *Router) udpReader(conn *net.UDPConn, po PacketSink) {
-	defer conn.Close()
-	dec := NewEthernetDecoder()
-	buf := make([]byte, MaxUDPPacketSize)
-	for {
-		n, sender, err := conn.ReadFromUDP(buf)
-		if err == io.EOF {
-			return
-		} else if err != nil {
-			log.Warnln("ignoring UDP read error", err)
-			continue
-		} else if n < NameSize {
-			log.Warnln("ignoring too short UDP packet from", sender)
-			continue
-		}
-		name := PeerNameFromBin(buf[:NameSize])
-		packet := make([]byte, n-NameSize)
-		copy(packet, buf[NameSize:n])
-		peerConn, found := router.Ourself.ConnectionTo(name)
-		if !found {
-			continue
-		}
-		relayConn, ok := peerConn.(*LocalConnection)
-		if !ok {
-			continue
-		}
-		if err := relayConn.Decryptor.IterateFrames(packet, router.handleUDPPacketFunc(relayConn, dec, sender, po)); err != nil {
-			// Errors during UDP packet decoding / processing are
-			// non-fatal. One common cause is that we receive and
-			// attempt to decrypt a "stray" packet. This can actually
-			// happen quite easily if there is some connection churn
-			// between two peers. After all, UDP isn't a
-			// connection-oriented protocol, yet we pretend it is.
-			//
-			// If anything really is seriously, unrecoverably amiss
-			// with a connection, that will typically result in missed
-			// heartbeats and the connection getting shut down because
-			// of that.
-			relayConn.Log(err)
-		}
-	}
-}
-
-func (router *Router) handleUDPPacketFunc(relayConn *LocalConnection, dec *EthernetDecoder, sender *net.UDPAddr, po PacketSink) FrameConsumer {
-	return func(srcNameByte, dstNameByte []byte, frame []byte) {
-		srcPeer := router.Peers.Fetch(PeerNameFromBin(srcNameByte))
-		dstPeer := router.Peers.Fetch(PeerNameFromBin(dstNameByte))
-		if srcPeer == nil || dstPeer == nil {
-			return
-		}
-
-		dec.DecodeLayers(frame)
-		decodedLen := len(dec.decoded)
-		if decodedLen == 0 {
-			return
-		}
-		// Handle special frames produced internally (rather than
-		// captured/forwarded) by the remote router.
-		//
-		// We really shouldn't be decoding these above, since they are
-		// not genuine Ethernet frames. However, it is actually more
-		// efficient to do so, as we want to optimise for the common
-		// (i.e. non-special) frames. These always need decoding, and
-		// detecting special frames is cheaper post decoding than pre.
-		if decodedLen == 1 && dec.IsSpecial() {
-			if srcPeer == relayConn.Remote() && dstPeer == router.Ourself.Peer {
-				handleSpecialFrame(relayConn, sender, frame)
-			}
-			return
-		}
-
-		if dstPeer != router.Ourself.Peer {
-			// it's not for us, we're just relaying it
-			router.LogFrame("Relaying", frame, dec)
-			err := router.Ourself.Relay(srcPeer, dstPeer, frame, dec)
-			if ftbe, ok := err.(FrameTooBigError); ok {
-				err = dec.sendICMPFragNeeded(ftbe.EPMTU, func(icmpFrame []byte) error {
-					return router.Ourself.Forward(srcPeer, icmpFrame, nil)
-				})
-			}
-
-			checkWarn(err)
-			return
-		}
-
-		srcMac := dec.Eth.SrcMAC
-		dstMac := dec.Eth.DstMAC
-
-		if router.Macs.Enter(srcMac, srcPeer) {
-			log.Println("Discovered remote MAC", srcMac, "at", srcPeer)
-		}
-		if po != nil {
-			router.LogFrame("Injecting", frame, dec)
-			checkWarn(po.WritePacket(frame))
-		}
-
-		dstPeer, found := router.Macs.Lookup(dstMac)
-		if !found || dstPeer != router.Ourself.Peer {
-			router.LogFrame("Relaying broadcast", frame, dec)
-			router.Ourself.RelayBroadcast(srcPeer, frame, dec)
-		}
-	}
-}
-
-func handleSpecialFrame(relayConn *LocalConnection, sender *net.UDPAddr, frame []byte) {
-	frameLen := len(frame)
-	switch {
-	case frameLen == EthernetOverhead+8:
-		relayConn.ReceivedHeartbeat(sender, binary.BigEndian.Uint64(frame[EthernetOverhead:]))
-	case frameLen == FragTestSize && bytes.Equal(frame, FragTest):
-		relayConn.SendProtocolMsg(ProtocolMsg{ProtocolFragmentationReceived, nil})
-	case frameLen == PMTUDiscoverySize && bytes.Equal(frame, PMTUDiscovery):
-	default:
-		frameLenBytes := []byte{0, 0}
-		binary.BigEndian.PutUint16(frameLenBytes, uint16(frameLen-EthernetOverhead))
-		relayConn.SendProtocolMsg(ProtocolMsg{ProtocolPMTUVerified, frameLenBytes})
-	}
-}
-
-// Gossiper methods - the Router is the topology Gossiper
-
-type TopologyGossipData struct {
-	peers  *Peers
-	update PeerNameSet
-}
-
-func NewTopologyGossipData(peers *Peers, update ...*Peer) *TopologyGossipData {
-	names := make(PeerNameSet)
-	for _, p := range update {
-		names[p.Name] = void
-	}
-	return &TopologyGossipData{peers: peers, update: names}
-}
-
-func (d *TopologyGossipData) Merge(other GossipData) {
-	for name := range other.(*TopologyGossipData).update {
-		d.update[name] = void
-	}
-}
-
-func (d *TopologyGossipData) Encode() [][]byte {
-	return [][]byte{d.peers.EncodePeers(d.update)}
-}
-
-func (router *Router) OnGossipUnicast(sender PeerName, msg []byte) error {
-	return fmt.Errorf("unexpected topology gossip unicast: %v", msg)
-}
-
-func (router *Router) OnGossipBroadcast(_ PeerName, update []byte) (GossipData, error) {
-	origUpdate, _, err := router.applyTopologyUpdate(update)
-	if err != nil || len(origUpdate) == 0 {
-		return nil, err
-	}
-	return &TopologyGossipData{peers: router.Peers, update: origUpdate}, nil
-}
-
-func (router *Router) Gossip() GossipData {
-	return &TopologyGossipData{peers: router.Peers, update: router.Peers.Names()}
-}
-
-func (router *Router) OnGossip(update []byte) (GossipData, error) {
-	_, newUpdate, err := router.applyTopologyUpdate(update)
-	if err != nil || len(newUpdate) == 0 {
-		return nil, err
-	}
-	return &TopologyGossipData{peers: router.Peers, update: newUpdate}, nil
-}
-
-func (router *Router) applyTopologyUpdate(update []byte) (PeerNameSet, PeerNameSet, error) {
-	origUpdate, newUpdate, err := router.Peers.ApplyUpdate(update)
-	if _, ok := err.(UnknownPeerError); err != nil && ok {
-		// That update contained a reference to a peer which wasn't
-		// itself included in the update, and we didn't know about
-		// already. We ignore this; eventually we should receive an
-		// update containing a complete topology.
-		log.Println("Topology gossip:", err)
-		return nil, nil, nil
-	}
-	if err != nil {
-		return nil, nil, err
-	}
-	if len(newUpdate) > 0 {
-		router.ConnectionMaker.Refresh()
-		router.Routes.Recalculate()
-	}
-	return origUpdate, newUpdate, nil
-}

router/udp_sender.go

@@ -1,138 +0,0 @@
-package router
-
-import (
-	"net"
-	"syscall"
-
-	"github.com/google/gopacket"
-	"github.com/google/gopacket/layers"
-)
-
-type UDPSender interface {
-	Send([]byte) error
-	Shutdown() error
-}
-
-type SimpleUDPSender struct {
-	conn    *LocalConnection
-	udpConn *net.UDPConn
-}
-
-type RawUDPSender struct {
-	ipBuf     gopacket.SerializeBuffer
-	opts      gopacket.SerializeOptions
-	udpHeader *layers.UDP
-	socket    *net.IPConn
-	conn      *LocalConnection
-}
-
-type MsgTooBigError struct {
-	PMTU int // actual pmtu, i.e. what the kernel told us
-}
-
-func NewSimpleUDPSender(conn *LocalConnection) *SimpleUDPSender {
-	return &SimpleUDPSender{udpConn: conn.Router.UDPListener, conn: conn}
-}
-
-func (sender *SimpleUDPSender) Send(msg []byte) error {
-	_, err := sender.udpConn.WriteToUDP(msg, sender.conn.RemoteUDPAddr())
-	return err
-}
-
-func (sender *SimpleUDPSender) Shutdown() error {
-	return nil
-}
-
-func NewRawUDPSender(conn *LocalConnection) (*RawUDPSender, error) {
-	ipSocket, err := dialIP(conn)
-	if err != nil {
-		return nil, err
-	}
-	udpHeader := &layers.UDP{SrcPort: layers.UDPPort(conn.Router.Port)}
-	ipBuf := gopacket.NewSerializeBuffer()
-	opts := gopacket.SerializeOptions{
-		FixLengths: true,
-		// UDP header is calculated with a phantom IP
-		// header. Yes, it's totally nuts. Thankfully, for UDP
-		// over IPv4, the checksum is optional. It's not
-		// optional for IPv6, but we'll ignore that for
-		// now. TODO
-		ComputeChecksums: false}
-
-	return &RawUDPSender{
-		ipBuf:     ipBuf,
-		opts:      opts,
-		udpHeader: udpHeader,
-		socket:    ipSocket,
-		conn:      conn}, nil
-}
-
-func (sender *RawUDPSender) Send(msg []byte) error {
-	payload := gopacket.Payload(msg)
-	sender.udpHeader.DstPort = layers.UDPPort(sender.conn.RemoteUDPAddr().Port)
-
-	err := gopacket.SerializeLayers(sender.ipBuf, sender.opts, sender.udpHeader, &payload)
-	if err != nil {
-		return err
-	}
-	packet := sender.ipBuf.Bytes()
-	_, err = sender.socket.Write(packet)
-	if err == nil || PosixError(err) != syscall.EMSGSIZE {
-		return err
-	}
-	f, err := sender.socket.File()
-	if err != nil {
-		return err
-	}
-	defer f.Close()
-	fd := int(f.Fd())
-	log.Println("EMSGSIZE on send, expecting PMTU update (IP packet was",
-		len(packet), "bytes, payload was", len(msg), "bytes)")
-	pmtu, err := syscall.GetsockoptInt(fd, syscall.IPPROTO_IP, syscall.IP_MTU)
-	if err != nil {
-		return err
-	}
-	return MsgTooBigError{PMTU: pmtu}
-}
-
-func (sender *RawUDPSender) Shutdown() error {
-	defer func() { sender.socket = nil }()
-	return sender.socket.Close()
-}
-
-func dialIP(conn *LocalConnection) (*net.IPConn, error) {
-	ipLocalAddr, err := ipAddr(conn.TCPConn.LocalAddr())
-	if err != nil {
-		return nil, err
-	}
-	ipRemoteAddr, err := ipAddr(conn.TCPConn.RemoteAddr())
-	if err != nil {
-		return nil, err
-	}
-	ipSocket, err := net.DialIP("ip4:UDP", ipLocalAddr, ipRemoteAddr)
-	if err != nil {
-		return nil, err
-	}
-	f, err := ipSocket.File()
-	if err != nil {
-		return nil, err
-	}
-	defer f.Close()
-	fd := int(f.Fd())
-	// This Makes sure all packets we send out have DF set on them.
-	err = syscall.SetsockoptInt(fd, syscall.IPPROTO_IP, syscall.IP_MTU_DISCOVER, syscall.IP_PMTUDISC_DO)
-	if err != nil {
-		return nil, err
-	}
-	return ipSocket, nil
-}
-
-func ipAddr(addr net.Addr) (*net.IPAddr, error) {
-	host, _, err := net.SplitHostPort(addr.String())
-	if err != nil {
-		return nil, err
-	}
-	return &net.IPAddr{
-		IP:   net.ParseIP(host),
-		Zone: ""}, nil
-}

router/utils.go

@@ -1,115 +0,0 @@
-package router
-
-import (
-	"bytes"
-	"crypto/rand"
-	"encoding/gob"
-	"fmt"
-	"net"
-
-	"github.com/weaveworks/weave/common"
-)
-
-var log = common.Log
-
-var void = struct{}{}
-
-func checkFatal(e error) {
-	if e != nil {
-		log.Fatal(e)
-	}
-}
-
-func checkWarn(e error) {
-	if e != nil {
-		log.Warnln(e)
-	}
-}
-
-func PosixError(err error) error {
-	if err == nil {
-		return nil
-	}
-	operr, ok := err.(*net.OpError)
-	if !ok {
-		return nil
-	}
-	return operr.Err
-}
-
-func (mtbe MsgTooBigError) Error() string {
-	return fmt.Sprint("Msg too big error. PMTU is ", mtbe.PMTU)
-}
-
-func (ftbe FrameTooBigError) Error() string {
-	return fmt.Sprint("Frame too big error. Effective PMTU is ", ftbe.EPMTU)
-}
-
-func (upe UnknownPeerError) Error() string {
-	return fmt.Sprint("Reference to unknown peer ", upe.Name)
-}
-
-func (nce NameCollisionError) Error() string {
-	return fmt.Sprint("Multiple peers found with same name: ", nce.Name)
-}
-
-func (pde PacketDecodingError) Error() string {
-	return fmt.Sprint("Failed to decode packet: ", pde.Desc)
-}
-
-func Concat(elems ...[]byte) []byte {
-	res := []byte{}
-	for _, e := range elems {
-		res = append(res, e...)
-	}
-	return res
-}
-
-func randUint64() (r uint64) {
-	buf := make([]byte, 8)
-	_, err := rand.Read(buf)
-	checkFatal(err)
-	for _, v := range buf {
-		r <<= 8
-		r |= uint64(v)
-	}
-	return
-}
-
-func GobEncode(items ...interface{}) []byte {
-	buf := new(bytes.Buffer)
-	enc := gob.NewEncoder(buf)
-	for _, i := range items {
-		checkFatal(enc.Encode(i))
-	}
-	return buf.Bytes()
-}
-
-func macint(mac net.HardwareAddr) (r uint64) {
-	for _, b := range mac {
-		r <<= 8
-		r |= uint64(b)
-	}
-	return
-}
-
-func intmac(key uint64) (r net.HardwareAddr) {
-	r = make([]byte, 6)
-	for i := 5; i >= 0; i-- {
-		r[i] = byte(key)
-		key >>= 8
-	}
-	return
-}
-
-type ListOfPeers []*Peer
-
-func (lop ListOfPeers) Len() int {
-	return len(lop)
-}
-func (lop ListOfPeers) Swap(i, j int) {
-	lop[i], lop[j] = lop[j], lop[i]
-}
-func (lop ListOfPeers) Less(i, j int) bool {
-	return lop[i].Name < lop[j].Name
-}

site/_includes/footer.html

@@ -69,3 +69,17 @@
       s.parentNode.insertBefore(wf, s);
     })();
 </script>
+<script type="text/javascript">
+piAId = '124932';
+piCId = '85570';
+
+(function() {
+function async_load(){
+var s = document.createElement('script'); s.type = 'text/javascript';
+s.src = ('https:' == document.location.protocol ? 'https://pi' : 'http://cdn') + '.pardot.com/pd.js';
+var c = document.getElementsByTagName('script')[0]; c.parentNode.insertBefore(s, c);
+}
+if(window.attachEvent) { window.attachEvent('onload', async_load); }
+else { window.addEventListener('load', async_load, false); }
+})();
+</script>

site/boot2docker.md

@@ -1,38 +0,0 @@
----
-title: Installing Weave with Boot2Docker
-layout: default
----
-
-# Installing Weave with Boot2Docker
-
-If you are running Docker inside the Boot2Docker VM, e.g. because you
-are on a Mac, then the following changes are required to the weave
-instructions. First, we need to get Boot2Docker running:
-
-    host1$ boot2docker init
-    host1$ boot2docker start
-    host1$ eval "$(boot2docker shellinit)"
-
-Assuming you have fetched the 'weave' script via curl or similar, and
-it is in the current directory, launch weave on the Boot2Docker VM and
-configure our shell. Because Boot2Docker uses TLS, we have to pass a
-few extra options when launching the docker API proxy:
-
-    host1$ ./weave launch-router
-    host1$ ./weave launch-proxy --tls \
-             --tlscacert /var/lib/boot2docker/tls/ca.pem \
-             --tlscert /var/lib/boot2docker/tls/server.pem \
-             --tlskey /var/lib/boot2docker/tls/serverkey.pem
-    host1$ eval "$(./weave env)"
-
-Now, we can use the docker command to run our containers, as shown in
-the [proxy documentation](proxy.html):
-
-    host1$ docker run -it ubuntu
-
-For more information about how to access services running in
-Boot2Docker from the host or other machines (i.e. outside of the weave
-network), see these pages:
-
-* https://github.com/docker/docker/issues/4007
-* http://viget.com/extend/how-to-use-docker-on-os-x-the-missing-guide

site/building.md

@@ -11,7 +11,7 @@ You only need to build weave if you want to work on the weave codebase
 Apart from the `weave` shell script, weave is delivered as a set of
 container images.  There is no distribution-specific packaging, so in
 principle it shouldn't matter which Linux distribution you build
-under.  But naturally, Docker is a prerequisite (version 1.3.1 or
+under.  But naturally, Docker is a prerequisite (version 1.6.0 or
 later).  And it is difficult to build under Fedora because [Fedora
 does not include static
 libraries](http://fedoraproject.org/wiki/Packaging:Guidelines#Packaging_Static_Libraries).
@@ -54,7 +54,7 @@ $ make
 
 This will build the weave components and package them into three
 Docker images (`weaveworks/weave`, `weaveworks/weavedns`, and
-`weaveworks/weaveexec`).  These are then exported as `weave.tar`.
+`weaveworks/weaveexec`).  These are then exported as `weave.tar.gz`.
 
 ## Building in a Docker container
 
@@ -167,13 +167,13 @@ you can do so with
 $ vagrant ssh -c 'make -C src/github.com/weaveworks/weave'
 ```
 
-you should then find a `weave.tar` container snapshot tarball in the
+you should then find a `weave.tar.gz` container snapshot tarball in the
 top-level directory. You can use that snapshot with `docker load`
 against a different host, e.g.
 
 ```bash
 $ export DOCKER_HOST=tcp://<HOST:PORT>
-$ docker load < weave.tar
+$ docker load < weave.tar.gz
 ```
 
 You can provide extra Vagrant configuration by putting a file

site/features.md

@@ -9,7 +9,9 @@ Weave has a few more features beyond those illustrated by the [basic
 example](https://github.com/weaveworks/weave#example):
 
  * [Virtual ethernet switch](#virtual-ethernet-switch)
+ * [Fast data path](#fast-data-path)
  * [Seamless Docker integration](#docker)
+ * [Docker network plugin](#plugin)
  * [Address allocation](#addressing)
  * [Naming and discovery](#naming-and-discovery)
  * [Application isolation](#application-isolation)
@@ -55,6 +57,25 @@ now re-use the same tools and techniques when deploying applications
 as containers as we would have done when deploying them 'on metal' in
 our data centre.
 
+### <a name="fast-data-path"></a>Fast data path
+
+Weave automatically chooses the fastest available method to transport
+data between peers. The most performant of these ('fastdp') offers
+near-native throughput and latency but does not support encryption;
+consequently supplying a password will cause the router to fall back
+to a slower mode ('sleeve') that does.
+
+Even when encryption is not in use, certain adverse network conditions
+will cause this fallback to occur dynamically; in these circumstances,
+weave will upgrade the connection back to the fastdp transport without
+user intervention once they abate. You can see which method is in use
+by examining the output of `weave status connections`.
+
+You can also administratively disable fastdp with the
+`WEAVE_NO_FASTDP` environment variable:
+
+    $ WEAVE_NO_FASTDP=true weave launch
+
 ### <a name="docker"></a>Seamless Docker integration
 
 Weave includes a [Docker API proxy](proxy.html) so that containers
@@ -64,6 +85,28 @@ or
 [remote API](https://docs.docker.com/reference/api/docker_remote_api/)
 are attached to the weave network before they begin execution.
 
+Containers started in this way that subsequently restart, either by an
+explicit `docker restart` command or by Docker restart policy, are
+re-attached to the weave network by the weave Docker API proxy.
+
+### <a name="plugin"></a>Docker network plugin
+
+Alternatively, you can use weave as a Docker plugin. First you must
+configure the Docker daemon to use a cluster store as documented
+[here](https://docs.docker.com/engine/userguide/networking/dockernetworks/#an-overlay-network);
+you can then launch weave and create a network:
+
+    $ weave launch
+    $ weave launch-plugin
+    $ docker network create --driver=weave weave
+
+and then start a container:
+
+    $ docker run --net=weave -ti ubuntu
+
+For more details see the
+[plugin README](https://github.com/weaveworks/docker-plugin).
+
 ### <a name="addressing"></a>Address allocation
 
 Containers are automatically allocated an IP address that is unique
@@ -195,7 +238,7 @@ NB: By default docker permits communication between containers on the
 same host, via their docker-assigned IP addresses. For complete
 isolation between application containers, that feature needs to be
 disabled by
-[setting `--icc=false`](https://docs.docker.com/articles/networking/#between-containers)
+[setting `--icc=false`](https://docs.docker.com/engine/userguide/networking/default_network/container-communication/#communication-between-containers)
 in the docker daemon configuration. Furthermore, containers should be
 prevented from capturing and injecting raw network packets - this can
 be accomplished by starting them with the `--cap-drop net_raw` option.
@@ -248,6 +291,9 @@ invocation:
     host1$ weave detach net:default net:10.2.2.0/24 net:10.2.3.0/24 $C
     10.2.1.3 10.2.2.3 10.2.3.1
 
+Note that addresses added by dynamic attachment are not re-attached
+if the container restarts.
+
 ### <a name="security"></a>Security
 
 In order to connect containers across untrusted networks, weave peers
@@ -275,7 +321,18 @@ way to generate a random password which satsifies this requirement is
 
     < /dev/urandom tr -dc A-Za-z0-9 | head -c9 ; echo
 
-The same password must be specified for all weave peers.
+The same password must be specified for all weave peers. Note that
+supplying a password will [cause weave to fall back to a slower
+method](#fast-data-path) for transporting data between
+peers.
+
+Be aware that:
+
+* Containers will be able to access the router REST API if you have
+  disabled fast datapath. You can prevent this by setting
+  [`--icc=false`](https://docs.docker.com/engine/userguide/networking/default_network/container-communication/#communication-between-containers)
+* Containers are able to access the router control and data plane
+  ports, but you can mitigate this by enabling encryption
 
 ### <a name="host-network-integration"></a>Host network integration
 
@@ -442,11 +499,13 @@ Weave can network containers hosted in different cloud providers /
 data centres. So, for example, one could run an application consisting
 of containers on GCE, EC2 and in local data centres.
 
-To enable this, the network must be configured to permit TCP and UDP
-connections to the weave port of the docker hosts. The weave port
-defaults to 6783. This can be overriden by setting `WEAVE_PORT`, but
-it is highly recommended that all peers in a weave network are given
-the same port setting.
+To enable this, the network must be configured to permit connections
+to weave's control and data ports on the docker hosts. The control
+port defaults to TCP 6783, and the data ports to UDP 6783/6784. You
+can override these defaults by setting `WEAVE_PORT` (this is a base
+value - setting `WEAVE_PORT=9000` will result in weave using TCP 9000
+for control and UDP 9000/9001 for data). Note that it is highly
+recommended that all peers be given the same setting.
 
 ### <a name="multi-hop-routing"></a>Multi-hop routing
 

site/ipam.md

@@ -105,21 +105,11 @@ range. This can be overridden with the `--ipalloc-range` option, e.g.
 
 and must be the same on every host.
 
-> The specified range is for *exclusive* use by the IP allocator. In
-> particular, when starting containers with manually assigned IP
-> addresses, these must be outside the IP allocator's range.  If, in
-> our example, you subsequently executed `docker run -e
-> WEAVE_CIDR=10.2.3.1/24 -ti ubuntu`, the IP allocator might assign
-> the same address to another container, which will break IP routing
-> for the containers sharing an address.
-
 The range parameter is written in
 [CIDR notation](http://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing) -
 in this example "/16" means the first 16 bits of the address form the
 network address and the allocator is to allocate container addresses
-that all start 10.2. The ".0" and ".-1" addresses in a subnet are not
-used, as required by
-[RFC 1122](https://tools.ietf.org/html/rfc1122#page-29).
+that all start 10.2.
 
 Weave shares the IP address range across all peers, dynamically
 according to their needs.  If a group of peers becomes isolated from
@@ -129,11 +119,16 @@ to the rest of the network without any conflicts arising.
 
 ## <a name="subnets"></a>Automatic allocation across multiple subnets
 
-When
-[running containers on different subnets](features.html#application-isolation),
-you may wish to request the allocation of an address from a particular
-subnet. This is done by specifying the subnet with `net:<subnet>`, in
-CIDR notation, e.g.
+IP subnets are used to define or restrict routing. By default, weave
+puts all containers in a subnet that spans the entire allocation
+range, so every weave-attached container can talk to every other
+weave-attached container.
+
+If you want some [isolation](features.html#application-isolation), you
+can choose to run containers on different subnets.  To request the
+allocation of an address from a particular subnet, set the
+`WEAVE_CIDR` environment variable to `net:<subnet>` when creating the
+container, e.g.
 
     host1$ docker run -e WEAVE_CIDR=net:10.2.7.0/24 -ti ubuntu
 
@@ -143,6 +138,9 @@ for instance:
 
     host1$ docker run -e WEAVE_CIDR="net:10.2.7.0/24 net:10.2.8.0/24 ip:10.3.9.1/24" -ti ubuntu
 
+(Note the ".0" and ".-1" addresses in a subnet are not used, as required by
+[RFC 1122](https://tools.ietf.org/html/rfc1122#page-29)).
+
 When working with multiple subnets in this way, it is usually
 desirable to constrain the default subnet - i.e. the one chosen by the
 allocator when no subnet is supplied - so that it does not overlap
@@ -216,7 +214,7 @@ reports on the current status of the weave router and IP allocator:
 
        Service: ipam
      Consensus: waiting(quorum: 2, known: 0)
-         Range: [10.32.0.0-10.48.0.0)
+         Range: 10.32.0.0-10.47.255.255
  DefaultSubnet: 10.32.0.0/12
 
 ...

site/proxy.md

@@ -38,15 +38,16 @@ modify the default behaviour you will have to use the latter.
 
 By default, the proxy decides where to listen based on how the
 launching client connects to docker. If the launching client connected
-over a unix socket, the proxy will listen on /var/run/weave.sock. If
+over a unix socket, the proxy will listen on /var/run/weave/weave.sock. If
 the launching client connected over TCP, the proxy will listen on port
 12375, on all network interfaces. This can be adjusted with the `-H`
 argument, e.g.
 
     host1$ weave launch-proxy -H tcp://127.0.0.1:9999
 
-When launching the proxy via TLS, `-H` and/or [TLS options](#tls) are
-required.
+If no TLS or listening interfaces are set, TLS will be autoconfigured
+based on the docker daemon's settings, and listening interfaces will
+be autoconfigured based on your docker client's settings.
 
 Multiple `-H` arguments can be specified. If you are working with a
 remote docker daemon, then any firewalls inbetween need to be
@@ -56,13 +57,13 @@ All docker commands can be run via the proxy, so it is safe to adjust
 your `DOCKER_HOST` to point at the proxy. Weave provides a convenient
 command for this:
 
-    host1$ eval "$(weave env)"
+    host1$ eval $(weave env)
     host1$ docker ps
     ...
 
 The prior settings can be restored with
 
-    host1$ eval "$(weave env --restore)"
+    host1$ eval $(weave env --restore)
 
 Alternatively, the proxy host can be set on a per-command basis with
 
@@ -145,10 +146,27 @@ container with a blank `WEAVE_CIDR`, e.g.
 
     host1$ docker run -ti -e WEAVE_CIDR="" ubuntu
 
-When launching weave-enabled containers, the proxy will automatically
-rewrite `/etc/hosts` to replace the docker IP with the container's
-weave IP. If you need the docker IP to remain in `/etc/hosts`, the
-proxy must be launched with the `--no-rewrite-hosts` flag.
+## <a name="etchosts"></a>Name resolution via `/etc/hosts`
+
+When starting weave-enabled containers, the proxy will automatically
+replace the container's `/etc/hosts` file, and disable Docker's control
+over it. The new file contains an entry for the container's hostname
+and weave IP address, as well as additional entries that have been
+specified with `--add-host` parameters. This ensures that
+
+- name resolution of the container's hostname, e.g. via `hostname -i`,
+returns the weave IP address. This is required for many cluster-aware
+applications to work.
+- unqualified names get resolved via DNS, i.e. typically via weaveDNS
+to weave IP addresses. This is required so that in a typical setup
+one can simply "ping <container-name>", i.e. without having to
+specify a `.weave.local` suffix.
+
+In case you prefer to keep `/etc/hosts` under Docker's control (for
+example, because you need the hostname to resolve to the Docker-assigned
+IP instead of the weave IP, or you require name resolution for
+Docker-managed networks), the proxy must be launched with the
+`--no-rewrite-hosts` flag.
 
     host1$ weave launch-router && weave launch-proxy --no-rewrite-hosts
 
@@ -176,7 +194,7 @@ ultimately be handed over to weaveDNS for registration.
 For instance, we can launch the proxy using all three flags
 
     host1$ weave launch-router && weave launch-proxy --hostname-from-label hostname-label --hostname-match '^aws-[0-9]+-(.*)$' --hostname-replacement 'my-app-$1'
-    host1$ eval "$(weave env)"
+    host1$ eval $(weave env)
 
 Note how regexp substitution groups should be prepended with a dollar sign
 (e.g. `$1`). For further details on the regular expression syntax please see
@@ -204,16 +222,23 @@ to the proxy, the specified label has precedence over the container's name.
 
 ## <a name="tls"></a>Securing the docker communication with TLS
 
-If you are
-[connecting to the docker daemon with TLS](https://docs.docker.com/articles/https/),
-you will probably want to do the same when connecting to the
-proxy. That is accomplished by launching the proxy with the same
-TLS-related command-line flags as supplied to the docker daemon. For
-example, if you have generated your certificates and keys into the
-docker host's `/tls` directory, we can launch the proxy with:
+If you are [connecting to the docker daemon with
+TLS](https://docs.docker.com/articles/https/), you will probably want
+to do the same when connecting to the proxy. The proxy will
+automatically detect the docker daemon's TLS configuration, and
+attempt to duplicate it. In the standard auto-detection case you will
+be able to launch a TLS-enabled proxy with:
 
-    host1$ weave launch-proxy --tls-verify --tls-cacert=/tls/ca.pem \
-             --tls-cert=/tls/server-cert.pem --tls-key=/tls/server-key.pem
+    host1$ weave launch-proxy
+
+You can also manually configure the proxy's TLS. This is accomplished
+by launching the proxy with the same TLS-related command-line flags as
+supplied to the docker daemon. For example, if you have generated your
+certificates and keys into the docker host's `/tls` directory, we can
+launch the proxy with:
+
+    host1$ weave launch-proxy --tlsverify --tlscacert=/tls/ca.pem \
+             --tlscert=/tls/server-cert.pem --tlskey=/tls/server-key.pem
 
 The paths to your certificates and key must be provided as absolute
 paths which exist on the docker host.
@@ -229,8 +254,8 @@ for an example.
 With the proxy running over TLS, we can configure our regular docker
 client to use TLS on a per-invocation basis with
 
-    $ docker --tls-verify --tls-cacert=ca.pem --tls-cert=cert.pem \
-         --tls-key=key.pem -H=tcp://host1:12375 version
+    $ docker --tlsverify --tlscacert=ca.pem --tlscert=cert.pem \
+         --tlskey=key.pem -H=tcp://host1:12375 version
     ...
 
 or,
@@ -239,7 +264,7 @@ with
 
     $ mkdir -pv ~/.docker
     $ cp -v {ca,cert,key}.pem ~/.docker
-    $ eval "$(weave env)"
+    $ eval $(weave env)
     $ export DOCKER_TLS_VERIFY=1
     $ docker version
     ...
@@ -263,6 +288,13 @@ specify addresses manually instead:
 
     $ weave run 10.2.1.1/24 -ti ubuntu
 
+`weave run` will rewrite `/etc/hosts` in the same way
+[the proxy does](#etchosts). In case you prefer to keep
+the original file, you must specify `--no-rewrite-hosts` when running
+the container:
+
+    $ weave run --no-rewrite-hosts 10.2.1.1/24 -ti ubuntu
+
 There are some limitations to starting containers with `weave run`:
 
 * containers are always started in the background, i.e. the equivalent