Add release v1.2.1

Prevent containers from accessing the weave router control port when
fast datapath is in use.

.gitignore

@@ -39,11 +39,11 @@ prog/weavehosts/weavehosts
 prog/weaveproxy/weaveproxy
 prog/sigproxy/sigproxy
 prog/weavewait/weavewait
+prog/weavewait/weavewait_noop
 prog/netcheck/netcheck
+prog/docker_tls_args/docker_tls_args
 testing/cover/cover
 testing/runner/runner
-tools/bin
-tools/build
 releases
 .weaver.uptodate
 .weavedns.uptodate
@@ -53,8 +53,10 @@ prog/weaveexec/weave
 prog/weaveexec/weavehosts
 prog/weaveexec/weaveproxy
 prog/weaveexec/weavewait
+prog/weaveexec/weavewait_noop
 prog/weaveexec/sigproxy
 prog/weaveexec/netcheck
+prog/weaveexec/docker_tls_args
 prog/weaveexec/docker*.tgz
 Vagrantfile.local
 test/tls/*.pem

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "tools"]
+	path = tools
+	url = https://github.com/weaveworks/build-tools

.travis.yml

@@ -21,7 +21,6 @@ install:
   - go get ./...
 
 script:
-  - bin/lint .
   - make tests
 
 after_success:

CHANGELOG.md

@@ -1,3 +1,43 @@
+## Release 1.2.1
+
+This release contains a number of bug fixes and minor
+enhancements. More details in the
+[change log](https://github.com/weaveworks/weave/issues?q=milestone%3A1.2.1).
+
+The release is fully compatible with 1.2.0 versions, so existing
+clusters can be upgraded incrementally.
+
+## Release 1.2.0
+
+**Highlights**
+
+This release introduces the 
+[Fast Data Path](http://docs.weave.works/weave/master/head/features.html#fast-data-path),
+which allows Weave networks to operate at near wire-level speeds. This
+new feature is enabled by default.
+
+Other highlights:
+
+- auto-configuration of TLS for the Weave Docker API proxy, making it
+  easier to run Weave on Macs and in conjunction with tools like
+  Docker Swarm
+- support for restart policies on application containers and weave
+  infrastructure containers
+- better compatibility with recent and future Docker versions
+
+More details in the 
+[change log](https://github.com/weaveworks/weave/issues?q=milestone%3A1.2.0).
+
+## Release 1.1.2
+
+This release contains a small number of bug fixes. More details in the
+[change log](https://github.com/weaveworks/weave/issues?q=milestone%3A1.1.2).
+
+The release is fully compatible with 1.1.0 versions, so existing
+clusters can be upgraded incrementally. When upgrading from 1.0.x,
+note the compatibility information in the
+[Installation & Upgrading instructions for Weave 1.1.0](https://github.com/weaveworks/weave/releases/tag/v1.1.0).
+
 ## Release 1.1.1
 
 This release contains a number of bug fixes and minor

Makefile

@@ -1,7 +1,7 @@
 PUBLISH=publish_weave publish_weaveexec
 
 .DEFAULT: all
-.PHONY: all update tests publish $(PUBLISH) clean prerequisites build travis run-smoketests
+.PHONY: all update tests lint publish $(PUBLISH) clean clean-bin prerequisites build travis run-smoketests
 
 # If you can use docker without being root, you can do "make SUDO="
 SUDO=sudo
@@ -13,12 +13,12 @@ WEAVER_EXE=prog/weaver/weaver
 WEAVEPROXY_EXE=prog/weaveproxy/weaveproxy
 SIGPROXY_EXE=prog/sigproxy/sigproxy
 WEAVEWAIT_EXE=prog/weavewait/weavewait
-WEAVEHOSTS_EXE=prog/weavehosts/weavehosts
+WEAVEWAIT_NOOP_EXE=prog/weavewait/weavewait_noop
 NETCHECK_EXE=prog/netcheck/netcheck
-COVER_EXE=testing/cover/cover
+DOCKERTLSARGS_EXE=prog/docker_tls_args/docker_tls_args
 RUNNER_EXE=testing/runner/runner
 
-EXES=$(WEAVER_EXE) $(SIGPROXY_EXE) $(WEAVEPROXY_EXE) $(WEAVEWAIT_EXE) $(WEAVEHOSTS_EXE) $(NETCHECK_EXE) $(COVER_EXE) $(RUNNER_EXE)
+EXES=$(WEAVER_EXE) $(SIGPROXY_EXE) $(WEAVEPROXY_EXE) $(WEAVEWAIT_EXE) $(WEAVEWAIT_NOOP_EXE) $(NETCHECK_EXE) $(DOCKERTLSARGS_EXE) $(RUNNER_EXE)
 
 WEAVER_UPTODATE=.weaver.uptodate
 WEAVEEXEC_UPTODATE=.weaveexec.uptodate
@@ -45,7 +45,9 @@ NETGO_CHECK=@strings $@ | grep cgo_stub\\\.go >/dev/null || { \
 }
 BUILD_FLAGS=-ldflags "-extldflags \"-static\" -X main.version $(WEAVE_VERSION)" -tags netgo
 
-all: $(WEAVE_EXPORT) $(COVER_EXE) $(RUNNER_EXE)
+PACKAGE_BASE=$(shell go list -e ./)
+
+all: $(WEAVE_EXPORT) $(RUNNER_EXE)
 
 travis: $(EXES)
 
@@ -54,7 +56,7 @@ update:
 
 $(WEAVER_EXE) $(WEAVEPROXY_EXE): common/*.go common/*/*.go net/*.go
 ifeq ($(COVERAGE),true)
-	$(eval COVERAGE_MODULES := $(shell (go list ./$(@D); go list -f '{{join .Deps "\n"}}' ./$(@D) | grep "weaveworks") | paste -s -d,))
+	$(eval COVERAGE_MODULES := $(shell (go list ./$(@D); go list -f '{{join .Deps "\n"}}' ./$(@D) | grep "^$(PACKAGE_BASE)/") | paste -s -d,))
 	go get -t -tags netgo ./$(@D)
 	go test -c -o ./$@ $(BUILD_FLAGS) -v -covermode=atomic -coverpkg $(COVERAGE_MODULES) ./$(@D)/
 else
@@ -75,26 +77,30 @@ $(NETCHECK_EXE): prog/netcheck/netcheck.go
 # Sigproxy and weavewait need separate rules as they fail the netgo check in
 # the main build stanza due to not importing net package
 $(SIGPROXY_EXE): prog/sigproxy/main.go
-$(WEAVEWAIT_EXE): prog/weavewait/main.go net/*.go
-$(WEAVEHOSTS_EXE): prog/weavehosts/weavehosts.go
-$(COVER_EXE): testing/cover/cover.go
+$(WEAVEWAIT_EXE): prog/weavewait/*.go net/*.go
 $(RUNNER_EXE): testing/runner/runner.go
+$(DOCKERTLSARGS_EXE): prog/docker_tls_args/*.go
 
-$(WEAVEWAIT_EXE) $(SIGPROXY_EXE) $(WEAVEHOSTS_EXE) $(COVER_EXE) $(RUNNER_EXE):
-	go get ./$(@D)
-	go build -o $@ ./$(@D)
+$(WEAVEWAIT_EXE) $(SIGPROXY_EXE) $(RUNNER_EXE) $(DOCKERTLSARGS_EXE):
+	go get -tags netgo ./$(@D)
+	go build $(BUILD_FLAGS) -o $@ ./$(@D)
+
+$(WEAVEWAIT_NOOP_EXE): prog/weavewait/*.go
+	go get -tags netgo ./$(@D)
+	go build $(BUILD_FLAGS) -tags noop -o $@ ./$(@D)
 
 $(WEAVER_UPTODATE): prog/weaver/Dockerfile $(WEAVER_EXE)
 	$(SUDO) docker build -t $(WEAVER_IMAGE) prog/weaver
 	touch $@
 
-$(WEAVEEXEC_UPTODATE): prog/weaveexec/Dockerfile $(DOCKER_DISTRIB) weave $(SIGPROXY_EXE) $(WEAVEPROXY_EXE) $(WEAVEWAIT_EXE) $(WEAVEHOSTS_EXE) $(NETCHECK_EXE)
+$(WEAVEEXEC_UPTODATE): prog/weaveexec/Dockerfile $(DOCKER_DISTRIB) weave $(SIGPROXY_EXE) $(WEAVEPROXY_EXE) $(WEAVEWAIT_EXE) $(WEAVEWAIT_NOOP_EXE) $(NETCHECK_EXE) $(DOCKERTLSARGS_EXE)
 	cp weave prog/weaveexec/weave
 	cp $(SIGPROXY_EXE) prog/weaveexec/sigproxy
 	cp $(WEAVEPROXY_EXE) prog/weaveexec/weaveproxy
 	cp $(WEAVEWAIT_EXE) prog/weaveexec/weavewait
-	cp $(WEAVEHOSTS_EXE) prog/weaveexec/weavehosts
+	cp $(WEAVEWAIT_NOOP_EXE) prog/weaveexec/weavewait_noop
 	cp $(NETCHECK_EXE) prog/weaveexec/netcheck
+	cp $(DOCKERTLSARGS_EXE) prog/weaveexec/docker_tls_args
 	cp $(DOCKER_DISTRIB) prog/weaveexec/docker.tgz
 	$(SUDO) docker build -t $(WEAVEEXEC_IMAGE) prog/weaveexec
 	touch $@
@@ -105,8 +111,14 @@ $(WEAVE_EXPORT): $(IMAGES_UPTODATE)
 $(DOCKER_DISTRIB):
 	curl -o $(DOCKER_DISTRIB) $(DOCKER_DISTRIB_URL)
 
-tests: $(COVER_EXE)
-	@test/units.sh
+tests: tools/.git
+	tools/test
+
+lint: tools/.git
+	tools/lint -nocomment -notestpackage .
+
+tools/.git:
+	git submodule update --init
 
 $(PUBLISH): publish_%: $(IMAGES_UPTODATE)
 	$(SUDO) docker tag -f $(DOCKERHUB_USER)/$* $(DOCKERHUB_USER)/$*:$(WEAVE_VERSION)
@@ -117,10 +129,12 @@ endif
 
 publish: $(PUBLISH)
 
-clean:
+clean-bin:
 	-$(SUDO) docker rmi $(IMAGES)
-	go clean -r ./...
+	go clean -r $(addprefix ./,$(dir $(EXES)))
 	rm -f $(EXES) $(IMAGES_UPTODATE) $(WEAVE_EXPORT)
+
+clean: clean-bin
 	rm -rf test/tls/*.pem test/coverage.* test/coverage
 
 build:

README.md

@@ -7,11 +7,9 @@
 [Weaveworks](http://weave.works) is the company that develops Weave -
 the most productive way for developers to connect, observe and control
 Docker containers. To learn about our products, including getting
-started tutorials, visit our [website](http://weave.works), read about
-the
-[1.0 release of Weave](http://blog.weave.works/2015/06/18/weave-1-0-is-out-and-it-is-good/),
-visit our [documentation](http://docs.weave.works) or continue to read
-about some of the more technical aspects of Weave in this readme.
+started tutorials, visit our [website](http://weave.works) and
+[documentation](http://docs.weave.works) or continue to read about
+some of the more technical aspects of Weave in this readme.
 
 # Weave
 
@@ -48,14 +46,24 @@ Ensure you are running Linux (kernel 3.8 or later) and have Docker
     sudo curl -L git.io/weave -o /usr/local/bin/weave
     sudo chmod a+x /usr/local/bin/weave
 
+For usage on OSX (with Docker Machine) you first need to make sure
+that a VM is running and configured, as shown in [the Docker Machine
+documentation](https://docs.docker.com/installation/mac/#from-your-shell).
+Then you can launch weave directly from the OSX host.
+
 CoreOS users see [here](https://github.com/fintanr/weave-gs/blob/master/coreos-simple/user-data) for an example of installing weave using cloud-config.
 
+Amazon ECS users see
+[here](https://github.com/weaveworks/guides/blob/master/aws-ecs/LATESTAMIs.md)
+for the latest Weave AMIs and
+[here](http://weave.works/guides/service-discovery-with-weave-aws-ecs.html) to get started with Weave on ECS.
+
 Weave respects the environment variable `DOCKER_HOST`, so you can run
 it locally to control a weave network on a remote host.
 
 ## Quick Start Screencast
 
-<a href="https://youtu.be/BF9YJSPQUcY" alt="Click to watch the screencast" target="_blank">
+<a href="https://youtu.be/kihQCCT1ykE" alt="Click to watch the screencast" target="_blank">
   <img src="/docs/hello-screencast.png" />
 </a>
 
@@ -99,9 +107,8 @@ The only difference, apart from the name of the application container,
 is that we tell our weave that it should peer with the weave on
 `$HOST1` (specified as the IP address or hostname, and optional
 `:port`, by which `$HOST2` can reach it). NB: if there is a firewall
-between `$HOST1` and `$HOST2`, you must open the weave port (6783 by
-default; this can be overriden by setting `WEAVE_PORT`) for TCP and
-UDP.
+between `$HOST1` and `$HOST2`, you must permit traffic to the weave
+control port (TCP 6783) and data ports (UDP 6783/6784).
 
 Note that we could instead have told the weave on `$HOST1` to connect to
 `$HOST2`, or told both about each other. Order does not matter here;
@@ -148,12 +155,16 @@ to each other.
 
 ## Contact Us
 
-Found a bug, want to suggest a feature, or have a question?
-[File an issue](https://github.com/weaveworks/weave/issues), or email
-help@weave.works. When reporting a bug, please include which version of
-weave you are running, as shown by `weave version`.
+Found a bug, want to suggest a feature, or have a question?  You can
+[File an issue](https://github.com/weaveworks/weave/issues).  And
+there is a [Weave Users Google
+Group](https://groups.google.com/a/weave.works/forum/#!forum/weave-users),
+which you can post to by email at weave-users@weave.works.  Or you can
+contact Weaveworks directly at help@weave.works. When reporting a
+bug, please include which version of weave you are running, as shown
+by `weave version`.
 
-Follow weave on Twitter:
+Follow on Twitter:
 [@weaveworks](https://twitter.com/weaveworks).
 
 Read the Weave blog:

bin/lint

@@ -1,86 +0,0 @@
-#!/bin/bash
-# This scipt lints go files for common errors.
-#
-# Its runs gofmt and go vet, and optionally golint and
-# gocyclo, if they are installed.
-#
-# With no arguments, it lints the current files staged
-# for git commit.  Or you can pass it explicit filenames
-# (or directories) and it will lint them.
-#
-# To use this script automatically, run:
-#   ln -s ../../bin/lint .git/hooks/pre-commit
-
-set -eu
-
-function lint_go {
-	filename="$1"
-	result=0
-
-	if [ -n "$(gofmt -l "${filename}")" ]; then
-		result=1
-		echo "${filename}: run gofmt -w ${filename}!"
-	fi
-
-	go tool vet "${filename}" || result=$?
-
-	# golint is completely optional.  If you don't like it
-	# don't have it installed.
-	if type golint >/dev/null 2>&1; then
-		# golint doesn't set an exit code it seems
-		lintoutput=$(golint "${filename}" | grep -vE 'comment|dot imports')
-		if [ "$lintoutput" != "" ]; then
-			result=1
-			echo "$lintoutput"
-		fi
-	fi
-
-	# gocyclo is completely optional.  If you don't like it
-	# don't have it installed.  Also never blocks a commit,
-	# it just warns.
-	if type gocyclo >/dev/null 2>&1; then
-		gocyclo -over 25 "${filename}" | while read line; do
-			echo "${filename}": higher than 25 cyclomatic complexity - "${line}"
-		done
-	fi
-
-	return $result
-}
-
-function lint {
-	filename="$1"
-	ext="${filename##*\.}"
-
-	# Don't lint this script
-	if [ "${filename}" = "bin/lint" ]; then
-		return
-	fi
-
-	# Don't lint files that don't exist
-	if ! [ -a "${filename}" ]; then
-		return
-	fi
-
-	case "$ext" in
-		go) lint_go "${filename}"
-		;;
-	esac
-}
-
-function lint_files {
-	lint_result=0
-	while read filename; do
-		lint "${filename}" || lint_result=1
-	done
-	exit $lint_result
-}
-
-function list_files {
-	if [ $# -gt 0 ]; then
-		find "$@" -type f
-	else
-		git diff --cached --name-only
-	fi
-}
-
-list_files "$@" | lint_files

build/build.sh

@@ -36,11 +36,7 @@ EOF
     echo "weave:*:::::::" >>/etc/shadow
     echo "weave	ALL=(ALL)	NOPASSWD: ALL" >>/etc/sudoers
 
-    if [ "$1" = "tests" ] ; then
-        su weave -c "PATH=$PATH make -C $WEAVE_SRC tests"
-    else
-        su weave -c "PATH=$PATH make -C $WEAVE_SRC build"
-    fi
+    su weave -c "PATH=$PATH make -C $WEAVE_SRC build"
 else
     # There are arguments to pass to git-clone
     mkdir -p ${WEAVE_SRC%/*}

circle.yml

@@ -12,18 +12,19 @@ machine:
     SRCDIR: /home/ubuntu/src/github.com/weaveworks/weave
     PATH: $PATH:$HOME/.local/bin:$HOME/google-cloud-sdk/bin/
     CLOUDSDK_CORE_DISABLE_PROMPTS: 1
-    WEAVE_BUILD: $HOME/docker/weave-build.tar
+    WEAVE_TEST: $HOME/docker/weave-test.tar
 
 dependencies:
   cache_directories:
     - "~/docker"
   override:
-    - if [[ -e "$WEAVE_BUILD" ]]; then
-        docker load -i $WEAVE_BUILD;
+    - if [[ -e "$WEAVE_TEST" ]]; then
+        docker load -i $WEAVE_TEST;
       else
         docker build -t weaveworks/weave-build build;
-        mkdir -p $(dirname "$WEAVE_BUILD");
-        docker save weaveworks/weave-build >$WEAVE_BUILD;
+        docker build -t weaveworks/weave-test test/image;
+        mkdir -p $(dirname "$WEAVE_TEST");
+        docker save weaveworks/weave-test >$WEAVE_TEST;
       fi
   post:
     - sudo apt-get update
@@ -33,10 +34,14 @@ dependencies:
     - bin/setup-circleci-secrets "$SECRET_PASSWORD"
     - mkdir -p $(dirname $SRCDIR)
     - cp -r $(pwd)/ $SRCDIR
+    - cd $SRCDIR/test; ./gce.sh make_template:
+        parallel: false
 
 test:
   override:
-    - docker run -v /var/run/docker.sock:/run/docker.sock -v /home/ubuntu:/home/go -e COVERDIR=test/coverage -e SLOW=true -e CIRCLECI -e CIRCLE_BUILD_NUM -e CIRCLE_NODE_TOTAL -e CIRCLE_NODE_INDEX weaveworks/weave-build tests:
+    - cd $SRCDIR; make lint:
+        parallel: true
+    - docker run -v /home/ubuntu:/home/go -e COVERDIR=test/coverage -e SLOW=true -e CIRCLECI -e CIRCLE_BUILD_NUM -e CIRCLE_NODE_TOTAL -e CIRCLE_NODE_INDEX weaveworks/weave-test:
         parallel: true
     - docker run -v /var/run/docker.sock:/run/docker.sock -v /home/ubuntu:/home/go -e COVERAGE=true weaveworks/weave-build:
         parallel: true
@@ -47,6 +52,10 @@ test:
     - cd $SRCDIR/test; eval $(./gce.sh hosts); export COVERAGE=true; ./run_all.sh:
         parallel: true
         timeout: 300
+    - cd $SRCDIR; make clean-bin:
+        parallel: true
+    - docker run -v /var/run/docker.sock:/run/docker.sock -v /home/ubuntu:/home/go weaveworks/weave-build:
+        parallel: true
   post:
     - cd $SRCDIR/test; ./gce.sh destroy:
         parallel: true
@@ -59,3 +68,11 @@ teardown:
         parallel: true
     - test "$CIRCLE_NODE_INDEX" != "0" || (cd $SRCDIR/test; cp coverage.* $CIRCLE_ARTIFACTS):
         parallel: true
+
+deployment:
+  hub:
+    branch: master
+    owner: weaveworks
+    commands:
+      - docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS
+      - cd $SRCDIR; make publish

common/docker/client.go

@@ -7,6 +7,7 @@ import (
 
 // An observer for container events
 type ContainerObserver interface {
+	ContainerStarted(ident string)
 	ContainerDied(ident string)
 }
 
@@ -22,12 +23,26 @@ func NewClient(apiPath string) (*Client, error) {
 	}
 	client := &Client{dc}
 
-	env, err := client.Version()
+	return client, client.checkWorking(apiPath)
+}
+
+func NewVersionedClient(apiPath string, apiVersionString string) (*Client, error) {
+	dc, err := docker.NewVersionedClient(apiPath, apiVersionString)
 	if err != nil {
 		return nil, err
 	}
+	client := &Client{dc}
+
+	return client, client.checkWorking(apiPath)
+}
+
+func (c *Client) checkWorking(apiPath string) error {
+	env, err := c.Version()
+	if err != nil {
+		return err
+	}
 	Log.Infof("[docker] Using Docker API on %s: %v", apiPath, env)
-	return client, nil
+	return nil
 }
 
 // AddObserver adds an observer for docker events
@@ -41,6 +56,9 @@ func (c *Client) AddObserver(ob ContainerObserver) error {
 	go func() {
 		for event := range events {
 			switch event.Status {
+			case "start":
+				id := event.ID
+				ob.ContainerStarted(id)
 			case "die":
 				id := event.ID
 				ob.ContainerDied(id)

common/signals.go

@@ -14,7 +14,7 @@ type SignalReceiver interface {
 
 func SignalHandlerLoop(ss ...SignalReceiver) {
 	sigs := make(chan os.Signal, 1)
-	signal.Notify(sigs, syscall.SIGINT, syscall.SIGQUIT, syscall.SIGUSR1, syscall.SIGTERM)
+	signal.Notify(sigs, syscall.SIGINT, syscall.SIGQUIT, syscall.SIGTERM)
 	buf := make([]byte, 1<<20)
 	for {
 		switch <-sigs {

docs/architecture.txt

@@ -66,23 +66,23 @@ Router:
     simply responds to traffic received from the remote peer via TCP
   5. We register this connection with the local peer
   6a. If we initiated this connection then we now start sending fast
-    heartbeats to the remote peer so that the remote peer can determine what
-    address/port it should use to send UDP back to us. To do this, we
-    spawn off two further threads which are the forwarder
-    loops. These receive frames which are to be sent to the remote
-    peer. One of them sends frames without the DF flag set. To do this
-    it just sends the packets out of the UDP Listener socket. The
-    other needs to send frames with the DF flag set and needs its own
-    socket so that it can do PMTU discovery easily. To do this, it
-    uses a Raw IP socket (IP has no ports, so there's no collision
-    issue with the UDP Listener socket) and so it must add UDP headers
-    itself.
-  6b. If we did not initiate this connection then the UDP Listener
+    heartbeats to the remote peer so that the remote peer can
+    determine what address/port it should use to send UDP back to
+    us. To do this, we spawn off a "forwarder" thread to send
+    heartbeats, monitor incoming heartbeats, and some other auxiliary
+    duties.  It also consumes frames to be encapsulated and send via
+    UDP from two channels, for DF and non-DF cases.  In the non-DF
+    case, it can just send the packets out of the UDP Listener
+    socket. In the DF case, it needs its own socket so that it can do
+    PMTU discovery easily. To do this, it uses a Raw IP socket (IP has
+    no ports, so there's no collision issue with the UDP Listener
+    socket) and so it must add UDP headers itself.
+ 6b. If we did not initiate this connection then the UDP Listener
     should start receiving fast heartbeats from the remote peer. From
     those it should be able to identify the local connection via the
     local peer. It will tell the local connection (communicating to
     the actor thread) about the UDP address of the remote peer. The
-    local connection will then start its forwarder threads as
+    local connection will then start its forwarder thread as
     described in 6a, and start sending fast heartbeats. We send to the
     remote peer via TCP a ConnectionEstablished message. The remote
     peer receives this (on the TCP receiver process), tells the

ipam/allocator.go

@@ -55,19 +55,21 @@ type Allocator struct {
 	paxos            *paxos.Node
 	paxosTicker      *time.Ticker
 	shuttingDown     bool // to avoid doing any requests while trying to shut down
+	isKnownPeer      func(router.PeerName) bool
 	now              func() time.Time
 }
 
 // NewAllocator creates and initialises a new Allocator
-func NewAllocator(ourName router.PeerName, ourUID router.PeerUID, ourNickname string, universe address.Range, quorum uint) *Allocator {
+func NewAllocator(ourName router.PeerName, ourUID router.PeerUID, ourNickname string, universe address.Range, quorum uint, isKnownPeer func(name router.PeerName) bool) *Allocator {
 	return &Allocator{
-		ourName:   ourName,
-		universe:  universe,
-		ring:      ring.New(universe.Start, universe.End, ourName),
-		owned:     make(map[string][]address.Address),
-		paxos:     paxos.NewNode(ourName, ourUID, quorum),
-		nicknames: map[router.PeerName]string{ourName: ourNickname},
-		now:       time.Now,
+		ourName:     ourName,
+		universe:    universe,
+		ring:        ring.New(universe.Start, universe.End, ourName),
+		owned:       make(map[string][]address.Address),
+		paxos:       paxos.NewNode(ourName, ourUID, quorum),
+		nicknames:   map[router.PeerName]string{ourName: ourNickname},
+		isKnownPeer: isKnownPeer,
+		now:         time.Now,
 	}
 }
 
@@ -175,17 +177,6 @@ func (alloc *Allocator) spaceRequestDenied(sender router.PeerName, r address.Ran
 	}
 }
 
-func hasBeenCancelled(cancelChan <-chan bool) func() bool {
-	return func() bool {
-		select {
-		case <-cancelChan:
-			return true
-		default:
-			return false
-		}
-	}
-}
-
 type errorCancelled struct {
 	kind  string
 	ident string
@@ -199,10 +190,9 @@ func (e *errorCancelled) Error() string {
 
 // Allocate (Sync) - get new IP address for container with given name in range
 // if there isn't any space in that range we block indefinitely
-func (alloc *Allocator) Allocate(ident string, r address.Range, cancelChan <-chan bool) (address.Address, error) {
+func (alloc *Allocator) Allocate(ident string, r address.Range, hasBeenCancelled func() bool) (address.Address, error) {
 	resultChan := make(chan allocateResult)
-	op := &allocate{resultChan: resultChan, ident: ident, r: r,
-		hasBeenCancelled: hasBeenCancelled(cancelChan)}
+	op := &allocate{resultChan: resultChan, ident: ident, r: r, hasBeenCancelled: hasBeenCancelled}
 	alloc.doOperation(op, &alloc.pendingAllocates)
 	result := <-resultChan
 	return result.addr, result.err
@@ -237,6 +227,8 @@ func (alloc *Allocator) ContainerDied(ident string) {
 	}
 }
 
+func (alloc *Allocator) ContainerStarted(ident string) {}
+
 // Delete (Sync) - release all IP addresses for container with given name
 func (alloc *Allocator) Delete(ident string) error {
 	errChan := make(chan error)
@@ -284,6 +276,33 @@ func (alloc *Allocator) Free(ident string, addrToFree address.Address) error {
 	return <-errChan
 }
 
+func (alloc *Allocator) pickPeerFromNicknames(isValid func(router.PeerName) bool) router.PeerName {
+	for name := range alloc.nicknames {
+		if name != alloc.ourName && isValid(name) {
+			return name
+		}
+	}
+	return router.UnknownPeerName
+}
+
+func (alloc *Allocator) pickPeerForTransfer() router.PeerName {
+	// first try alive peers that actively participate in IPAM (i.e. have entries)
+	if heir := alloc.ring.PickPeerForTransfer(alloc.isKnownPeer); heir != router.UnknownPeerName {
+		return heir
+	}
+	// next try alive peers that have IPAM enabled but have no entries
+	if heir := alloc.pickPeerFromNicknames(alloc.isKnownPeer); heir != router.UnknownPeerName {
+		return heir
+	}
+	// next try disappeared peers that still have entries
+	t := func(router.PeerName) bool { return true }
+	if heir := alloc.ring.PickPeerForTransfer(t); heir != router.UnknownPeerName {
+		return heir
+	}
+	// finally, disappeared peers that that passively participated in IPAM
+	return alloc.pickPeerFromNicknames(t)
+}
+
 // Shutdown (Sync)
 func (alloc *Allocator) Shutdown() {
 	alloc.infof("Shutdown")
@@ -292,7 +311,7 @@ func (alloc *Allocator) Shutdown() {
 		alloc.shuttingDown = true
 		alloc.cancelOps(&alloc.pendingClaims)
 		alloc.cancelOps(&alloc.pendingAllocates)
-		if heir := alloc.ring.PickPeerForTransfer(); heir != router.UnknownPeerName {
+		if heir := alloc.pickPeerForTransfer(); heir != router.UnknownPeerName {
 			alloc.ring.Transfer(alloc.ourName, heir)
 			alloc.space.Clear()
 			alloc.gossip.GossipBroadcast(alloc.Gossip())
@@ -320,7 +339,6 @@ func (alloc *Allocator) AdminTakeoverRanges(peerNameOrNickname string) error {
 			return
 		}
 
-		delete(alloc.nicknames, peername)
 		newRanges, err := alloc.ring.Transfer(peername, alloc.ourName)
 		alloc.space.AddRanges(newRanges)
 		resultChan <- err
@@ -342,16 +360,28 @@ func (alloc *Allocator) lookupPeername(name string) (router.PeerName, error) {
 	return router.PeerNameFromString(name)
 }
 
-// Restrict the peers in "nicknames" to those in the ring and our own
+// Restrict the peers in "nicknames" to those in the ring plus peers known to the router
 func (alloc *Allocator) pruneNicknames() {
 	ringPeers := alloc.ring.PeerNames()
 	for name := range alloc.nicknames {
-		if _, ok := ringPeers[name]; !ok && name != alloc.ourName {
+		if _, ok := ringPeers[name]; !ok && !alloc.isKnownPeer(name) {
 			delete(alloc.nicknames, name)
 		}
 	}
 }
 
+func (alloc *Allocator) annotatePeernames(names []router.PeerName) []string {
+	var res []string
+	for _, name := range names {
+		if nickname, found := alloc.nicknames[name]; found {
+			res = append(res, fmt.Sprint(name, "(", nickname, ")"))
+		} else {
+			res = append(res, name.String())
+		}
+	}
+	return res
+}
+
 func decodeRange(msg []byte) (r address.Range, err error) {
 	decoder := gob.NewDecoder(bytes.NewReader(msg))
 	return r, decoder.Decode(&r)
@@ -616,16 +646,20 @@ func (alloc *Allocator) update(sender router.PeerName, msg []byte) error {
 	// shouldn't get updates for a empty Ring. But tolerate
 	// them just in case.
 	if data.Ring != nil {
-		if data.Ring.Range() != alloc.universe {
-			return fmt.Errorf("Incompatible IP allocation range %s; ours is %s",
-				data.Ring.Range().AsCIDRString(), alloc.universe.AsCIDRString())
+		switch err = alloc.ring.Merge(*data.Ring); err {
+		case ring.ErrDifferentSeeds:
+			return fmt.Errorf("IP allocation was seeded by different peers (received: %v, ours: %v)",
+				alloc.annotatePeernames(data.Ring.Seeds), alloc.annotatePeernames(alloc.ring.Seeds))
+		case ring.ErrDifferentRange:
+			return fmt.Errorf("Incompatible IP allocation ranges (received: %s, ours: %s)",
+				data.Ring.Range().AsCIDRString(), alloc.ring.Range().AsCIDRString())
+		default:
+			if err == nil && !alloc.ring.Empty() {
+				alloc.pruneNicknames()
+				alloc.ringUpdated()
+			}
+			return err
 		}
-		err = alloc.ring.Merge(*data.Ring)
-		if !alloc.ring.Empty() {
-			alloc.pruneNicknames()
-			alloc.ringUpdated()
-		}
-		return err
 	}
 
 	if data.Paxos != nil {

ipam/allocator_test.go

@@ -20,6 +20,8 @@ const (
 	testStart3 = "10.0.3.0"
 )
 
+func returnFalse() bool { return false }
+
 func TestAllocFree(t *testing.T) {
 	const (
 		container1 = "abcdef"
@@ -39,31 +41,31 @@ func TestAllocFree(t *testing.T) {
 	_, cidr2, _ := address.ParseCIDR(subnet2)
 
 	alloc.claimRingForTesting()
-	addr1, err := alloc.Allocate(container1, cidr1.HostRange(), nil)
+	addr1, err := alloc.Allocate(container1, cidr1.HostRange(), returnFalse)
 	require.NoError(t, err)
 	require.Equal(t, testAddr1, addr1.String(), "address")
 
-	addr2, err := alloc.Allocate(container1, cidr2.HostRange(), nil)
+	addr2, err := alloc.Allocate(container1, cidr2.HostRange(), returnFalse)
 	require.NoError(t, err)
 	require.Equal(t, testAddr2, addr2.String(), "address")
 
 	// Ask for another address for a different container and check it's different
-	addr1b, _ := alloc.Allocate(container2, cidr1.HostRange(), nil)
+	addr1b, _ := alloc.Allocate(container2, cidr1.HostRange(), returnFalse)
 	if addr1b.String() == testAddr1 {
 		t.Fatalf("Expected different address but got %s", addr1b.String())
 	}
 
 	// Ask for the first container again and we should get the same addresses again
-	addr1a, _ := alloc.Allocate(container1, cidr1.HostRange(), nil)
+	addr1a, _ := alloc.Allocate(container1, cidr1.HostRange(), returnFalse)
 	require.Equal(t, testAddr1, addr1a.String(), "address")
-	addr2a, _ := alloc.Allocate(container1, cidr2.HostRange(), nil)
+	addr2a, _ := alloc.Allocate(container1, cidr2.HostRange(), returnFalse)
 	require.Equal(t, testAddr2, addr2a.String(), "address")
 
 	// Now delete the first container, and we should get its addresses back
 	require.NoError(t, alloc.Delete(container1))
-	addr3, _ := alloc.Allocate(container3, cidr1.HostRange(), nil)
+	addr3, _ := alloc.Allocate(container3, cidr1.HostRange(), returnFalse)
 	require.Equal(t, testAddr1, addr3.String(), "address")
-	addr4, _ := alloc.Allocate(container3, cidr2.HostRange(), nil)
+	addr4, _ := alloc.Allocate(container3, cidr2.HostRange(), returnFalse)
 	require.Equal(t, testAddr2, addr4.String(), "address")
 
 	alloc.ContainerDied(container2)
@@ -93,7 +95,7 @@ func TestBootstrap(t *testing.T) {
 	ExpectBroadcastMessage(alloc1, nil) // alloc1 will try to form consensus
 	done := make(chan bool)
 	go func() {
-		alloc1.Allocate("somecontainer", subnet, nil)
+		alloc1.Allocate("somecontainer", subnet, returnFalse)
 		done <- true
 	}()
 	time.Sleep(100 * time.Millisecond)
@@ -147,15 +149,15 @@ func TestAllocatorClaim(t *testing.T) {
 	require.NoError(t, err)
 
 	// Do one allocate to ensure paxos is all done
-	alloc.Allocate("unused", subnet, nil)
+	alloc.Allocate("unused", subnet, returnFalse)
 	// Do an allocate on the other peer, which we will try to claim later
-	addrx, err := allocs[0].Allocate(container1, subnet, nil)
+	addrx, err := allocs[0].Allocate(container1, subnet, returnFalse)
 
 	// Now try the claim again
 	err = alloc.Claim(container3, addr1, true)
 	require.NoError(t, err)
 	// Check we get this address back if we try an allocate
-	addr3, _ := alloc.Allocate(container3, subnet, nil)
+	addr3, _ := alloc.Allocate(container3, subnet, returnFalse)
 	require.Equal(t, testAddr1, addr3.String(), "address")
 	// one more claim should still work
 	err = alloc.Claim(container3, addr1, true)
@@ -206,9 +208,9 @@ func TestCancel(t *testing.T) {
 	alloc1.OnGossipBroadcast(alloc2.ourName, alloc2.Encode())
 
 	// Get some IPs, so each allocator has some space
-	res1, _ := alloc1.Allocate("foo", subnet, nil)
+	res1, _ := alloc1.Allocate("foo", subnet, returnFalse)
 	common.Log.Debugf("res1 = %s", res1.String())
-	res2, _ := alloc2.Allocate("bar", subnet, nil)
+	res2, _ := alloc2.Allocate("bar", subnet, returnFalse)
 	common.Log.Debugf("res2 = %s", res2.String())
 	if res1 == res2 {
 		require.FailNow(t, "Error: got same ips!")
@@ -220,12 +222,20 @@ func TestCancel(t *testing.T) {
 
 	// Use up all the IPs that alloc1 owns, so the allocation after this will prompt a request to alloc2
 	for i := 0; alloc1.NumFreeAddresses(subnet) > 0; i++ {
-		alloc1.Allocate(fmt.Sprintf("tmp%d", i), subnet, nil)
+		alloc1.Allocate(fmt.Sprintf("tmp%d", i), subnet, returnFalse)
 	}
 	cancelChan := make(chan bool, 1)
 	doneChan := make(chan bool)
 	go func() {
-		_, ok := alloc1.Allocate("baz", subnet, cancelChan)
+		_, ok := alloc1.Allocate("baz", subnet,
+			func() bool {
+				select {
+				case <-cancelChan:
+					return true
+				default:
+					return false
+				}
+			})
 		doneChan <- ok == nil
 	}()
 
@@ -252,7 +262,7 @@ func TestCancelOnDied(t *testing.T) {
 
 	doneChan := make(chan bool)
 	f := func() {
-		_, ok := alloc1.Allocate(container1, subnet, nil)
+		_, ok := alloc1.Allocate(container1, subnet, returnFalse)
 		doneChan <- ok == nil
 	}
 
@@ -283,11 +293,11 @@ func TestGossipShutdown(t *testing.T) {
 	defer alloc.Stop()
 
 	alloc.claimRingForTesting()
-	alloc.Allocate(container1, subnet, nil)
+	alloc.Allocate(container1, subnet, returnFalse)
 
 	alloc.Shutdown()
 
-	_, err := alloc.Allocate(container2, subnet, nil) // trying to allocate after shutdown should fail
+	_, err := alloc.Allocate(container2, subnet, returnFalse) // trying to allocate after shutdown should fail
 	require.False(t, err == nil, "no address")
 
 	CheckAllExpectedMessagesSent(alloc)
@@ -302,10 +312,10 @@ func TestTransfer(t *testing.T) {
 	alloc2 := allocs[1]
 	alloc3 := allocs[2] // This will be 'master' and get the first range
 
-	_, err := alloc2.Allocate("foo", subnet, nil)
+	_, err := alloc2.Allocate("foo", subnet, returnFalse)
 	require.True(t, err == nil, "Failed to get address")
 
-	_, err = alloc3.Allocate("bar", subnet, nil)
+	_, err = alloc3.Allocate("bar", subnet, returnFalse)
 	require.True(t, err == nil, "Failed to get address")
 
 	alloc2.gossip.GossipBroadcast(alloc2.Gossip())
@@ -323,7 +333,7 @@ func TestTransfer(t *testing.T) {
 
 	require.Equal(t, address.Offset(1022), alloc1.NumFreeAddresses(subnet))
 
-	_, err = alloc1.Allocate("foo", subnet, nil)
+	_, err = alloc1.Allocate("foo", subnet, returnFalse)
 	require.True(t, err == nil, "Failed to get address")
 	alloc1.Stop()
 }
@@ -338,7 +348,7 @@ func TestFakeRouterSimple(t *testing.T) {
 	alloc1 := allocs[0]
 	//alloc2 := allocs[1]
 
-	_, err := alloc1.Allocate("foo", subnet, nil)
+	_, err := alloc1.Allocate("foo", subnet, returnFalse)
 	require.NoError(t, err, "Failed to get address")
 }
 
@@ -419,7 +429,7 @@ func TestAllocatorFuzz(t *testing.T) {
 		allocIndex := rand.Int31n(nodes)
 		alloc := allocs[allocIndex]
 		//common.Log.Infof("Allocate: asking allocator %d", allocIndex)
-		addr, err := alloc.Allocate(name, subnet, nil)
+		addr, err := alloc.Allocate(name, subnet, returnFalse)
 
 		if err != nil {
 			panic(fmt.Sprintf("Could not allocate addr"))
@@ -476,7 +486,7 @@ func TestAllocatorFuzz(t *testing.T) {
 
 		//common.Log.Infof("Asking for %s (%s) on allocator %d again", res.name, addr, res.alloc)
 
-		newAddr, _ := alloc.Allocate(res.name, subnet, nil)
+		newAddr, _ := alloc.Allocate(res.name, subnet, returnFalse)
 		oldAddr, _ := address.ParseIP(addr)
 		if newAddr != oldAddr {
 			panic(fmt.Sprintf("Got different address for repeat request for %s: %s != %s", res.name, newAddr, oldAddr))

ipam/http.go

@@ -31,7 +31,17 @@ func parseCIDR(w http.ResponseWriter, cidrStr string) (address.CIDR, bool) {
 
 func (alloc *Allocator) handleHTTPAllocate(dockerCli *docker.Client, w http.ResponseWriter, ident string, checkAlive bool, subnet address.CIDR) {
 	closedChan := w.(http.CloseNotifier).CloseNotify()
-	addr, err := alloc.Allocate(ident, subnet.HostRange(), closedChan)
+	addr, err := alloc.Allocate(ident, subnet.HostRange(),
+		func() bool {
+			select {
+			case <-closedChan:
+				return true
+			default:
+				res := checkAlive && dockerCli != nil && dockerCli.IsContainerNotRunning(ident)
+				checkAlive = false // we check only once; if the container dies later we learn about that through events
+				return res
+			}
+		})
 	if err != nil {
 		if _, ok := err.(*errorCancelled); ok { // cancellation is not really an error
 			common.Log.Infoln("[allocator]:", err.Error())
@@ -41,12 +51,6 @@ func (alloc *Allocator) handleHTTPAllocate(dockerCli *docker.Client, w http.Resp
 		badRequest(w, err)
 		return
 	}
-	if checkAlive && dockerCli != nil && dockerCli.IsContainerNotRunning(ident) {
-		common.Log.Infof("[allocator] '%s' is not running: freeing %s", ident, addr)
-		alloc.Free(ident, addr)
-		fmt.Fprint(w, "cancelled")
-		return
-	}
 
 	fmt.Fprintf(w, "%s/%d", addr, subnet.PrefixLen)
 }

ipam/ring/ring.go

@@ -19,9 +19,10 @@ import (
 
 // Ring represents the ring itself
 type Ring struct {
-	Start, End address.Address // [min, max) tokens in this ring.  Due to wrapping, min == max (effectively)
-	Peer       router.PeerName // name of peer owning this ring instance
-	Entries    entries         // list of entries sorted by token
+	Start, End address.Address   // [min, max) tokens in this ring.  Due to wrapping, min == max (effectively)
+	Peer       router.PeerName   // name of peer owning this ring instance
+	Entries    entries           // list of entries sorted by token
+	Seeds      []router.PeerName // peers with which the ring was seeded
 }
 
 func (r *Ring) assertInvariants() {
@@ -36,6 +37,7 @@ var (
 	ErrNotSorted       = errors.New("Ring not sorted")
 	ErrTokenRepeated   = errors.New("Token appears twice in ring")
 	ErrTokenOutOfRange = errors.New("Token is out of range")
+	ErrDifferentSeeds  = errors.New("Received ring was seeded differently from ours!")
 	ErrDifferentRange  = errors.New("Received range differs from ours!")
 	ErrNewerVersion    = errors.New("Received new version for entry I own!")
 	ErrInvalidEntry    = errors.New("Received invalid state update!")
@@ -188,6 +190,17 @@ func (r *Ring) Merge(gossip Ring) error {
 		return err
 	}
 
+	if len(gossip.Seeds) > 0 && len(r.Seeds) > 0 {
+		if len(gossip.Seeds) != len(r.Seeds) {
+			return ErrDifferentSeeds
+		}
+		for i, seed := range gossip.Seeds {
+			if seed != r.Seeds[i] {
+				return ErrDifferentSeeds
+			}
+		}
+	}
+
 	if r.Start != gossip.Start || r.End != gossip.End {
 		return ErrDifferentRange
 	}
@@ -254,6 +267,9 @@ func (r *Ring) Merge(gossip Ring) error {
 		previousOwner = nil
 	}
 
+	if len(r.Seeds) == 0 {
+		r.Seeds = gossip.Seeds
+	}
 	r.Entries = result
 	return nil
 }
@@ -330,6 +346,8 @@ func (r *Ring) ClaimForPeers(peers []router.PeerName) {
 	}
 
 	common.Assert(pos == r.End)
+
+	r.Seeds = peers
 }
 
 func (r *Ring) FprintWithNicknames(w io.Writer, m map[router.PeerName]string) {
@@ -452,9 +470,9 @@ func (r *Ring) ChoosePeersToAskForSpace(start, end address.Address) []router.Pee
 	return result
 }
 
-func (r *Ring) PickPeerForTransfer() router.PeerName {
+func (r *Ring) PickPeerForTransfer(isValid func(router.PeerName) bool) router.PeerName {
 	for _, entry := range r.Entries {
-		if entry.Peer != r.Peer {
+		if entry.Peer != r.Peer && isValid(entry.Peer) {
 			return entry.Peer
 		}
 	}

ipam/testutils_test.go

@@ -135,7 +135,7 @@ func makeAllocator(name string, cidrStr string, quorum uint) (*Allocator, addres
 	}
 
 	alloc := NewAllocator(peername, router.PeerUID(rand.Int63()),
-		"nick-"+name, cidr.Range(), quorum)
+		"nick-"+name, cidr.Range(), quorum, func(router.PeerName) bool { return true })
 
 	return alloc, cidr.HostRange()
 }

nameserver/dns.go

@@ -216,7 +216,7 @@ func (h *handler) handleRecursive(w dns.ResponseWriter, req *dns.Msg) {
 		reqCopy := req.Copy()
 		reqCopy.Id = dns.Id()
 		response, _, err := h.client.Exchange(reqCopy, fmt.Sprintf("%s:%s", server, h.upstream.Port))
-		if err != nil || response == nil {
+		if (err != nil && err != dns.ErrTruncated) || response == nil {
 			h.ns.debugf("error trying %s: %v", server, err)
 			continue
 		}

nameserver/dns_test.go

@@ -18,7 +18,7 @@ import (
 func startServer(t *testing.T, upstream *dns.ClientConfig) (*DNSServer, *Nameserver, int, int) {
 	peername, err := router.PeerNameFromString("00:00:00:02:00:00")
 	require.Nil(t, err)
-	nameserver := New(peername, nil, nil, "")
+	nameserver := New(peername, nil, "")
 	dnsserver, err := NewDNSServer(nameserver, "weave.local.", "0.0.0.0:0", "", 30, 5*time.Second)
 	require.Nil(t, err)
 	udpPort := dnsserver.servers[0].PacketConn.LocalAddr().(*net.UDPAddr).Port
@@ -42,17 +42,17 @@ func TestTruncation(t *testing.T) {
 		nameserver.AddEntry("foo.weave.local.", "", router.UnknownPeerName, i)
 	}
 
-	doRequest := func(client *dns.Client, request *dns.Msg, port int) *dns.Msg {
+	doRequest := func(client *dns.Client, request *dns.Msg, port int, expectedErr error) *dns.Msg {
 		request.SetQuestion("foo.weave.local.", dns.TypeA)
 		response, _, err := client.Exchange(request, fmt.Sprintf("127.0.0.1:%d", port))
-		require.Nil(t, err)
+		require.Equal(t, expectedErr, err)
 		return response
 	}
 
 	// do a udp query, ensure we get a truncated response
 	{
 		udpClient := dns.Client{Net: "udp", UDPSize: minUDPSize}
-		response := doRequest(&udpClient, &dns.Msg{}, udpPort)
+		response := doRequest(&udpClient, &dns.Msg{}, udpPort, dns.ErrTruncated)
 		require.True(t, response.MsgHdr.Truncated)
 		require.True(t, len(response.Answer) < 100)
 	}
@@ -62,7 +62,7 @@ func TestTruncation(t *testing.T) {
 		udpClient := dns.Client{Net: "udp", UDPSize: 65535}
 		request := &dns.Msg{}
 		request.SetEdns0(65535, false)
-		response := doRequest(&udpClient, request, udpPort)
+		response := doRequest(&udpClient, request, udpPort, nil)
 		require.False(t, response.MsgHdr.Truncated)
 		require.Equal(t, len(response.Answer), 100)
 	}
@@ -70,7 +70,7 @@ func TestTruncation(t *testing.T) {
 	// do a tcp query, ensure we don't get a truncated response
 	{
 		tcpClient := dns.Client{Net: "tcp"}
-		response := doRequest(&tcpClient, &dns.Msg{}, tcpPort)
+		response := doRequest(&tcpClient, &dns.Msg{}, tcpPort, nil)
 		require.False(t, response.MsgHdr.Truncated)
 		require.Equal(t, len(response.Answer), 100)
 	}

nameserver/entry.go

@@ -5,6 +5,7 @@ import (
 	"encoding/gob"
 	"fmt"
 	"sort"
+	"strings"
 	"time"
 
 	"github.com/weaveworks/weave/net/address"
@@ -17,14 +18,33 @@ type Entry struct {
 	ContainerID string
 	Origin      router.PeerName
 	Addr        address.Address
-	Hostname    string
+	Hostname    string // as supplied
+	lHostname   string // lowercased (not exported, so not encoded by gob)
 	Version     int
 	Tombstone   int64 // timestamp of when it was deleted
 }
 
 type Entries []Entry
+type CaseSensitive Entries
+type CaseInsensitive Entries
+type SortableEntries interface {
+	sort.Interface
+	Get(i int) Entry
+}
 
-func (e1 *Entry) equal(e2 *Entry) bool {
+// Gossip messages are sorted in a case sensitive order...
+func (es CaseSensitive) Len() int           { return len(es) }
+func (es CaseSensitive) Swap(i, j int)      { es[i], es[j] = es[j], es[i] }
+func (es CaseSensitive) Get(i int) Entry    { return es[i] }
+func (es CaseSensitive) Less(i, j int) bool { return es[i].less(&es[j]) }
+
+// ... but we store entries in a case insensitive order.
+func (es CaseInsensitive) Len() int           { return len(es) }
+func (es CaseInsensitive) Swap(i, j int)      { es[i], es[j] = es[j], es[i] }
+func (es CaseInsensitive) Get(i int) Entry    { return es[i] }
+func (es CaseInsensitive) Less(i, j int) bool { return es[i].insensitiveLess(&es[j]) }
+
+func (e1 Entry) equal(e2 Entry) bool {
 	return e1.ContainerID == e2.ContainerID &&
 		e1.Origin == e2.Origin &&
 		e1.Addr == e2.Addr &&
@@ -48,6 +68,24 @@ func (e1 *Entry) less(e2 *Entry) bool {
 	}
 }
 
+func (e1 *Entry) insensitiveLess(e2 *Entry) bool {
+	// Entries are kept sorted by Hostname, Origin, ContainerID then address
+	e1Hostname, e2Hostname := e1.lHostname, e2.lHostname
+	switch {
+	case e1Hostname != e2Hostname:
+		return e1Hostname < e2Hostname
+
+	case e1.Origin != e2.Origin:
+		return e1.Origin < e2.Origin
+
+	case e1.ContainerID != e2.ContainerID:
+		return e1.ContainerID < e2.ContainerID
+
+	default:
+		return e1.Addr < e2.Addr
+	}
+}
+
 // returns true to indicate a change
 func (e1 *Entry) merge(e2 *Entry) bool {
 	// we know container id, origin, add and hostname are equal
@@ -63,37 +101,42 @@ func (e1 *Entry) String() string {
 	return fmt.Sprintf("%s -> %s", e1.Hostname, e1.Addr.String())
 }
 
-func (es Entries) Len() int           { return len(es) }
-func (es Entries) Swap(i, j int)      { panic("Swap") }
-func (es Entries) Less(i, j int) bool { return es[i].less(&es[j]) }
+func (e1 *Entry) addLowercase() {
+	e1.lHostname = strings.ToLower(e1.Hostname)
+}
 
-func (es Entries) check() error {
+func check(es SortableEntries) error {
 	if !sort.IsSorted(es) {
 		return fmt.Errorf("Not sorted!")
 	}
-	for i := 1; i < len(es); i++ {
-		if es[i].equal(&es[i-1]) {
-			return fmt.Errorf("Duplicate entry: %d:%v and %d:%v", i-1, es[i-1], i, es[i])
+	for i := 1; i < es.Len(); i++ {
+		if es.Get(i).equal(es.Get(i - 1)) {
+			return fmt.Errorf("Duplicate entry: %d:%v and %d:%v", i-1, es.Get(i-1), i, es.Get(i))
 		}
 	}
 	return nil
 }
 
-func (es *Entries) checkAndPanic() {
-	if err := es.check(); err != nil {
+func checkAndPanic(es SortableEntries) {
+	if err := check(es); err != nil {
 		panic(err)
 	}
 }
 
-func (es *Entries) add(hostname, containerid string, origin router.PeerName, addr address.Address) Entry {
-	es.checkAndPanic()
-	defer es.checkAndPanic()
+func (es *Entries) checkAndPanic() *Entries {
+	checkAndPanic(CaseInsensitive(*es))
+	return es
+}
 
-	entry := Entry{Hostname: hostname, Origin: origin, ContainerID: containerid, Addr: addr}
+func (es *Entries) add(hostname, containerid string, origin router.PeerName, addr address.Address) Entry {
+	defer es.checkAndPanic().checkAndPanic()
+
+	entry := Entry{Hostname: hostname, lHostname: strings.ToLower(hostname),
+		Origin: origin, ContainerID: containerid, Addr: addr}
 	i := sort.Search(len(*es), func(i int) bool {
-		return !(*es)[i].less(&entry)
+		return !(*es)[i].insensitiveLess(&entry)
 	})
-	if i < len(*es) && (*es)[i].equal(&entry) {
+	if i < len(*es) && (*es)[i].equal(entry) {
 		if (*es)[i].Tombstone > 0 {
 			(*es)[i].Tombstone = 0
 			(*es)[i].Version++
@@ -107,16 +150,17 @@ func (es *Entries) add(hostname, containerid string, origin router.PeerName, add
 }
 
 func (es *Entries) merge(incoming Entries) Entries {
-	es.checkAndPanic()
-	defer es.checkAndPanic()
+	defer es.checkAndPanic().checkAndPanic()
+	incoming.checkAndPanic()
+
 	newEntries := Entries{}
 	i := 0
 
 	for _, entry := range incoming {
-		for i < len(*es) && (*es)[i].less(&entry) {
+		for i < len(*es) && (*es)[i].insensitiveLess(&entry) {
 			i++
 		}
-		if i < len(*es) && (*es)[i].equal(&entry) {
+		if i < len(*es) && (*es)[i].equal(entry) {
 			if (*es)[i].merge(&entry) {
 				newEntries = append(newEntries, entry)
 			}
@@ -133,8 +177,8 @@ func (es *Entries) merge(incoming Entries) Entries {
 
 // f returning true means keep the entry.
 func (es *Entries) tombstone(ourname router.PeerName, f func(*Entry) bool) Entries {
-	es.checkAndPanic()
-	defer es.checkAndPanic()
+	defer es.checkAndPanic().checkAndPanic()
+
 	tombstoned := Entries{}
 	for i, e := range *es {
 		if f(&e) && e.Origin == ourname {
@@ -148,8 +192,8 @@ func (es *Entries) tombstone(ourname router.PeerName, f func(*Entry) bool) Entri
 }
 
 func (es *Entries) filter(f func(*Entry) bool) {
-	es.checkAndPanic()
-	defer es.checkAndPanic()
+	defer es.checkAndPanic().checkAndPanic()
+
 	i := 0
 	for _, e := range *es {
 		if !f(&e) {
@@ -163,23 +207,26 @@ func (es *Entries) filter(f func(*Entry) bool) {
 
 func (es Entries) lookup(hostname string) Entries {
 	es.checkAndPanic()
+
+	lowerHostname := strings.ToLower(hostname)
 	i := sort.Search(len(es), func(i int) bool {
-		return es[i].Hostname >= hostname
+		return es[i].lHostname >= lowerHostname
 	})
-	if i >= len(es) || es[i].Hostname != hostname {
+	if i >= len(es) || es[i].lHostname != lowerHostname {
 		return Entries{}
 	}
 
 	j := sort.Search(len(es)-i, func(j int) bool {
-		return es[i+j].Hostname > hostname
+		return es[i+j].lHostname > lowerHostname
 	})
 
 	return es[i : i+j]
 }
 
-func (es *Entries) first(f func(*Entry) bool) (*Entry, error) {
+func (es Entries) first(f func(*Entry) bool) (*Entry, error) {
 	es.checkAndPanic()
-	for _, e := range *es {
+
+	for _, e := range es {
 		if f(&e) {
 			return &e, nil
 		}
@@ -187,6 +234,12 @@ func (es *Entries) first(f func(*Entry) bool) (*Entry, error) {
 	return nil, fmt.Errorf("Not found")
 }
 
+func (es Entries) addLowercase() {
+	for i := range es {
+		es[i].addLowercase()
+	}
+}
+
 type GossipData struct {
 	Timestamp int64
 	Entries
@@ -200,9 +253,23 @@ func (g *GossipData) Merge(o router.GossipData) {
 	}
 }
 
+func (g *GossipData) Decode(msg []byte) error {
+	if err := gob.NewDecoder(bytes.NewReader(msg)).Decode(g); err != nil {
+		return err
+	}
+
+	g.Entries.addLowercase() // lowercase strings not sent on the wire
+	sort.Sort(CaseInsensitive(g.Entries))
+	return nil
+}
+
 func (g *GossipData) Encode() [][]byte {
+	// Make a copy so we can sort: all outgoing data is sent in case-sensitive order
+	g2 := GossipData{Timestamp: g.Timestamp, Entries: make(Entries, len(g.Entries))}
+	copy(g2.Entries, g.Entries)
+	sort.Sort(CaseSensitive(g2.Entries))
 	buf := &bytes.Buffer{}
-	if err := gob.NewEncoder(buf).Encode(g); err != nil {
+	if err := gob.NewEncoder(buf).Encode(g2); err != nil {
 		panic(err)
 	}
 	return [][]byte{buf.Bytes()}

nameserver/entry_test.go

@@ -9,6 +9,19 @@ import (
 	"github.com/weaveworks/weave/router"
 )
 
+func l(es Entries) Entries {
+	es.addLowercase()
+	return es
+}
+
+func makeEntries(values string) Entries {
+	entries := make(Entries, len(values))
+	for i, c := range values {
+		entries[i] = Entry{Hostname: string(c)}
+	}
+	return l(entries)
+}
+
 func TestAdd(t *testing.T) {
 	oldNow := now
 	defer func() { now = oldNow }()
@@ -16,68 +29,46 @@ func TestAdd(t *testing.T) {
 
 	entries := Entries{}
 	entries.add("A", "", router.UnknownPeerName, address.Address(0))
-	expected := Entries{
+	expected := l(Entries{
 		Entry{Hostname: "A", Origin: router.UnknownPeerName, Addr: address.Address(0)},
-	}
+	})
 	require.Equal(t, entries, expected)
 
 	entries.tombstone(router.UnknownPeerName, func(e *Entry) bool { return e.Hostname == "A" })
-	expected = Entries{
+	expected = l(Entries{
 		Entry{Hostname: "A", Origin: router.UnknownPeerName, Addr: address.Address(0), Version: 1, Tombstone: 1234},
-	}
+	})
 	require.Equal(t, entries, expected)
 
 	entries.add("A", "", router.UnknownPeerName, address.Address(0))
-	expected = Entries{
+	expected = l(Entries{
 		Entry{Hostname: "A", Origin: router.UnknownPeerName, Addr: address.Address(0), Version: 2},
-	}
+	})
 	require.Equal(t, entries, expected)
 }
 
 func TestMerge(t *testing.T) {
-	e1 := Entries{
-		Entry{Hostname: "A"},
-		Entry{Hostname: "C"},
-		Entry{Hostname: "D"},
-		Entry{Hostname: "F"},
-	}
-
-	e2 := Entries{
-		Entry{Hostname: "B"},
-		Entry{Hostname: "E"},
-		Entry{Hostname: "F"},
-	}
+	e1 := makeEntries("ACDF")
+	e2 := makeEntries("BEF")
 
 	diff := e1.merge(e2)
-	expectedDiff := Entries{
-		Entry{Hostname: "B"},
-		Entry{Hostname: "E"},
-	}
-	require.Equal(t, expectedDiff, diff)
 
-	expected := Entries{
-		Entry{Hostname: "A"},
-		Entry{Hostname: "B"},
-		Entry{Hostname: "C"},
-		Entry{Hostname: "D"},
-		Entry{Hostname: "E"},
-		Entry{Hostname: "F"},
-	}
-	require.Equal(t, expected, e1)
+	require.Equal(t, makeEntries("BE"), diff)
+	require.Equal(t, makeEntries("ABCDEF"), e1)
 
 	diff = e1.merge(e1)
 	require.Equal(t, Entries{}, diff)
 }
 
 func TestOldMerge(t *testing.T) {
-	e1 := Entries{Entry{Hostname: "A", Version: 0}}
-	diff := e1.merge(Entries{Entry{Hostname: "A", Version: 1}})
-	require.Equal(t, Entries{Entry{Hostname: "A", Version: 1}}, diff)
-	require.Equal(t, Entries{Entry{Hostname: "A", Version: 1}}, e1)
+	e1 := l(Entries{Entry{Hostname: "A", Version: 0}})
+	diff := e1.merge(l(Entries{Entry{Hostname: "A", Version: 1}}))
+	require.Equal(t, l(Entries{Entry{Hostname: "A", Version: 1}}), diff)
+	require.Equal(t, l(Entries{Entry{Hostname: "A", Version: 1}}), e1)
 
-	diff = e1.merge(Entries{Entry{Hostname: "A", Version: 0}})
+	diff = e1.merge(l(Entries{Entry{Hostname: "A", Version: 0}}))
 	require.Equal(t, Entries{}, diff)
-	require.Equal(t, Entries{Entry{Hostname: "A", Version: 1}}, e1)
+	require.Equal(t, l(Entries{Entry{Hostname: "A", Version: 1}}), e1)
 }
 
 func TestTombstone(t *testing.T) {
@@ -85,48 +76,48 @@ func TestTombstone(t *testing.T) {
 	defer func() { now = oldNow }()
 	now = func() int64 { return 1234 }
 
-	es := Entries{
-		Entry{Hostname: "A"},
-		Entry{Hostname: "B"},
-	}
+	es := makeEntries("AB")
 
 	es.tombstone(router.UnknownPeerName, func(e *Entry) bool {
 		return e.Hostname == "B"
 	})
-	expected := Entries{
+	expected := l(Entries{
 		Entry{Hostname: "A"},
 		Entry{Hostname: "B", Version: 1, Tombstone: 1234},
-	}
+	})
 	require.Equal(t, expected, es)
 }
 
 func TestDelete(t *testing.T) {
-	es := Entries{
-		Entry{Hostname: "A"},
-		Entry{Hostname: "B"},
-	}
+	es := makeEntries("AB")
 
 	es.filter(func(e *Entry) bool {
 		return e.Hostname != "A"
 	})
-	expected := Entries{
-		Entry{Hostname: "B"},
-	}
-	require.Equal(t, expected, es)
+	require.Equal(t, makeEntries("B"), es)
 }
 
 func TestLookup(t *testing.T) {
-	es := Entries{
+	es := l(Entries{
 		Entry{Hostname: "A"},
 		Entry{Hostname: "B", ContainerID: "bar"},
 		Entry{Hostname: "B", ContainerID: "foo"},
 		Entry{Hostname: "C"},
-	}
+	})
 
 	have := es.lookup("B")
-	want := Entries{
+	want := l(Entries{
 		Entry{Hostname: "B", ContainerID: "bar"},
 		Entry{Hostname: "B", ContainerID: "foo"},
-	}
+	})
 	require.Equal(t, have, want)
 }
+
+func TestGossipDataMerge(t *testing.T) {
+	g1 := GossipData{Entries: makeEntries("AcDf")}
+	g2 := GossipData{Entries: makeEntries("BEf")}
+
+	g1.Merge(&g2)
+
+	require.Equal(t, GossipData{Entries: makeEntries("ABcDEf")}, g1)
+}

nameserver/http.go

@@ -8,6 +8,7 @@ import (
 	"github.com/gorilla/mux"
 	"github.com/miekg/dns"
 
+	"github.com/weaveworks/weave/common/docker"
 	"github.com/weaveworks/weave/net/address"
 )
 
@@ -16,7 +17,7 @@ func (n *Nameserver) badRequest(w http.ResponseWriter, err error) {
 	n.infof("%v", err)
 }
 
-func (n *Nameserver) HandleHTTP(router *mux.Router) {
+func (n *Nameserver) HandleHTTP(router *mux.Router, dockerCli *docker.Client) {
 	router.Methods("GET").Path("/domain").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		fmt.Fprint(w, n.domain)
 	})
@@ -34,12 +35,17 @@ func (n *Nameserver) HandleHTTP(router *mux.Router) {
 			return
 		}
 
+		if !dns.IsSubDomain(n.domain, hostname) {
+			n.infof("Ignoring registration %s %s %s (not a subdomain of %s)", hostname, ipStr, container, n.domain)
+			return
+		}
+
 		if err := n.AddEntry(hostname, container, n.ourName, ip); err != nil {
 			n.badRequest(w, fmt.Errorf("Unable to add entry: %v", err))
 			return
 		}
 
-		if r.FormValue("check-alive") == "true" && n.docker.IsContainerNotRunning(container) {
+		if r.FormValue("check-alive") == "true" && dockerCli != nil && dockerCli.IsContainerNotRunning(container) {
 			n.infof("container '%s' is not running: removing", container)
 			if err := n.Delete(hostname, container, ipStr, ip); err != nil {
 				n.infof("failed to remove: %v", err)

nameserver/nameserver.go

@@ -1,8 +1,6 @@
 package nameserver
 
 import (
-	"bytes"
-	"encoding/gob"
 	"fmt"
 	"sync"
 	"time"
@@ -10,7 +8,6 @@ import (
 	"github.com/miekg/dns"
 
 	. "github.com/weaveworks/weave/common"
-	"github.com/weaveworks/weave/common/docker"
 	"github.com/weaveworks/weave/net/address"
 	"github.com/weaveworks/weave/router"
 )
@@ -40,18 +37,16 @@ type Nameserver struct {
 	ourName router.PeerName
 	domain  string
 	gossip  router.Gossip
-	docker  *docker.Client
 	entries Entries
 	peers   *router.Peers
 	quit    chan struct{}
 }
 
-func New(ourName router.PeerName, peers *router.Peers, docker *docker.Client, domain string) *Nameserver {
+func New(ourName router.PeerName, peers *router.Peers, domain string) *Nameserver {
 	ns := &Nameserver{
 		ourName: ourName,
 		domain:  dns.Fqdn(domain),
 		peers:   peers,
-		docker:  docker,
 		quit:    make(chan struct{}),
 	}
 	if peers != nil {
@@ -130,6 +125,8 @@ func (n *Nameserver) ReverseLookup(ip address.Address) (string, error) {
 	return match.Hostname, nil
 }
 
+func (n *Nameserver) ContainerStarted(ident string) {}
+
 func (n *Nameserver) ContainerDied(ident string) {
 	n.Lock()
 	entries := n.entries.tombstone(n.ourName, func(e *Entry) bool {
@@ -205,18 +202,13 @@ func (n *Nameserver) OnGossipUnicast(sender router.PeerName, msg []byte) error {
 
 func (n *Nameserver) receiveGossip(msg []byte) (router.GossipData, router.GossipData, error) {
 	var gossip GossipData
-	if err := gob.NewDecoder(bytes.NewReader(msg)).Decode(&gossip); err != nil {
+	if err := gossip.Decode(msg); err != nil {
 		return nil, nil, err
 	}
-
 	if delta := gossip.Timestamp - now(); delta > gossipWindow || delta < -gossipWindow {
 		return nil, nil, fmt.Errorf("clock skew of %d detected", delta)
 	}
 
-	if err := gossip.Entries.check(); err != nil {
-		return nil, nil, err
-	}
-
 	n.Lock()
 	defer n.Unlock()
 

nameserver/nameserver_test.go

@@ -23,7 +23,7 @@ func makeNetwork(size int) ([]*Nameserver, *gossip.TestRouter) {
 
 	for i := 0; i < size; i++ {
 		name, _ := router.PeerNameFromString(fmt.Sprintf("%02d:00:00:02:00:00", i))
-		nameserver := New(name, nil, nil, "")
+		nameserver := New(name, nil, "")
 		nameserver.SetGossip(gossipRouter.Connect(nameserver.ourName, nameserver))
 		nameserver.Start()
 		nameservers[i] = nameserver
@@ -72,7 +72,7 @@ func (m mapping) Addrs() []address.Address {
 }
 
 func TestNameservers(t *testing.T) {
-	wt.RunWithTimeout(t, time.Minute, func() {
+	wt.RunWithTimeout(t, 2*time.Minute, func() {
 		testNameservers(t)
 	})
 }
@@ -81,8 +81,12 @@ func testNameservers(t *testing.T) {
 	//common.SetLogLevel("debug")
 
 	lookupTimeout := 10 // ms
-	nameservers, grouter := makeNetwork(50)
+	nameservers, grouter := makeNetwork(30)
 	defer stopNetwork(nameservers, grouter)
+	// This subset will sometimes lose touch with some of the others
+	badNameservers := nameservers[25:]
+	// This subset will remain well-connected, and we will deal mainly with them
+	nameservers = nameservers[:25]
 	nameserversByName := map[router.PeerName]*Nameserver{}
 	for _, n := range nameservers {
 		nameserversByName[n.ourName] = n
@@ -106,7 +110,13 @@ func testNameservers(t *testing.T) {
 	addMapping := func() {
 		nameserver := nameservers[rand.Intn(len(nameservers))]
 		addr := address.Address(rand.Int31())
-		hostname := fmt.Sprintf("hostname%d", rand.Int63())
+		// Create a hostname which has some upper and lowercase letters,
+		// and a unique number so we don't have to check if we allocated it already
+		randomBits := rand.Int63()
+		firstLetter := 'H' + (randomBits&1)*32
+		secondLetter := 'O' + (randomBits&2)*16
+		randomBits = randomBits >> 2
+		hostname := fmt.Sprintf("%c%cstname%d", firstLetter, secondLetter, randomBits)
 		mapping := mapping{hostname, []pair{{nameserver.ourName, addr}}}
 		mappings = append(mappings, mapping)
 
@@ -129,6 +139,12 @@ func testNameservers(t *testing.T) {
 		check(nameserver, mapping)
 	}
 
+	loseConnection := func() {
+		nameserver1 := badNameservers[rand.Intn(len(badNameservers))]
+		nameserver2 := nameservers[rand.Intn(len(nameservers))]
+		nameserver1.PeerGone(&router.Peer{Name: nameserver2.ourName})
+	}
+
 	deleteMapping := func() {
 		if len(mappings) <= 0 {
 			return
@@ -179,7 +195,7 @@ func testNameservers(t *testing.T) {
 		require.Equal(t, mapping.hostname, hostname)
 	}
 
-	for i := 0; i < 1000; i++ {
+	for i := 0; i < 800; i++ {
 		r := rand.Float32()
 		switch {
 		case r < 0.1:
@@ -191,7 +207,10 @@ func testNameservers(t *testing.T) {
 		case 0.2 <= r && r < 0.3:
 			deleteMapping()
 
-		case 0.3 <= r && r < 0.9:
+		case 0.3 <= r && r < 0.35:
+			loseConnection()
+
+		case 0.35 <= r && r < 0.9:
 			doLookup()
 
 		case 0.9 <= r:
@@ -205,7 +224,7 @@ func testNameservers(t *testing.T) {
 func TestContainerAndPeerDeath(t *testing.T) {
 	peername, err := router.PeerNameFromString("00:00:00:02:00:00")
 	require.Nil(t, err)
-	nameserver := New(peername, nil, nil, "")
+	nameserver := New(peername, nil, "")
 
 	err = nameserver.AddEntry("hostname", "containerid", peername, address.Address(0))
 	require.Nil(t, err)
@@ -229,7 +248,7 @@ func TestTombstoneDeletion(t *testing.T) {
 
 	peername, err := router.PeerNameFromString("00:00:00:02:00:00")
 	require.Nil(t, err)
-	nameserver := New(peername, nil, nil, "")
+	nameserver := New(peername, nil, "")
 
 	err = nameserver.AddEntry("hostname", "containerid", peername, address.Address(0))
 	require.Nil(t, err)
@@ -241,14 +260,14 @@ func TestTombstoneDeletion(t *testing.T) {
 	err = nameserver.Delete("hostname", "containerid", "", address.Address(0))
 	require.Nil(t, err)
 	require.Equal(t, []address.Address{}, nameserver.Lookup("hostname"))
-	require.Equal(t, Entries{{
+	require.Equal(t, l(Entries{Entry{
 		ContainerID: "containerid",
 		Origin:      peername,
 		Addr:        address.Address(0),
 		Hostname:    "hostname",
 		Version:     1,
 		Tombstone:   1234,
-	}}, nameserver.entries)
+	}}), nameserver.entries)
 
 	now = func() int64 { return 1234 + int64(tombstoneTimeout/time.Second) + 1 }
 	nameserver.deleteTombstones()

prog/docker_tls_args/main.go

@@ -0,0 +1,77 @@
+// docker_tls_args: find the docker daemon's tls args
+// This reimplements pgrep, because pgrep will only look in /proc, but not in
+// $PROCFS.
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+)
+
+func main() {
+	procRoot := os.Getenv("PROCFS")
+	if procRoot == "" {
+		procRoot = "/proc"
+	}
+
+	dirEntries, err := ioutil.ReadDir(procRoot)
+	checkErr(err)
+
+	for _, dirEntry := range dirEntries {
+		dirName := dirEntry.Name()
+		if _, err := strconv.Atoi(dirName); err != nil {
+			continue
+		}
+
+		if comm, err := ioutil.ReadFile(filepath.Join(procRoot, dirName, "comm")); err != nil || string(comm) != "docker\n" {
+			continue
+		}
+
+		cmdline, err := ioutil.ReadFile(filepath.Join(procRoot, dirName, "cmdline"))
+		if err != nil {
+			continue
+		}
+
+		isDaemon := false
+		tlsArgs := []string{}
+		args := bytes.Split(cmdline, []byte{'\000'})
+		for i := 0; i < len(args); i++ {
+			arg := string(args[i])
+			switch {
+			case arg == "-d" || arg == "daemon":
+				isDaemon = true
+				break
+			case arg == "--tls", arg == "--tlsverify":
+				tlsArgs = append(tlsArgs, arg)
+			case strings.HasPrefix(arg, "--tls"):
+				tlsArgs = append(tlsArgs, arg)
+				if len(args) > i+1 &&
+					!strings.Contains(arg, "=") &&
+					!strings.HasPrefix(string(args[i+1]), "-") {
+					tlsArgs = append(tlsArgs, string(args[i+1]))
+					i++
+				}
+			}
+		}
+		if !isDaemon {
+			continue
+		}
+
+		fmt.Println(strings.Join(tlsArgs, " "))
+		os.Exit(0)
+	}
+
+	os.Exit(1)
+}
+
+func checkErr(err error) {
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+}

prog/weaveexec/Dockerfile

@@ -1,10 +1,11 @@
-FROM gliderlabs/alpine
+FROM alpine
 
 MAINTAINER Weaveworks Inc <help@weave.works>
 LABEL works.weave.role=system
 
 WORKDIR /home/weave
 VOLUME /w
+VOLUME /w-noop
 ENTRYPOINT ["/home/weave/sigproxy", "/home/weave/weave"]
 
 RUN apk add --update \
@@ -18,6 +19,7 @@ RUN apk add --update \
   && rm -rf /var/cache/apk/*
 
 ADD ./weave ./sigproxy ./weaveproxy /home/weave/
-ADD ./netcheck ./weavehosts /usr/bin/
+ADD ./netcheck ./docker_tls_args /usr/bin/
 ADD ./weavewait /w/w
+ADD ./weavewait_noop /w-noop/w
 ADD ./docker.tgz /

prog/weavehosts/weavehosts.go

@@ -1,120 +0,0 @@
-/* weavehosts: rewrite /etc/hosts as necessary to put the weave address(es) at the top */
-package main
-
-import (
-	"bufio"
-	"bytes"
-	"fmt"
-	"io/ioutil"
-	"net"
-	"os"
-	"sort"
-	"strings"
-)
-
-func main() {
-	args := os.Args
-	if len(args) < 4 {
-		fmt.Fprintln(os.Stderr, "usage: weavehosts hostpath hostname cidr [cidr ...]")
-		os.Exit(1)
-	}
-
-	checkErr(updateHosts(args[1], args[2], args[3:]))
-
-	os.Exit(0)
-}
-
-func checkErr(err error) {
-	if err != nil {
-		fmt.Fprintln(os.Stderr, err)
-		os.Exit(1)
-	}
-}
-
-func updateHosts(path, hostname string, cidrs []string) error {
-	hosts, err := parseHosts(path)
-	if err != nil {
-		return err
-	}
-
-	// Remove existing ips pointing to our hostname
-	toRemove := []string{}
-	for ip, addrs := range hosts {
-		for _, addr := range addrs {
-			if addr == hostname {
-				toRemove = append(toRemove, ip)
-				break
-			}
-		}
-	}
-	for _, ip := range toRemove {
-		delete(hosts, ip)
-	}
-
-	// Add the weave ip(s)
-	for _, cidr := range cidrs {
-		ip, _, err := net.ParseCIDR(cidr)
-		checkErr(err)
-		ipStr := ip.String()
-		hosts[ipStr] = append(hosts[ipStr], hostname)
-	}
-
-	return writeHosts(path, hosts)
-}
-
-func parseHosts(path string) (map[string][]string, error) {
-	f, err := os.Open(path)
-	if err != nil {
-		return nil, err
-	}
-	defer f.Close()
-	ips := map[string][]string{}
-	scanner := bufio.NewScanner(f)
-	for scanner.Scan() {
-		line := scanner.Text()
-
-		// Remove any comments
-		if i := strings.IndexByte(line, '#'); i != -1 {
-			line = line[:i]
-		}
-
-		fields := strings.Fields(line)
-		if len(fields) > 0 {
-			ips[fields[0]] = append(ips[fields[0]], fields[1:]...)
-		}
-	}
-	if scanner.Err() != nil {
-		return nil, scanner.Err()
-	}
-	return ips, nil
-}
-
-func writeHosts(path string, contents map[string][]string) error {
-	ips := []string{}
-	for ip := range contents {
-		ips = append(ips, ip)
-	}
-	sort.Strings(ips)
-
-	buf := &bytes.Buffer{}
-	fmt.Fprintln(buf, "# modified by weave")
-	for _, ip := range ips {
-		if addrs := contents[ip]; len(addrs) > 0 {
-			fmt.Fprintf(buf, "%s\t%s\n", ip, strings.Join(uniqueStrs(addrs), " "))
-		}
-	}
-	return ioutil.WriteFile(path, buf.Bytes(), 644)
-}
-
-func uniqueStrs(s []string) []string {
-	m := map[string]struct{}{}
-	result := []string{}
-	for _, str := range s {
-		if _, ok := m[str]; !ok {
-			m[str] = struct{}{}
-			result = append(result, str)
-		}
-	}
-	sort.Strings(result)
-	return result
-}

prog/weaveproxy/main.go

@@ -43,7 +43,7 @@ func main() {
 	mflag.Parse()
 
 	if justVersion {
-		fmt.Printf("weave proxy %s\n", version)
+		fmt.Printf("weave proxy  %s\n", version)
 		os.Exit(0)
 	}
 
@@ -61,6 +61,9 @@ func main() {
 		Log.Fatalf("Could not start proxy: %s", err)
 	}
 
-	go p.ListenAndServe()
+	listeners := p.Listen()
+	p.AttachExistingContainers()
+	go p.Serve(listeners)
+	go p.ListenAndServeStatus("/home/weave/status.sock")
 	SignalHandlerLoop()
 }

prog/weaver/http.go

@@ -89,7 +89,7 @@ var statusTemplate = defTemplate("status", `\
  PeerDiscovery: {{printState .Router.PeerDiscovery}}
        Targets: {{len .Router.Targets}}
    Connections: {{len .Router.Connections}}{{with printConnectionCounts .Router.Connections}} ({{.}}){{end}}
-         Peers: {{len .Router.Peers}}{{with printPeerConnectionCounts .Router.Peers}} (with {{.}} connections between them){{end}}
+         Peers: {{len .Router.Peers}}{{with printPeerConnectionCounts .Router.Peers}} (with {{.}} connections){{end}}
 {{if .IPAM}}\
 
        Service: ipam
@@ -161,7 +161,12 @@ func HandleHTTP(muxRouter *mux.Router, version string, router *weave.Router, all
 	}
 	muxRouter.Methods("GET").Path("/report").Headers("Accept", "application/json").HandlerFunc(
 		func(w http.ResponseWriter, r *http.Request) {
-			json, _ := json.MarshalIndent(status(), "", "    ")
+			json, err := json.MarshalIndent(status(), "", "    ")
+			if err != nil {
+				http.Error(w, err.Error(), http.StatusInternalServerError)
+				Log.Error("Error during report marshalling: ", err)
+				return
+			}
 			w.Header().Set("Content-Type", "application/json")
 			w.Write(json)
 		})

prog/weaver/main.go

@@ -1,7 +1,6 @@
 package main
 
 import (
-	"crypto/sha256"
 	"fmt"
 	"net"
 	"net/http"
@@ -36,34 +35,44 @@ func main() {
 	runtime.GOMAXPROCS(procs)
 
 	var (
-		config             weave.Config
-		justVersion        bool
-		protocolMinVersion int
-		ifaceName          string
-		routerName         string
-		nickName           string
-		password           string
-		pktdebug           bool
-		logLevel           string
-		prof               string
-		bufSzMB            int
-		noDiscovery        bool
-		httpAddr           string
-		iprangeCIDR        string
-		ipsubnetCIDR       string
-		peerCount          int
-		apiPath            string
-		peers              []string
+		// flags that cause immediate exit
+		justVersion          bool
+		createDatapath       bool
+		deleteDatapath       bool
+		addDatapathInterface string
 
+		config                    weave.Config
+		protocolMinVersion        int
+		ifaceName                 string
+		routerName                string
+		nickName                  string
+		password                  string
+		pktdebug                  bool
+		logLevel                  string
+		prof                      string
+		bufSzMB                   int
+		noDiscovery               bool
+		httpAddr                  string
+		iprangeCIDR               string
+		ipsubnetCIDR              string
+		peerCount                 int
+		dockerAPI                 string
+		peers                     []string
 		noDNS                     bool
 		dnsDomain                 string
 		dnsListenAddress          string
 		dnsTTL                    int
 		dnsClientTimeout          time.Duration
 		dnsEffectiveListenAddress string
+		iface                     *net.Interface
+		datapathName              string
 	)
 
 	mflag.BoolVar(&justVersion, []string{"#version", "-version"}, false, "print version and exit")
+	mflag.BoolVar(&createDatapath, []string{"-create-datapath"}, false, "create ODP datapath and exit")
+	mflag.BoolVar(&deleteDatapath, []string{"-delete-datapath"}, false, "delete ODP datapath and exit")
+	mflag.StringVar(&addDatapathInterface, []string{"-add-datapath-iface"}, "", "add a network interface to the ODP datapath and exit")
+
 	mflag.IntVar(&config.Port, []string{"#port", "-port"}, weave.Port, "router port")
 	mflag.IntVar(&protocolMinVersion, []string{"-min-protocol-version"}, weave.ProtocolMinVersion, "minimum weave protocol version")
 	mflag.StringVar(&ifaceName, []string{"#iface", "-iface"}, "", "name of interface to capture/inject from (disabled if blank)")
@@ -80,13 +89,14 @@ func main() {
 	mflag.StringVar(&iprangeCIDR, []string{"#iprange", "#-iprange", "-ipalloc-range"}, "", "IP address range reserved for automatic allocation, in CIDR notation")
 	mflag.StringVar(&ipsubnetCIDR, []string{"#ipsubnet", "#-ipsubnet", "-ipalloc-default-subnet"}, "", "subnet to allocate within by default, in CIDR notation")
 	mflag.IntVar(&peerCount, []string{"#initpeercount", "#-initpeercount", "-init-peer-count"}, 0, "number of peers in network (for IP address allocation)")
-	mflag.StringVar(&apiPath, []string{"#api", "-api"}, "unix:///var/run/docker.sock", "Path to Docker API socket")
+	mflag.StringVar(&dockerAPI, []string{"#api", "#-api", "-docker-api"}, "", "Docker API endpoint, e.g. unix:///var/run/docker.sock")
 	mflag.BoolVar(&noDNS, []string{"-no-dns"}, false, "disable DNS server")
 	mflag.StringVar(&dnsDomain, []string{"-dns-domain"}, nameserver.DefaultDomain, "local domain to server requests for")
 	mflag.StringVar(&dnsListenAddress, []string{"-dns-listen-address"}, nameserver.DefaultListenAddress, "address to listen on for DNS requests")
 	mflag.IntVar(&dnsTTL, []string{"-dns-ttl"}, nameserver.DefaultTTL, "TTL for DNS request from our domain")
 	mflag.DurationVar(&dnsClientTimeout, []string{"-dns-fallback-timeout"}, nameserver.DefaultClientTimeout, "timeout for fallback DNS requests")
 	mflag.StringVar(&dnsEffectiveListenAddress, []string{"-dns-effective-listen-address"}, "", "address DNS will actually be listening, after Docker port mapping")
+	mflag.StringVar(&datapathName, []string{"-datapath"}, "", "ODP datapath name")
 
 	// crude way of detecting that we probably have been started in a
 	// container, with `weave launch` --> suppress misleading paths in
@@ -95,14 +105,41 @@ func main() {
 		os.Args[0] = "weave"
 		mflag.CommandLine.Init("weave", mflag.ExitOnError)
 	}
+
 	mflag.Parse()
 
 	peers = mflag.Args()
 
 	SetLogLevel(logLevel)
-	if justVersion {
+
+	switch {
+	case justVersion:
 		fmt.Printf("weave router %s\n", version)
 		os.Exit(0)
+
+	case createDatapath:
+		err, odp_supported := weave.CreateDatapath(datapathName)
+		if !odp_supported {
+			if err != nil {
+				Log.Error(err)
+			}
+
+			// When the kernel lacks ODP support, exit
+			// with a special status to distinguish it for
+			// the weave script.
+			os.Exit(17)
+		}
+
+		checkFatal(err)
+		os.Exit(0)
+
+	case deleteDatapath:
+		checkFatal(weave.DeleteDatapath(datapathName))
+		os.Exit(0)
+
+	case addDatapathInterface != "":
+		checkFatal(weave.AddDatapathInterface(datapathName, addDatapathInterface))
+		os.Exit(0)
 	}
 
 	Log.Println("Command line options:", options())
@@ -113,41 +150,74 @@ func main() {
 	}
 	config.ProtocolMinVersion = byte(protocolMinVersion)
 
-	var err error
+	var fastDPOverlay weave.Overlay
+	if datapathName != "" {
+		// A datapath name implies that "Bridge" and "Overlay"
+		// packet handling use fast datapath, although other
+		// options can override that below.  Even if both
+		// things are overridden, we might need bridging on
+		// the datapath.
+		fastdp, err := weave.NewFastDatapath(weave.FastDatapathConfig{
+			DatapathName: datapathName,
+			Port:         config.Port,
+		})
+
+		checkFatal(err)
+		config.Bridge = fastdp.Bridge()
+		fastDPOverlay = fastdp.Overlay()
+	}
 
 	if ifaceName != "" {
-		config.Iface, err = weavenet.EnsureInterface(ifaceName)
-		if err != nil {
-			Log.Fatal(err)
-		}
-	}
+		// -iface can coexist with -datapath, because
+		// pcap-based packet capture is a bit more efficient
+		// than capture via ODP misses, even when using an
+		// ODP-based bridge.  So when using weave encyption,
+		// it's preferable to use -iface.
+		var err error
+		iface, err = weavenet.EnsureInterface(ifaceName)
+		checkFatal(err)
 
-	if routerName == "" {
-		if config.Iface == nil {
-			Log.Fatal("Either an interface must be specified with --iface or a name with -name")
-		}
-		routerName = config.Iface.HardwareAddr.String()
-	}
-	name, err := weave.PeerNameFromUserInput(routerName)
-	if err != nil {
-		Log.Fatal(err)
-	}
-
-	if nickName == "" {
-		nickName, err = os.Hostname()
-		if err != nil {
-			Log.Fatal(err)
-		}
+		// bufsz flag is in MB
+		config.Bridge, err = weave.NewPcap(iface, bufSzMB*1024*1024)
+		checkFatal(err)
 	}
 
 	if password == "" {
 		password = os.Getenv("WEAVE_PASSWORD")
 	}
+
 	if password == "" {
 		Log.Println("Communication between peers is unencrypted.")
 	} else {
 		config.Password = []byte(password)
 		Log.Println("Communication between peers is encrypted.")
+
+		// fastdp doesn't support encryption
+		fastDPOverlay = nil
+	}
+
+	overlays := weave.NewOverlaySwitch()
+	if fastDPOverlay != nil {
+		overlays.Add("fastdp", fastDPOverlay)
+	}
+	sleeve := weave.NewSleeveOverlay(config.Port)
+	overlays.Add("sleeve", sleeve)
+	overlays.SetCompatOverlay(sleeve)
+	config.Overlay = overlays
+
+	if routerName == "" {
+		if iface == nil {
+			Log.Fatal("Either an interface must be specified with --iface or a name with -name")
+		}
+		routerName = iface.HardwareAddr.String()
+	}
+
+	name, err := weave.PeerNameFromUserInput(routerName)
+	checkFatal(err)
+
+	if nickName == "" {
+		nickName, err = os.Hostname()
+		checkFatal(err)
 	}
 
 	if prof != "" {
@@ -157,25 +227,37 @@ func main() {
 		defer profile.Start(&p).Stop()
 	}
 
-	config.BufSz = bufSzMB * 1024 * 1024
-	config.LogFrame = logFrameFunc(pktdebug)
 	config.PeerDiscovery = !noDiscovery
 
+	if pktdebug {
+		config.PacketLogging = packetLogging{}
+	} else {
+		config.PacketLogging = nopPacketLogging{}
+	}
+
 	router := weave.NewRouter(config, name, nickName)
 	Log.Println("Our name is", router.Ourself)
 
-	dockerCli, err := docker.NewClient(apiPath)
-	if err != nil {
-		Log.Fatal("Unable to start docker client: ", err)
+	var dockerCli *docker.Client
+	if dockerAPI != "" {
+		dc, err := docker.NewClient(dockerAPI)
+		if err != nil {
+			Log.Fatal("Unable to start docker client: ", err)
+		}
+		dockerCli = dc
+	}
+	observeContainers := func(o docker.ContainerObserver) {
+		if dockerCli != nil {
+			if err = dockerCli.AddObserver(o); err != nil {
+				Log.Fatal("Unable to start watcher", err)
+			}
+		}
 	}
-
 	var allocator *ipam.Allocator
 	var defaultSubnet address.CIDR
 	if iprangeCIDR != "" {
 		allocator, defaultSubnet = createAllocator(router, iprangeCIDR, ipsubnetCIDR, determineQuorum(peerCount, peers))
-		if err = dockerCli.AddObserver(allocator); err != nil {
-			Log.Fatal("Unable to start watcher", err)
-		}
+		observeContainers(allocator)
 	} else if peerCount > 0 {
 		Log.Fatal("--init-peer-count flag specified without --ipalloc-range")
 	}
@@ -185,11 +267,9 @@ func main() {
 		dnsserver *nameserver.DNSServer
 	)
 	if !noDNS {
-		ns = nameserver.New(router.Ourself.Peer.Name, router.Peers, dockerCli, dnsDomain)
+		ns = nameserver.New(router.Ourself.Peer.Name, router.Peers, dnsDomain)
 		ns.SetGossip(router.NewGossip("nameserver", ns))
-		if err = dockerCli.AddObserver(ns); err != nil {
-			Log.Fatal("Unable to start watcher", err)
-		}
+		observeContainers(ns)
 		ns.Start()
 		defer ns.Stop()
 		dnsserver, err = nameserver.NewDNSServer(ns, dnsDomain, dnsListenAddress,
@@ -197,6 +277,11 @@ func main() {
 		if err != nil {
 			Log.Fatal("Unable to start dns server: ", err)
 		}
+		listenAddr := dnsListenAddress
+		if dnsEffectiveListenAddress != "" {
+			listenAddr = dnsEffectiveListenAddress
+		}
+		Log.Println("Listening for DNS queries on", listenAddr)
 		dnsserver.ActivateAndServe()
 		defer dnsserver.Stop()
 	}
@@ -215,11 +300,12 @@ func main() {
 			allocator.HandleHTTP(muxRouter, defaultSubnet, dockerCli)
 		}
 		if ns != nil {
-			ns.HandleHTTP(muxRouter)
+			ns.HandleHTTP(muxRouter, dockerCli)
 		}
 		router.HandleHTTP(muxRouter)
 		HandleHTTP(muxRouter, version, router, allocator, defaultSubnet, ns, dnsserver)
 		http.Handle("/", muxRouter)
+		Log.Println("Listening for HTTP control messages on", httpAddr)
 		go listenAndServeHTTP(httpAddr, muxRouter)
 	}
 
@@ -248,31 +334,28 @@ func canonicalName(f *mflag.Flag) string {
 	return ""
 }
 
-func logFrameFunc(debug bool) weave.LogFrameFunc {
-	if !debug {
-		return func(prefix string, frame []byte, dec *weave.EthernetDecoder) {}
-	}
-	return func(prefix string, frame []byte, dec *weave.EthernetDecoder) {
-		h := fmt.Sprintf("%x", sha256.Sum256(frame))
-		parts := []interface{}{prefix, len(frame), "bytes (", h, ")"}
+type packetLogging struct{}
 
-		if dec != nil {
-			parts = append(parts, dec.Eth.SrcMAC, "->", dec.Eth.DstMAC)
+func (packetLogging) LogPacket(msg string, key weave.PacketKey) {
+	Log.Println(msg, key.SrcMAC, "->", key.DstMAC)
+}
 
-			if dec.DF() {
-				parts = append(parts, "(DF)")
-			}
-		}
+func (packetLogging) LogForwardPacket(msg string, key weave.ForwardPacketKey) {
+	Log.Println(msg, key.SrcPeer, key.SrcMAC, "->", key.DstPeer, key.DstMAC)
+}
 
-		Log.Println(parts...)
-	}
+type nopPacketLogging struct{}
+
+func (nopPacketLogging) LogPacket(string, weave.PacketKey) {
+}
+
+func (nopPacketLogging) LogForwardPacket(string, weave.ForwardPacketKey) {
 }
 
 func parseAndCheckCIDR(cidrStr string) address.CIDR {
 	_, cidr, err := address.ParseCIDR(cidrStr)
-	if err != nil {
-		Log.Fatal(err)
-	}
+	checkFatal(err)
+
 	if cidr.Size() < ipam.MinSubnetSize {
 		Log.Fatalf("Allocation range smaller than minimum size %d: %s", ipam.MinSubnetSize, cidrStr)
 	}
@@ -288,7 +371,8 @@ func createAllocator(router *weave.Router, ipRangeStr string, defaultSubnetStr s
 			Log.Fatalf("IP address allocation default subnet %s does not overlap with allocation range %s", defaultSubnet, ipRange)
 		}
 	}
-	allocator := ipam.NewAllocator(router.Ourself.Peer.Name, router.Ourself.Peer.UID, router.Ourself.Peer.NickName, ipRange.Range(), quorum)
+	isKnownPeer := func(name weave.PeerName) bool { return router.Peers.Fetch(name) != nil }
+	allocator := ipam.NewAllocator(router.Ourself.Peer.Name, router.Ourself.Peer.UID, router.Ourself.Peer.NickName, ipRange.Range(), quorum, isKnownPeer)
 
 	allocator.SetInterfaces(router.NewGossip("IPallocation", allocator))
 	allocator.Start()
@@ -331,3 +415,9 @@ func listenAndServeHTTP(httpAddr string, muxRouter *mux.Router) {
 		Log.Fatal("Unable to create http server", err)
 	}
 }
+
+func checkFatal(e error) {
+	if e != nil {
+		Log.Fatal(e)
+	}
+}

prog/weavewait/check_network.go

@@ -0,0 +1,12 @@
+// +build !noop
+
+package main
+
+import (
+	weavenet "github.com/weaveworks/weave/net"
+)
+
+func checkNetwork() error {
+	_, err := weavenet.EnsureInterfaceAndMcastRoute("ethwe")
+	return err
+}

prog/weavewait/main.go

@@ -6,8 +6,6 @@ import (
 	"os"
 	"os/exec"
 	"syscall"
-
-	weavenet "github.com/weaveworks/weave/net"
 )
 
 var (
@@ -19,8 +17,7 @@ func main() {
 		args = os.Args[1:]
 	)
 
-	_, err := weavenet.EnsureInterfaceAndMcastRoute("ethwe")
-	checkErr(err)
+	checkErr(checkNetwork())
 
 	if len(args) == 0 {
 		checkErr(ErrNoCommandSpecified)

prog/weavewait/noop.go

@@ -0,0 +1,7 @@
+// +build noop
+
+package main
+
+func checkNetwork() error {
+	return nil
+}

proxy/common.go

@@ -20,19 +20,34 @@ import (
 var (
 	containerIDRegexp   = regexp.MustCompile("^(/v[0-9\\.]*)?/containers/([^/]*)/.*")
 	weaveWaitEntrypoint = []string{"/w/w"}
+	weaveEntrypoint     = "/home/weave/weaver"
+	weaveContainerName  = "/weave"
 )
 
 func callWeave(args ...string) ([]byte, []byte, error) {
 	args = append([]string{"--local"}, args...)
-	Log.Debug("Calling weave", args)
 	cmd := exec.Command("./weave", args...)
 	cmd.Env = []string{
 		"PATH=/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
 		"PROCFS=/hostproc",
 	}
-	if bridge := os.Getenv("DOCKER_BRIDGE"); bridge != "" {
-		cmd.Env = append(cmd.Env, fmt.Sprintf("DOCKER_BRIDGE=%s", bridge))
+
+	propagateEnv := func(key string) {
+		if val := os.Getenv(key); val != "" {
+			cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", key, val))
+		}
 	}
+
+	propagateEnv("DOCKER_BRIDGE")
+
+	// Propogage WEAVE_DEBUG, to make debugging easier.
+	propagateEnv("WEAVE_DEBUG")
+
+	// This prevents the code coverage contortions in our
+	// integration test suite breaking things.
+	propagateEnv("COVERAGE")
+
+	Log.Debug("Calling weave args: ", args, "env: ", cmd.Env)
 	var stdout, stderr bytes.Buffer
 	cmd.Stdout = &stdout
 	cmd.Stderr = &stderr
@@ -45,6 +60,7 @@ func unmarshalRequestBody(r *http.Request, target interface{}) error {
 	if err != nil {
 		return err
 	}
+	Log.Debugf("->requestBody: %s", body)
 	if err := r.Body.Close(); err != nil {
 		return err
 	}
@@ -60,6 +76,7 @@ func marshalRequestBody(r *http.Request, body interface{}) error {
 	if err != nil {
 		return err
 	}
+	Log.Debugf("<-requestBody: %s", newBody)
 	r.Body = ioutil.NopCloser(bytes.NewReader(newBody))
 	r.ContentLength = int64(len(newBody))
 	return nil
@@ -70,6 +87,7 @@ func unmarshalResponseBody(r *http.Response, target interface{}) error {
 	if err != nil {
 		return err
 	}
+	Log.Debugf("->responseBody: %s", body)
 	if err := r.Body.Close(); err != nil {
 		return err
 	}
@@ -85,12 +103,12 @@ func marshalResponseBody(r *http.Response, body interface{}) error {
 	if err != nil {
 		return err
 	}
+	Log.Debugf("<-responseBody: %s", newBody)
 	r.Body = ioutil.NopCloser(bytes.NewReader(newBody))
 	r.ContentLength = int64(len(newBody))
 	// Stop it being chunked, because that hangs
 	r.TransferEncoding = nil
 	return nil
-
 }
 
 func inspectContainerInPath(client *docker.Client, path string) (*docker.Container, error) {
@@ -137,3 +155,25 @@ func weaveContainerIPs(containerID string) (mac string, ips []net.IP, nets []*ne
 	}
 	return
 }
+
+func addVolume(hostConfig jsonObject, source, target, mode string) error {
+	configBinds, err := hostConfig.StringArray("Binds")
+	if err != nil {
+		return err
+	}
+
+	var binds []string
+	for _, bind := range configBinds {
+		s := strings.Split(bind, ":")
+		if len(s) >= 2 && s[1] == target {
+			continue
+		}
+		binds = append(binds, bind)
+	}
+	bind := source + ":" + target
+	if mode != "" {
+		bind += ":" + mode
+	}
+	hostConfig["Binds"] = append(binds, bind)
+	return nil
+}

proxy/create_container_interceptor.go

@@ -2,16 +2,11 @@ package proxy
 
 import (
 	"errors"
-	"fmt"
-	"io/ioutil"
 	"net/http"
 	"strings"
 
 	"github.com/fsouza/go-dockerclient"
-
 	. "github.com/weaveworks/weave/common"
-	"github.com/weaveworks/weave/nameserver"
-	"github.com/weaveworks/weave/router"
 )
 
 const MaxDockerHostname = 64
@@ -22,12 +17,6 @@ var (
 
 type createContainerInterceptor struct{ proxy *Proxy }
 
-type createContainerRequestBody struct {
-	*docker.Config
-	HostConfig *docker.HostConfig `json:"HostConfig,omitempty" yaml:"HostConfig,omitempty"`
-	MacAddress string             `json:"MacAddress,omitempty" yaml:"MacAddress,omitempty"`
-}
-
 // ErrNoSuchImage replaces docker.NoSuchImage, which does not contain the image
 // name, which in turn breaks docker clients post 1.7.0 since they expect the
 // image name to be present in errors.
@@ -40,34 +29,47 @@ func (err *ErrNoSuchImage) Error() string {
 }
 
 func (i *createContainerInterceptor) InterceptRequest(r *http.Request) error {
-	container := createContainerRequestBody{}
+	container := jsonObject{}
 	if err := unmarshalRequestBody(r, &container); err != nil {
 		return err
 	}
 
-	if cidrs, err := i.proxy.weaveCIDRsFromConfig(container.Config, container.HostConfig); err != nil {
+	hostConfig, err := container.Object("HostConfig")
+	if err != nil {
+		return err
+	}
+
+	networkMode, err := hostConfig.String("NetworkMode")
+	if err != nil {
+		return err
+	}
+
+	env, err := container.StringArray("Env")
+	if err != nil {
+		return err
+	}
+
+	if cidrs, err := i.proxy.weaveCIDRs(networkMode, env); err != nil {
 		Log.Infof("Leaving container alone because %s", err)
 	} else {
 		Log.Infof("Creating container with WEAVE_CIDR \"%s\"", strings.Join(cidrs, " "))
-		if container.HostConfig == nil {
-			container.HostConfig = &docker.HostConfig{}
-		}
-		if container.Config == nil {
-			container.Config = &docker.Config{}
-		}
-		i.addWeaveWaitVolume(container.HostConfig)
-		if err := i.setWeaveWaitEntrypoint(container.Config); err != nil {
+		if err := addVolume(hostConfig, i.proxy.weaveWaitVolume, "/w", "ro"); err != nil {
 			return err
 		}
-		hostname := r.URL.Query().Get("name")
-		if i.proxy.Config.HostnameFromLabel != "" {
-			if labelValue, ok := container.Labels[i.proxy.Config.HostnameFromLabel]; ok {
-				hostname = labelValue
+		if err := i.setWeaveWaitEntrypoint(container); err != nil {
+			return err
+		}
+		hostname, err := i.containerHostname(r, container)
+		if err != nil {
+			return err
+		}
+		if dnsDomain := i.proxy.getDNSDomain(); dnsDomain != "" {
+			if err := i.setHostname(container, hostname, dnsDomain); err != nil {
+				return err
+			}
+			if err := i.proxy.setWeaveDNS(hostConfig, hostname, dnsDomain); err != nil {
+				return err
 			}
-		}
-		hostname = i.proxy.hostnameMatchRegexp.ReplaceAllString(hostname, i.proxy.HostnameReplacement)
-		if err := i.setWeaveDNS(&container, hostname); err != nil {
-			return err
 		}
 
 		return marshalRequestBody(r, container)
@@ -76,105 +78,97 @@ func (i *createContainerInterceptor) InterceptRequest(r *http.Request) error {
 	return nil
 }
 
-func (i *createContainerInterceptor) addWeaveWaitVolume(hostConfig *docker.HostConfig) {
-	var binds []string
-	for _, bind := range hostConfig.Binds {
-		s := strings.Split(bind, ":")
-		if len(s) >= 2 && s[1] == "/w" {
-			continue
-		}
-		binds = append(binds, bind)
+func (i *createContainerInterceptor) setWeaveWaitEntrypoint(container jsonObject) error {
+	var entrypoint []string
+	entrypoint, err := container.StringArray("Entrypoint")
+	if err != nil {
+		return err
 	}
-	hostConfig.Binds = append(binds, fmt.Sprintf("%s:/w:ro", i.proxy.weaveWaitVolume))
-}
 
-func (i *createContainerInterceptor) setWeaveWaitEntrypoint(container *docker.Config) error {
-	if len(container.Entrypoint) == 0 {
-		image, err := i.proxy.client.InspectImage(container.Image)
+	cmd, err := container.StringArray("Cmd")
+	if err != nil {
+		return err
+	}
+
+	if len(entrypoint) == 0 {
+		containerImage, err := container.String("Image")
+		if err != nil {
+			return err
+		}
+
+		image, err := i.proxy.client.InspectImage(containerImage)
 		if err == docker.ErrNoSuchImage {
-			return &ErrNoSuchImage{container.Image}
+			return &ErrNoSuchImage{containerImage}
 		} else if err != nil {
 			return err
 		}
 
-		if len(container.Cmd) == 0 {
-			container.Cmd = image.Config.Cmd
+		if len(cmd) == 0 {
+			cmd = image.Config.Cmd
+			container["Cmd"] = cmd
 		}
 
-		if container.Entrypoint == nil {
-			container.Entrypoint = image.Config.Entrypoint
+		if entrypoint == nil {
+			entrypoint = image.Config.Entrypoint
+			container["Entrypoint"] = entrypoint
 		}
 	}
 
-	if len(container.Entrypoint) == 0 && len(container.Cmd) == 0 {
+	if len(entrypoint) == 0 && len(cmd) == 0 {
 		return ErrNoCommandSpecified
 	}
 
-	if len(container.Entrypoint) == 0 || container.Entrypoint[0] != weaveWaitEntrypoint[0] {
-		container.Entrypoint = append(weaveWaitEntrypoint, container.Entrypoint...)
+	if len(entrypoint) == 0 || entrypoint[0] != weaveWaitEntrypoint[0] {
+		container["Entrypoint"] = append(weaveWaitEntrypoint, entrypoint...)
 	}
 
 	return nil
 }
 
-func (i *createContainerInterceptor) setWeaveDNS(container *createContainerRequestBody, name string) error {
-	if i.proxy.WithoutDNS {
-		return nil
+func (i *createContainerInterceptor) setHostname(container jsonObject, name, dnsDomain string) error {
+	hostname, err := container.String("Hostname")
+	if err != nil {
+		return err
 	}
-
-	dnsDomain, dnsRunning := i.getDNSDomain()
-	if !(dnsRunning || i.proxy.WithDNS) {
-		return nil
-	}
-
-	container.HostConfig.DNS = append(container.HostConfig.DNS, i.proxy.dockerBridgeIP)
-
-	if container.Hostname == "" && name != "" {
+	if hostname == "" && name != "" {
 		// Strip trailing period because it's unusual to see it used on the end of a host name
 		trimmedDNSDomain := strings.TrimSuffix(dnsDomain, ".")
 		if len(name)+1+len(trimmedDNSDomain) > MaxDockerHostname {
 			Log.Warningf("Container name [%s] too long to be used as hostname", name)
 		} else {
-			container.Hostname = name
-			container.Domainname = trimmedDNSDomain
-		}
-	}
-
-	if len(container.HostConfig.DNSSearch) == 0 {
-		if container.Hostname == "" {
-			container.HostConfig.DNSSearch = []string{dnsDomain}
-		} else {
-			container.HostConfig.DNSSearch = []string{"."}
+			container["Hostname"] = name
+			container["Domainname"] = trimmedDNSDomain
 		}
 	}
 
 	return nil
 }
 
-func (i *createContainerInterceptor) getDNSDomain() (domain string, running bool) {
-	domain = nameserver.DefaultDomain
-	weaveContainer, err := i.proxy.client.InspectContainer("weave")
-	if err != nil ||
-		weaveContainer.NetworkSettings == nil ||
-		weaveContainer.NetworkSettings.IPAddress == "" {
-		return
-	}
-
-	url := fmt.Sprintf("http://%s:%d/domain", weaveContainer.NetworkSettings.IPAddress, router.HTTPPort)
-	resp, err := http.Get(url)
-	if err != nil || resp.StatusCode != http.StatusOK {
-		return
-	}
-	defer resp.Body.Close()
-
-	b, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return
-	}
-
-	return string(b), true
-}
-
 func (i *createContainerInterceptor) InterceptResponse(r *http.Response) error {
 	return nil
 }
+
+func (i *createContainerInterceptor) containerHostname(r *http.Request, container jsonObject) (hostname string, err error) {
+	hostname = r.URL.Query().Get("name")
+	if i.proxy.Config.HostnameFromLabel != "" {
+		hostname, err = i.hostnameFromLabel(hostname, container)
+	}
+	hostname = i.proxy.hostnameMatchRegexp.ReplaceAllString(hostname, i.proxy.HostnameReplacement)
+	return
+}
+
+func (i *createContainerInterceptor) hostnameFromLabel(hostname string, container jsonObject) (string, error) {
+	labels, err := container.Object("Labels")
+	if err != nil {
+		return "", err
+	}
+	label, err := labels.String(i.proxy.Config.HostnameFromLabel)
+	if err != nil {
+		return "", err
+	}
+	if label == "" {
+		return hostname, nil
+	}
+
+	return label, nil
+}

proxy/create_exec_interceptor.go

@@ -4,14 +4,13 @@ import (
 	"net/http"
 	"strings"
 
-	"github.com/fsouza/go-dockerclient"
 	. "github.com/weaveworks/weave/common"
 )
 
 type createExecInterceptor struct{ proxy *Proxy }
 
 func (i *createExecInterceptor) InterceptRequest(r *http.Request) error {
-	options := docker.CreateExecOptions{}
+	options := jsonObject{}
 	if err := unmarshalRequestBody(r, &options); err != nil {
 		return err
 	}
@@ -25,14 +24,19 @@ func (i *createExecInterceptor) InterceptRequest(r *http.Request) error {
 		return nil
 	}
 
-	cidrs, err := i.proxy.weaveCIDRsFromConfig(container.Config, container.HostConfig)
+	cidrs, err := i.proxy.weaveCIDRs(container.HostConfig.NetworkMode, container.Config.Env)
 	if err != nil {
 		Log.Infof("Leaving container %s alone because %s", container.ID, err)
 		return nil
 	}
 
+	cmd, err := options.StringArray("Cmd")
+	if err != nil {
+		return err
+	}
+
 	Log.Infof("Exec in container %s with WEAVE_CIDR \"%s\"", container.ID, strings.Join(cidrs, " "))
-	options.Cmd = append(weaveWaitEntrypoint, options.Cmd...)
+	options["Cmd"] = append(weaveWaitEntrypoint, cmd...)
 
 	return marshalRequestBody(r, options)
 }

proxy/inspect_container_interceptor.go

@@ -13,39 +13,18 @@ func (i *inspectContainerInterceptor) InterceptRequest(r *http.Request) error {
 }
 
 func (i *inspectContainerInterceptor) InterceptResponse(r *http.Response) error {
-	if !i.proxy.RewriteInspect {
+	if !i.proxy.RewriteInspect || r.StatusCode != 200 {
 		return nil
 	}
 
-	container := map[string]interface{}{}
+	container := jsonObject{}
 	if err := unmarshalResponseBody(r, &container); err != nil {
 		return err
 	}
 
-	if err := updateContainerNetworkSettings(container); err != nil {
+	if err := i.proxy.updateContainerNetworkSettings(container); err != nil {
 		Log.Warningf("Inspecting container %s failed: %s", container["Id"], err)
 	}
 
 	return marshalResponseBody(r, container)
 }
-
-func updateContainerNetworkSettings(container map[string]interface{}) error {
-	containerID, err := lookupString(container, "Id")
-	if err != nil {
-		return err
-	}
-
-	mac, ips, nets, err := weaveContainerIPs(containerID)
-	if err != nil || len(ips) == 0 {
-		return err
-	}
-
-	networkSettings, err := lookupObject(container, "NetworkSettings")
-	if err != nil {
-		return err
-	}
-	networkSettings["MacAddress"] = mac
-	networkSettings["IPAddress"] = ips[0].String()
-	networkSettings["IPPrefixLen"], _ = nets[0].Mask.Size()
-	return nil
-}

proxy/inspect_exec_interceptor.go

@@ -17,17 +17,17 @@ func (i *inspectExecInterceptor) InterceptResponse(r *http.Response) error {
 		return nil
 	}
 
-	exec := map[string]interface{}{}
+	exec := jsonObject{}
 	if err := unmarshalResponseBody(r, &exec); err != nil {
 		return err
 	}
 
-	container, err := lookupObject(exec, "Container")
+	container, err := exec.Object("Container")
 	if err != nil {
 		return err
 	}
 
-	if err := updateContainerNetworkSettings(container); err != nil {
+	if err := i.proxy.updateContainerNetworkSettings(container); err != nil {
 		Log.Warningf("Inspecting exec %s failed: %s", exec["Id"], err)
 	}
 

proxy/json.go

@@ -13,12 +13,13 @@ func (e *UnmarshalWrongTypeError) Error() string {
 	return fmt.Sprintf("Wrong type for %s field, expected %s, but got %T", e.Field, e.Expected, e.Got)
 }
 
-// For parsing json objects. Look up (or create) a child object in the given map.
-func lookupObject(m map[string]interface{}, key string) (map[string]interface{}, error) {
-	iface, ok := m[key]
+type jsonObject map[string]interface{}
+
+func (j jsonObject) Object(key string) (jsonObject, error) {
+	iface, ok := j[key]
 	if !ok || iface == nil {
-		result := map[string]interface{}{}
-		m[key] = result
+		result := jsonObject{}
+		j[key] = result
 		return result, nil
 	}
 
@@ -27,12 +28,11 @@ func lookupObject(m map[string]interface{}, key string) (map[string]interface{},
 		return nil, &UnmarshalWrongTypeError{key, "object", iface}
 	}
 
-	return result, nil
+	return jsonObject(result), nil
 }
 
-// For parsing json objects. Look up (or create) a string in the given map.
-func lookupString(m map[string]interface{}, key string) (string, error) {
-	iface, ok := m[key]
+func (j jsonObject) String(key string) (string, error) {
+	iface, ok := j[key]
 	if !ok || iface == nil {
 		return "", nil
 	}
@@ -44,3 +44,29 @@ func lookupString(m map[string]interface{}, key string) (string, error) {
 
 	return result, nil
 }
+
+func (j jsonObject) StringArray(key string) ([]string, error) {
+	iface, ok := j[key]
+	if !ok || iface == nil {
+		return nil, nil
+	}
+
+	switch o := iface.(type) {
+	case string:
+		return []string{o}, nil
+	case []string:
+		return o, nil
+	case []interface{}:
+		result := []string{}
+		for _, s := range o {
+			if s, ok := s.(string); ok {
+				result = append(result, s)
+			} else {
+				return nil, &UnmarshalWrongTypeError{key, "string or array of strings", iface}
+			}
+		}
+		return result, nil
+	}
+
+	return nil, &UnmarshalWrongTypeError{key, "string or array of strings", iface}
+}

proxy/json_test.go

@@ -0,0 +1,116 @@
+package proxy
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestLookupObject(t *testing.T) {
+	tests := []struct {
+		root   jsonObject
+		key    string
+		result jsonObject
+		err    error
+	}{
+		{
+			jsonObject{},
+			"a",
+			jsonObject{},
+			nil,
+		},
+		{
+			jsonObject{"a": map[string]interface{}{"b": int(1)}},
+			"a",
+			jsonObject{"b": int(1)},
+			nil,
+		},
+		{
+			jsonObject{"nonObject": int(1)},
+			"nonObject",
+			nil,
+			&UnmarshalWrongTypeError{Field: "nonObject", Expected: "object", Got: 1},
+		},
+	}
+	for _, test := range tests {
+		gotResult, gotErr := test.root.Object(test.key)
+		msg := fmt.Sprintf("%q.Object(%q) => %q, %q", test.root, test.key, gotResult, gotErr)
+		assert.Equal(t, test.result, gotResult, msg)
+		assert.Equal(t, test.err, gotErr, msg)
+	}
+}
+
+func TestLookupString(t *testing.T) {
+	tests := []struct {
+		root   jsonObject
+		key    string
+		result string
+		err    error
+	}{
+		{
+			jsonObject{},
+			"a",
+			"",
+			nil,
+		},
+		{
+			jsonObject{"nonString": int(1)},
+			"nonString",
+			"",
+			&UnmarshalWrongTypeError{Field: "nonString", Expected: "string", Got: 1},
+		},
+	}
+	for _, test := range tests {
+		gotResult, gotErr := test.root.String(test.key)
+		msg := fmt.Sprintf("%q.String(%q) => %q, %q", test.root, test.key, gotResult, gotErr)
+		assert.Equal(t, test.result, gotResult, msg)
+		assert.Equal(t, test.err, gotErr, msg)
+	}
+}
+
+func TestLookupStringArray(t *testing.T) {
+	tests := []struct {
+		root   jsonObject
+		key    string
+		result []string
+		err    error
+	}{
+		{
+			jsonObject{},
+			"a",
+			nil,
+			nil,
+		},
+		{
+			jsonObject{"a": []string{"foo"}},
+			"a",
+			[]string{"foo"},
+			nil,
+		},
+		{
+			jsonObject{"a": []string{}},
+			"a",
+			[]string{},
+			nil,
+		},
+		{
+			jsonObject{"a": "foo"},
+			"a",
+			[]string{"foo"},
+			nil,
+		},
+		{
+			jsonObject{"int": 5},
+			"int",
+			nil,
+			&UnmarshalWrongTypeError{Field: "int", Expected: "string or array of strings", Got: 5},
+		},
+	}
+	for _, test := range tests {
+		gotResult, gotErr := test.root.StringArray(test.key)
+		msg := fmt.Sprintf("%q.String(%q) => %q, %q", test.root, test.key, gotResult, gotErr)
+		assert.Equal(t, test.result, gotResult, msg)
+		assert.Equal(t, test.err, gotErr, msg)
+	}
+}

proxy/proxy.go

@@ -4,15 +4,20 @@ import (
 	"crypto/tls"
 	"errors"
 	"fmt"
+	"io/ioutil"
 	"net"
 	"net/http"
 	"os"
 	"regexp"
 	"strings"
+	"sync"
 	"syscall"
 
-	"github.com/fsouza/go-dockerclient"
+	docker "github.com/fsouza/go-dockerclient"
 	. "github.com/weaveworks/weave/common"
+	weavedocker "github.com/weaveworks/weave/common/docker"
+	"github.com/weaveworks/weave/nameserver"
+	"github.com/weaveworks/weave/router"
 )
 
 const (
@@ -52,18 +57,28 @@ type Config struct {
 	WithoutDNS          bool
 }
 
+type wait struct {
+	ident string
+	ch    chan error
+	done  bool
+}
+
 type Proxy struct {
+	sync.Mutex
 	Config
 	client              *docker.Client
 	dockerBridgeIP      string
 	hostnameMatchRegexp *regexp.Regexp
 	weaveWaitVolume     string
+	weaveWaitNoopVolume string
+	normalisedAddrs     []string
+	waiters             map[*http.Request]*wait
 }
 
 func NewProxy(c Config) (*Proxy, error) {
-	p := &Proxy{Config: c}
+	p := &Proxy{Config: c, waiters: make(map[*http.Request]*wait)}
 
-	if err := p.TLSConfig.loadCerts(); err != nil {
+	if err := p.TLSConfig.LoadCerts(); err != nil {
 		Log.Fatalf("Could not configure tls for proxy: %s", err)
 	}
 
@@ -72,11 +87,11 @@ func NewProxy(c Config) (*Proxy, error) {
 	// to insulate ourselves from breaking changes to the API, as
 	// happened in 1.20 (Docker 1.8.0) when the presentation of
 	// volumes changed in `inspect`.
-	client, err := docker.NewVersionedClient(dockerSockUnix, "1.15")
+	client, err := weavedocker.NewVersionedClient(dockerSockUnix, "1.15")
 	if err != nil {
 		return nil, err
 	}
-	p.client = client
+	p.client = client.Client
 
 	if !p.WithoutDNS {
 		dockerBridgeIP, stderr, err := callWeave("docker-bridge-ip")
@@ -92,34 +107,60 @@ func NewProxy(c Config) (*Proxy, error) {
 		return nil, err
 	}
 
-	if err = p.findWeaveWaitVolume(); err != nil {
+	if err = p.findWeaveWaitVolumes(); err != nil {
 		return nil, err
 	}
 
+	client.AddObserver(p)
+
 	return p, nil
 }
 
+func (proxy *Proxy) AttachExistingContainers() {
+	containers, _ := proxy.client.ListContainers(docker.ListContainersOptions{})
+	for _, c := range containers {
+		container, err := proxy.client.InspectContainer(c.ID)
+		if err != nil {
+			if _, ok := err.(*docker.NoSuchContainer); !ok {
+				Log.Warningf("unable to attach existing container %s since inspecting it failed: %v", c.ID, err)
+			}
+			continue
+		}
+		if containerShouldAttach(container) && (container.State.Running || container.State.Paused) {
+			proxy.attach(container, false)
+		}
+	}
+}
+
 func (proxy *Proxy) Dial() (net.Conn, error) {
 	return net.Dial("unix", dockerSock)
 }
 
-func (proxy *Proxy) findWeaveWaitVolume() error {
+func (proxy *Proxy) findWeaveWaitVolumes() error {
+	var err error
+	if proxy.weaveWaitVolume, err = proxy.findVolume("/w"); err != nil {
+		return err
+	}
+	proxy.weaveWaitNoopVolume, err = proxy.findVolume("/w-noop")
+	return err
+}
+
+func (proxy *Proxy) findVolume(v string) (string, error) {
 	container, err := proxy.client.InspectContainer("weaveproxy")
 	if err != nil {
-		return fmt.Errorf("Could not find the weavewait volume: %s", err)
+		return "", fmt.Errorf("Could not find the weavewait volume: %s", err)
 	}
 
 	if container.Volumes == nil {
-		return fmt.Errorf("Could not find the weavewait volume")
+		return "", fmt.Errorf("Could not find the weavewait volume")
 	}
 
-	volume, ok := container.Volumes["/w"]
+	volume, ok := container.Volumes[v]
 	if !ok {
-		return fmt.Errorf("Could not find the weavewait volume")
+		return "", fmt.Errorf("Could not find the weavewait volume")
 	}
 
-	proxy.weaveWaitVolume = volume
-	return nil
+	return volume, nil
 }
 
 func (proxy *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
@@ -143,22 +184,25 @@ func (proxy *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	proxy.Intercept(i, w, r)
 }
 
-func (proxy *Proxy) ListenAndServe() {
+func (proxy *Proxy) Listen() []net.Listener {
 	listeners := []net.Listener{}
-	addrs := []string{}
+	proxy.normalisedAddrs = []string{}
 	for _, addr := range proxy.ListenAddrs {
 		listener, normalisedAddr, err := proxy.listen(addr)
 		if err != nil {
 			Log.Fatalf("Cannot listen on %s: %s", addr, err)
 		}
 		listeners = append(listeners, listener)
-		addrs = append(addrs, normalisedAddr)
+		proxy.normalisedAddrs = append(proxy.normalisedAddrs, normalisedAddr)
 	}
 
-	for _, addr := range addrs {
+	for _, addr := range proxy.normalisedAddrs {
 		Log.Infoln("proxy listening on", addr)
 	}
+	return listeners
+}
 
+func (proxy *Proxy) Serve(listeners []net.Listener) {
 	errs := make(chan error)
 	for _, listener := range listeners {
 		go func(listener net.Listener) {
@@ -173,6 +217,26 @@ func (proxy *Proxy) ListenAndServe() {
 	}
 }
 
+func (proxy *Proxy) ListenAndServeStatus(socket string) {
+	if err := os.Remove(socket); err != nil && !os.IsNotExist(err) {
+		Log.Fatalf("Error removing existing status socket: %s", err)
+	}
+	listener, err := net.Listen("unix", socket)
+	if err != nil {
+		Log.Fatalf("ListenAndServeStatus failed: %s", err)
+	}
+	handler := http.HandlerFunc(proxy.StatusHTTP)
+	if err := (&http.Server{Handler: handler}).Serve(listener); err != nil {
+		Log.Fatalf("ListenAndServeStatus failed: %s", err)
+	}
+}
+
+func (proxy *Proxy) StatusHTTP(w http.ResponseWriter, r *http.Request) {
+	for _, addr := range proxy.normalisedAddrs {
+		fmt.Fprintln(w, addr)
+	}
+}
+
 func copyOwnerAndPermissions(from, to string) error {
 	stat, err := os.Stat(from)
 	if err != nil {
@@ -215,7 +279,7 @@ func (proxy *Proxy) listen(protoAndAddr string) (net.Listener, string, error) {
 		if err != nil {
 			return nil, "", err
 		}
-		if proxy.TLSConfig.enabled() {
+		if proxy.TLSConfig.IsEnabled() {
 			listener = tls.NewListener(listener, proxy.TLSConfig.Config)
 		}
 
@@ -239,15 +303,140 @@ func (proxy *Proxy) listen(protoAndAddr string) (net.Listener, string, error) {
 	return listener, fmt.Sprintf("%s://%s", proto, addr), nil
 }
 
-func (proxy *Proxy) weaveCIDRsFromConfig(config *docker.Config, hostConfig *docker.HostConfig) ([]string, error) {
-	netMode := ""
-	if hostConfig != nil {
-		netMode = hostConfig.NetworkMode
+// weavedocker.ContainerObserver interface
+func (proxy *Proxy) ContainerStarted(ident string) {
+	container, err := proxy.client.InspectContainer(ident)
+	if err == nil {
+		if containerShouldAttach(container) {
+			err = proxy.attach(container, true)
+		} else if containerIsWeaveRouter(container) {
+			err = proxy.attachRouter(container)
+		}
 	}
-	if netMode == "host" || strings.HasPrefix(netMode, "container:") {
-		return nil, fmt.Errorf("the container was created with the '--net=%s'", netMode)
+	if _, ok := err.(*docker.NoSuchContainer); err != nil && !ok {
+		Log.Warningf("unable to attach new container %s since inspecting it failed: %v", ident, err)
 	}
-	for _, e := range config.Env {
+	proxy.notifyWaiters(ident, err)
+}
+
+func containerShouldAttach(container *docker.Container) bool {
+	return len(container.Config.Entrypoint) > 0 && container.Config.Entrypoint[0] == weaveWaitEntrypoint[0]
+}
+
+func containerIsWeaveRouter(container *docker.Container) bool {
+	return container.Name == weaveContainerName &&
+		len(container.Config.Entrypoint) > 0 && container.Config.Entrypoint[0] == weaveEntrypoint
+}
+
+func (proxy *Proxy) createWait(r *http.Request, ident string) {
+	proxy.Lock()
+	proxy.waiters[r] = &wait{ident: ident, ch: make(chan error, 1)}
+	proxy.Unlock()
+}
+
+func (proxy *Proxy) removeWait(r *http.Request) {
+	proxy.Lock()
+	delete(proxy.waiters, r)
+	proxy.Unlock()
+}
+
+func (proxy *Proxy) notifyWaiters(ident string, err error) {
+	proxy.Lock()
+	for _, wait := range proxy.waiters {
+		if ident == wait.ident && !wait.done {
+			wait.ch <- err
+			close(wait.ch)
+			wait.done = true
+		}
+	}
+	proxy.Unlock()
+}
+
+func (proxy *Proxy) waitForStart(r *http.Request) error {
+	var ch chan error
+	proxy.Lock()
+	wait, found := proxy.waiters[r]
+	if found {
+		ch = wait.ch
+	}
+	proxy.Unlock()
+	if ch != nil {
+		Log.Debugf("Wait for start of container %s", wait.ident)
+		return <-ch
+	}
+	return nil
+}
+
+// If some other operation is waiting for a container to start, join in the wait
+func (proxy *Proxy) waitForStartByIdent(ident string) error {
+	var ch chan error
+	proxy.Lock()
+	for _, wait := range proxy.waiters {
+		if ident == wait.ident && !wait.done {
+			ch = wait.ch
+			break
+		}
+	}
+	proxy.Unlock()
+	if ch != nil {
+		Log.Debugf("Wait for start of container %s", ident)
+		return <-ch
+	}
+	return nil
+}
+
+func (proxy *Proxy) ContainerDied(ident string) {
+}
+
+func (proxy *Proxy) attach(container *docker.Container, orDie bool) error {
+	cidrs, err := proxy.weaveCIDRs(container.HostConfig.NetworkMode, container.Config.Env)
+	if err != nil {
+		Log.Infof("Leaving container %s alone because %s", container.ID, err)
+		return nil
+	}
+	Log.Infof("Attaching container %s with WEAVE_CIDR \"%s\" to weave network", container.ID, strings.Join(cidrs, " "))
+	args := []string{"attach"}
+	args = append(args, cidrs...)
+	if !proxy.NoRewriteHosts {
+		args = append(args, "--rewrite-hosts")
+
+		if container.HostConfig != nil {
+			for _, eh := range container.HostConfig.ExtraHosts {
+				args = append(args, fmt.Sprintf("--add-host=%s", eh))
+			}
+		}
+	}
+	if orDie {
+		args = append(args, "--or-die")
+	}
+	args = append(args, container.ID)
+	if _, stderr, err := callWeave(args...); err != nil {
+		Log.Warningf("Attaching container %s to weave network failed: %s", container.ID, string(stderr))
+		return errors.New(string(stderr))
+	} else if len(stderr) > 0 {
+		Log.Warningf("Attaching container %s to weave network: %s", container.ID, string(stderr))
+	}
+
+	return nil
+}
+
+func (proxy *Proxy) attachRouter(container *docker.Container) error {
+	Log.Infof("Attaching weave router container: %s", container.ID)
+	args := []string{"attach-router"}
+	if _, stderr, err := callWeave(args...); err != nil {
+		Log.Warningf("Attaching container %s to weave network failed: %s", container.ID, string(stderr))
+		return errors.New(string(stderr))
+	} else if len(stderr) > 0 {
+		Log.Warningf("Attaching container %s to weave network: %s", container.ID, string(stderr))
+	}
+	return nil
+}
+
+func (proxy *Proxy) weaveCIDRs(networkMode string, env []string) ([]string, error) {
+	if networkMode == "host" || strings.HasPrefix(networkMode, "container:") {
+		return nil, fmt.Errorf("the container has '--net=%s'", networkMode)
+	}
+	for _, e := range env {
 		if strings.HasPrefix(e, "WEAVE_CIDR=") {
 			if e[11:] == "none" {
 				return nil, ErrWeaveCIDRNone
@@ -260,3 +449,82 @@ func (proxy *Proxy) weaveCIDRsFromConfig(config *docker.Config, hostConfig *dock
 	}
 	return nil, nil
 }
+
+func (proxy *Proxy) setWeaveDNS(hostConfig jsonObject, hostname, dnsDomain string) error {
+	dns, err := hostConfig.StringArray("Dns")
+	if err != nil {
+		return err
+	}
+	hostConfig["Dns"] = append(dns, proxy.dockerBridgeIP)
+
+	dnsSearch, err := hostConfig.StringArray("DnsSearch")
+	if err != nil {
+		return err
+	}
+	if len(dnsSearch) == 0 {
+		if hostname == "" {
+			hostConfig["DnsSearch"] = []string{dnsDomain}
+		} else {
+			hostConfig["DnsSearch"] = []string{"."}
+		}
+	}
+
+	return nil
+}
+
+func (proxy *Proxy) getDNSDomain() (domain string) {
+	if proxy.WithoutDNS {
+		return ""
+	}
+
+	if proxy.WithDNS {
+		domain = nameserver.DefaultDomain
+	}
+
+	weaveContainer, err := proxy.client.InspectContainer("weave")
+	var weaveIP string
+	if err == nil && weaveContainer.NetworkSettings != nil {
+		weaveIP = weaveContainer.NetworkSettings.IPAddress
+	}
+	if weaveIP == "" {
+		weaveIP = "127.0.0.1"
+	}
+
+	url := fmt.Sprintf("http://%s:%d/domain", weaveIP, router.HTTPPort)
+	resp, err := http.Get(url)
+	if err != nil || resp.StatusCode != http.StatusOK {
+		return
+	}
+	defer resp.Body.Close()
+
+	b, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return
+	}
+
+	return string(b)
+}
+
+func (proxy *Proxy) updateContainerNetworkSettings(container jsonObject) error {
+	containerID, err := container.String("Id")
+	if err != nil {
+		return err
+	}
+
+	if err := proxy.waitForStartByIdent(containerID); err != nil {
+		return err
+	}
+	mac, ips, nets, err := weaveContainerIPs(containerID)
+	if err != nil || len(ips) == 0 {
+		return err
+	}
+
+	networkSettings, err := container.Object("NetworkSettings")
+	if err != nil {
+		return err
+	}
+	networkSettings["MacAddress"] = mac
+	networkSettings["IPAddress"] = ips[0].String()
+	networkSettings["IPPrefixLen"], _ = nets[0].Mask.Size()
+	return nil
+}

proxy/start_container_interceptor.go

@@ -1,47 +1,70 @@
 package proxy
 
 import (
-	"errors"
 	"net/http"
 	"strings"
-
-	. "github.com/weaveworks/weave/common"
 )
 
 type startContainerInterceptor struct{ proxy *Proxy }
 
 func (i *startContainerInterceptor) InterceptRequest(r *http.Request) error {
-	return nil
-}
-
-func (i *startContainerInterceptor) InterceptResponse(r *http.Response) error {
-	if r.StatusCode < 200 || r.StatusCode >= 300 { // Docker didn't do the start
-		return nil
-	}
-
-	container, err := inspectContainerInPath(i.proxy.client, r.Request.URL.Path)
+	container, err := inspectContainerInPath(i.proxy.client, r.URL.Path)
 	if err != nil {
 		return err
 	}
 
-	cidrs, err := i.proxy.weaveCIDRsFromConfig(container.Config, container.HostConfig)
-	if err != nil {
-		Log.Infof("Leaving container %s alone because %s", container.ID, err)
-		return nil
-	}
-	Log.Infof("Attaching container %s with WEAVE_CIDR \"%s\" to weave network", container.ID, strings.Join(cidrs, " "))
-	args := []string{"attach"}
-	args = append(args, cidrs...)
-	if !i.proxy.NoRewriteHosts {
-		args = append(args, "--rewrite-hosts")
-	}
-	args = append(args, "--or-die", container.ID)
-	if _, stderr, err := callWeave(args...); err != nil {
-		Log.Warningf("Attaching container %s to weave network failed: %s", container.ID, string(stderr))
-		return errors.New(string(stderr))
-	} else if len(stderr) > 0 {
-		Log.Warningf("Attaching container %s to weave network: %s", container.ID, string(stderr))
-	}
+	// If the client has sent some JSON which might be a HostConfig, add our
+	// parameters back into it, otherwise Docker will consider them overwritten
+	if containerShouldAttach(container) && r.Header.Get("Content-Type") == "application/json" {
+		params := map[string]interface{}{}
+		if err := unmarshalRequestBody(r, &params); err != nil {
+			return err
+		}
+		// HostConfig can be sent either as a struct named HostConfig, or unnamed at top level
+		var hostConfig map[string]interface{}
+		if subParam, found := params["HostConfig"]; found {
+			if typecast, ok := subParam.(map[string]interface{}); ok {
+				hostConfig = typecast
+			}
+			// We can't reliably detect what Docker will see as a top-level HostConfig,
+			// so just assume any parameter at top level indicates there is one.
+		} else if len(params) > 0 {
+			hostConfig = params
+		}
+		if hostConfig != nil {
+			networkMode, err := jsonObject(hostConfig).String("NetworkMode")
+			if err != nil {
+				return err
+			}
+			if strings.HasPrefix(networkMode, "container:") || networkMode == "host" {
+				if err := addVolume(hostConfig, i.proxy.weaveWaitNoopVolume, "/w", "ro"); err != nil {
+					return err
+				}
+			} else {
+				if err := addVolume(hostConfig, i.proxy.weaveWaitVolume, "/w", "ro"); err != nil {
+					return err
+				}
+				if dnsDomain := i.proxy.getDNSDomain(); dnsDomain != "" {
+					if err := i.proxy.setWeaveDNS(hostConfig, container.Config.Hostname, dnsDomain); err != nil {
+						return err
+					}
+				}
+			}
 
+			// Note we marshal the original top-level dictionary to avoid disturbing anything else
+			if err := marshalRequestBody(r, params); err != nil {
+				return err
+			}
+		}
+	}
+	i.proxy.createWait(r, container.ID)
 	return nil
 }
+
+func (i *startContainerInterceptor) InterceptResponse(r *http.Response) error {
+	defer i.proxy.removeWait(r.Request)
+	if r.StatusCode < 200 || r.StatusCode >= 300 { // Docker didn't do the start
+		return nil
+	}
+	return i.proxy.waitForStart(r.Request)
+}

proxy/tls.go

@@ -17,15 +17,17 @@ type TLSConfig struct {
 	*tls.Config
 }
 
-func (c *TLSConfig) enabled() bool {
+// IsEnabled returns true if TLS is enable, according to the config.
+func (c *TLSConfig) IsEnabled() bool {
 	if c == nil {
 		return false
 	}
 	return c.Enabled || c.Verify
 }
 
-func (c *TLSConfig) loadCerts() error {
-	if !c.enabled() {
+// LoadCerts loads the certificates into c.Config, if TLS is enabled.
+func (c *TLSConfig) LoadCerts() error {
+	if !c.IsEnabled() {
 		return nil
 	}
 

router/bridge.go

@@ -0,0 +1,35 @@
+package router
+
+// Interface to packet handling on the local virtual bridge
+type Bridge interface {
+	// Inject a packet to be delivered locally
+	InjectPacket(PacketKey) FlowOp
+
+	// Start consuming packets from the bridge.  Injected packets
+	// should not be included.
+	StartConsumingPackets(BridgeConsumer) error
+
+	String() string
+	Stats() map[string]int
+}
+
+// A function that determines how to handle locally captured packets.
+type BridgeConsumer func(PacketKey) FlowOp
+
+type NullBridge struct{}
+
+func (NullBridge) InjectPacket(PacketKey) FlowOp {
+	return nil
+}
+
+func (NullBridge) StartConsumingPackets(BridgeConsumer) error {
+	return nil
+}
+
+func (NullBridge) String() string {
+	return "<no bridge networking>"
+}
+
+func (NullBridge) Stats() map[string]int {
+	return nil
+}

router/connection.go

@@ -1,8 +1,6 @@
 package router
 
 import (
-	"encoding/binary"
-	"errors"
 	"fmt"
 	"net"
 	"strconv"
@@ -29,7 +27,7 @@ const (
 	TieBreakTied
 )
 
-var ErrConnectToSelf = errors.New("Cannot connect to ourself")
+var ErrConnectToSelf = fmt.Errorf("Cannot connect to ourself")
 
 type RemoteConnection struct {
 	local         *Peer
@@ -42,26 +40,17 @@ type RemoteConnection struct {
 type LocalConnection struct {
 	sync.RWMutex
 	RemoteConnection
-	TCPConn           *net.TCPConn
-	version           byte
-	tcpSender         TCPSender
-	remoteUDPAddr     *net.UDPAddr
-	receivedHeartbeat bool
-	stackFrag         bool
-	effectivePMTU     int
-	SessionKey        *[32]byte
-	heartbeatTCP      *time.Ticker
-	heartbeatTimeout  *time.Timer
-	heartbeatFrame    []byte
-	heartbeat         *time.Ticker
-	fragTest          *time.Ticker
-	forwarder         *Forwarder
-	forwarderDF       *ForwarderDF
-	Decryptor         Decryptor
-	Router            *Router
-	uid               uint64
-	actionChan        chan<- ConnectionAction
-	finished          <-chan struct{} // closed to signal that actorLoop has finished
+	TCPConn       *net.TCPConn
+	version       byte
+	tcpSender     TCPSender
+	remoteUDPAddr *net.UDPAddr
+	SessionKey    *[32]byte
+	heartbeatTCP  *time.Ticker
+	Router        *Router
+	uid           uint64
+	actionChan    chan<- ConnectionAction
+	finished      <-chan struct{} // closed to signal that actorLoop has finished
+	forwarder     OverlayForwarder
 }
 
 type ConnectionAction func() error
@@ -106,7 +95,6 @@ func StartLocalConnection(connRemote *RemoteConnection, tcpConn *net.TCPConn, ud
 		Router:           router,
 		TCPConn:          tcpConn,
 		remoteUDPAddr:    udpAddr,
-		effectivePMTU:    DefaultPMTU,
 		uid:              randUint64(),
 		actionChan:       actionChan,
 		finished:         finished}
@@ -126,48 +114,10 @@ func (conn *LocalConnection) BreakTie(dupConn Connection) ConnectionTieBreak {
 	}
 }
 
-// Read by the forwarder processes when in the UDP senders
-func (conn *LocalConnection) RemoteUDPAddr() *net.UDPAddr {
-	conn.RLock()
-	defer conn.RUnlock()
-	return conn.remoteUDPAddr
-}
-
 func (conn *LocalConnection) Established() bool {
-	conn.RLock()
-	defer conn.RUnlock()
 	return conn.established
 }
 
-// Called by forwarder processes, read in Forward (by sniffer and udp
-// listener process in router).
-func (conn *LocalConnection) setEffectivePMTU(pmtu int) {
-	conn.Lock()
-	defer conn.Unlock()
-	if conn.effectivePMTU != pmtu {
-		conn.effectivePMTU = pmtu
-		conn.Log("Effective PMTU set to", pmtu)
-	}
-}
-
-// Called by the connection's actor process, and by the connection's
-// TCP receiver process. StackFrag is read in conn.forward
-func (conn *LocalConnection) setStackFrag(frag bool) {
-	conn.Lock()
-	defer conn.Unlock()
-	conn.stackFrag = frag
-}
-
-// Called by the connection's TCP receiver process.
-func (conn *LocalConnection) pmtuVerified(pmtu int) {
-	conn.RLock()
-	fwd := conn.forwarderDF
-	conn.RUnlock()
-	if fwd != nil {
-		fwd.PMTUVerified(pmtu)
-	}
-}
-
 // Send directly, not via the Actor.  If it goes via the Actor we can
 // get a deadlock where LocalConnection is blocked talking to
 // LocalPeer and LocalPeer is blocked trying send a ProtocolMsg via
@@ -197,64 +147,6 @@ func (conn *LocalConnection) Shutdown(err error) {
 	go func() { conn.sendAction(func() error { return err }) }()
 }
 
-// Async
-//
-// Heartbeating serves two purposes: a) keeping NAT paths alive, and
-// b) updating a remote peer's knowledge of our address, in the event
-// it changes (e.g. because NAT paths expired).
-func (conn *LocalConnection) ReceivedHeartbeat(remoteUDPAddr *net.UDPAddr, connUID uint64) {
-	if remoteUDPAddr == nil || connUID != conn.uid {
-		return
-	}
-	conn.sendAction(func() error {
-		oldRemoteUDPAddr := conn.remoteUDPAddr
-		old := conn.receivedHeartbeat
-		conn.Lock()
-		conn.remoteUDPAddr = remoteUDPAddr
-		conn.receivedHeartbeat = true
-		conn.Unlock()
-		conn.heartbeatTimeout.Reset(HeartbeatTimeout)
-		if !old {
-			if err := conn.sendSimpleProtocolMsg(ProtocolConnectionEstablished); err != nil {
-				return err
-			}
-		}
-		if oldRemoteUDPAddr == nil {
-			return conn.sendFastHeartbeats()
-		} else if oldRemoteUDPAddr.String() != remoteUDPAddr.String() {
-			log.Println("Peer", conn.remote, "moved from", oldRemoteUDPAddr, "to", remoteUDPAddr)
-		}
-		return nil
-	})
-}
-
-// Async
-func (conn *LocalConnection) SetEstablished() {
-	conn.sendAction(func() error {
-		stopTicker(conn.heartbeat)
-		old := conn.established
-		conn.Lock()
-		conn.established = true
-		conn.Unlock()
-		if old {
-			return nil
-		}
-		conn.Router.Ourself.ConnectionEstablished(conn)
-		if err := conn.ensureForwarders(); err != nil {
-			return err
-		}
-		// Send a large frame down the DF channel in order to prompt
-		// PMTU discovery to start.
-		conn.Send(true, PMTUDiscovery)
-		conn.heartbeat = time.NewTicker(SlowHeartbeat)
-		conn.fragTest = time.NewTicker(FragTestInterval)
-		// avoid initial waits for timers to fire
-		conn.Send(true, conn.heartbeatFrame)
-		conn.performFragTest()
-		return nil
-	})
-}
-
 // Send an actor request to the actorLoop, but don't block if
 // actorLoop has exited - see http://blog.golang.org/pipelines for
 // pattern
@@ -298,85 +190,75 @@ func (conn *LocalConnection) run(actionChan <-chan ConnectionAction, finished ch
 		return
 	}
 
-	if conn.SessionKey == nil {
-		conn.Decryptor = NewNonDecryptor()
-	} else {
-		conn.Decryptor = NewNaClDecryptor(conn.SessionKey, conn.outbound)
-	}
-
 	conn.Log("connection ready; using protocol version", conn.version)
 
-	// The ordering of the following is very important. [1]
-
-	if conn.remoteUDPAddr != nil {
-		if err = conn.ensureForwarders(); err != nil {
-			return
-		}
+	params := ForwarderParams{
+		RemotePeer:         conn.remote,
+		LocalIP:            conn.TCPConn.LocalAddr().(*net.TCPAddr).IP,
+		RemoteAddr:         conn.remoteUDPAddr,
+		ConnUID:            conn.uid,
+		Crypto:             conn.forwarderCrypto(),
+		SendControlMessage: conn.sendOverlayControlMessage,
+		Features:           intro.Features,
 	}
+	if conn.forwarder, err = conn.Router.Overlay.MakeForwarder(params); err != nil {
+		return
+	}
+
+	// As soon as we do AddConnection, the new connection becomes
+	// visible to the packet routing logic.  So AddConnection must
+	// come after MakeForwarder
 	if err = conn.Router.Ourself.AddConnection(conn); err != nil {
 		return
 	}
-	if err = conn.initHeartbeats(); err != nil {
-		return
-	}
+	conn.Router.ConnectionMaker.ConnectionCreated(conn)
 
+	// Forwarder confirmation comes after AddConnection, because
+	// only after that completes do we know the connection is
+	// valid: in particular that it is not a duplicate connection
+	// to the same peer. Sending heartbeats on a duplicate
+	// connection can trip up crypto at the other end, since the
+	// associated UDP packets may get decoded by the other
+	// connection. It is also generally wasteful to engage in any
+	// interaction with the remote on a connection that turns out
+	// to be invalid.
+	conn.forwarder.Confirm()
+
+	// receiveTCP must follow also AddConnection. In the absence
+	// of any indirect connectivity to the remote peer, the first
+	// we hear about it (and any peers reachable from it) is
+	// through topology gossip it sends us on the connection. We
+	// must ensure that the connection has been added to Ourself
+	// prior to processing any such gossip, otherwise we risk
+	// immediately gc'ing part of that newly received portion of
+	// the topology (though not the remote peer itself, since that
+	// will have a positive ref count), leaving behind dangling
+	// references to peers. Hence we must invoke AddConnection,
+	// which is *synchronous*, first.
+	conn.heartbeatTCP = time.NewTicker(TCPHeartbeat)
 	go conn.receiveTCP(intro.Receiver)
+
+	// AddConnection must precede actorLoop. More precisely, it
+	// must precede shutdown, since that invokes DeleteConnection
+	// and is invoked on termination of this entire
+	// function. Essentially this boils down to a prohibition on
+	// running AddConnection in a separate goroutine, at least not
+	// without some synchronisation. Which in turn requires the
+	// launching of the receiveTCP goroutine to precede actorLoop.
 	err = conn.actorLoop(actionChan)
 }
 
-// [1] Ordering constraints:
-//
-// (a) AddConnections must precede initHeartbeats. It is only after
-// the former completes that we know the connection is valid, in
-// particular is not a duplicate connection to the same peer. Sending
-// heartbeats on a duplicate connection can trip up crypto at the
-// other end, since the associated UDP packets may get decoded by the
-// other connection. It is also generally wasteful to engage in any
-// interaction with the remote on a connection that turns out to be
-// invald.
-//
-// (b) AddConnection must precede receiveTCP. In the absence of any
-// indirect connectivity to the remote peer, the first we hear about
-// it (and any peers reachable from it) is through topology gossip it
-// sends us on the connection. We must ensure that the connection has
-// been added to Ourself prior to processing any such gossip,
-// otherwise we risk immediately gc'ing part of that newly received
-// portion of the topology (though not the remote peer itself, since
-// that will have a positive ref count), leaving behind dangling
-// references to peers. Hence we must invoke AddConnection, which is
-// *synchronous*, first.
-//
-// (c) AddConnection must precede actorLoop. More precisely, it must
-// precede shutdown, since that invokes DeleteConnection and is
-// invoked on termination of this entire function. Essentially this
-// boils down to a prohibition on running AddConnection in a separate
-// goroutine, at least not without some synchronisation. Which in turn
-// requires us the launching of the receiveTCP goroutine to precede
-// actorLoop.
-//
-// (d) AddConnection should precede receiveTCP. There is no point
-// starting the latter if the former fails.
-//
-// (e) initHeartbeats should precede actorLoop. The former is setting
-// LocalConnection fields accessed by the latter. Since the latter
-// runs in a separate goroutine, we'd have to add some synchronisation
-// if initHeartbeats isn't run first.
-//
-// (f) ensureForwarders should precede AddConnection. As soon as a
-// connection has been added to LocalPeer by the latter, it becomes
-// visible to the packet routing logic, which will end up dropping
-// packets if the forwarders haven't been created yet. We cannot
-// prevent that completely, since, for example, forwarder can only be
-// created when we know the remote UDP address, but it helps to try.
-
 func (conn *LocalConnection) makeFeatures() map[string]string {
-	return map[string]string{
+	features := map[string]string{
 		"PeerNameFlavour": PeerNameFlavour,
 		"Name":            conn.local.Name.String(),
 		"NickName":        conn.local.NickName,
+		"ShortID":         fmt.Sprint(conn.local.ShortID),
 		"UID":             fmt.Sprint(conn.local.UID),
 		"ConnID":          fmt.Sprint(conn.uid),
 	}
+	conn.Router.Overlay.AddFeaturesTo(features)
+	return features
 }
 
 type features map[string]string
@@ -411,6 +293,17 @@ func (conn *LocalConnection) parseFeatures(features features) (*Peer, error) {
 
 	nickName := features.Get("NickName")
 
+	var shortID uint64
+	var hasShortID bool
+	if shortIDStr, present := features["ShortID"]; present {
+		hasShortID = true
+		shortID, err = strconv.ParseUint(shortIDStr, 10,
+			PeerShortIDBits)
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	uid, err := ParsePeerUID(features.Get("UID"))
 	if err != nil {
 		return nil, err
@@ -422,7 +315,9 @@ func (conn *LocalConnection) parseFeatures(features features) (*Peer, error) {
 	}
 
 	conn.uid ^= remoteConnID
-	return NewPeer(name, nickName, uid, 0), nil
+	peer := NewPeer(name, nickName, uid, 0, PeerShortID(shortID))
+	peer.HasShortID = hasShortID
+	return peer, nil
 }
 
 func (conn *LocalConnection) registerRemote(remote *Peer, acceptNewPeer bool) error {
@@ -446,30 +341,24 @@ func (conn *LocalConnection) registerRemote(remote *Peer, acceptNewPeer bool) er
 	return nil
 }
 
-func (conn *LocalConnection) initHeartbeats() error {
-	conn.heartbeatTCP = time.NewTicker(TCPHeartbeat)
-	conn.heartbeatTimeout = time.NewTimer(HeartbeatTimeout)
-	conn.heartbeatFrame = make([]byte, EthernetOverhead+8)
-	binary.BigEndian.PutUint64(conn.heartbeatFrame[EthernetOverhead:], conn.uid)
-	if conn.remoteUDPAddr == nil {
-		return nil
-	}
-	return conn.sendFastHeartbeats()
-}
-
 func (conn *LocalConnection) actorLoop(actionChan <-chan ConnectionAction) (err error) {
+	fwdErrorChan := conn.forwarder.ErrorChannel()
+	fwdEstablishedChan := conn.forwarder.EstablishedChannel()
+
 	for err == nil {
 		select {
 		case action := <-actionChan:
 			err = action()
+
 		case <-conn.heartbeatTCP.C:
 			err = conn.sendSimpleProtocolMsg(ProtocolHeartbeat)
-		case <-conn.heartbeatTimeout.C:
-			err = fmt.Errorf("timed out waiting for UDP heartbeat")
-		case <-tickerChan(conn.heartbeat):
-			conn.Send(true, conn.heartbeatFrame)
-		case <-tickerChan(conn.fragTest):
-			conn.performFragTest()
+
+		case <-fwdEstablishedChan:
+			conn.established = true
+			fwdEstablishedChan = nil
+			conn.Router.Ourself.ConnectionEstablished(conn)
+
+		case err = <-fwdErrorChan:
 		}
 	}
 	return
@@ -491,19 +380,30 @@ func (conn *LocalConnection) shutdown(err error) {
 		conn.Router.Ourself.DeleteConnection(conn)
 	}
 
-	if conn.heartbeatTimeout != nil {
-		conn.heartbeatTimeout.Stop()
+	stopTicker(conn.heartbeatTCP)
+
+	if conn.forwarder != nil {
+		conn.forwarder.Stop()
 	}
 
-	stopTicker(conn.heartbeatTCP)
-	stopTicker(conn.heartbeat)
-	stopTicker(conn.fragTest)
+	conn.Router.ConnectionMaker.ConnectionTerminated(conn, err)
+}
 
-	// blank out the forwardChan so that the router processes don't
-	// try to send any more
-	conn.stopForwarders()
+func (conn *LocalConnection) forwarderCrypto() *OverlayCrypto {
+	if !conn.Router.UsingPassword() {
+		return nil
+	}
 
-	conn.Router.ConnectionMaker.ConnectionTerminated(conn.remoteTCPAddr, err)
+	name := conn.local.NameByte
+	return &OverlayCrypto{
+		Dec:   NewNaClDecryptor(conn.SessionKey, conn.outbound),
+		Enc:   NewNaClEncryptor(name, conn.SessionKey, conn.outbound, false),
+		EncDF: NewNaClEncryptor(name, conn.SessionKey, conn.outbound, true),
+	}
+}
+
+func (conn *LocalConnection) sendOverlayControlMessage(tag byte, msg []byte) error {
+	return conn.sendProtocolMsg(ProtocolMsg{ProtocolTag(tag), msg})
 }
 
 // Helpers
@@ -539,16 +439,8 @@ func (conn *LocalConnection) receiveTCP(receiver TCPReceiver) {
 func (conn *LocalConnection) handleProtocolMsg(tag ProtocolTag, payload []byte) error {
 	switch tag {
 	case ProtocolHeartbeat:
-	case ProtocolConnectionEstablished:
-		// We sent fast heartbeats to the remote peer, which has now
-		// received at least one of them and told us via this message.
-		// We can now consider the connection as established from our
-		// end.
-		conn.SetEstablished()
-	case ProtocolFragmentationReceived:
-		conn.setStackFrag(true)
-	case ProtocolPMTUVerified:
-		conn.pmtuVerified(int(binary.BigEndian.Uint16(payload)))
+	case ProtocolConnectionEstablished, ProtocolFragmentationReceived, ProtocolPMTUVerified, ProtocolOverlayControlMsg:
+		conn.forwarder.ControlMessage(byte(tag), payload)
 	case ProtocolGossipUnicast, ProtocolGossipBroadcast, ProtocolGossip:
 		return conn.Router.handleGossip(tag, payload)
 	default:
@@ -561,18 +453,8 @@ func (conn *LocalConnection) extendReadDeadline() {
 	conn.TCPConn.SetReadDeadline(time.Now().Add(TCPHeartbeat * 2))
 }
 
-func (conn *LocalConnection) sendFastHeartbeats() error {
-	err := conn.ensureForwarders()
-	if err == nil {
-		conn.heartbeat = time.NewTicker(FastHeartbeat)
-		conn.Send(true, conn.heartbeatFrame) // avoid initial wait
-	}
-	return err
-}
-
-func (conn *LocalConnection) performFragTest() {
-	conn.setStackFrag(false)
-	conn.Send(false, FragTest)
+func (conn *LocalConnection) Forward(key ForwardPacketKey) FlowOp {
+	return conn.forwarder.Forward(key)
 }
 
 func tickerChan(ticker *time.Ticker) <-chan time.Time {

router/connection_maker.go

@@ -10,6 +10,7 @@ import (
 const (
 	InitialInterval = 2 * time.Second
 	MaxInterval     = 6 * time.Minute
+	ResetAfter      = 1 * time.Minute
 )
 
 type peerAddrs map[string]*net.TCPAddr
@@ -20,13 +21,22 @@ type ConnectionMaker struct {
 	port        int
 	discovery   bool
 	targets     map[string]*Target
+	connections map[Connection]struct{}
 	directPeers peerAddrs
 	actionChan  chan<- ConnectionMakerAction
 }
 
+type TargetState int
+
+const (
+	TargetWaiting    TargetState = iota // we are waiting to connect there
+	TargetAttempting                    // we are attempting to connect there
+	TargetConnected                     // we are connected to there
+)
+
 // Information about an address where we may find a peer
 type Target struct {
-	attempting  bool          // are we currently attempting to connect there?
+	state       TargetState
 	lastError   error         // reason for disconnection last time
 	tryAfter    time.Time     // next time to try this address
 	tryInterval time.Duration // retry delay on next failure
@@ -43,6 +53,7 @@ func NewConnectionMaker(ourself *LocalPeer, peers *Peers, port int, discovery bo
 		discovery:   discovery,
 		directPeers: peerAddrs{},
 		targets:     make(map[string]*Target),
+		connections: make(map[Connection]struct{}),
 		actionChan:  actionChan}
 	go cm.queryLoop(actionChan)
 	return cm
@@ -71,7 +82,7 @@ func (cm *ConnectionMaker) InitiateConnections(peers []string, replace bool) []e
 			cm.directPeers[peer] = addr
 			// curtail any existing reconnect interval
 			if target, found := cm.targets[addr.String()]; found {
-				target.tryNow()
+				target.nextTryNow()
 			}
 		}
 		return true
@@ -88,12 +99,42 @@ func (cm *ConnectionMaker) ForgetConnections(peers []string) {
 	}
 }
 
-func (cm *ConnectionMaker) ConnectionTerminated(address string, err error) {
+func (cm *ConnectionMaker) ConnectionAborted(address string, err error) {
 	cm.actionChan <- func() bool {
-		if target, found := cm.targets[address]; found {
-			target.attempting = false
+		target := cm.targets[address]
+		target.state = TargetWaiting
+		target.lastError = err
+		target.nextTryLater()
+		return true
+	}
+}
+
+func (cm *ConnectionMaker) ConnectionCreated(conn Connection) {
+	cm.actionChan <- func() bool {
+		cm.connections[conn] = void
+		if conn.Outbound() {
+			target := cm.targets[conn.RemoteTCPAddr()]
+			target.state = TargetConnected
+		}
+		return false
+	}
+}
+
+func (cm *ConnectionMaker) ConnectionTerminated(conn Connection, err error) {
+	cm.actionChan <- func() bool {
+		delete(cm.connections, conn)
+		if conn.Outbound() {
+			target := cm.targets[conn.RemoteTCPAddr()]
+			target.state = TargetWaiting
 			target.lastError = err
-			target.retry()
+			switch {
+			case err == ErrConnectToSelf:
+				target.nextTryNever()
+			case time.Now().After(target.tryAfter.Add(ResetAfter)):
+				target.nextTryNow()
+			default:
+				target.nextTryLater()
+			}
 		}
 		return true
 	}
@@ -123,9 +164,6 @@ func (cm *ConnectionMaker) checkStateAndAttemptConnections() time.Duration {
 		validTarget  = make(map[string]struct{})
 		directTarget = make(map[string]struct{})
 	)
-	// Copy the set of things we are connected to, so we can access
-	// them without locking.  Also clear out any entries in cm.targets
-	// for existing connections.
 	ourConnectedPeers, ourConnectedTargets, ourInboundIPs := cm.ourConnections()
 
 	addTarget := func(address string) {
@@ -136,8 +174,8 @@ func (cm *ConnectionMaker) checkStateAndAttemptConnections() time.Duration {
 		if _, found := cm.targets[address]; found {
 			return
 		}
-		target := &Target{}
-		target.tryNow()
+		target := &Target{state: TargetWaiting}
+		target.nextTryNow()
 		cm.targets[address] = target
 	}
 
@@ -176,9 +214,8 @@ func (cm *ConnectionMaker) ourConnections() (PeerNameSet, map[string]struct{}, m
 		ourConnectedTargets = make(map[string]struct{})
 		ourInboundIPs       = make(map[string]struct{})
 	)
-	for conn := range cm.ourself.Connections() {
+	for conn := range cm.connections {
 		address := conn.RemoteTCPAddr()
-		delete(cm.targets, address)
 		ourConnectedPeers[conn.Remote().Name] = void
 		ourConnectedTargets[address] = void
 		if conn.Outbound() {
@@ -212,7 +249,7 @@ func (cm *ConnectionMaker) addPeerTargets(ourConnectedPeers PeerNameSet, addTarg
 			} else if ip, _, err := net.SplitHostPort(address); err == nil {
 				// There is no point connecting to the (likely
 				// ephemeral) remote port of an inbound connection
-				// that some peer has. Let's try to connect to on the
+				// that some peer has. Let's try to connect on the
 				// weave port instead.
 				addTarget(fmt.Sprintf("%s:%d", ip, cm.port))
 			}
@@ -224,7 +261,7 @@ func (cm *ConnectionMaker) connectToTargets(validTarget map[string]struct{}, dir
 	now := time.Now() // make sure we catch items just added
 	after := MaxDuration
 	for address, target := range cm.targets {
-		if target.attempting {
+		if target.state != TargetWaiting {
 			continue
 		}
 		if _, valid := validTarget[address]; !valid {
@@ -236,7 +273,7 @@ func (cm *ConnectionMaker) connectToTargets(validTarget map[string]struct{}, dir
 		}
 		switch duration := target.tryAfter.Sub(now); {
 		case duration <= 0:
-			target.attempting = true
+			target.state = TargetAttempting
 			_, isCmdLineTarget := directTarget[address]
 			go cm.attemptConnection(address, isCmdLineTarget)
 		case duration < after:
@@ -250,23 +287,23 @@ func (cm *ConnectionMaker) attemptConnection(address string, acceptNewPeer bool)
 	log.Printf("->[%s] attempting connection", address)
 	if err := cm.ourself.CreateConnection(address, acceptNewPeer); err != nil {
 		log.Errorf("->[%s] error during connection attempt: %v", address, err)
-		cm.ConnectionTerminated(address, err)
+		cm.ConnectionAborted(address, err)
 	}
 }
 
-func (t *Target) tryNow() {
+func (t *Target) nextTryNever() {
+	t.tryAfter = time.Time{}
+	t.tryInterval = MaxInterval
+}
+
+func (t *Target) nextTryNow() {
 	t.tryAfter = time.Now()
 	t.tryInterval = InitialInterval
 }
 
 // The delay at the nth retry is a random value in the range
 // [i-i/2,i+i/2], where i = InitialInterval * 1.5^(n-1).
-func (t *Target) retry() {
-	if t.lastError == ErrConnectToSelf {
-		t.tryAfter = time.Time{}
-		t.tryInterval = MaxInterval
-		return
-	}
+func (t *Target) nextTryLater() {
 	t.tryAfter = time.Now().Add(t.tryInterval/2 + time.Duration(rand.Int63n(int64(t.tryInterval))))
 	t.tryInterval = t.tryInterval * 3 / 2
 	if t.tryInterval > MaxInterval {

router/consts.go

@@ -6,29 +6,16 @@ import (
 )
 
 const (
-	EthernetOverhead    = 14
-	UDPOverhead         = 28 // 20 bytes for IPv4, 8 bytes for UDP
 	Port                = 6783
 	HTTPPort            = Port + 1
-	DefaultPMTU         = 65535
 	MaxUDPPacketSize    = 65535
 	ChannelSize         = 16
-	FragTestSize        = 60001
-	PMTUDiscoverySize   = 60000
 	TCPHeartbeat        = 30 * time.Second
+	GossipInterval      = 30 * time.Second
+	MaxDuration         = time.Duration(math.MaxInt64)
+	MaxTCPMsgSize       = 10 * 1024 * 1024
 	FastHeartbeat       = 500 * time.Millisecond
 	SlowHeartbeat       = 10 * time.Second
-	FragTestInterval    = 5 * time.Minute
-	GossipInterval      = 30 * time.Second
-	PMTUVerifyAttempts  = 8
-	PMTUVerifyTimeout   = 10 * time.Millisecond // gets doubled with every attempt
-	MaxDuration         = time.Duration(math.MaxInt64)
 	MaxMissedHeartbeats = 6
 	HeartbeatTimeout    = MaxMissedHeartbeats * SlowHeartbeat
-	MaxTCPMsgSize       = 10 * 1024 * 1024
-)
-
-var (
-	FragTest      = make([]byte, FragTestSize)
-	PMTUDiscovery = make([]byte, PMTUDiscoverySize)
 )

router/ethernet_decoder.go

@@ -29,7 +29,13 @@ func (dec *EthernetDecoder) DecodeLayers(data []byte) {
 	dec.parser.DecodeLayers(data, &dec.decoded)
 }
 
-func (dec *EthernetDecoder) sendICMPFragNeeded(mtu int, sendFrame func([]byte) error) error {
+func (dec *EthernetDecoder) PacketKey() (key PacketKey) {
+	copy(key.SrcMAC[:], dec.Eth.SrcMAC)
+	copy(key.DstMAC[:], dec.Eth.DstMAC)
+	return
+}
+
+func (dec *EthernetDecoder) makeICMPFragNeeded(mtu int) ([]byte, error) {
 	buf := gopacket.NewSerializeBuffer()
 	opts := gopacket.SerializeOptions{
 		FixLengths:       true,
@@ -57,23 +63,17 @@ func (dec *EthernetDecoder) sendICMPFragNeeded(mtu int, sendFrame func([]byte) e
 			Seq:      uint16(mtu)},
 		&payload)
 	if err != nil {
-		return err
+		return nil, err
 	}
 
-	log.Printf("Sending ICMP 3,4 (%v -> %v): PMTU= %v", dec.IP.DstIP, dec.IP.SrcIP, mtu)
-	return sendFrame(buf.Bytes())
+	log.Printf("Sending ICMP 3,4 (%v -> %v): PMTU=%v", dec.IP.DstIP, dec.IP.SrcIP, mtu)
+	return buf.Bytes(), nil
 }
 
 var (
-	// see http://en.wikipedia.org/wiki/Multicast_address#Ethernet
-	stpMACPrefix = []byte{0x01, 0x80, 0xC2, 0x00, 0x00}
-	zeroMAC, _   = net.ParseMAC("00:00:00:00:00:00")
+	zeroMAC, _ = net.ParseMAC("00:00:00:00:00:00")
 )
 
-func (dec *EthernetDecoder) DropFrame() bool {
-	return bytes.Equal(stpMACPrefix, dec.Eth.DstMAC[:len(stpMACPrefix)])
-}
-
 func (dec *EthernetDecoder) IsSpecial() bool {
 	return dec.Eth.Length == 0 && dec.Eth.EthernetType == layers.EthernetTypeLLC &&
 		bytes.Equal(zeroMAC, dec.Eth.SrcMAC) && bytes.Equal(zeroMAC, dec.Eth.DstMAC)

router/flow.go

@@ -0,0 +1,102 @@
+package router
+
+import "net"
+
+// Just enough flow machinery for the weave router
+
+type MAC [6]byte
+
+func (mac MAC) String() string {
+	return net.HardwareAddr(mac[:]).String()
+}
+
+type PacketKey struct {
+	SrcMAC MAC
+	DstMAC MAC
+}
+
+type ForwardPacketKey struct {
+	SrcPeer *Peer
+	DstPeer *Peer
+	PacketKey
+}
+
+type FlowOp interface {
+	// The caller must supply an EthernetDecoder specific to this
+	// thread, which has already been used to decode the frame.
+	// The broadcast parameter is a hint whether the packet is
+	// being broadcast.
+	Process(frame []byte, dec *EthernetDecoder, broadcast bool)
+
+	// Does the FlowOp discard the packet?
+	Discards() bool
+}
+
+type DiscardingFlowOp struct{}
+
+func (DiscardingFlowOp) Process([]byte, *EthernetDecoder, bool) {
+}
+
+func (DiscardingFlowOp) Discards() bool {
+	return true
+}
+
+type NonDiscardingFlowOp struct{}
+
+func (NonDiscardingFlowOp) Discards() bool {
+	return false
+}
+
+type MultiFlowOp struct {
+	broadcast bool
+	ops       []FlowOp
+}
+
+func NewMultiFlowOp(broadcast bool, ops ...FlowOp) *MultiFlowOp {
+	mfop := &MultiFlowOp{broadcast: broadcast}
+	for _, op := range ops {
+		mfop.Add(op)
+	}
+	return mfop
+}
+
+func (mfop *MultiFlowOp) Add(op FlowOp) {
+	mfop.ops = append(mfop.ops, op)
+}
+
+func (mfop *MultiFlowOp) Process(frame []byte, dec *EthernetDecoder,
+	broadcast bool) {
+	for _, op := range mfop.ops {
+		op.Process(frame, dec, mfop.broadcast)
+	}
+}
+
+func (mfop *MultiFlowOp) Discards() bool {
+	for _, op := range mfop.ops {
+		if !op.Discards() {
+			return false
+		}
+	}
+
+	return true
+}
+
+func FlattenFlowOp(fop FlowOp) []FlowOp {
+	return collectFlowOps(nil, fop)
+}
+
+func collectFlowOps(into []FlowOp, fop FlowOp) []FlowOp {
+	if fop == nil {
+		return into
+	}
+
+	if mfop, ok := fop.(*MultiFlowOp); ok {
+		for _, op := range mfop.ops {
+			into = collectFlowOps(into, op)
+		}
+
+		return into
+	}
+
+	return append(into, fop)
+}

router/forwarder.go

@@ -1,419 +0,0 @@
-package router
-
-import (
-	"syscall"
-	"time"
-
-	"github.com/google/gopacket"
-	"github.com/google/gopacket/layers"
-)
-
-type ForwardedFrame struct {
-	srcPeer *Peer
-	dstPeer *Peer
-	frame   []byte
-}
-
-type FrameTooBigError struct {
-	EPMTU int // effective pmtu, i.e. what we tell packet senders
-}
-
-func (conn *LocalConnection) ensureForwarders() error {
-	if conn.forwarder != nil || conn.forwarderDF != nil {
-		return nil
-	}
-	udpSender := NewSimpleUDPSender(conn)
-	udpSenderDF, err := NewRawUDPSender(conn) // only thing that can error, so do it early
-	if err != nil {
-		return err
-	}
-
-	usingPassword := conn.SessionKey != nil
-	var encryptor, encryptorDF Encryptor
-	if usingPassword {
-		encryptor = NewNaClEncryptor(conn.local.NameByte, conn.SessionKey, conn.outbound, false)
-		encryptorDF = NewNaClEncryptor(conn.local.NameByte, conn.SessionKey, conn.outbound, true)
-	} else {
-		encryptor = NewNonEncryptor(conn.local.NameByte)
-		encryptorDF = NewNonEncryptor(conn.local.NameByte)
-	}
-
-	forwarder := NewForwarder(conn, encryptor, udpSender, DefaultPMTU)
-	forwarderDF := NewForwarderDF(conn, encryptorDF, udpSenderDF, DefaultPMTU)
-
-	// Various fields in the conn struct are read by other processes,
-	// so we have to use locks.
-	conn.Lock()
-	conn.forwarder = forwarder
-	conn.forwarderDF = forwarderDF
-	conn.Unlock()
-
-	return nil
-}
-
-func (conn *LocalConnection) stopForwarders() {
-	if conn.forwarder == nil || conn.forwarderDF == nil {
-		return
-	}
-	conn.forwarder.Shutdown()
-	conn.forwarderDF.Shutdown()
-}
-
-// Called from connection's actor process, and from the connection's
-// TCP receiver process.
-func (conn *LocalConnection) Send(df bool, frameBytes []byte) error {
-	frame := &ForwardedFrame{
-		srcPeer: conn.local,
-		dstPeer: conn.remote,
-		frame:   frameBytes}
-	return conn.forward(frame, nil, df)
-}
-
-// Called from LocalPeer.Relay[Broadcast] which is itself invoked from
-// router (both UDP listener process and sniffer process).
-func (conn *LocalConnection) Forward(frame *ForwardedFrame, dec *EthernetDecoder) error {
-	return conn.forward(frame, dec, dec != nil && dec.DF())
-}
-
-func (conn *LocalConnection) forward(frame *ForwardedFrame, dec *EthernetDecoder, df bool) error {
-	conn.RLock()
-	var (
-		forwarder     = conn.forwarder
-		forwarderDF   = conn.forwarderDF
-		effectivePMTU = conn.effectivePMTU
-		stackFrag     = conn.stackFrag
-	)
-	conn.RUnlock()
-
-	if forwarder == nil || forwarderDF == nil {
-		conn.Log("Cannot forward frame yet - awaiting contact")
-		return nil
-	}
-
-	// We could use non-blocking channel sends here, i.e. drop frames
-	// on the floor when the forwarder is busy. This would allow our
-	// caller - the capturing loop in the router - to read frames more
-	// quickly when under load, i.e. we'd drop fewer frames on the
-	// floor during capture. And we could maximise CPU utilisation
-	// since we aren't stalling a thread. However, a lot of work has
-	// already been done by the time we get here. Since any packet we
-	// drop will likely get re-transmitted we end up paying that cost
-	// multiple times. So it's better to drop things at the beginning
-	// of our pipeline.
-	if df {
-		if !frameTooBig(frame, effectivePMTU) {
-			forwarderDF.Forward(frame)
-			return nil
-		}
-		return FrameTooBigError{EPMTU: effectivePMTU}
-	}
-
-	if stackFrag || dec == nil || len(dec.decoded) < 2 {
-		forwarder.Forward(frame)
-		return nil
-	}
-	// Don't have trustworthy stack, so we're going to have to
-	// send it DF in any case.
-	if !frameTooBig(frame, effectivePMTU) {
-		forwarderDF.Forward(frame)
-		return nil
-	}
-	// We can't trust the stack to fragment, we have IP, and we
-	// have a frame that's too big for the MTU, so we have to
-	// fragment it ourself.
-	return fragment(dec.Eth, dec.IP, effectivePMTU, frame, func(segFrame *ForwardedFrame) {
-		forwarderDF.Forward(segFrame)
-	})
-}
-
-func frameTooBig(frame *ForwardedFrame, effectivePMTU int) bool {
-	// We capture/forward complete ethernet frames. Therefore the
-	// frame length includes the ethernet header. However, MTUs
-	// operate at the IP layer and thus do not include the ethernet
-	// header. To put it another way, when a sender that was told an
-	// MTU of M sends an IP packet of exactly that length, we will
-	// capture/forward M + EthernetOverhead bytes of data.
-	return len(frame.frame) > effectivePMTU+EthernetOverhead
-}
-
-func fragment(eth layers.Ethernet, ip layers.IPv4, pmtu int, frame *ForwardedFrame, forward func(*ForwardedFrame)) error {
-	// We are not doing any sort of NAT, so we don't need to worry
-	// about checksums of IP payload (eg UDP checksum).
-	headerSize := int(ip.IHL) * 4
-	// &^ is bit clear (AND NOT). So here we're clearing the lowest 3
-	// bits.
-	maxSegmentSize := (pmtu - headerSize) &^ 7
-	opts := gopacket.SerializeOptions{
-		FixLengths:       false,
-		ComputeChecksums: true}
-	payloadSize := int(ip.Length) - headerSize
-	payload := ip.BaseLayer.Payload[:payloadSize]
-	offsetBase := int(ip.FragOffset) << 3
-	origFlags := ip.Flags
-	ip.Flags = ip.Flags | layers.IPv4MoreFragments
-	ip.Length = uint16(headerSize + maxSegmentSize)
-	if eth.EthernetType == layers.EthernetTypeLLC {
-		// using LLC, so must set eth length correctly. eth length
-		// is just the length of the payload
-		eth.Length = ip.Length
-	} else {
-		eth.Length = 0
-	}
-	for offset := 0; offset < payloadSize; offset += maxSegmentSize {
-		var segmentPayload []byte
-		if len(payload) <= maxSegmentSize {
-			// last one
-			segmentPayload = payload
-			ip.Length = uint16(len(payload) + headerSize)
-			ip.Flags = origFlags
-			if eth.EthernetType == layers.EthernetTypeLLC {
-				eth.Length = ip.Length
-			} else {
-				eth.Length = 0
-			}
-		} else {
-			segmentPayload = payload[:maxSegmentSize]
-			payload = payload[maxSegmentSize:]
-		}
-		ip.FragOffset = uint16((offset + offsetBase) >> 3)
-		buf := gopacket.NewSerializeBuffer()
-		segPayload := gopacket.Payload(segmentPayload)
-		err := gopacket.SerializeLayers(buf, opts, &eth, &ip, &segPayload)
-		if err != nil {
-			return err
-		}
-		// make copies of the frame we received
-		segFrame := *frame
-		segFrame.frame = buf.Bytes()
-		forward(&segFrame)
-	}
-	return nil
-}
-
-// Forwarder
-
-type Forwarder struct {
-	conn             *LocalConnection
-	ch               chan<- *ForwardedFrame
-	finished         <-chan struct{}
-	enc              Encryptor
-	udpSender        UDPSender
-	maxPayload       int
-	processSendError func(error) error
-}
-
-func NewForwarder(conn *LocalConnection, enc Encryptor, udpSender UDPSender, pmtu int) *Forwarder {
-	ch := make(chan *ForwardedFrame, ChannelSize)
-	finished := make(chan struct{})
-	fwd := &Forwarder{
-		conn:             conn,
-		ch:               ch,
-		finished:         finished,
-		enc:              enc,
-		udpSender:        udpSender,
-		maxPayload:       pmtu - UDPOverhead,
-		processSendError: func(err error) error { return err }}
-	go fwd.run(ch, finished)
-	return fwd
-}
-
-func (fwd *Forwarder) Shutdown() {
-	fwd.ch <- nil
-}
-
-func (fwd *Forwarder) Forward(frame *ForwardedFrame) {
-	select {
-	case fwd.ch <- frame:
-	case <-fwd.finished:
-	}
-}
-
-func (fwd *Forwarder) run(ch <-chan *ForwardedFrame, finished chan<- struct{}) {
-	defer fwd.udpSender.Shutdown()
-	for {
-		if !fwd.accumulateAndSendFrames(ch, <-ch) {
-			close(finished)
-			return
-		}
-	}
-}
-
-func (fwd *Forwarder) effectiveOverhead() int {
-	return UDPOverhead + fwd.enc.PacketOverhead() + fwd.enc.FrameOverhead() + EthernetOverhead
-}
-
-// Drain the inbound channel of frames, aggregating them into larger
-// packets for efficient transmission.
-//
-// FIXME Depending on the golang scheduler, and the rate at which
-// franes get sent to the forwarder, we can be going around this loop
-// forever. That is bad since there may be other stuff for us to do,
-// i.e. the other branches in the run loop.
-func (fwd *Forwarder) accumulateAndSendFrames(ch <-chan *ForwardedFrame, frame *ForwardedFrame) bool {
-	if frame == nil {
-		return false
-	}
-	if !fwd.appendFrame(frame) {
-		fwd.logDrop(frame)
-		// [1] The buffer is empty at this point and therefore we must
-		// not flush it. The easiest way to accomplish that is simply
-		// by returning to the surrounding run loop.
-		return true
-	}
-	for {
-		select {
-		case frame = <-ch:
-			if frame == nil {
-				return false
-			}
-			if !fwd.appendFrame(frame) {
-				fwd.flush()
-				if !fwd.appendFrame(frame) {
-					fwd.logDrop(frame)
-					return true // see [1]
-				}
-			}
-		default:
-			fwd.flush()
-			return true
-		}
-	}
-}
-
-func (fwd *Forwarder) logDrop(frame *ForwardedFrame) {
-	fwd.conn.ErrorLog("Dropping too big frame during forwarding: frame len:", len(frame.frame), "; effective PMTU:", fwd.maxPayload+UDPOverhead-fwd.effectiveOverhead())
-}
-
-func (fwd *Forwarder) appendFrame(frame *ForwardedFrame) bool {
-	frameLen := len(frame.frame)
-	if fwd.enc.TotalLen()+fwd.enc.FrameOverhead()+frameLen > fwd.maxPayload {
-		return false
-	}
-	fwd.enc.AppendFrame(frame.srcPeer.NameByte, frame.dstPeer.NameByte, frame.frame)
-	return true
-}
-
-func (fwd *Forwarder) flush() {
-	msg, err := fwd.enc.Bytes()
-	if err != nil {
-		fwd.conn.Shutdown(err)
-	}
-	err = fwd.processSendError(fwd.udpSender.Send(msg))
-	if err != nil && PosixError(err) != syscall.ENOBUFS {
-		fwd.conn.Shutdown(err)
-	}
-}
-
-type ForwarderDF struct {
-	Forwarder
-	verifyPMTUTick  <-chan time.Time
-	verifyPMTU      chan<- int
-	pmtuVerifyCount uint
-	pmtuVerified    bool
-	highestGoodPMTU int
-	unverifiedPMTU  int
-	lowestBadPMTU   int
-}
-
-func NewForwarderDF(conn *LocalConnection, enc Encryptor, udpSender UDPSender, pmtu int) *ForwarderDF {
-	ch := make(chan *ForwardedFrame, ChannelSize)
-	finished := make(chan struct{})
-	verifyPMTU := make(chan int, ChannelSize)
-	fwd := &ForwarderDF{
-		Forwarder: Forwarder{
-			conn:       conn,
-			ch:         ch,
-			finished:   finished,
-			enc:        enc,
-			udpSender:  udpSender,
-			maxPayload: pmtu - UDPOverhead},
-		verifyPMTU: verifyPMTU}
-	fwd.Forwarder.processSendError = fwd.processSendError
-	fwd.unverifiedPMTU = pmtu - fwd.effectiveOverhead()
-	conn.setEffectivePMTU(fwd.unverifiedPMTU)
-	go fwd.run(ch, finished, verifyPMTU)
-	return fwd
-}
-
-func (fwd *ForwarderDF) PMTUVerified(pmtu int) {
-	select {
-	case fwd.verifyPMTU <- pmtu:
-	case <-fwd.finished:
-	}
-}
-
-func (fwd *ForwarderDF) run(ch <-chan *ForwardedFrame, finished chan<- struct{}, verifyPMTU <-chan int) {
-	defer fwd.udpSender.Shutdown()
-	for {
-		select {
-		case <-fwd.verifyPMTUTick:
-			// We only do this case here when we know the
-			// buffers are all empty so that we don't risk
-			// appending verify-frames to other data.
-			fwd.verifyPMTUTick = nil
-			if fwd.pmtuVerified {
-				continue
-			}
-			if fwd.pmtuVerifyCount > 0 {
-				fwd.pmtuVerifyCount--
-				fwd.attemptVerifyEffectivePMTU()
-			} else {
-				// we've exceeded the verification
-				// attempts of the unverifiedPMTU
-				fwd.lowestBadPMTU = fwd.unverifiedPMTU
-				fwd.verifyEffectivePMTU((fwd.highestGoodPMTU + fwd.lowestBadPMTU) / 2)
-			}
-		case epmtu := <-verifyPMTU:
-			if fwd.pmtuVerified || epmtu != fwd.unverifiedPMTU {
-				continue
-			}
-			if epmtu+1 < fwd.lowestBadPMTU {
-				fwd.highestGoodPMTU = fwd.unverifiedPMTU // = epmtu
-				fwd.verifyEffectivePMTU((fwd.highestGoodPMTU + fwd.lowestBadPMTU) / 2)
-			} else {
-				fwd.pmtuVerified = true
-				fwd.maxPayload = epmtu + fwd.effectiveOverhead() - UDPOverhead
-				fwd.conn.setEffectivePMTU(epmtu)
-				fwd.conn.Log("Effective PMTU verified at", epmtu)
-			}
-		case frame := <-ch:
-			if !fwd.accumulateAndSendFrames(ch, frame) {
-				close(finished)
-				return
-			}
-		}
-	}
-}
-
-func (fwd *ForwarderDF) verifyEffectivePMTU(newUnverifiedPMTU int) {
-	fwd.unverifiedPMTU = newUnverifiedPMTU
-	fwd.pmtuVerifyCount = PMTUVerifyAttempts
-	fwd.attemptVerifyEffectivePMTU()
-}
-
-func (fwd *ForwarderDF) attemptVerifyEffectivePMTU() {
-	fwd.enc.AppendFrame(fwd.conn.local.NameByte, fwd.conn.remote.NameByte,
-		make([]byte, fwd.unverifiedPMTU+EthernetOverhead))
-	fwd.flush()
-	if fwd.verifyPMTUTick == nil {
-		fwd.verifyPMTUTick = time.After(PMTUVerifyTimeout << (PMTUVerifyAttempts - fwd.pmtuVerifyCount))
-	}
-}
-
-func (fwd *ForwarderDF) processSendError(err error) error {
-	if mtbe, ok := err.(MsgTooBigError); ok {
-		newUnverifiedPMTU := mtbe.PMTU - fwd.effectiveOverhead()
-		if newUnverifiedPMTU < fwd.unverifiedPMTU {
-			fwd.pmtuVerified = false
-			fwd.maxPayload = mtbe.PMTU - UDPOverhead
-			fwd.highestGoodPMTU = 8
-			fwd.lowestBadPMTU = newUnverifiedPMTU + 1
-			fwd.conn.setEffectivePMTU(newUnverifiedPMTU)
-			fwd.verifyEffectivePMTU(newUnverifiedPMTU)
-		}
-
-		return nil
-	}
-
-	return err
-}

router/gossip_test.go

@@ -42,11 +42,8 @@ func sendPendingGossip(routers ...*Router) {
 }
 
 func (router *Router) AddTestChannelConnection(r *Router) {
-	fromName := router.Ourself.Peer.Name
-	toName := r.Ourself.Peer.Name
-
-	fromPeer := NewPeer(fromName, "", router.Ourself.Peer.UID, 0)
-	toPeer := NewPeer(toName, "", r.Ourself.Peer.UID, 0)
+	fromPeer := NewPeerFrom(router.Ourself.Peer)
+	toPeer := NewPeerFrom(r.Ourself.Peer)
 
 	r.Peers.FetchWithDefault(fromPeer)    // Has side-effect of incrementing refcount
 	router.Peers.FetchWithDefault(toPeer) //
@@ -80,13 +77,13 @@ func TestGossipTopology(t *testing.T) {
 // the routers supplied as arguments, carrying across all UID and
 // version information.
 func (router *Router) tp(routers ...*Router) *Peer {
-	peer := NewPeer(router.Ourself.Peer.Name, "", router.Ourself.Peer.UID, 0)
+	peer := NewPeerFrom(router.Ourself.Peer)
 	connections := make(map[PeerName]Connection)
 	for _, r := range routers {
-		p := NewPeer(r.Ourself.Peer.Name, "", r.Ourself.Peer.UID, r.Ourself.Peer.version)
+		p := NewPeerFrom(r.Ourself.Peer)
 		connections[r.Ourself.Peer.Name] = newMockConnection(peer, p)
 	}
-	peer.version = router.Ourself.Peer.version
+	peer.Version = router.Ourself.Peer.Version
 	peer.connections = connections
 	return peer
 }

router/local_peer.go

@@ -19,7 +19,8 @@ type LocalPeerAction func()
 func NewLocalPeer(name PeerName, nickName string, router *Router) *LocalPeer {
 	actionChan := make(chan LocalPeerAction, ChannelSize)
 	peer := &LocalPeer{
-		Peer:       NewPeer(name, nickName, 0, 0),
+		Peer: NewPeer(name, nickName, randomPeerUID(), 0,
+			randomPeerShortID()),
 		router:     router,
 		actionChan: actionChan,
 	}
@@ -27,56 +28,6 @@ func NewLocalPeer(name PeerName, nickName string, router *Router) *LocalPeer {
 	return peer
 }
 
-func (peer *LocalPeer) Forward(dstPeer *Peer, frame []byte, dec *EthernetDecoder) error {
-	return peer.Relay(peer.Peer, dstPeer, frame, dec)
-}
-
-func (peer *LocalPeer) Broadcast(frame []byte, dec *EthernetDecoder) {
-	peer.RelayBroadcast(peer.Peer, frame, dec)
-}
-
-func (peer *LocalPeer) Relay(srcPeer, dstPeer *Peer, frame []byte, dec *EthernetDecoder) error {
-	relayPeerName, found := peer.router.Routes.Unicast(dstPeer.Name)
-	if !found {
-		// Not necessarily an error as there could be a race with the
-		// dst disappearing whilst the frame is in flight
-		log.Println("Received packet for unknown destination:", dstPeer)
-		return nil
-	}
-	conn, found := peer.ConnectionTo(relayPeerName)
-	if !found {
-		// Again, could just be a race, not necessarily an error
-		log.Println("Unable to find connection to relay peer", relayPeerName)
-		return nil
-	}
-	return conn.(*LocalConnection).Forward(&ForwardedFrame{
-		srcPeer: srcPeer,
-		dstPeer: dstPeer,
-		frame:   frame},
-		dec)
-}
-
-func (peer *LocalPeer) RelayBroadcast(srcPeer *Peer, frame []byte, dec *EthernetDecoder) {
-	nextHops := peer.router.Routes.Broadcast(srcPeer.Name)
-	if len(nextHops) == 0 {
-		return
-	}
-	for _, conn := range peer.ConnectionsTo(nextHops) {
-		err := conn.(*LocalConnection).Forward(&ForwardedFrame{
-			srcPeer: srcPeer,
-			dstPeer: conn.Remote(),
-			frame:   frame},
-			dec)
-		if err != nil {
-			if ftbe, ok := err.(FrameTooBigError); ok {
-				log.Warningf("dropping too big DF broadcast frame (%v -> %v): PMTU= %v", dec.IP.DstIP, dec.IP.SrcIP, ftbe.EPMTU)
-			} else {
-				log.Errorln(err)
-			}
-		}
-	}
-}
-
 func (peer *LocalPeer) Connections() ConnectionSet {
 	connections := make(ConnectionSet)
 	peer.RLock()
@@ -210,7 +161,10 @@ func (peer *LocalPeer) handleAddConnection(conn Connection) error {
 		conn.Log("connection added (new peer)")
 		peer.router.SendAllGossipDown(conn)
 	}
+
+	peer.router.Routes.Recalculate()
 	peer.broadcastPeerUpdate(conn.Remote())
+
 	return nil
 }
 
@@ -224,6 +178,8 @@ func (peer *LocalPeer) handleConnectionEstablished(conn Connection) {
 	}
 	peer.connectionEstablished(conn)
 	conn.Log("connection fully established")
+
+	peer.router.Routes.Recalculate()
 	peer.broadcastPeerUpdate()
 }
 
@@ -243,14 +199,20 @@ func (peer *LocalPeer) handleDeleteConnection(conn Connection) {
 	// Must do garbage collection first to ensure we don't send out an
 	// update with unreachable peers (can cause looping)
 	peer.router.Peers.GarbageCollect()
+	peer.router.Routes.Recalculate()
 	peer.broadcastPeerUpdate()
 }
 
 // helpers
 
 func (peer *LocalPeer) broadcastPeerUpdate(peers ...*Peer) {
-	peer.router.Routes.Recalculate()
-	peer.router.TopologyGossip.GossipBroadcast(NewTopologyGossipData(peer.router.Peers, append(peers, peer.Peer)...))
+	// Some tests run without a router.  This should be fixed so
+	// that the relevant part of Router can be easily run in the
+	// context of a test, but that will involve significant
+	// reworking of tests.
+	if peer.router != nil {
+		peer.router.BroadcastTopologyUpdate(append(peers, peer.Peer))
+	}
 }
 
 func (peer *LocalPeer) checkConnectionLimit() error {
@@ -265,20 +227,20 @@ func (peer *LocalPeer) addConnection(conn Connection) {
 	peer.Lock()
 	defer peer.Unlock()
 	peer.connections[conn.Remote().Name] = conn
-	peer.version++
+	peer.Version++
 }
 
 func (peer *LocalPeer) deleteConnection(conn Connection) {
 	peer.Lock()
 	defer peer.Unlock()
 	delete(peer.connections, conn.Remote().Name)
-	peer.version++
+	peer.Version++
 }
 
 func (peer *LocalPeer) connectionEstablished(conn Connection) {
 	peer.Lock()
 	defer peer.Unlock()
-	peer.version++
+	peer.Version++
 }
 
 func (peer *LocalPeer) connectionCount() int {
@@ -286,3 +248,10 @@ func (peer *LocalPeer) connectionCount() int {
 	defer peer.RUnlock()
 	return len(peer.connections)
 }
+
+func (peer *LocalPeer) setShortID(shortID PeerShortID) {
+	peer.Lock()
+	defer peer.Unlock()
+	peer.ShortID = shortID
+	peer.Version++
+}

router/mac_cache.go

@@ -28,43 +28,59 @@ func NewMacCache(maxAge time.Duration, onExpiry func(net.HardwareAddr, *Peer)) *
 	return cache
 }
 
-func (cache *MacCache) Enter(mac net.HardwareAddr, peer *Peer) bool {
+func (cache *MacCache) add(mac net.HardwareAddr, peer *Peer, force bool) (bool, *Peer) {
 	key := macint(mac)
 	now := time.Now()
+
 	cache.RLock()
 	entry, found := cache.table[key]
 	if found && entry.peer == peer && now.Before(entry.lastSeen.Add(cache.maxAge/10)) {
 		cache.RUnlock()
-		return false
+		return false, nil
 	}
 	cache.RUnlock()
+
 	cache.Lock()
 	defer cache.Unlock()
+
 	entry, found = cache.table[key]
 	if !found {
 		cache.table[key] = &MacCacheEntry{lastSeen: now, peer: peer}
-		return true
+		return true, nil
 	}
+
 	if entry.peer != peer {
-		entry.lastSeen = now
+		if !force {
+			return false, entry.peer
+		}
+
 		entry.peer = peer
-		return true
 	}
+
 	if now.After(entry.lastSeen.Add(cache.maxAge / 10)) {
 		entry.lastSeen = now
 	}
-	return false
+
+	return false, nil
 }
 
-func (cache *MacCache) Lookup(mac net.HardwareAddr) (*Peer, bool) {
+func (cache *MacCache) Add(mac net.HardwareAddr, peer *Peer) (bool, *Peer) {
+	return cache.add(mac, peer, false)
+}
+
+func (cache *MacCache) AddForced(mac net.HardwareAddr, peer *Peer) (bool, *Peer) {
+	return cache.add(mac, peer, true)
+}
+
+func (cache *MacCache) Lookup(mac net.HardwareAddr) *Peer {
 	key := macint(mac)
 	cache.RLock()
 	defer cache.RUnlock()
 	entry, found := cache.table[key]
 	if !found {
-		return nil, false
+		return nil
 	}
-	return entry.peer, true
+	return entry.peer
 }
 
 func (cache *MacCache) Delete(peer *Peer) bool {

router/mocks_test.go

@@ -14,7 +14,9 @@ import (
 
 // Add to peers a connection from peers.ourself to p
 func (peers *Peers) AddTestConnection(p *Peer) {
-	toPeer := NewPeer(p.Name, "", p.UID, 0)
+	summary := p.PeerSummary
+	summary.Version = 0
+	toPeer := NewPeerFromSummary(summary)
 	peers.FetchWithDefault(toPeer) // Has side-effect of incrementing refcount
 	conn := newMockConnection(peers.ourself.Peer, toPeer)
 	peers.ourself.addConnection(conn)
@@ -23,11 +25,9 @@ func (peers *Peers) AddTestConnection(p *Peer) {
 
 // Add to peers a connection from p1 to p2
 func (peers *Peers) AddTestRemoteConnection(p1, p2 *Peer) {
-	fromName := p1.Name
-	fromPeer := NewPeer(fromName, "", p1.UID, 0)
+	fromPeer := NewPeerFrom(p1)
 	fromPeer = peers.FetchWithDefault(fromPeer)
-	toName := p2.Name
-	toPeer := NewPeer(toName, "", p2.UID, 0)
+	toPeer := NewPeerFrom(p2)
 	toPeer = peers.FetchWithDefault(toPeer)
 	peers.ourself.addConnection(&RemoteConnection{fromPeer, toPeer, "", false, false})
 }
@@ -70,7 +70,7 @@ func checkEqualConns(t *testing.T, ourName PeerName, got, wanted map[PeerName]Co
 // Get all the peers from a Peers in a slice
 func (peers *Peers) allPeers() []*Peer {
 	var res []*Peer
-	for _, peer := range peers.table {
+	for _, peer := range peers.byName {
 		res = append(res, peer)
 	}
 	return res

router/odp.go

@@ -0,0 +1,93 @@
+package router
+
+import (
+	"fmt"
+	"net"
+	"syscall"
+
+	"github.com/weaveworks/go-odp/odp"
+)
+
+// ODP admin functionality
+
+func CreateDatapath(dpname string) (err error, supported bool) {
+	dpif, err := odp.NewDpif()
+	if err != nil {
+		if odp.IsKernelLacksODPError(err) {
+			return nil, false
+		}
+
+		return err, true
+	}
+
+	defer dpif.Close()
+
+	dp, err := dpif.CreateDatapath(dpname)
+	if err != nil && !odp.IsDatapathNameAlreadyExistsError(err) {
+		return err, true
+	}
+
+	// Pick an ephemeral port number to use in probing for vxlan
+	// support.
+	udpconn, err := net.ListenUDP("udp4", nil)
+	if err != nil {
+		return err, true
+	}
+
+	// we leave the UDP socket open, so creating a vxlan vport on
+	// the same port number should fail.  But that's fine: It's
+	// still sufficient to probe for support.
+	portno := uint16(udpconn.LocalAddr().(*net.UDPAddr).Port)
+	vpid, err := dp.CreateVport(odp.NewVxlanVportSpec(
+		fmt.Sprintf("vxlan-%d", portno), portno))
+	if nlerr, ok := err.(odp.NetlinkError); ok {
+		if syscall.Errno(nlerr) == syscall.EAFNOSUPPORT {
+			dp.Delete()
+			return fmt.Errorf("kernel does not have Open vSwitch VXLAN support"), false
+		}
+	}
+
+	if err == nil {
+		dp.DeleteVport(vpid)
+	}
+
+	udpconn.Close()
+	return nil, true
+}
+
+func DeleteDatapath(dpname string) error {
+	dpif, err := odp.NewDpif()
+	if err != nil {
+		return err
+	}
+
+	defer dpif.Close()
+
+	dp, err := dpif.LookupDatapath(dpname)
+	if err != nil {
+		if odp.IsNoSuchDatapathError(err) {
+			return nil
+		}
+
+		return err
+	}
+
+	return dp.Delete()
+}
+
+func AddDatapathInterface(dpname string, ifname string) error {
+	dpif, err := odp.NewDpif()
+	if err != nil {
+		return err
+	}
+
+	defer dpif.Close()
+
+	dp, err := dpif.LookupDatapath(dpname)
+	if err != nil {
+		return err
+	}
+
+	_, err = dp.CreateVport(odp.NewNetdevVportSpec(ifname))
+	return err
+}

router/overlay.go

@@ -0,0 +1,146 @@
+package router
+
+import (
+	"net"
+)
+
+// Interface to overlay network packet handling
+type Overlay interface {
+	// Start consuming forwarded packets.
+	StartConsumingPackets(*Peer, *Peers, OverlayConsumer) error
+
+	// Form a packet-forwarding connection.
+	MakeForwarder(ForwarderParams) (OverlayForwarder, error)
+
+	// The routes have changed, so any cached information should
+	// be discarded.
+	InvalidateRoutes()
+
+	// A mapping of a short id to a peer has changed
+	InvalidateShortIDs()
+
+	// Enhance a features map with overlay-related features
+	AddFeaturesTo(map[string]string)
+
+	// Obtain diagnostic information specific to the overlay
+	Diagnostics() interface{}
+}
+
+type ForwarderParams struct {
+	RemotePeer *Peer
+
+	// The local IP address to use for sending.  Derived from the
+	// local address of the corresponding TCP socket, so may
+	// differ for different forwarders.
+	LocalIP net.IP
+
+	// The remote address to send to.  nil if unknown, i.e. an
+	// incoming connection, in which case the Overlay needs to
+	// discover it (e.g. from incoming datagrams).
+	RemoteAddr *net.UDPAddr
+
+	// Unique identifier for this connection
+	ConnUID uint64
+
+	// Crypto bits.  Nil if not encrypting
+	Crypto *OverlayCrypto
+
+	// Function to send a control message to the counterpart
+	// forwarder.
+	SendControlMessage func(tag byte, msg []byte) error
+
+	// Features passed at connection initiation
+	Features map[string]string
+}
+
+// When a consumer is called, the decoder will already have been used
+// to decode the frame.
+type OverlayConsumer func(ForwardPacketKey) FlowOp
+
+// Crypto settings for a forwarder.
+type OverlayCrypto struct {
+	Dec   Decryptor
+	Enc   Encryptor
+	EncDF Encryptor
+}
+
+// All of the machinery to forward packets to a particular peer
+type OverlayForwarder interface {
+	// Forward a packet across the connection.  May be called as
+	// soon as the forwarder is created, in particular before
+	// Confirm().  The return value nil means the key could not be
+	// handled by this forwarder.
+	Forward(ForwardPacketKey) FlowOp
+
+	// Confirm that the connection is really wanted, and so the
+	// Overlay should begin heartbeats etc. to verify the
+	// operation of the forwarder.
+	Confirm()
+
+	// A channel indicating that the forwarder is established,
+	// i.e. its operation has been confirmed.
+	EstablishedChannel() <-chan struct{}
+
+	// A channel indicating an error from the forwarder.  The
+	// forwarder is not expected to be operational after the first
+	// error, so the channel only needs to buffer a single error.
+	ErrorChannel() <-chan error
+
+	Stop()
+
+	// Handle a message from the peer.  'tag' exists for
+	// compatibility, and should always be
+	// ProtocolOverlayControlMessage for non-sleeve overlays.
+	ControlMessage(tag byte, msg []byte)
+
+	// User facing overlay name
+	DisplayName() string
+}
+
+type NullOverlay struct{}
+
+func (NullOverlay) StartConsumingPackets(*Peer, *Peers, OverlayConsumer) error {
+	return nil
+}
+
+func (NullOverlay) MakeForwarder(ForwarderParams) (OverlayForwarder, error) {
+	return NullOverlay{}, nil
+}
+
+func (NullOverlay) InvalidateRoutes() {
+}
+
+func (NullOverlay) InvalidateShortIDs() {
+}
+
+func (NullOverlay) Confirm() {
+}
+
+func (NullOverlay) EstablishedChannel() <-chan struct{} {
+	return nil
+}
+
+func (NullOverlay) ErrorChannel() <-chan error {
+	return nil
+}
+
+func (NullOverlay) AddFeaturesTo(map[string]string) {
+}
+
+func (NullOverlay) Forward(ForwardPacketKey) FlowOp {
+	return DiscardingFlowOp{}
+}
+
+func (NullOverlay) Stop() {
+}
+
+func (NullOverlay) ControlMessage(byte, []byte) {
+}
+
+func (NullOverlay) DisplayName() string {
+	return "null"
+}
+
+func (NullOverlay) Diagnostics() interface{} {
+	return nil
+}

router/overlay_switch.go

@@ -0,0 +1,437 @@
+package router
+
+import (
+	"fmt"
+	"strings"
+	"sync"
+)
+
+// OverlaySwitch selects which overlay to use, from a set of
+// subsidiary overlays.  First, it passes a list of supported overlays
+// in the connection features, and uses that to determine which
+// overlays are in common.  Then it tries those common overlays, and
+// uses the best one that seems to be working.
+
+type OverlaySwitch struct {
+	overlays      map[string]Overlay
+	overlayNames  []string
+	compatOverlay Overlay
+}
+
+func NewOverlaySwitch() *OverlaySwitch {
+	return &OverlaySwitch{overlays: make(map[string]Overlay)}
+}
+
+func (osw *OverlaySwitch) Add(name string, overlay Overlay) {
+	// check for repeated names
+	if _, present := osw.overlays[name]; present {
+		log.Fatal("OverlaySwitch: repeated overlay name")
+	}
+
+	osw.overlays[name] = overlay
+	osw.overlayNames = append(osw.overlayNames, name)
+}
+
+func (osw *OverlaySwitch) SetCompatOverlay(overlay Overlay) {
+	osw.compatOverlay = overlay
+}
+
+func (osw *OverlaySwitch) AddFeaturesTo(features map[string]string) {
+	features["Overlays"] = strings.Join(osw.overlayNames, " ")
+}
+
+func (osw *OverlaySwitch) Diagnostics() interface{} {
+	diagnostics := make(map[string]interface{})
+	for name, overlay := range osw.overlays {
+		diagnostics[name] = overlay.Diagnostics()
+	}
+	return diagnostics
+}
+
+func (osw *OverlaySwitch) InvalidateRoutes() {
+	for _, overlay := range osw.overlays {
+		overlay.InvalidateRoutes()
+	}
+}
+
+func (osw *OverlaySwitch) InvalidateShortIDs() {
+	for _, overlay := range osw.overlays {
+		overlay.InvalidateShortIDs()
+	}
+}
+
+func (osw *OverlaySwitch) StartConsumingPackets(localPeer *Peer, peers *Peers,
+	consumer OverlayConsumer) error {
+	for _, overlay := range osw.overlays {
+		if err := overlay.StartConsumingPackets(localPeer, peers,
+			consumer); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+type namedOverlay struct {
+	Overlay
+	name string
+}
+
+// Find the common set of overlays supported by both sides, with the
+// ordering being the same on both sides too.
+func (osw *OverlaySwitch) commonOverlays(params ForwarderParams) ([]namedOverlay, error) {
+	var peerOverlays []string
+	if overlaysFeature, present := params.Features["Overlays"]; present {
+		peerOverlays = strings.Split(overlaysFeature, " ")
+	}
+
+	common := make(map[string]Overlay)
+	for _, name := range peerOverlays {
+		if overlay := osw.overlays[name]; overlay != nil {
+			common[name] = overlay
+		}
+	}
+
+	if len(common) == 0 {
+		return nil, fmt.Errorf("no overlays in common with peer")
+	}
+
+	// we order them according to the connecting node
+	ordering := osw.overlayNames
+	if params.RemoteAddr == nil {
+		// we are the connectee
+		ordering = peerOverlays
+	}
+
+	res := make([]namedOverlay, 0, len(common))
+	for _, name := range ordering {
+		overlay := common[name]
+		if overlay != nil {
+			res = append(res, namedOverlay{overlay, name})
+		}
+	}
+
+	// we use bytes to represent forwarder indices in control
+	// messages, so just in case:
+	if len(res) > 256 {
+		res = res[:256]
+	}
+
+	return res, nil
+}
+
+type overlaySwitchForwarder struct {
+	remotePeer *Peer
+
+	lock sync.Mutex
+
+	// the index of the forwarder to send on
+	best int
+
+	// the subsidiary forwarders
+	forwarders []subForwarder
+
+	// closed to tell the main goroutine to stop
+	stopChan chan<- struct{}
+
+	alreadyEstablished bool
+	establishedChan    chan struct{}
+	errorChan          chan error
+}
+
+// A subsidiary forwarder
+type subForwarder struct {
+	fwd         OverlayForwarder
+	overlayName string
+
+	// Has the forwarder signalled that it is established?
+	established bool
+
+	// closed to tell the forwarder monitor goroutine to stop
+	stopChan chan<- struct{}
+}
+
+// An event from a subsidiary forwarder
+type subForwarderEvent struct {
+	// the index of the forwarder
+	index int
+
+	// is this an "established" event?
+	established bool
+
+	// is this an error event?
+	err error
+}
+
+func (osw *OverlaySwitch) MakeForwarder(
+	params ForwarderParams) (OverlayForwarder, error) {
+	if _, present := params.Features["Overlays"]; !present && osw.compatOverlay != nil {
+		return osw.compatOverlay.MakeForwarder(params)
+	}
+
+	overlays, err := osw.commonOverlays(params)
+	if err != nil {
+		return nil, err
+	}
+
+	// channel to carry events from the subforwarder monitors to
+	// the main goroutine
+	eventsChan := make(chan subForwarderEvent)
+
+	// channel to stop the main goroutine
+	stopChan := make(chan struct{})
+
+	fwd := &overlaySwitchForwarder{
+		remotePeer: params.RemotePeer,
+
+		best:       -1,
+		forwarders: make([]subForwarder, len(overlays)),
+		stopChan:   stopChan,
+
+		establishedChan: make(chan struct{}),
+		errorChan:       make(chan error, 1),
+	}
+
+	origSendControlMessage := params.SendControlMessage
+	for i, overlay := range overlays {
+		// Prefix control messages to indicate the relevant forwarder
+		index := i
+		params.SendControlMessage = func(tag byte, msg []byte) error {
+			xmsg := make([]byte, len(msg)+2)
+			xmsg[0] = byte(index)
+			xmsg[1] = tag
+			copy(xmsg[2:], msg)
+			return origSendControlMessage(ProtocolOverlayControlMsg,
+				xmsg)
+		}
+
+		subFwd, err := overlay.MakeForwarder(params)
+		if err != nil {
+			fwd.stopFrom(0)
+			return nil, err
+		}
+
+		subStopChan := make(chan struct{})
+		go monitorForwarder(i, eventsChan, subStopChan, subFwd)
+		fwd.forwarders[i] = subForwarder{
+			fwd:         subFwd,
+			overlayName: overlay.name,
+			stopChan:    subStopChan,
+		}
+	}
+
+	fwd.chooseBest()
+	go fwd.run(eventsChan, stopChan)
+	return fwd, nil
+}
+
+func monitorForwarder(index int, eventsChan chan<- subForwarderEvent,
+	stopChan <-chan struct{}, fwd OverlayForwarder) {
+	establishedChan := fwd.EstablishedChannel()
+loop:
+	for {
+		e := subForwarderEvent{index: index}
+
+		select {
+		case <-establishedChan:
+			e.established = true
+			establishedChan = nil
+
+		case err := <-fwd.ErrorChannel():
+			e.err = err
+
+		case <-stopChan:
+			break loop
+		}
+
+		select {
+		case eventsChan <- e:
+		case <-stopChan:
+			break loop
+		}
+
+		if e.err != nil {
+			break loop
+		}
+	}
+
+	fwd.Stop()
+}
+
+func (fwd *overlaySwitchForwarder) run(eventsChan <-chan subForwarderEvent,
+	stopChan <-chan struct{}) {
+loop:
+	for {
+		select {
+		case <-stopChan:
+			break loop
+
+		case e := <-eventsChan:
+			switch {
+			case e.established:
+				fwd.established(e.index)
+			case e.err != nil:
+				fwd.error(e.index, e.err)
+			}
+		}
+	}
+
+	fwd.lock.Lock()
+	defer fwd.lock.Unlock()
+	fwd.stopFrom(0)
+}
+
+func (fwd *overlaySwitchForwarder) established(index int) {
+	fwd.lock.Lock()
+	defer fwd.lock.Unlock()
+
+	fwd.forwarders[index].established = true
+
+	if !fwd.alreadyEstablished {
+		fwd.alreadyEstablished = true
+		close(fwd.establishedChan)
+	}
+
+	fwd.chooseBest()
+}
+
+func (fwd *overlaySwitchForwarder) logPrefix() string {
+	return fmt.Sprintf("overlay_switch ->[%s] ", fwd.remotePeer)
+}
+
+func (fwd *overlaySwitchForwarder) error(index int, err error) {
+	fwd.lock.Lock()
+	defer fwd.lock.Unlock()
+
+	log.Info(fwd.logPrefix(), fwd.forwarders[index].overlayName, " ", err)
+	fwd.forwarders[index].fwd = nil
+	fwd.chooseBest()
+}
+
+func (fwd *overlaySwitchForwarder) stopFrom(index int) {
+	for index < len(fwd.forwarders) {
+		subFwd := &fwd.forwarders[index]
+		if subFwd.fwd != nil {
+			subFwd.fwd = nil
+			close(subFwd.stopChan)
+		}
+		index++
+	}
+}
+
+func (fwd *overlaySwitchForwarder) chooseBest() {
+	// the most preferred established forwarder is the best
+	// otherwise, the most preferred working forwarder is the best
+	bestEstablished := -1
+	bestWorking := -1
+
+	for i := range fwd.forwarders {
+		subFwd := &fwd.forwarders[i]
+		if subFwd.fwd == nil {
+			continue
+		}
+
+		if bestWorking < 0 {
+			bestWorking = i
+		}
+
+		if bestEstablished < 0 && subFwd.established {
+			bestEstablished = i
+		}
+	}
+
+	best := bestEstablished
+	if best < 0 {
+		if bestWorking < 0 {
+			select {
+			case fwd.errorChan <- fmt.Errorf("no working forwarders to %s", fwd.remotePeer):
+			default:
+			}
+
+			return
+		}
+
+		best = bestWorking
+	}
+
+	if fwd.best != best {
+		fwd.best = best
+		log.Info(fwd.logPrefix(),
+			"using ", fwd.forwarders[best].overlayName)
+	}
+}
+
+func (fwd *overlaySwitchForwarder) Confirm() {
+	var forwarders []OverlayForwarder
+
+	fwd.lock.Lock()
+	for _, subFwd := range fwd.forwarders {
+		if subFwd.fwd != nil {
+			forwarders = append(forwarders, subFwd.fwd)
+		}
+	}
+	fwd.lock.Unlock()
+
+	for _, subFwd := range forwarders {
+		subFwd.Confirm()
+	}
+}
+
+func (fwd *overlaySwitchForwarder) Forward(pk ForwardPacketKey) FlowOp {
+	fwd.lock.Lock()
+
+	if fwd.best >= 0 {
+		for i := fwd.best; i < len(fwd.forwarders); i++ {
+			best := fwd.forwarders[i].fwd
+			if best != nil {
+				fwd.lock.Unlock()
+				if op := best.Forward(pk); op != nil {
+					return op
+				}
+				fwd.lock.Lock()
+			}
+		}
+	}
+
+	fwd.lock.Unlock()
+
+	return DiscardingFlowOp{}
+}
+
+func (fwd *overlaySwitchForwarder) EstablishedChannel() <-chan struct{} {
+	return fwd.establishedChan
+}
+
+func (fwd *overlaySwitchForwarder) ErrorChannel() <-chan error {
+	return fwd.errorChan
+}
+
+func (fwd *overlaySwitchForwarder) Stop() {
+	fwd.lock.Lock()
+	defer fwd.lock.Unlock()
+	fwd.stopFrom(0)
+}
+
+func (fwd *overlaySwitchForwarder) ControlMessage(tag byte, msg []byte) {
+	fwd.lock.Lock()
+	subFwd := fwd.forwarders[msg[0]].fwd
+	fwd.lock.Unlock()
+	if subFwd != nil {
+		subFwd.ControlMessage(msg[1], msg[2:])
+	}
+}
+
+func (fwd *overlaySwitchForwarder) DisplayName() string {
+	var best OverlayForwarder
+
+	fwd.lock.Lock()
+	if fwd.best >= 0 {
+		best = fwd.forwarders[fwd.best].fwd
+	}
+	fwd.lock.Unlock()
+
+	if best != nil {
+		return best.DisplayName()
+	}
+
+	return "none"
+}

router/pcap.go

@@ -2,18 +2,44 @@ package router
 
 import (
 	"fmt"
+	"net"
+	"sync"
 
 	"github.com/google/gopacket/pcap"
 )
 
-type PcapIO struct {
-	handle *pcap.Handle
+type Pcap struct {
+	NonDiscardingFlowOp
+
+	iface *net.Interface
+	bufSz int
+
+	// The libpcap handle for writing packets. It's possible that a
+	// single handle could be used for reading and writing, but
+	// we'd have to examine the performance implications.
+	writeHandle *pcap.Handle
+
+	// pcap handles are single-threaded, so we need to lock around
+	// uses of writeHandle.
+	mutex sync.Mutex
+
+	// The libpcap handle for reading packets
+	readHandle *pcap.Handle
 }
 
-func NewPcapIO(ifName string, bufSz int) (PacketSourceSink, error) {
-	pio, err := newPcapIO(ifName, true, 65535, bufSz)
+func NewPcap(iface *net.Interface, bufSz int) (Bridge, error) {
+	wh, err := newPcapHandle(iface.Name, false, 0, 0)
 	if err != nil {
-		return pio, err
+		return nil, err
+	}
+
+	return &Pcap{iface: iface, bufSz: bufSz, writeHandle: wh}, nil
+}
+
+func (p *Pcap) StartConsumingPackets(consumer BridgeConsumer) error {
+	rh, err := newPcapHandle(p.iface.Name, true, 65535, p.bufSz)
+	if err != nil {
+		return err
 	}
 
 	// Under Linux, libpcap implements the SetDirection filtering
@@ -21,16 +47,25 @@ func NewPcapIO(ifName string, bufSz int) (PacketSourceSink, error) {
 	// packets inside the kernel.  We do this here rather than in
 	// newPcapIO because libpcap doesn't like this in combination
 	// with a 0 snaplen.
-	err = pio.handle.SetBPFFilter("inbound")
-	return pio, err
+	err = rh.SetBPFFilter("inbound")
+	if err != nil {
+		rh.Close()
+		return err
+	}
+
+	// readHandle is just for the benefit of Stats.
+	p.mutex.Lock()
+	defer p.mutex.Unlock()
+	if p.readHandle != nil {
+		panic("already consuming")
+	}
+
+	p.readHandle = rh
+	go p.sniff(rh, consumer)
+	return nil
 }
 
-func NewPcapO(ifName string) (po PacketSink, err error) {
-	po, err = newPcapIO(ifName, false, 0, 0)
-	return
-}
-
-func newPcapIO(ifName string, promisc bool, snaplen int, bufSz int) (handle *PcapIO, err error) {
+func newPcapHandle(ifName string, promisc bool, snaplen int, bufSz int) (handle *pcap.Handle, err error) {
 	inactive, err := pcap.NewInactiveHandle(ifName)
 	if err != nil {
 		return
@@ -61,38 +96,72 @@ func newPcapIO(ifName string, promisc bool, snaplen int, bufSz int) (handle *Pca
 	if err = inactive.SetBufferSize(bufSz); err != nil {
 		return
 	}
-	active, err := inactive.Activate()
+	handle, err = inactive.Activate()
 	if err != nil {
 		return
 	}
-	if err = active.SetDirection(pcap.DirectionIn); err != nil {
-		return
-	}
-	return &PcapIO{handle: active}, nil
-}
-
-func (pio *PcapIO) ReadPacket() (data []byte, err error) {
-	for {
-		data, _, err = pio.handle.ZeroCopyReadPacketData()
-		if err == nil || err != pcap.NextErrorTimeoutExpired {
-			break
-		}
-	}
+	err = handle.SetDirection(pcap.DirectionIn)
 	return
 }
 
-func (pio *PcapIO) WritePacket(data []byte) error {
-	return pio.handle.WritePacketData(data)
+func (p *Pcap) String() string {
+	return fmt.Sprint(p.iface.Name, " (via pcap)")
 }
 
-func (pio *PcapIO) Stats() map[string]int {
-	stats, err := pio.handle.Stats()
-	if err != nil {
-		return nil
+func (p *Pcap) InjectPacket(PacketKey) FlowOp {
+	return p
+}
+
+func (p *Pcap) Process(frame []byte, dec *EthernetDecoder, broadcast bool) {
+	p.mutex.Lock()
+	defer p.mutex.Unlock()
+	checkWarn(p.writeHandle.WritePacketData(frame))
+}
+
+func (p *Pcap) sniff(readHandle *pcap.Handle, consumer BridgeConsumer) {
+	dec := NewEthernetDecoder()
+
+	for {
+		pkt, _, err := readHandle.ZeroCopyReadPacketData()
+		if err == pcap.NextErrorTimeoutExpired {
+			continue
+		}
+
+		checkFatal(err)
+		dec.DecodeLayers(pkt)
+		if len(dec.decoded) == 0 {
+			continue
+		}
+
+		if fop := consumer(dec.PacketKey()); !fop.Discards() {
+			// We are handing over the frame to
+			// forwarders, so we need to make a copy of it
+			// in order to prevent the next capture from
+			// overwriting the data
+			pktLen := len(pkt)
+			pktCopy := make([]byte, pktLen, pktLen)
+			copy(pktCopy, pkt)
+
+			fop.Process(pktCopy, dec, false)
+		}
 	}
-	res := make(map[string]int)
-	res["PacketsReceived"] = stats.PacketsReceived
-	res["PacketsDropped"] = stats.PacketsDropped
-	res["PacketsIfDropped"] = stats.PacketsIfDropped
-	return res
+}
+
+func (p *Pcap) Stats() map[string]int {
+	p.mutex.Lock()
+	rh := p.readHandle
+	p.mutex.Unlock()
+
+	if rh != nil {
+		stats, err := rh.Stats()
+		if err == nil {
+			return map[string]int{
+				"PacketsReceived":  stats.PacketsReceived,
+				"PacketsDropped":   stats.PacketsDropped,
+				"PacketsIfDropped": stats.PacketsIfDropped,
+			}
+		}
+	}
+
+	return nil
 }

router/peer.go

@@ -8,34 +8,66 @@ import (
 
 type PeerUID uint64
 
+func randomPeerUID() PeerUID {
+	return PeerUID(randUint64())
+}
+
 func ParsePeerUID(s string) (PeerUID, error) {
 	uid, err := strconv.ParseUint(s, 10, 64)
 	return PeerUID(uid), err
 }
 
+// Short IDs exist for the sake of fast datapath.  They are 12 bits,
+// randomly assigned, but we detect and recover from collisions.  This
+// does limit us to 4096 peers, but that should be sufficient for a
+// while.
+type PeerShortID uint16
+
+const PeerShortIDBits = 12
+
+func randomPeerShortID() PeerShortID {
+	return PeerShortID(randUint16() & (1<<PeerShortIDBits - 1))
+}
+
+type PeerSummary struct {
+	NameByte   []byte
+	NickName   string
+	UID        PeerUID
+	Version    uint64
+	ShortID    PeerShortID
+	HasShortID bool
+}
+
 type Peer struct {
-	Name          PeerName
-	NameByte      []byte
-	NickName      string
-	UID           PeerUID
-	version       uint64
+	Name PeerName
+	PeerSummary
 	localRefCount uint64 // maintained by Peers
 	connections   map[PeerName]Connection
 }
 
 type ConnectionSet map[Connection]struct{}
 
-func NewPeer(name PeerName, nickName string, uid PeerUID, version uint64) *Peer {
-	if uid == 0 {
-		uid = PeerUID(randUint64())
-	}
+func NewPeerFromSummary(summary PeerSummary) *Peer {
 	return &Peer{
-		Name:        name,
-		NameByte:    name.Bin(),
-		NickName:    nickName,
-		UID:         uid,
-		version:     version,
-		connections: make(map[PeerName]Connection)}
+		Name:        PeerNameFromBin(summary.NameByte),
+		PeerSummary: summary,
+		connections: make(map[PeerName]Connection),
+	}
+}
+
+func NewPeer(name PeerName, nickName string, uid PeerUID, version uint64, shortID PeerShortID) *Peer {
+	return NewPeerFromSummary(PeerSummary{
+		NameByte:   name.Bin(),
+		NickName:   nickName,
+		UID:        uid,
+		Version:    version,
+		ShortID:    shortID,
+		HasShortID: true,
+	})
+}
+
+func NewPeerFrom(peer *Peer) *Peer {
+	return NewPeerFromSummary(peer.PeerSummary)
 }
 
 func (peer *Peer) String() string {
@@ -66,7 +98,7 @@ func (peer *Peer) String() string {
 // NB: This function should generally be invoked while holding a read
 // lock on Peers and LocalPeer.
 func (peer *Peer) Routes(stopAt *Peer, establishedAndSymmetric bool) (bool, map[PeerName]PeerName) {
-	routes := make(map[PeerName]PeerName)
+	routes := make(unicastRoutes)
 	routes[peer.Name] = UnknownPeerName
 	nextWorklist := []*Peer{peer}
 	for len(nextWorklist) > 0 {

router/peers.go

@@ -4,14 +4,29 @@ import (
 	"bytes"
 	"encoding/gob"
 	"io"
+	"math/rand"
 	"sync"
 )
 
 type Peers struct {
 	sync.RWMutex
-	ourself *LocalPeer
-	table   map[PeerName]*Peer
-	onGC    []func(*Peer)
+	ourself   *LocalPeer
+	byName    map[PeerName]*Peer
+	byShortID map[PeerShortID]ShortIDPeers
+	onGC      []func(*Peer)
+
+	// Called when the mapping from short ids to peers changes
+	onInvalidateShortIDs []func()
+}
+
+type ShortIDPeers struct {
+	// If we know about a single peer with the short id, this is
+	// that peer.  If there is a collision, this is the peer with
+	// the lowest Name.
+	peer *Peer
+
+	// In case of a collision, this holds the other peers.
+	others []*Peer
 }
 
 type UnknownPeerError struct {
@@ -24,13 +39,6 @@ type NameCollisionError struct {
 
 type PeerNameSet map[PeerName]struct{}
 
-type PeerSummary struct {
-	NameByte []byte
-	NickName string
-	UID      PeerUID
-	Version  uint64
-}
-
 type ConnectionSummary struct {
 	NameByte      []byte
 	RemoteTCPAddr string
@@ -38,32 +46,231 @@ type ConnectionSummary struct {
 	Established   bool
 }
 
+// Pending notifications due to changes to Peers that need to be sent
+// out once the Peers is unlocked.
+type PeersPendingNotifications struct {
+	// Peers that have been GCed
+	removed []*Peer
+
+	// The mapping from shorts ids to peers changed
+	invalidateShortIDs bool
+
+	// The local short ID needs reassigning due to a collision
+	reassignLocalShortID bool
+}
+
 func NewPeers(ourself *LocalPeer) *Peers {
-	return &Peers{ourself: ourself, table: make(map[PeerName]*Peer)}
+	peers := &Peers{
+		ourself:   ourself,
+		byName:    make(map[PeerName]*Peer),
+		byShortID: make(map[PeerShortID]ShortIDPeers),
+	}
+	peers.FetchWithDefault(ourself.Peer)
+	return peers
 }
 
 func (peers *Peers) OnGC(callback func(*Peer)) {
 	peers.Lock()
 	defer peers.Unlock()
+
+	// Although the array underlying peers.onGC might be accessed
+	// without holding the lock in unlockAndNotify, we don't
+	// support removing callbacks, so a simple append here is
+	// safe.
 	peers.onGC = append(peers.onGC, callback)
 }
 
-func (peers *Peers) invokeOnGCCallbacks(removed []*Peer) {
-	for _, callback := range peers.onGC {
-		for _, peer := range removed {
-			callback(peer)
+func (peers *Peers) OnInvalidateShortIDs(callback func()) {
+	peers.Lock()
+	defer peers.Unlock()
+
+	// Safe, as in OnGC
+	peers.onInvalidateShortIDs = append(peers.onInvalidateShortIDs, callback)
+}
+
+func (peers *Peers) unlockAndNotify(pending *PeersPendingNotifications) {
+	broadcastLocalPeer := pending.reassignLocalShortID && peers.reassignLocalShortID(pending)
+	onGC := peers.onGC
+	onInvalidateShortIDs := peers.onInvalidateShortIDs
+	peers.Unlock()
+
+	if pending.removed != nil {
+		for _, callback := range onGC {
+			for _, peer := range pending.removed {
+				callback(peer)
+			}
 		}
 	}
+
+	if pending.invalidateShortIDs {
+		for _, callback := range onInvalidateShortIDs {
+			callback()
+		}
+	}
+
+	if broadcastLocalPeer {
+		peers.ourself.broadcastPeerUpdate()
+	}
+}
+
+func (peers *Peers) addByShortID(peer *Peer, pending *PeersPendingNotifications) {
+	if !peer.HasShortID {
+		return
+	}
+
+	entry, ok := peers.byShortID[peer.ShortID]
+	if !ok {
+		entry = ShortIDPeers{peer: peer}
+	} else if entry.peer == nil {
+		// This short ID is free, but was used in the past.
+		// Because we are reusing it, it's an invalidation
+		// event.
+		entry.peer = peer
+		pending.invalidateShortIDs = true
+	} else if peer.Name < entry.peer.Name {
+		// Short ID collision, this peer becomes the principal
+		// peer for the short ID, bumping the previous one
+		// into others.
+
+		if entry.peer == peers.ourself.Peer {
+			// The bumped peer is peers.ourself, so we
+			// need to look for a new short id
+			pending.reassignLocalShortID = true
+		}
+
+		entry.others = append(entry.others, entry.peer)
+		entry.peer = peer
+		pending.invalidateShortIDs = true
+	} else {
+		// Short ID collision, this peer is secondary
+		entry.others = append(entry.others, peer)
+	}
+
+	peers.byShortID[peer.ShortID] = entry
+}
+
+func (peers *Peers) deleteByShortID(peer *Peer, pending *PeersPendingNotifications) {
+	if !peer.HasShortID {
+		return
+	}
+
+	entry := peers.byShortID[peer.ShortID]
+	var otherIndex int
+
+	if peer != entry.peer {
+		// peer is secondary, find its index in others
+		otherIndex = -1
+
+		for i, other := range entry.others {
+			if peer == other {
+				otherIndex = i
+				break
+			}
+		}
+
+		if otherIndex < 0 {
+			return
+		}
+	} else if len(entry.others) != 0 {
+		// need to find the peer with the lowest name to
+		// become the new principal one
+		otherIndex = 0
+		minName := entry.others[0].Name
+
+		for i := 1; i < len(entry.others); i++ {
+			otherName := entry.others[i].Name
+			if otherName < minName {
+				minName = otherName
+				otherIndex = i
+			}
+		}
+
+		entry.peer = entry.others[otherIndex]
+		pending.invalidateShortIDs = true
+	} else {
+		// This is the last peer with the short id.  We clear
+		// the entry, don't delete it, in order to detect when
+		// it gets re-used.
+		peers.byShortID[peer.ShortID] = ShortIDPeers{}
+		return
+	}
+
+	entry.others[otherIndex] = entry.others[len(entry.others)-1]
+	entry.others = entry.others[:len(entry.others)-1]
+	peers.byShortID[peer.ShortID] = entry
+}
+
+func (peers *Peers) reassignLocalShortID(pending *PeersPendingNotifications) bool {
+	newShortID, ok := peers.chooseShortID()
+	if ok {
+		peers.setLocalShortID(newShortID, pending)
+		return true
+	}
+
+	// Otherwise we'll try again later on in garbageColleect
+	return false
+}
+
+func (peers *Peers) setLocalShortID(newShortID PeerShortID, pending *PeersPendingNotifications) {
+	peers.deleteByShortID(peers.ourself.Peer, pending)
+	peers.ourself.setShortID(newShortID)
+	peers.addByShortID(peers.ourself.Peer, pending)
+}
+
+// Choose an available short id at random
+func (peers *Peers) chooseShortID() (PeerShortID, bool) {
+	rng := rand.New(rand.NewSource(int64(randUint64())))
+
+	// First, just try picking some short ids at random, and
+	// seeing if they are available:
+	for i := 0; i < 10; i++ {
+		shortID := PeerShortID(rng.Intn(1 << PeerShortIDBits))
+		if peers.byShortID[shortID].peer == nil {
+			return shortID, true
+		}
+	}
+
+	// Looks like most short ids are used.  So count the number of
+	// unused ones, and pick one at random.
+	available := int(1 << PeerShortIDBits)
+	for _, entry := range peers.byShortID {
+		if entry.peer != nil {
+			available--
+		}
+	}
+
+	if available == 0 {
+		// All short ids are used.
+		return 0, false
+	}
+
+	n := rng.Intn(available)
+	var i PeerShortID
+	for {
+		if peers.byShortID[i].peer == nil {
+			if n == 0 {
+				return PeerShortID(i), true
+			}
+
+			n--
+		}
+
+		i++
+	}
 }
 
 func (peers *Peers) FetchWithDefault(peer *Peer) *Peer {
 	peers.Lock()
-	defer peers.Unlock()
-	if existingPeer, found := peers.table[peer.Name]; found {
+	var pending PeersPendingNotifications
+	defer peers.unlockAndNotify(&pending)
+
+	if existingPeer, found := peers.byName[peer.Name]; found {
 		existingPeer.localRefCount++
 		return existingPeer
 	}
-	peers.table[peer.Name] = peer
+
+	peers.byName[peer.Name] = peer
+	peers.addByShortID(peer, &pending)
 	peer.localRefCount++
 	return peer
 }
@@ -71,19 +278,25 @@ func (peers *Peers) FetchWithDefault(peer *Peer) *Peer {
 func (peers *Peers) Fetch(name PeerName) *Peer {
 	peers.RLock()
 	defer peers.RUnlock()
-	return peers.table[name]
+	return peers.byName[name]
 }
 
 func (peers *Peers) FetchAndAddRef(name PeerName) *Peer {
 	peers.Lock()
 	defer peers.Unlock()
-	peer := peers.table[name]
+	peer := peers.byName[name]
 	if peer != nil {
 		peer.localRefCount++
 	}
 	return peer
 }
 
+func (peers *Peers) FetchByShortID(shortID PeerShortID) *Peer {
+	peers.RLock()
+	defer peers.RUnlock()
+	return peers.byShortID[shortID].peer
+}
+
 func (peers *Peers) Dereference(peer *Peer) {
 	peers.Lock()
 	defer peers.Unlock()
@@ -93,7 +306,7 @@ func (peers *Peers) Dereference(peer *Peer) {
 func (peers *Peers) ForEach(fun func(*Peer)) {
 	peers.RLock()
 	defer peers.RUnlock()
-	for _, peer := range peers.table {
+	for _, peer := range peers.byName {
 		fun(peer)
 	}
 }
@@ -106,42 +319,46 @@ func (peers *Peers) ForEach(fun func(*Peer)) {
 // "improved" update containing just these new/updated elements.
 func (peers *Peers) ApplyUpdate(update []byte) (PeerNameSet, PeerNameSet, error) {
 	peers.Lock()
+	var pending PeersPendingNotifications
+	defer peers.unlockAndNotify(&pending)
 
 	newPeers, decodedUpdate, decodedConns, err := peers.decodeUpdate(update)
 	if err != nil {
-		peers.Unlock()
 		return nil, nil, err
 	}
 
 	// By this point, we know the update doesn't refer to any peers we
 	// have no knowledge of. We can now apply the update. Start by
-	// adding in any new peers into the cache.
+	// adding in any new peers.
 	for name, newPeer := range newPeers {
-		peers.table[name] = newPeer
+		peers.byName[name] = newPeer
+		peers.addByShortID(newPeer, &pending)
 	}
 
 	// Now apply the updates
-	newUpdate := peers.applyUpdate(decodedUpdate, decodedConns)
-	removed := peers.garbageCollect()
-	for _, peerRemoved := range removed {
+	newUpdate := peers.applyUpdate(decodedUpdate, decodedConns, &pending)
+	peers.garbageCollect(&pending)
+	for _, peerRemoved := range pending.removed {
 		delete(newUpdate, peerRemoved.Name)
 	}
 
-	// Don't need to hold peers lock any longer
-	peers.Unlock()
-	peers.invokeOnGCCallbacks(removed)
-
 	updateNames := make(PeerNameSet)
 	for _, peer := range decodedUpdate {
 		updateNames[peer.Name] = void
 	}
-	return updateNames, setFromPeersMap(newUpdate), nil
+
+	return updateNames, newUpdate, nil
 }
 
 func (peers *Peers) Names() PeerNameSet {
 	peers.RLock()
 	defer peers.RUnlock()
-	return setFromPeersMap(peers.table)
+
+	names := make(PeerNameSet)
+	for name := range peers.byName {
+		names[name] = void
+	}
+	return names
 }
 
 func (peers *Peers) EncodePeers(names PeerNameSet) []byte {
@@ -150,7 +367,7 @@ func (peers *Peers) EncodePeers(names PeerNameSet) []byte {
 	peers.RLock()
 	defer peers.RUnlock()
 	for name := range names {
-		if peer, found := peers.table[name]; found {
+		if peer, found := peers.byName[name]; found {
 			if peer == peers.ourself.Peer {
 				peers.ourself.Encode(enc)
 			} else {
@@ -161,34 +378,33 @@ func (peers *Peers) EncodePeers(names PeerNameSet) []byte {
 	return buf.Bytes()
 }
 
-func (peers *Peers) GarbageCollect() []*Peer {
+func (peers *Peers) GarbageCollect() {
 	peers.Lock()
-	removed := peers.garbageCollect()
-	peers.Unlock()
-	peers.invokeOnGCCallbacks(removed)
-	return removed
+	var pending PeersPendingNotifications
+	defer peers.unlockAndNotify(&pending)
+
+	peers.garbageCollect(&pending)
 }
 
-func (peers *Peers) garbageCollect() []*Peer {
-	removed := []*Peer{}
+func (peers *Peers) garbageCollect(pending *PeersPendingNotifications) {
 	peers.ourself.RLock()
 	_, reached := peers.ourself.Routes(nil, false)
 	peers.ourself.RUnlock()
-	for name, peer := range peers.table {
+
+	for name, peer := range peers.byName {
 		if _, found := reached[peer.Name]; !found && peer.localRefCount == 0 {
-			delete(peers.table, name)
-			removed = append(removed, peer)
+			delete(peers.byName, name)
+			peers.deleteByShortID(peer, pending)
+			pending.removed = append(pending.removed, peer)
 		}
 	}
-	return removed
-}
 
-func setFromPeersMap(peers map[PeerName]*Peer) PeerNameSet {
-	names := make(PeerNameSet)
-	for name := range peers {
-		names[name] = void
+	if peers.byShortID[peers.ourself.ShortID].peer != peers.ourself.Peer {
+		// The local peer doesn't own its short id.  Garbage
+		// collection might have freed some up, so try to
+		// reassign.
+		pending.reassignLocalShortID = true
 	}
-	return names
 }
 
 func (peers *Peers) decodeUpdate(update []byte) (newPeers map[PeerName]*Peer, decodedUpdate []*Peer, decodedConns [][]ConnectionSummary, err error) {
@@ -206,13 +422,12 @@ func (peers *Peers) decodeUpdate(update []byte) (newPeers map[PeerName]*Peer, de
 			err = decErr
 			return
 		}
-		name := PeerNameFromBin(peerSummary.NameByte)
-		newPeer := NewPeer(name, peerSummary.NickName, peerSummary.UID, peerSummary.Version)
+		newPeer := NewPeerFromSummary(peerSummary)
 		decodedUpdate = append(decodedUpdate, newPeer)
 		decodedConns = append(decodedConns, connSummaries)
-		existingPeer, found := peers.table[name]
+		existingPeer, found := peers.byName[newPeer.Name]
 		if !found {
-			newPeers[name] = newPeer
+			newPeers[newPeer.Name] = newPeer
 		} else if existingPeer.UID != newPeer.UID {
 			err = NameCollisionError{Name: newPeer.Name}
 			return
@@ -225,7 +440,7 @@ func (peers *Peers) decodeUpdate(update []byte) (newPeers map[PeerName]*Peer, de
 			if _, found := newPeers[remoteName]; found {
 				continue
 			}
-			if _, found := peers.table[remoteName]; found {
+			if _, found := peers.byName[remoteName]; found {
 				continue
 			}
 			// Update refers to a peer which we have no knowledge
@@ -237,15 +452,15 @@ func (peers *Peers) decodeUpdate(update []byte) (newPeers map[PeerName]*Peer, de
 	return
 }
 
-func (peers *Peers) applyUpdate(decodedUpdate []*Peer, decodedConns [][]ConnectionSummary) map[PeerName]*Peer {
-	newUpdate := make(map[PeerName]*Peer)
+func (peers *Peers) applyUpdate(decodedUpdate []*Peer, decodedConns [][]ConnectionSummary, pending *PeersPendingNotifications) PeerNameSet {
+	newUpdate := make(PeerNameSet)
 	for idx, newPeer := range decodedUpdate {
 		connSummaries := decodedConns[idx]
 		name := newPeer.Name
-		// guaranteed to find peer in the peers.table
-		peer := peers.table[name]
+		// guaranteed to find peer in the peers.byName
+		peer := peers.byName[name]
 		if peer != newPeer &&
-			(peer == peers.ourself.Peer || peer.version >= newPeer.version) {
+			(peer == peers.ourself.Peer || peer.Version >= newPeer.Version) {
 			// Nobody but us updates us. And if we know more about a
 			// peer than what's in the the update, we ignore the
 			// latter.
@@ -261,19 +476,23 @@ func (peers *Peers) applyUpdate(decodedUpdate []*Peer, decodedConns [][]Connecti
 		// for an update would be someone else calling
 		// router.Peers.ApplyUpdate. But ApplyUpdate takes the Lock on
 		// the router.Peers, so there can be no race here.
-		peer.version = newPeer.version
-		peer.connections = makeConnsMap(peer, connSummaries, peers.table)
-		newUpdate[name] = peer
+		peer.Version = newPeer.Version
+		peer.connections = makeConnsMap(peer, connSummaries, peers.byName)
+
+		if newPeer.ShortID != peer.ShortID {
+			peers.deleteByShortID(peer, pending)
+			peer.ShortID = newPeer.ShortID
+			peers.addByShortID(peer, pending)
+		}
+
+		newUpdate[name] = void
 	}
+
 	return newUpdate
 }
 
 func (peer *Peer) Encode(enc *gob.Encoder) {
-	checkFatal(enc.Encode(PeerSummary{
-		peer.NameByte,
-		peer.NickName,
-		peer.UID,
-		peer.version}))
+	checkFatal(enc.Encode(peer.PeerSummary))
 
 	connSummaries := []ConnectionSummary{}
 	for _, conn := range peer.connections {
@@ -304,11 +523,11 @@ func decodePeer(dec *gob.Decoder) (peerSummary PeerSummary, connSummaries []Conn
 	return
 }
 
-func makeConnsMap(peer *Peer, connSummaries []ConnectionSummary, table map[PeerName]*Peer) map[PeerName]Connection {
+func makeConnsMap(peer *Peer, connSummaries []ConnectionSummary, byName map[PeerName]*Peer) map[PeerName]Connection {
 	conns := make(map[PeerName]Connection)
 	for _, connSummary := range connSummaries {
 		name := PeerNameFromBin(connSummary.NameByte)
-		remotePeer := table[name]
+		remotePeer := byName[name]
 		conn := NewRemoteConnection(peer, remotePeer, connSummary.RemoteTCPAddr, connSummary.Outbound, connSummary.Established)
 		conns[name] = conn
 	}

router/peers_test.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"math/rand"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/require"
 )
@@ -20,7 +21,6 @@ import (
 func newNode(name PeerName) (*Peer, *Peers) {
 	peer := NewLocalPeer(name, "", nil)
 	peers := NewPeers(peer)
-	peers.FetchWithDefault(peer.Peer)
 	return peer.Peer, peers
 }
 
@@ -72,6 +72,13 @@ func TestPeersEncoding(t *testing.T) {
 	}
 }
 
+func garbageCollect(peers *Peers) []*Peer {
+	var removed []*Peer
+	peers.OnGC(func(peer *Peer) { removed = append(removed, peer) })
+	peers.GarbageCollect()
+	return removed
+}
+
 func TestPeersGarbageCollection(t *testing.T) {
 	const (
 		peer1NameString = "01:00:00:01:00:00"
@@ -98,17 +105,272 @@ func TestPeersGarbageCollection(t *testing.T) {
 	ps2.AddTestRemoteConnection(p3, p1)
 
 	// Every peer is referenced, so nothing should be dropped
-	require.Empty(t, ps1.GarbageCollect(), "peers removed")
-	require.Empty(t, ps2.GarbageCollect(), "peers removed")
-	require.Empty(t, ps3.GarbageCollect(), "peers removed")
+	require.Empty(t, garbageCollect(ps1), "peers removed")
+	require.Empty(t, garbageCollect(ps2), "peers removed")
+	require.Empty(t, garbageCollect(ps3), "peers removed")
 
 	// Drop the connection from 2 to 3, and 3 isn't garbage-collected
 	// because 1 has a connection to 3
 	ps2.DeleteTestConnection(p3)
-	require.Empty(t, ps2.GarbageCollect(), "peers removed")
+	require.Empty(t, garbageCollect(ps2), "peers removed")
 
 	// Drop the connection from 1 to 3, and 3 will get removed by
 	// garbage-collection
 	ps1.DeleteTestConnection(p3)
-	checkPeerArray(t, ps1.GarbageCollect(), p3)
+	checkPeerArray(t, garbageCollect(ps1), p3)
+}
+
+func TestShortIDCollisions(t *testing.T) {
+	rng := rand.New(rand.NewSource(time.Now().UnixNano()))
+	_, peers := newNode(PeerName(1 << PeerShortIDBits))
+
+	// Make enough peers that short id collisions are
+	// overwhelmingly likely
+	ps := make([]*Peer, 1<<PeerShortIDBits)
+	for i := 0; i < 1<<PeerShortIDBits; i++ {
+		ps[i] = NewPeer(PeerName(i), "", PeerUID(i), 0,
+			PeerShortID(rng.Intn(1<<PeerShortIDBits)))
+	}
+
+	shuffle := func() {
+		for i := range ps {
+			j := rng.Intn(i + 1)
+			ps[i], ps[j] = ps[j], ps[i]
+		}
+	}
+
+	// Fill peers
+	shuffle()
+	var pending PeersPendingNotifications
+	for _, p := range ps {
+		peers.addByShortID(p, &pending)
+	}
+
+	// Check invariants
+	counts := make([]int, 1<<PeerShortIDBits)
+	saw := func(p *Peer) {
+		if p != peers.ourself.Peer {
+			counts[p.UID]++
+		}
+	}
+
+	for shortID, entry := range peers.byShortID {
+		if entry.peer == nil {
+			// no principal peer for this short id, so
+			// others must be empty
+			require.Empty(t, entry.others)
+			continue
+		}
+
+		require.Equal(t, shortID, entry.peer.ShortID)
+		saw(entry.peer)
+
+		for _, p := range entry.others {
+			saw(p)
+			require.Equal(t, shortID, p.ShortID)
+
+			// the principal peer should have the lowest name
+			require.True(t, p.Name > entry.peer.Name)
+		}
+	}
+
+	// Check that every peer was seen
+	for _, n := range counts {
+		require.Equal(t, 1, n)
+	}
+
+	// Delete all the peers
+	shuffle()
+	for _, p := range ps {
+		peers.deleteByShortID(p, &pending)
+	}
+
+	for _, entry := range peers.byShortID {
+		if entry.peer != peers.ourself.Peer {
+			require.Nil(t, entry.peer)
+		}
+
+		require.Empty(t, entry.others)
+	}
+}
+
+// Test the easy case of short id reassignment, when few short ids are taken
+func TestShortIDReassignmentEasy(t *testing.T) {
+	rng := rand.New(rand.NewSource(time.Now().UnixNano()))
+	_, peers := newNode(PeerName(0))
+
+	for i := 1; i <= 10; i++ {
+		peers.FetchWithDefault(NewPeer(PeerName(i), "", PeerUID(i), 0,
+			PeerShortID(rng.Intn(1<<PeerShortIDBits))))
+	}
+
+	checkShortIDReassignment(t, peers)
+}
+
+// Test the hard case of short id reassignment, when most short ids are taken
+func TestShortIDReassignmentHard(t *testing.T) {
+	rng := rand.New(rand.NewSource(time.Now().UnixNano()))
+	_, peers := newNode(PeerName(1 << PeerShortIDBits))
+
+	// Take all short ids
+	ps := make([]*Peer, 1<<PeerShortIDBits)
+	var pending PeersPendingNotifications
+	for i := 0; i < 1<<PeerShortIDBits; i++ {
+		ps[i] = NewPeer(PeerName(i), "", PeerUID(i), 0,
+			PeerShortID(i))
+		peers.addByShortID(ps[i], &pending)
+	}
+
+	// As all short ids are taken, an attempted reassigment won't
+	// do anything
+	oldShortID := peers.ourself.ShortID
+	require.False(t, peers.reassignLocalShortID(&pending))
+	require.Equal(t, oldShortID, peers.ourself.ShortID)
+
+	// Free up a few ids
+	for i := 0; i < 10; i++ {
+		x := rng.Intn(len(ps))
+		if ps[x] != nil {
+			peers.deleteByShortID(ps[x], &pending)
+			ps[x] = nil
+		}
+	}
+
+	checkShortIDReassignment(t, peers)
+}
+
+func checkShortIDReassignment(t *testing.T, peers *Peers) {
+	oldShortID := peers.ourself.ShortID
+	peers.reassignLocalShortID(&PeersPendingNotifications{})
+	require.NotEqual(t, oldShortID, peers.ourself.ShortID)
+	require.Equal(t, peers.ourself.Peer, peers.byShortID[peers.ourself.ShortID].peer)
+}
+
+func TestShortIDInvalidation(t *testing.T) {
+	_, peers := newNode(PeerName(1 << PeerShortIDBits))
+
+	// need to use a short id that is not the local peer's
+	shortID := peers.ourself.ShortID + 1
+
+	var pending PeersPendingNotifications
+
+	requireInvalidateShortIDs := func(expect bool) {
+		require.Equal(t, expect, pending.invalidateShortIDs)
+		pending.invalidateShortIDs = false
+	}
+
+	// The use of a fresh short id does not cause invalidation
+	a := NewPeer(PeerName(1), "", PeerUID(1), 0, shortID)
+	peers.addByShortID(a, &pending)
+	requireInvalidateShortIDs(false)
+
+	// An addition which does not change the mapping
+	b := NewPeer(PeerName(2), "", PeerUID(2), 0, shortID)
+	peers.addByShortID(b, &pending)
+	requireInvalidateShortIDs(false)
+
+	// An addition which does change the mapping
+	c := NewPeer(PeerName(0), "", PeerUID(0), 0, shortID)
+	peers.addByShortID(c, &pending)
+	requireInvalidateShortIDs(true)
+
+	// A deletion which does not change the mapping
+	peers.deleteByShortID(b, &pending)
+	requireInvalidateShortIDs(false)
+
+	// A deletion which does change the mapping
+	peers.deleteByShortID(c, &pending)
+	requireInvalidateShortIDs(true)
+
+	// Deleting the last peer with a short id does not cause invalidation
+	peers.deleteByShortID(a, &pending)
+	requireInvalidateShortIDs(false)
+
+	// .. but subsequent reuse of that short id does cause invalidation
+	peers.addByShortID(a, &pending)
+	requireInvalidateShortIDs(true)
+}
+
+func TestShortIDPropagation(t *testing.T) {
+	_, peers1 := newNode(PeerName(1))
+	_, peers2 := newNode(PeerName(2))
+
+	peers1.AddTestConnection(peers2.ourself.Peer)
+	peers1.ApplyUpdate(peers2.EncodePeers(peers2.Names()))
+	peers12 := peers1.Fetch(PeerName(2))
+	old := peers12.PeerSummary
+
+	require.True(t,
+		peers2.reassignLocalShortID(&PeersPendingNotifications{}))
+	peers1.ApplyUpdate(peers2.EncodePeers(peers2.Names()))
+	require.NotEqual(t, old.Version, peers12.Version)
+	require.NotEqual(t, old.ShortID, peers12.ShortID)
+}
+
+func TestShortIDCollision(t *testing.T) {
+	// Create 3 peers
+	_, peers1 := newNode(PeerName(1))
+	_, peers2 := newNode(PeerName(2))
+	_, peers3 := newNode(PeerName(3))
+
+	var pending PeersPendingNotifications
+	peers1.setLocalShortID(1, &pending)
+	peers2.setLocalShortID(2, &pending)
+	peers3.setLocalShortID(3, &pending)
+
+	peers2.AddTestConnection(peers1.ourself.Peer)
+	peers3.AddTestConnection(peers2.ourself.Peer)
+
+	// Propogate from 1 to 2 to 3
+	peers2.ApplyUpdate(peers1.EncodePeers(peers1.Names()))
+	peers3.ApplyUpdate(peers2.EncodePeers(peers2.Names()))
+
+	// Force the short id of peer 1 to collide with peer 2.  Peer
+	// 1 has the lowest name, so it gets to keep the short id
+	peers1.setLocalShortID(2, &pending)
+
+	oldShortID := peers2.ourself.ShortID
+	_, updated, _ := peers2.ApplyUpdate(peers1.EncodePeers(peers1.Names()))
+
+	// peer 2 should have noticed the collision and resolved it
+	require.NotEqual(t, oldShortID, peers2.ourself.ShortID)
+
+	// The Peers do not have a Router, so broadcastPeerUpdate does
+	// nothing in the context of this test.  So we fake what it
+	// would do.
+	updated[PeerName(2)] = void
+
+	// the update from peer 2 should include its short id change
+	peers3.ApplyUpdate(peers2.EncodePeers(updated))
+	require.Equal(t, peers2.ourself.ShortID,
+		peers3.Fetch(PeerName(2)).ShortID)
+}
+
+// Test the case where all short ids are taken, but then some peers go
+// away, so the local peer reassigns
+func TestDeferredShortIDReassignment(t *testing.T) {
+	rng := rand.New(rand.NewSource(time.Now().UnixNano()))
+	_, us := newNode(PeerName(1 << PeerShortIDBits))
+
+	// Connect us to other peers occupying all short ids
+	others := make([]*Peers, 1<<PeerShortIDBits)
+	var pending PeersPendingNotifications
+	for i := range others {
+		_, others[i] = newNode(PeerName(i))
+		others[i].setLocalShortID(PeerShortID(i), &pending)
+		us.AddTestConnection(others[i].ourself.Peer)
+	}
+
+	// Check that, as expected, the local peer does not own its
+	// short id
+	require.NotEqual(t, us.ourself.Peer,
+		us.byShortID[us.ourself.ShortID].peer)
+
+	// Disconnect one peer, and we should now be able to claim its
+	// short id
+	other := others[rng.Intn(1<<PeerShortIDBits)]
+	us.DeleteTestConnection(other.ourself.Peer)
+	us.GarbageCollect()
+
+	require.Equal(t, us.ourself.Peer, us.byShortID[us.ourself.ShortID].peer)
 }

router/protocol.go

@@ -328,13 +328,14 @@ func (res *ProtocolIntroResults) setupCrypto(params ProtocolIntroParams, remoteP
 type ProtocolTag byte
 
 const (
-	ProtocolHeartbeat ProtocolTag = iota
+	ProtocolHeartbeat = iota
 	ProtocolConnectionEstablished
 	ProtocolFragmentationReceived
 	ProtocolPMTUVerified
 	ProtocolGossip
 	ProtocolGossipUnicast
 	ProtocolGossipBroadcast
+	ProtocolOverlayControlMsg
 )
 
 type ProtocolMsg struct {

router/router.go

@@ -1,13 +1,9 @@
 package router
 
 import (
-	"bytes"
-	"encoding/binary"
 	"fmt"
-	"io"
 	"net"
 	"sync"
-	"syscall"
 	"time"
 )
 
@@ -24,17 +20,21 @@ const (
 
 // [3] control rate at which new tokens are added to the bucket
 
-type LogFrameFunc func(string, []byte, *EthernetDecoder)
-
 type Config struct {
 	Port               int
 	ProtocolMinVersion byte
-	Iface              *net.Interface
 	Password           []byte
 	ConnLimit          int
 	PeerDiscovery      bool
 	BufSz              int
-	LogFrame           LogFrameFunc
+	PacketLogging      PacketLogging
+	Bridge             Bridge
+	Overlay            Overlay
+}
+
+type PacketLogging interface {
+	LogPacket(string, PacketKey)
+	LogForwardPacket(string, ForwardPacketKey)
 }
 
 type Router struct {
@@ -47,40 +47,34 @@ type Router struct {
 	gossipLock      sync.RWMutex
 	gossipChannels  GossipChannels
 	TopologyGossip  Gossip
-	PacketSource    PacketSource
 	UDPListener     *net.UDPConn
 	acceptLimiter   *TokenBucket
 }
 
-type PacketSource interface {
-	ReadPacket() ([]byte, error)
-	Stats() map[string]int
-}
-
-type PacketSink interface {
-	WritePacket([]byte) error
-}
-
-type PacketSourceSink interface {
-	PacketSource
-	PacketSink
-}
-
 func NewRouter(config Config, name PeerName, nickName string) *Router {
 	router := &Router{Config: config, gossipChannels: make(GossipChannels)}
-	onMacExpiry := func(mac net.HardwareAddr, peer *Peer) {
-		log.Println("Expired MAC", mac, "at", peer)
+
+	if router.Bridge == nil {
+		router.Bridge = NullBridge{}
 	}
-	onPeerGC := func(peer *Peer) {
+
+	if router.Overlay == nil {
+		router.Overlay = NullOverlay{}
+	}
+
+	router.Ourself = NewLocalPeer(name, nickName, router)
+	router.Macs = NewMacCache(macMaxAge,
+		func(mac net.HardwareAddr, peer *Peer) {
+			log.Println("Expired MAC", mac, "at", peer)
+		})
+	router.Peers = NewPeers(router.Ourself)
+	router.Peers.OnGC(func(peer *Peer) {
 		router.Macs.Delete(peer)
 		log.Println("Removed unreachable peer", peer)
-	}
-	router.Ourself = NewLocalPeer(name, nickName, router)
-	router.Macs = NewMacCache(macMaxAge, onMacExpiry)
-	router.Peers = NewPeers(router.Ourself)
-	router.Peers.OnGC(onPeerGC)
-	router.Peers.FetchWithDefault(router.Ourself.Peer)
-	router.Routes = NewRoutes(router.Ourself, router.Peers)
+	})
+	router.Peers.OnInvalidateShortIDs(router.Overlay.InvalidateShortIDs)
+
+	router.Routes = NewRoutes(router.Ourself, router.Peers, router.Overlay.InvalidateRoutes)
 	router.ConnectionMaker = NewConnectionMaker(router.Ourself, router.Peers, router.Port, router.PeerDiscovery)
 	router.TopologyGossip = router.NewGossip("topology", router)
 	router.acceptLimiter = NewTokenBucket(acceptMaxTokens, acceptTokenDelay)
@@ -88,25 +82,13 @@ func NewRouter(config Config, name PeerName, nickName string) *Router {
 }
 
 // Start listening for TCP connections, locally captured packets, and
-// packets forwarded over UDP.  This is separate from NewRouter so
+// forwarded packets.  This is separate from NewRouter so
 // that gossipers can register before we start forming connections.
 func (router *Router) Start() {
-	// we need two pcap handles since they aren't thread-safe
-	var pio PacketSourceSink
-	var po PacketSink
-	var err error
-	if router.Iface != nil {
-		pio, err = NewPcapIO(router.Iface.Name, router.BufSz)
-		checkFatal(err)
-		po, err = NewPcapO(router.Iface.Name)
-		checkFatal(err)
-	}
-	router.UDPListener = router.listenUDP(router.Port, po)
+	log.Println("Sniffing traffic on", router.Bridge)
+	checkFatal(router.Bridge.StartConsumingPackets(router.handleCapturedPacket))
+	checkFatal(router.Overlay.StartConsumingPackets(router.Ourself.Peer, router.Peers, router.handleForwardedPacket))
 	router.listenTCP(router.Port)
-	if pio != nil {
-		router.PacketSource = pio
-		router.sniff(pio)
-	}
 }
 
 func (router *Router) Stop() error {
@@ -118,70 +100,49 @@ func (router *Router) UsingPassword() bool {
 	return router.Password != nil
 }
 
-func (router *Router) sniff(pio PacketSourceSink) {
-	log.Println("Sniffing traffic on", router.Iface)
+func (router *Router) handleCapturedPacket(key PacketKey) FlowOp {
+	router.PacketLogging.LogPacket("Captured", key)
+	srcMac := net.HardwareAddr(key.SrcMAC[:])
 
-	dec := NewEthernetDecoder()
-	mac := router.Iface.HardwareAddr
-	if router.Macs.Enter(mac, router.Ourself.Peer) {
-		log.Println("Discovered our MAC", mac)
-	}
-	go func() {
-		for {
-			pkt, err := pio.ReadPacket()
-			checkFatal(err)
-			router.LogFrame("Sniffed", pkt, nil)
-			router.handleCapturedPacket(pkt, dec, pio)
-		}
-	}()
-}
-
-func (router *Router) handleCapturedPacket(frameData []byte, dec *EthernetDecoder, po PacketSink) {
-	dec.DecodeLayers(frameData)
-	decodedLen := len(dec.decoded)
-	if decodedLen == 0 {
-		return
-	}
-	srcMac := dec.Eth.SrcMAC
-	srcPeer, found := router.Macs.Lookup(srcMac)
-	// We need to filter out frames we injected ourselves. For such
-	// frames, the srcMAC will have been recorded as associated with a
-	// different peer.
-	if found && srcPeer != router.Ourself.Peer {
-		return
-	}
-	if router.Macs.Enter(srcMac, router.Ourself.Peer) {
+	switch newSrcMac, conflictPeer := router.Macs.Add(srcMac, router.Ourself.Peer); {
+	case newSrcMac:
 		log.Println("Discovered local MAC", srcMac)
-	}
-	if dec.DropFrame() {
-		return
-	}
-	dstMac := dec.Eth.DstMAC
-	dstPeer, found := router.Macs.Lookup(dstMac)
-	if found && dstPeer == router.Ourself.Peer {
-		return
-	}
-	router.LogFrame("Forwarding", frameData, dec)
 
-	// at this point we are handing over the frame to forwarders, so
-	// we need to make a copy of it in order to prevent the next
-	// capture from overwriting the data
-	frameLen := len(frameData)
-	frameCopy := make([]byte, frameLen, frameLen)
-	copy(frameCopy, frameData)
-
-	// If we don't know which peer corresponds to the dest MAC,
-	// broadcast it.
-	if !found {
-		router.Ourself.Broadcast(frameCopy, dec)
-		return
+	case conflictPeer != nil:
+		// The MAC cache has an entry for the source MAC
+		// associated with another peer.  This probably means
+		// we are seeing a frame we injected ourself.  That
+		// shouldn't happen, but discard it just in case.
+		log.Error("Captured frame from MAC (", srcMac, ") associated with another peer ", conflictPeer)
+		return DiscardingFlowOp{}
 	}
 
-	err := router.Ourself.Forward(dstPeer, frameCopy, dec)
-	if ftbe, ok := err.(FrameTooBigError); ok {
-		err = dec.sendICMPFragNeeded(ftbe.EPMTU, po.WritePacket)
+	// Discard STP broadcasts
+	if key.DstMAC == [...]byte{0x01, 0x80, 0xC2, 0x00, 0x00, 0x00} {
+		return DiscardingFlowOp{}
+	}
+
+	dstMac := net.HardwareAddr(key.DstMAC[:])
+	switch dstPeer := router.Macs.Lookup(dstMac); dstPeer {
+	case router.Ourself.Peer:
+		// The packet is destined for a local MAC.  The bridge
+		// won't normally send us such packets, and if it does
+		// it's likely to be broadcasting the packet to all
+		// ports.  So if it happens, just drop the packet to
+		// avoid warnings if we try to forward it.
+		return DiscardingFlowOp{}
+	case nil:
+		// If we don't know which peer corresponds to the dest
+		// MAC, broadcast it.
+		router.PacketLogging.LogPacket("Broadcasting", key)
+		return router.relayBroadcast(router.Ourself.Peer, key)
+	default:
+		router.PacketLogging.LogPacket("Forwarding", key)
+		return router.relay(ForwardPacketKey{
+			PacketKey: key,
+			SrcPeer:   router.Ourself.Peer,
+			DstPeer:   dstPeer})
 	}
-	checkWarn(err)
 }
 
 func (router *Router) listenTCP(localPort int) {
@@ -213,141 +174,95 @@ func (router *Router) acceptTCP(tcpConn *net.TCPConn) {
 	StartLocalConnection(connRemote, tcpConn, nil, router, true)
 }
 
-func (router *Router) listenUDP(localPort int, po PacketSink) *net.UDPConn {
-	localAddr, err := net.ResolveUDPAddr("udp4", fmt.Sprint(":", localPort))
-	checkFatal(err)
-	conn, err := net.ListenUDP("udp4", localAddr)
-	checkFatal(err)
-	f, err := conn.File()
-	defer f.Close()
-	checkFatal(err)
-	fd := int(f.Fd())
-	// This one makes sure all packets we send out do not have DF set on them.
-	err = syscall.SetsockoptInt(fd, syscall.IPPROTO_IP, syscall.IP_MTU_DISCOVER, syscall.IP_PMTUDISC_DONT)
-	checkFatal(err)
-	go router.udpReader(conn, po)
-	return conn
-}
-
-func (router *Router) udpReader(conn *net.UDPConn, po PacketSink) {
-	defer conn.Close()
-	dec := NewEthernetDecoder()
-	buf := make([]byte, MaxUDPPacketSize)
-	for {
-		n, sender, err := conn.ReadFromUDP(buf)
-		if err == io.EOF {
-			return
-		} else if err != nil {
-			log.Warnln("ignoring UDP read error", err)
-			continue
-		} else if n < NameSize {
-			log.Warnln("ignoring too short UDP packet from", sender)
-			continue
-		}
-		name := PeerNameFromBin(buf[:NameSize])
-		packet := make([]byte, n-NameSize)
-		copy(packet, buf[NameSize:n])
-		peerConn, found := router.Ourself.ConnectionTo(name)
-		if !found {
-			continue
-		}
-		relayConn, ok := peerConn.(*LocalConnection)
-		if !ok {
-			continue
-		}
-		if err := relayConn.Decryptor.IterateFrames(packet, router.handleUDPPacketFunc(relayConn, dec, sender, po)); err != nil {
-			// Errors during UDP packet decoding / processing are
-			// non-fatal. One common cause is that we receive and
-			// attempt to decrypt a "stray" packet. This can actually
-			// happen quite easily if there is some connection churn
-			// between two peers. After all, UDP isn't a
-			// connection-oriented protocol, yet we pretend it is.
-			//
-			// If anything really is seriously, unrecoverably amiss
-			// with a connection, that will typically result in missed
-			// heartbeats and the connection getting shut down because
-			// of that.
-			relayConn.Log(err)
-		}
+func (router *Router) handleForwardedPacket(key ForwardPacketKey) FlowOp {
+	if key.DstPeer != router.Ourself.Peer {
+		// it's not for us, we're just relaying it
+		router.PacketLogging.LogForwardPacket("Relaying", key)
+		return router.relay(key)
 	}
-}
 
-func (router *Router) handleUDPPacketFunc(relayConn *LocalConnection, dec *EthernetDecoder, sender *net.UDPAddr, po PacketSink) FrameConsumer {
-	return func(srcNameByte, dstNameByte []byte, frame []byte) {
-		srcPeer := router.Peers.Fetch(PeerNameFromBin(srcNameByte))
-		dstPeer := router.Peers.Fetch(PeerNameFromBin(dstNameByte))
-		if srcPeer == nil || dstPeer == nil {
-			return
-		}
+	// At this point, it's either unicast to us, or a broadcast
+	// (because the DstPeer on a forwarded broadcast packet is
+	// always set to the peer being forwarded to)
 
-		dec.DecodeLayers(frame)
-		decodedLen := len(dec.decoded)
-		if decodedLen == 0 {
-			return
-		}
-		// Handle special frames produced internally (rather than
-		// captured/forwarded) by the remote router.
-		//
-		// We really shouldn't be decoding these above, since they are
-		// not genuine Ethernet frames. However, it is actually more
-		// efficient to do so, as we want to optimise for the common
-		// (i.e. non-special) frames. These always need decoding, and
-		// detecting special frames is cheaper post decoding than pre.
-		if decodedLen == 1 && dec.IsSpecial() {
-			if srcPeer == relayConn.Remote() && dstPeer == router.Ourself.Peer {
-				handleSpecialFrame(relayConn, sender, frame)
-			}
-			return
-		}
+	srcMac := net.HardwareAddr(key.SrcMAC[:])
+	dstMac := net.HardwareAddr(key.DstMAC[:])
 
-		if dstPeer != router.Ourself.Peer {
-			// it's not for us, we're just relaying it
-			router.LogFrame("Relaying", frame, dec)
-			err := router.Ourself.Relay(srcPeer, dstPeer, frame, dec)
-			if ftbe, ok := err.(FrameTooBigError); ok {
-				err = dec.sendICMPFragNeeded(ftbe.EPMTU, func(icmpFrame []byte) error {
-					return router.Ourself.Forward(srcPeer, icmpFrame, nil)
-				})
-			}
+	switch newSrcMac, conflictPeer := router.Macs.AddForced(srcMac, key.SrcPeer); {
+	case newSrcMac:
+		log.Print("Discovered remote MAC ", srcMac, " at ", key.SrcPeer)
 
-			checkWarn(err)
-			return
-		}
+	case conflictPeer != nil:
+		log.Print("Discovered remote MAC ", srcMac, " at ", key.SrcPeer, " (was at ", conflictPeer, ")")
 
-		srcMac := dec.Eth.SrcMAC
-		dstMac := dec.Eth.DstMAC
-
-		if router.Macs.Enter(srcMac, srcPeer) {
-			log.Println("Discovered remote MAC", srcMac, "at", srcPeer)
-		}
-		if po != nil {
-			router.LogFrame("Injecting", frame, dec)
-			checkWarn(po.WritePacket(frame))
-		}
-
-		dstPeer, found := router.Macs.Lookup(dstMac)
-		if !found || dstPeer != router.Ourself.Peer {
-			router.LogFrame("Relaying broadcast", frame, dec)
-			router.Ourself.RelayBroadcast(srcPeer, frame, dec)
-		}
+		// We need to clear out any flows destined to the MAC
+		// that forward to the old peer.
+		router.Overlay.InvalidateRoutes()
 	}
-}
 
-func handleSpecialFrame(relayConn *LocalConnection, sender *net.UDPAddr, frame []byte) {
-	frameLen := len(frame)
+	router.PacketLogging.LogForwardPacket("Injecting", key)
+	injectFop := router.Bridge.InjectPacket(key.PacketKey)
+	dstPeer := router.Macs.Lookup(dstMac)
+	if dstPeer == router.Ourself.Peer {
+		return injectFop
+	}
+
+	router.PacketLogging.LogForwardPacket("Relaying broadcast", key)
+	relayFop := router.relayBroadcast(key.SrcPeer, key.PacketKey)
 	switch {
-	case frameLen == EthernetOverhead+8:
-		relayConn.ReceivedHeartbeat(sender, binary.BigEndian.Uint64(frame[EthernetOverhead:]))
-	case frameLen == FragTestSize && bytes.Equal(frame, FragTest):
-		relayConn.SendProtocolMsg(ProtocolMsg{ProtocolFragmentationReceived, nil})
-	case frameLen == PMTUDiscoverySize && bytes.Equal(frame, PMTUDiscovery):
+	case injectFop == nil:
+		return relayFop
+
+	case relayFop == nil:
+		return injectFop
+
 	default:
-		frameLenBytes := []byte{0, 0}
-		binary.BigEndian.PutUint16(frameLenBytes, uint16(frameLen-EthernetOverhead))
-		relayConn.SendProtocolMsg(ProtocolMsg{ProtocolPMTUVerified, frameLenBytes})
+		mfop := NewMultiFlowOp(false)
+		mfop.Add(injectFop)
+		mfop.Add(relayFop)
+		return mfop
 	}
 }
 
+// Routing
+
+func (router *Router) relay(key ForwardPacketKey) FlowOp {
+	relayPeerName, found := router.Routes.Unicast(key.DstPeer.Name)
+	if !found {
+		// Not necessarily an error as there could be a race with the
+		// dst disappearing whilst the frame is in flight
+		log.Println("Received packet for unknown destination:", key.DstPeer)
+		return DiscardingFlowOp{}
+	}
+
+	conn, found := router.Ourself.ConnectionTo(relayPeerName)
+	if !found {
+		// Again, could just be a race, not necessarily an error
+		log.Println("Unable to find connection to relay peer", relayPeerName)
+		return DiscardingFlowOp{}
+	}
+
+	return conn.(*LocalConnection).Forward(key)
+}
+
+func (router *Router) relayBroadcast(srcPeer *Peer, key PacketKey) FlowOp {
+	nextHops := router.Routes.Broadcast(srcPeer.Name)
+	if len(nextHops) == 0 {
+		return DiscardingFlowOp{}
+	}
+
+	op := NewMultiFlowOp(true)
+
+	for _, conn := range router.Ourself.ConnectionsTo(nextHops) {
+		op.Add(conn.(*LocalConnection).Forward(ForwardPacketKey{
+			PacketKey: key,
+			SrcPeer:   srcPeer,
+			DstPeer:   conn.Remote()}))
+	}
+
+	return op
+}
+
 // Gossiper methods - the Router is the topology Gossiper
 
 type TopologyGossipData struct {
@@ -355,12 +270,16 @@ type TopologyGossipData struct {
 	update PeerNameSet
 }
 
-func NewTopologyGossipData(peers *Peers, update ...*Peer) *TopologyGossipData {
+func (router *Router) BroadcastTopologyUpdate(update []*Peer) {
 	names := make(PeerNameSet)
 	for _, p := range update {
 		names[p.Name] = void
 	}
-	return &TopologyGossipData{peers: peers, update: names}
+
+	router.TopologyGossip.GossipBroadcast(&TopologyGossipData{
+		peers:  router.Peers,
+		update: names,
+	})
 }
 
 func (d *TopologyGossipData) Merge(other GossipData) {

router/routes.go

@@ -5,30 +5,35 @@ import (
 	"sync"
 )
 
+type unicastRoutes map[PeerName]PeerName
+type broadcastRoutes map[PeerName][]PeerName
+
 type Routes struct {
 	sync.RWMutex
 	ourself      *LocalPeer
 	peers        *Peers
-	unicast      map[PeerName]PeerName
-	unicastAll   map[PeerName]PeerName // [1]
-	broadcast    map[PeerName][]PeerName
-	broadcastAll map[PeerName][]PeerName // [1]
+	onChange     func()
+	unicast      unicastRoutes
+	unicastAll   unicastRoutes // [1]
+	broadcast    broadcastRoutes
+	broadcastAll broadcastRoutes // [1]
 	recalculate  chan<- *struct{}
 	wait         chan<- chan struct{}
 	// [1] based on *all* connections, not just established &
 	// symmetric ones
 }
 
-func NewRoutes(ourself *LocalPeer, peers *Peers) *Routes {
+func NewRoutes(ourself *LocalPeer, peers *Peers, onChange func()) *Routes {
 	recalculate := make(chan *struct{}, 1)
 	wait := make(chan chan struct{})
 	routes := &Routes{
 		ourself:      ourself,
 		peers:        peers,
-		unicast:      make(map[PeerName]PeerName),
-		unicastAll:   make(map[PeerName]PeerName),
-		broadcast:    make(map[PeerName][]PeerName),
-		broadcastAll: make(map[PeerName][]PeerName),
+		onChange:     onChange,
+		unicast:      make(unicastRoutes),
+		unicastAll:   make(unicastRoutes),
+		broadcast:    make(broadcastRoutes),
+		broadcastAll: make(broadcastRoutes),
 		recalculate:  recalculate,
 		wait:         wait}
 	routes.unicast[ourself.Name] = UnknownPeerName
@@ -60,21 +65,13 @@ func (routes *Routes) UnicastAll(name PeerName) (PeerName, bool) {
 func (routes *Routes) Broadcast(name PeerName) []PeerName {
 	routes.RLock()
 	defer routes.RUnlock()
-	hops, found := routes.broadcast[name]
-	if !found {
-		return []PeerName{}
-	}
-	return hops
+	return routes.broadcast[name]
 }
 
 func (routes *Routes) BroadcastAll(name PeerName) []PeerName {
 	routes.RLock()
 	defer routes.RUnlock()
-	hops, found := routes.broadcastAll[name]
-	if !found {
-		return []PeerName{}
-	}
-	return hops
+	return routes.broadcastAll[name]
 }
 
 // Choose min(log2(n_peers), n_neighbouring_peers) neighbours, with a
@@ -147,6 +144,8 @@ func (routes *Routes) calculate() {
 	routes.peers.RLock()
 	routes.ourself.RLock()
 	var (
+		oldUnicast   = routes.unicast
+		oldBroadcast = routes.broadcast
 		unicast      = routes.calculateUnicast(true)
 		unicastAll   = routes.calculateUnicast(false)
 		broadcast    = routes.calculateBroadcast(true)
@@ -161,6 +160,10 @@ func (routes *Routes) calculate() {
 	routes.broadcast = broadcast
 	routes.broadcastAll = broadcastAll
 	routes.Unlock()
+
+	if !unicast.equals(oldUnicast) || !broadcast.equals(oldBroadcast) {
+		routes.onChange()
+	}
 }
 
 // Calculate all the routes for the question: if *we* want to send a
@@ -172,7 +175,7 @@ func (routes *Routes) calculate() {
 // any knowledge of the MAC address at all. Thus there's no need
 // to exchange knowledge of MAC addresses, nor any constraints on
 // the routes that we construct.
-func (routes *Routes) calculateUnicast(establishedAndSymmetric bool) map[PeerName]PeerName {
+func (routes *Routes) calculateUnicast(establishedAndSymmetric bool) unicastRoutes {
 	_, unicast := routes.ourself.Routes(nil, establishedAndSymmetric)
 	return unicast
 }
@@ -195,9 +198,9 @@ func (routes *Routes) calculateUnicast(establishedAndSymmetric bool) map[PeerNam
 //     Y =/= Z /\ X.Routes(Y) <= X.Routes(Z) =>
 //     X.Routes(Y) u [P | Y.HasSymmetricConnectionTo(P)] <= X.Routes(Z)
 // where <= is the subset relationship on keys of the returned map.
-func (routes *Routes) calculateBroadcast(establishedAndSymmetric bool) map[PeerName][]PeerName {
-	broadcast := make(map[PeerName][]PeerName)
-	for _, peer := range routes.peers.table {
+func (routes *Routes) calculateBroadcast(establishedAndSymmetric bool) broadcastRoutes {
+	broadcast := make(broadcastRoutes)
+	for _, peer := range routes.peers.byName {
 		hops := []PeerName{}
 		if found, reached := peer.Routes(routes.ourself.Peer, establishedAndSymmetric); found {
 			routes.ourself.ForEachConnectedPeer(establishedAndSymmetric, reached,
@@ -207,3 +210,55 @@ func (routes *Routes) calculateBroadcast(establishedAndSymmetric bool) map[PeerN
 	}
 	return broadcast
 }
+
+func (a unicastRoutes) equals(b unicastRoutes) bool {
+	for key, aval := range a {
+		if bval, ok := b[key]; !ok || bval != aval {
+			return false
+		}
+	}
+
+	for key := range b {
+		if _, ok := a[key]; !ok {
+			return false
+		}
+	}
+
+	return true
+}
+
+func (a broadcastRoutes) equals(b broadcastRoutes) bool {
+	set := make(map[PeerName]struct{})
+
+	for key, aval := range a {
+		bval, ok := b[key]
+		if !ok {
+			return false
+		}
+
+		for _, peer := range aval {
+			set[peer] = struct{}{}
+		}
+
+		for _, peer := range bval {
+			if _, ok := set[peer]; !ok {
+				return false
+			}
+
+			delete(set, peer)
+		}
+
+		if len(set) != 0 {
+			return false
+		}
+
+	}
+
+	for key := range b {
+		if _, ok := a[key]; !ok {
+			return false
+		}
+	}
+
+	return true
+}

router/status.go

@@ -1,6 +1,7 @@
 package router
 
 import (
+	"fmt"
 	"time"
 )
 
@@ -21,6 +22,7 @@ type Status struct {
 	BroadcastRoutes    []BroadcastRouteStatus
 	Connections        []LocalConnectionStatus
 	Targets            []string
+	OverlayDiagnostics interface{}
 }
 
 type MACStatus struct {
@@ -34,6 +36,7 @@ type PeerStatus struct {
 	Name        string
 	NickName    string
 	UID         PeerUID
+	ShortID     PeerShortID
 	Version     uint64
 	Connections []ConnectionStatus
 }
@@ -63,14 +66,6 @@ type LocalConnectionStatus struct {
 }
 
 func NewStatus(router *Router) *Status {
-	var ifaceName string
-	if router.Iface != nil {
-		ifaceName = router.Iface.Name
-	}
-	var captureStats map[string]int
-	if router.PacketSource != nil {
-		captureStats = router.PacketSource.Stats()
-	}
 	return &Status{
 		Protocol,
 		ProtocolMinVersion,
@@ -80,14 +75,15 @@ func NewStatus(router *Router) *Status {
 		router.Ourself.Name.String(),
 		router.Ourself.NickName,
 		router.Port,
-		ifaceName,
-		captureStats,
+		router.Bridge.String(),
+		router.Bridge.Stats(),
 		NewMACStatusSlice(router.Macs),
 		NewPeerStatusSlice(router.Peers),
 		NewUnicastRouteStatusSlice(router.Routes),
 		NewBroadcastRouteStatusSlice(router.Routes),
 		NewLocalConnectionStatusSlice(router.ConnectionMaker),
-		NewTargetSlice(router.ConnectionMaker)}
+		NewTargetSlice(router.ConnectionMaker),
+		router.Overlay.Diagnostics()}
 }
 
 func NewMACStatusSlice(cache *MacCache) []MACStatus {
@@ -127,7 +123,8 @@ func NewPeerStatusSlice(peers *Peers) []PeerStatus {
 			peer.Name.String(),
 			peer.NickName,
 			peer.UID,
-			peer.version,
+			peer.ShortID,
+			peer.Version,
 			connections})
 	})
 
@@ -170,40 +167,41 @@ func NewBroadcastRouteStatusSlice(routes *Routes) []BroadcastRouteStatus {
 }
 
 func NewLocalConnectionStatusSlice(cm *ConnectionMaker) []LocalConnectionStatus {
-	// We need to Refresh first in order to clear out any 'attempting'
-	// connections from cm.targets that have been established since
-	// the last run of cm.checkStateAndAttemptConnections. These
-	// entries are harmless but do represent stale state that we do
-	// not want to report.
-	cm.Refresh()
 	resultChan := make(chan []LocalConnectionStatus, 0)
 	cm.actionChan <- func() bool {
 		var slice []LocalConnectionStatus
-		for conn := range cm.ourself.Connections() {
+		for conn := range cm.connections {
 			state := "pending"
 			if conn.Established() {
 				state = "established"
 			}
-			slice = append(slice, LocalConnectionStatus{conn.RemoteTCPAddr(), conn.Outbound(), state, conn.Remote().String()})
+			lc, _ := conn.(*LocalConnection)
+			info := fmt.Sprintf("%-6v %v", lc.forwarder.DisplayName(), conn.Remote())
+			slice = append(slice, LocalConnectionStatus{conn.RemoteTCPAddr(), conn.Outbound(), state, info})
 		}
 		for address, target := range cm.targets {
-			var state, info string
-			switch {
-			case target.lastError == nil:
-				state = "connecting"
-				info = ""
-			case target.attempting:
-				state = "retrying"
-				info = target.lastError.Error()
-			default:
-				state = "failed"
-				retry := "never"
+			add := func(state, info string) {
+				slice = append(slice, LocalConnectionStatus{address, true, state, info})
+			}
+			switch target.state {
+			case TargetWaiting:
+				until := "never"
 				if !target.tryAfter.IsZero() {
-					retry = target.tryAfter.String()
+					until = target.tryAfter.String()
 				}
-				info = target.lastError.Error() + ", retry: " + retry
+				if target.lastError == nil { // shouldn't happen
+					add("waiting", "until: "+until)
+				} else {
+					add("failed", target.lastError.Error()+", retry: "+until)
+				}
+			case TargetAttempting:
+				if target.lastError == nil {
+					add("connecting", "")
+				} else {
+					add("retrying", target.lastError.Error())
+				}
+			case TargetConnected:
 			}
-			slice = append(slice, LocalConnectionStatus{address, true, state, info})
 		}
 		resultChan <- slice
 		return false

router/udp_sender.go

@@ -1,138 +0,0 @@
-package router
-
-import (
-	"net"
-	"syscall"
-
-	"github.com/google/gopacket"
-	"github.com/google/gopacket/layers"
-)
-
-type UDPSender interface {
-	Send([]byte) error
-	Shutdown() error
-}
-
-type SimpleUDPSender struct {
-	conn    *LocalConnection
-	udpConn *net.UDPConn
-}
-
-type RawUDPSender struct {
-	ipBuf     gopacket.SerializeBuffer
-	opts      gopacket.SerializeOptions
-	udpHeader *layers.UDP
-	socket    *net.IPConn
-	conn      *LocalConnection
-}
-
-type MsgTooBigError struct {
-	PMTU int // actual pmtu, i.e. what the kernel told us
-}
-
-func NewSimpleUDPSender(conn *LocalConnection) *SimpleUDPSender {
-	return &SimpleUDPSender{udpConn: conn.Router.UDPListener, conn: conn}
-}
-
-func (sender *SimpleUDPSender) Send(msg []byte) error {
-	_, err := sender.udpConn.WriteToUDP(msg, sender.conn.RemoteUDPAddr())
-	return err
-}
-
-func (sender *SimpleUDPSender) Shutdown() error {
-	return nil
-}
-
-func NewRawUDPSender(conn *LocalConnection) (*RawUDPSender, error) {
-	ipSocket, err := dialIP(conn)
-	if err != nil {
-		return nil, err
-	}
-	udpHeader := &layers.UDP{SrcPort: layers.UDPPort(conn.Router.Port)}
-	ipBuf := gopacket.NewSerializeBuffer()
-	opts := gopacket.SerializeOptions{
-		FixLengths: true,
-		// UDP header is calculated with a phantom IP
-		// header. Yes, it's totally nuts. Thankfully, for UDP
-		// over IPv4, the checksum is optional. It's not
-		// optional for IPv6, but we'll ignore that for
-		// now. TODO
-		ComputeChecksums: false}
-
-	return &RawUDPSender{
-		ipBuf:     ipBuf,
-		opts:      opts,
-		udpHeader: udpHeader,
-		socket:    ipSocket,
-		conn:      conn}, nil
-}
-
-func (sender *RawUDPSender) Send(msg []byte) error {
-	payload := gopacket.Payload(msg)
-	sender.udpHeader.DstPort = layers.UDPPort(sender.conn.RemoteUDPAddr().Port)
-
-	err := gopacket.SerializeLayers(sender.ipBuf, sender.opts, sender.udpHeader, &payload)
-	if err != nil {
-		return err
-	}
-	packet := sender.ipBuf.Bytes()
-	_, err = sender.socket.Write(packet)
-	if err == nil || PosixError(err) != syscall.EMSGSIZE {
-		return err
-	}
-	f, err := sender.socket.File()
-	if err != nil {
-		return err
-	}
-	defer f.Close()
-	fd := int(f.Fd())
-	log.Println("EMSGSIZE on send, expecting PMTU update (IP packet was",
-		len(packet), "bytes, payload was", len(msg), "bytes)")
-	pmtu, err := syscall.GetsockoptInt(fd, syscall.IPPROTO_IP, syscall.IP_MTU)
-	if err != nil {
-		return err
-	}
-	return MsgTooBigError{PMTU: pmtu}
-}
-
-func (sender *RawUDPSender) Shutdown() error {
-	defer func() { sender.socket = nil }()
-	return sender.socket.Close()
-}
-
-func dialIP(conn *LocalConnection) (*net.IPConn, error) {
-	ipLocalAddr, err := ipAddr(conn.TCPConn.LocalAddr())
-	if err != nil {
-		return nil, err
-	}
-	ipRemoteAddr, err := ipAddr(conn.TCPConn.RemoteAddr())
-	if err != nil {
-		return nil, err
-	}
-	ipSocket, err := net.DialIP("ip4:UDP", ipLocalAddr, ipRemoteAddr)
-	if err != nil {
-		return nil, err
-	}
-	f, err := ipSocket.File()
-	if err != nil {
-		return nil, err
-	}
-	defer f.Close()
-	fd := int(f.Fd())
-	// This Makes sure all packets we send out have DF set on them.
-	err = syscall.SetsockoptInt(fd, syscall.IPPROTO_IP, syscall.IP_MTU_DISCOVER, syscall.IP_PMTUDISC_DO)
-	if err != nil {
-		return nil, err
-	}
-	return ipSocket, nil
-}
-
-func ipAddr(addr net.Addr) (*net.IPAddr, error) {
-	host, _, err := net.SplitHostPort(addr.String())
-	if err != nil {
-		return nil, err
-	}
-	return &net.IPAddr{
-		IP:   net.ParseIP(host),
-		Zone: ""}, nil
-}

router/utils.go

@@ -3,9 +3,11 @@ package router
 import (
 	"bytes"
 	"crypto/rand"
+	"encoding/binary"
 	"encoding/gob"
 	"fmt"
 	"net"
+	"os"
 
 	"github.com/weaveworks/weave/common"
 )
@@ -26,23 +28,19 @@ func checkWarn(e error) {
 	}
 }
 
+// Look inside an error produced by the net package to get to the
+// syscall.Errno at the root of the problem.
 func PosixError(err error) error {
-	if err == nil {
-		return nil
+	if operr, ok := err.(*net.OpError); ok {
+		err = operr.Err
 	}
-	operr, ok := err.(*net.OpError)
-	if !ok {
-		return nil
+
+	// go1.5 wraps an Errno inside a SyscallError inside an OpError
+	if scerr, ok := err.(*os.SyscallError); ok {
+		err = scerr.Err
 	}
-	return operr.Err
-}
 
-func (mtbe MsgTooBigError) Error() string {
-	return fmt.Sprint("Msg too big error. PMTU is ", mtbe.PMTU)
-}
-
-func (ftbe FrameTooBigError) Error() string {
-	return fmt.Sprint("Frame too big error. Effective PMTU is ", ftbe.EPMTU)
+	return err
 }
 
 func (upe UnknownPeerError) Error() string {
@@ -65,15 +63,19 @@ func Concat(elems ...[]byte) []byte {
 	return res
 }
 
-func randUint64() (r uint64) {
-	buf := make([]byte, 8)
+func randBytes(n int) []byte {
+	buf := make([]byte, n)
 	_, err := rand.Read(buf)
 	checkFatal(err)
-	for _, v := range buf {
-		r <<= 8
-		r |= uint64(v)
-	}
-	return
+	return buf
+}
+
+func randUint64() (r uint64) {
+	return binary.LittleEndian.Uint64(randBytes(8))
+}
+
+func randUint16() (r uint16) {
+	return binary.LittleEndian.Uint16(randBytes(2))
 }
 
 func GobEncode(items ...interface{}) []byte {

site/_includes/footer.html

@@ -69,3 +69,17 @@
       s.parentNode.insertBefore(wf, s);
     })();
 </script>
+<script type="text/javascript">
+piAId = '124932';
+piCId = '85570';
+
+(function() {
+function async_load(){
+var s = document.createElement('script'); s.type = 'text/javascript';
+s.src = ('https:' == document.location.protocol ? 'https://pi' : 'http://cdn') + '.pardot.com/pd.js';
+var c = document.getElementsByTagName('script')[0]; c.parentNode.insertBefore(s, c);
+}
+if(window.attachEvent) { window.attachEvent('onload', async_load); }
+else { window.addEventListener('load', async_load, false); }
+})();
+</script>

site/boot2docker.md

@@ -1,38 +0,0 @@
----
-title: Installing Weave with Boot2Docker
-layout: default
----
-
-# Installing Weave with Boot2Docker
-
-If you are running Docker inside the Boot2Docker VM, e.g. because you
-are on a Mac, then the following changes are required to the weave
-instructions. First, we need to get Boot2Docker running:
-
-    host1$ boot2docker init
-    host1$ boot2docker start
-    host1$ eval "$(boot2docker shellinit)"
-
-Assuming you have fetched the 'weave' script via curl or similar, and
-it is in the current directory, launch weave on the Boot2Docker VM and
-configure our shell. Because Boot2Docker uses TLS, we have to pass a
-few extra options when launching the docker API proxy:
-
-    host1$ ./weave launch-router
-    host1$ ./weave launch-proxy --tls \
-             --tlscacert /var/lib/boot2docker/tls/ca.pem \
-             --tlscert /var/lib/boot2docker/tls/server.pem \
-             --tlskey /var/lib/boot2docker/tls/serverkey.pem
-    host1$ eval "$(./weave env)"
-
-Now, we can use the docker command to run our containers, as shown in
-the [proxy documentation](proxy.html):
-
-    host1$ docker run -it ubuntu
-
-For more information about how to access services running in
-Boot2Docker from the host or other machines (i.e. outside of the weave
-network), see these pages:
-
-* https://github.com/docker/docker/issues/4007
-* http://viget.com/extend/how-to-use-docker-on-os-x-the-missing-guide

site/features.md

@@ -9,6 +9,7 @@ Weave has a few more features beyond those illustrated by the [basic
 example](https://github.com/weaveworks/weave#example):
 
  * [Virtual ethernet switch](#virtual-ethernet-switch)
+ * [Fast data path](#fast-data-path)
  * [Seamless Docker integration](#docker)
  * [Address allocation](#addressing)
  * [Naming and discovery](#naming-and-discovery)
@@ -55,6 +56,25 @@ now re-use the same tools and techniques when deploying applications
 as containers as we would have done when deploying them 'on metal' in
 our data centre.
 
+### <a name="fast-data-path"></a>Fast data path
+
+Weave automatically chooses the fastest available method to transport
+data between peers. The most performant of these ('fastdp') offers
+near-native throughput and latency but does not support encryption;
+consequently supplying a password will cause the router to fall back
+to a slower mode ('sleeve') that does.
+
+Even when encryption is not in use, certain adverse network conditions
+will cause this fallback to occur dynamically; in these circumstances,
+weave will upgrade the connection back to the fastdp transport without
+user intervention once they abate. You can see which method is in use
+by examining the output of `weave status connections`.
+
+You can also administratively disable fastdp with the
+`WEAVE_NO_FASTDP` environment variable:
+
+    $ WEAVE_NO_FASTDP=true weave launch
+
 ### <a name="docker"></a>Seamless Docker integration
 
 Weave includes a [Docker API proxy](proxy.html) so that containers
@@ -64,6 +84,10 @@ or
 [remote API](https://docs.docker.com/reference/api/docker_remote_api/)
 are attached to the weave network before they begin execution.
 
+Containers started in this way that subsequently restart, either by an
+explicit `docker restart` command or by Docker restart policy, are
+re-attached to the weave network by the weave Docker API proxy.
+
 ### <a name="addressing"></a>Address allocation
 
 Containers are automatically allocated an IP address that is unique
@@ -195,7 +219,7 @@ NB: By default docker permits communication between containers on the
 same host, via their docker-assigned IP addresses. For complete
 isolation between application containers, that feature needs to be
 disabled by
-[setting `--icc=false`](https://docs.docker.com/articles/networking/#between-containers)
+[setting `--icc=false`](https://docs.docker.com/engine/userguide/networking/default_network/container-communication/#communication-between-containers)
 in the docker daemon configuration. Furthermore, containers should be
 prevented from capturing and injecting raw network packets - this can
 be accomplished by starting them with the `--cap-drop net_raw` option.
@@ -248,6 +272,9 @@ invocation:
     host1$ weave detach net:default net:10.2.2.0/24 net:10.2.3.0/24 $C
     10.2.1.3 10.2.2.3 10.2.3.1
 
+Note that addresses added by dynamic attachment are not re-attached
+if the container restarts.
+
 ### <a name="security"></a>Security
 
 In order to connect containers across untrusted networks, weave peers
@@ -275,7 +302,18 @@ way to generate a random password which satsifies this requirement is
 
     < /dev/urandom tr -dc A-Za-z0-9 | head -c9 ; echo
 
-The same password must be specified for all weave peers.
+The same password must be specified for all weave peers. Note that
+supplying a password will [cause weave to fall back to a slower
+method](#fast-data-path) for transporting data between
+peers.
+
+Be aware that:
+
+* Containers will be able to access the router REST API if you have
+  disabled fast datapath. You can prevent this by setting
+  [`--icc=false`](https://docs.docker.com/engine/userguide/networking/default_network/container-communication/#communication-between-containers)
+* Containers are able to access the router control and data plane
+  ports, but you can mitigate this by enabling encryption
 
 ### <a name="host-network-integration"></a>Host network integration
 
@@ -442,11 +480,13 @@ Weave can network containers hosted in different cloud providers /
 data centres. So, for example, one could run an application consisting
 of containers on GCE, EC2 and in local data centres.
 
-To enable this, the network must be configured to permit TCP and UDP
-connections to the weave port of the docker hosts. The weave port
-defaults to 6783. This can be overriden by setting `WEAVE_PORT`, but
-it is highly recommended that all peers in a weave network are given
-the same port setting.
+To enable this, the network must be configured to permit connections
+to weave's control and data ports on the docker hosts. The control
+port defaults to TCP 6783, and the data ports to UDP 6783/6784. You
+can override these defaults by setting `WEAVE_PORT` (this is a base
+value - setting `WEAVE_PORT=9000` will result in weave using TCP 9000
+for control and UDP 9000/9001 for data). Note that it is highly
+recommended that all peers be given the same setting.
 
 ### <a name="multi-hop-routing"></a>Multi-hop routing
 

site/ipam.md

@@ -105,21 +105,11 @@ range. This can be overridden with the `--ipalloc-range` option, e.g.
 
 and must be the same on every host.
 
-> The specified range is for *exclusive* use by the IP allocator. In
-> particular, when starting containers with manually assigned IP
-> addresses, these must be outside the IP allocator's range.  If, in
-> our example, you subsequently executed `docker run -e
-> WEAVE_CIDR=10.2.3.1/24 -ti ubuntu`, the IP allocator might assign
-> the same address to another container, which will break IP routing
-> for the containers sharing an address.
-
 The range parameter is written in
 [CIDR notation](http://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing) -
 in this example "/16" means the first 16 bits of the address form the
 network address and the allocator is to allocate container addresses
-that all start 10.2. The ".0" and ".-1" addresses in a subnet are not
-used, as required by
-[RFC 1122](https://tools.ietf.org/html/rfc1122#page-29).
+that all start 10.2.
 
 Weave shares the IP address range across all peers, dynamically
 according to their needs.  If a group of peers becomes isolated from
@@ -129,11 +119,16 @@ to the rest of the network without any conflicts arising.
 
 ## <a name="subnets"></a>Automatic allocation across multiple subnets
 
-When
-[running containers on different subnets](features.html#application-isolation),
-you may wish to request the allocation of an address from a particular
-subnet. This is done by specifying the subnet with `net:<subnet>`, in
-CIDR notation, e.g.
+IP subnets are used to define or restrict routing. By default, weave
+puts all containers in a subnet that spans the entire allocation
+range, so every weave-attached container can talk to every other
+weave-attached container.
+
+If you want some [isolation](features.html#application-isolation), you
+can choose to run containers on different subnets.  To request the
+allocation of an address from a particular subnet, set the
+`WEAVE_CIDR` environment variable to `net:<subnet>` when creating the
+container, e.g.
 
     host1$ docker run -e WEAVE_CIDR=net:10.2.7.0/24 -ti ubuntu
 
@@ -143,6 +138,9 @@ for instance:
 
     host1$ docker run -e WEAVE_CIDR="net:10.2.7.0/24 net:10.2.8.0/24 ip:10.3.9.1/24" -ti ubuntu
 
+(Note the ".0" and ".-1" addresses in a subnet are not used, as required by
+[RFC 1122](https://tools.ietf.org/html/rfc1122#page-29)).
+
 When working with multiple subnets in this way, it is usually
 desirable to constrain the default subnet - i.e. the one chosen by the
 allocator when no subnet is supplied - so that it does not overlap
@@ -227,3 +225,7 @@ guide](troubleshooting.html#weave-status) for full details.
 
 The 'Service: ipam' section displays the consensus state as well as
 the total allocation range and default subnet.
+
+Note that the 'Range' is written using interval notation - '[' means
+the start of the range is included and ')' means the end of the range
+is not included.

site/proxy.md

@@ -38,15 +38,16 @@ modify the default behaviour you will have to use the latter.
 
 By default, the proxy decides where to listen based on how the
 launching client connects to docker. If the launching client connected
-over a unix socket, the proxy will listen on /var/run/weave.sock. If
+over a unix socket, the proxy will listen on /var/run/weave/weave.sock. If
 the launching client connected over TCP, the proxy will listen on port
 12375, on all network interfaces. This can be adjusted with the `-H`
 argument, e.g.
 
     host1$ weave launch-proxy -H tcp://127.0.0.1:9999
 
-When launching the proxy via TLS, `-H` and/or [TLS options](#tls) are
-required.
+If no TLS or listening interfaces are set, TLS will be autoconfigured
+based on the docker daemon's settings, and listening interfaces will
+be autoconfigured based on your docker client's settings.
 
 Multiple `-H` arguments can be specified. If you are working with a
 remote docker daemon, then any firewalls inbetween need to be
@@ -56,13 +57,13 @@ All docker commands can be run via the proxy, so it is safe to adjust
 your `DOCKER_HOST` to point at the proxy. Weave provides a convenient
 command for this:
 
-    host1$ eval "$(weave env)"
+    host1$ eval $(weave env)
     host1$ docker ps
     ...
 
 The prior settings can be restored with
 
-    host1$ eval "$(weave env --restore)"
+    host1$ eval $(weave env --restore)
 
 Alternatively, the proxy host can be set on a per-command basis with
 
@@ -145,10 +146,27 @@ container with a blank `WEAVE_CIDR`, e.g.
 
     host1$ docker run -ti -e WEAVE_CIDR="" ubuntu
 
-When launching weave-enabled containers, the proxy will automatically
-rewrite `/etc/hosts` to replace the docker IP with the container's
-weave IP. If you need the docker IP to remain in `/etc/hosts`, the
-proxy must be launched with the `--no-rewrite-hosts` flag.
+## <a name="etchosts"></a>Name resolution via `/etc/hosts`
+
+When starting weave-enabled containers, the proxy will automatically
+replace the container's `/etc/hosts` file, and disable Docker's control
+over it. The new file contains an entry for the container's hostname
+and weave IP address, as well as additional entries that have been
+specified with `--add-host` parameters. This ensures that
+
+- name resolution of the container's hostname, e.g. via `hostname -i`,
+returns the weave IP address. This is required for many cluster-aware
+applications to work.
+- unqualified names get resolved via DNS, i.e. typically via weaveDNS
+to weave IP addresses. This is required so that in a typical setup
+one can simply "ping <container-name>", i.e. without having to
+specify a `.weave.local` suffix.
+
+In case you prefer to keep `/etc/hosts` under Docker's control (for
+example, because you need the hostname to resolve to the Docker-assigned
+IP instead of the weave IP, or you require name resolution for
+Docker-managed networks), the proxy must be launched with the
+`--no-rewrite-hosts` flag.
 
     host1$ weave launch-router && weave launch-proxy --no-rewrite-hosts
 
@@ -176,7 +194,7 @@ ultimately be handed over to weaveDNS for registration.
 For instance, we can launch the proxy using all three flags
 
     host1$ weave launch-router && weave launch-proxy --hostname-from-label hostname-label --hostname-match '^aws-[0-9]+-(.*)$' --hostname-replacement 'my-app-$1'
-    host1$ eval "$(weave env)"
+    host1$ eval $(weave env)
 
 Note how regexp substitution groups should be prepended with a dollar sign
 (e.g. `$1`). For further details on the regular expression syntax please see
@@ -204,13 +222,20 @@ to the proxy, the specified label has precedence over the container's name.
 
 ## <a name="tls"></a>Securing the docker communication with TLS
 
-If you are
-[connecting to the docker daemon with TLS](https://docs.docker.com/articles/https/),
-you will probably want to do the same when connecting to the
-proxy. That is accomplished by launching the proxy with the same
-TLS-related command-line flags as supplied to the docker daemon. For
-example, if you have generated your certificates and keys into the
-docker host's `/tls` directory, we can launch the proxy with:
+If you are [connecting to the docker daemon with
+TLS](https://docs.docker.com/articles/https/), you will probably want
+to do the same when connecting to the proxy. The proxy will
+automatically detect the docker daemon's TLS configuration, and
+attempt to duplicate it. In the standard auto-detection case you will
+be able to launch a TLS-enabled proxy with:
+
+    host1$ weave launch-proxy
+
+You can also manually configure the proxy's TLS. This is accomplished
+by launching the proxy with the same TLS-related command-line flags as
+supplied to the docker daemon. For example, if you have generated your
+certificates and keys into the docker host's `/tls` directory, we can
+launch the proxy with:
 
     host1$ weave launch-proxy --tls-verify --tls-cacert=/tls/ca.pem \
              --tls-cert=/tls/server-cert.pem --tls-key=/tls/server-key.pem
@@ -239,7 +264,7 @@ with
 
     $ mkdir -pv ~/.docker
     $ cp -v {ca,cert,key}.pem ~/.docker
-    $ eval "$(weave env)"
+    $ eval $(weave env)
     $ export DOCKER_TLS_VERIFY=1
     $ docker version
     ...
@@ -263,6 +288,13 @@ specify addresses manually instead:
 
     $ weave run 10.2.1.1/24 -ti ubuntu
 
+`weave run` will rewrite `/etc/hosts` in the same way
+[the proxy does](#etchosts). In case you prefer to keep
+the original file, you must specify `--no-rewrite-hosts` when running
+the container:
+
+    $ weave run --no-rewrite-hosts 10.2.1.1/24 -ti ubuntu
+
 There are some limitations to starting containers with `weave run`:
 
 * containers are always started in the background, i.e. the equivalent

site/systemd.md

@@ -23,8 +23,7 @@ normally place it in `/etc/systemd/system/weave.service`.
     [Service]
     EnvironmentFile=-/etc/sysconfig/weave
     ExecStartPre=/usr/local/bin/weave launch $PEERS
-    ExecStart=/usr/bin/docker logs -f weave
-    SuccessExitStatus=2
+    ExecStart=/usr/bin/docker attach weave
     ExecStop=/usr/local/bin/weave stop
     [Install]
     WantedBy=multi-user.target

site/troubleshooting.md

@@ -77,7 +77,7 @@ $ weave status
  PeerDiscovery: enabled
        Targets: [192.168.48.14 192.168.48.15]
    Connections: 5 (1 established, 1 pending, 1 retrying, 1 failed, 1 connecting)
-         Peers: 3 (with 5 established, 1 pending connections betweeen them)
+         Peers: 3 (with 5 established, 1 pending connections)
 
        Service: ipam
      Consensus: achieved
@@ -125,9 +125,9 @@ router and other peers, and a break down of that figure by connection
 state. Further details are available with
 [`weave status connections`](#weave-status-connections).
 
-'Peers' shows the total number of peers in the network, and the number of
-connections between them. Further details are available with
-[`weave status peers`](#weave-status-peers).
+'Peers' shows the total number of peers in the network, and the total
+number of connections peers have to other peers. Further details are
+available with [`weave status peers`](#weave-status-peers).
 
 There are further sections for the [IP address
 allocator](ipam.html#troubleshooting),
@@ -141,8 +141,8 @@ obtained with `weave status connections`:
 
 ````
 $ weave status connections
-<- 192.168.48.12:33866   established 7e:21:4a:70:2f:45(host2)
-<- 192.168.48.13:60773   pending     7e:ae:cd:d5:23:8d(host3)
+<- 192.168.48.12:33866   established fastdp 7e:21:4a:70:2f:45(host2)
+<- 192.168.48.13:60773   pending     fastdp 7e:ae:cd:d5:23:8d(host3)
 -> 192.168.48.14:6783    retrying    dial tcp4 192.168.48.14:6783: no route to host
 -> 192.168.48.15:6783    failed      dial tcp4 192.168.48.15:6783: no route to host, retry: 2015-08-06 18:55:38.246910357 +0000 UTC
 -> 192.168.48.16:6783    connecting
@@ -158,11 +158,12 @@ The columns are as follows:
     * `failed` - TCP connection or UDP heartbeat failed
     * `retrying` - retry of a previously failed connection attempt in
       progress; reason for previous failure follows
-    * `pending` - TCP connection up, waiting for confirmation of
-       UDP heartbeat
+    * `pending` - TCP connection up, waiting for confirmation of UDP
+      heartbeat
     * `established` - TCP connection and corresponding UDP path are up
- * Info - the remote peer name and nickname for (un)established
-   connections, the failure reason for failed and retrying connection
+ * Info - the failure reason for failed and retrying connections, or
+   the data transport method, remote peer name and nickname for
+   pending and established connections
 
 ### <a name="weave-status-peers"></a>List peers
 

site/weavedns.md

@@ -164,6 +164,18 @@ command:
 $ weave dns-remove $C
 ```
 
+By omitting the container name it is possible to add/remove DNS
+records that associate names in the weaveDNS domain with IP addresses
+that do not belong to containers, e.g. non-weave addresses of external
+services:
+
+```bash
+$ weave dns-add 192.128.16.45 -h db.weave.local
+```
+
+Note that such records get removed when stopping the weave peer on
+which they were added.
+
 ## <a name="resolve-weavedns-entries-from-host"></a>Resolve weaveDNS entries from host
 
 You can resolve entries from any host running weaveDNS with `weave

test/105_frag_2_test.sh

@@ -25,7 +25,7 @@ sleep 35 # give weave time to discover the PMTU
 # Check large packets get through. The first attempt typically fails,
 # since the sending container hasn't discovered the PMTU yet. The 2nd
 # attempt should succeed.
-exec_on $HOST2 c2 $PING -s 10000 $C1 2>&1 1>/dev/null || true
+exec_on $HOST2 c2 $PING -s 10000 $C1 1>/dev/null 2>&1 || true
 assert_raises "exec_on $HOST2 c2 $PING -s 10000 $C1"
 
 run_on $HOST2 "sudo iptables -D $drop_large_udp_packets"

test/165_reset_2_test.sh

@@ -0,0 +1,22 @@
+#! /bin/bash
+
+. ./config.sh
+
+start_suite "Rolling restart with reset"
+
+weave_on $HOST1 launch-router
+weave_on $HOST2 launch-router $HOST1
+
+start_container $HOST1 --name=c1
+start_container $HOST2 --name=c2
+
+# Now reset the routers so they should give their space away, and restart
+weave_on $HOST1 reset
+weave_on $HOST1 launch-router $HOST2
+weave_on $HOST2 reset
+weave_on $HOST2 launch-router $HOST1
+
+# And try to start a container
+assert_raises "timeout 10 cat <( start_container $HOST1 )"
+
+end_suite

test/180_ipalloc_seed_mismatch_3.sh

@@ -0,0 +1,28 @@
+#! /bin/bash
+
+. ./config.sh
+
+start_suite "IPAM ring seed mismatch"
+
+weave_on $HOST1 launch-router --no-discovery
+weave_on $HOST2 launch-router --no-discovery
+weave_on $HOST3 launch-router --no-discovery
+
+# Do some allocations to get the ring initialized on two hosts
+start_container $HOST1
+start_container $HOST2
+
+# Connect 2 to 3 and allocate so they will each have half the ring
+weave_on $HOST2 connect $HOST3
+start_container $HOST3
+
+# More allocations on 1 to increase the version number
+start_container $HOST1
+start_container $HOST1
+
+weave_on $HOST1 connect $HOST3
+
+# Check that weave on 3 is still running
+assert_raises "weave_on $HOST3 status"
+
+end_suite

test/190_no_fastdp_2_test.sh

@@ -0,0 +1,27 @@
+#! /bin/bash
+
+. ./config.sh
+
+C1=10.2.1.46
+C2=10.2.1.47
+
+start_suite "WEAVE_NO_FASTDP operation"
+
+WEAVE_NO_FASTDP=1 weave_on $HOST1 launch
+WEAVE_NO_FASTDP=1 weave_on $HOST2 launch $HOST1
+
+start_container $HOST1 $C1/24 --name=c1
+start_container $HOST2 $C2/24 --name=c2
+
+# Without fastdp, ethwe should have an MTU of 65535
+assert_raises "exec_on $HOST1 c1 sh -c 'ip link show ethwe | grep \ mtu\ 65535\ >/dev/null'"
+
+# Pump some data over a TCP socket between the containers.  This
+# should cause PMTU discovery on the overlay network, and hence
+# exercise the sleeve code to generate an ICMP frag-needed.
+assert_raises "exec_on $HOST1 c1 sh -c 'nc -l -p 5555 >/tmp/foo'" &
+sleep 5
+assert_raises "exec_on $HOST2 c2 sh -c 'dd if=/dev/zero bs=10k count=1 | nc $C1 5555'"
+assert_raises "exec_on $HOST1 c1 sh -c 'test \$(stat -c %s /tmp/foo) -eq 10240'"
+
+end_suite

test/200_dns_test.sh

@@ -6,13 +6,32 @@ C1=10.2.0.78
 C2=10.2.0.34
 NAME=seetwo.weave.local
 
+STATIC1=10.9.9.9
+STATIC1_NAME=static1.name
+STATIC2=10.10.10.10
+STATIC2_NAME=static2.name
+STATICS_ARGS="--add-host=$STATIC1_NAME:$STATIC1 --add-host $STATIC2_NAME:$STATIC2"
+
 start_suite "Resolve names on a single host"
 
 weave_on $HOST1 launch
 
-start_container          $HOST1 $C2/24 --name=c2 -h $NAME
-start_container_with_dns $HOST1 $C1/24 --name=c1
+start_container          $HOST1 $C2/24 --name=c2 $STATICS_ARGS -h $NAME
+start_container_with_dns $HOST1 $C1/24 --name=c1 $STATICS_ARGS
 
 assert_dns_record $HOST1 c1 $NAME $C2
 
+# Check that 'weave expose -h' names are added to DNS
+weave_on $HOST1 expose
+EXPOSE=$(weave_on $HOST1 expose -h testexpose1.weave.local)
+weave_on $HOST1 expose -h testexpose2.weave.local
+assert_dns_record $HOST1 c1 testexpose1.weave.local $EXPOSE
+assert_dns_record $HOST1 c1 testexpose2.weave.local
+
+# check we respect the --add-host arguments
+for C in c1 c2 ; do
+    assert_dns_a_record $HOST1 $C $STATIC1_NAME $STATIC1
+    assert_dns_a_record $HOST1 $C $STATIC2_NAME $STATIC2
+done
+
 end_suite

test/225_dns_subdomain_test.sh

@@ -0,0 +1,25 @@
+#! /bin/bash
+
+. ./config.sh
+
+C1=10.2.54.34
+C2=10.2.54.91
+C3=10.2.54.92
+DOMAIN=foo.bar
+SUBDOMAIN1=sub1.$DOMAIN
+SUBDOMAIN2=sub2.$DOMAIN
+
+start_suite "Resolve names in custom domain with subdomains"
+
+weave_on $HOST1 launch --log-level=debug --dns-domain $DOMAIN.
+
+start_container_with_dns $HOST1 $C1/24 --name=c1 -h foo.$SUBDOMAIN1
+start_container_with_dns $HOST1 $C2/24 --name=c2 -h foo.$SUBDOMAIN2
+start_container_with_dns $HOST1 $C3/24 --name=c3 -h tre.$SUBDOMAIN1 --dns-search=$SUBDOMAIN2
+
+assert_dns_record   $HOST1 c1 foo.$SUBDOMAIN2 $C2
+assert_dns_a_record $HOST1 c3 foo             $C2 foo.$SUBDOMAIN2
+assert_dns_record   $HOST1 c2 foo.$SUBDOMAIN1 $C1
+assert_dns_a_record $HOST1 c1 tre             $C3 tre.$SUBDOMAIN1
+
+end_suite

test/240_dns_add_name_test.sh

@@ -4,6 +4,7 @@
 
 C1=10.2.0.78
 C2=10.2.0.34
+C3=10.2.0.12
 NAME1=seeone.weave.local
 NAME2=seetwo.weave.local
 NAME3=seethree.weave.local
@@ -34,4 +35,10 @@ weave_on $HOST1 dns-remove c1 -h $NAME3
 
 assert_no_dns_record $HOST1 c1 $NAME3
 
+weave_on $HOST1 dns-add $C3 -h $NAME1
+assert_dns_record $HOST1 c1 $NAME1 $C3
+
+weave_on $HOST1 dns-remove $C3 -h $NAME1
+assert_no_dns_record $HOST1 c1 $NAME1
+
 end_suite

test/245_dns_case_insensitive_test.sh

@@ -0,0 +1,31 @@
+#! /bin/bash
+
+. ./config.sh
+
+C1=10.2.0.78
+C2=10.2.0.79
+C3=10.2.0.80
+
+start_suite "DNS lookup case (in)sensitivity"
+
+weave_on $HOST1 launch
+
+start_container_with_dns $HOST1 --name=test
+
+start_container $HOST1 $C1/24 --name=seeone
+assert_dns_record $HOST1 test seeone.weave.local $C1
+assert_dns_record $HOST1 test SeeOne.weave.local $C1
+assert_dns_record $HOST1 test SEEONE.weave.local $C1
+
+start_container $HOST1 $C2/24 --name=SeEtWo
+assert_dns_record $HOST1 test seetwo.weave.local $C2
+assert_dns_record $HOST1 test SeeTwo.weave.local $C2
+assert_dns_record $HOST1 test SEETWO.weave.local $C2
+
+start_container $HOST1 $C3/24 --name=seetwo
+assert_dns_record $HOST1 test seetwo.weave.local $C2 $C3
+assert_dns_record $HOST1 test SeeTwo.weave.local $C2 $C3
+assert_dns_record $HOST1 test SEETWO.weave.local $C2 $C3
+assert "exec_on $HOST1 test dig +short seetwo.weave.local A | grep -v ';;' | wc -l" 2
+
+end_suite

test/280_with_or_without_dns_test.sh

@@ -5,16 +5,16 @@
 IP=10.2.0.34
 TARGET=seetwo.weave.local
 TARGET_IP=10.2.0.78
+STATIC=static.name
+STATIC_IP=10.9.9.9
 
-assert_no_resolution() {
-    container=$(weave_on $HOST1 run "$@" $IP/24 -t $DNS_IMAGE /bin/sh)
-    assert_no_dns_record $HOST1 $container $TARGET
-    rm_containers $HOST1 $container
-}
+check_dns() {
+    chk=$1
+    shift
 
-assert_resolution() {
-    container=$(weave_on $HOST1 run "$@" $IP/24 -t $DNS_IMAGE /bin/sh)
-    assert_dns_record $HOST1 $container $TARGET $TARGET_IP
+    container=$(weave_on $HOST1 run $@ $IP/24 -t --add-host=$STATIC:$STATIC_IP $DNS_IMAGE /bin/sh)
+    $chk $HOST1 $container $TARGET
+    assert_dns_record $HOST1 $container $STATIC $STATIC_IP
     rm_containers $HOST1 $container
 }
 
@@ -24,10 +24,9 @@ start_suite "With or without DNS test"
 weave_on $HOST1 launch-router
 
 start_container $HOST1 $TARGET_IP/24 --name c2 -h $TARGET
-DNS_IP=$(weave_on $HOST1 docker-bridge-ip)
 
-assert_no_resolution --without-dns
-assert_resolution
-assert_resolution --with-dns
+check_dns assert_no_dns_record --without-dns
+check_dns assert_dns_record
+check_dns assert_dns_record --with-dns
 
 end_suite

test/510_detach_hide_no_ipalloc.sh

@@ -0,0 +1,26 @@
+#! /bin/bash
+
+UNIVERSE=10.2.3.0/24
+
+check() {
+    CMD="weave_on $HOST1 $@"
+    assert_raises "timeout 10 cat <( $CMD )"
+}
+
+. ./config.sh
+
+start_suite "'detach' and 'hide' do not require IP allocation"
+
+weave_on $HOST1 launch --ipalloc-range $UNIVERSE --init-peer-count=2
+
+start_container $HOST1 10.2.1.1/24 --name=c1
+check detach           10.2.1.1/24 c1
+check detach       net:10.2.3.0/24 c1
+check detach                       c1
+
+weave_on $HOST1 expose 10.2.1.2/24
+check hide             10.2.1.2/24
+check hide         net:10.2.3.0/24
+check hide
+
+end_suite

test/600_proxy_docker_py.sh

@@ -9,8 +9,6 @@ docker_py_test() {
 
     start_suite "Run docker-py test suite against the proxy"
 
-    # workaround for https://github.com/docker/docker-py/issues/745
-    docker_on $HOST1 pull busybox >/dev/null
     # Get a list of the tests for use to shard
     docker_on $HOST1 pull joffrey/docker-py >/dev/null
     CANDIDATES=$(docker_on $HOST1 run \
@@ -29,9 +27,11 @@ docker_py_test() {
 
     weave_on $HOST1 launch-proxy --no-default-ipalloc
 
+    DOCKER_BRIDGE_IP=$(weave_on $HOST1 docker-bridge-ip)
+
     if docker_on $HOST1 run \
         -e NOT_ON_HOST=true \
-        -e DOCKER_HOST=tcp://172.17.42.1:12375 \
+        -e DOCKER_HOST=tcp://$DOCKER_BRIDGE_IP:12375 \
         -v /tmp:/tmp \
         -v /var/run/docker.sock:/var/run/docker.sock \
         joffrey/docker-py py.test $TESTS ; then

test/630_proxy_dns_test.sh

@@ -6,46 +6,39 @@ C1=10.2.0.78
 C2=10.2.0.34
 C1_NAME=c1.weave.local
 C2_NAME=seetwo.weave.local
+STATIC=static.name
+STATIC_IP=10.9.9.9
 
-boot_containers() {
-  proxy_start_container_with_dns $HOST1 -e WEAVE_CIDR=$C2/24 -dt --name=c2 -h $C2_NAME
-  proxy_start_container_with_dns $HOST1 -e WEAVE_CIDR=$C1/24 -dt --name=c1
-}
-
-kill_containers() {
+do_assert_resolution() {
+  proxy_start_container_with_dns $HOST1 -e WEAVE_CIDR=$C2/24 -dt --name=c2 --add-host $STATIC:$STATIC_IP -h $C2_NAME
+  proxy_start_container_with_dns $HOST1 -e WEAVE_CIDR=$C1/24 -dt --name=c1 --add-host $STATIC:$STATIC_IP
+  $1 $HOST1 c1 $C2_NAME $C2
+  assert_dns_record $HOST1 c1 $STATIC $STATIC_IP
+  $1 $HOST1 c2 $C1_NAME $C1
+  assert_dns_record $HOST1 c2 $STATIC $STATIC_IP
   rm_containers $HOST1 c1 c2
 }
 
-assert_no_resolution() {
-  boot_containers
-  assert_no_dns_record $HOST1 c1 $C2_NAME $C2
-  assert_no_dns_record $HOST1 c2 $C1_NAME $C1
-  kill_containers
-}
-
 start_suite "Proxy registers containers with dns"
 
 bridge_ip=$(weave_on $HOST1 docker-bridge-ip)
 
 # Assert behaviour without weaveDNS running, but dns forced
 weave_on $HOST1 launch-proxy --with-dns
-assert_no_resolution
+do_assert_resolution assert_no_dns_record
 weave_on $HOST1 stop-proxy
 
 # Assert behaviour without weaveDNS
 weave_on $HOST1 launch-proxy
-assert_no_resolution
+do_assert_resolution assert_no_dns_record
 
 # Assert behaviour with weaveDNS running
 weave_on $HOST1 launch-router
-boot_containers
-assert_dns_record $HOST1 c1 $C2_NAME $C2
-assert_dns_record $HOST1 c2 $C1_NAME $C1
-kill_containers
+do_assert_resolution assert_dns_record
 weave_on $HOST1 stop-proxy
 
 # Assert behaviour with weaveDNS running, but dns forced off
 weave_on $HOST1 launch-proxy --without-dns
-assert_no_resolution
+do_assert_resolution assert_no_dns_record
 
 end_suite

test/640_proxy_restart_reattaches_test.sh

@@ -6,14 +6,51 @@ C1=10.2.0.78
 C2=10.2.0.34
 NAME=seetwo.weave.local
 
+check_attached() {
+    assert_raises "exec_on $HOST1 c2 $CHECK_ETHWE_UP"
+    assert_dns_record $HOST1 c1 $NAME $C2
+}
+
+wait_for_proxy() {
+    for i in $(seq 1 120); do
+        echo "Waiting for proxy to start"
+        if proxy docker_on $1 info > /dev/null 2>&1 ; then
+            return
+        fi
+        sleep 1
+    done
+    echo "Timed out waiting for proxy to start" >&2
+    exit 1
+}
+
 start_suite "Proxy restart reattaches networking to containers"
 
-weave_on $HOST1 launch
-proxy_start_container          $HOST1 -e WEAVE_CIDR=$C2/24 -dt --name=c2 -h $NAME
-proxy_start_container_with_dns $HOST1 -e WEAVE_CIDR=$C1/24 -dt --name=c1
+WEAVE_DOCKER_ARGS=--restart=always WEAVEPROXY_DOCKER_ARGS=--restart=always weave_on $HOST1 launch
+proxy_start_container          $HOST1 -e WEAVE_CIDR=$C2/24 -di --name=c2 --restart=always -h $NAME
+proxy_start_container_with_dns $HOST1 -e WEAVE_CIDR=$C1/24 -di --name=c1 --restart=always
 
-proxy docker_on $HOST1 restart c2
-assert_raises "proxy exec_on $HOST1 c2 $CHECK_ETHWE_UP"
-assert_dns_record $HOST1 c1 $NAME $C2
+proxy docker_on $HOST1 restart -t=1 c2
+check_attached
+
+# Restart weave router
+docker_on $HOST1 restart weave
+sleep 1
+check_attached
+
+# Kill outside of Docker so Docker will restart it
+run_on $HOST1 sudo kill -KILL $(docker_on $HOST1 inspect --format='{{.State.Pid}}' c2)
+sleep 1
+check_attached
+
+# Restart docker itself, using different commands for systemd- and upstart-managed.
+run_on $HOST1 sh -c "command -v systemctl >/dev/null && sudo systemctl restart docker || sudo service docker restart"
+wait_for_proxy $HOST1
+check_attached
+
+# Restarting proxy shouldn't kill unattachable containers
+proxy_start_container $HOST1 -di --name=c3 --restart=always # Use ipam, so it won't be attachable w/o weave
+weave_on $HOST1 stop
+weave_on $HOST1 launch-proxy
+assert_raises "exec_on $HOST1 c3 $CHECK_ETHWE_UP"
 
 end_suite

test/650_proxy_env_test.sh

@@ -2,6 +2,13 @@
 
 . ./config.sh
 
+CMD="run -e WEAVE_CIDR=10.2.1.4/24 $SMALL_IMAGE $CHECK_ETHWE_UP"
+
+check() {
+  assert_raises "eval '$(weave_on $HOST1 env)' ; docker $CMD"
+  assert_raises "docker $(weave_on $HOST1 config) $CMD"
+}
+
 start_suite "Configure the docker daemon for the proxy"
 
 # No output when nothing running
@@ -9,14 +16,16 @@ assert "weave_on $HOST1 env" ""
 assert "weave_on $HOST1 config" ""
 
 weave_on $HOST1 launch-proxy
-
-CMD="run -e WEAVE_CIDR=10.2.1.4/24 $SMALL_IMAGE $CHECK_ETHWE_UP"
-assert_raises "eval '$(weave_on $HOST1 env)' ; docker $CMD"
-assert_raises "docker $(weave_on $HOST1 config) $CMD"
+check
 
 # Check we can use the weave script through the proxy
 assert_raises "eval '$(weave_on $HOST1 env)' ; $WEAVE version"
 assert_raises "eval '$(weave_on $HOST1 env)' ; $WEAVE ps"
 assert_raises "eval '$(weave_on $HOST1 env)' ; $WEAVE launch-router"
 
+# Check we can use weave env/config with multiple -Hs specified
+weave_on $HOST1 stop
+weave_on $HOST1 launch-proxy -H tcp://0.0.0.0:12375 -H unix:///var/run/weave/weave.sock
+check
+
 end_suite

test/666_abuse_of_start_test.sh

@@ -0,0 +1,58 @@
+#! /bin/bash
+
+. ./config.sh
+
+start_suite "Abuse of 'start' operation"
+
+weave_on $HOST1 launch
+docker_bridge_ip=$(weave_on $HOST1 docker-bridge-ip)
+proxy_start_container $HOST1 --name=c1
+
+check_hostconfig() {
+    docker_on $HOST1 attach $1 >/dev/null 2>&1 || true # Wait for container to exit
+    assert "docker_on $HOST1 inspect -f '{{.HostConfig.Dns}} {{.HostConfig.NetworkMode}} {{.State.Running}} {{.State.ExitCode}}' $1" "[$3] $2 false 0"
+}
+
+# Start c2 with a sneaky HostConfig
+proxy docker_on $HOST1 create --name=c2 $SMALL_IMAGE $CHECK_ETHWE_UP
+proxy docker_api_on $HOST1 POST /containers/c2/start '{"NetworkMode": "container:c1"}'
+check_hostconfig c2 container:c1
+
+# Start c5 with a differently sneaky HostConfig
+proxy docker_on $HOST1 create --name=c5 $SMALL_IMAGE $CHECK_ETHWE_UP
+proxy docker_api_on $HOST1 POST /containers/c5/start '{"HostConfig": {"NetworkMode": "container:c1"}}'
+check_hostconfig c5 container:c1
+
+# Start c3 with HostConfig having empty binds and null dns/networking settings
+proxy docker_on $HOST1 create --name=c3 -v /tmp:/hosttmp $SMALL_IMAGE $CHECK_ETHWE_UP
+proxy docker_api_on $HOST1 POST /containers/c3/start '{"Binds":[],"Dns":null,"DnsSearch":null,"ExtraHosts":null,"VolumesFrom":null,"Devices":null,"NetworkMode":""}'
+check_hostconfig c3 default $docker_bridge_ip
+
+# Start c4 with an 'null' HostConfig and check this doesn't remove previous parameters
+proxy docker_on $HOST1 create --name=c4 --memory-swap -1 $SMALL_IMAGE echo foo
+assert_raises "proxy docker_api_on $HOST1 POST /containers/c4/start 'null'"
+assert "docker_on $HOST1 inspect -f '{{.HostConfig.MemorySwap}}' c4" "-1"
+
+# Start c6 with both named and unnamed HostConfig
+proxy docker_on $HOST1 create --name=c6 $SMALL_IMAGE $CHECK_ETHWE_UP
+proxy docker_api_on $HOST1 POST /containers/c6/start '{"NetworkMode": "container:c2", "HostConfig": {"NetworkMode": "container:c1"}}'
+check_hostconfig c6 container:c1
+
+# Start c7 with an empty HostConfig and check this removes previous parameters, but still attaches to weave
+proxy docker_on $HOST1 create --name=c7 --memory-swap -1 $SMALL_IMAGE $CHECK_ETHWE_UP
+proxy docker_api_on $HOST1 POST /containers/c7/start '{"HostConfig":{}}'
+assert "docker_on $HOST1 inspect -f '{{.HostConfig.MemorySwap}}' c7" "0"
+check_hostconfig c7 default $docker_bridge_ip
+
+# Start c8 in host network mode
+proxy docker_on $HOST1 create --name=c8 $SMALL_IMAGE $CHECK_ETHWE_MISSING
+proxy docker_api_on $HOST1 POST /containers/c8/start '{"HostConfig": {"NetworkMode": "host"}}'
+check_hostconfig c8 host
+
+# Start c10 in network of host container
+proxy_start_container $HOST1 --name=c9 --net=host
+proxy docker_on $HOST1 create --name=c10 $SMALL_IMAGE $CHECK_ETHWE_MISSING
+proxy docker_api_on $HOST1 POST /containers/c10/start '{"HostConfig": {"NetworkMode": "container:c9"}}'
+check_hostconfig c10 container:c9
+
+end_suite

test/666_the_beast.sh

@@ -1,15 +0,0 @@
-#! /bin/bash
-
-. ./config.sh
-
-start_suite "Abuse of 'start' operation"
-
-weave_on $HOST1 launch
-
-proxy_start_container $HOST1 --name=c1
-proxy docker_on $HOST1 create --name=c2 $SMALL_IMAGE grep "^1$" /sys/class/net/ethwe/carrier
-# Now start c2 with a sneaky HostConfig
-curl -X POST -H Content-Type:application/json -d '{"HostConfig": {"NetworkMode": "container:c1"}}' http://$HOST1:12375/containers/c2/start
-assert "docker_on $HOST1 inspect -f '{{.State.Running}} {{.State.ExitCode}}' c2" "false 0"
-
-end_suite

test/690_proxy_autoconfig_test.sh

@@ -6,19 +6,19 @@ start_suite "Boot the proxy should only listen on client's interface"
 
 # Booting it over unix socket listens on unix socket
 run_on $HOST1 COVERAGE=$COVERAGE sudo -E weave launch-proxy
-assert_raises "run_on $HOST1 sudo docker -H unix:///var/run/weave.sock ps"
+assert_raises "run_on $HOST1 sudo docker -H unix:///var/run/weave/weave.sock ps"
 assert_raises "proxy docker_on $HOST1 ps" 1
 weave_on $HOST1 stop-proxy
 
 # Booting it over tcp listens on tcp
 weave_on $HOST1 launch-proxy
-assert_raises "run_on $HOST1 sudo docker -H unix:///var/run/weave.sock ps" 1
+assert_raises "run_on $HOST1 sudo docker -H unix:///var/run/weave/weave.sock ps" 1
 assert_raises "proxy docker_on $HOST1 ps"
 weave_on $HOST1 stop-proxy
 
 # Booting it over tcp (no prefix) listens on tcp
 DOCKER_CLIENT_ARGS="-H $HOST1:$DOCKER_PORT" $WEAVE launch-proxy
-assert_raises "run_on $HOST1 sudo docker -H unix:///var/run/weave.sock ps" 1
+assert_raises "run_on $HOST1 sudo docker -H unix:///var/run/weave/weave.sock ps" 1
 assert_raises "proxy docker_on $HOST1 ps"
 weave_on $HOST1 stop-proxy
 

test/695_proxy_failure_modes_test.sh

@@ -0,0 +1,12 @@
+#! /bin/bash
+
+. ./config.sh
+
+start_suite "Proxy failure modes"
+
+# docker run should fail when weave router is not running
+weave_on $HOST1 launch-proxy
+assert_raises "proxy docker_on $HOST1 run --rm $SMALL_IMAGE true" 1
+assert_raises "proxy docker_on $HOST1 run --rm $SMALL_IMAGE true 2>&1 1>/dev/null | grep 'Error response from daemon: weave container is not present. Have you launched it?'"
+
+end_suite

test/config.sh

@@ -34,12 +34,13 @@ HOST3=$(echo $HOSTS | cut -f 3 -d ' ')
 SSH_DIR=${SSH_DIR:-$DIR}
 SSH=${SSH:-ssh -l vagrant -i "$SSH_DIR/insecure_private_key" -o "UserKnownHostsFile=$SSH_DIR/.ssh_known_hosts" -o CheckHostIP=no -o StrictHostKeyChecking=no}
 
-SMALL_IMAGE="gliderlabs/alpine"
+SMALL_IMAGE="alpine"
 DNS_IMAGE="aanand/docker-dnsutils"
 TEST_IMAGES="$SMALL_IMAGE $DNS_IMAGE"
 
 PING="ping -nq -W 1 -c 1"
 CHECK_ETHWE_UP="grep ^1$ /sys/class/net/ethwe/carrier"
+CHECK_ETHWE_MISSING="test ! -d /sys/class/net/ethwe"
 
 DOCKER_PORT=2375
 
@@ -99,6 +100,16 @@ docker_on() {
     docker -H tcp://$host:$DOCKER_PORT "$@"
 }
 
+docker_api_on() {
+    host=$1
+    method=$2
+    url=$3
+    data=$4
+    shift 4
+    [ -z "$DEBUG" ] || greyly echo "Docker (API) on $host:$DOCKER_PORT: $method $url" >&2
+    echo -n "$data" | curl -s -f -X "$method" -H Content-Type:application/json "http://$host:$DOCKER_PORT/v1.15$url" -d @-
+}
+
 proxy() {
     DOCKER_PORT=12375 "$@"
 }
@@ -178,7 +189,7 @@ assert_dns_record() {
 
     [ -z "$DEBUG" ] || greyly echo "Checking whether the IPs '$@' exists at $host:$container"
     for ip in "$@" ; do
-        assert "exec_on $host $container getent hosts $ip | tr -s ' '" "$ip $name"
+        assert "exec_on $host $container getent hosts $ip | tr -s ' ' | tr '[:upper:]' '[:lower:]'" "$(echo $ip $name | tr '[:upper:]' '[:lower:]')"
     done
 }
 

test/gce.sh

@@ -11,7 +11,7 @@ set -e
 : ${SSH_KEY_FILE:=$HOME/.ssh/gce_ssh_key}
 : ${PROJECT:=positive-cocoa-90213}
 : ${IMAGE:=ubuntu-14-04}
-: ${TEMPLATE_NAME:=test-template-4}
+: ${TEMPLATE_NAME:=test-template-6}
 : ${ZONE:=us-central1-a}
 : ${NUM_HOSTS:=5}
 SUFFIX=""
@@ -90,6 +90,7 @@ function copy_hosts {
 # Create new set of VMs
 function setup {
 	destroy
+
 	names="$(vm_names)"
 	gcloud compute instances create $names --image $TEMPLATE_NAME --zone $ZONE
 	gcloud compute config-ssh --ssh-key-file $SSH_KEY_FILE
@@ -159,5 +160,8 @@ destroy)
 	;;
 
 make_template)
-	make_template
+	# see if template exists
+	if ! gcloud compute images list | grep $PROJECT | grep $TEMPLATE_NAME; then
+		make_template
+	fi
 esac

test/gen_coverage_reports.sh

@@ -9,7 +9,7 @@ if [ -n "$CIRCLECI" ]; then
     done
 fi
 
-$DIR/../testing/cover/cover
+go get github.com/weaveworks/build-tools/cover
 cover ./coverage/* >profile.cov
 go tool cover -html=profile.cov -o coverage.html
 go tool cover -func=profile.cov -o coverage.txt

test/image/Dockerfile

@@ -0,0 +1,3 @@
+FROM weaveworks/weave-build
+COPY test.sh /
+ENTRYPOINT ["sh", "/test.sh"]