Add release 1.0.0

Update documentation to leverage IPAM and DNS

.gitignore

@@ -30,20 +30,23 @@ coverage.html
 *~
 
 # Project specific
-weaver/weaver
-weavedns/weavedns
-weaveproxy/weaveproxy
-sigproxy/sigproxy
-weavewait/weavewait
+prog/weaver/weaver
+prog/weavedns/weavedns
+prog/weaveproxy/weaveproxy
+prog/sigproxy/sigproxy
+prog/weavewait/weavewait
+prog/netcheck/netcheck
 tools/bin
 tools/build
 releases
 weave.tar
 weavedns.tar
 weaveexec.tar
-weaveexec/weave
-weaveexec/weaveproxy
-weaveexec/weavewait
-weaveexec/sigproxy
-weaveexec/docker*.tgz
+prog/weaveexec/weave
+prog/weaveexec/weaveproxy
+prog/weaveexec/weavewait
+prog/weaveexec/sigproxy
+prog/weaveexec/netcheck
+prog/weaveexec/docker*.tgz
 Vagrantfile.local
+test/tls/*.pem

.travis.yml

@@ -14,6 +14,7 @@ addons:
 install:
   - go clean -i net
   - go install -tags netgo std
+  - go get github.com/golang/lint/golint
   - pip install --user --upgrade gcloud gsutil
   - bin/setup-ci-secrets $encrypted_5ba036e89377_key $encrypted_5ba036e89377_iv
   - make travis

CHANGELOG.md

@@ -1,3 +1,52 @@
+## Release 1.0.0
+
+**Highlights**:
+
+- It is now easier than ever to start containers and for them to
+  communicate, across multiple hosts. Automatic IP address allocation,
+  and name resolution via weaveDNS are now enabled by default, and the
+  proxy has become more fully-featured. In short, once weave has been
+  launched the following is possible:
+
+      ````
+      host1$ docker run --name=pingme -dti ubuntu
+      host2$ docker run -ti ubuntu
+      root@d11e9287f65b:/# ping pingme
+      ````
+- Containers can now be
+  [load-balanced](http://docs.weave.works/weave/master/head/weavedns.html#load-balancing)
+  easily.
+- IP address allocation is now available across [multiple
+  subnets](http://docs.weave.works/weave/master/head/ipam.html#range),
+  and hence can be employed when running multiple, isolated
+  applications.
+- The proxy now supports [TLS
+  connections](http://docs.weave.works/weave/master/head/proxy.html#tls),
+  enabling its deployment when the communication between docker
+  clients and the server must be secured.
+
+There are many other new features, plus the usual assortment of bug
+fixes and improvements under the hood. More detail below and in the
+[change log](https://github.com/weaveworks/weave/issues?q=milestone%3A1.0.0).
+
+*NB: This release changes the weave protocol version. Therefore, when
+ upgrading an existing installation, all hosts need to be upgraded in
+ order to for them to be able to communicate and form a network.*
+
+## Release 0.11.2
+
+This is a bug fix release, addressing the following issues:
+
+- `weave run` did not respect DOCKER_CLIENT_ARGS. #855
+- Negative result cache did not expire if requeried within TTL. #845
+
+More details in the [change
+log](https://github.com/weaveworks/weave/issues?q=milestone%3A0.11.2).
+
+*NB: This release does not change the weave protocol version.
+Therefore, when upgrading an existing 0.11 installation incrementally,
+connectivity between peers will be retained.*
+
 ## Release 0.11.1
 
 This is a bug fix release, addressing the following issues:

Makefile

@@ -8,30 +8,40 @@ SUDO=sudo
 
 DOCKERHUB_USER=weaveworks
 WEAVE_VERSION=git-$(shell git rev-parse --short=12 HEAD)
-WEAVER_EXE=weaver/weaver
-WEAVEDNS_EXE=weavedns/weavedns
-WEAVEPROXY_EXE=weaveproxy/weaveproxy
-SIGPROXY_EXE=sigproxy/sigproxy
-WEAVEWAIT_EXE=weavewait/weavewait
+
+WEAVER_EXE=prog/weaver/weaver
+WEAVEDNS_EXE=prog/weavedns/weavedns
+WEAVEPROXY_EXE=prog/weaveproxy/weaveproxy
+SIGPROXY_EXE=prog/sigproxy/sigproxy
+WEAVEWAIT_EXE=prog/weavewait/weavewait
+NETCHECK_EXE=prog/netcheck/netcheck
+
+EXES=$(WEAVER_EXE) $(WEAVEDNS_EXE) $(SIGPROXY_EXE) $(WEAVEPROXY_EXE) $(WEAVEWAIT_EXE) $(NETCHECK_EXE)
+
 WEAVER_IMAGE=$(DOCKERHUB_USER)/weave
 WEAVEDNS_IMAGE=$(DOCKERHUB_USER)/weavedns
 WEAVEEXEC_IMAGE=$(DOCKERHUB_USER)/weaveexec
+
+IMAGES=$(WEAVER_IMAGE) $(WEAVEDNS_IMAGE) $(WEAVEEXEC_IMAGE)
+
 WEAVER_EXPORT=weave.tar
 WEAVEDNS_EXPORT=weavedns.tar
 WEAVEEXEC_EXPORT=weaveexec.tar
 
+EXPORTS=$(WEAVER_EXPORT) $(WEAVEDNS_EXPORT) $(WEAVEEXEC_EXPORT)
+
 WEAVEEXEC_DOCKER_VERSION=1.3.1
-DOCKER_DISTRIB=weaveexec/docker-$(WEAVEEXEC_DOCKER_VERSION).tgz
+DOCKER_DISTRIB=prog/weaveexec/docker-$(WEAVEEXEC_DOCKER_VERSION).tgz
 DOCKER_DISTRIB_URL=https://get.docker.com/builds/Linux/x86_64/docker-$(WEAVEEXEC_DOCKER_VERSION).tgz
 
-all: $(WEAVER_EXPORT) $(WEAVEDNS_EXPORT) $(WEAVEEXEC_EXPORT)
+all: $(EXPORTS)
 
-travis: $(WEAVER_EXE) $(WEAVEDNS_EXE)
+travis: $(EXES)
 
-update:
-	go get -u -f -v -tags -netgo ./$(dir $(WEAVER_EXE)) ./$(dir $(WEAVEDNS_EXE)) ./$(dir $(SIGPROXY_EXE)) ./$(dir $(WEAVEPROXY_EXE))
+update: $(EXES)
+	go get -u -f -v -tags -netgo $(addprefix ./,$(dir $(EXES)))
 
-$(WEAVER_EXE) $(WEAVEDNS_EXE) $(WEAVEPROXY_EXE) $(WEAVEWAIT_EXE): common/*.go
+$(WEAVER_EXE) $(WEAVEDNS_EXE) $(WEAVEPROXY_EXE) $(NETCHECK_EXE): common/*.go common/*/*.go net/*.go
 	go get -tags netgo ./$(@D)
 	go build -ldflags "-extldflags \"-static\" -X main.version $(WEAVE_VERSION)" -tags netgo -o $@ ./$(@D)
 	@strings $@ | grep cgo_stub\\\.go >/dev/null || { \
@@ -43,31 +53,35 @@ $(WEAVER_EXE) $(WEAVEDNS_EXE) $(WEAVEPROXY_EXE) $(WEAVEWAIT_EXE): common/*.go
 		false; \
 	}
 
-$(WEAVER_EXE): router/*.go ipam/*.go ipam/*/*.go weaver/main.go
-$(WEAVEDNS_EXE): nameserver/*.go weavedns/main.go
-$(WEAVEPROXY_EXE): proxy/*.go weaveproxy/main.go
-$(WEAVEWAIT_EXE): weavewait/*.go weavewait/main.go
+$(WEAVER_EXE): router/*.go ipam/*.go ipam/*/*.go prog/weaver/main.go
+$(WEAVEDNS_EXE): nameserver/*.go prog/weavedns/main.go
+$(WEAVEPROXY_EXE): proxy/*.go prog/weaveproxy/main.go
 
-# Sigproxy needs separate rule as it fails the netgo check in the main
-# build stanza due to not importing net package
-$(SIGPROXY_EXE): sigproxy/main.go
+# Sigproxy and weavewait need separate rules as they fail the netgo check in
+# the main build stanza due to not importing net package
+
+$(SIGPROXY_EXE): prog/sigproxy/main.go
 	go build -o $@ ./$(@D)
 
-$(WEAVER_EXPORT): weaver/Dockerfile $(WEAVER_EXE)
-	$(SUDO) docker build -t $(WEAVER_IMAGE) weaver
+$(WEAVEWAIT_EXE): prog/weavewait/main.go
+	go build -o $@ ./$(@D)
+
+$(WEAVER_EXPORT): prog/weaver/Dockerfile $(WEAVER_EXE)
+	$(SUDO) docker build -t $(WEAVER_IMAGE) prog/weaver
 	$(SUDO) docker save $(WEAVER_IMAGE):latest > $@
 
-$(WEAVEDNS_EXPORT): weavedns/Dockerfile $(WEAVEDNS_EXE)
-	$(SUDO) docker build -t $(WEAVEDNS_IMAGE) weavedns
+$(WEAVEDNS_EXPORT): prog/weavedns/Dockerfile $(WEAVEDNS_EXE)
+	$(SUDO) docker build -t $(WEAVEDNS_IMAGE) prog/weavedns
 	$(SUDO) docker save $(WEAVEDNS_IMAGE):latest > $@
 
-$(WEAVEEXEC_EXPORT): weaveexec/Dockerfile $(DOCKER_DISTRIB) weave $(SIGPROXY_EXE) $(WEAVEPROXY_EXE) $(WEAVEWAIT_EXE)
-	cp weave weaveexec/weave
-	cp $(SIGPROXY_EXE) weaveexec/sigproxy
-	cp $(WEAVEWAIT_EXE) weaveexec/weavewait
-	cp $(WEAVEPROXY_EXE) weaveexec/weaveproxy
-	cp $(DOCKER_DISTRIB) weaveexec/docker.tgz
-	$(SUDO) docker build -t $(WEAVEEXEC_IMAGE) weaveexec
+$(WEAVEEXEC_EXPORT): prog/weaveexec/Dockerfile $(DOCKER_DISTRIB) weave $(SIGPROXY_EXE) $(WEAVEPROXY_EXE) $(WEAVEWAIT_EXE) $(NETCHECK_EXE)
+	cp weave prog/weaveexec/weave
+	cp $(SIGPROXY_EXE) prog/weaveexec/sigproxy
+	cp $(WEAVEPROXY_EXE) prog/weaveexec/weaveproxy
+	cp $(WEAVEWAIT_EXE) prog/weaveexec/weavewait
+	cp $(NETCHECK_EXE) prog/weaveexec/netcheck
+	cp $(DOCKER_DISTRIB) prog/weaveexec/docker.tgz
+	$(SUDO) docker build -t $(WEAVEEXEC_IMAGE) prog/weaveexec
 	$(SUDO) docker save $(WEAVEEXEC_IMAGE):latest > $@
 
 $(DOCKER_DISTRIB):
@@ -78,7 +92,7 @@ tests:
 	fail=0 ;                                                                              \
 	for dir in $$(find . -type f -name '*_test.go' | xargs -n1 dirname | sort -u); do     \
 	    output=$$(mktemp cover.XXXXXXXXXX) ;                                              \
-	    if ! go test -tags netgo -covermode=count -coverprofile=$$output $$dir ; then     \
+	    if ! go test -cpu 4 -tags netgo -covermode=count -coverprofile=$$output $$dir ; then     \
             fail=1 ;                                                                          \
         fi ;                                                                                  \
 	    if [ -f $$output ]; then                                                          \
@@ -97,8 +111,9 @@ $(PUBLISH): publish_%:
 publish: $(PUBLISH)
 
 clean:
-	-$(SUDO) docker rmi $(WEAVER_IMAGE) $(WEAVEDNS_IMAGE) $(WEAVEEXEC_IMAGE)
-	rm -f $(WEAVER_EXE) $(WEAVEDNS_EXE) $(SIGPROXY_EXE) $(WEAVEPROXY_EXE) $(WEAVEWAIT_EXE) $(WEAVER_EXPORT) $(WEAVEDNS_EXPORT) $(WEAVEEXEC_EXPORT)
+	-$(SUDO) docker rmi $(IMAGES)
+	rm -f $(EXES) $(EXPORTS)
+	rm -f test/tls/*.pem
 
 build:
 	$(SUDO) go clean -i net

README.md

@@ -1,4 +1,4 @@
-# Weave - the Docker network
+# Weave - weaving containers into applications
 
 [![Build Status](https://travis-ci.org/weaveworks/weave.svg?branch=master)](https://travis-ci.org/weaveworks/weave) [![Integration Tests](https://circleci.com/gh/weaveworks/weave/tree/master.svg?style=shield&circle-token=4933c7dabb3d0383e62117565cb9d16df7b1a811)](https://circleci.com/gh/weaveworks/weave) [![Coverage Status](https://coveralls.io/repos/weaveworks/weave/badge.svg)](https://coveralls.io/r/weaveworks/weave)
 
@@ -32,8 +32,7 @@ capabilities, so these can continue to be used by containers.
 Ensure you are running Linux (kernel 3.8 or later) and have Docker
 (version 1.3.1 or later) installed. Then install weave with
 
-    sudo wget -O /usr/local/bin/weave \
-      https://github.com/weaveworks/weave/releases/download/latest_release/weave
+    sudo curl -L git.io/weave -o /usr/local/bin/weave
     sudo chmod a+x /usr/local/bin/weave
 
 CoreOS users see [here](https://github.com/fintanr/weave-gs/blob/master/coreos-simple/user-data) for an example of installing weave using cloud-config.
@@ -56,47 +55,41 @@ two containers, one on each host.
 On `$HOST1` run
 
     host1$ weave launch
-    host1$ C=$(weave run 10.2.1.1/24 -t -i ubuntu)
+    host1$ weave launch-dns
+    host1$ weave run --name a1 -ti ubuntu
 
-The first line starts the weave router, in a container. This needs to
-be done once on each host. The required docker image for the weave
-router container is downloaded automatically. There is also a `weave
-setup` command for downloading this and other images required for
+The first two lines start the weave router and DNS, each inside their
+own container - this needs to be done once per host. Whilst the
+required docker images are downloaded automatically, there is also a
+`weave setup` command for downloading all the images required for
 weave operation; this is a strictly optional step which is especially
 useful for automated installation of weave and ensures that any
 subsequent weave commands do not encounter delays due to image
 downloading.
 
-The second line runs our application container. We give it an IP
-address and network, in
-[CIDR notation](http://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing#CIDR_notation).
-`weave run` invokes `docker run -d` with all the parameters following
-the IP address and network. So we could be launching any container
-this way; here we just take a stock ubuntu container and launch a
-shell in it. There's also a `weave start` command, which invokes
-`docker start` for starting existing containers.
+The third line runs our application container, allocating it an IP
+address on the weave network and registering it in DNS.  `weave run`
+invokes `docker run -d` behind the scenes, so we could be launching
+any container this way; here we just take a stock ubuntu container and
+launch a shell in it.  There's also a `weave start` command, which
+invokes `docker start` for starting existing containers.
 
 If our application consists of more than one container on this host we
-simply launch them with a variation on that second line.
+simply launch them with a variation on third second line.
 
-The IP addresses and netmasks can be anything you like, but make sure
-they don't conflict with any IP ranges in use on the hosts or IP
-addresses of external services the hosts or containers need to connect
-to. The same IP range must be used everywhere, and the individual IP
-addresses must, of course, be unique.
-
-We repeat similar steps on `$HOST2`...
+Next we repeat similar steps on `$HOST2`...
 
     host2$ weave launch $HOST1
-    host2$ C=$(weave run 10.2.1.2/24 -t -i ubuntu)
+    host2$ weave launch-dns
+    host2$ weave run --name a2 -ti ubuntu
 
-The only difference, apart from the choice of IP address for the
-application container, is that we tell our weave that it should peer
-with the weave on `$HOST1` (specified as the IP address or hostname,
-and optional `:port`, by which `$HOST2` can reach it). NB: if there is
-a firewall between `$HOST1` and `$HOST2`, you must open the weave port
-(6783 by default; this can be overriden by setting `WEAVE_PORT`) for
-TCP and UDP.
+The only difference, apart from the name of the application container,
+is that we tell our weave that it should peer with the weave on
+`$HOST1` (specified as the IP address or hostname, and optional
+`:port`, by which `$HOST2` can reach it). NB: if there is a firewall
+between `$HOST1` and `$HOST2`, you must open the weave port (6783 by
+default; this can be overriden by setting `WEAVE_PORT`) for TCP and
+UDP.
 
 Note that we could instead have told the weave on `$HOST1` to connect to
 `$HOST2`, or told both about each other. Order does not matter here;
@@ -110,21 +103,21 @@ can talk to each other...
 
 On `$HOST1`...
 
-    host1$ docker attach $C
-    root@28841bd02eff:/# ping -c 1 -q 10.2.1.2
-    PING 10.2.1.2 (10.2.1.2): 48 data bytes
-    --- 10.2.1.2 ping statistics ---
-    1 packets transmitted, 1 packets received, 0% packet loss
-    round-trip min/avg/max/stddev = 1.048/1.048/1.048/0.000 ms
+    host1$ docker attach a1
+    root@a1:/# ping -c 1 -q a2
+    PING a2.weave.local (10.160.0.2) 56(84) bytes of data.
+    --- a2.weave.local ping statistics ---
+    1 packets transmitted, 1 received, 0% packet loss, time 0ms
+    rtt min/avg/max/mdev = 0.341/0.341/0.341/0.000 ms
 
 Similarly, on `$HOST2`...
 
-    host2$ docker attach $C
-    root@f76829496120:/# ping -c 1 -q 10.2.1.1
-    PING 10.2.1.1 (10.2.1.1): 48 data bytes
-    --- 10.2.1.1 ping statistics ---
-    1 packets transmitted, 1 packets received, 0% packet loss
-    round-trip min/avg/max/stddev = 1.034/1.034/1.034/0.000 ms
+    host2$ docker attach a2
+    root@a2:/# ping -c 1 -q a1
+    PING a1.weave.local (10.128.0.2) 56(84) bytes of data.
+    --- a1.weave.local ping statistics ---
+    1 packets transmitted, 1 received, 0% packet loss, time 0ms
+    rtt min/avg/max/mdev = 0.366/0.366/0.366/0.000 ms
 
 So there we have it, two containers on separate hosts happily talking
 to each other.

bin/lint

@@ -28,7 +28,7 @@ function lint_go {
 	# don't have it installed.
 	if type golint >/dev/null 2>&1; then
 		# golint doesn't set an exit code it seems
-		lintoutput=$(golint "${filename}")
+		lintoutput=$(golint "${filename}" | grep -vE 'comment|dot imports')
 		if [ "$lintoutput" != "" ]; then
 			result=1
 			echo "$lintoutput"

bin/release

@@ -61,7 +61,7 @@ build() {
     fi
 
     ## Run tests with the distributables, including version check
-    v=$(./weaver/weaver --version | grep -o '[0-9].*')
+    v=$(./prog/weaver/weaver --version | grep -o '[0-9].*')
     if ! [ "$v" == "$VERSION" ]; then
         echo "Version of distributable "$v" does not match release version $VERSION" >&2
         exit 1

circle.yml

@@ -21,11 +21,13 @@ dependencies:
     - if [[ -e "$WEAVE_BUILD" ]]; then
         docker load -i $WEAVE_BUILD;
       else
-        docker pull weaveworks/weave-build;
+        docker build -t weaveworks/weave-build build;
         mkdir -p $(dirname "$WEAVE_BUILD");
         docker save weaveworks/weave-build >$WEAVE_BUILD;
       fi
   post:
+    - sudo apt-get install bc
+    - pip install requests
     - curl https://sdk.cloud.google.com | bash
     - bin/setup-circleci-secrets "$SECRET_PASSWORD"
     - mkdir -p $(dirname $SRCDIR)
@@ -41,6 +43,7 @@ test:
         parallel: true
     - cd $SRCDIR/test; . ./gce.sh hosts; ./run_all.sh:
         parallel: true
+        timeout: 300
   post:
     - cd $SRCDIR/test; ./gce.sh destroy:
         parallel: true

common/docker/client.go

@@ -0,0 +1,80 @@
+package docker
+
+import (
+	"regexp"
+
+	"github.com/fsouza/go-dockerclient"
+	. "github.com/weaveworks/weave/common"
+)
+
+// Regexp that matches container IDs
+var containerIDregexp = regexp.MustCompile("^[a-f0-9]+$")
+
+// IsContainerID returns True if the string provided is a valid container id
+func IsContainerID(idStr string) bool {
+	return containerIDregexp.MatchString(idStr)
+}
+
+// An observer for container events
+type ContainerObserver interface {
+	ContainerDied(ident string) error
+}
+
+type Client struct {
+	*docker.Client
+}
+
+// NewClient creates a new Docker client and checks we can talk to Docker
+func NewClient(apiPath string) (*Client, error) {
+	dc, err := docker.NewClient(apiPath)
+	if err != nil {
+		return nil, err
+	}
+	client := &Client{dc}
+
+	env, err := client.Version()
+	if err != nil {
+		return nil, err
+	}
+	Info.Printf("[docker] Using Docker API on %s: %v", apiPath, env)
+	return client, nil
+}
+
+// AddObserver adds an observer for docker events
+func (c *Client) AddObserver(ob ContainerObserver) error {
+	events := make(chan *docker.APIEvents)
+	if err := c.AddEventListener(events); err != nil {
+		Error.Printf("[docker] Unable to add listener to Docker API: %s", err)
+		return err
+	}
+
+	go func() {
+		for event := range events {
+			switch event.Status {
+			case "die":
+				id := event.ID
+				ob.ContainerDied(id)
+			}
+		}
+	}()
+	return nil
+}
+
+// IsContainerNotRunning returns true if we have checked with Docker that the ID is not running
+func (c *Client) IsContainerNotRunning(idStr string) bool {
+	if !IsContainerID(idStr) {
+		Debug.Printf("[docker] '%s' does not seem to be a container id", idStr)
+		return false
+	}
+
+	// check with Docker whether the container is really running
+	container, err := c.InspectContainer(idStr)
+	if err == nil {
+		return !container.State.Running
+	}
+	if _, notThere := err.(*docker.NoSuchContainer); notThere {
+		return true
+	}
+	Error.Printf("[docker] Could not check container status: %s", err)
+	return false
+}

common/logging.go

@@ -19,6 +19,7 @@ var (
 	Info    *log.Logger
 	Warning *log.Logger
 	Error   *log.Logger
+	debugF  bool
 )
 
 func InitLogging(debugHandle io.Writer,
@@ -33,6 +34,10 @@ func InitLogging(debugHandle io.Writer,
 }
 
 func InitDefaultLogging(debug bool) {
+	if debug == debugF {
+		return
+	}
+	debugF = debug
 	debugOut := ioutil.Discard
 	if debug {
 		debugOut = os.Stderr

common/sched_queue.go

@@ -0,0 +1,155 @@
+package common
+
+import (
+	"container/heap"
+	"math"
+	"time"
+
+	"sync/atomic"
+
+	"github.com/benbjohnson/clock"
+)
+
+// A callable is a function that is called and return an (optional) next scheduled time
+type callable func() time.Time
+
+type schedCall struct {
+	t time.Time
+	c callable
+}
+
+func (sc *schedCall) FromNow(now time.Time) time.Duration {
+	if sc.t.After(now) {
+		return time.Duration(sc.t.Sub(now).Nanoseconds())
+	}
+	return time.Duration(0)
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+// A min-heap of scheduled callables
+type schedCallsHeap []*schedCall
+
+func (ch *schedCallsHeap) Len() int           { return len(*ch) }
+func (ch *schedCallsHeap) Less(i, j int) bool { return (*ch)[i].t.Before((*ch)[j].t) }
+func (ch *schedCallsHeap) Swap(i, j int) {
+	(*ch)[i], (*ch)[j] = (*ch)[j], (*ch)[i]
+}
+
+func (ch *schedCallsHeap) Push(x interface{}) {
+	// Push and Pop use pointer receivers because they modify the slice's length,
+	// not just its contents.
+	entry := x.(*schedCall)
+	*ch = append(*ch, entry)
+}
+
+func (ch *schedCallsHeap) Pop() interface{} {
+	old := *ch
+	n := len(old)
+	item := old[n-1]
+	*ch = old[0 : n-1]
+	return item
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+// SchedQueue is queue of scheduled callables
+type SchedQueue struct {
+	clock      clock.Clock
+	callablesH schedCallsHeap
+	schedChan  chan *schedCall
+	closeChan  chan bool
+	counter    uint64 // number of calls invoked so far (used for stats). Note: it will wrap.
+}
+
+// NewSchedQueue creates a new scheduled queue
+func NewSchedQueue(clock clock.Clock) *SchedQueue {
+	cq := SchedQueue{
+		clock:     clock,
+		schedChan: make(chan *schedCall),
+		closeChan: make(chan bool),
+	}
+	heap.Init(&cq.callablesH)
+	return &cq
+}
+
+// Start starts the scheduled queue
+func (cq *SchedQueue) Start() {
+	go func() {
+		defer func() { Debug.Printf("[sched-q] Quitting (%d calls pending)", len(cq.callablesH)) }()
+
+		var now time.Time
+		timer := cq.clock.Timer(time.Duration(math.MaxInt64))
+
+		for {
+			// Wait until an insertion, until the next callable or until we are Stop()ed
+			select {
+			case ce := <-cq.schedChan:
+				heap.Push(&cq.callablesH, ce)
+				now = cq.clock.Now()
+			case <-cq.closeChan:
+				return
+			case now = <-timer.C:
+			}
+
+			timer.Stop()
+			durationUntilNext := cq.durationUntilNext(now)
+			for durationUntilNext == 0 {
+				sched := heap.Pop(&cq.callablesH).(*schedCall)
+
+				atomic.AddUint64(&cq.counter, 1)
+				schedNextTime := sched.c()
+
+				if !schedNextTime.IsZero() {
+					sched.t = schedNextTime
+					heap.Push(&cq.callablesH, sched) // TODO: use a Fix() instead of Pop()&Push()
+				}
+
+				now = cq.clock.Now()
+				durationUntilNext = cq.durationUntilNext(now)
+			}
+
+			timer = cq.clock.Timer(durationUntilNext)
+		}
+	}()
+}
+
+// Stop stops the scheduled queue
+func (cq *SchedQueue) Stop() {
+	Debug.Printf("[sched-q] Stopping...")
+	cq.closeChan <- true
+}
+
+// Add schedules a call.
+// The callable should not modify the scheduled queue in any way.
+func (cq *SchedQueue) Add(c callable, t time.Time) {
+	Debug.Printf("[sched-q] Adding call at %s", t)
+	ce := schedCall{c: c, t: t}
+	cq.schedChan <- &ce
+}
+
+// Counter returns the number of calls invoked in the queued
+// Note: the result will wrap over time.
+func (cq *SchedQueue) Count() uint64 {
+	return atomic.LoadUint64(&cq.counter)
+}
+
+func (cq *SchedQueue) durationUntilNext(now time.Time) time.Duration {
+	if len(cq.callablesH) > 0 {
+		firstSched := cq.callablesH[0]
+		return firstSched.FromNow(now)
+	}
+	return time.Duration(math.MaxInt64)
+}
+
+// ensure that the background goroutine has executed any pending callbacks
+func (cq *SchedQueue) Flush() {
+	ch := make(chan struct{})
+	flushFunc := func() time.Time {
+		atomic.AddUint64(&cq.counter, ^uint64(0)) // decrement so nobody sees this
+		close(ch)
+		return time.Time{}
+	}
+	cq.schedChan <- &schedCall{c: flushFunc, t: time.Time{}}
+	<-ch
+}

common/sched_queue_test.go

@@ -0,0 +1,92 @@
+package common
+
+import (
+	"testing"
+	"time"
+
+	"github.com/benbjohnson/clock"
+	wt "github.com/weaveworks/weave/testing"
+)
+
+// Ensure we can add new calls while forwarding the clock
+func TestSchedCallsBasic(t *testing.T) {
+	InitDefaultLogging(testing.Verbose())
+	Info.Println("TestSchedCallsBasic starting")
+
+	const testSecs = 1000
+	clk := clock.NewMock()
+	schedQueue := NewSchedQueue(clk)
+	schedQueue.Start()
+	defer schedQueue.Stop()
+
+	c := func() time.Time {
+		return clk.Now().Add(time.Second)
+	}
+
+	schedQueue.Add(c, clk.Now().Add(time.Second))
+	schedQueue.Flush()
+	for i := 0; i < testSecs; i++ {
+		clk.Add(time.Second)
+		schedQueue.Flush()
+	}
+
+	t.Logf("Now: %s - calls: %d", clk.Now(), schedQueue.Count())
+	wt.AssertEqualInt(t, (int)(schedQueue.Count()), testSecs, "Number of calls")
+}
+
+// Ensure we can create a 100 seconds gap in the middle of the time travel
+func TestSchedCallsGap(t *testing.T) {
+	InitDefaultLogging(testing.Verbose())
+	Info.Println("TestSchedCallsGap starting")
+
+	const testSecs = 1000
+	clk := clock.NewMock()
+	schedQueue := NewSchedQueue(clk)
+	schedQueue.Start()
+	defer schedQueue.Stop()
+
+	c2 := func() time.Time {
+		if schedQueue.Count() == testSecs/2 {
+			return clk.Now().Add(time.Duration(100) * time.Second)
+		}
+		return clk.Now().Add(time.Second)
+	}
+
+	schedQueue.Add(c2, clk.Now().Add(time.Second))
+	schedQueue.Flush()
+	for i := 0; i < testSecs; i++ {
+		clk.Add(time.Second)
+		schedQueue.Flush()
+	}
+
+	t.Logf("Now: %s - calls: %d", clk.Now(), schedQueue.Count())
+	wt.AssertEqualInt(t, (int)(schedQueue.Count()), testSecs-100+1, "Number of calls")
+}
+
+func TestSchedCallsStop(t *testing.T) {
+	InitDefaultLogging(testing.Verbose())
+	Info.Println("TestSchedCallsStop starting")
+
+	const testSecs = 1000
+	clk := clock.NewMock()
+	schedQueue := NewSchedQueue(clk)
+	schedQueue.Start()
+	defer schedQueue.Stop()
+
+	c2 := func() time.Time {
+		if schedQueue.Count() == testSecs/2 {
+			return time.Time{}
+		}
+		return clk.Now().Add(time.Second)
+	}
+
+	schedQueue.Add(c2, clk.Now().Add(time.Second))
+	schedQueue.Flush()
+	for i := 0; i < testSecs; i++ {
+		clk.Add(time.Second)
+		schedQueue.Flush()
+	}
+
+	t.Logf("Now: %s - calls: %d", clk.Now(), schedQueue.Count())
+	wt.AssertEqualInt(t, (int)(schedQueue.Count()), testSecs/2, "Number of calls")
+}

common/updater/updater.go

@@ -1,47 +0,0 @@
-package updater
-
-import (
-	"github.com/fsouza/go-dockerclient"
-	. "github.com/weaveworks/weave/common"
-)
-
-type ContainerObserver interface {
-	ContainerDied(ident string) error
-}
-
-func checkError(err error, apiPath string) {
-	if err != nil {
-		Error.Fatalf("[updater] Unable to connect to Docker API on %s: %s",
-			apiPath, err)
-	}
-}
-
-func Start(apiPath string, ob ContainerObserver) error {
-	client, err := docker.NewClient(apiPath)
-	checkError(err, apiPath)
-
-	env, err := client.Version()
-	checkError(err, apiPath)
-
-	events := make(chan *docker.APIEvents)
-	err = client.AddEventListener(events)
-	checkError(err, apiPath)
-
-	Info.Printf("[updater] Using Docker API on %s: %v", apiPath, env)
-
-	go func() {
-		for event := range events {
-			handleEvent(ob, event, client)
-		}
-	}()
-	return nil
-}
-
-func handleEvent(ob ContainerObserver, event *docker.APIEvents, client *docker.Client) error {
-	switch event.Status {
-	case "die":
-		id := event.ID
-		ob.ContainerDied(id)
-	}
-	return nil
-}

common/utils.go

@@ -0,0 +1,8 @@
+package common
+
+func OnOff(b bool) string {
+	if b {
+		return "on"
+	}
+	return "off"
+}

docs/architecture.txt

@@ -34,7 +34,7 @@ Router:
   the known locations of peers as they're discovered and is informed
   when connections we have made die. It proactively tries to create
   connections to remote peers we're not connected to using a random
-  binary exponential backoff to regulate the period between connection
+  exponential backoff to regulate the period between connection
   attempts.
 
 - Local Peer

docs/ipam.md

@@ -20,11 +20,16 @@ plugin) uses to request IP addresses.
 
 The commands supported by the allocator via the http interface are:
 
-- Allocate: request one IP address for a container
+- Allocate: request one IP address for a container in a subnet
+- Lookup: fetch the previously-allocated IP address for a container in a subnet
 - Free: return an IP address that is currently allocated
 - Claim: request a specific IP address for a container (e.g. because
   it is already using that address)
 
+Each http request either specifies a subnet, or if no subnet is
+specified this is taken as a request to allocate in a pre-defined
+default subnet.
+
 The allocator also watches via the Docker event mechanism: if a
 container dies then all IP addresses allocated to that container are
 freed.
@@ -48,16 +53,17 @@ freed.
 
 When a peer owns some range(s), it can allocate freely from within
 those ranges to containers on the same machine. If it runs out of
-space (all owned ranges are full), it will ask another peer for space:
+space in a subnet (all owned ranges are full), it will ask another
+peer for space:
   - it picks a peer to ask at random, weighted by the amount of space
-    owned by each peer
+    owned by each peer in the subnet
     - if the target peer decides to give up space, it unicasts a message
       back to the asker with the newly-updated ring.
     - if the target peer has no space, it unicasts a message back to the
       asker with its current copy of the ring, on the basis that the
       requestor must have acted on out-of-date information.
   - it will continue to ask for space until it receives some, or its
-    copy of the ring tells it all peers are full.
+    copy of the ring tells it all peers are full in that subnet.
 
 ### Claiming an address
 

ipam/address/address.go

@@ -1,8 +1,10 @@
 package address
 
 import (
-	"github.com/weaveworks/weave/common"
+	"fmt"
 	"net"
+
+	"github.com/weaveworks/weave/common"
 )
 
 // Using 32-bit integer to represent IPv4 address
@@ -13,12 +15,49 @@ type Range struct {
 	Start, End Address // [Start, End); Start <= End
 }
 
+func NewRange(start Address, size Offset) Range {
+	return Range{Start: start, End: Add(start, size)}
+}
+func (r Range) Size() Offset               { return Subtract(r.End, r.Start) }
+func (r Range) String() string             { return fmt.Sprintf("[%s-%s)", r.Start, r.End) }
+func (r Range) Overlaps(or Range) bool     { return !(r.Start >= or.End || r.End <= or.Start) }
+func (r Range) Contains(addr Address) bool { return addr >= r.Start && addr < r.End }
+
+type CIDR struct {
+	Start     Address
+	PrefixLen int
+}
+
 func ParseIP(s string) (Address, error) {
-	if ip := net.ParseIP(s); ip == nil {
-		return 0, &net.ParseError{Type: "IP Address", Text: s}
-	} else {
+	if ip := net.ParseIP(s); ip != nil {
 		return FromIP4(ip), nil
 	}
+	return 0, &net.ParseError{Type: "IP Address", Text: s}
+}
+
+func ParseCIDR(s string) (Address, CIDR, error) {
+	if ip, ipnet, err := net.ParseCIDR(s); err != nil {
+		return 0, CIDR{}, err
+	} else if ipnet.IP.To4() == nil {
+		return 0, CIDR{}, &net.ParseError{Type: "Non-IPv4 address not supported", Text: s}
+	} else {
+		prefixLen, _ := ipnet.Mask.Size()
+		return FromIP4(ip), CIDR{Start: FromIP4(ipnet.IP), PrefixLen: prefixLen}, nil
+	}
+}
+
+func (cidr CIDR) Size() Offset { return 1 << uint(32-cidr.PrefixLen) }
+
+func (cidr CIDR) Range() Range {
+	return NewRange(cidr.Start, cidr.Size())
+}
+func (cidr CIDR) HostRange() Range {
+	// Respect RFC1122 exclusions of first and last addresses
+	return NewRange(cidr.Start+1, cidr.Size()-2)
+}
+
+func (cidr CIDR) String() string {
+	return fmt.Sprintf("%s/%d", cidr.Start.String(), cidr.PrefixLen)
 }
 
 // FromIP4 converts an ipv4 address to our integer address type

ipam/allocate.go

@@ -12,8 +12,9 @@ type allocateResult struct {
 
 type allocate struct {
 	resultChan       chan<- allocateResult
-	hasBeenCancelled func() bool
 	ident            string
+	r                address.Range // Range we are trying to allocate within
+	hasBeenCancelled func() bool
 }
 
 // Try returns true if the request is completed, false if pending
@@ -23,23 +24,27 @@ func (g *allocate) Try(alloc *Allocator) bool {
 		return true
 	}
 
-	// If we have previously stored an address for this container, return it.
-	if addr, found := alloc.owned[g.ident]; found {
+	if addr, found := alloc.lookupOwned(g.ident, g.r); found {
 		g.resultChan <- allocateResult{addr, nil}
 		return true
 	}
 
-	if ok, addr := alloc.space.Allocate(); ok {
-		alloc.debugln("Allocated", addr, "for", g.ident)
+	if !alloc.universe.Overlaps(g.r) {
+		g.resultChan <- allocateResult{0, fmt.Errorf("range %s out of bounds: %s", g.r, alloc.universe)}
+		return true
+	}
+
+	if ok, addr := alloc.space.Allocate(g.r); ok {
+		alloc.debugln("Allocated", addr, "for", g.ident, "in", g.r)
 		alloc.addOwned(g.ident, addr)
 		g.resultChan <- allocateResult{addr, nil}
 		return true
 	}
 
 	// out of space
-	if donor, err := alloc.ring.ChoosePeerToAskForSpace(); err == nil {
-		alloc.debugln("Decided to ask peer", donor, "for space")
-		alloc.sendRequest(donor, msgSpaceRequest)
+	if donor, err := alloc.ring.ChoosePeerToAskForSpace(g.r.Start, g.r.End); err == nil {
+		alloc.debugln("Decided to ask peer", donor, "for space in range", g.r)
+		alloc.sendSpaceRequest(donor, g.r)
 	}
 
 	return false

ipam/allocator.go

@@ -3,9 +3,7 @@ package ipam
 import (
 	"bytes"
 	"encoding/gob"
-	"errors"
 	"fmt"
-	"net"
 	"sort"
 	"time"
 
@@ -23,6 +21,7 @@ const (
 	msgRingUpdate
 
 	paxosInterval = time.Second * 5
+	MinSubnetSize = 4 // first and last addresses are excluded, so 2 would be too small
 )
 
 // operation represents something which Allocator wants to do, but
@@ -46,16 +45,14 @@ type operation interface {
 type Allocator struct {
 	actionChan       chan<- func()
 	ourName          router.PeerName
-	subnetStart      address.Address            // start address of space all peers are allocating from
-	subnetSize       address.Offset             // length of space all peers are allocating from
-	prefixLen        int                        // network prefix length, e.g. 24 for a /24 network
-	ring             *ring.Ring                 // information on ranges owned by all peers
-	space            space.Space                // more detail on ranges owned by us
-	owned            map[string]address.Address // who owns what address, indexed by container-ID
-	nicknames        map[router.PeerName]string // so we can map nicknames for rmpeer
-	pendingAllocates []operation                // held until we get some free space
-	pendingClaims    []operation                // held until we know who owns the space
-	gossip           router.Gossip              // our link to the outside world for sending messages
+	universe         address.Range                // superset of all ranges
+	ring             *ring.Ring                   // information on ranges owned by all peers
+	space            space.Space                  // more detail on ranges owned by us
+	owned            map[string][]address.Address // who owns what addresses, indexed by container-ID
+	nicknames        map[router.PeerName]string   // so we can map nicknames for rmpeer
+	pendingAllocates []operation                  // held until we get some free space
+	pendingClaims    []operation                  // held until we know who owns the space
+	gossip           router.Gossip                // our link to the outside world for sending messages
 	paxos            *paxos.Node
 	paxosTicker      *time.Ticker
 	shuttingDown     bool // to avoid doing any requests while trying to shut down
@@ -63,34 +60,16 @@ type Allocator struct {
 }
 
 // NewAllocator creates and initialises a new Allocator
-func NewAllocator(ourName router.PeerName, ourUID router.PeerUID, ourNickname string, subnetCIDR string, quorum uint) (*Allocator, error) {
-	_, subnet, err := net.ParseCIDR(subnetCIDR)
-	if err != nil {
-		return nil, err
-	}
-	if subnet.IP.To4() == nil {
-		return nil, errors.New("Non-IPv4 address not supported")
-	}
-	// Get the size of the network from the mask
-	ones, bits := subnet.Mask.Size()
-	var subnetSize address.Offset = 1 << uint(bits-ones)
-	if subnetSize < 4 {
-		return nil, errors.New("Allocation subnet too small")
-	}
-	subnetStart := address.FromIP4(subnet.IP)
-	alloc := &Allocator{
-		ourName:     ourName,
-		subnetStart: subnetStart,
-		subnetSize:  subnetSize,
-		prefixLen:   ones,
-		// per RFC 1122, don't allocate the first and last address in the subnet
-		ring:      ring.New(address.Add(subnetStart, 1), address.Add(subnetStart, subnetSize-1), ourName),
-		owned:     make(map[string]address.Address),
+func NewAllocator(ourName router.PeerName, ourUID router.PeerUID, ourNickname string, universe address.Range, quorum uint) *Allocator {
+	return &Allocator{
+		ourName:   ourName,
+		universe:  universe,
+		ring:      ring.New(universe.Start, address.Add(universe.Start, universe.Size()), ourName),
+		owned:     make(map[string][]address.Address),
 		paxos:     paxos.NewNode(ourName, ourUID, quorum),
 		nicknames: map[router.PeerName]string{ourName: ourNickname},
 		now:       time.Now,
 	}
-	return alloc, nil
 }
 
 // Start runs the allocator goroutine
@@ -197,17 +176,31 @@ func hasBeenCancelled(cancelChan <-chan bool) func() bool {
 
 // Actor client API
 
-// Allocate (Sync) - get IP address for container with given name
-// if there isn't any space we block indefinitely
-func (alloc *Allocator) Allocate(ident string, cancelChan <-chan bool) (address.Address, error) {
+// Allocate (Sync) - get new IP address for container with given name in range
+// if there isn't any space in that range we block indefinitely
+func (alloc *Allocator) Allocate(ident string, r address.Range, cancelChan <-chan bool) (address.Address, error) {
 	resultChan := make(chan allocateResult)
-	op := &allocate{resultChan: resultChan, ident: ident,
+	op := &allocate{resultChan: resultChan, ident: ident, r: r,
 		hasBeenCancelled: hasBeenCancelled(cancelChan)}
 	alloc.doOperation(op, &alloc.pendingAllocates)
 	result := <-resultChan
 	return result.addr, result.err
 }
 
+// Lookup (Sync) - get existing IP address for container with given name in range
+func (alloc *Allocator) Lookup(ident string, r address.Range) (address.Address, error) {
+	resultChan := make(chan allocateResult)
+	alloc.actionChan <- func() {
+		if addr, found := alloc.lookupOwned(ident, r); found {
+			resultChan <- allocateResult{addr: addr}
+			return
+		}
+		resultChan <- allocateResult{err: fmt.Errorf("lookup: no address found for %s in range %s", ident, r)}
+	}
+	result := <-resultChan
+	return result.addr, result.err
+}
+
 // Claim an address that we think we should own (Sync)
 func (alloc *Allocator) Claim(ident string, addr address.Address, cancelChan <-chan bool) error {
 	resultChan := make(chan error)
@@ -217,22 +210,21 @@ func (alloc *Allocator) Claim(ident string, addr address.Address, cancelChan <-c
 	return <-resultChan
 }
 
-// Free (Sync) - release IP address for container with given name
-func (alloc *Allocator) Free(ident string) error {
-	return alloc.free(ident)
-}
-
-// ContainerDied is provided to satisfy the updater interface; does a free underneath.  Async.
+// ContainerDied is provided to satisfy the updater interface; does a 'Delete' underneath.  Sync.
 func (alloc *Allocator) ContainerDied(ident string) error {
-	alloc.debugln("Container", ident, "died; releasing addresses")
-	return alloc.free(ident)
+	err := alloc.Delete(ident)
+	if err == nil {
+		alloc.debugln("Container", ident, "died; released addresses")
+	}
+	return err
 }
 
-func (alloc *Allocator) free(ident string) error {
+// Delete (Sync) - release all IP addresses for container with given name
+func (alloc *Allocator) Delete(ident string) error {
 	errChan := make(chan error)
 	alloc.actionChan <- func() {
-		addr, found := alloc.owned[ident]
-		if found {
+		addrs, found := alloc.owned[ident]
+		for _, addr := range addrs {
 			alloc.space.Free(addr)
 		}
 		delete(alloc.owned, ident)
@@ -242,7 +234,7 @@ func (alloc *Allocator) free(ident string) error {
 		found = alloc.cancelOpsFor(&alloc.pendingClaims, ident) || found
 
 		if !found {
-			errChan <- fmt.Errorf("Free: no addresses for %s", ident)
+			errChan <- fmt.Errorf("Delete: no addresses for %s", ident)
 			return
 		}
 		errChan <- nil
@@ -250,6 +242,30 @@ func (alloc *Allocator) free(ident string) error {
 	return <-errChan
 }
 
+// Free (Sync) - release single IP address for container
+func (alloc *Allocator) Free(ident string, addrToFree address.Address) error {
+	errChan := make(chan error)
+	alloc.actionChan <- func() {
+		addrs := alloc.owned[ident]
+		for i, ownedAddr := range addrs {
+			if ownedAddr == addrToFree {
+				alloc.debugln("Freed", addrToFree, "for", ident)
+				if len(addrs) == 1 {
+					delete(alloc.owned, ident)
+				} else {
+					alloc.owned[ident] = append(addrs[:i], addrs[i+1:]...)
+				}
+				alloc.space.Free(addrToFree)
+				errChan <- nil
+				return
+			}
+		}
+
+		errChan <- fmt.Errorf("Free: address %s not found for %s", addrToFree, ident)
+	}
+	return <-errChan
+}
+
 // Sync.
 func (alloc *Allocator) String() string {
 	resultChan := make(chan string)
@@ -296,7 +312,7 @@ func (alloc *Allocator) AdminTakeoverRanges(peerNameOrNickname string) error {
 		}
 
 		delete(alloc.nicknames, peername)
-		err, newRanges := alloc.ring.Transfer(peername, alloc.ourName)
+		newRanges, err := alloc.ring.Transfer(peername, alloc.ourName)
 		alloc.space.AddRanges(newRanges)
 		resultChan <- err
 	}
@@ -320,7 +336,7 @@ func (alloc *Allocator) lookupPeername(name string) (router.PeerName, error) {
 // Restrict the peers in "nicknames" to those in the ring and our own
 func (alloc *Allocator) pruneNicknames() {
 	ringPeers := alloc.ring.PeerNames()
-	for name, _ := range alloc.nicknames {
+	for name := range alloc.nicknames {
 		if _, ok := ringPeers[name]; !ok && name != alloc.ourName {
 			delete(alloc.nicknames, name)
 		}
@@ -335,7 +351,13 @@ func (alloc *Allocator) OnGossipUnicast(sender router.PeerName, msg []byte) erro
 		switch msg[0] {
 		case msgSpaceRequest:
 			// some other peer asked us for space
-			alloc.donateSpace(sender)
+			decoder := gob.NewDecoder(bytes.NewReader(msg[1:]))
+			var r address.Range
+			if err := decoder.Decode(&r); err != nil {
+				resultChan <- err
+				return
+			}
+			alloc.donateSpace(r, sender)
 			resultChan <- nil
 		case msgRingUpdate:
 			resultChan <- alloc.update(msg[1:])
@@ -356,7 +378,7 @@ func (alloc *Allocator) OnGossipBroadcast(msg []byte) (router.GossipData, error)
 
 type gossipState struct {
 	// We send a timstamp along with the information to be
-	// gossipped in order to do detect skewed clocks
+	// gossipped in order to detect skewed clocks
 	Now       int64
 	Nicknames map[router.PeerName]string
 
@@ -413,8 +435,8 @@ func (d *ipamGossipData) Merge(other router.GossipData) {
 	// no-op
 }
 
-func (d *ipamGossipData) Encode() []byte {
-	return d.alloc.Encode()
+func (d *ipamGossipData) Encode() [][]byte {
+	return [][]byte{d.alloc.Encode()}
 }
 
 // Gossip returns a GossipData implementation, which in this case always
@@ -456,20 +478,14 @@ func (alloc *Allocator) actorLoop(actionChan <-chan func()) {
 
 func (alloc *Allocator) string() string {
 	var buf bytes.Buffer
-	fmt.Fprintf(&buf, "Allocator subnet %s/%d", alloc.subnetStart.String(), alloc.prefixLen)
+	fmt.Fprintf(&buf, "Allocator range %s", alloc.universe)
 
 	if alloc.ring.Empty() {
 		if alloc.paxosTicker != nil {
 			fmt.Fprintf(&buf, " awaiting consensus: %s", alloc.paxos.String())
 		}
 	} else {
-		localFreeSpace := alloc.space.NumFreeAddresses()
-		remoteFreeSpace := alloc.ring.TotalRemoteFree()
-		percentFree := 100 * float64(localFreeSpace+remoteFreeSpace) / float64(alloc.subnetSize)
-		fmt.Fprintf(&buf, "  Free IPs: ~%.1f%%, %d local, ~%d remote\n",
-			percentFree, localFreeSpace, remoteFreeSpace)
-
-		fmt.Fprint(&buf, "Owned Ranges:")
+		fmt.Fprint(&buf, "\nOwned Ranges:")
 		alloc.ring.FprintWithNicknames(&buf, alloc.nicknames)
 	}
 	if len(alloc.pendingAllocates)+len(alloc.pendingClaims) > 0 {
@@ -484,11 +500,6 @@ func (alloc *Allocator) string() string {
 	return buf.String()
 }
 
-func (alloc *Allocator) consensus() []router.PeerName {
-	_, val := alloc.paxos.Consensus()
-	return val.Value
-}
-
 // Ensure we are making progress towards an established ring
 func (alloc *Allocator) establishRing() {
 	if !alloc.ring.Empty() || alloc.paxosTicker != nil {
@@ -496,10 +507,10 @@ func (alloc *Allocator) establishRing() {
 	}
 
 	alloc.propose()
-	if cons := alloc.consensus(); cons != nil {
+	if ok, cons := alloc.paxos.Consensus(); ok {
 		// If the quorum was 1, then proposing immediately
 		// leads to consensus
-		alloc.createRing(cons)
+		alloc.createRing(cons.Value)
 	} else {
 		// re-try until we get consensus
 		alloc.paxosTicker = time.NewTicker(paxosInterval)
@@ -564,8 +575,18 @@ func (alloc *Allocator) propose() {
 	alloc.gossip.GossipBroadcast(alloc.Gossip())
 }
 
-func (alloc *Allocator) sendRequest(dest router.PeerName, kind byte) {
-	msg := router.Concat([]byte{kind}, alloc.encode())
+func (alloc *Allocator) sendSpaceRequest(dest router.PeerName, r address.Range) {
+	buf := new(bytes.Buffer)
+	enc := gob.NewEncoder(buf)
+	if err := enc.Encode(r); err != nil {
+		panic(err)
+	}
+	msg := router.Concat([]byte{msgSpaceRequest}, buf.Bytes())
+	alloc.gossip.GossipUnicast(dest, msg)
+}
+
+func (alloc *Allocator) sendRingUpdate(dest router.PeerName) {
+	msg := router.Concat([]byte{msgRingUpdate}, alloc.encode())
 	alloc.gossip.GossipUnicast(dest, msg)
 }
 
@@ -608,8 +629,8 @@ func (alloc *Allocator) update(msg []byte) error {
 				alloc.gossip.GossipBroadcast(alloc.Gossip())
 			}
 
-			if cons := alloc.consensus(); cons != nil {
-				alloc.createRing(cons)
+			if ok, cons := alloc.paxos.Consensus(); ok {
+				alloc.createRing(cons.Value)
 			}
 		}
 	}
@@ -617,25 +638,24 @@ func (alloc *Allocator) update(msg []byte) error {
 	return nil
 }
 
-func (alloc *Allocator) donateSpace(to router.PeerName) {
+func (alloc *Allocator) donateSpace(r address.Range, to router.PeerName) {
 	// No matter what we do, we'll send a unicast gossip
 	// of our ring back to tha chap who asked for space.
 	// This serves to both tell him of any space we might
 	// have given him, or tell him where he might find some
 	// more.
-	defer alloc.sendRequest(to, msgRingUpdate)
+	defer alloc.sendRingUpdate(to)
 
 	alloc.debugln("Peer", to, "asked me for space")
-	start, size, ok := alloc.space.Donate()
+	chunk, ok := alloc.space.Donate(r)
 	if !ok {
-		free := alloc.space.NumFreeAddresses()
+		free := alloc.space.NumFreeAddressesInRange(r)
 		common.Assert(free == 0)
 		alloc.debugln("No space to give to peer", to)
 		return
 	}
-	end := address.Add(start, size)
-	alloc.debugln("Giving range", start, end, size, "to", to)
-	alloc.ring.GrantRangeToHost(start, end, to)
+	alloc.debugln("Giving range", chunk, "to", to)
+	alloc.ring.GrantRangeToHost(chunk.Start, chunk.End, to)
 }
 
 func (alloc *Allocator) assertInvariants() {
@@ -663,21 +683,33 @@ func (alloc *Allocator) reportFreeSpace() {
 
 	freespace := make(map[address.Address]address.Offset)
 	for _, r := range ranges {
-		freespace[r.Start] = alloc.space.NumFreeAddressesInRange(r.Start, r.End)
+		freespace[r.Start] = alloc.space.NumFreeAddressesInRange(r)
 	}
 	alloc.ring.ReportFree(freespace)
 }
 
 // Owned addresses
 
+// NB: addr must not be owned by ident already
 func (alloc *Allocator) addOwned(ident string, addr address.Address) {
-	alloc.owned[ident] = addr
+	alloc.owned[ident] = append(alloc.owned[ident], addr)
+}
+
+func (alloc *Allocator) lookupOwned(ident string, r address.Range) (address.Address, bool) {
+	for _, addr := range alloc.owned[ident] {
+		if r.Contains(addr) {
+			return addr, true
+		}
+	}
+	return 0, false
 }
 
 func (alloc *Allocator) findOwner(addr address.Address) string {
-	for ident, candidate := range alloc.owned {
-		if candidate == addr {
-			return ident
+	for ident, addrs := range alloc.owned {
+		for _, candidate := range addrs {
+			if candidate == addr {
+				return ident
+			}
 		}
 	}
 	return ""

ipam/allocator_test.go

@@ -24,37 +24,50 @@ func TestAllocFree(t *testing.T) {
 		container1 = "abcdef"
 		container2 = "baddf00d"
 		container3 = "b01df00d"
-		universe   = "10.0.3.0/28"
-		testAddr1  = "10.0.3.1" // first address allocated should be .1 because .0 is network addr
-		spaceSize  = 14         // 16 IP addresses in /28, minus .0 and .15
+		universe   = "10.0.3.0/26"
+		subnet1    = "10.0.3.0/28"
+		subnet2    = "10.0.3.32/28"
+		testAddr1  = "10.0.3.1"
+		testAddr2  = "10.0.3.33"
+		spaceSize  = 62 // 64 IP addresses in /26, minus .0 and .63
 	)
 
-	alloc := makeAllocatorWithMockGossip(t, "01:00:00:01:00:00", universe, 1)
+	alloc, subnet := makeAllocatorWithMockGossip(t, "01:00:00:01:00:00", universe, 1)
 	defer alloc.Stop()
+	_, cidr1, _ := address.ParseCIDR(subnet1)
+	_, cidr2, _ := address.ParseCIDR(subnet2)
 
 	alloc.claimRingForTesting()
-	addr1, _ := alloc.Allocate(container1, nil)
+	addr1, err := alloc.Allocate(container1, cidr1.HostRange(), nil)
+	wt.AssertNoErr(t, err)
 	wt.AssertEqualString(t, addr1.String(), testAddr1, "address")
 
+	addr2, err := alloc.Allocate(container1, cidr2.HostRange(), nil)
+	wt.AssertNoErr(t, err)
+	wt.AssertEqualString(t, addr2.String(), testAddr2, "address")
+
 	// Ask for another address for a different container and check it's different
-	addr2, _ := alloc.Allocate(container2, nil)
-	if addr2.String() == testAddr1 {
-		t.Fatalf("Expected different address but got %s", addr2.String())
+	addr1b, _ := alloc.Allocate(container2, cidr1.HostRange(), nil)
+	if addr1b.String() == testAddr1 {
+		t.Fatalf("Expected different address but got %s", addr1b.String())
 	}
 
-	// Ask for the first container again and we should get the same address again
-	addr1a, _ := alloc.Allocate(container1, nil)
+	// Ask for the first container again and we should get the same addresses again
+	addr1a, _ := alloc.Allocate(container1, cidr1.HostRange(), nil)
 	wt.AssertEqualString(t, addr1a.String(), testAddr1, "address")
+	addr2a, _ := alloc.Allocate(container1, cidr2.HostRange(), nil)
+	wt.AssertEqualString(t, addr2a.String(), testAddr2, "address")
 
-	// Now free the first one, and we should get it back when we ask
-	wt.AssertSuccess(t, alloc.Free(container1))
-	addr3, _ := alloc.Allocate(container3, nil)
+	// Now delete the first container, and we should get its addresses back
+	wt.AssertSuccess(t, alloc.Delete(container1))
+	addr3, _ := alloc.Allocate(container3, cidr1.HostRange(), nil)
 	wt.AssertEqualString(t, addr3.String(), testAddr1, "address")
+	addr4, _ := alloc.Allocate(container3, cidr2.HostRange(), nil)
+	wt.AssertEqualString(t, addr4.String(), testAddr2, "address")
 
 	alloc.ContainerDied(container2)
 	alloc.ContainerDied(container3)
-	alloc.String() // force sync-up after async call
-	wt.AssertEquals(t, alloc.space.NumFreeAddresses(), address.Offset(spaceSize))
+	wt.AssertEquals(t, alloc.NumFreeAddresses(subnet), address.Offset(spaceSize))
 }
 
 func TestBootstrap(t *testing.T) {
@@ -66,21 +79,21 @@ func TestBootstrap(t *testing.T) {
 		peerNameString = "02:00:00:02:00:00"
 	)
 
-	alloc1 := makeAllocatorWithMockGossip(t, ourNameString, testStart1+"/22", 2)
+	alloc1, subnet := makeAllocatorWithMockGossip(t, ourNameString, testStart1+"/22", 2)
 	defer alloc1.Stop()
 
 	// Simulate another peer on the gossip network
-	alloc2 := makeAllocatorWithMockGossip(t, peerNameString, testStart1+"/22", 2)
+	alloc2, _ := makeAllocatorWithMockGossip(t, peerNameString, testStart1+"/22", 2)
 	defer alloc2.Stop()
 
-	alloc1.OnGossipBroadcast(alloc2.encode())
+	alloc1.OnGossipBroadcast(alloc2.Encode())
 
-	alloc1.tryPendingOps()
+	alloc1.actionChan <- func() { alloc1.tryPendingOps() }
 
 	ExpectBroadcastMessage(alloc1, nil) // alloc1 will try to form consensus
 	done := make(chan bool)
 	go func() {
-		alloc1.Allocate("somecontainer", nil)
+		alloc1.Allocate("somecontainer", subnet, nil)
 		done <- true
 	}()
 	time.Sleep(100 * time.Millisecond)
@@ -88,26 +101,26 @@ func TestBootstrap(t *testing.T) {
 
 	CheckAllExpectedMessagesSent(alloc1, alloc2)
 
-	alloc1.tryPendingOps()
+	alloc1.actionChan <- func() { alloc1.tryPendingOps() }
 	AssertNothingSent(t, done)
 
 	CheckAllExpectedMessagesSent(alloc1, alloc2)
 
 	// alloc2 receives paxos update and broadcasts its reply
 	ExpectBroadcastMessage(alloc2, nil)
-	alloc2.OnGossipBroadcast(alloc1.encode())
+	alloc2.OnGossipBroadcast(alloc1.Encode())
 
 	ExpectBroadcastMessage(alloc1, nil)
-	alloc1.OnGossipBroadcast(alloc2.encode())
+	alloc1.OnGossipBroadcast(alloc2.Encode())
 
 	// both nodes will get consensus now so initialize the ring
 	ExpectBroadcastMessage(alloc2, nil)
 	ExpectBroadcastMessage(alloc2, nil)
-	alloc2.OnGossipBroadcast(alloc1.encode())
+	alloc2.OnGossipBroadcast(alloc1.Encode())
 
 	CheckAllExpectedMessagesSent(alloc1, alloc2)
 
-	alloc1.OnGossipBroadcast(alloc2.encode())
+	alloc1.OnGossipBroadcast(alloc2.Encode())
 	// now alloc1 should have space
 
 	AssertSent(t, done)
@@ -124,19 +137,16 @@ func TestAllocatorClaim(t *testing.T) {
 		testAddr1  = "10.0.3.1" // first address allocated should be .1 because .0 is network addr
 	)
 
-	alloc := makeAllocatorWithMockGossip(t, "01:00:00:01:00:00", universe, 1)
+	alloc, subnet := makeAllocatorWithMockGossip(t, "01:00:00:01:00:00", universe, 1)
 	defer alloc.Stop()
 
 	alloc.claimRingForTesting()
-	addr1, _ := alloc.Allocate(container1, nil)
-	alloc.Allocate(container2, nil)
+	addr1, _ := address.ParseIP(testAddr1)
 
-	// Now free the first one, and try to claim it
-	wt.AssertSuccess(t, alloc.Free(container1))
-	t.Log(alloc)
 	err := alloc.Claim(container3, addr1, nil)
 	wt.AssertNoErr(t, err)
-	addr3, _ := alloc.Allocate(container3, nil)
+	// Check we get this address back if we try an allocate
+	addr3, _ := alloc.Allocate(container3, subnet, nil)
 	wt.AssertEqualString(t, addr3.String(), testAddr1, "address")
 }
 
@@ -160,10 +170,10 @@ func TestCancel(t *testing.T) {
 
 	router := TestGossipRouter{make(map[router.PeerName]chan gossipMessage), 0.0}
 
-	alloc1 := makeAllocator("01:00:00:02:00:00", CIDR, 2)
+	alloc1, subnet := makeAllocator("01:00:00:02:00:00", CIDR, 2)
 	alloc1.SetInterfaces(router.connect(alloc1.ourName, alloc1))
 
-	alloc2 := makeAllocator("02:00:00:02:00:00", CIDR, 2)
+	alloc2, _ := makeAllocator("02:00:00:02:00:00", CIDR, 2)
 	alloc2.SetInterfaces(router.connect(alloc2.ourName, alloc2))
 	alloc1.claimRingForTesting(alloc1, alloc2)
 	alloc2.claimRingForTesting(alloc1, alloc2)
@@ -172,12 +182,12 @@ func TestCancel(t *testing.T) {
 	alloc2.Start()
 
 	// tell peers about each other
-	alloc1.OnGossipBroadcast(alloc2.encode())
+	alloc1.OnGossipBroadcast(alloc2.Encode())
 
 	// Get some IPs, so each allocator has some space
-	res1, _ := alloc1.Allocate("foo", nil)
+	res1, _ := alloc1.Allocate("foo", subnet, nil)
 	common.Debug.Printf("res1 = %s", res1.String())
-	res2, _ := alloc2.Allocate("bar", nil)
+	res2, _ := alloc2.Allocate("bar", subnet, nil)
 	common.Debug.Printf("res2 = %s", res2.String())
 	if res1 == res2 {
 		wt.Fatalf(t, "Error: got same ips!")
@@ -188,13 +198,13 @@ func TestCancel(t *testing.T) {
 	unpause := alloc2.pause()
 
 	// Use up all the IPs that alloc1 owns, so the allocation after this will prompt a request to alloc2
-	for i := 0; alloc1.space.NumFreeAddresses() > 0; i++ {
-		alloc1.Allocate(fmt.Sprintf("tmp%d", i), nil)
+	for i := 0; alloc1.NumFreeAddresses(subnet) > 0; i++ {
+		alloc1.Allocate(fmt.Sprintf("tmp%d", i), subnet, nil)
 	}
 	cancelChan := make(chan bool, 1)
 	doneChan := make(chan bool)
 	go func() {
-		_, ok := alloc1.Allocate("baz", cancelChan)
+		_, ok := alloc1.Allocate("baz", subnet, cancelChan)
 		doneChan <- ok == nil
 	}()
 
@@ -214,19 +224,17 @@ func TestGossipShutdown(t *testing.T) {
 		container1 = "abcdef"
 		container2 = "baddf00d"
 		universe   = "10.0.3.0/30"
-		testAddr1  = "10.0.3.1" // first address allocated should be .1 because .0 is network addr
 	)
 
-	alloc := makeAllocatorWithMockGossip(t, "01:00:00:01:00:00", universe, 1)
+	alloc, subnet := makeAllocatorWithMockGossip(t, "01:00:00:01:00:00", universe, 1)
 	defer alloc.Stop()
 
 	alloc.claimRingForTesting()
-	addr1, _ := alloc.Allocate(container1, nil)
-	wt.AssertEqualString(t, addr1.String(), testAddr1, "address")
+	alloc.Allocate(container1, subnet, nil)
 
 	alloc.Shutdown()
 
-	_, err := alloc.Allocate(container2, nil) // trying to allocate after shutdown should fail
+	_, err := alloc.Allocate(container2, subnet, nil) // trying to allocate after shutdown should fail
 	wt.AssertFalse(t, err == nil, "no address")
 
 	CheckAllExpectedMessagesSent(alloc)
@@ -236,15 +244,15 @@ func TestTransfer(t *testing.T) {
 	const (
 		cidr = "10.0.1.7/22"
 	)
-	allocs, router := makeNetworkOfAllocators(3, cidr)
+	allocs, router, subnet := makeNetworkOfAllocators(3, cidr)
 	alloc1 := allocs[0]
 	alloc2 := allocs[1]
 	alloc3 := allocs[2] // This will be 'master' and get the first range
 
-	_, err := alloc2.Allocate("foo", nil)
+	_, err := alloc2.Allocate("foo", subnet, nil)
 	wt.AssertTrue(t, err == nil, "Failed to get address")
 
-	_, err = alloc3.Allocate("bar", nil)
+	_, err = alloc3.Allocate("bar", subnet, nil)
 	wt.AssertTrue(t, err == nil, "Failed to get address")
 
 	router.GossipBroadcast(alloc2.Gossip())
@@ -256,9 +264,9 @@ func TestTransfer(t *testing.T) {
 	wt.AssertSuccess(t, alloc1.AdminTakeoverRanges(alloc2.ourName.String()))
 	wt.AssertSuccess(t, alloc1.AdminTakeoverRanges(alloc3.ourName.String()))
 
-	wt.AssertEquals(t, alloc1.space.NumFreeAddresses(), address.Offset(1022))
+	wt.AssertEquals(t, alloc1.NumFreeAddresses(subnet), address.Offset(1022))
 
-	_, err = alloc1.Allocate("foo", nil)
+	_, err = alloc1.Allocate("foo", subnet, nil)
 	wt.AssertTrue(t, err == nil, "Failed to get address")
 	alloc1.Stop()
 }
@@ -268,27 +276,26 @@ func TestFakeRouterSimple(t *testing.T) {
 	const (
 		cidr = "10.0.1.7/22"
 	)
-	allocs, _ := makeNetworkOfAllocators(2, cidr)
+	allocs, _, subnet := makeNetworkOfAllocators(2, cidr)
 	defer stopNetworkOfAllocators(allocs)
 
 	alloc1 := allocs[0]
 	//alloc2 := allocs[1]
 
-	addr, _ := alloc1.Allocate("foo", nil)
-	println("Got addr", addr)
+	alloc1.Allocate("foo", subnet, nil)
 }
 
 func TestAllocatorFuzz(t *testing.T) {
 	common.InitDefaultLogging(false)
 	const (
 		firstpass    = 1000
-		secondpass   = 5000
+		secondpass   = 20000
 		nodes        = 10
 		maxAddresses = 1000
-		concurrency  = 10
+		concurrency  = 30
 		cidr         = "10.0.1.7/22"
 	)
-	allocs, _ := makeNetworkOfAllocators(nodes, cidr)
+	allocs, _, subnet := makeNetworkOfAllocators(nodes, cidr)
 	defer stopNetworkOfAllocators(allocs)
 
 	// Test state
@@ -298,6 +305,7 @@ func TestAllocatorFuzz(t *testing.T) {
 	type result struct {
 		name  string
 		alloc int32
+		block bool
 	}
 	stateLock := sync.Mutex{}
 	state := make(map[string]result)
@@ -331,7 +339,7 @@ func TestAllocatorFuzz(t *testing.T) {
 		allocIndex := rand.Int31n(nodes)
 		alloc := allocs[allocIndex]
 		//common.Info.Printf("Allocate: asking allocator %d", allocIndex)
-		addr, err := alloc.Allocate(name, nil)
+		addr, err := alloc.Allocate(name, subnet, nil)
 
 		if err != nil {
 			panic(fmt.Sprintf("Could not allocate addr"))
@@ -348,7 +356,7 @@ func TestAllocatorFuzz(t *testing.T) {
 				name, res.name))
 		}
 
-		state[addrStr] = result{name, allocIndex}
+		state[addrStr] = result{name, allocIndex, false}
 		addrs = append(addrs, addrStr)
 		numPending--
 	}
@@ -365,14 +373,22 @@ func TestAllocatorFuzz(t *testing.T) {
 		addrIndex := rand.Int31n(int32(len(addrs)))
 		addr := addrs[addrIndex]
 		res := state[addr]
+		if res.block {
+			stateLock.Unlock()
+			return
+		}
 		addrs = rm(addrs, addrIndex)
 		delete(state, addr)
 		stateLock.Unlock()
 
 		alloc := allocs[res.alloc]
-		//common.Info.Printf("Freeing %s on allocator %d", addr, res.alloc)
+		//common.Info.Printf("Freeing %s (%s) on allocator %d", res.name, addr, res.alloc)
 
-		wt.AssertSuccess(t, alloc.Free(res.name))
+		oldAddr, err := address.ParseIP(addr)
+		if err != nil {
+			panic(err)
+		}
+		wt.AssertSuccess(t, alloc.Free(res.name, oldAddr))
 	}
 
 	// Do a Allocate on an existing container & allocator
@@ -382,16 +398,27 @@ func TestAllocatorFuzz(t *testing.T) {
 		addrIndex := rand.Int31n(int32(len(addrs)))
 		addr := addrs[addrIndex]
 		res := state[addr]
+		if res.block {
+			stateLock.Unlock()
+			return
+		}
+		res.block = true
+		state[addr] = res
 		stateLock.Unlock()
 		alloc := allocs[res.alloc]
 
-		//common.Info.Printf("Asking for %s on allocator %d again", addr, res.alloc)
+		//common.Info.Printf("Asking for %s (%s) on allocator %d again", res.name, addr, res.alloc)
 
-		newAddr, _ := alloc.Allocate(res.name, nil)
+		newAddr, _ := alloc.Allocate(res.name, subnet, nil)
 		oldAddr, _ := address.ParseIP(addr)
 		if newAddr != oldAddr {
-			panic(fmt.Sprintf("Got different address for repeat request"))
+			panic(fmt.Sprintf("Got different address for repeat request for %s: %s != %s", res.name, newAddr, oldAddr))
 		}
+
+		stateLock.Lock()
+		res.block = false
+		state[addr] = res
+		stateLock.Unlock()
 	}
 
 	// Run function _f_ _iterations_ times, in _concurrency_
@@ -441,13 +468,13 @@ func TestAllocatorFuzz(t *testing.T) {
 }
 
 func TestGossipSkew(t *testing.T) {
-	alloc1 := makeAllocatorWithMockGossip(t, "01:00:00:01:00:00", "10.0.1.0/22", 2)
+	alloc1, _ := makeAllocatorWithMockGossip(t, "01:00:00:01:00:00", "10.0.1.0/22", 2)
 	defer alloc1.Stop()
-	alloc2 := makeAllocatorWithMockGossip(t, "02:00:00:02:00:00", "10.0.1.0/22", 2)
+	alloc2, _ := makeAllocatorWithMockGossip(t, "02:00:00:02:00:00", "10.0.1.0/22", 2)
 	alloc2.now = func() time.Time { return time.Now().Add(time.Hour * 2) }
 	defer alloc2.Stop()
 
-	if _, err := alloc1.OnGossipBroadcast(alloc2.encode()); err == nil {
+	if _, err := alloc1.OnGossipBroadcast(alloc2.Encode()); err == nil {
 		t.Fail()
 	}
 }

ipam/claim.go

@@ -9,9 +9,9 @@ import (
 
 type claim struct {
 	resultChan       chan<- error
-	hasBeenCancelled func() bool
 	ident            string
 	addr             address.Address
+	hasBeenCancelled func() bool
 }
 
 // Try returns true for success (or failure), false if we need to try again later
@@ -57,6 +57,7 @@ func (c *claim) Try(alloc *Allocator) bool {
 			c.resultChan <- err
 			return true
 		}
+		alloc.debugln("Claimed", c.addr, "for", c.ident)
 		alloc.addOwned(c.ident, c.addr)
 		c.resultChan <- nil
 		return true

ipam/dummy.go

@@ -1,36 +0,0 @@
-package ipam
-
-import (
-	"errors"
-
-	"github.com/weaveworks/weave/router"
-)
-
-// Exists solely for the purpose of detecting a network in which some
-// peers are configured with IPAM on and some with IPAM off.
-type DummyAllocator struct {
-}
-
-var (
-	ErrMixedConfig = errors.New("Mixed IPAM/non-IPAM configuration not supported")
-)
-
-func (alloc *DummyAllocator) OnGossipUnicast(sender router.PeerName, msg []byte) error {
-	return ErrMixedConfig
-}
-
-func (alloc *DummyAllocator) OnGossipBroadcast(msg []byte) (router.GossipData, error) {
-	return nil, ErrMixedConfig
-}
-
-func (alloc *DummyAllocator) OnGossip(msg []byte) (router.GossipData, error) {
-	return nil, ErrMixedConfig
-}
-
-func (alloc *DummyAllocator) Gossip() router.GossipData {
-	return nil
-}
-
-func (alloc *DummyAllocator) Encode() []byte {
-	return nil
-}

ipam/http.go

@@ -7,6 +7,7 @@ import (
 	"github.com/gorilla/mux"
 
 	"github.com/weaveworks/weave/common"
+	"github.com/weaveworks/weave/common/docker"
 	"github.com/weaveworks/weave/ipam/address"
 )
 
@@ -16,7 +17,7 @@ func badRequest(w http.ResponseWriter, err error) {
 }
 
 // HandleHTTP wires up ipams HTTP endpoints to the provided mux.
-func (alloc *Allocator) HandleHTTP(router *mux.Router) {
+func (alloc *Allocator) HandleHTTP(router *mux.Router, defaultSubnet address.CIDR, dockerCli *docker.Client) {
 	router.Methods("PUT").Path("/ip/{id}/{ip}").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		closedChan := w.(http.CloseNotifier).CloseNotify()
 		vars := mux.Vars(r)
@@ -33,21 +34,94 @@ func (alloc *Allocator) HandleHTTP(router *mux.Router) {
 		w.WriteHeader(204)
 	})
 
-	router.Methods("POST").Path("/ip/{id}").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		closedChan := w.(http.CloseNotifier).CloseNotify()
-		ident := mux.Vars(r)["id"]
-		newAddr, err := alloc.Allocate(ident, closedChan)
+	router.Methods("GET").Path("/ip/{id}/{ip}/{prefixlen}").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		vars := mux.Vars(r)
+		cidr := vars["ip"] + "/" + vars["prefixlen"]
+		_, subnet, err := address.ParseCIDR(cidr)
 		if err != nil {
 			badRequest(w, err)
 			return
 		}
+		addr, err := alloc.Lookup(vars["id"], subnet.HostRange())
+		if err != nil {
+			http.NotFound(w, r)
+			return
+		}
+		fmt.Fprintf(w, "%s/%d", addr, subnet.PrefixLen)
+	})
 
-		fmt.Fprintf(w, "%s/%d", newAddr.String(), alloc.prefixLen)
+	router.Methods("GET").Path("/ip/{id}").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		addr, err := alloc.Lookup(mux.Vars(r)["id"], defaultSubnet.HostRange())
+		if err != nil {
+			http.NotFound(w, r)
+			return
+		}
+		fmt.Fprintf(w, "%s/%d", addr, defaultSubnet.PrefixLen)
+	})
+
+	router.Methods("POST").Path("/ip/{id}/{ip}/{prefixlen}").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		closedChan := w.(http.CloseNotifier).CloseNotify()
+		vars := mux.Vars(r)
+		ident := vars["id"]
+		cidrStr := vars["ip"] + "/" + vars["prefixlen"]
+		subnetAddr, cidr, err := address.ParseCIDR(cidrStr)
+		if err != nil {
+			badRequest(w, err)
+			return
+		}
+		if cidr.Start != subnetAddr {
+			badRequest(w, fmt.Errorf("Invalid subnet %s - bits after network prefix are not all zero", cidrStr))
+			return
+		}
+		addr, err := alloc.Allocate(ident, cidr.HostRange(), closedChan)
+		if err != nil {
+			badRequest(w, fmt.Errorf("Unable to allocate: %s", err))
+			return
+		}
+		if dockerCli != nil && dockerCli.IsContainerNotRunning(ident) {
+			common.Info.Printf("[allocator] '%s' is not running: freeing %s", ident, addr)
+			alloc.Free(ident, addr)
+			return
+		}
+
+		fmt.Fprintf(w, "%s/%d", addr, cidr.PrefixLen)
+	})
+
+	router.Methods("POST").Path("/ip/{id}").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		closedChan := w.(http.CloseNotifier).CloseNotify()
+		ident := mux.Vars(r)["id"]
+		newAddr, err := alloc.Allocate(ident, defaultSubnet.HostRange(), closedChan)
+		if err != nil {
+			badRequest(w, err)
+			return
+		}
+		if dockerCli != nil && dockerCli.IsContainerNotRunning(ident) {
+			common.Info.Printf("[allocator] '%s' is not running: freeing %s", ident, newAddr)
+			alloc.Free(ident, newAddr)
+			return
+		}
+
+		fmt.Fprintf(w, "%s/%d", newAddr, defaultSubnet.PrefixLen)
+	})
+
+	router.Methods("DELETE").Path("/ip/{id}/{ip}").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		vars := mux.Vars(r)
+		ident := vars["id"]
+		ipStr := vars["ip"]
+		if ip, err := address.ParseIP(ipStr); err != nil {
+			badRequest(w, err)
+			return
+		} else if err := alloc.Free(ident, ip); err != nil {
+			badRequest(w, fmt.Errorf("Unable to free: %s", err))
+			return
+		}
+
+		w.WriteHeader(204)
 	})
 
 	router.Methods("DELETE").Path("/ip/{id}").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		ident := mux.Vars(r)["id"]
-		if err := alloc.Free(ident); err != nil {
+		if err := alloc.Delete(ident); err != nil {
 			badRequest(w, err)
 			return
 		}

ipam/http_test.go

@@ -4,7 +4,7 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
-	"math/rand"
+	"net"
 	"net/http"
 	"strings"
 	"testing"
@@ -13,6 +13,7 @@ import (
 	"github.com/gorilla/mux"
 
 	"github.com/weaveworks/weave/common"
+	"github.com/weaveworks/weave/ipam/address"
 	wt "github.com/weaveworks/weave/testing"
 )
 
@@ -25,65 +26,89 @@ func HTTPPost(t *testing.T, url string) string {
 	return string(body)
 }
 
+func HTTPGet(t *testing.T, url string) string {
+	resp, err := http.Get(url)
+	wt.AssertNoErr(t, err)
+	defer resp.Body.Close()
+	body, _ := ioutil.ReadAll(resp.Body)
+	return string(body)
+}
+
 func doHTTP(method string, url string) (resp *http.Response, err error) {
 	req, _ := http.NewRequest(method, url, nil)
 	return http.DefaultClient.Do(req)
 }
 
-func listenHTTP(port int, alloc *Allocator) {
+func listenHTTP(alloc *Allocator, subnet address.CIDR) int {
 	router := mux.NewRouter()
 	router.HandleFunc("/status", func(w http.ResponseWriter, r *http.Request) {
 		io.WriteString(w, fmt.Sprintln(alloc))
 	})
-	alloc.HandleHTTP(router)
+	alloc.HandleHTTP(router, subnet, nil)
 
-	srv := &http.Server{
-		Addr:    fmt.Sprintf(":%d", port),
-		Handler: router,
-	}
-	if err := srv.ListenAndServe(); err != nil {
+	httpListener, err := net.Listen("tcp", ":0")
+	if err != nil {
 		common.Error.Fatal("Unable to create http listener: ", err)
 	}
+
+	go func() {
+		srv := &http.Server{Handler: router}
+		if err := srv.Serve(httpListener); err != nil {
+			common.Error.Fatal("Unable to serve http: ", err)
+		}
+	}()
+	return httpListener.Addr().(*net.TCPAddr).Port
 }
 
-func allocURL(port int, containerID string) string {
+func identURL(port int, containerID string) string {
 	return fmt.Sprintf("http://localhost:%d/ip/%s", port, containerID)
 }
 
+func allocURL(port int, cidr string, containerID string) string {
+	return fmt.Sprintf("http://localhost:%d/ip/%s/%s", port, containerID, cidr)
+}
+
 func TestHttp(t *testing.T) {
 	var (
 		containerID = "deadbeef"
 		container2  = "baddf00d"
 		container3  = "b01df00d"
-		testAddr1   = "10.0.3.9"
-		netSuffix   = "/29"
-		testCIDR1   = "10.0.3.8" + netSuffix
+		universe    = "10.0.0.0/8"
+		testCIDR1   = "10.0.3.8/29"
+		testCIDR2   = "10.2.0.0/16"
+		testAddr1   = "10.0.3.9/29"
+		testAddr2   = "10.2.0.1/16"
 	)
 
-	alloc := makeAllocatorWithMockGossip(t, "08:00:27:01:c3:9a", testCIDR1, 1)
-	port := rand.Intn(10000) + 32768
-	fmt.Println("Http test on port", port)
-	go listenHTTP(port, alloc)
-
-	time.Sleep(100 * time.Millisecond) // Allow for http server to get going
-
+	alloc, _ := makeAllocatorWithMockGossip(t, "08:00:27:01:c3:9a", universe, 1)
+	_, cidr, _ := address.ParseCIDR(universe)
+	port := listenHTTP(alloc, cidr)
 	alloc.claimRingForTesting()
-	// Ask the http server for a new address
-	cidr1 := HTTPPost(t, allocURL(port, containerID))
-	wt.AssertEqualString(t, cidr1, testAddr1+netSuffix, "address")
 
-	// Ask the http server for another address and check it's different
-	cidr2 := HTTPPost(t, allocURL(port, container2))
-	wt.AssertFalse(t, cidr2 == testAddr1+netSuffix, "address")
+	// Allocate an address in each subnet, and check we got what we expected
+	cidr1a := HTTPPost(t, allocURL(port, testCIDR1, containerID))
+	wt.AssertEqualString(t, cidr1a, testAddr1, "address")
+	cidr2a := HTTPPost(t, allocURL(port, testCIDR2, containerID))
+	wt.AssertEqualString(t, cidr2a, testAddr2, "address")
+	// Now, make the same requests again to check the operation is idempotent
+	check := HTTPGet(t, allocURL(port, testCIDR1, containerID))
+	wt.AssertEqualString(t, check, cidr1a, "address")
+	check = HTTPGet(t, allocURL(port, testCIDR2, containerID))
+	wt.AssertEqualString(t, check, cidr2a, "address")
 
-	// Ask for the first container again and we should get the same address again
-	cidr1a := HTTPPost(t, allocURL(port, containerID))
-	wt.AssertEqualString(t, cidr1a, testAddr1+netSuffix, "address")
+	// Ask the http server for a pair of addresses for another container and check they're different
+	cidr1b := HTTPPost(t, allocURL(port, testCIDR1, container2))
+	wt.AssertFalse(t, cidr1b == testAddr1, "address")
+	cidr2b := HTTPPost(t, allocURL(port, testCIDR2, container2))
+	wt.AssertFalse(t, cidr2b == testAddr2, "address")
 
-	// Now free the first one, and we should get it back when we ask
-	doHTTP("DELETE", allocURL(port, containerID))
-	cidr3 := HTTPPost(t, allocURL(port, container3))
-	wt.AssertEqualString(t, cidr3, testAddr1+netSuffix, "address")
+	// Now free the first container, and we should get its addresses back when we ask
+	doHTTP("DELETE", identURL(port, containerID))
+
+	cidr1c := HTTPPost(t, allocURL(port, testCIDR1, container3))
+	wt.AssertEqualString(t, cidr1c, testAddr1, "address")
+	cidr2c := HTTPPost(t, allocURL(port, testCIDR2, container3))
+	wt.AssertEqualString(t, cidr2c, testAddr2, "address")
 
 	// Would like to shut down the http server at the end of this test
 	// but it's complicated.
@@ -96,14 +121,13 @@ func TestBadHttp(t *testing.T) {
 		testCIDR1   = "10.0.0.0/8"
 	)
 
-	alloc := makeAllocatorWithMockGossip(t, "08:00:27:01:c3:9a", testCIDR1, 1)
+	alloc, _ := makeAllocatorWithMockGossip(t, "08:00:27:01:c3:9a", testCIDR1, 1)
 	defer alloc.Stop()
-	port := rand.Intn(10000) + 32768
-	fmt.Println("BadHttp test on port", port)
-	go listenHTTP(port, alloc)
+	_, cidr, _ := address.ParseCIDR(testCIDR1)
+	port := listenHTTP(alloc, cidr)
 
 	alloc.claimRingForTesting()
-	cidr1 := HTTPPost(t, allocURL(port, containerID))
+	cidr1 := HTTPPost(t, allocURL(port, testCIDR1, containerID))
 	parts := strings.Split(cidr1, "/")
 	testAddr1 := parts[0]
 	// Verb that's not handled
@@ -130,24 +154,21 @@ func impTestHTTPCancel(t *testing.T) {
 	common.InitDefaultLogging(true)
 	var (
 		containerID = "deadbeef"
-		testCIDR1   = "10.0.3.5/29"
+		testCIDR1   = "10.0.3.0/29"
 	)
 
-	alloc := makeAllocatorWithMockGossip(t, "08:00:27:01:c3:9a", testCIDR1, 1)
+	alloc, _ := makeAllocatorWithMockGossip(t, "08:00:27:01:c3:9a", testCIDR1, 1)
 	defer alloc.Stop()
 	alloc.claimRingForTesting()
-	port := rand.Intn(10000) + 32768
-	fmt.Println("Http test on port", port)
-	go listenHTTP(port, alloc)
-
-	time.Sleep(100 * time.Millisecond) // Allow for http server to get going
+	_, cidr, _ := address.ParseCIDR(testCIDR1)
+	port := listenHTTP(alloc, cidr)
 
 	// Stop the alloc so nothing actually works
 	unpause := alloc.pause()
 
 	// Ask the http server for a new address
 	done := make(chan *http.Response)
-	req, _ := http.NewRequest("POST", allocURL(port, containerID), nil)
+	req, _ := http.NewRequest("POST", allocURL(port, testCIDR1, containerID), nil)
 	go func() {
 		res, _ := http.DefaultClient.Do(req)
 		done <- res

ipam/paxos/paxos.go

@@ -117,9 +117,8 @@ func (node *Node) Update(from GossipState) bool {
 func max(a uint, b uint) uint {
 	if a > b {
 		return a
-	} else {
-		return b
 	}
+	return b
 }
 
 // Initiate a new proposal, i.e. the Paxos "Prepare" step.  This is
@@ -255,7 +254,6 @@ func (node *Node) Consensus() (bool, AcceptedValue) {
 func (node *Node) String() string {
 	if ok, val := node.Consensus(); ok {
 		return fmt.Sprintf("Consensus reached with size %d", len(val.Value))
-	} else {
-		return fmt.Sprintf("Nodes known: %d, Quorum size: %d", len(node.knows), node.quorum)
 	}
+	return fmt.Sprintf("Nodes known: %d, Quorum size: %d", len(node.knows), node.quorum)
 }

ipam/ring/ring.go

@@ -91,18 +91,6 @@ func New(start, end address.Address, peer router.PeerName) *Ring {
 	return ring
 }
 
-// TotalRemoteFree returns the approximate number of free IPs
-// on other hosts.
-func (r *Ring) TotalRemoteFree() address.Offset {
-	result := address.Offset(0)
-	for _, entry := range r.Entries {
-		if entry.Peer != r.Peer {
-			result += entry.Free
-		}
-	}
-	return result
-}
-
 // Returns the distance between two tokens on this ring, dealing
 // with ranges which cross the origin
 func (r *Ring) distance(start, end address.Address) address.Offset {
@@ -341,10 +329,6 @@ func (r *Ring) ClaimForPeers(peers []router.PeerName) {
 	common.Assert(pos == r.End)
 }
 
-func (r *Ring) ClaimItAll() {
-	r.ClaimForPeers([]router.PeerName{r.Peer})
-}
-
 func (r *Ring) FprintWithNicknames(w io.Writer, m map[router.PeerName]string) {
 	for _, entry := range r.Entries {
 		nickname, found := m[entry.Peer]
@@ -409,15 +393,22 @@ func (r *Ring) ReportFree(freespace map[address.Address]address.Offset) {
 }
 
 // ChoosePeerToAskForSpace chooses a weighted-random peer to ask
-// for space.
-func (r *Ring) ChoosePeerToAskForSpace() (result router.PeerName, err error) {
+// for space in the range [start, end).  Assumes start<end.
+func (r *Ring) ChoosePeerToAskForSpace(start, end address.Address) (result router.PeerName, err error) {
 	var (
 		sum               address.Offset
 		totalSpacePerPeer = make(map[router.PeerName]address.Offset) // Compute total free space per peer
 	)
 
 	// iterate through tokens
-	for _, entry := range r.Entries {
+	for i, entry := range r.Entries {
+		// Ignore entries that don't span the range we want
+		if i+1 < len(r.Entries) && r.Entries.entry(i+1).Token-1 < start {
+			continue
+		}
+		if entry.Token >= end {
+			break
+		}
 		// Ignore ranges with no free space
 		if entry.Free <= 0 {
 			continue
@@ -460,7 +451,7 @@ func (r *Ring) PickPeerForTransfer() router.PeerName {
 
 // Transfer will mark all entries associated with 'from' peer as owned by 'to' peer
 // and return ranges indicating the new space we picked up
-func (r *Ring) Transfer(from, to router.PeerName) (error, []address.Range) {
+func (r *Ring) Transfer(from, to router.PeerName) ([]address.Range, error) {
 	r.assertInvariants()
 	defer r.assertInvariants()
 	defer r.updateExportedVariables()
@@ -478,10 +469,10 @@ func (r *Ring) Transfer(from, to router.PeerName) (error, []address.Range) {
 	}
 
 	if !found {
-		return ErrNotFound, nil
+		return nil, ErrNotFound
 	}
 
-	return nil, r.splitRangesOverZero(newRanges)
+	return r.splitRangesOverZero(newRanges), nil
 }
 
 // Contains returns true if addr is in this ring

ipam/ring/ring_test.go

@@ -345,34 +345,40 @@ func TestGossip(t *testing.T) {
 func TestFindFree(t *testing.T) {
 	ring1 := New(start, end, peer1name)
 
-	_, err := ring1.ChoosePeerToAskForSpace()
+	_, err := ring1.ChoosePeerToAskForSpace(start, end)
 	wt.AssertTrue(t, err == ErrNoFreeSpace, "Expected ErrNoFreeSpace")
 
 	ring1.Entries = []*entry{{Token: start, Peer: peer1name}}
-	_, err = ring1.ChoosePeerToAskForSpace()
+	_, err = ring1.ChoosePeerToAskForSpace(start, end)
 	wt.AssertTrue(t, err == ErrNoFreeSpace, "Expected ErrNoFreeSpace")
 
 	// We shouldn't return outselves
 	ring1.ReportFree(map[address.Address]address.Offset{start: 10})
-	_, err = ring1.ChoosePeerToAskForSpace()
+	_, err = ring1.ChoosePeerToAskForSpace(start, end)
 	wt.AssertTrue(t, err == ErrNoFreeSpace, "Expected ErrNoFreeSpace")
 
 	ring1.Entries = []*entry{{Token: start, Peer: peer1name, Free: 1},
-		{Token: start, Peer: peer1name, Free: 1}}
-	_, err = ring1.ChoosePeerToAskForSpace()
+		{Token: middle, Peer: peer1name, Free: 1}}
+	_, err = ring1.ChoosePeerToAskForSpace(start, end)
 	wt.AssertTrue(t, err == ErrNoFreeSpace, "Expected ErrNoFreeSpace")
+	ring1.assertInvariants()
 
 	// We should return others
 	ring1.Entries = []*entry{{Token: start, Peer: peer2name, Free: 1}}
-	peer, err := ring1.ChoosePeerToAskForSpace()
+	peer, err := ring1.ChoosePeerToAskForSpace(start, end)
 	wt.AssertSuccess(t, err)
 	wt.AssertEquals(t, peer, peer2name)
 
 	ring1.Entries = []*entry{{Token: start, Peer: peer2name, Free: 1},
-		{Token: start, Peer: peer2name, Free: 1}}
-	peer, err = ring1.ChoosePeerToAskForSpace()
+		{Token: middle, Peer: peer3name, Free: 1}}
+	peer, err = ring1.ChoosePeerToAskForSpace(start, middle)
 	wt.AssertSuccess(t, err)
 	wt.AssertEquals(t, peer, peer2name)
+
+	peer, err = ring1.ChoosePeerToAskForSpace(middle, end)
+	wt.AssertSuccess(t, err)
+	wt.AssertEquals(t, peer, peer3name)
+	ring1.assertInvariants()
 }
 
 func TestReportFree(t *testing.T) {
@@ -710,6 +716,10 @@ func TestFuzzRingHard(t *testing.T) {
 	}
 }
 
+func (r *Ring) ClaimItAll() {
+	r.ClaimForPeers([]router.PeerName{r.Peer})
+}
+
 func (es entries) String() string {
 	var buffer bytes.Buffer
 	fmt.Fprintf(&buffer, "[")

ipam/space/space.go

@@ -9,8 +9,6 @@ import (
 	"github.com/weaveworks/weave/ipam/address"
 )
 
-type Addr address.Address
-
 type Space struct {
 	// ours and free represent a set of addresses as a sorted
 	// sequences of ranges.  Even elements give the inclusive
@@ -37,15 +35,47 @@ func (s *Space) Clear() {
 	s.ours = s.ours[:0]
 }
 
-func (s *Space) Allocate() (bool, address.Address) {
-	if len(s.free) == 0 {
-		return false, 0
-	} else {
-		res := s.free[0]
-		s.ours = add(s.ours, res, res+1)
-		s.free = subtract(s.free, res, res+1)
-		return true, res
+// Walk down the free list calling f() on the in-range portions, until
+// f() returns true or we run out of free space.  Return true iff f() returned true
+func (s *Space) walkFree(r address.Range, f func(address.Range) bool) bool {
+	if r.Start >= r.End { // degenerate case
+		return false
 	}
+	for i := 0; i < len(s.free); i += 2 {
+		chunk := address.Range{Start: s.free[i], End: s.free[i+1]}
+		if chunk.End <= r.Start { // this chunk comes before the range
+			continue
+		}
+		if chunk.Start >= r.End {
+			// all remaining free space is completely after range
+			break
+		}
+		// at this point we know chunk.End>chunk.Start &&
+		// chunk.End>r.Start && r.End>chunk.Start && r.End>r.Start
+		// therefore max(start, r.Start) < min(end, r.End)
+		// Restrict this block of free space to be in range
+		if chunk.Start < r.Start {
+			chunk.Start = r.Start
+		}
+		if chunk.End > r.End {
+			chunk.End = r.End
+		}
+		// at this point we know start<end
+		if f(chunk) {
+			return true
+		}
+	}
+	return false
+}
+
+func (s *Space) Allocate(r address.Range) (bool, address.Address) {
+	var result address.Address
+	return s.walkFree(r, func(chunk address.Range) bool {
+		result = chunk.Start
+		s.ours = add(s.ours, result, result+1)
+		s.free = subtract(s.free, result, result+1)
+		return true
+	}), result
 }
 
 func (s *Space) Claim(addr address.Address) error {
@@ -58,29 +88,12 @@ func (s *Space) Claim(addr address.Address) error {
 	return nil
 }
 
-func (s *Space) NumFreeAddresses() address.Offset {
+func (s *Space) NumFreeAddressesInRange(r address.Range) address.Offset {
 	res := address.Offset(0)
-	for i := 0; i < len(s.free); i += 2 {
-		res += address.Subtract(s.free[i+1], s.free[i])
-	}
-	return res
-}
-
-func (s *Space) NumFreeAddressesInRange(start, end address.Address) address.Offset {
-	res := address.Offset(0)
-	for i := 0; i < len(s.free); i += 2 {
-		s, e := s.free[i], s.free[i+1]
-		if s < start {
-			s = start
-		}
-		if e > end {
-			e = end
-		}
-		if s >= e {
-			continue
-		}
-		res += address.Subtract(e, s)
-	}
+	s.walkFree(r, func(chunk address.Range) bool {
+		res += chunk.Size()
+		return false
+	})
 	return res
 }
 
@@ -97,35 +110,32 @@ func (s *Space) Free(addr address.Address) error {
 	return nil
 }
 
-func (s *Space) biggestFreeRange() (int, address.Offset) {
-	pos := -1
-	biggest := address.Offset(0)
-
-	for i := 0; i < len(s.free); i += 2 {
-		size := address.Subtract(s.free[i+1], s.free[i])
-		if size >= biggest {
-			pos = i
-			biggest = size
+func (s *Space) biggestFreeRange(r address.Range) (biggest address.Range) {
+	biggestSize := address.Offset(0)
+	s.walkFree(r, func(chunk address.Range) bool {
+		if size := chunk.Size(); size >= biggestSize {
+			biggest = chunk
+			biggestSize = size
 		}
-	}
-	return pos, biggest
+		return false
+	})
+	return
 }
 
-func (s *Space) Donate() (address.Address, address.Offset, bool) {
-	if len(s.free) == 0 {
-		return 0, 0, false
+func (s *Space) Donate(r address.Range) (address.Range, bool) {
+	biggest := s.biggestFreeRange(r)
+
+	if biggest.Size() == 0 {
+		return address.Range{}, false
 	}
 
-	pos, biggest := s.biggestFreeRange()
+	// Donate half of that biggest free range. Note size/2 rounds down, so
+	// the resulting donation size rounds up, and in particular can't be empty.
+	biggest.Start = address.Add(biggest.Start, biggest.Size()/2)
 
-	// Donate half of that biggest free range, rounding up so
-	// that the donation can't be empty
-	end := s.free[pos+1]
-	start := end - address.Address((biggest+1)/2)
-
-	s.ours = subtract(s.ours, start, end)
-	s.free = subtract(s.free, start, end)
-	return start, address.Subtract(end, start), true
+	s.ours = subtract(s.ours, biggest.Start, biggest.End)
+	s.free = subtract(s.free, biggest.Start, biggest.End)
+	return biggest, true
 }
 
 func firstGreater(a []address.Address, x address.Address) int {
@@ -152,25 +162,25 @@ func subtract(addrs []address.Address, start address.Address, end address.Addres
 }
 
 func addSub(addrs []address.Address, start address.Address, end address.Address, sense int) []address.Address {
-	start_pos := firstGreaterOrEq(addrs, start)
-	end_pos := firstGreater(addrs[start_pos:], end) + start_pos
+	startPos := firstGreaterOrEq(addrs, start)
+	endPos := firstGreater(addrs[startPos:], end) + startPos
 
-	// Boundaries up to start_pos are unaffected
-	res := make([]address.Address, start_pos, len(addrs)+2)
+	// Boundaries up to startPos are unaffected
+	res := make([]address.Address, startPos, len(addrs)+2)
 	copy(res, addrs)
 
 	// Include start and end as new boundaries if they lie
 	// outside/inside existing ranges (according to sense).
-	if start_pos&1 == sense {
+	if startPos&1 == sense {
 		res = append(res, start)
 	}
 
-	if end_pos&1 == sense {
+	if endPos&1 == sense {
 		res = append(res, end)
 	}
 
-	// Boundaries after end_pos are unaffected
-	return append(res, addrs[end_pos:]...)
+	// Boundaries after endPos are unaffected
+	return append(res, addrs[endPos:]...)
 }
 
 func (s *Space) String() string {

ipam/space/space_test.go

@@ -1,7 +1,6 @@
 package space
 
 import (
-	"reflect"
 	"testing"
 
 	"github.com/weaveworks/weave/ipam/address"
@@ -19,6 +18,16 @@ func ip(s string) address.Address {
 	return addr
 }
 
+// Helper function to avoid 'NumFreeAddressesInRange(start, end)'
+// dozens of times in tests
+func (s *Space) NumFreeAddresses() address.Offset {
+	res := address.Offset(0)
+	for i := 0; i < len(s.free); i += 2 {
+		res += address.Subtract(s.free[i+1], s.free[i])
+	}
+	return res
+}
+
 func TestLowlevel(t *testing.T) {
 	a := []address.Address{}
 	a = add(a, 100, 200)
@@ -52,12 +61,12 @@ func TestLowlevel(t *testing.T) {
 
 	s := New()
 	wt.AssertEquals(t, s.NumFreeAddresses(), address.Offset(0))
-	ok, got := s.Allocate()
+	ok, got := s.Allocate(address.NewRange(0, 1000))
 	wt.AssertFalse(t, ok, "allocate in empty space should fail")
 
 	s.Add(100, 100)
 	wt.AssertEquals(t, s.NumFreeAddresses(), address.Offset(100))
-	ok, got = s.Allocate()
+	ok, got = s.Allocate(address.NewRange(0, 1000))
 	wt.AssertTrue(t, ok && got == 100, "allocate")
 	wt.AssertEquals(t, s.NumFreeAddresses(), address.Offset(99))
 	wt.AssertNoErr(t, s.Claim(150))
@@ -67,44 +76,49 @@ func TestLowlevel(t *testing.T) {
 	wt.AssertErrorInterface(t, s.Free(0), (*error)(nil), "free not allocated")
 	wt.AssertErrorInterface(t, s.Free(100), (*error)(nil), "double free")
 
-	start, size, ok := s.Donate()
-	wt.AssertTrue(t, ok && start == 125 && size == 25, "donate")
+	r, ok := s.Donate(address.NewRange(0, 1000))
+	wt.AssertTrue(t, ok && r.Start == 125 && r.Size() == 25, "donate")
 
 	// test Donate when addresses are scarce
 	s = New()
-	start, size, ok = s.Donate()
+	r, ok = s.Donate(address.NewRange(0, 1000))
 	wt.AssertTrue(t, !ok, "donate on empty space should fail")
 	s.Add(0, 3)
 	wt.AssertNoErr(t, s.Claim(0))
 	wt.AssertNoErr(t, s.Claim(2))
-	start, size, ok = s.Donate()
-	wt.AssertTrue(t, ok && start == 1 && size == 1, "donate")
-	start, size, ok = s.Donate()
+	r, ok = s.Donate(address.NewRange(0, 1000))
+	wt.AssertTrue(t, ok && r.Start == 1 && r.End == 2, "donate")
+	r, ok = s.Donate(address.NewRange(0, 1000))
 	wt.AssertTrue(t, !ok, "donate should fail")
 }
 
 func TestSpaceAllocate(t *testing.T) {
 	const (
 		testAddr1   = "10.0.3.4"
+		testAddr2   = "10.0.3.5"
 		testAddrx   = "10.0.3.19"
 		testAddry   = "10.0.9.19"
 		containerID = "deadbeef"
+		size        = 20
+	)
+	var (
+		start = ip(testAddr1)
 	)
 
-	space1 := makeSpace(ip(testAddr1), 20)
+	space1 := makeSpace(start, size)
 	wt.AssertEquals(t, space1.NumFreeAddresses(), address.Offset(20))
 	space1.assertInvariants()
 
-	_, addr1 := space1.Allocate()
+	_, addr1 := space1.Allocate(address.NewRange(start, size))
 	wt.AssertEqualString(t, addr1.String(), testAddr1, "address")
 	wt.AssertEquals(t, space1.NumFreeAddresses(), address.Offset(19))
 	space1.assertInvariants()
 
-	_, addr2 := space1.Allocate()
+	_, addr2 := space1.Allocate(address.NewRange(start, size))
 	wt.AssertFalse(t, addr2.String() == testAddr1, "address")
 	wt.AssertEquals(t, space1.NumFreeAddresses(), address.Offset(18))
-	wt.AssertEquals(t, space1.NumFreeAddressesInRange(ip(testAddr1), ip(testAddrx)), address.Offset(13))
-	wt.AssertEquals(t, space1.NumFreeAddressesInRange(ip(testAddr1), ip(testAddry)), address.Offset(18))
+	wt.AssertEquals(t, space1.NumFreeAddressesInRange(address.Range{Start: ip(testAddr1), End: ip(testAddrx)}), address.Offset(13))
+	wt.AssertEquals(t, space1.NumFreeAddressesInRange(address.Range{Start: ip(testAddr1), End: ip(testAddry)}), address.Offset(18))
 	space1.assertInvariants()
 
 	space1.Free(addr2)
@@ -125,32 +139,33 @@ func TestSpaceFree(t *testing.T) {
 		containerID = "deadbeef"
 	)
 
+	entireRange := address.NewRange(ip(testAddr1), 20)
 	space := makeSpace(ip(testAddr1), 20)
 
 	// Check we are prepared to give up the entire space
-	pos, size := space.biggestFreeRange()
-	wt.AssertTrue(t, space.free[pos] == ip(testAddr1) && size == 20, "Wrong space")
+	r := space.biggestFreeRange(entireRange)
+	wt.AssertTrue(t, r.Start == ip(testAddr1) && r.Size() == 20, "Wrong space")
 
 	for i := 0; i < 20; i++ {
-		ok, _ := space.Allocate()
+		ok, _ := space.Allocate(entireRange)
 		wt.AssertTrue(t, ok, "Failed to get address")
 	}
 
 	// Check we are full
-	ok, _ := space.Allocate()
+	ok, _ := space.Allocate(entireRange)
 	wt.AssertTrue(t, !ok, "Should have failed to get address")
-	_, size, ok = space.Donate()
-	wt.AssertTrue(t, size == 0, "Wrong space")
+	r, ok = space.Donate(entireRange)
+	wt.AssertTrue(t, r.Size() == 0, "Wrong space")
 
 	// Free in the middle
 	wt.AssertSuccess(t, space.Free(ip("10.0.3.13")))
-	pos, size = space.biggestFreeRange()
-	wt.AssertTrue(t, space.free[pos] == ip("10.0.3.13") && size == 1, "Wrong space")
+	r = space.biggestFreeRange(entireRange)
+	wt.AssertTrue(t, r.Start == ip("10.0.3.13") && r.Size() == 1, "Wrong space")
 
 	// Free one at the end
 	wt.AssertSuccess(t, space.Free(ip("10.0.3.23")))
-	pos, size = space.biggestFreeRange()
-	wt.AssertTrue(t, space.free[pos] == ip("10.0.3.23") && size == 1, "Wrong space")
+	r = space.biggestFreeRange(entireRange)
+	wt.AssertTrue(t, r.Start == ip("10.0.3.23") && r.Size() == 1, "Wrong space")
 
 	// Now free a few at the end
 	wt.AssertSuccess(t, space.Free(ip("10.0.3.22")))
@@ -159,8 +174,8 @@ func TestSpaceFree(t *testing.T) {
 	wt.AssertEquals(t, space.NumFreeAddresses(), address.Offset(4))
 
 	// Now get the biggest free space; should be 3.21
-	pos, size = space.biggestFreeRange()
-	wt.AssertTrue(t, space.free[pos] == ip("10.0.3.21") && size == 3, "Wrong space")
+	r = space.biggestFreeRange(entireRange)
+	wt.AssertTrue(t, r.Start == ip("10.0.3.21") && r.Size() == 3, "Wrong space")
 
 	// Now free a few in the middle
 	wt.AssertSuccess(t, space.Free(ip("10.0.3.12")))
@@ -170,46 +185,44 @@ func TestSpaceFree(t *testing.T) {
 	wt.AssertEquals(t, space.NumFreeAddresses(), address.Offset(7))
 
 	// Now get the biggest free space; should be 3.21
-	pos, size = space.biggestFreeRange()
-	wt.AssertTrue(t, space.free[pos] == ip("10.0.3.10") && size == 4, "Wrong space")
+	r = space.biggestFreeRange(entireRange)
+	wt.AssertTrue(t, r.Start == ip("10.0.3.10") && r.Size() == 4, "Wrong space")
 
 	wt.AssertEquals(t, space.OwnedRanges(), []address.Range{address.Range{Start: ip("10.0.3.4"), End: ip("10.0.3.24")}})
 }
 
-func (s1 *Space) Equal(s2 *Space) bool {
-	return reflect.DeepEqual(s1.ours, s2.ours) && reflect.DeepEqual(s1.free, s2.free)
-}
-
 func TestDonateSimple(t *testing.T) {
 	const (
 		testAddr1 = "10.0.1.0"
 		testAddr2 = "10.0.1.32"
+		size      = 48
 	)
 
 	var (
 		ipAddr1 = ip(testAddr1)
 	)
 
-	ps1 := makeSpace(ipAddr1, 48)
+	ps1 := makeSpace(ipAddr1, size)
 
 	// Empty space set should split in two and give me the second half
-	start, numGivenUp, ok := ps1.Donate()
+	r, ok := ps1.Donate(address.NewRange(ip(testAddr1), size))
+	numGivenUp := r.Size()
 	wt.AssertTrue(t, ok, "Donate result")
-	wt.AssertEqualString(t, start.String(), "10.0.1.24", "Invalid start")
-	wt.AssertEquals(t, numGivenUp, address.Offset(24))
-	wt.AssertEquals(t, ps1.NumFreeAddresses(), address.Offset(24))
+	wt.AssertEqualString(t, r.Start.String(), "10.0.1.24", "Invalid start")
+	wt.AssertEquals(t, numGivenUp, address.Offset(size/2))
+	wt.AssertEquals(t, ps1.NumFreeAddresses(), address.Offset(size/2))
 
 	// Now check we can give the rest up.
 	count := 0 // count to avoid infinite loop
 	for ; count < 1000; count++ {
-		_, size, ok := ps1.Donate()
+		r, ok := ps1.Donate(address.NewRange(ip(testAddr1), size))
 		if !ok {
 			break
 		}
-		numGivenUp += size
+		numGivenUp += r.Size()
 	}
 	wt.AssertEquals(t, ps1.NumFreeAddresses(), address.Offset(0))
-	wt.AssertEquals(t, numGivenUp, address.Offset(48))
+	wt.AssertEquals(t, numGivenUp, address.Offset(size))
 }
 
 func TestDonateHard(t *testing.T) {
@@ -222,7 +235,7 @@ func TestDonateHard(t *testing.T) {
 	// Fill a fresh space
 	spaceset := makeSpace(start, size)
 	for i := address.Offset(0); i < size; i++ {
-		ok, _ := spaceset.Allocate()
+		ok, _ := spaceset.Allocate(address.NewRange(start, size))
 		wt.AssertTrue(t, ok, "Failed to get IP!")
 	}
 
@@ -235,10 +248,10 @@ func TestDonateHard(t *testing.T) {
 	}
 
 	// Now split
-	newRange, numGivenUp, ok := spaceset.Donate()
+	newRange, ok := spaceset.Donate(address.NewRange(start, size))
 	wt.AssertTrue(t, ok, "GiveUpSpace result")
-	wt.AssertEquals(t, newRange, ip("10.0.1.23"))
-	wt.AssertEquals(t, numGivenUp, address.Offset(24))
+	wt.AssertEquals(t, newRange.Start, ip("10.0.1.23"))
+	wt.AssertEquals(t, newRange.Size(), address.Offset(24))
 	wt.AssertEquals(t, spaceset.NumFreeAddresses(), address.Offset(23))
 
 	//Space set should now have 2 spaces

ipam/testutils_test.go

@@ -3,10 +3,12 @@ package ipam
 import (
 	"fmt"
 	"math/rand"
+	"sync"
 	"testing"
 	"time"
 
 	"github.com/weaveworks/weave/common"
+	"github.com/weaveworks/weave/ipam/address"
 	"github.com/weaveworks/weave/router"
 	wt "github.com/weaveworks/weave/testing"
 )
@@ -30,12 +32,15 @@ func toStringArray(messages []mockMessage) []string {
 }
 
 type mockGossipComms struct {
+	sync.RWMutex
 	t        *testing.T
 	name     string
 	messages []mockMessage
 }
 
 func (m *mockGossipComms) String() string {
+	m.RLock()
+	defer m.RUnlock()
 	return fmt.Sprintf("[mockGossipComms %s]", m.name)
 }
 
@@ -44,6 +49,8 @@ func (m *mockGossipComms) String() string {
 // requires they are not based off iterating through a map.
 
 func (m *mockGossipComms) GossipBroadcast(update router.GossipData) error {
+	m.Lock()
+	defer m.Unlock()
 	buf := []byte{}
 	if len(m.messages) == 0 {
 		m.Fatalf("%s: Gossip broadcast message unexpected: \n%x", m.name, buf)
@@ -76,6 +83,8 @@ func (m *mockGossipComms) Fatalf(format string, args ...interface{}) {
 }
 
 func (m *mockGossipComms) GossipUnicast(dstPeerName router.PeerName, buf []byte) error {
+	m.Lock()
+	defer m.Unlock()
 	if len(m.messages) == 0 {
 		m.Fatalf("%s: Gossip message to %s unexpected: \n%s", m.name, dstPeerName, buf)
 	} else if msg := m.messages[0]; msg.dst == router.UnknownPeerName {
@@ -96,44 +105,52 @@ func (m *mockGossipComms) GossipUnicast(dstPeerName router.PeerName, buf []byte)
 func ExpectMessage(alloc *Allocator, dst string, msgType byte, buf []byte) {
 	m := alloc.gossip.(*mockGossipComms)
 	dstPeerName, _ := router.PeerNameFromString(dst)
+	m.Lock()
 	m.messages = append(m.messages, mockMessage{dstPeerName, msgType, buf})
+	m.Unlock()
 }
 
 func ExpectBroadcastMessage(alloc *Allocator, buf []byte) {
 	m := alloc.gossip.(*mockGossipComms)
+	m.Lock()
 	m.messages = append(m.messages, mockMessage{router.UnknownPeerName, 0, buf})
+	m.Unlock()
 }
 
 func CheckAllExpectedMessagesSent(allocs ...*Allocator) {
 	for _, alloc := range allocs {
 		m := alloc.gossip.(*mockGossipComms)
+		m.RLock()
 		if len(m.messages) > 0 {
 			wt.Fatalf(m.t, "%s: Gossip message(s) not sent as expected: \n%x", m.name, m.messages)
 		}
+		m.RUnlock()
 	}
 }
 
-func makeAllocator(name string, cidr string, quorum uint) *Allocator {
+func makeAllocator(name string, cidrStr string, quorum uint) (*Allocator, address.Range) {
 	peername, err := router.PeerNameFromString(name)
 	if err != nil {
 		panic(err)
 	}
 
-	alloc, err := NewAllocator(peername, router.PeerUID(rand.Int63()),
-		"nick-"+name, cidr, quorum)
+	_, cidr, err := address.ParseCIDR(cidrStr)
 	if err != nil {
 		panic(err)
 	}
 
-	return alloc
+	alloc := NewAllocator(peername, router.PeerUID(rand.Int63()),
+		"nick-"+name, cidr.Range(), quorum)
+
+	return alloc, cidr.HostRange()
 }
 
-func makeAllocatorWithMockGossip(t *testing.T, name string, universeCIDR string, quorum uint) *Allocator {
-	alloc := makeAllocator(name, universeCIDR, quorum)
+func makeAllocatorWithMockGossip(t *testing.T, name string, universeCIDR string, quorum uint) (*Allocator, address.Range) {
+	alloc, subnet := makeAllocator(name, universeCIDR, quorum)
 	gossip := &mockGossipComms{t: t, name: name}
 	alloc.SetInterfaces(gossip)
 	alloc.Start()
-	return alloc
+	return alloc, subnet
 }
 
 func (alloc *Allocator) claimRingForTesting(allocs ...*Allocator) {
@@ -145,6 +162,14 @@ func (alloc *Allocator) claimRingForTesting(allocs ...*Allocator) {
 	alloc.space.AddRanges(alloc.ring.OwnedRanges())
 }
 
+func (alloc *Allocator) NumFreeAddresses(r address.Range) address.Offset {
+	resultChan := make(chan address.Offset)
+	alloc.actionChan <- func() {
+		resultChan <- alloc.space.NumFreeAddressesInRange(r)
+	}
+	return <-resultChan
+}
+
 // Check whether or not something was sent on a channel
 func AssertSent(t *testing.T, ch <-chan bool) {
 	timeout := time.After(10 * time.Second)
@@ -178,7 +203,8 @@ func AssertNothingSentErr(t *testing.T, ch <-chan error) {
 type gossipMessage struct {
 	isUnicast bool
 	sender    *router.PeerName
-	buf       []byte
+	buf       []byte            // for unicast
+	data      router.GossipData // for broadcast
 	exitChan  chan bool
 }
 
@@ -190,7 +216,7 @@ type TestGossipRouter struct {
 func (grouter *TestGossipRouter) GossipBroadcast(update router.GossipData) error {
 	for _, gossipChan := range grouter.gossipChans {
 		select {
-		case gossipChan <- gossipMessage{buf: update.(*ipamGossipData).alloc.encode()}:
+		case gossipChan <- gossipMessage{data: update}:
 		default: // drop the message if we cannot send it
 		}
 	}
@@ -228,14 +254,14 @@ func (grouter *TestGossipRouter) connect(sender router.PeerName, gossiper router
 				}
 
 				if message.isUnicast {
-					err := gossiper.OnGossipUnicast(*message.sender, message.buf)
-					if err != nil {
+					if err := gossiper.OnGossipUnicast(*message.sender, message.buf); err != nil {
 						panic(fmt.Sprintf("Error doing gossip unicast to %s: %s", sender, err))
 					}
 				} else {
-					_, err := gossiper.OnGossipBroadcast(message.buf)
-					if err != nil {
-						panic(fmt.Sprintf("Error doing gossip broadcast to %s: %s", sender, err))
+					for _, msg := range message.data.Encode() {
+						if _, err := gossiper.OnGossipBroadcast(msg); err != nil {
+							panic(fmt.Sprintf("Error doing gossip broadcast to %s: %s", sender, err))
+						}
 					}
 				}
 			case <-gossipTimer:
@@ -261,13 +287,15 @@ func (client TestGossipRouterClient) GossipBroadcast(update router.GossipData) e
 	return client.router.GossipBroadcast(update)
 }
 
-func makeNetworkOfAllocators(size int, cidr string) ([]*Allocator, TestGossipRouter) {
+func makeNetworkOfAllocators(size int, cidr string) ([]*Allocator, TestGossipRouter, address.Range) {
 
 	gossipRouter := TestGossipRouter{make(map[router.PeerName]chan gossipMessage), 0.0}
 	allocs := make([]*Allocator, size)
+	var subnet address.Range
 
 	for i := 0; i < size; i++ {
-		alloc := makeAllocator(fmt.Sprintf("%02d:00:00:02:00:00", i),
+		var alloc *Allocator
+		alloc, subnet = makeAllocator(fmt.Sprintf("%02d:00:00:02:00:00", i),
 			cidr, uint(size/2+1))
 		alloc.SetInterfaces(gossipRouter.connect(alloc.ourName, alloc))
 		alloc.Start()
@@ -276,7 +304,7 @@ func makeNetworkOfAllocators(size int, cidr string) ([]*Allocator, TestGossipRou
 
 	gossipRouter.GossipBroadcast(allocs[size-1].Gossip())
 	time.Sleep(1000 * time.Millisecond)
-	return allocs, gossipRouter
+	return allocs, gossipRouter, subnet
 }
 
 func stopNetworkOfAllocators(allocs []*Allocator) {

nameserver/addrs.go

@@ -25,13 +25,16 @@ type ZoneLookup interface {
 	LookupInaddr(inaddr string) ([]ZoneRecord, error)
 }
 
-type ZoneObserver interface {
+type ZoneLookupFunc func(target string) ([]ZoneRecord, error)
+
+type ZoneObservable interface {
 	// Observe anything that affects a particular name in the zone
 	ObserveName(name string, observer ZoneRecordObserver) error
 	// Observe anything that affects a particular IP in the zone
 	ObserveInaddr(inaddr string, observer ZoneRecordObserver) error
 }
 
+type ZoneObserverFunc func(string, ZoneRecordObserver) error
 type ZoneRecordObserver func()
 
 /////////////////////////////////////////////////////////////

nameserver/addrs_test.go

@@ -1,10 +1,11 @@
 package nameserver
 
 import (
-	. "github.com/weaveworks/weave/common"
-	wt "github.com/weaveworks/weave/testing"
 	"net"
 	"testing"
+
+	. "github.com/weaveworks/weave/common"
+	wt "github.com/weaveworks/weave/testing"
 )
 
 func TestAddrs(t *testing.T) {

nameserver/cache.go

@@ -5,13 +5,13 @@ import (
 	"container/heap"
 	"errors"
 	"fmt"
+	"math"
+	"sync"
+	"time"
+
 	"github.com/benbjohnson/clock"
 	"github.com/miekg/dns"
 	. "github.com/weaveworks/weave/common"
-	"math"
-	"math/rand"
-	"sync"
-	"time"
 )
 
 var (
@@ -43,21 +43,6 @@ const (
 	CacheNoLocalReplies uint8 = 1 << iota // not found in local network (stored in the cache so we skip another local lookup or some time)
 )
 
-// shuffleAnswers reorders answers for very basic load balancing
-func shuffleAnswers(answers []dns.RR) []dns.RR {
-	n := len(answers)
-	if n > 1 {
-		rand.Seed(time.Now().UTC().UnixNano())
-
-		for i := 0; i < n; i++ {
-			r := i + rand.Intn(n-i)
-			answers[r], answers[i] = answers[i], answers[r]
-		}
-	}
-
-	return answers
-}
-
 // a cache entry
 type cacheEntry struct {
 	Status   entryStatus // status of the entry
@@ -124,9 +109,6 @@ func (e *cacheEntry) getReply(request *dns.Msg, maxLen int, now time.Time) (*dns
 	reply.Rcode = e.reply.Rcode
 	reply.Authoritative = true
 
-	// shuffle the values, etc...
-	reply.Answer = shuffleAnswers(reply.Answer)
-
 	return reply, nil
 }
 
@@ -145,9 +127,12 @@ func (e *cacheEntry) setReply(reply *dns.Msg, ttl int, flags uint8, now time.Tim
 		prevValidUntil = e.validUntil
 	}
 
-	e.Status = stResolved
-	e.Flags = flags
-	e.putTime = now
+	// make sure we do not overwrite noLocalReplies entries
+	if flags&CacheNoLocalReplies != 0 {
+		if e.Flags&CacheNoLocalReplies != 0 {
+			return false
+		}
+	}
 
 	if ttl != nullTTL {
 		e.validUntil = now.Add(time.Second * time.Duration(ttl))
@@ -161,10 +146,16 @@ func (e *cacheEntry) setReply(reply *dns.Msg, ttl int, flags uint8, now time.Tim
 			}
 		}
 		e.validUntil = now.Add(time.Second * time.Duration(minTTL))
+	} else {
+		Warning.Printf("[cache] no valid TTL could be calculated")
 	}
 
+	e.Status = stResolved
+	e.Flags = flags
+	e.putTime = now
+
 	if reply != nil {
-		e.reply = *reply
+		e.reply = *reply.Copy()
 		e.ReplyLen = reply.Len()
 	}
 

nameserver/cache_test.go

@@ -2,13 +2,14 @@ package nameserver
 
 import (
 	"fmt"
+	"net"
+	"testing"
+	"time"
+
 	"github.com/benbjohnson/clock"
 	"github.com/miekg/dns"
 	. "github.com/weaveworks/weave/common"
 	wt "github.com/weaveworks/weave/testing"
-	"net"
-	"testing"
-	"time"
 )
 
 // Check that the cache keeps its intended capacity constant
@@ -35,7 +36,7 @@ func TestCacheLength(t *testing.T) {
 		ip := net.ParseIP(fmt.Sprintf("10.0.1.%d", i))
 		records := []ZoneRecord{Record{name, ip, 0, 0, 0}}
 
-		reply := makeAddressReply(questionMsg, question, records)
+		reply := makeAddressReply(questionMsg, question, records, DefaultLocalTTL)
 		reply.Answer[0].Header().Ttl = uint32(i)
 
 		l.Put(questionMsg, reply, 0, 0)
@@ -88,7 +89,7 @@ func TestCacheEntries(t *testing.T) {
 
 	t.Logf("Inserting the reply")
 	records := []ZoneRecord{Record{"some.name", net.ParseIP("10.0.1.1"), 0, 0, 0}}
-	reply1 := makeAddressReply(questionMsg, question, records)
+	reply1 := makeAddressReply(questionMsg, question, records, DefaultLocalTTL)
 	l.Put(questionMsg, reply1, nullTTL, 0)
 
 	t.Logf("Checking we can Get() the reply now")
@@ -109,8 +110,8 @@ func TestCacheEntries(t *testing.T) {
 	ttlGet2 := resp.Answer[0].Header().Ttl
 	wt.AssertEqualInt(t, int(ttlGet1-ttlGet2), 1, "TTL difference")
 
-	clk.Add(time.Duration(localTTL) * time.Second)
-	t.Logf("Checking that a third Get(), after %d second, gets no result", localTTL)
+	clk.Add(time.Duration(DefaultLocalTTL) * time.Second)
+	t.Logf("Checking that a third Get(), after %d second, gets no result", DefaultLocalTTL)
 	resp, err = l.Get(questionMsg, minUDPSize)
 	wt.AssertNoErr(t, err)
 	if resp != nil {
@@ -120,7 +121,7 @@ func TestCacheEntries(t *testing.T) {
 
 	t.Logf("Checking that an Remove() results in Get() returning nothing")
 	records = []ZoneRecord{Record{"some.name", net.ParseIP("10.0.9.9"), 0, 0, 0}}
-	replyTemp := makeAddressReply(questionMsg, question, records)
+	replyTemp := makeAddressReply(questionMsg, question, records, DefaultLocalTTL)
 	l.Put(questionMsg, replyTemp, nullTTL, 0)
 	lenBefore := l.Len()
 	l.Remove(question)
@@ -134,11 +135,11 @@ func TestCacheEntries(t *testing.T) {
 
 	t.Logf("Inserting a two replies for the same query")
 	records = []ZoneRecord{Record{"some.name", net.ParseIP("10.0.1.2"), 0, 0, 0}}
-	reply2 := makeAddressReply(questionMsg, question, records)
+	reply2 := makeAddressReply(questionMsg, question, records, DefaultLocalTTL)
 	l.Put(questionMsg, reply2, nullTTL, 0)
 	clk.Add(time.Duration(1) * time.Second)
 	records = []ZoneRecord{Record{"some.name", net.ParseIP("10.0.1.3"), 0, 0, 0}}
-	reply3 := makeAddressReply(questionMsg, question, records)
+	reply3 := makeAddressReply(questionMsg, question, records, DefaultLocalTTL)
 	l.Put(questionMsg, reply3, nullTTL, 0)
 
 	t.Logf("Checking we get the last one...")
@@ -148,9 +149,9 @@ func TestCacheEntries(t *testing.T) {
 	t.Logf("Received '%s'", resp.Answer[0])
 	wt.AssertType(t, resp.Answer[0], (*dns.A)(nil), "DNS record")
 	wt.AssertEqualString(t, resp.Answer[0].(*dns.A).A.String(), "10.0.1.3", "IP address")
-	wt.AssertEqualInt(t, int(resp.Answer[0].Header().Ttl), int(localTTL), "TTL")
+	wt.AssertEqualInt(t, int(resp.Answer[0].Header().Ttl), int(DefaultLocalTTL), "TTL")
 
-	clk.Add(time.Duration(localTTL-1) * time.Second)
+	clk.Add(time.Duration(DefaultLocalTTL-1) * time.Second)
 	resp, err = l.Get(questionMsg, minUDPSize)
 	wt.AssertNoErr(t, err)
 	wt.AssertTrue(t, resp != nil, "reponse from the Get()")
@@ -161,7 +162,7 @@ func TestCacheEntries(t *testing.T) {
 
 	t.Logf("Checking we get empty replies when they are expired...")
 	lenBefore = l.Len()
-	clk.Add(time.Duration(localTTL) * time.Second)
+	clk.Add(time.Duration(DefaultLocalTTL) * time.Second)
 	resp, err = l.Get(questionMsg, minUDPSize)
 	wt.AssertNoErr(t, err)
 	if resp != nil {
@@ -182,7 +183,7 @@ func TestCacheEntries(t *testing.T) {
 
 	t.Logf("Checking that an Remove() between Get() and Put() does not break things")
 	records = []ZoneRecord{Record{"some.name", net.ParseIP("10.0.9.9"), 0, 0, 0}}
-	replyTemp2 := makeAddressReply(questionMsg2, question2, records)
+	replyTemp2 := makeAddressReply(questionMsg2, question2, records, DefaultLocalTTL)
 	l.Remove(question2)
 	l.Put(questionMsg2, replyTemp2, nullTTL, 0)
 	resp, err = l.Get(questionMsg2, minUDPSize)
@@ -195,13 +196,26 @@ func TestCacheEntries(t *testing.T) {
 	question3 := &questionMsg3.Question[0]
 
 	t.Logf("Checking that a entry with CacheNoLocalReplies return an error")
-	l.Put(questionMsg3, nil, nullTTL, CacheNoLocalReplies)
+	l.Put(questionMsg3, nil, DefaultLocalTTL, CacheNoLocalReplies)
 	resp, err = l.Get(questionMsg3, minUDPSize)
 	wt.AssertNil(t, resp, "Get() response with CacheNoLocalReplies")
-	wt.AssertNotNil(t, err, "Get() error with CacheNoLocalReplies")
+	wt.AssertTrue(t, err == errNoLocalReplies, "Get() error with CacheNoLocalReplies")
 
-	clk.Add(time.Second * time.Duration(negLocalTTL+1))
-	t.Logf("Checking that we get an expired response after %f seconds", negLocalTTL)
+	t.Logf("Checking that more Put's do not make the negative TTL longer")
+	clk.Add(time.Duration(DefaultLocalTTL/2) * time.Second)
+	l.Put(questionMsg3, nil, DefaultLocalTTL, CacheNoLocalReplies)
+	clk.Add(time.Duration(1) * time.Second)
+	l.Put(questionMsg3, nil, DefaultLocalTTL, CacheNoLocalReplies)
+	clk.Add(time.Duration(1) * time.Second)
+	l.Put(questionMsg3, nil, DefaultLocalTTL, CacheNoLocalReplies)
+	resp, err = l.Get(questionMsg3, minUDPSize)
+	wt.AssertNil(t, resp, "Get() response with CacheNoLocalReplies")
+	wt.AssertTrue(t, err == errNoLocalReplies, "Get() error with CacheNoLocalReplies")
+
+	clk.Add(time.Duration(DefaultLocalTTL/2) * time.Second)
+	t.Logf("Checking that we get an expired response after %f seconds", DefaultLocalTTL)
+	clk.Add(time.Second * time.Duration(DefaultLocalTTL+1))
+	t.Logf("Checking that we get an expired response after %d seconds", DefaultLocalTTL)
 	resp, err = l.Get(questionMsg3, minUDPSize)
 	wt.AssertNil(t, resp, "expired Get() response with CacheNoLocalReplies")
 	wt.AssertNil(t, err, "expired Get() error with CacheNoLocalReplies")

nameserver/dns.go

@@ -1,20 +1,24 @@
 package nameserver
 
 import (
+	"math/rand"
+	"time"
+
 	"github.com/miekg/dns"
 )
 
 const (
-	localTTL    uint32 = 30 // somewhat arbitrary; we don't expect anyone downstream to cache results
-	negLocalTTL        = 30 // TTL for negative local resolutions
-	minUDPSize         = 512
-	maxUDPSize         = 65535
+	minUDPSize = 512
+	maxUDPSize = 65535
 )
 
-func makeHeader(r *dns.Msg, q *dns.Question) *dns.RR_Header {
+func makeHeader(r *dns.Msg, q *dns.Question, ttl int) *dns.RR_Header {
 	return &dns.RR_Header{
-		Name: q.Name, Rrtype: q.Qtype,
-		Class: dns.ClassINET, Ttl: localTTL}
+		Name:   q.Name,
+		Rrtype: q.Qtype,
+		Class:  dns.ClassINET,
+		Ttl:    uint32(ttl),
+	}
 }
 
 func makeReply(r *dns.Msg, as []dns.RR) *dns.Msg {
@@ -33,9 +37,11 @@ func makeTruncatedReply(r *dns.Msg) *dns.Msg {
 	return reply
 }
 
-func makeAddressReply(r *dns.Msg, q *dns.Question, addrs []ZoneRecord) *dns.Msg {
+type DNSResponseBuilder func(r *dns.Msg, q *dns.Question, addrs []ZoneRecord, ttl int) *dns.Msg
+
+func makeAddressReply(r *dns.Msg, q *dns.Question, addrs []ZoneRecord, ttl int) *dns.Msg {
 	answers := make([]dns.RR, len(addrs))
-	header := makeHeader(r, q)
+	header := makeHeader(r, q, ttl)
 	count := 0
 	for _, addr := range addrs {
 		ip := addr.IP()
@@ -57,9 +63,9 @@ func makeAddressReply(r *dns.Msg, q *dns.Question, addrs []ZoneRecord) *dns.Msg
 	return makeReply(r, answers[:count])
 }
 
-func makePTRReply(r *dns.Msg, q *dns.Question, names []ZoneRecord) *dns.Msg {
+func makePTRReply(r *dns.Msg, q *dns.Question, names []ZoneRecord, ttl int) *dns.Msg {
 	answers := make([]dns.RR, len(names))
-	header := makeHeader(r, q)
+	header := makeHeader(r, q, ttl)
 	for i, name := range names {
 		answers[i] = &dns.PTR{Hdr: *header, Ptr: name.Name()}
 	}
@@ -84,3 +90,28 @@ func getMaxReplyLen(r *dns.Msg, proto dnsProtocol) int {
 	}
 	return maxLen
 }
+
+func init() {
+	rand.Seed(time.Now().UTC().UnixNano())
+}
+
+// shuffleAnswers reorders answers for very basic load balancing
+func shuffleAnswers(answers []dns.RR) []dns.RR {
+	if len(answers) > 1 {
+		for i := range answers {
+			j := rand.Intn(i + 1)
+			answers[i], answers[j] = answers[j], answers[i]
+		}
+	}
+
+	return answers
+}
+
+// only take the first `num` answers
+func pruneAnswers(answers []dns.RR, num int) []dns.RR {
+	if num > 0 && len(answers) > num {
+		// TODO: we should have some prefer locally-introduced answers, etc...
+		return answers[:num]
+	}
+	return answers
+}

nameserver/dns_test.go

@@ -0,0 +1,39 @@
+package nameserver
+
+import (
+	"net"
+	"testing"
+
+	"github.com/miekg/dns"
+	. "github.com/weaveworks/weave/common"
+	wt "github.com/weaveworks/weave/testing"
+)
+
+// Check that we can prune an answer
+func TestPrune(t *testing.T) {
+	InitDefaultLogging(testing.Verbose())
+	Info.Println("TestPrune starting")
+
+	questionMsg := new(dns.Msg)
+	questionMsg.SetQuestion("name", dns.TypeA)
+	questionMsg.RecursionDesired = true
+	question := &questionMsg.Question[0]
+	records := []ZoneRecord{
+		Record{"name", net.ParseIP("10.0.1.1"), 0, 0, 0},
+		Record{"name", net.ParseIP("10.0.1.2"), 0, 0, 0},
+		Record{"name", net.ParseIP("10.0.1.3"), 0, 0, 0},
+		Record{"name", net.ParseIP("10.0.1.4"), 0, 0, 0},
+	}
+
+	reply := makeAddressReply(questionMsg, question, records, DefaultLocalTTL)
+	reply.Answer[0].Header().Ttl = DefaultLocalTTL
+
+	pruned := pruneAnswers(reply.Answer, 1)
+	wt.AssertEqualInt(t, len(pruned), 1, "wrong number of answers")
+
+	pruned = pruneAnswers(reply.Answer, 2)
+	wt.AssertEqualInt(t, len(pruned), 2, "wrong number of answers")
+
+	pruned = pruneAnswers(reply.Answer, 0)
+	wt.AssertEqualInt(t, len(pruned), len(records), "wrong number of answers")
+}

nameserver/http.go

@@ -2,12 +2,14 @@ package nameserver
 
 import (
 	"fmt"
-	"github.com/gorilla/mux"
-	"github.com/miekg/dns"
-	. "github.com/weaveworks/weave/common"
 	"log"
 	"net"
 	"net/http"
+
+	"github.com/gorilla/mux"
+	"github.com/miekg/dns"
+	. "github.com/weaveworks/weave/common"
+	"github.com/weaveworks/weave/common/docker"
 )
 
 func httpErrorAndLog(level *log.Logger, w http.ResponseWriter, msg string,
@@ -16,13 +18,25 @@ func httpErrorAndLog(level *log.Logger, w http.ResponseWriter, msg string,
 	level.Printf("[http] "+logmsg, logargs...)
 }
 
-func ListenHTTP(version string, server *DNSServer, domain string, db Zone, port int) {
+func extractFQDN(r *http.Request) (string, string) {
+	if fqdnStr := r.FormValue("fqdn"); fqdnStr != "" {
+		return fqdnStr, fqdnStr
+	}
+	return "*", ""
+}
+
+func ServeHTTP(listener net.Listener, version string, server *DNSServer, dockerCli *docker.Client) {
 
 	muxRouter := mux.NewRouter()
 
 	muxRouter.Methods("GET").Path("/status").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		fmt.Fprintln(w, "weave DNS", version)
 		fmt.Fprintln(w, server.Status())
+		fmt.Fprintln(w, server.Zone.Status())
+	})
+
+	muxRouter.Methods("GET").Path("/domain").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprint(w, server.Zone.Domain())
 	})
 
 	muxRouter.Methods("PUT").Path("/name/{id:.+}/{ip:.+}").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -46,50 +60,66 @@ func ListenHTTP(version string, server *DNSServer, domain string, db Zone, port
 			return
 		}
 
-		if dns.IsSubDomain(domain, name) {
-			Info.Printf("[http] Adding %s -> %s", name, ipStr)
-			if err := db.AddRecord(idStr, name, ip); err != nil {
-				if _, ok := err.(DuplicateError); !ok {
-					httpErrorAndLog(
-						Error, w, "Internal error", http.StatusInternalServerError,
-						"Unexpected error from DB: %s", err)
-					return
-				} // oh, I already know this. whatever.
-			}
-		} else {
+		domain := server.Zone.Domain()
+		if !dns.IsSubDomain(domain, name) {
 			Info.Printf("[http] Ignoring name %s, not in %s", name, domain)
+			return
+		}
+		Info.Printf("[http] Adding %s -> %s", name, ipStr)
+		if err := server.Zone.AddRecord(idStr, name, ip); err != nil {
+			if _, ok := err.(DuplicateError); !ok {
+				httpErrorAndLog(
+					Error, w, "Internal error", http.StatusInternalServerError,
+					"Unexpected error from DB: %s", err)
+				return
+			} // oh, I already know this. whatever.
+		}
+
+		if dockerCli != nil && dockerCli.IsContainerNotRunning(idStr) {
+			Info.Printf("[http] '%s' is not running: removing", idStr)
+			server.Zone.DeleteRecords(idStr, name, ip)
 		}
 	})
 
 	muxRouter.Methods("DELETE").Path("/name/{id:.+}/{ip:.+}").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		reqError := func(msg string, logmsg string, logargs ...interface{}) {
-			httpErrorAndLog(Warning, w, msg, http.StatusBadRequest, logmsg, logargs...)
-		}
-
 		vars := mux.Vars(r)
 		idStr := vars["id"]
 		ipStr := vars["ip"]
 
 		ip := net.ParseIP(ipStr)
 		if ip == nil {
-			reqError("Invalid IP in request", "Invalid IP in request: %s", ipStr)
+			httpErrorAndLog(
+				Warning, w, "Invalid IP in request", http.StatusBadRequest,
+				"Invalid IP in request: %s", ipStr)
 			return
 		}
-		Info.Printf("[http] Deleting %s (%s)", idStr, ipStr)
-		if err := db.DeleteRecord(idStr, ip); err != nil {
-			if _, ok := err.(LookupError); !ok {
-				httpErrorAndLog(
-					Error, w, "Internal error", http.StatusInternalServerError,
-					"Unexpected error from DB: %s", err)
-				return
-			}
-		}
+
+		fqdnStr, fqdn := extractFQDN(r)
+
+		Info.Printf("[http] Deleting ID %s, IP %s, FQDN %s", idStr, ipStr, fqdnStr)
+		server.Zone.DeleteRecords(idStr, fqdn, ip)
+	})
+
+	muxRouter.Methods("DELETE").Path("/name/{id:.+}").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		vars := mux.Vars(r)
+		idStr := vars["id"]
+
+		fqdnStr, fqdn := extractFQDN(r)
+
+		Info.Printf("[http] Deleting ID %s, IP *, FQDN %s", idStr, fqdnStr)
+		server.Zone.DeleteRecords(idStr, fqdn, net.IP{})
+	})
+
+	muxRouter.Methods("DELETE").Path("/name").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		fqdnStr, fqdn := extractFQDN(r)
+
+		Info.Printf("[http] Deleting ID *, IP *, FQDN %s", fqdnStr)
+		server.Zone.DeleteRecords("", fqdn, net.IP{})
 	})
 
 	http.Handle("/", muxRouter)
 
-	address := fmt.Sprintf(":%d", port)
-	if err := http.ListenAndServe(address, nil); err != nil {
-		Error.Fatal("[http] Unable to create http listener: ", err)
+	if err := http.Serve(listener, nil); err != nil {
+		Error.Fatal("[http] Unable to serve http: ", err)
 	}
 }

nameserver/http_test.go

@@ -2,14 +2,13 @@ package nameserver
 
 import (
 	"fmt"
-	wt "github.com/weaveworks/weave/testing"
-	"math/rand"
 	"net"
 	"net/http"
 	"net/url"
 	"strings"
 	"testing"
-	"time"
+
+	wt "github.com/weaveworks/weave/testing"
 )
 
 func genForm(method string, url string, data url.Values) (resp *http.Response, err error) {
@@ -24,16 +23,26 @@ func TestHttp(t *testing.T) {
 		testDomain      = "weave.local."
 		successTestName = "test1." + testDomain
 		testAddr1       = "10.2.2.1/24"
+		testAddr2       = "10.2.2.2/24"
 		dockerIP        = "9.8.7.6"
 	)
 
-	zone := NewZoneDb(DefaultLocalDomain)
+	zone, err := NewZoneDb(ZoneConfig{Domain: testDomain})
+	wt.AssertNoErr(t, err)
+	err = zone.Start()
+	wt.AssertNoErr(t, err)
+	defer zone.Stop()
 
-	port := rand.Intn(10000) + 32768
-	fmt.Println("Http test on port", port)
-	go ListenHTTP("", nil, testDomain, zone, port)
+	httpListener, err := net.Listen("tcp", ":0")
+	if err != nil {
+		t.Fatal("Unable to create http listener: ", err)
+	}
 
-	time.Sleep(100 * time.Millisecond) // Allow for http server to get going
+	// Create a useless server... it will not even be started
+	srv, _ := NewDNSServer(DNSServerConfig{Zone: zone})
+
+	port := httpListener.Addr().(*net.TCPAddr).Port
+	go ServeHTTP(httpListener, "", srv, nil)
 
 	// Ask the http server to add our test address into the database
 	addrParts := strings.Split(testAddr1, "/")
@@ -65,6 +74,31 @@ func TestHttp(t *testing.T) {
 	wt.AssertNoErr(t, err)
 	wt.AssertStatus(t, resp.StatusCode, http.StatusOK, "http response")
 
+	// Adding a new IP for the same name should be OK
+	addrParts2 := strings.Split(testAddr2, "/")
+	addrURL2 := fmt.Sprintf("http://localhost:%d/name/%s/%s", port, containerID, addrParts2[0])
+	resp, err = genForm("PUT", addrURL2,
+		url.Values{"fqdn": {successTestName}, "local_ip": {dockerIP}, "routing_prefix": {addrParts2[1]}})
+	wt.AssertNoErr(t, err)
+	wt.AssertStatus(t, resp.StatusCode, http.StatusOK, "http success response for second IP")
+
+	// Check that we can get two IPs for that name
+	ip2, _, _ := net.ParseCIDR(testAddr2)
+	foundIP, err = zone.LookupName(successTestName)
+	wt.AssertNoErr(t, err)
+	if len(foundIP) != 2 {
+		t.Logf("IPs found: %s", foundIP)
+		t.Fatalf("Unexpected result length: received %d responses", len(foundIP))
+	}
+	if !(foundIP[0].IP().Equal(ip) || foundIP[0].IP().Equal(ip2)) {
+		t.Logf("IPs found: %s", foundIP)
+		t.Fatalf("Unexpected result for %s: received %s, expected %s", successTestName, foundIP, ip)
+	}
+	if !(foundIP[1].IP().Equal(ip) || foundIP[1].IP().Equal(ip2)) {
+		t.Logf("IPs found: %s", foundIP)
+		t.Fatalf("Unexpected result for %s: received %s, expected %s", successTestName, foundIP, ip)
+	}
+
 	// Delete the address
 	resp, err = genForm("DELETE", addrURL, nil)
 	wt.AssertNoErr(t, err)
@@ -80,6 +114,11 @@ func TestHttp(t *testing.T) {
 	wt.AssertNoErr(t, err)
 	wt.AssertStatus(t, resp.StatusCode, http.StatusOK, "http response")
 
+	// Delete the second IP
+	resp, err = genForm("DELETE", addrURL2, nil)
+	wt.AssertNoErr(t, err)
+	wt.AssertStatus(t, resp.StatusCode, http.StatusOK, "http response")
+
 	// Check that the address is gone
 	x, err = zone.LookupName(successTestName)
 	t.Logf("Got %s", x)

nameserver/mdns.go

@@ -0,0 +1,28 @@
+package nameserver
+
+import "net"
+
+// A basic mDNS service
+type ZoneMDNS interface {
+	// Start the service
+	Start(*net.Interface) error
+	// Stop the service
+	Stop() error
+}
+
+// A mDNS server
+type ZoneMDNSServer interface {
+	ZoneMDNS
+	// Return the Zone database used by the server
+	Zone() Zone
+}
+
+// A mDNS client interface
+type ZoneMDNSClient interface {
+	ZoneMDNS
+	ZoneLookup
+	// Perform an insistent lookup for a name
+	InsistentLookupName(name string) ([]ZoneRecord, error)
+	// Perform an insistent lookup for a reverse address
+	InsistentLookupInaddr(inaddr string) ([]ZoneRecord, error)
+}

nameserver/mdns_client.go

@@ -3,10 +3,11 @@ package nameserver
 import (
 	"bytes"
 	"fmt"
-	"github.com/miekg/dns"
 	"math"
 	"net"
 	"time"
+
+	"github.com/miekg/dns"
 )
 
 // Portions of this code taken from github.com/armon/mdns
@@ -96,7 +97,6 @@ type MDNSClient struct {
 	addr       *net.UDPAddr
 	inflight   map[string]*inflightQuery
 	actionChan chan<- MDNSAction
-	running    bool
 }
 
 type mDNSQueryInfo struct {
@@ -108,15 +108,10 @@ type mDNSQueryInfo struct {
 func NewMDNSClient() (*MDNSClient, error) {
 	return &MDNSClient{
 		addr:     ipv4Addr,
-		running:  false,
 		inflight: make(map[string]*inflightQuery)}, nil
 }
 
 func (c *MDNSClient) Start(ifi *net.Interface) (err error) {
-	if c.running {
-		return nil
-	}
-
 	if c.conn, err = net.ListenUDP("udp4", &net.UDPAddr{IP: net.IPv4zero, Port: 0}); err != nil {
 		return err
 	}
@@ -133,19 +128,18 @@ func (c *MDNSClient) Start(ifi *net.Interface) (err error) {
 		}
 	}
 
-	c.listener = &dns.Server{Unsafe: true, PacketConn: multicast, Handler: dns.HandlerFunc(handleMDNS)}
-	go c.listener.ActivateAndServe()
-
 	actionChan := make(chan MDNSAction, MailboxSize)
 	c.actionChan = actionChan
 	go c.actorLoop(actionChan)
+
+	c.listener = &dns.Server{Unsafe: true, PacketConn: multicast, Handler: dns.HandlerFunc(handleMDNS)}
+	go c.listener.ActivateAndServe()
+
 	return nil
 }
 
 func (c *MDNSClient) Stop() error {
-	if c.running {
-		c.actionChan <- nil
-	}
+	c.actionChan <- nil
 	return nil
 }
 
@@ -271,13 +265,13 @@ func (c *MDNSClient) checkInFlightQueries() time.Duration {
 func (c *MDNSClient) actorLoop(actionChan <-chan MDNSAction) {
 	timer := time.NewTimer(MaxDuration)
 	run := func() { timer.Reset(c.checkInFlightQueries()) }
-	c.running = true
-	for c.running {
+loop:
+	for {
 		select {
 		case action := <-actionChan:
 			if action == nil {
 				c.listener.Shutdown()
-				c.running = false
+				break loop
 			} else {
 				action()
 				run()

nameserver/mdns_client_test.go

@@ -1,13 +1,14 @@
 package nameserver
 
 import (
-	"github.com/miekg/dns"
-	. "github.com/weaveworks/weave/common"
-	wt "github.com/weaveworks/weave/testing"
 	"log"
 	"net"
 	"testing"
 	"time"
+
+	"github.com/miekg/dns"
+	. "github.com/weaveworks/weave/common"
+	wt "github.com/weaveworks/weave/testing"
 )
 
 var (

nameserver/mdns_server.go

@@ -1,9 +1,10 @@
 package nameserver
 
 import (
+	"net"
+
 	"github.com/miekg/dns"
 	. "github.com/weaveworks/weave/common"
-	"net"
 )
 
 type MDNSServer struct {
@@ -11,13 +12,18 @@ type MDNSServer struct {
 	sendconn   *net.UDPConn
 	srv        *dns.Server
 	zone       Zone
-	running    bool
+	allowLocal bool
+	ttl        int
 }
 
 // Create a new mDNS server
 // Nothing will be done (including port bindings) until you `Start()` the server
-func NewMDNSServer(zone Zone) (*MDNSServer, error) {
-	return &MDNSServer{zone: zone}, nil
+func NewMDNSServer(zone Zone, local bool, ttl int) (*MDNSServer, error) {
+	return &MDNSServer{
+		zone:       zone,
+		allowLocal: local,
+		ttl:        ttl,
+	}, nil
 }
 
 // Return true if testaddr is a UDP address with IP matching my local i/f
@@ -36,11 +42,6 @@ func (s *MDNSServer) addrIsLocal(testaddr net.Addr) bool {
 
 // Start the mDNS server
 func (s *MDNSServer) Start(ifi *net.Interface) (err error) {
-	// skip double initialization
-	if s.running {
-		return nil
-	}
-
 	// This is a bit of a kludge - per the RFC we should send responses from 5353, but that doesn't seem to work
 	s.sendconn, err = net.ListenUDP("udp4", &net.UDPAddr{IP: net.IPv4zero, Port: 0})
 	if err != nil {
@@ -64,7 +65,7 @@ func (s *MDNSServer) Start(ifi *net.Interface) (err error) {
 	handleLocal := s.makeHandler(dns.TypeA,
 		func(zone ZoneLookup, r *dns.Msg, q *dns.Question) *dns.Msg {
 			if ips, err := zone.LookupName(q.Name); err == nil {
-				return makeAddressReply(r, q, ips)
+				return makeAddressReply(r, q, ips, s.ttl)
 			}
 			return nil
 		})
@@ -72,7 +73,7 @@ func (s *MDNSServer) Start(ifi *net.Interface) (err error) {
 	handleReverse := s.makeHandler(dns.TypePTR,
 		func(zone ZoneLookup, r *dns.Msg, q *dns.Question) *dns.Msg {
 			if names, err := zone.LookupInaddr(q.Name); err == nil {
-				return makePTRReply(r, q, names)
+				return makePTRReply(r, q, names, s.ttl)
 			}
 			return nil
 		})
@@ -87,13 +88,11 @@ func (s *MDNSServer) Start(ifi *net.Interface) (err error) {
 		Handler:    mux,
 	}
 	go s.srv.ActivateAndServe()
-	s.running = true
 	return err
 }
 
 // Stop the mDNS server
 func (s *MDNSServer) Stop() error {
-	s.running = false
 	return s.srv.Shutdown()
 }
 
@@ -112,6 +111,12 @@ func (s *MDNSServer) makeHandler(qtype uint16, lookup LookupFunc) dns.HandlerFun
 		// assumption that the client wouldn't ask if it already knew
 		// the answer, and if it does ask, it'll be happy to get an
 		// answer.
+		remoteAddr := rw.RemoteAddr()
+		if s.addrIsLocal(remoteAddr) && !s.allowLocal {
+			Debug.Printf("[mdns] srv: mDNS query from local host ('%s'): ignored", remoteAddr)
+			return
+		}
+
 		// TODO: use any Answer in the query for adding records in the Zone database...
 		if len(r.Answer) == 0 && len(r.Question) > 0 {
 			q := &r.Question[0]

nameserver/mdns_server_test.go

@@ -1,12 +1,13 @@
 package nameserver
 
 import (
-	"github.com/miekg/dns"
-	. "github.com/weaveworks/weave/common"
-	wt "github.com/weaveworks/weave/testing"
 	"net"
 	"testing"
 	"time"
+
+	"github.com/miekg/dns"
+	. "github.com/weaveworks/weave/common"
+	wt "github.com/weaveworks/weave/testing"
 )
 
 func TestServerSimpleQuery(t *testing.T) {
@@ -20,7 +21,7 @@ func TestServerSimpleQuery(t *testing.T) {
 	Info.Println("TestServerSimpleQuery starting")
 
 	mzone := newMockedZoneWithRecords([]ZoneRecord{testRecord1, testRecord2})
-	mdnsServer, err := NewMDNSServer(mzone)
+	mdnsServer, err := NewMDNSServer(mzone, true, DefaultLocalTTL)
 	wt.AssertNoErr(t, err)
 	err = mdnsServer.Start(nil)
 	wt.AssertNoErr(t, err)
@@ -40,16 +41,7 @@ func TestServerSimpleQuery(t *testing.T) {
 		if len(r.Answer) > 0 {
 			t.Logf("Received %d answer(s)", len(r.Answer))
 			for _, answer := range r.Answer {
-				switch rr := answer.(type) {
-				case *dns.A:
-					t.Logf("... A:\n%+v", rr)
-					receivedAddrs = append(receivedAddrs, rr.A)
-					receivedCount++
-				case *dns.PTR:
-					t.Logf("... PTR:\n%+v", rr)
-					receivedName = rr.Ptr
-					receivedCount++
-				}
+				recvChan <- answer
 			}
 			recvChan <- "ok"
 		}
@@ -59,7 +51,6 @@ func TestServerSimpleQuery(t *testing.T) {
 		receivedAddrs = make([]net.IP, 0)
 		receivedName = ""
 		receivedCount = 0
-		recvChan = make(chan interface{})
 
 		m := new(dns.Msg)
 		m.SetQuestion(name, querytype)
@@ -73,12 +64,25 @@ func TestServerSimpleQuery(t *testing.T) {
 		wt.AssertNoErr(t, err)
 
 		Debug.Printf("Waiting for response")
-		select {
-		case <-recvChan:
-			return
-		case <-time.After(100 * time.Millisecond):
-			Debug.Printf("Timeout while waiting for response")
-			return
+		for {
+			select {
+			case x := <-recvChan:
+				switch rr := x.(type) {
+				case *dns.A:
+					t.Logf("... A:\n%+v", rr)
+					receivedAddrs = append(receivedAddrs, rr.A)
+					receivedCount++
+				case *dns.PTR:
+					t.Logf("... PTR:\n%+v", rr)
+					receivedName = rr.Ptr
+					receivedCount++
+				case string:
+					return
+				}
+			case <-time.After(100 * time.Millisecond):
+				Debug.Printf("Timeout while waiting for response")
+				return
+			}
 		}
 	}
 

nameserver/mdns_test.go

@@ -0,0 +1,170 @@
+package nameserver
+
+import (
+	"net"
+	"testing"
+	"time"
+
+	"github.com/miekg/dns"
+	. "github.com/weaveworks/weave/common"
+	wt "github.com/weaveworks/weave/testing"
+)
+
+// Check that we can use a regular mDNS server with a regular mDNS client
+func TestClientServerSimpleQuery(t *testing.T) {
+	InitDefaultLogging(testing.Verbose())
+
+	testRecord1 := Record{"test.weave.local.", net.ParseIP("10.2.2.1"), 0, 0, 0}
+	testInAddr1 := "1.2.2.10.in-addr.arpa."
+
+	mzone := newMockedZoneWithRecords([]ZoneRecord{testRecord1})
+	mdnsServer, err := NewMDNSServer(mzone, true, DefaultLocalTTL)
+	wt.AssertNoErr(t, err)
+	err = mdnsServer.Start(nil)
+	wt.AssertNoErr(t, err)
+	defer mdnsServer.Stop()
+
+	var receivedAddr net.IP
+	var receivedName string
+	receivedCount := 0
+
+	mdnsCli, err := NewMDNSClient()
+	wt.AssertNoErr(t, err)
+	err = mdnsCli.Start(nil)
+	wt.AssertNoErr(t, err)
+
+	sendQuery := func(name string, querytype uint16) {
+		receivedAddr = nil
+		receivedName = ""
+		receivedCount = 0
+		Debug.Printf("Sending query...")
+		switch querytype {
+		case dns.TypeA:
+			r, err := mdnsCli.LookupName(name)
+			if len(r) > 0 {
+				receivedAddr = r[0].IP()
+				if err == nil {
+					receivedCount++
+				}
+			}
+		case dns.TypePTR:
+			r, err := mdnsCli.LookupInaddr(name)
+			if len(r) > 0 {
+				receivedName = r[0].Name()
+				if err == nil {
+					receivedCount++
+				}
+			}
+		}
+	}
+
+	time.Sleep(100 * time.Millisecond) // Allow for server to get going
+
+	Debug.Printf("Query: %s dns.TypeA", testRecord1.Name())
+	sendQuery(testRecord1.Name(), dns.TypeA)
+	if receivedCount != 1 {
+		t.Fatalf("Unexpected result count %d for %s", receivedCount, testRecord1.Name())
+	}
+	if !receivedAddr.Equal(testRecord1.IP()) {
+		t.Fatalf("Unexpected result %s for %s", receivedAddr, testRecord1.Name())
+	}
+
+	Debug.Printf("Query: testfail.weave. dns.TypeA")
+	sendQuery("testfail.weave.", dns.TypeA)
+	if receivedCount != 0 {
+		t.Fatalf("Unexpected result count %d for testfail.weave", receivedCount)
+	}
+
+	Debug.Printf("Query: %s dns.TypePTR", testInAddr1)
+	sendQuery(testInAddr1, dns.TypePTR)
+	if receivedCount != 1 {
+		t.Fatalf("Expected an answer to %s, got %d answers", testInAddr1, receivedCount)
+	} else if !(testRecord1.Name() == receivedName) {
+		t.Fatalf("Expected answer %s to query for %s, got %s", testRecord1.Name(), testInAddr1, receivedName)
+	}
+}
+
+// Check that we can use a use "insistent" queries
+func TestClientServerInsistentQuery(t *testing.T) {
+	InitDefaultLogging(testing.Verbose())
+
+	testRecord1 := Record{"test.weave.local.", net.ParseIP("10.2.2.1"), 0, 0, 0}
+	testInAddr1 := "1.2.2.10.in-addr.arpa."
+	testRecord2 := Record{"test.weave.local.", net.ParseIP("10.2.2.2"), 0, 0, 0}
+	testInAddr2 := "2.2.2.10.in-addr.arpa."
+
+	mzone1 := newMockedZoneWithRecords([]ZoneRecord{testRecord1})
+	mdnsServer1, err := NewMDNSServer(mzone1, true, DefaultLocalTTL)
+	wt.AssertNoErr(t, err)
+	err = mdnsServer1.Start(nil)
+	wt.AssertNoErr(t, err)
+	defer mdnsServer1.Stop()
+
+	mzone2 := newMockedZoneWithRecords([]ZoneRecord{testRecord2})
+	mdnsServer2, err := NewMDNSServer(mzone2, true, DefaultLocalTTL)
+	wt.AssertNoErr(t, err)
+	err = mdnsServer2.Start(nil)
+	wt.AssertNoErr(t, err)
+	defer mdnsServer2.Stop()
+
+	// create a third server with exactly the same info as the second server (so we can test duplicates removals)
+	mdnsServer3, err := NewMDNSServer(mzone2, true, DefaultLocalTTL)
+	wt.AssertNoErr(t, err)
+	err = mdnsServer3.Start(nil)
+	wt.AssertNoErr(t, err)
+	defer mdnsServer3.Stop()
+
+	var receivedAddrs []ZoneRecord
+	var receivedNames []ZoneRecord
+	receivedCount := 0
+
+	mdnsCli, err := NewMDNSClient()
+	wt.AssertNoErr(t, err)
+	err = mdnsCli.Start(nil)
+	wt.AssertNoErr(t, err)
+
+	sendQuery := func(name string, querytype uint16) {
+		receivedAddrs = nil
+		receivedNames = nil
+		receivedCount = 0
+		Debug.Printf("Sending query...")
+		switch querytype {
+		case dns.TypeA:
+			receivedAddrs, err = mdnsCli.InsistentLookupName(name)
+			if err == nil {
+				receivedCount = len(receivedAddrs)
+			}
+		case dns.TypePTR:
+			receivedNames, err = mdnsCli.InsistentLookupInaddr(name)
+			if err == nil {
+				receivedCount = len(receivedNames)
+			}
+		}
+	}
+
+	time.Sleep(100 * time.Millisecond) // Allow for server to get going
+
+	Debug.Printf("Query: %s dns.TypeA", testRecord1.Name())
+	sendQuery(testRecord1.Name(), dns.TypeA)
+	if receivedCount != 2 {
+		t.Fatalf("Unexpected result count %d for %s", receivedCount, testRecord1.Name())
+	}
+
+	Debug.Printf("Query: testfail.weave. dns.TypeA")
+	sendQuery("testfail.weave.", dns.TypeA)
+	if receivedCount != 0 {
+		t.Fatalf("Unexpected result count %d for testfail.weave", receivedCount)
+	}
+
+	Debug.Printf("Query: %s dns.TypePTR", testInAddr1)
+	sendQuery(testInAddr1, dns.TypePTR)
+	if receivedCount != 1 {
+		t.Fatalf("Expected an answer to %s, got %d answers", testInAddr1, receivedCount)
+	}
+
+	Debug.Printf("Query: %s dns.TypePTR", testInAddr2)
+	sendQuery(testInAddr2, dns.TypePTR)
+	if receivedCount != 1 {
+		t.Fatalf("Expected an answer to %s, got %d answers", testInAddr2, receivedCount)
+	}
+}

nameserver/mocks_test.go

@@ -1,9 +1,17 @@
 package nameserver
 
 import (
-	. "github.com/weaveworks/weave/common"
+	"fmt"
 	"net"
+	"strconv"
 	"sync"
+	"testing"
+	"time"
+
+	"github.com/benbjohnson/clock"
+	"github.com/miekg/dns"
+	. "github.com/weaveworks/weave/common"
+	wt "github.com/weaveworks/weave/testing"
 )
 
 const (
@@ -75,12 +83,11 @@ func (mz *mockedZoneWithRecords) AddRecord(ident string, name string, ip net.IP)
 	notImplWarn()
 	return nil
 }
-func (mz *mockedZoneWithRecords) DeleteRecord(ident string, ip net.IP) error {
+func (mz *mockedZoneWithRecords) DeleteRecords(ident string, name string, ip net.IP) int {
 	notImplWarn()
-	return nil
+	return 0
 }
-func (mz *mockedZoneWithRecords) DeleteRecordsFor(ident string) error { notImplWarn(); return nil }
-func (mz *mockedZoneWithRecords) Status() string                      { notImplWarn(); return "nothing" }
+func (mz *mockedZoneWithRecords) Status() string { notImplWarn(); return "nothing" }
 func (mz *mockedZoneWithRecords) ObserveName(name string, observer ZoneRecordObserver) error {
 	notImplWarn()
 	return nil
@@ -89,3 +96,412 @@ func (mz *mockedZoneWithRecords) ObserveInaddr(inaddr string, observer ZoneRecor
 	notImplWarn()
 	return nil
 }
+
+//////////////////////////////////////////////////////////////////
+
+// A mocked mDNS server
+type mockedMDNSServer struct {
+	zone Zone
+}
+
+func newMockedMDNSServer(z Zone) *mockedMDNSServer        { return &mockedMDNSServer{zone: z} }
+func (ms *mockedMDNSServer) Start(_ *net.Interface) error { return nil }
+func (ms *mockedMDNSServer) Stop() error                  { return nil }
+func (ms *mockedMDNSServer) Zone() Zone                   { return ms.zone }
+
+// a useful mix of NewMockedZone and NewMockedMDNSServer
+func newMockedMDNSServerWithRecord(zr ZoneRecord) *mockedMDNSServer {
+	return newMockedMDNSServer(newMockedZoneWithRecords([]ZoneRecord{zr}))
+}
+func newMockedMDNSServerWithRecords(zrs []ZoneRecord) *mockedMDNSServer {
+	return newMockedMDNSServer(newMockedZoneWithRecords(zrs))
+}
+
+//////////////////////////////////////////////////////////////////
+
+// A mocked mDNS client.
+// This mock asks a group of (potentially mocked) mDNS servers
+type mockedMDNSClient struct {
+	sync.RWMutex
+
+	servers map[ZoneMDNSServer]ZoneMDNSServer
+
+	// Statistics
+	NumLookupsName      int
+	NumLookupsInaddr    int
+	NumInsLookupsName   int
+	NumInsLookupsInaddr int
+}
+
+func newMockedMDNSClient(srvrs []*mockedMDNSServer) *mockedMDNSClient {
+	r := mockedMDNSClient{
+		servers: make(map[ZoneMDNSServer]ZoneMDNSServer),
+	}
+	if srvrs != nil {
+		for _, server := range srvrs {
+			r.servers[server] = server
+		}
+	}
+	return &r
+}
+
+func (mc *mockedMDNSClient) Start(ifi *net.Interface) error { return nil }
+func (mc *mockedMDNSClient) Stop() error                    { return nil }
+func (mc *mockedMDNSClient) Domain() string                 { return DefaultLocalDomain }
+func (mc *mockedMDNSClient) LookupName(name string) ([]ZoneRecord, error) {
+	mc.Lock()
+	defer mc.Unlock()
+
+	// get a random result
+	mc.NumLookupsName++
+	for _, server := range mc.servers {
+		r, err := server.Zone().LookupName(name)
+		if err == nil {
+			return r, nil // return the first answer
+		}
+	}
+	return nil, LookupError(name)
+}
+
+func (mc *mockedMDNSClient) LookupInaddr(inaddr string) ([]ZoneRecord, error) {
+	mc.Lock()
+	defer mc.Unlock()
+
+	// get a random result
+	mc.NumLookupsInaddr++
+	for _, server := range mc.servers {
+		r, err := server.Zone().LookupInaddr(inaddr)
+		if err == nil {
+			return r, nil // return the first answer
+		}
+	}
+	return nil, LookupError(inaddr)
+}
+
+func (mc *mockedMDNSClient) InsistentLookupName(name string) ([]ZoneRecord, error) {
+	mc.Lock()
+	defer mc.Unlock()
+
+	var res []ZoneRecord
+	mc.NumInsLookupsName++
+	for _, server := range mc.servers {
+		r, err := server.Zone().LookupName(name)
+		if err == nil {
+			res = append(res, r...)
+		}
+	}
+	if len(res) == 0 {
+		return nil, LookupError(name)
+	}
+	return res, nil
+}
+
+func (mc *mockedMDNSClient) InsistentLookupInaddr(inaddr string) ([]ZoneRecord, error) {
+	mc.Lock()
+	defer mc.Unlock()
+
+	var res []ZoneRecord
+	mc.NumInsLookupsInaddr++
+	for _, server := range mc.servers {
+		r, err := server.Zone().LookupInaddr(inaddr)
+		if err == nil {
+			res = append(res, r...)
+		}
+	}
+	if len(res) == 0 {
+		return nil, LookupError(inaddr)
+	}
+	return res, nil
+}
+
+func (mc *mockedMDNSClient) AddServer(srv ZoneMDNSServer) {
+	mc.Lock()
+	defer mc.Unlock()
+	mc.servers[srv] = srv
+}
+
+func (mc *mockedMDNSClient) RemoveServer(srv ZoneMDNSServer) {
+	mc.Lock()
+	defer mc.Unlock()
+	delete(mc.servers, srv)
+}
+
+//////////////////////////////////////////////////////////////////
+
+type zbmEntry struct {
+	Server *mockedMDNSServer
+	Client *mockedMDNSClient
+	Zone   *ZoneDb
+}
+type zoneDbsWithMockedMDns []*zbmEntry
+
+// Creates a group of (real) zone databases linked through mocked mDNS servers and clients
+// This effectively creates a cluster of WeaveDNS peers
+func newZoneDbsWithMockedMDns(num int, config ZoneConfig) zoneDbsWithMockedMDns {
+	res := make(zoneDbsWithMockedMDns, num)
+
+	for i := 0; i < num; i++ {
+		res[i] = new(zbmEntry)
+
+		Debug.Printf("[test] Creating mocked mDNS server #%d", i)
+		res[i].Server = newMockedMDNSServer(nil)
+
+		Debug.Printf("[test] Creating mocked mDNS client #%d", i)
+		res[i].Client = newMockedMDNSClient([]*mockedMDNSServer{})
+
+		cfg := config
+		cfg.MDNSClient = res[i].Client
+		cfg.MDNSServer = res[i].Server
+
+		Debug.Printf("[test] Creating ZoneDb #%d", i)
+		res[i].Zone, _ = NewZoneDb(cfg)
+
+		// link the mDNS server to its zone
+		// ZoneDbs are connected to mDNS in two directions: they use their mDNS
+		// clients for asking other peers about names/IPs, and export their records
+		// through a mDNS server.
+		res[i].Server.zone = res[i].Zone
+	}
+
+	// link all the clients to all the servers
+	for i := 0; i < num; i++ {
+		for j := 0; j < num; j++ {
+			// with this check, the local client will not ask to the local server
+			// this is not what we currently do, but can be convenient in some cases
+			// for finding some bugs...
+			if i != j {
+				Debug.Printf("[test] Linking mocked mDNS client #%d to server #%d", i, j)
+				res[i].Client.AddServer(res[j].Server)
+			}
+		}
+	}
+
+	return res
+}
+
+func (zbs zoneDbsWithMockedMDns) Start() {
+	for _, entry := range zbs {
+		entry.Zone.Start()
+	}
+}
+
+func (zbs zoneDbsWithMockedMDns) Stop() {
+	for _, entry := range zbs {
+		entry.Zone.Stop()
+	}
+}
+
+func (zbs zoneDbsWithMockedMDns) Flush() {
+	for _, entry := range zbs {
+		entry.Zone.refreshScheds.Flush()
+	}
+}
+
+//////////////////////////////////////////////////////////////////
+
+// A mocked cache where we never find a single thing... ;)
+type mockedCache struct {
+	NumGets     int
+	NumPuts     int
+	NumRemovals int
+}
+
+func newMockedCache() *mockedCache { return &mockedCache{} }
+
+func (c *mockedCache) Get(request *dns.Msg, maxLen int) (reply *dns.Msg, err error) {
+	c.NumGets++
+	return nil, nil
+}
+func (c *mockedCache) Put(request *dns.Msg, reply *dns.Msg, ttl int, flags uint8) int {
+	c.NumPuts++
+	return 0
+}
+func (c *mockedCache) Remove(question *dns.Question) { c.NumRemovals++ }
+func (c *mockedCache) Purge()                        {}
+func (c *mockedCache) Clear()                        {}
+func (c *mockedCache) Len() int                      { return 0 }
+func (c *mockedCache) Capacity() int                 { return DefaultCacheLen }
+
+//////////////////////////////////////////////////////////////////
+
+// A mocked fallback server
+type mockedFallback struct {
+	CliConfig *dns.ClientConfig
+	Addr      string
+	Port      int
+
+	udpSrv *dns.Server
+	tcpSrv *dns.Server
+}
+
+// Create a mocked fallback server
+// You can use the server's `CliConfig` as the `UpstreamCfg` of a `DNSServer`...
+func newMockedFallback(udpH dns.HandlerFunc, tcpH dns.HandlerFunc) (*mockedFallback, error) {
+	udpSrv, fallbackAddr, err := runLocalUDPServer("127.0.0.1:0", udpH)
+	if err != nil {
+		return nil, err
+	}
+	_, fallbackPort, err := net.SplitHostPort(fallbackAddr)
+	if err != nil {
+		return nil, err
+	}
+
+	fallbackPortI, _ := strconv.Atoi(fallbackPort)
+
+	res := mockedFallback{
+		udpSrv: udpSrv,
+		Addr:   fallbackAddr,
+		Port:   fallbackPortI,
+		CliConfig: &dns.ClientConfig{
+			Servers: []string{"127.0.0.1"},
+			Port:    fallbackPort,
+		},
+	}
+
+	if tcpH != nil {
+		fallbackTCPAddr := fmt.Sprintf("127.0.0.1:%s", fallbackPort)
+		res.tcpSrv, _, err = runLocalTCPServer(fallbackTCPAddr, tcpH)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return &res, nil
+}
+
+// Start the fallback server
+func (mf *mockedFallback) Start() error {
+	return nil
+}
+
+// Stop the fallback server
+func (mf *mockedFallback) Stop() error {
+	if mf.tcpSrv != nil {
+		mf.udpSrv.Shutdown()
+		mf.udpSrv = nil
+	}
+	if mf.tcpSrv != nil {
+		mf.tcpSrv.Shutdown()
+		mf.udpSrv = nil
+	}
+	return nil
+}
+
+// Run a UDP fallback server
+func runLocalUDPServer(laddr string, handler dns.HandlerFunc) (*dns.Server, string, error) {
+	Debug.Printf("[mocked fallback] Starting fallback UDP server at %s", laddr)
+	pc, err := net.ListenPacket("udp", laddr)
+	if err != nil {
+		return nil, "", err
+	}
+	server := &dns.Server{PacketConn: pc, Handler: handler, ReadTimeout: testSocketTimeout * time.Millisecond}
+
+	go func() {
+		server.ActivateAndServe()
+		pc.Close()
+	}()
+
+	Debug.Printf("[mocked fallback] Fallback UDP server listening at %s", pc.LocalAddr())
+	return server, pc.LocalAddr().String(), nil
+}
+
+// Run a TCP fallback server
+func runLocalTCPServer(laddr string, handler dns.HandlerFunc) (*dns.Server, string, error) {
+	Debug.Printf("[mocked fallback] Starting fallback TCP server at %s", laddr)
+	laddrTCP, err := net.ResolveTCPAddr("tcp", laddr)
+	if err != nil {
+		return nil, "", err
+	}
+
+	l, err := net.ListenTCP("tcp", laddrTCP)
+	if err != nil {
+		return nil, "", err
+	}
+	server := &dns.Server{Listener: l, Handler: handler, ReadTimeout: testSocketTimeout * time.Millisecond}
+
+	go func() {
+		server.ActivateAndServe()
+		l.Close()
+	}()
+
+	Debug.Printf("[mocked fallback] Fallback TCP server listening at %s", l.Addr().String())
+	return server, l.Addr().String(), nil
+}
+
+//////////////////////////////////////////////////////////////////
+
+type mockedClock struct {
+	*clock.Mock
+}
+
+func newMockedClock() *mockedClock {
+	return &mockedClock{clock.NewMock()}
+}
+
+// Note: moving the clock forward does not take into account the data that could
+//       be waiting in channels. So your code should not depend on the channels
+//       for creating new timers. Otherwise, time travelling will not be reliable...
+func (clk *mockedClock) Forward(secs int) {
+	Debug.Printf(">>>>>>> Moving clock forward %d seconds - Time traveling >>>>>>>", secs)
+	clk.Add(time.Duration(secs) * time.Second)
+	Debug.Printf("<<<<<<< Time travel finished! We are at %s <<<<<<<", clk.Now())
+}
+
+//////////////////////////////////////////////////////////////////
+
+// Perform a DNS query and assert the reply code, number or answers, etc
+func assertExchange(t *testing.T, z string, ty uint16, port int, minAnswers int, maxAnswers int, expErr int) (*dns.Msg, *dns.Msg) {
+	wt.AssertNotEqualInt(t, port, 0, "invalid DNS server port")
+
+	c := &dns.Client{
+		UDPSize: testUDPBufSize,
+	}
+
+	m := new(dns.Msg)
+	m.RecursionDesired = true
+	m.SetQuestion(z, ty)
+	m.SetEdns0(testUDPBufSize, false) // we don't want to play with truncation here...
+
+	lstAddr := fmt.Sprintf("127.0.0.1:%d", port)
+	r, _, err := c.Exchange(m, lstAddr)
+	t.Logf("Response from '%s':\n%+v\n", lstAddr, r)
+	if err != nil {
+		t.Errorf("Error when querying DNS server at %s: %s", lstAddr, err)
+	}
+	wt.AssertNoErr(t, err)
+	if minAnswers == 0 && maxAnswers == 0 {
+		wt.AssertStatus(t, r.Rcode, expErr, "DNS response code")
+	} else {
+		wt.AssertStatus(t, r.Rcode, dns.RcodeSuccess, "DNS response code")
+	}
+	answers := len(r.Answer)
+	if minAnswers >= 0 && answers < minAnswers {
+		wt.Fatalf(t, "Number of answers >= %d", minAnswers)
+	}
+	if maxAnswers >= 0 && answers > maxAnswers {
+		wt.Fatalf(t, "Number of answers <= %d", maxAnswers)
+	}
+	return m, r
+}
+
+// Assert that we have a response in the cache for a query `q`
+func assertInCache(t *testing.T, cache ZoneCache, q *dns.Msg, desc string) {
+	r, err := cache.Get(q, maxUDPSize)
+	wt.AssertNoErr(t, err)
+	wt.AssertNotNil(t, r, fmt.Sprintf("value in the cache: %s", desc))
+}
+
+// Assert that we have a response in the cache for a query `q`
+func assertNotLocalInCache(t *testing.T, cache ZoneCache, q *dns.Msg, desc string) {
+	r, err := cache.Get(q, maxUDPSize)
+	if !(r == nil && err == errNoLocalReplies) {
+		t.Fatalf("Cache does not return a noLocalReplies error for query %s", q)
+	}
+}
+
+// Assert that we do not have a response in the cache for a query `q`
+func assertNotInCache(t *testing.T, cache ZoneCache, q *dns.Msg, desc string) {
+	r, err := cache.Get(q, maxUDPSize)
+	wt.AssertNoErr(t, err)
+	wt.AssertNil(t, r, fmt.Sprintf("value in the cache: %s\n%s", desc, r))
+}

nameserver/server.go

@@ -3,37 +3,56 @@ package nameserver
 import (
 	"bytes"
 	"fmt"
+	"net"
+	"strconv"
+	"sync"
+	"time"
+
+	"github.com/benbjohnson/clock"
 	"github.com/miekg/dns"
 	. "github.com/weaveworks/weave/common"
-	"net"
-	"sync"
 )
 
 const (
-	DefaultLocalDomain   = "weave.local."     // The default name used for the local domain
-	DefaultServerPort    = 53                 // The default server port
-	DefaultCLICfgFile    = "/etc/resolv.conf" // default "resolv.conf" file to try to load
-	DefaultUDPBuflen     = 4096               // bigger than the default 512
-	DefaultCacheLen      = 8192               // default cache capacity
-	DefaultResolvWorkers = 8                  // default number of resolution workers
-	DefaultTimeout       = 5                  // default timeout for DNS resolutions
+	DefaultServerPort = 53                 // The default server port
+	DefaultHTTPPort   = 6785               // The default http port
+	DefaultCLICfgFile = "/etc/resolv.conf" // default "resolv.conf" file to try to load
+	DefaultLocalTTL   = 30                 // default TTL for responses for local domain queries
+	DefaultUDPBuflen  = 4096               // bigger than the default 512
+	DefaultCacheLen   = 8192               // default cache capacity
+	DefaultTimeout    = 5000               // default timeout for DNS resolutions (millisecs)
+	DefaultMaxAnswers = 1                  // default number of answers provided to users
 )
 
 type DNSServerConfig struct {
+	// The zone
+	Zone Zone
 	// (Optional) client config file for resolving upstream servers
 	UpstreamCfgFile string
 	// (Optional) DNS client config for the fallback server(s)
 	UpstreamCfg *dns.ClientConfig
 	// (Optional) port number (for TCP and UDP)
 	Port int
-	// (Optional) local domain (ie, "weave.local.")
-	LocalDomain string
 	// (Optional) cache size
 	CacheLen int
+	// (Optional) disable the cache
+	CacheDisabled bool
 	// (Optional) timeout for DNS queries
 	Timeout int
 	// (Optional) UDP buffer length
 	UDPBufLen int
+	// (Optional) force a specific cache
+	Cache ZoneCache
+	// (Optional) TTL for local domain responses
+	LocalTTL int
+	// (Optional) TTL for negative results in the local domain (defaults to LocalTTL)
+	CacheNegLocalTTL int
+	// (Optional) for a specific clock provider
+	Clock clock.Clock
+	// (Optional) Listening socket read timeout (in milliseconds)
+	ListenReadTimeout int
+	// Maximum number of answers provided to users
+	MaxAnswers int
 }
 
 type dnsProtocol uint8
@@ -54,51 +73,68 @@ func (proto dnsProtocol) String() string {
 }
 
 // get a new dns.Client for a protocol
-func (proto dnsProtocol) GetNewClient(bufsize int) *dns.Client {
+func (proto dnsProtocol) GetNewClient(bufsize int, timeout time.Duration) *dns.Client {
+	Debug.Printf("[dns] Creating %s DNS client with %s timeout", proto, timeout)
 	switch proto {
 	case protTCP:
-		return &dns.Client{Net: "tcp"}
+		return &dns.Client{Net: "tcp", ReadTimeout: timeout}
 	case protUDP:
-		return &dns.Client{Net: "udp", UDPSize: uint16(bufsize)}
+		return &dns.Client{Net: "udp", ReadTimeout: timeout, UDPSize: uint16(bufsize)}
 	}
 	return nil
 }
 
+// a DNS server
 type DNSServer struct {
-	Zone        Zone
-	Iface       *net.Interface
-	Upstream    *dns.ClientConfig
-	udpSrv      *dns.Server
-	tcpSrv      *dns.Server
-	mdnsCli     *MDNSClient
-	mdnsSrv     *MDNSServer
-	cache       *Cache
-	timeout     int
-	udpBuf      int
-	listenersWg *sync.WaitGroup
-
+	Zone       Zone
+	Upstream   *dns.ClientConfig
 	Domain     string // the local domain
 	ListenAddr string // the address the server is listening at
+
+	udpSrv        *dns.Server
+	tcpSrv        *dns.Server
+	pc            net.PacketConn
+	lst           net.Listener
+	cache         ZoneCache
+	cacheDisabled bool
+	maxAnswers    int
+	localTTL      int
+	negLocalTTL   int
+	timeout       time.Duration
+	readTimeout   time.Duration
+	udpBuf        int
+	listenersWg   *sync.WaitGroup
+	clock         clock.Clock
 }
 
 // Creates a new DNS server
-func NewDNSServer(config DNSServerConfig, zone Zone, iface *net.Interface) (s *DNSServer, err error) {
+func NewDNSServer(config DNSServerConfig) (s *DNSServer, err error) {
 	s = &DNSServer{
-		Zone:        zone,
-		Iface:       iface,
-		listenersWg: new(sync.WaitGroup),
-
+		Zone:       config.Zone,
 		Domain:     DefaultLocalDomain,
-		ListenAddr: fmt.Sprintf(":%d", DefaultServerPort),
+		ListenAddr: fmt.Sprintf(":%d", config.Port),
+
+		listenersWg:   new(sync.WaitGroup),
+		timeout:       DefaultTimeout * time.Millisecond,
+		readTimeout:   DefaultTimeout * time.Millisecond,
+		cacheDisabled: false,
+		maxAnswers:    DefaultMaxAnswers,
+		localTTL:      DefaultLocalTTL,
+		clock:         config.Clock,
+	}
+
+	// check some basic parameters are valid
+	if s.Zone == nil {
+		return nil, fmt.Errorf("No valid Zone provided in server initialization")
+	}
+	if len(s.Domain) == 0 {
+		return nil, fmt.Errorf("No valid Domain provided in server initialization")
+	}
+	if s.clock == nil {
+		s.clock = clock.New()
 	}
 
 	// fill empty parameters with defaults...
-	if config.Port != 0 {
-		s.ListenAddr = fmt.Sprintf(":%d", config.Port)
-	}
-	if len(config.LocalDomain) > 0 {
-		s.Domain = config.LocalDomain
-	}
 	if config.UpstreamCfg != nil {
 		s.Upstream = config.UpstreamCfg
 	} else {
@@ -111,207 +147,236 @@ func NewDNSServer(config DNSServerConfig, zone Zone, iface *net.Interface) (s *D
 		}
 	}
 	if config.Timeout > 0 {
-		s.timeout = config.Timeout
+		s.timeout = time.Duration(config.Timeout) * time.Millisecond
+	}
+	if config.ListenReadTimeout > 0 {
+		s.readTimeout = time.Duration(config.ListenReadTimeout) * time.Millisecond
 	}
 	if config.UDPBufLen > 0 {
 		s.udpBuf = config.UDPBufLen
 	}
-	s.mdnsCli, err = NewMDNSClient()
-	if err != nil {
-		return
+	if config.MaxAnswers > 0 {
+		s.maxAnswers = config.MaxAnswers
 	}
-	s.mdnsSrv, err = NewMDNSServer(s.Zone)
-	if err != nil {
-		return
+	if config.LocalTTL > 0 {
+		s.localTTL = config.LocalTTL
 	}
-	cacheLen := DefaultCacheLen
-	if config.CacheLen > 0 {
-		cacheLen = config.CacheLen
+	if config.CacheNegLocalTTL > 0 {
+		s.negLocalTTL = config.CacheNegLocalTTL
+	} else {
+		s.negLocalTTL = s.localTTL
 	}
-	Debug.Printf("[dns] Initializing cache: %d entries", config.CacheLen)
-	s.cache, err = NewCache(cacheLen, nil)
-	if err != nil {
-		return
+	if config.CacheDisabled {
+		s.cacheDisabled = true
 	}
-
-	// create two DNS request multiplexerers, depending on the protocol used by clients
-	// (we use the same protocol for asking upstream servers)
-	mux := func(proto dnsProtocol) *dns.ServeMux {
-		m := dns.NewServeMux()
-		m.HandleFunc(s.Domain, s.queryHandler(proto))
-		m.HandleFunc(RDNSDomain, s.rdnsHandler(proto))
-		m.HandleFunc(".", s.notUsHandler(proto))
-		return m
+	if !s.cacheDisabled {
+		if config.Cache != nil {
+			s.cache = config.Cache
+		} else {
+			cacheLen := DefaultCacheLen
+			if config.CacheLen > 0 {
+				cacheLen = config.CacheLen
+			}
+			if s.cache, err = NewCache(cacheLen, s.clock); err != nil {
+				return
+			}
+		}
 	}
 
-	s.udpSrv = &dns.Server{Addr: s.ListenAddr, Net: "udp", Handler: mux(protUDP)}
-	s.tcpSrv = &dns.Server{Addr: s.ListenAddr, Net: "tcp", Handler: mux(protTCP)}
-
 	return
 }
 
 // Start the DNS server
 func (s *DNSServer) Start() error {
-	Info.Printf("Using mDNS on %v", s.Iface)
-	err := s.mdnsCli.Start(s.Iface)
-	CheckFatal(err)
-	err = s.mdnsSrv.Start(s.Iface)
-	CheckFatal(err)
+	Info.Printf("[dns] Upstream server(s): %+v", s.Upstream)
+	if s.cacheDisabled {
+		Info.Printf("[dns] Cache: disabled")
+	} else {
+		Info.Printf("[dns] Cache: %d entries", s.cache.Capacity())
+	}
+
+	pc, err := net.ListenPacket("udp", s.ListenAddr)
+	if err != nil {
+		return err
+	}
+	s.pc = pc
+
+	_, port, err := net.SplitHostPort(pc.LocalAddr().String())
+	if err != nil {
+		return err
+	}
+	s.ListenAddr = fmt.Sprintf(":%s", port)
+	s.udpSrv = &dns.Server{PacketConn: s.pc, Handler: s.createMux(protUDP), ReadTimeout: s.readTimeout}
+
+	// Bind the TCP socket at the same port, aborting otherwise
+	l, err := net.Listen("tcp", s.ListenAddr)
+	if err != nil {
+		s.Stop()
+		return err
+	}
+	s.lst = l
+	s.tcpSrv = &dns.Server{Listener: l, Handler: s.createMux(protTCP), ReadTimeout: s.readTimeout}
 
 	s.listenersWg.Add(2)
+	return nil
+}
 
+func (s *DNSServer) ActivateAndServe() {
 	go func() {
 		defer s.listenersWg.Done()
 
-		Debug.Printf("Listening for DNS on %s (UDP)", s.ListenAddr)
-		err = s.udpSrv.ListenAndServe()
+		Info.Printf("[dns] Listening for DNS on %s (UDP)", s.ListenAddr)
+		err := s.udpSrv.ActivateAndServe()
 		CheckFatal(err)
-		Debug.Printf("DNS UDP server exiting...")
+		Debug.Printf("[dns] DNS UDP server exiting...")
 	}()
 
 	go func() {
 		defer s.listenersWg.Done()
 
-		Debug.Printf("Listening for DNS on %s (TCP)", s.ListenAddr)
-		err = s.tcpSrv.ListenAndServe()
+		Info.Printf("[dns] Listening for DNS on %s (TCP)", s.ListenAddr)
+		err := s.tcpSrv.ActivateAndServe()
 		CheckFatal(err)
-		Debug.Printf("DNS TCP server exiting...")
+		Debug.Printf("[dns] DNS TCP server exiting...")
 	}()
 
 	// Waiting for all goroutines to finish (otherwise they die as main routine dies)
 	s.listenersWg.Wait()
 
-	Info.Printf("WeaveDNS server exiting...")
-	return nil
+	Info.Printf("[dns] Server exiting...")
 }
 
 // Return status string
 func (s *DNSServer) Status() string {
 	var buf bytes.Buffer
-	fmt.Fprintln(&buf, "Local domain", s.Domain)
 	fmt.Fprintln(&buf, "Listen address", s.ListenAddr)
-	fmt.Fprintln(&buf, "mDNS interface", s.Iface)
 	fmt.Fprintln(&buf, "Fallback DNS config", s.Upstream)
-	fmt.Fprintf(&buf, "Zone database:\n%s", s.Zone)
 	return buf.String()
 }
 
 // Perform a graceful shutdown
 func (s *DNSServer) Stop() error {
 	// Stop the listeners/handlers
-	if err := s.tcpSrv.Shutdown(); err != nil {
-		return err
+	if s.tcpSrv != nil {
+		if err := s.tcpSrv.Shutdown(); err != nil {
+			return err
+		}
+		s.lst.Close()
+		s.tcpSrv = nil
 	}
-	if err := s.udpSrv.Shutdown(); err != nil {
-		return err
+	if s.udpSrv != nil {
+		if err := s.udpSrv.Shutdown(); err != nil {
+			return err
+		}
+		s.pc.Close()
+		s.udpSrv = nil
 	}
 	s.listenersWg.Wait()
-
-	// shutdown the mDNS server
-	s.mdnsSrv.Stop()
-
 	return nil
 }
 
-func (s *DNSServer) queryHandler(proto dnsProtocol) dns.HandlerFunc {
-	zoneLookup := func(lookup ZoneLookup, q *dns.Question, r *dns.Msg) (*dns.Msg, []ZoneRecord, error) {
-		ips, err := lookup.LookupName(q.Name)
-		if err != nil {
-			return nil, nil, err
-		}
-		return makeAddressReply(r, q, ips), ips, nil
-	}
-
-	fallback := func(w dns.ResponseWriter, r *dns.Msg) {
+// Create a multiplexer for requests
+// We must create two DNS request multiplexers, depending on the protocol used by
+// clients (as we use the same protocol for asking upstream servers)
+func (s *DNSServer) createMux(proto dnsProtocol) *dns.ServeMux {
+	failFallback := func(w dns.ResponseWriter, r *dns.Msg) {
 		w.WriteMsg(makeDNSFailResponse(r))
 	}
-
-	return s.commonQueryHandler(proto, "Query", dns.TypeA, zoneLookup, fallback)
-}
-
-func (s *DNSServer) rdnsHandler(proto dnsProtocol) dns.HandlerFunc {
-	zoneLookup := func(lookup ZoneLookup, q *dns.Question, r *dns.Msg) (*dns.Msg, []ZoneRecord, error) {
-		names, err := lookup.LookupInaddr(q.Name)
-		if err != nil {
-			return nil, nil, err
-		}
-		return makePTRReply(r, q, names), names, nil
-	}
-
 	notUsHandler := s.notUsHandler(proto)
-	fallback := func(w dns.ResponseWriter, r *dns.Msg) {
+	notUsFallback := func(w dns.ResponseWriter, r *dns.Msg) {
 		Info.Printf("[dns msgid %d] -> sending to fallback server", r.MsgHdr.Id)
 		notUsHandler(w, r)
 	}
 
-	return s.commonQueryHandler(proto, "Reverse query", dns.TypePTR, zoneLookup, fallback)
+	// create the multiplexer
+	m := dns.NewServeMux()
+	m.HandleFunc(s.Zone.Domain(), s.localHandler(proto, "Query", dns.TypeA,
+		s.Zone.DomainLookupName, makeAddressReply, s.Zone.ObserveName, failFallback))
+	m.HandleFunc(RDNSDomain, s.localHandler(proto, "Reverse query", dns.TypePTR,
+		s.Zone.DomainLookupInaddr, makePTRReply, s.Zone.ObserveInaddr, notUsFallback))
+	m.HandleFunc(".", s.notUsHandler(proto))
+	return m
 }
 
-func (s *DNSServer) commonQueryHandler(proto dnsProtocol, kind string, qtype uint16,
-	zoneLookup func(ZoneLookup, *dns.Question, *dns.Msg) (*dns.Msg, []ZoneRecord, error), fallback dns.HandlerFunc) dns.HandlerFunc {
+// Process a request for a name/address in the '.weave.local.' zone
+func (s *DNSServer) localHandler(proto dnsProtocol, kind string, qtype uint16,
+	lookup func(inaddr string) ([]ZoneRecord, error),
+	msgBuilder DNSResponseBuilder, observer ZoneObserverFunc, fallback dns.HandlerFunc) dns.HandlerFunc {
 
 	return func(w dns.ResponseWriter, r *dns.Msg) {
 		q := r.Question[0]
 		Debug.Printf("[dns msgid %d] %s: %+v", r.MsgHdr.Id, kind, q)
 		maxLen := getMaxReplyLen(r, proto)
-		lookups := []ZoneLookup{s.Zone, s.mdnsCli}
 
-		reply, err := s.cache.Get(r, maxLen)
-		if err != nil {
-			if err == errNoLocalReplies {
-				Debug.Printf("[dns msgid %d] Cached 'no local replies' - skipping local lookup", r.MsgHdr.Id)
-				lookups = []ZoneLookup{s.Zone}
-			} else {
-				Debug.Printf("[dns msgid %d] Error from cache: %s", r.MsgHdr.Id, err)
-				w.WriteMsg(makeDNSFailResponse(r))
-				return
+		// cache a response if the cache is enabled, and observe the name/IP
+		maybeCache := func(m *dns.Msg, ttl int, flags uint8) {
+			if !s.cacheDisabled {
+				s.cache.Put(r, m, ttl, flags)
+				// any change in the Zone database for this IP/name will lead to this
+				// cache entry being removed...
+				// TODO: this closure results in unnecessary `Remove`s and some wasted
+				// mem... but we can live with that.
+				observer(q.Name, func() { s.cache.Remove(&q) })
 			}
 		}
-		if reply != nil {
-			Debug.Printf("[dns msgid %d] Returning reply from cache: %s/%d answers",
-				r.MsgHdr.Id, dns.RcodeToString[reply.MsgHdr.Rcode], len(reply.Answer))
-			w.WriteMsg(reply)
-			return
+
+		if !s.cacheDisabled {
+			reply, err := s.cache.Get(r, maxLen)
+			if err != nil {
+				if err == errNoLocalReplies {
+					Debug.Printf("[dns msgid %d] Cached no-local-replies", r.MsgHdr.Id)
+					fallback(w, r)
+				} else {
+					Debug.Printf("[dns msgid %d] Error from cache: %s", r.MsgHdr.Id, err)
+					w.WriteMsg(makeDNSFailResponse(r))
+				}
+				return
+			}
+			if reply != nil {
+				reply.Answer = pruneAnswers(shuffleAnswers(reply.Answer), s.maxAnswers)
+				Debug.Printf("[dns msgid %d] Returning reply from cache: %s/%d answers",
+					r.MsgHdr.Id, dns.RcodeToString[reply.MsgHdr.Rcode], len(reply.Answer))
+				w.WriteMsg(reply)
+				return
+			}
 		}
 
 		// catch unsupported queries
 		if q.Qtype != qtype {
 			Debug.Printf("[dns msgid %d] Unsupported query type %s", r.MsgHdr.Id, dns.TypeToString[q.Qtype])
 			m := makeDNSFailResponse(r)
-			s.cache.Put(r, m, negLocalTTL, 0)
+			maybeCache(m, s.negLocalTTL, 0)
 			w.WriteMsg(m)
 			return
 		}
 
-		for _, lookup := range lookups {
-			if m, answers, err := zoneLookup(lookup, &q, r); err == nil {
-				m.Authoritative = true
-				Debug.Printf("[dns msgid %d] Caching response for type %s query for '%s': %s [code:%s]",
-					m.MsgHdr.Id, dns.TypeToString[q.Qtype], q.Name, answers, dns.RcodeToString[m.Rcode])
-				s.cache.Put(r, m, nullTTL, 0)
-				w.WriteMsg(m)
-				return
-			}
+		if answers, err := lookup(q.Name); err != nil {
+			Info.Printf("[dns msgid %d] No results for type %s query for '%s' [caching no-local]",
+				r.MsgHdr.Id, dns.TypeToString[q.Qtype], q.Name)
+			maybeCache(nil, s.negLocalTTL, CacheNoLocalReplies)
+			fallback(w, r)
+		} else {
+			m := msgBuilder(r, &q, answers, s.localTTL)
+			m.Authoritative = true
+			maybeCache(m, nullTTL, 0)
+			m.Answer = pruneAnswers(shuffleAnswers(m.Answer), s.maxAnswers)
+			Debug.Printf("[dns msgid %d] Sending response: %s/%d answers [code:%s]",
+				m.MsgHdr.Id, dns.TypeToString[q.Qtype], len(m.Answer), dns.RcodeToString[m.Rcode])
+			w.WriteMsg(m)
 		}
-
-		Info.Printf("[dns msgid %d] No results for type %s query for '%s' [caching no-local]",
-			r.MsgHdr.Id, dns.TypeToString[q.Qtype], q.Name)
-		s.cache.Put(r, nil, negLocalTTL, CacheNoLocalReplies)
-
-		fallback(w, r)
 	}
 }
 
 // When we receive a request for a name outside of our '.weave.local.'
 // domain, ask the configured DNS server as a fallback.
 func (s *DNSServer) notUsHandler(proto dnsProtocol) dns.HandlerFunc {
-	dnsClient := proto.GetNewClient(DefaultUDPBuflen)
+	dnsClient := proto.GetNewClient(DefaultUDPBuflen, s.timeout)
 
 	return func(w dns.ResponseWriter, r *dns.Msg) {
-		maxLen := getMaxReplyLen(r, proto)
 		q := r.Question[0]
 
-		// create a request where we announce our max payload size
+		// announce our max payload size as the max payload our client supports
+		maxLen := getMaxReplyLen(r, proto)
 		rcopy := r
 		rcopy.SetEdns0(uint16(maxLen), false)
 
@@ -337,3 +402,12 @@ func (s *DNSServer) notUsHandler(proto dnsProtocol) dns.HandlerFunc {
 		w.WriteMsg(makeDNSFailResponse(r))
 	}
 }
+
+// Get the listen port
+func (s *DNSServer) GetPort() (int, error) {
+	_, portS, err := net.SplitHostPort(s.ListenAddr)
+	if err != nil {
+		return 0, err
+	}
+	return strconv.Atoi(portS)
+}

nameserver/server_cache_test.go

@@ -0,0 +1,351 @@
+package nameserver
+
+import (
+	"fmt"
+	"net"
+	"testing"
+	"time"
+
+	"github.com/miekg/dns"
+	. "github.com/weaveworks/weave/common"
+	wt "github.com/weaveworks/weave/testing"
+)
+
+// Check that AddRecord/DeleteRecord/... in the Zone database lead to cache invalidations
+func TestServerDbCacheInvalidation(t *testing.T) {
+	const (
+		containerID = "somecontainer"
+		testName1   = "first.weave.local."
+		testName2   = "second.weave.local."
+	)
+
+	InitDefaultLogging(testing.Verbose())
+	Info.Println("TestServerDbCacheInvalidation starting")
+
+	clk := newMockedClock()
+
+	Debug.Printf("Creating mocked mDNS client and server")
+	mdnsServer1 := newMockedMDNSServerWithRecord(Record{testName1, net.ParseIP("10.2.2.9"), 0, 0, 0})
+	mdnsCli1 := newMockedMDNSClient([]*mockedMDNSServer{mdnsServer1})
+
+	Debug.Printf("Creating zone database with the mocked mDNS client and server")
+	zoneConfig := ZoneConfig{
+		MDNSServer: mdnsServer1,
+		MDNSClient: mdnsCli1,
+		Clock:      clk,
+	}
+	zone, err := NewZoneDb(zoneConfig)
+	wt.AssertNoErr(t, err)
+	err = zone.Start()
+	wt.AssertNoErr(t, err)
+	defer zone.Stop()
+
+	Debug.Printf("Creating a cache")
+	cache, err := NewCache(1024, clk)
+	wt.AssertNoErr(t, err)
+
+	fallbackHandler := func(w dns.ResponseWriter, req *dns.Msg) {
+		m := new(dns.Msg)
+		m.SetReply(req)
+		if len(req.Question) == 1 {
+			m.Rcode = dns.RcodeNameError
+		}
+		w.WriteMsg(m)
+	}
+
+	// Run another DNS server for fallback
+	fallback, err := newMockedFallback(fallbackHandler, nil)
+	wt.AssertNoErr(t, err)
+	fallback.Start()
+	defer fallback.Stop()
+
+	Debug.Printf("Creating a real DNS server with a mocked cache")
+	srv, err := NewDNSServer(DNSServerConfig{
+		Zone:              zone,
+		Cache:             cache,
+		Clock:             clk,
+		ListenReadTimeout: testSocketTimeout,
+		UpstreamCfg:       fallback.CliConfig,
+		MaxAnswers:        4,
+	})
+	wt.AssertNoErr(t, err)
+	err = srv.Start()
+	wt.AssertNoErr(t, err)
+	go srv.ActivateAndServe()
+	defer srv.Stop()
+	time.Sleep(100 * time.Millisecond) // Allow server goroutine to start
+
+	testPort, err := srv.GetPort()
+	wt.AssertNoErr(t, err)
+	wt.AssertNotEqualInt(t, testPort, 0, "invalid listen port")
+
+	Debug.Printf("Adding two IPs to %s", testName1)
+	zone.AddRecord(containerID, testName1, net.ParseIP("10.2.2.1"))
+	zone.AddRecord(containerID, testName1, net.ParseIP("10.2.2.2"))
+	q, _ := assertExchange(t, testName1, dns.TypeA, testPort, 2, 2, 0)
+	assertInCache(t, cache, q, fmt.Sprintf("after asking for %s", testName1))
+
+	// Zone database at this point:
+	//   first.weave.local  = 10.2.2.1 10.2.2.2
+
+	zone.AddRecord(containerID, testName2, net.ParseIP("10.9.9.1"))
+	assertInCache(t, cache, q, fmt.Sprintf("after adding a new IP for %s", testName2))
+
+	// we should have an entry in the cache for this query
+	// if we add another IP, that cache entry should be removed
+	Debug.Printf("Adding a new IP to %s: the cache entry should be removed", testName1)
+	zone.AddRecord(containerID, testName1, net.ParseIP("10.2.2.3"))
+	assertNotInCache(t, cache, q, fmt.Sprintf("after adding a new IP for %s", testName1))
+
+	// Zone database at this point:
+	//   first.weave.local  = 10.2.2.1 10.2.2.2 10.2.2.3
+	//   second.weave.local = 10.9.9.1
+
+	Debug.Printf("Querying again (so a cache entry will be created)")
+	q, _ = assertExchange(t, testName1, dns.TypeA, testPort, 3, 4, 0)
+	assertInCache(t, cache, q, "after asking about the name")
+	Debug.Printf("... and removing one of the IP addresses")
+	zone.DeleteRecords(containerID, "", net.ParseIP("10.2.2.2"))
+	assertNotInCache(t, cache, q, "after deleting IP for 10.2.2.2")
+
+	// Zone database at this point:
+	//   first.weave.local  = 10.2.2.1 10.2.2.3
+	//   second.weave.local = 10.9.9.1
+
+	// generate cache responses
+	Debug.Printf("Querying for a raddr")
+	qname, _ := assertExchange(t, testName1, dns.TypeA, testPort, 2, 2, 0)
+	qptr, _ := assertExchange(t, "1.2.2.10.in-addr.arpa.", dns.TypePTR, testPort, 1, 1, 0)
+	qotherName, _ := assertExchange(t, testName2, dns.TypeA, testPort, 1, 1, 0)
+	qotherPtr, _ := assertExchange(t, "1.9.9.10.in-addr.arpa.", dns.TypePTR, testPort, 1, 1, 0)
+	qwrongName, _ := assertExchange(t, "wrong.weave.local.", dns.TypeA, testPort, 0, 0, dns.RcodeNameError)
+	assertInCache(t, cache, qname, "after asking for name")
+	assertInCache(t, cache, qptr, "after asking for address")
+	assertInCache(t, cache, qotherName, "after asking for second name")
+	assertInCache(t, cache, qotherPtr, "after asking for second address")
+	assertNotLocalInCache(t, cache, qwrongName, "after asking for a wrong name")
+
+	// now we will check if a removal affects all the responses
+	Debug.Printf("... and removing an IP should invalidate both the cached responses for name and raddr")
+	zone.DeleteRecords(containerID, "", net.ParseIP("10.2.2.1"))
+	assertNotInCache(t, cache, qptr, "after deleting record")
+	assertNotInCache(t, cache, qname, "after deleting record")
+	assertInCache(t, cache, qotherName, "after deleting record")
+
+	// Zone database at this point:
+	//   first.weave.local  = 10.2.2.3
+	//   second.weave.local = 10.9.9.1
+
+	// generate cache responses
+	Debug.Printf("Querying for a raddr")
+	qptr, _ = assertExchange(t, "3.2.2.10.in-addr.arpa.", dns.TypePTR, testPort, 1, 1, 0)
+	qname, _ = assertExchange(t, testName1, dns.TypeA, testPort, 1, 1, 0)
+	qotherName, _ = assertExchange(t, testName2, dns.TypeA, testPort, 1, 1, 0)
+	qotherPtr, _ = assertExchange(t, "1.9.9.10.in-addr.arpa.", dns.TypePTR, testPort, 1, 1, 0)
+	assertInCache(t, cache, qname, "after asking for name")
+	assertInCache(t, cache, qptr, "after asking for PTR")
+	assertInCache(t, cache, qotherName, "after asking for second name")
+	assertInCache(t, cache, qotherPtr, "after asking for second address")
+
+	// let's repeat this, but adding an IP
+	Debug.Printf("... and adding a new IP should invalidate both the cached responses for the name")
+	zone.AddRecord(containerID, testName1, net.ParseIP("10.2.2.7"))
+	assertNotInCache(t, cache, qname, "after adding a new IP")
+	assertInCache(t, cache, qotherName, "after adding a new IP")
+	assertInCache(t, cache, qotherPtr, "after adding a new IP")
+
+	// check that after some time, the cache entry is expired
+	clk.Forward(int(DefaultLocalTTL) + 1)
+	assertNotInCache(t, cache, qotherName, "after passing some time")
+	assertNotInCache(t, cache, qwrongName, "after passing some time")
+
+	// Zone database at this point:
+	//   first.weave.local  = 10.2.2.3 10.2.2.7
+	//   second.weave.local = 10.9.9.1
+
+	zone.DeleteRecords(containerID, "", net.IP{})
+	assertNotInCache(t, cache, qotherName, "after removing container")
+	assertNotInCache(t, cache, qotherPtr, "after removing container")
+}
+
+// Check if the entries in the cache expire
+func TestServerCacheExpiration(t *testing.T) {
+	const (
+		containerID      = "somecontainer"
+		testName1        = "first.weave.local."
+		testName2        = "second.weave.local."
+		negativeLocalTTL = 10
+	)
+
+	InitDefaultLogging(testing.Verbose())
+	Info.Println("TestServerCacheExpiration starting")
+
+	clk := newMockedClock()
+
+	Debug.Printf("Creating 2 zone databases")
+	zoneConfig := ZoneConfig{
+		RefreshInterval: 0, // no name updates
+		Clock:           clk,
+	}
+	dbs := newZoneDbsWithMockedMDns(2, zoneConfig)
+	dbs.Start()
+	defer dbs.Stop()
+
+	Debug.Printf("Creating a cache")
+	cache, err := NewCache(1024, clk)
+	wt.AssertNoErr(t, err)
+
+	Debug.Printf("Creating a real DNS server for the first zone database and with the cache")
+	srv, err := NewDNSServer(DNSServerConfig{
+		Zone:              dbs[0].Zone,
+		Cache:             cache,
+		Clock:             clk,
+		ListenReadTimeout: testSocketTimeout,
+		CacheNegLocalTTL:  negativeLocalTTL,
+	})
+	wt.AssertNoErr(t, err)
+	err = srv.Start()
+	wt.AssertNoErr(t, err)
+	go srv.ActivateAndServe()
+	defer srv.Stop()
+	time.Sleep(100 * time.Millisecond) // Allow server goroutine to start
+
+	testPort, err := srv.GetPort()
+	wt.AssertNoErr(t, err)
+	wt.AssertNotEqualInt(t, testPort, 0, "invalid listen port")
+
+	// Check that the DNS server knows nothing about testName1
+	qName1, _ := assertExchange(t, testName1, dns.TypeA, testPort, 0, 0, dns.RcodeNameError)
+	// there should be a negative reply cached during [0, negativeLocalTTL]
+	assertNotLocalInCache(t, cache, qName1, fmt.Sprintf("after asking for %s", testName1))
+	clk.Forward(negativeLocalTTL - 1)
+	assertNotLocalInCache(t, cache, qName1, fmt.Sprintf("after asking for %s", testName1))
+	// after negativeLocalTTL, there should be absolutely nothing in the cache
+	clk.Forward(2)
+	assertNotInCache(t, cache, qName1, fmt.Sprintf("after asking for %s", testName1))
+
+	// We add the IP in the second peer right after
+	Debug.Printf("Adding an IPs to %s in database #2", testName1)
+	dbs[1].Zone.AddRecord(containerID, testName1, net.ParseIP("10.2.2.1"))
+	t.Logf("Zone database #2:\n%s", dbs[1].Zone)
+
+	// Check that the DNS server returns updated information, and it will be
+	// cached during [0, DefaultLocalTTL]
+	qName1, _ = assertExchange(t, testName1, dns.TypeA, testPort, 1, 1, 0)
+
+	assertInCache(t, cache, qName1, fmt.Sprintf("after asking for %s", testName1))
+	clk.Forward(int(DefaultLocalTTL) - 1)
+	assertInCache(t, cache, qName1, fmt.Sprintf("after asking for %s", testName1))
+	// and it will expire after DefaultLocalTTL
+	clk.Forward(2)
+	assertNotInCache(t, cache, qName1, fmt.Sprintf("after asking for %s", testName1))
+
+	// We delete the IP in the second peer right after
+	Debug.Printf("Removing the IPs from %s in database #2", testName1)
+	dbs[1].Zone.DeleteRecords(containerID, "", net.ParseIP("10.2.2.1"))
+
+	// Check that we return to the initial state: a neg-local entry in the cache
+	qName1, _ = assertExchange(t, testName1, dns.TypeA, testPort, 0, 0, dns.RcodeNameError)
+	assertNotLocalInCache(t, cache, qName1, fmt.Sprintf("after asking for %s", testName1))
+	clk.Forward(negativeLocalTTL + 1)
+
+	// Check that all entries should be outdated
+	cache.Purge()
+	if cache.Len() != 0 {
+		t.Fatalf("Some entries still valid in cache:\n%s", cache)
+	}
+}
+
+// Check if the names updates lead to cache invalidations
+func TestServerCacheRefresh(t *testing.T) {
+	const (
+		containerID     = "somecontainer"
+		testName1       = "first.weave.local."
+		testName2       = "second.weave.local."
+		refreshInterval = int(DefaultLocalTTL) / 3
+	)
+
+	InitDefaultLogging(testing.Verbose())
+	Info.Println("TestServerCacheRefresh starting")
+	clk := newMockedClock()
+
+	Debug.Printf("Creating 2 zone databases")
+	zoneConfig := ZoneConfig{
+		RefreshInterval: refreshInterval,
+		Clock:           clk,
+	}
+	dbs := newZoneDbsWithMockedMDns(2, zoneConfig)
+	dbs.Start()
+	defer dbs.Stop()
+
+	Debug.Printf("Creating a cache")
+	cache, err := NewCache(1024, clk)
+	wt.AssertNoErr(t, err)
+
+	Debug.Printf("Creating a real DNS server for the first zone database and with the cache")
+	srv, err := NewDNSServer(DNSServerConfig{
+		Zone:              dbs[0].Zone,
+		Cache:             cache,
+		Clock:             clk,
+		ListenReadTimeout: testSocketTimeout,
+		MaxAnswers:        4,
+	})
+	wt.AssertNoErr(t, err)
+	err = srv.Start()
+	wt.AssertNoErr(t, err)
+	go srv.ActivateAndServe()
+	defer srv.Stop()
+	time.Sleep(100 * time.Millisecond) // Allow sever goroutine to start
+
+	testPort, err := srv.GetPort()
+	wt.AssertNoErr(t, err)
+	wt.AssertNotEqualInt(t, testPort, 0, "listen port")
+
+	Debug.Printf("Adding an IPs to %s", testName1)
+	dbs[1].Zone.AddRecord(containerID, testName1, net.ParseIP("10.2.2.1"))
+	dbs.Flush() // ensure the immediate refresh is done
+
+	// Zone database #2 at this point:
+	//   first.weave.local  = 10.2.2.1
+
+	// testName1 and testName2 should have no IPs yet
+	qName1, _ := assertExchange(t, testName1, dns.TypeA, testPort, 1, 1, 0)
+	qName2, _ := assertExchange(t, testName2, dns.TypeA, testPort, 0, 0, dns.RcodeNameError)
+	assertInCache(t, cache, qName1, "after asking for first name")
+	assertNotLocalInCache(t, cache, qName2, "after asking for second name")
+
+	clk.Forward(refreshInterval / 2)
+	dbs.Flush()
+
+	Debug.Printf("Adding an IP to %s and to %s", testName1, testName2)
+	dbs[1].Zone.AddRecord(containerID, testName1, net.ParseIP("10.2.2.2"))
+	dbs[1].Zone.AddRecord(containerID, testName2, net.ParseIP("10.9.9.2"))
+
+	// Zone database #2 at this point:
+	//   first.weave.local    = 10.2.2.1 10.2.2.2
+	//   second.weave.local   = 10.9.9.2
+	clk.Forward(refreshInterval/2 + 2)
+	dbs.Flush()
+
+	// at this point, the testName1 should have been refreshed
+	// so it should have two IPs, and the cache entry should have been invalidated
+	assertNotInCache(t, cache, qName1, fmt.Sprintf("after asking for %s", testName1))
+	assertNotLocalInCache(t, cache, qName2, fmt.Sprintf("after asking for %s", testName2))
+
+	qName1, _ = assertExchange(t, testName1, dns.TypeA, testPort, 2, 2, 0)
+	qName2, _ = assertExchange(t, testName2, dns.TypeA, testPort, 0, 0, dns.RcodeNameError)
+	assertInCache(t, cache, qName1, fmt.Sprintf("after asking for %s", testName1))
+	assertNotLocalInCache(t, cache, qName2, "after asking for a unknown name")
+
+	// delete the IPs, and some time passes by so the cache should be purged...
+	dbs[1].Zone.DeleteRecords(containerID, "", net.ParseIP("10.2.2.1"))
+	dbs[1].Zone.DeleteRecords(containerID, "", net.ParseIP("10.2.2.2"))
+	clk.Forward(refreshInterval + 1)
+	dbs.Flush()
+
+	qName1, _ = assertExchange(t, testName1, dns.TypeA, testPort, 0, 0, dns.RcodeNameError)
+	qName2, _ = assertExchange(t, testName2, dns.TypeA, testPort, 0, 0, dns.RcodeNameError)
+	assertNotLocalInCache(t, cache, qName1, "after asking for a unknown name")
+	assertNotLocalInCache(t, cache, qName2, "after asking for a unknown name")
+
+}

nameserver/server_test.go

@@ -3,12 +3,13 @@ package nameserver
 import (
 	"encoding/binary"
 	"fmt"
-	"github.com/miekg/dns"
-	. "github.com/weaveworks/weave/common"
-	wt "github.com/weaveworks/weave/testing"
 	"net"
 	"testing"
 	"time"
+
+	"github.com/miekg/dns"
+	. "github.com/weaveworks/weave/common"
+	wt "github.com/weaveworks/weave/testing"
 )
 
 const (
@@ -18,10 +19,6 @@ const (
 	testUDPBufSize   = 16384
 )
 
-var (
-	testPort = 17625
-)
-
 func setupForTest(t *testing.T) {
 	// fail early if we cannot find a default multicast interface
 	multicast, err := LinkLocalMulticastListener(nil)
@@ -29,89 +26,104 @@ func setupForTest(t *testing.T) {
 		t.Fatalf("Unable to create multicast listener: %s. No default multicast interface?", err)
 	}
 	multicast.Close()
-
-	testPort++
 }
 
 func TestUDPDNSServer(t *testing.T) {
 	setupForTest(t)
+
 	const (
-		containerID     = "foobar"
 		successTestName = "test1.weave.local."
-		failTestName    = "test2.weave.local."
+		failTestName    = "fail.weave.local."
 		nonLocalName    = "weave.works."
 		testAddr1       = "10.2.2.1"
+		containerID     = "somecontainer"
 	)
 	testCIDR1 := testAddr1 + "/24"
 
-	InitDefaultLogging(true)
-	var zone = NewZoneDb(DefaultLocalDomain)
+	InitDefaultLogging(testing.Verbose())
+	Info.Println("TestUDPDNSServer starting")
+
+	zone, err := NewZoneDb(ZoneConfig{})
+	wt.AssertNoErr(t, err)
+	err = zone.Start()
+	wt.AssertNoErr(t, err)
+	defer zone.Stop()
+
 	ip, _, _ := net.ParseCIDR(testCIDR1)
 	zone.AddRecord(containerID, successTestName, ip)
 
 	fallbackHandler := func(w dns.ResponseWriter, req *dns.Msg) {
+		if len(req.Question) == 0 {
+			return // ignore empty queries (sent when shutting down the server)
+		}
 		m := new(dns.Msg)
 		m.SetReply(req)
-		if len(req.Question) == 1 {
-			q := req.Question[0]
-			if q.Name == nonLocalName && q.Qtype == dns.TypeMX {
-				m.Answer = make([]dns.RR, 1)
-				m.Answer[0] = &dns.MX{Hdr: dns.RR_Header{Name: m.Question[0].Name, Rrtype: dns.TypeMX, Class: dns.ClassINET, Ttl: 0}, Mx: "mail." + nonLocalName}
-			} else if q.Name == nonLocalName && q.Qtype == dns.TypeANY {
-				m.Answer = make([]dns.RR, 512/len("mailn."+nonLocalName)+1)
-				for i := range m.Answer {
-					m.Answer[i] = &dns.MX{Hdr: dns.RR_Header{Name: m.Question[0].Name, Rrtype: dns.TypeMX, Class: dns.ClassINET, Ttl: 0}, Mx: fmt.Sprintf("mail%d.%s", i, nonLocalName)}
-				}
-			} else if q.Name == testRDNSnonlocal && q.Qtype == dns.TypePTR {
-				m.Answer = make([]dns.RR, 1)
-				m.Answer[0] = &dns.PTR{Hdr: dns.RR_Header{Name: m.Question[0].Name, Rrtype: dns.TypePTR, Class: dns.ClassINET, Ttl: 0}, Ptr: "ns1.google.com."}
-			} else if q.Name == testRDNSfail && q.Qtype == dns.TypePTR {
-				m.Rcode = dns.RcodeNameError
+		q := req.Question[0]
+		if q.Name == nonLocalName && q.Qtype == dns.TypeMX {
+			m.Answer = make([]dns.RR, 1)
+			m.Answer[0] = &dns.MX{Hdr: dns.RR_Header{Name: m.Question[0].Name, Rrtype: dns.TypeMX, Class: dns.ClassINET, Ttl: 0}, Mx: "mail." + nonLocalName}
+		} else if q.Name == nonLocalName && q.Qtype == dns.TypeANY {
+			m.Answer = make([]dns.RR, 512/len("mailn."+nonLocalName)+1)
+			for i := range m.Answer {
+				m.Answer[i] = &dns.MX{Hdr: dns.RR_Header{Name: m.Question[0].Name, Rrtype: dns.TypeMX, Class: dns.ClassINET, Ttl: 0}, Mx: fmt.Sprintf("mail%d.%s", i, nonLocalName)}
 			}
+		} else if q.Name == testRDNSnonlocal && q.Qtype == dns.TypePTR {
+			m.Answer = make([]dns.RR, 1)
+			m.Answer[0] = &dns.PTR{Hdr: dns.RR_Header{Name: m.Question[0].Name, Rrtype: dns.TypePTR, Class: dns.ClassINET, Ttl: 0}, Ptr: "ns1.google.com."}
+		} else if q.Name == testRDNSfail && q.Qtype == dns.TypePTR {
+			m.Rcode = dns.RcodeNameError
 		}
 		w.WriteMsg(m)
 	}
 
 	// Run another DNS server for fallback
-	s, fallbackAddr, err := runLocalUDPServer(t, "127.0.0.1:0", fallbackHandler)
+	fallback, err := newMockedFallback(fallbackHandler, nil)
 	wt.AssertNoErr(t, err)
-	defer s.Shutdown()
+	fallback.Start()
+	defer fallback.Stop()
 
-	_, fallbackPort, err := net.SplitHostPort(fallbackAddr)
+	srv, err := NewDNSServer(DNSServerConfig{
+		Zone:              zone,
+		UpstreamCfg:       fallback.CliConfig,
+		CacheDisabled:     true,
+		ListenReadTimeout: testSocketTimeout,
+	})
 	wt.AssertNoErr(t, err)
-
-	config := &dns.ClientConfig{Servers: []string{"127.0.0.1"}, Port: fallbackPort}
-	srv, err := NewDNSServer(DNSServerConfig{UpstreamCfg: config, Port: testPort}, zone, nil)
+	err = srv.Start()
 	wt.AssertNoErr(t, err)
+	go srv.ActivateAndServe()
 	defer srv.Stop()
-	go srv.Start()
 	time.Sleep(100 * time.Millisecond) // Allow sever goroutine to start
 
 	var r *dns.Msg
 
-	r = assertExchange(t, successTestName, dns.TypeA, 1, 1, 0)
+	testPort, err := srv.GetPort()
+	wt.AssertNoErr(t, err)
+	wt.AssertNotEqualInt(t, testPort, 0, "invalid listen port")
+
+	_, r = assertExchange(t, successTestName, dns.TypeA, testPort, 1, 1, 0)
 	wt.AssertType(t, r.Answer[0], (*dns.A)(nil), "DNS record")
 	wt.AssertEqualString(t, r.Answer[0].(*dns.A).A.String(), testAddr1, "IP address")
 
-	assertExchange(t, failTestName, dns.TypeA, 0, 0, dns.RcodeNameError)
+	assertExchange(t, failTestName, dns.TypeA, testPort, 0, 0, dns.RcodeNameError)
 
-	r = assertExchange(t, testRDNSsuccess, dns.TypePTR, 1, 1, 0)
+	_, r = assertExchange(t, testRDNSsuccess, dns.TypePTR, testPort, 1, 1, 0)
 	wt.AssertType(t, r.Answer[0], (*dns.PTR)(nil), "DNS record")
 	wt.AssertEqualString(t, r.Answer[0].(*dns.PTR).Ptr, successTestName, "IP address")
 
-	assertExchange(t, testRDNSfail, dns.TypePTR, 0, 0, dns.RcodeNameError)
+	assertExchange(t, testRDNSfail, dns.TypePTR, testPort, 0, 0, dns.RcodeNameError)
 
-	// This should fail because we don't have information about MX records
-	assertExchange(t, successTestName, dns.TypeMX, 0, 0, dns.RcodeNameError)
+	// This should fail because we don't handle MX records
+	assertExchange(t, successTestName, dns.TypeMX, testPort, 0, 0, dns.RcodeNameError)
 
 	// This non-local query for an MX record should succeed by being
 	// passed on to the fallback server
-	assertExchange(t, nonLocalName, dns.TypeMX, 1, -1, 0)
+	assertExchange(t, nonLocalName, dns.TypeMX, testPort, 1, -1, 0)
 
 	// Now ask a query that we expect to return a lot of data.
-	assertExchange(t, nonLocalName, dns.TypeANY, 5, -1, 0)
+	assertExchange(t, nonLocalName, dns.TypeANY, testPort, 5, -1, 0)
 
-	assertExchange(t, testRDNSnonlocal, dns.TypePTR, 1, -1, 0)
+	assertExchange(t, testRDNSnonlocal, dns.TypePTR, testPort, 1, -1, 0)
 
 	// Not testing MDNS functionality of server here (yet), since it
 	// needs two servers, each listening on its own address
@@ -119,30 +131,39 @@ func TestUDPDNSServer(t *testing.T) {
 
 func TestTCPDNSServer(t *testing.T) {
 	setupForTest(t)
+
 	const (
 		numAnswers   = 512
 		nonLocalName = "weave.works."
 	)
-	dnsAddr := fmt.Sprintf("localhost:%d", testPort)
 
-	InitDefaultLogging(true)
-	var zone = NewZoneDb(DefaultLocalDomain)
+	InitDefaultLogging(testing.Verbose())
+	Info.Println("TestTCPDNSServer starting")
+
+	zone, err := NewZoneDb(ZoneConfig{})
+	wt.AssertNoErr(t, err)
+	err = zone.Start()
+	wt.AssertNoErr(t, err)
+	defer zone.Stop()
 
 	// generate a list of `numAnswers` IP addresses
 	var addrs []ZoneRecord
 	bs := make([]byte, 4)
 	for i := 0; i < numAnswers; i++ {
 		binary.LittleEndian.PutUint32(bs, uint32(i))
-		addrs = append(addrs, Record{"", net.IPv4(bs[0], bs[1], bs[2], bs[3]), 0, 0, 0})
+		ip := net.IPv4(bs[0], bs[1], bs[2], bs[3])
+		addrs = append(addrs, ZoneRecord(Record{"", ip, 0, 0, 0}))
 	}
 
 	// handler for the fallback server: it will just return a very long response
 	fallbackUDPHandler := func(w dns.ResponseWriter, req *dns.Msg) {
+		if len(req.Question) == 0 {
+			return // ignore empty queries (sent when shutting down the server)
+		}
 		maxLen := getMaxReplyLen(req, protUDP)
-
 		t.Logf("Fallback UDP server got asked: returning %d answers", numAnswers)
 		q := req.Question[0]
-		m := makeAddressReply(req, &q, addrs)
+		m := makeAddressReply(req, &q, addrs, DefaultLocalTTL)
 		mLen := m.Len()
 		m.SetEdns0(uint16(maxLen), false)
 
@@ -153,34 +174,40 @@ func TestTCPDNSServer(t *testing.T) {
 		w.WriteMsg(m)
 	}
 	fallbackTCPHandler := func(w dns.ResponseWriter, req *dns.Msg) {
+		if len(req.Question) == 0 {
+			return // ignore empty queries (sent when shutting down the server)
+		}
 		t.Logf("Fallback TCP server got asked: returning %d answers", numAnswers)
 		q := req.Question[0]
-		m := makeAddressReply(req, &q, addrs)
+		m := makeAddressReply(req, &q, addrs, DefaultLocalTTL)
 		w.WriteMsg(m)
 	}
 
 	t.Logf("Running a DNS fallback server with UDP")
-	us, fallbackUDPAddr, err := runLocalUDPServer(t, "127.0.0.1:0", fallbackUDPHandler)
+	fallback, err := newMockedFallback(fallbackUDPHandler, fallbackTCPHandler)
 	wt.AssertNoErr(t, err)
-	defer us.Shutdown()
+	fallback.Start()
+	defer fallback.Stop()
 
-	_, fallbackPort, err := net.SplitHostPort(fallbackUDPAddr)
+	t.Logf("Creating a WeaveDNS server instance, falling back to 127.0.0.1:%d", fallback.Port)
+	srv, err := NewDNSServer(DNSServerConfig{
+		Zone:              zone,
+		UpstreamCfg:       fallback.CliConfig,
+		CacheDisabled:     true,
+		ListenReadTimeout: testSocketTimeout,
+	})
 	wt.AssertNoErr(t, err)
-
-	t.Logf("Starting another fallback server, with TCP, on the same port as the UDP server")
-	fallbackTCPAddr := fmt.Sprintf("127.0.0.1:%s", fallbackPort)
-	ts, fallbackTCPAddr, err := runLocalTCPServer(t, fallbackTCPAddr, fallbackTCPHandler)
-	wt.AssertNoErr(t, err)
-	defer ts.Shutdown()
-
-	t.Logf("Creating a WeaveDNS server instance, falling back to 127.0.0.1:%s", fallbackPort)
-	config := &dns.ClientConfig{Servers: []string{"127.0.0.1"}, Port: fallbackPort}
-	srv, err := NewDNSServer(DNSServerConfig{UpstreamCfg: config, Port: testPort}, zone, nil)
+	err = srv.Start()
 	wt.AssertNoErr(t, err)
+	go srv.ActivateAndServe()
 	defer srv.Stop()
-	go srv.Start()
 	time.Sleep(100 * time.Millisecond) // Allow sever goroutine to start
 
+	testPort, err := srv.GetPort()
+	wt.AssertNoErr(t, err)
+	wt.AssertNotEqualInt(t, testPort, 0, "listen port")
+	dnsAddr := fmt.Sprintf("127.0.0.1:%d", testPort)
+
 	t.Logf("Creating a UDP and a TCP client")
 	uc := new(dns.Client)
 	uc.UDPSize = minUDPSize
@@ -192,8 +219,8 @@ func TestTCPDNSServer(t *testing.T) {
 	m.RecursionDesired = true
 	m.SetQuestion(nonLocalName, dns.TypeA)
 
-	t.Logf("Checking the fallback server at %s returns a truncated response with UDP", fallbackUDPAddr)
-	r, _, err := uc.Exchange(m, fallbackUDPAddr)
+	t.Logf("Checking the fallback server at %s returns a truncated response with UDP", fallback.Addr)
+	r, _, err := uc.Exchange(m, fallback.Addr)
 	t.Logf("Got response from fallback server (UDP) with %d answers", len(r.Answer))
 	t.Logf("Response:\n%+v\n", r)
 	wt.AssertNoErr(t, err)
@@ -229,73 +256,3 @@ func TestTCPDNSServer(t *testing.T) {
 	wt.AssertFalse(t, r.MsgHdr.Truncated, "DNS truncated response flag")
 	wt.AssertEqualInt(t, len(r.Answer), numAnswers, "number of answers (UDP-long)")
 }
-
-//////////////////////////////////////////////////////////////////////////////
-
-// perform a DNS query and assert the reply code, number or answers, etc
-func assertExchange(t *testing.T, z string, ty uint16, minAnswers int, maxAnswers int, expErr int) *dns.Msg {
-	c := new(dns.Client)
-	c.UDPSize = testUDPBufSize
-	m := new(dns.Msg)
-	m.RecursionDesired = true
-	m.SetQuestion(z, ty)
-	m.SetEdns0(testUDPBufSize, false) // we don't want to play with truncation here...
-
-	r, _, err := c.Exchange(m, fmt.Sprintf("127.0.0.1:%d", testPort))
-	t.Logf("Response:\n%+v\n", r)
-	wt.AssertNoErr(t, err)
-	if minAnswers == 0 && maxAnswers == 0 {
-		wt.AssertStatus(t, r.Rcode, expErr, "DNS response code")
-	} else {
-		wt.AssertStatus(t, r.Rcode, dns.RcodeSuccess, "DNS response code")
-	}
-	answers := len(r.Answer)
-	if minAnswers >= 0 && answers < minAnswers {
-		wt.Fatalf(t, "Number of answers >= %d", minAnswers)
-	}
-	if maxAnswers >= 0 && answers > maxAnswers {
-		wt.Fatalf(t, "Number of answers <= %d", maxAnswers)
-	}
-	return r
-}
-
-// run a UDP fallback server
-func runLocalUDPServer(t *testing.T, laddr string, handler dns.HandlerFunc) (*dns.Server, string, error) {
-	t.Logf("Starting fallback UDP server at %s", laddr)
-	pc, err := net.ListenPacket("udp", laddr)
-	if err != nil {
-		return nil, "", err
-	}
-	server := &dns.Server{PacketConn: pc, Handler: handler, ReadTimeout: 100 * time.Millisecond}
-
-	go func() {
-		server.ActivateAndServe()
-		pc.Close()
-	}()
-
-	t.Logf("Fallback UDP server listening at %s", pc.LocalAddr())
-	return server, pc.LocalAddr().String(), nil
-}
-
-// run a TCP fallback server
-func runLocalTCPServer(t *testing.T, laddr string, handler dns.HandlerFunc) (*dns.Server, string, error) {
-	t.Logf("Starting fallback TCP server at %s", laddr)
-	laddrTCP, err := net.ResolveTCPAddr("tcp", laddr)
-	if err != nil {
-		return nil, "", err
-	}
-
-	l, err := net.ListenTCP("tcp", laddrTCP)
-	if err != nil {
-		return nil, "", err
-	}
-	server := &dns.Server{Listener: l, Handler: handler, ReadTimeout: 100 * time.Millisecond}
-
-	go func() {
-		server.ActivateAndServe()
-		l.Close()
-	}()
-
-	t.Logf("Fallback TCP server listening at %s", l.Addr().String())
-	return server, l.Addr().String(), nil
-}

nameserver/zone.go

@@ -3,41 +3,30 @@ package nameserver
 import (
 	"bytes"
 	"fmt"
-	"github.com/miekg/dns"
-	. "github.com/weaveworks/weave/common"
 	"net"
 	"sync"
+	"time"
+
+	"github.com/benbjohnson/clock"
+	"github.com/miekg/dns"
+	. "github.com/weaveworks/weave/common"
 )
 
 const (
 	RDNSDomain = "in-addr.arpa."
 )
 
+const (
+	DefaultLocalDomain     = "weave.local."       // The default name used for the local domain
+	DefaultRefreshInterval = int(DefaultLocalTTL) // Period for background updates with mDNS
+	DefaultRelevantTime    = 60                   // When to forget info about remote info if nobody asks...
+
+	defaultRemoteIdent = "weave:remote" // Ident used for info obtained from mDNS
+)
+
 // +1 to also exclude a dot
 var rdnsDomainLen = len(RDNSDomain) + 1
 
-type Zone interface {
-	AddRecord(ident string, name string, ip net.IP) error
-	DeleteRecord(ident string, ip net.IP) error
-	DeleteRecordsFor(ident string) error
-	Domain() string
-	ZoneLookup
-}
-
-type dbRecord struct {
-	Ident string
-	Name  string
-	IP    net.IP
-}
-
-// Very simple data structure for now, with linear searching.
-// TODO: make more sophisticated to improve performance.
-type ZoneDb struct {
-	mx     sync.RWMutex
-	recs   []dbRecord
-	domain string
-}
-
 type LookupError string
 
 func (ops LookupError) Error() string {
@@ -51,103 +40,700 @@ func (dup DuplicateError) Error() string {
 	return "Tried to add a duplicate entry"
 }
 
-func NewZoneDb(domain string) *ZoneDb {
-	return &ZoneDb{
-		domain: domain,
+type Zone interface {
+	ZoneObservable
+	ZoneLookup
+	// The domain where we operate (eg, "weave.local.")
+	Domain() string
+	// Add a record in the local database
+	AddRecord(ident string, name string, ip net.IP) error
+	// Delete matching records (uninitialised values act as wildcards)
+	DeleteRecords(ident string, name string, ip net.IP) (count int)
+	// Lookup for a name in the whole domain
+	DomainLookupName(name string) ([]ZoneRecord, error)
+	// Lookup for an address in the whole domain
+	DomainLookupInaddr(inaddr string) ([]ZoneRecord, error)
+	// Return a status string
+	Status() string
+}
+
+///////////////////////////////////////////////////////////////////
+//
+// Zone database overview:
+//
+//           idents            names             entry
+//         *-------*          *------*         *------*
+// zone -> | ident |  --*-->  | name |  --*->  | IPv4 |
+//         *-------*    |     *------*    |    *------*
+//         | ident |    *-->  | name |    *->  | IPv4 |
+//         *-------*          *------*         *------*
+//            ...               ...
+//
+// The zone database keeps an ident per container, but also a special
+// ident for names/IPs obtained from remote peers.
+//
+// Each name can have multiple IPs (so far, we only consider IPv4s), and
+// each IP is stored in a `recordEntry` in the database. Entries store
+// some additional information like priority and weight of the IP (for
+// example, for future use with SRV records)
+//
+
+// An entry in the zone database
+type recordEntry struct {
+	Record
+
+	observers []ZoneRecordObserver // the observers for this record
+	insTime   time.Time            // time when this record was inserted
+}
+
+func newRecordEntry(zr ZoneRecord, now time.Time) *recordEntry {
+	return &recordEntry{
+		Record: Record{
+			name:     zr.Name(),
+			ip:       zr.IP(),
+			priority: zr.Priority(),
+			weight:   zr.Weight(),
+			ttl:      zr.TTL(),
+		},
+		insTime:   now,
+		observers: make([]ZoneRecordObserver, 0),
 	}
 }
 
-func (zone *ZoneDb) Domain() string {
-	return zone.domain
+func (re *recordEntry) addIPObserver(zro ZoneRecordObserver) {
+	re.observers = append(re.observers, zro)
 }
 
-func (zone *ZoneDb) indexOf(match func(dbRecord) bool) int {
-	for i, r := range zone.recs {
-		if match(r) {
-			return i
+// Notify all the IP observers and then remove them all
+// IP address observers are notified when
+// - the IP is removed
+// - the name is removed
+func (re *recordEntry) notifyIPObservers() {
+	numObservers := len(re.observers)
+	if numObservers > 0 {
+		Debug.Printf("[zonedb] Notifying %d observers of '%s'", numObservers, re.ip)
+		for _, observer := range re.observers {
+			observer()
 		}
+		re.observers = make([]ZoneRecordObserver, 0)
 	}
-	return -1
 }
 
-func (zone *ZoneDb) String() string {
-	zone.mx.RLock()
-	defer zone.mx.RUnlock()
+func (re *recordEntry) TTL() int {
+	// if we never set the TTL (eg, when using AddRecord()), return the standard value...
+	if re.ttl == 0 {
+		return int(DefaultLocalTTL)
+	}
+	return re.ttl
+}
+
+// Check if the info in this record is still valid according to the TTL
+func (re *recordEntry) hasExpired(now time.Time) bool {
+	return re.ttl > 0 && now.Sub(re.insTime).Seconds() > float64(re.ttl)
+}
+
+func (re *recordEntry) String() string {
 	var buf bytes.Buffer
-	for _, r := range zone.recs {
-		fmt.Fprintf(&buf, "%.12s %s %v\n", r.Ident, r.IP, r.Name)
+	fmt.Fprintf(&buf, "%s", re.Record)
+	lobs := len(re.observers)
+	if lobs > 0 {
+		fmt.Fprintf(&buf, "/OBS:%d", lobs)
 	}
 	return buf.String()
 }
 
-func (zone *ZoneDb) LookupName(name string) ([]ZoneRecord, error) {
-	zone.mx.RLock()
-	defer zone.mx.RUnlock()
-	for _, r := range zone.recs {
-		if r.Name == name {
-			return []ZoneRecord{Record{r.Name, r.IP, 0, 0, 0}}, nil
-		}
-	}
-	return nil, LookupError(name)
+///////////////////////////////////////////////////////////////////
+
+// a name, with group of records by IPv4
+type name struct {
+	name            string
+	ipv4            map[IPv4]*recordEntry // all the IPv4 records for this name
+	observers       []ZoneRecordObserver  // the observers for this name
+	lastRefreshTime time.Time             // last time this name was updated by broadcasting a query
+	lastAccessTime  time.Time             // last time a Lookup* used info from this name
 }
 
-func (zone *ZoneDb) LookupInaddr(inaddr string) ([]ZoneRecord, error) {
-	if revIP := net.ParseIP(inaddr[:len(inaddr)-rdnsDomainLen]); revIP != nil {
-		revIP4 := revIP.To4()
-		ip := []byte{revIP4[3], revIP4[2], revIP4[1], revIP4[0]}
-		Debug.Printf("[zonedb] Looking for address: %+v", ip)
-		zone.mx.RLock()
-		defer zone.mx.RUnlock()
-		for _, r := range zone.recs {
-			if r.IP.Equal(ip) {
-				return []ZoneRecord{Record{r.Name, r.IP, 0, 0, 0}}, nil
+func newName(n string) *name {
+	return &name{
+		name:      n,
+		ipv4:      make(map[IPv4]*recordEntry),
+		observers: make([]ZoneRecordObserver, 0),
+	}
+}
+func (n *name) len() int             { return len(n.ipv4) }
+func (n *name) empty() bool          { return n.len() == 0 }
+func (n *name) hasIP(ip net.IP) bool { return n.hasIPv4(ipToIPv4(ip)) }
+func (n *name) hasIPv4(ip IPv4) bool { _, found := n.ipv4[ip]; return found }
+
+// Get all the entries for this name
+func (n *name) getAllEntries() (res []*recordEntry) {
+	for _, entry := range n.ipv4 {
+		res = append(res, entry)
+	}
+	return
+}
+
+// Get the entry for an IP in this name
+func (n *name) getEntryForIP(ip IPv4) (res *recordEntry) {
+	if entry, found := n.ipv4[ip]; found {
+		res = entry
+	}
+	return
+}
+
+// Add a new IPv4 to this name
+func (n *name) addIP(zr ZoneRecord, now time.Time) (*recordEntry, error) {
+	ne := newRecordEntry(zr, now)
+	ipv4 := ipToIPv4(zr.IP())
+	n.ipv4[ipv4] = ne
+	n.notifyNameObservers()
+	return ne, nil
+}
+
+// Delete by IP. Pass uninitialized value for wildcard
+func (n *name) deleteIP(ip net.IP) int {
+	count := 0
+	if !ip.Equal(net.IP{}) {
+		ipv4 := ipToIPv4(ip)
+		if ipv4V, found := n.ipv4[ipv4]; found {
+			ipv4V.notifyIPObservers()
+			delete(n.ipv4, ipv4)
+			count++
+		}
+	} else {
+		for ipv4K, ipv4V := range n.ipv4 {
+			ipv4V.notifyIPObservers()
+			delete(n.ipv4, ipv4K)
+			count++
+		}
+	}
+	if count > 0 {
+		n.notifyNameObservers()
+	}
+	return count
+}
+
+// Update the list of IPs for this name, adding new ones and removing old IPs...
+func (n *name) updateIPs(records []ZoneRecord, now time.Time) (added int, removed int) {
+	ipsMap := make(map[IPv4]bool)
+
+	// add the new IPs
+	for _, record := range records {
+		ipv4 := ipToIPv4(record.IP())
+		ipsMap[ipv4] = true
+		if !n.hasIPv4(ipv4) {
+			n.addIP(record, now)
+			added++
+		}
+	}
+
+	// remove the old IPs
+	for ipv4, ipRecord := range n.ipv4 {
+		if _, isNew := ipsMap[ipv4]; !isNew {
+			ipRecord.notifyIPObservers() // we must notify the observers before removing the IP record...
+			delete(n.ipv4, ipv4)
+			removed++
+		}
+	}
+
+	// notify observers and update times
+	if added > 0 || removed > 0 {
+		n.lastRefreshTime = now
+		n.notifyNameObservers()
+	}
+	return
+}
+
+// Add a observer for this name
+func (n *name) addNameObserver(observer ZoneRecordObserver) {
+	n.observers = append(n.observers, observer)
+}
+
+// Notify and flush all the observers
+// Name observers are notified when
+// - an IP is added
+// - an IP is removed
+// - the name is removed
+func (n *name) notifyNameObservers() {
+	numObservers := len(n.observers)
+	if numObservers > 0 {
+		Debug.Printf("[zonedb] Notifying %d observers of '%s'", numObservers, n.name)
+		for _, observer := range n.observers {
+			observer()
+		}
+		n.observers = make([]ZoneRecordObserver, 0)
+	}
+}
+
+// Convert a set of IP records to a comma-separated string
+func (n *name) String() string {
+	var buf bytes.Buffer
+	for _, ip := range n.ipv4 {
+		fmt.Fprintf(&buf, "%s, ", ip)
+	}
+	l := buf.Len()
+	if l > 2 {
+		buf.Truncate(l - 2)
+	}
+	return buf.String()
+}
+
+///////////////////////////////////////////////////////////////////
+
+// all the names in a ident
+type nameSet struct {
+	names map[string]*name
+}
+
+func newNameSet() *nameSet      { return &nameSet{names: make(map[string]*name)} }
+func (ns *nameSet) len() int    { return len(ns.names) }
+func (ns *nameSet) empty() bool { return ns.len() == 0 }
+func (ns *nameSet) hasName(name string) bool {
+	n, found := ns.names[name]
+	if !found {
+		return false
+	}
+	// the name must have some valid IPs in order to be considered as "existent"
+	return len(n.ipv4) > 0
+}
+
+// Get the zone entries for a name
+// If the name does not exist and `create` is true, a new name will be created
+func (ns *nameSet) getName(name string, create bool) (n *name) {
+	n, found := ns.names[name]
+	if !found && create {
+		newName := newName(name)
+		ns.names[name] = newName
+		return newName
+	}
+	return n
+}
+
+// Get all the entries for a name
+func (ns *nameSet) getEntriesForName(name string) (res []*recordEntry) {
+	n := ns.getName(name, false)
+	if n != nil {
+		res = n.getAllEntries()
+	}
+	return
+}
+
+// Get all the entries for an IP, for all names
+func (ns *nameSet) getEntriesForIP(ip IPv4) (res []*recordEntry) {
+	for _, name := range ns.names {
+		entry := name.getEntryForIP(ip)
+		if entry != nil {
+			res = append(res, entry)
+		}
+	}
+	return
+}
+
+// Add a new IPv4 to a name
+func (ns *nameSet) addIPToName(zr ZoneRecord, now time.Time) (*recordEntry, error) {
+	n := ns.getName(zr.Name(), true)
+	ipv4 := ipToIPv4(zr.IP())
+	if n.hasIPv4(ipv4) {
+		return n.getEntryForIP(ipv4), DuplicateError{}
+	}
+	ns.touchName(zr.Name(), now)
+	return n.addIP(zr, now)
+}
+
+// Delete by name and IP. Pass uninitialized values for wildcard
+func (ns nameSet) deleteNameIP(name string, ip net.IP) int {
+	count := 0
+	if name != "" {
+		if nameV, found := ns.names[name]; found {
+			count += nameV.deleteIP(ip)
+			if nameV.empty() {
+				delete(ns.names, name)
 			}
 		}
-		return nil, LookupError(inaddr)
-	}
-	Warning.Printf("[zonedb] Asked to reverse lookup %s", inaddr)
-	return nil, LookupError(inaddr)
-}
-
-func (zone *ZoneDb) AddRecord(ident string, name string, ip net.IP) error {
-	zone.mx.Lock()
-	defer zone.mx.Unlock()
-	fqdn := dns.Fqdn(name)
-	pred := func(r dbRecord) bool { return r.Name == fqdn && r.IP.Equal(ip) && r.Ident == ident }
-	if index := zone.indexOf(pred); index != -1 {
-		return DuplicateError{}
-	}
-	zone.recs = append(zone.recs, dbRecord{ident, fqdn, ip})
-	return nil
-}
-
-func (zone *ZoneDb) DeleteRecord(ident string, ip net.IP) error {
-	zone.mx.Lock()
-	defer zone.mx.Unlock()
-	pred := func(r dbRecord) bool { return r.Ident == ident && r.IP.Equal(ip) }
-	if index := zone.indexOf(pred); index != -1 {
-		zone.recs = append(zone.recs[:index], zone.recs[index+1:]...)
-		return nil
-	}
-	return LookupError(ident)
-}
-
-func (zone *ZoneDb) DeleteRecordsFor(ident string) error {
-	zone.mx.Lock()
-	defer zone.mx.Unlock()
-	w := 0 // write index
-
-	for _, r := range zone.recs {
-		if r.Ident != ident {
-			zone.recs[w] = r
-			w++
+	} else {
+		for nameK, nameV := range ns.names {
+			count += nameV.deleteIP(ip)
+			if nameV.empty() {
+				delete(ns.names, nameK)
+			}
 		}
 	}
-	zone.recs = zone.recs[:w]
+	return count
+}
+
+// Get the name query time
+func (ns *nameSet) getNameLastAccess(n string) time.Time {
+	if name, found := ns.names[n]; found {
+		return name.lastAccessTime
+	}
+	return time.Time{}
+}
+
+// Touch the last access time for a name
+// The access time is saved only for locally-introduced records (otherwise it could
+// be lost when names are irrelevant...)
+func (ns *nameSet) touchName(name string, now time.Time) {
+	Debug.Printf("[zonedb] Touching name %s", name)
+	n := ns.getName(name, false)
+	if n != nil {
+		n.lastAccessTime = now
+	}
+}
+
+func (ns *nameSet) String() string {
+	var buf bytes.Buffer
+	for _, name := range ns.names {
+		fmt.Fprintf(&buf, "%s, ", name)
+	}
+	l := buf.Len()
+	if l > 2 {
+		buf.Truncate(l - 2)
+	}
+	return buf.String()
+}
+
+///////////////////////////////////////////////////////////////////
+
+// Name sets, by ident name
+type identRecordSet map[string]*nameSet
+
+func (irs identRecordSet) String() string {
+	var buf bytes.Buffer
+	for ident, nameset := range irs {
+		fmt.Fprintf(&buf, "%.12s: %s\n", ident, nameset)
+	}
+	return buf.String()
+}
+
+///////////////////////////////////////////////////////////////////
+
+// Invariant on updateRequests: they are in ascending order of time.
+type refreshRequest struct {
+	name string
+	time time.Time
+}
+
+type ZoneConfig struct {
+	// The local domain
+	Domain string
+	// The interface where mDNS operates
+	Iface *net.Interface
+	// Refresh interval for local names in the zone database (disabled by default) (in seconds)
+	RefreshInterval int
+	// Forget about remote info if nobody asks for it in this time (in seconds)
+	RelevantTime int
+	// TTL returnined in local asnwers (with mDNS)
+	LocalTTL int
+	// Force a specific mDNS client
+	MDNSClient ZoneMDNSClient
+	// Force a specific mDNS server
+	MDNSServer ZoneMDNSServer
+	// For a specific clock provider
+	Clock clock.Clock
+}
+
+// Zone database
+// The zone database contains the locally known information about the zone.
+// The zone database uses some other mechanisms (eg, mDNS) for finding unknown
+// information or for keeping current information up to date.
+type ZoneDb struct {
+	mx     sync.RWMutex
+	idents identRecordSet
+
+	mdnsCli       ZoneMDNSClient
+	mdnsSrv       ZoneMDNSServer
+	iface         *net.Interface
+	domain        string        // the local domain
+	relevantLimit time.Duration // if no one asks for something in this time, it is not relevant a anymore...
+	clock         clock.Clock
+
+	refreshScheds    *SchedQueue
+	refreshCloseChan chan bool
+	refreshInterval  time.Duration
+}
+
+// Create a new zone database
+func NewZoneDb(config ZoneConfig) (zone *ZoneDb, err error) {
+	zone = &ZoneDb{
+		domain:           config.Domain,
+		idents:           make(identRecordSet),
+		mdnsCli:          config.MDNSClient,
+		mdnsSrv:          config.MDNSServer,
+		iface:            config.Iface,
+		clock:            config.Clock,
+		relevantLimit:    time.Duration(DefaultRelevantTime) * time.Second,
+		refreshCloseChan: make(chan bool),
+	}
+
+	// fix the default configuration parameters
+	if zone.clock == nil {
+		zone.clock = clock.New()
+	}
+	if len(zone.domain) == 0 {
+		zone.domain = DefaultLocalDomain
+	}
+	if config.RefreshInterval > 0 {
+		zone.refreshInterval = time.Duration(config.RefreshInterval) * time.Second
+	}
+	if config.RelevantTime > 0 {
+		zone.relevantLimit = time.Duration(config.RelevantTime) * time.Second
+	}
+
+	if zone.refreshInterval > 0 {
+		zone.refreshScheds = NewSchedQueue(zone.clock)
+	}
+
+	// Create the mDNS client and server
+	if zone.mdnsCli == nil {
+		if zone.mdnsCli, err = NewMDNSClient(); err != nil {
+			return
+		}
+	}
+	if zone.mdnsSrv == nil {
+		mdnsTTL := DefaultLocalTTL
+		if config.LocalTTL > 0 {
+			mdnsTTL = config.LocalTTL
+		}
+
+		if zone.mdnsSrv, err = NewMDNSServer(zone, false, mdnsTTL); err != nil {
+			return
+		}
+	}
+
+	return
+}
+
+// Start the zone database
+func (zone *ZoneDb) Start() (err error) {
+	if zone.iface != nil {
+		Info.Printf("[zonedb] Using mDNS on %+v", zone.iface)
+	} else {
+		Info.Printf("[zonedb] Using mDNS on all interfaces")
+	}
+
+	if err = zone.mdnsCli.Start(zone.iface); err != nil {
+		return
+	}
+	if err = zone.mdnsSrv.Start(zone.iface); err != nil {
+		return
+	}
+	if zone.refreshInterval > 0 {
+		zone.refreshScheds.Start()
+	}
+
+	return
+}
+
+// Perform a graceful shutdown of the zone database
+func (zone *ZoneDb) Stop() error {
+	if zone.refreshInterval > 0 {
+		Debug.Printf("[zonedb] Closing background updaters...")
+		zone.refreshScheds.Stop()
+	}
+
+	Debug.Printf("[zonedb] Exiting mDNS client and server...")
+	zone.mdnsCli.Stop()
+	zone.mdnsSrv.Stop()
 	return nil
 }
 
+// Obtain the domain where this database keeps information for
+func (zone *ZoneDb) Domain() string {
+	return zone.domain
+}
+
+// Return the status string
+func (zone *ZoneDb) Status() string {
+	var buf bytes.Buffer
+	fmt.Fprintln(&buf, "Local domain", zone.domain)
+	fmt.Fprintln(&buf, "Interface", zone.iface)
+	fmt.Fprintf(&buf, "Zone database:\n%s", zone)
+	return buf.String()
+}
+
+// Add a record with `name` pointing to `ip`
+func (zone *ZoneDb) AddRecord(ident string, name string, ip net.IP) (err error) {
+	zone.mx.Lock()
+	defer zone.mx.Unlock()
+	Debug.Printf("[zonedb] Adding record: '%s'/'%s'[%s]", ident, name, ip)
+	record := Record{dns.Fqdn(name), ip, 0, 0, 0}
+	_, err = zone.getNameSet(ident).addIPToName(record, zone.clock.Now())
+	return
+}
+
+// Delete matching records (uninitialised values act as wildcards)
+func (zone *ZoneDb) DeleteRecords(ident string, name string, ip net.IP) int {
+	if name != "" {
+		name = dns.Fqdn(name)
+	}
+
+	zone.mx.Lock()
+	defer zone.mx.Unlock()
+
+	return zone.deleteIdentNameIP(ident, name, ip)
+}
+
+// Observe a name.
+// The name must have at least one valid IP address.
+// Name observers are notified when
+// - an IP is added
+// - an IP is removed
+// - the name is removed
+// The observer will be invoked on any change/removal that affects this name.
+// Each observer will be invoked at least once (but possibly more). After that, they will be removed.
+// The observer should not try to lock the ZoneDB (you will get a deadlock)
+func (zone *ZoneDb) ObserveName(name string, observer ZoneRecordObserver) (err error) {
+	zone.mx.Lock()
+	defer zone.mx.Unlock()
+
+	cnt := 0
+	name = dns.Fqdn(name)
+	for _, nameset := range zone.idents {
+		if n := nameset.getName(name, false); n != nil {
+			n.addNameObserver(observer)
+			cnt++
+		}
+	}
+	if cnt == 0 {
+		err = LookupError(fmt.Sprintf("%s", name))
+	}
+	return
+}
+
+// Observe a IP address.
+// The IP address must be exit in the database.
+// IP address observers are notified when
+// - the IP is removed
+// - the name is removed
+// The observer will be invoked on any change/removal that affects this IP.
+// Each observer will be invoked at least once (but possibly more). After that, they will be removed.
+// The observer should not try to lock the ZoneDB (you will get a deadlock)
+func (zone *ZoneDb) ObserveInaddr(inaddr string, observer ZoneRecordObserver) (err error) {
+	zone.mx.Lock()
+	defer zone.mx.Unlock()
+
+	cnt := 0
+	revIP, err := raddrToIPv4(inaddr)
+	if err != nil {
+		return newParseError("lookup address", inaddr)
+	}
+	for _, nameset := range zone.idents {
+		for _, entry := range nameset.getEntriesForIP(revIP) {
+			entry.addIPObserver(observer)
+			cnt++
+		}
+	}
+	if cnt == 0 {
+		err = LookupError(fmt.Sprintf("%+v/%+v", inaddr, revIP))
+	}
+	return
+}
+
+// Get the string representation of a zone
+func (zone *ZoneDb) String() string {
+	zone.mx.RLock()
+	defer zone.mx.RUnlock()
+	return zone.idents.String()
+}
+
+// Notify that a container has died
 func (zone *ZoneDb) ContainerDied(ident string) error {
 	Info.Printf("[zonedb] Container %s down. Removing records", ident)
-	return zone.DeleteRecordsFor(ident)
+	zone.DeleteRecords(ident, "", net.IP{})
+	return nil
+}
+
+// Return true if a remote name is still relevant
+func (zone *ZoneDb) IsNameRelevant(n string) bool {
+	zone.mx.Lock()
+	defer zone.mx.Unlock()
+
+	lastAccess := time.Time{}
+	for _, nameset := range zone.idents {
+		t := nameset.getNameLastAccess(n)
+		if !t.IsZero() && (lastAccess.IsZero() || t.After(lastAccess)) {
+			lastAccess = t
+		}
+	}
+
+	return !lastAccess.IsZero() && zone.clock.Now().Sub(lastAccess) <= zone.relevantLimit
+}
+
+// Returns true if the info we have for a remote name has expired. Local names are never expired
+// Returns false if we do not have info for this name from remote peers.
+func (zone *ZoneDb) IsNameExpired(n string) bool {
+	zone.mx.Lock()
+	defer zone.mx.Unlock()
+
+	now := zone.clock.Now()
+	remoteNameSet := zone.getNameSet(defaultRemoteIdent)
+	entries := remoteNameSet.getEntriesForName(n)
+	if entries != nil {
+		for _, entry := range entries {
+			if entry.hasExpired(now) {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+func (zone *ZoneDb) HasNameLocalInfo(n string) bool {
+	zone.mx.Lock()
+	defer zone.mx.Unlock()
+
+	remoteNameSet := zone.getNameSet(defaultRemoteIdent)
+	for _, nameset := range zone.idents {
+		if nameset != remoteNameSet {
+			if nameset.hasName(n) {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+// Return true if the we have obtained some information for a name from remote peers
+// We can have both local (eg, introduced through the HTTP API) and remote (eg, mDNS)
+// information for a name. This method only checks if there is remote information...
+func (zone *ZoneDb) HasNameRemoteInfo(n string) bool {
+	zone.mx.Lock()
+	defer zone.mx.Unlock()
+
+	remoteNameSet := zone.getNameSet(defaultRemoteIdent)
+	if remoteNameSet != nil {
+		return remoteNameSet.hasName(n)
+	}
+	return false
+}
+
+// Get or create a name set
+func (zone *ZoneDb) getNameSet(ident string) *nameSet {
+	ns, found := zone.idents[ident]
+	if !found {
+		ns = newNameSet()
+		zone.idents[ident] = ns
+	}
+	return ns
+}
+
+// Delete by ident, name and IP. Pass uninitialized value for wildcard
+func (zone *ZoneDb) deleteIdentNameIP(ident string, name string, ip net.IP) int {
+	count := 0
+	if ident != "" {
+		if identV, found := zone.idents[ident]; found {
+			count += identV.deleteNameIP(name, ip)
+			if identV.empty() {
+				delete(zone.idents, ident)
+			}
+		}
+	} else {
+		for identK, identV := range zone.idents {
+			count += identV.deleteNameIP(name, ip)
+			if identV.empty() {
+				delete(zone.idents, identK)
+			}
+		}
+	}
+	return count
 }

nameserver/zone_lookup.go

@@ -0,0 +1,291 @@
+package nameserver
+
+import (
+	"net"
+	"time"
+
+	"github.com/miekg/dns"
+	. "github.com/weaveworks/weave/common"
+)
+
+type uniqZoneRecordKey struct {
+	name string
+	ipv4 IPv4
+}
+
+// A group of ZoneRecords where there are no duplicates (according to the name & IPv4)
+type uniqZoneRecords map[uniqZoneRecordKey]ZoneRecord
+
+func newUniqZoneRecords() uniqZoneRecords {
+	return make(uniqZoneRecords, 0)
+}
+
+// Add a new ZoneRecord to the group
+func (uzr *uniqZoneRecords) add(zr ZoneRecord) {
+	key := uniqZoneRecordKey{zr.Name(), ipToIPv4(zr.IP())}
+	(*uzr)[key] = zr
+}
+
+// Return the group as an slice
+func (uzr *uniqZoneRecords) toSlice() []ZoneRecord {
+	res := make([]ZoneRecord, len(*uzr))
+	i := 0
+	for _, r := range *uzr {
+		res[i] = r
+		i++
+	}
+	return res
+}
+
+//////////////////////////////////////////////////////////////////////////////
+
+// Lookup in the database for locally-introduced information
+func (zone *ZoneDb) lookup(target string, lfun func(ns *nameSet) []*recordEntry) (res []ZoneRecord, err error) {
+	uniq := newUniqZoneRecords()
+	for identName, nameset := range zone.idents {
+		if identName != defaultRemoteIdent {
+			for _, ze := range lfun(nameset) {
+				uniq.add(ze)
+			}
+		}
+	}
+	if len(uniq) == 0 {
+		return nil, LookupError(target)
+	}
+	return uniq.toSlice(), nil
+}
+
+// Perform a lookup for a name in the zone
+// The name can be resolved locally with the local database
+func (zone *ZoneDb) LookupName(name string) (res []ZoneRecord, err error) {
+	zone.mx.RLock()
+	defer zone.mx.RUnlock()
+
+	// note: LookupName() is usually called from the mDNS server, so we do not touch the name
+	name = dns.Fqdn(name)
+	Debug.Printf("[zonedb] Looking for name '%s' in local database", name)
+	return zone.lookup(name, func(ns *nameSet) []*recordEntry { return ns.getEntriesForName(name) })
+}
+
+// Perform a lookup for a IP address in the zone
+// The address can be resolved locally with the local database
+func (zone *ZoneDb) LookupInaddr(inaddr string) (res []ZoneRecord, err error) {
+	zone.mx.RLock()
+	defer zone.mx.RUnlock()
+
+	// note: LookupInaddr() is usually called from the mDNS server, so we do not touch the name
+	revIPv4, err := raddrToIPv4(inaddr)
+	if err != nil {
+		return nil, newParseError("lookup address", inaddr)
+	}
+	Debug.Printf("[zonedb] Looking for address in local database: '%s' (%s)", revIPv4, inaddr)
+	return zone.lookup(inaddr, func(ns *nameSet) []*recordEntry { return ns.getEntriesForIP(revIPv4) })
+}
+
+// Perform a domain lookup with mDNS
+func (zone *ZoneDb) domainLookup(target string, lfun ZoneLookupFunc) (res []ZoneRecord, err error) {
+	// no local results have been obtained in the local database: try with a mDNS query
+	Debug.Printf("[zonedb] '%s' not in local database... trying with mDNS", target)
+	lanswers, err := lfun(target)
+	if err != nil {
+		Debug.Printf("[zonedb] mDNS lookup error for '%s': %s", target, err)
+		return nil, err
+	}
+
+	// if the request has been successful, save the IP in the local database and return the corresponding ZoneRecord
+	// (we do not get the remote ident in the mDNS reply, so we save it in a "remote" ident)
+	Debug.Printf("[zonedb] adding '%s' (obtained with mDNS) to '%s'", lanswers, target)
+	res = make([]ZoneRecord, len(lanswers))
+	zone.mx.Lock()
+	now := zone.clock.Now()
+	uniq := newUniqZoneRecords()
+	remoteIdent := zone.getNameSet(defaultRemoteIdent)
+	for _, answer := range lanswers {
+		r, err := remoteIdent.addIPToName(answer, now)
+		if err != nil {
+			zone.mx.Unlock()
+			Warning.Printf("[zonedb] '%s' insertion for %s failed: %s", answer, target, err)
+			return nil, err
+		}
+		uniq.add(r)
+	}
+	zone.mx.Unlock()
+
+	return uniq.toSlice(), nil
+}
+
+// Perform a lookup for a name in the zone
+// The name can be resolved locally with the local database or with some other resolution method (eg, a mDNS query)
+func (zone *ZoneDb) DomainLookupName(name string) (res []ZoneRecord, err error) {
+	name = dns.Fqdn(name)
+	Debug.Printf("[zonedb] Looking for name '%s' in local(&remote) database", name)
+
+	zone.mx.Lock()
+	now := zone.clock.Now()
+	uniq := newUniqZoneRecords()
+	for identName, nameset := range zone.idents {
+		for _, ze := range nameset.getEntriesForName(name) {
+			// filter the entries with expired TTL
+			// locally introduced entries are never expired: they always have TTL=0
+			if ze.hasExpired(now) {
+				Debug.Printf("[zonedb] '%s': expired entry '%s' ignored: removing", name, ze)
+				nameset.deleteNameIP(name, net.IP{})
+			} else {
+				uniq.add(ze)
+			}
+		}
+		if identName != defaultRemoteIdent {
+			nameset.touchName(name, now)
+		}
+	}
+	zone.mx.Unlock()
+
+	if len(uniq) > 0 {
+		Debug.Printf("[zonedb] '%s' resolved in local database", name)
+		res = uniq.toSlice()
+	} else {
+		res, err = zone.domainLookup(name, zone.mdnsCli.LookupName)
+	}
+
+	if len(res) > 0 {
+		zone.startUpdatingName(name)
+		return res, nil
+	}
+	return nil, LookupError(name)
+}
+
+// Perform a lookup for a IP address in the zone
+// The address can be resolved either with the local database or
+// with some other resolution method (eg, a mDNS query)
+func (zone *ZoneDb) DomainLookupInaddr(inaddr string) (res []ZoneRecord, err error) {
+	revIPv4, err := raddrToIPv4(inaddr)
+	if err != nil {
+		return nil, newParseError("lookup address", inaddr)
+	}
+
+	Debug.Printf("[zonedb] Looking for address in local(&remote) database: '%s' (%s)", revIPv4, inaddr)
+
+	zone.mx.Lock()
+	now := zone.clock.Now()
+	uniq := newUniqZoneRecords()
+	for identName, nameset := range zone.idents {
+		for _, ze := range nameset.getEntriesForIP(revIPv4) {
+			// filter the entries with expired TTL
+			// locally introduced entries are never expired: they always have TTL=0
+			if ze.hasExpired(now) {
+				Debug.Printf("[zonedb] '%s': expired entry '%s' ignored: removing", revIPv4, ze)
+				nameset.deleteNameIP("", revIPv4.toNetIP())
+			} else {
+				uniq.add(ze)
+				if identName != defaultRemoteIdent {
+					nameset.touchName(ze.Name(), now)
+				}
+			}
+		}
+	}
+	zone.mx.Unlock()
+
+	if len(uniq) > 0 {
+		Debug.Printf("[zonedb] '%s' resolved in local database", inaddr)
+		res = uniq.toSlice()
+	} else {
+		res, err = zone.domainLookup(inaddr, zone.mdnsCli.LookupInaddr)
+	}
+
+	if len(res) > 0 {
+		// note: even for reverse addresses, we perform the background updates in the name, not in the IP
+		//       this simplifies the process and produces basically the same results...
+		// note: we do not spend time trying to update names that did not return an initial response...
+		for _, r := range res {
+			zone.startUpdatingName(r.Name())
+		}
+
+		return res, nil
+	}
+	return nil, LookupError(inaddr)
+}
+
+//////////////////////////////////////////////////////////////////////////////
+
+// Names updates try to find all the IPs for a given name with a mDNS query
+//
+// There are two types of names updates:
+//
+// - immediate updates.
+//   After a `DomainLookup*()` for a name not in the database we will return the
+//   first IP we can get with mDNS from other peers. Waiting for more responses would
+//   mean more latency in the response to the client, so we send only one answer BUT
+//   we also trigger an immediate update request for that name in order to get all
+//   the other IPs we didn't wait for...
+//
+// - periodic updates
+//   once we have obtained the first group of IPs for a name, we schedule a periodic
+//   refresh for that name, so we keep the list of IPs for that name up to date.
+//
+// These names updates are repeated until either
+//
+//  a) there is no interest in the name, determined by a global 'relevant time'
+//     and the last time some local client asked about the name,
+//     or
+//  b) no peers answer one of our refresh requests (because the name has probably
+//     disappeared from the network)
+//
+
+// Check if we must start updating a name and, in that case, trigger a immediate update
+func (zone *ZoneDb) startUpdatingName(name string) {
+	if zone.refreshInterval > 0 {
+		zone.mx.Lock()
+
+		// check if we should enqueue a refresh request for this name
+		n := zone.getNameSet(defaultRemoteIdent).getName(name, true)
+		if n.lastRefreshTime.IsZero() {
+			now := zone.clock.Now()
+			n.lastRefreshTime = now
+			zone.mx.Unlock()
+
+			Debug.Printf("[zonedb] Creating new immediate refresh request for '%s'", name)
+			zone.refreshScheds.Add(func() time.Time { return zone.updater(name) }, now)
+		} else {
+			zone.mx.Unlock()
+		}
+	}
+}
+
+// Update the IPs we have for a name
+func (zone *ZoneDb) updater(name string) (nextTime time.Time) {
+	deleteRemoteInfo := func() {
+		zone.mx.Lock()
+		zone.getNameSet(defaultRemoteIdent).deleteNameIP(name, net.IP{})
+		zone.mx.Unlock()
+	}
+
+	// if nobody has asked for this name for long time, just forget about it...
+	if !zone.IsNameRelevant(name) || zone.IsNameExpired(name) {
+		Debug.Printf("[zonedb] '%s' seem to be irrelevant now: removing any remote information", name)
+		deleteRemoteInfo()
+		return
+	}
+
+	// perform the refresh for this name
+	fullName := dns.Fqdn(name)
+	startTime := zone.clock.Now()
+	Debug.Printf("[zonedb] Refreshing name '%s' with mDNS...", fullName)
+	res, _ := zone.mdnsCli.InsistentLookupName(fullName)
+	if res != nil && len(res) > 0 {
+		numIps := len(res)
+		zone.mx.Lock()
+		now := zone.clock.Now()
+		added, removed := zone.getNameSet(defaultRemoteIdent).getName(name, true).updateIPs(res, now)
+		zone.mx.Unlock()
+		Debug.Printf("[zonedb] Obtained %d IPs for name '%s' with mDNS: %d added, %d removed",
+			numIps, name, added, removed)
+
+		// once the name has been updated, we re-schedule the update
+		nextTime = startTime.Add(zone.refreshInterval)
+		Debug.Printf("[zonedb] Rescheduling update for '%s' in %s", name, nextTime.Sub(zone.clock.Now()))
+	} else {
+		Debug.Printf("[zonedb] nobody knows about '%s'... removing", name)
+		deleteRemoteInfo()
+	}
+	return
+}

nameserver/zone_lookup_test.go

@@ -0,0 +1,116 @@
+package nameserver
+
+import (
+	"net"
+	"testing"
+	"time"
+
+	. "github.com/weaveworks/weave/common"
+	wt "github.com/weaveworks/weave/testing"
+)
+
+// Check that the refreshing mechanism works as expected
+func TestZoneRefresh(t *testing.T) {
+	const (
+		refreshInterval = 5
+		relevantTime    = 30
+	)
+	var (
+		name  = "mysql.weave.local."
+		addr1 = "10.2.8.4"
+		addr2 = "10.2.8.5"
+		addr3 = "10.2.8.6"
+		addr4 = "10.2.8.7"
+	)
+
+	InitDefaultLogging(testing.Verbose())
+	Info.Println("TestZoneRefresh starting")
+
+	clk := newMockedClock()
+
+	// Create multiple zone databases, linked through mocked mDNS connections
+	zc := ZoneConfig{
+		RefreshInterval: refreshInterval,
+		RelevantTime:    relevantTime,
+		Clock:           clk,
+	}
+	dbs := newZoneDbsWithMockedMDns(2, zc)
+	dbs.Start()
+	defer dbs.Stop()
+
+	time.Sleep(100 * time.Millisecond) // allow for server to get going
+
+	Debug.Printf("Adding '%s' to Db #1", name)
+	dbs[0].Zone.AddRecord("someident", name, net.ParseIP(addr1))
+
+	Debug.Printf("Checking that the name %s is relevant (as it has been locally inserted) and not remote", name)
+	wt.AssertTrue(t, dbs[0].Zone.IsNameRelevant(name), "name relevant")
+	wt.AssertTrue(t, dbs[0].Zone.HasNameLocalInfo(name), "local name info")
+	wt.AssertFalse(t, dbs[0].Zone.HasNameRemoteInfo(name), "remote name info")
+
+	Debug.Printf("Asking for '%s' to Db #1: should get 1 IP from the local database...", name)
+	res, err := dbs[0].Zone.DomainLookupName(name)
+	wt.AssertNoErr(t, err)
+	Debug.Printf("Got: %s", res)
+	t.Logf("Db #1 after the lookup:\n%s", dbs[0].Zone)
+	wt.AssertEqualInt(t, len(res), 1, "lookup result")
+
+	clk.Forward(refreshInterval / 2)
+	dbs.Flush()
+	Debug.Printf("A couple of seconds later, we should still have one IP for that name")
+	res, err = dbs[0].Zone.DomainLookupName(name)
+	wt.AssertNoErr(t, err)
+	wt.AssertEqualInt(t, len(res), 1, "lookup result")
+
+	Debug.Printf("And then we add 2 IPs for that name at ZoneDb 2")
+	clk.Forward(1)
+	dbs.Flush()
+	Debug.Printf("Adding 2 IPs to '%s' in Db #2", name)
+	dbs[1].Zone.AddRecord("someident", name, net.ParseIP(addr2))
+	dbs[1].Zone.AddRecord("someident", name, net.ParseIP(addr3))
+
+	Debug.Printf("Perform a lookup, to ensure the name will be updated in the background...")
+	dbs[0].Zone.DomainLookupName(name)
+
+	Debug.Printf("Wait for a while, until a refresh is performed...")
+	clk.Forward(refreshInterval + 1)
+	dbs.Flush()
+	Debug.Printf("A refresh should have been scheduled now: we should have 3 IPs:")
+	Debug.Printf("the first (local) IP and the others obtained from zone2 with a mDNS query")
+	Debug.Printf("Asking for '%s' again... we should have 3 IPs now", name)
+	res, err = dbs[0].Zone.DomainLookupName(name)
+	Debug.Printf("Got: %s", res)
+	t.Logf("Db #1 after the second lookup:\n%s", dbs[0].Zone)
+	wt.AssertEqualInt(t, len(res), 3, "lookup result length")
+
+	Debug.Printf("We will not ask for `name` for a while, so it will become irrelevant and will be removed...")
+	clk.Forward(refreshInterval + relevantTime + 1)
+	dbs.Flush()
+
+	// the name should be irrelevant now, and all remote info should have been
+	// removed from the zone database
+	Debug.Printf("Name '%s' should not be in the remote database in ZoneDb 1", name)
+	wt.AssertFalse(t, dbs[0].Zone.IsNameRelevant(name), "name still relevant after some inactivity time")
+	wt.AssertTrue(t, dbs[0].Zone.HasNameLocalInfo(name), "local name info")
+	wt.AssertFalse(t, dbs[0].Zone.HasNameRemoteInfo(name), "remote name info")
+
+	Debug.Printf("There is no remote info about this name at zone 1: a new IP appears remotely meanwhile...")
+	clk.Forward(1)
+	dbs.Flush()
+
+	Debug.Printf("Adding '%s' to Db #2", name)
+	dbs[1].Zone.AddRecord("someident", name, net.ParseIP(addr4))
+
+	Debug.Printf("When we ask about this name again, we get 4 IPs (1 local, 3 remote)")
+	Debug.Printf("Asking for '%s' again... the first lookup will return only the local results", name)
+	res, err = dbs[0].Zone.DomainLookupName(name)
+	wt.AssertEqualInt(t, len(res), 1, "lookup result length")
+	Debug.Printf("... but a second lookup should return all the results in the network")
+
+	clk.Forward(refreshInterval + 1)
+	dbs.Flush()
+
+	res, err = dbs[0].Zone.DomainLookupName(name)
+	Debug.Printf("Got: %s", res)
+	wt.AssertEqualInt(t, len(res), 4, "lookup result length")
+}

nameserver/zone_test.go

@@ -1,88 +1,159 @@
 package nameserver
 
 import (
-	wt "github.com/weaveworks/weave/testing"
+	"fmt"
 	"net"
 	"testing"
+
+	. "github.com/weaveworks/weave/common"
+	wt "github.com/weaveworks/weave/testing"
 )
 
 func TestZone(t *testing.T) {
 	var (
-		containerID      = "deadbeef"
-		otherContainerID = "cowjuice"
-		successTestName  = "test1.weave."
-		testAddr1        = "10.2.2.1/24"
+		container1    = "deadbeef"
+		container2    = "cowjuice"
+		name1         = "test1.weave."
+		name1Addr1    = "10.9.2.1/24"
+		name1Addr2    = "10.9.2.2/24"
+		revName1Addr1 = "1.2.9.10.in-addr.arpa."
 	)
 
-	var zone = NewZoneDb(DefaultLocalDomain)
+	InitDefaultLogging(testing.Verbose())
 
-	ip, _, _ := net.ParseCIDR(testAddr1)
-	err := zone.AddRecord(containerID, successTestName, ip)
+	zone, err := NewZoneDb(ZoneConfig{})
+	wt.AssertNoErr(t, err)
+	err = zone.Start()
+	wt.AssertNoErr(t, err)
+	defer zone.Stop()
+
+	ip1, _, _ := net.ParseCIDR(name1Addr1)
+	t.Logf("Adding '%s'/%s to '%s'", name1, ip1, container1)
+	err = zone.AddRecord(container1, name1, ip1)
 	wt.AssertNoErr(t, err)
 
 	// Add a few more records to make the job harder
-	err = zone.AddRecord("abcdef0123", "adummy.weave.", net.ParseIP("10.2.0.1"))
+	t.Logf("Adding 'adummy.weave.' to 'abcdef0123'")
+	err = zone.AddRecord("abcdef0123", "adummy.weave.", net.ParseIP("10.9.0.1"))
 	wt.AssertNoErr(t, err)
-	err = zone.AddRecord("0123abcdef", "zdummy.weave.", net.ParseIP("10.2.0.2"))
+	t.Logf("Adding 'zdummy.weave.' to '0123abcdef'")
+	err = zone.AddRecord("0123abcdef", "zdummy.weave.", net.ParseIP("10.9.0.2"))
+	wt.AssertNoErr(t, err)
+	t.Logf("Zone database:\n%s", zone)
+
+	t.Logf("Checking if we can find the name '%s'", name1)
+	foundIPs, err := zone.LookupName(name1)
 	wt.AssertNoErr(t, err)
 
-	// Check that the address is now there.
-	foundIP, err := zone.LookupName(successTestName)
-	wt.AssertNoErr(t, err)
-
-	if !foundIP[0].IP().Equal(ip) {
-		t.Fatal("Unexpected result for", successTestName, foundIP)
+	if !foundIPs[0].IP().Equal(ip1) {
+		t.Fatal("Unexpected result for", name1, foundIPs)
 	}
 
-	// See if we can find the address by IP.
-	foundName, err := zone.LookupInaddr("1.2.2.10.in-addr.arpa.")
+	t.Logf("Checking if we cannot find some silly name like 'something.wrong'")
+	foundIPs, err = zone.LookupName("something.wrong")
+	wt.AssertErrorType(t, err, (*LookupError)(nil), fmt.Sprintf("unknown name: %+v", foundIPs))
+
+	ip2, _, _ := net.ParseCIDR(name1Addr2)
+	t.Logf("Adding a second IP for '%s'/%s to '%s'", name1, ip2, container1)
+	err = zone.AddRecord(container1, name1, ip2)
 	wt.AssertNoErr(t, err)
 
-	if foundName[0].Name() != successTestName {
-		t.Fatal("Unexpected result for", ip, foundName)
+	t.Logf("Checking if we can find both the old IP and the new IP for '%s'", name1)
+	foundIPs, err = zone.LookupName(name1)
+	wt.AssertNoErr(t, err)
+	if !(foundIPs[0].IP().Equal(ip1) || foundIPs[1].IP().Equal(ip1)) {
+		t.Fatal("Unexpected result for", name1, foundIPs)
+	}
+	if !(foundIPs[0].IP().Equal(ip2) || foundIPs[1].IP().Equal(ip2)) {
+		t.Fatal("Unexpected result for", name1, foundIPs)
 	}
 
-	err = zone.AddRecord(containerID, successTestName, ip)
+	t.Logf("Checking if we can find the address by IP '1.2.9.10.in-addr.arpa.'")
+	foundNames, err := zone.LookupInaddr("1.2.9.10.in-addr.arpa.")
+	wt.AssertNoErr(t, err)
+
+	if foundNames[0].Name() != name1 {
+		t.Fatal("Unexpected result for", ip1, foundNames)
+	}
+
+	t.Logf("Checking we can not find an unknown address '30.20.10.1.in-addr.arpa.'")
+	foundNames, err = zone.LookupInaddr("30.20.10.1.in-addr.arpa.")
+	wt.AssertErrorType(t, err, (*LookupError)(nil), fmt.Sprintf("unknown IP: %+v", foundNames))
+
+	t.Logf("Checking if adding again '%s'/%s in %s results in an error", name1, ip1, container1)
+	err = zone.AddRecord(container1, name1, ip1)
 	wt.AssertErrorType(t, err, (*DuplicateError)(nil), "duplicate add")
 
-	err = zone.AddRecord(otherContainerID, successTestName, ip)
-	// Delete the record for the original container
-	err = zone.DeleteRecord(containerID, ip)
+	t.Logf("Adding '%s'/%s in %s too", name1, ip1, container2)
+	err = zone.AddRecord(container2, name1, ip1)
 	wt.AssertNoErr(t, err)
 
-	_, err = zone.LookupName(successTestName)
+	name1Removed := 0
+	err = zone.ObserveName(name1, func() { t.Logf("Observer #1 for '%s' notified.", name1); name1Removed++ })
+	wt.AssertNoErr(t, err)
+	err = zone.ObserveName(name1, func() { t.Logf("Observer #2 for '%s' notified.", name1); name1Removed++ })
+	wt.AssertNoErr(t, err)
+	err = zone.ObserveInaddr(revName1Addr1, func() { t.Logf("Observer #1 for '%s' notified.", revName1Addr1); name1Removed++ })
 	wt.AssertNoErr(t, err)
 
-	err = zone.DeleteRecord(otherContainerID, ip)
+	t.Logf("Zone database:\n%s", zone)
+	t.Logf("Deleting the %s in %s", ip1, container1)
+	count := zone.DeleteRecords(container1, "", ip1)
+	wt.AssertEqualInt(t, count, 1, "delete failed")
+	t.Logf("Zone database:\n%s", zone)
+
+	t.Logf("Checking %s's observers have been notified on removal", name1)
+	if name1Removed < 3 {
+		t.Logf("Zone database:\n%s", zone)
+		t.Fatalf("Unexpected number (%d) of calls to observers", name1Removed)
+	}
+
+	t.Logf("Checking %s can be found", name1)
+	_, err = zone.LookupName(name1)
 	wt.AssertNoErr(t, err)
 
-	// Check that the address is not there now.
-	_, err = zone.LookupName(successTestName)
+	t.Logf("Checking %s is not found after removing %s it in %s and %s in %s",
+		name1, ip1, container2, ip2, container1)
+	count = zone.DeleteRecords(container1, "", ip2)
+	wt.AssertEqualInt(t, count, 1, "delete failed")
+	count = zone.DeleteRecords(container2, "", ip1)
+	wt.AssertEqualInt(t, count, 1, "delete failed")
+	t.Logf("Zone database:\n%s", zone)
+	_, err = zone.LookupName(name1)
 	wt.AssertErrorType(t, err, (*LookupError)(nil), "after deleting record")
 
-	// Delete a record that isn't there
-	err = zone.DeleteRecord(containerID, net.ParseIP("0.0.0.0"))
-	wt.AssertErrorType(t, err, (*LookupError)(nil), "when deleting record that doesn't exist")
+	t.Logf("Checking if removing an unknown record results in an error")
+	count = zone.DeleteRecords(container1, "", net.ParseIP("0.0.0.0"))
+	wt.AssertEqualInt(t, count, 0, "delete failed")
 }
 
-func TestDeleteFor(t *testing.T) {
+func TestDeleteRecords(t *testing.T) {
 	var (
 		id    = "foobar"
 		name  = "foo.weave."
 		addr1 = "10.2.2.3/24"
 		addr2 = "10.2.7.8/24"
 	)
-	zone := NewZoneDb(DefaultLocalDomain)
+
+	InitDefaultLogging(testing.Verbose())
+
+	zone, err := NewZoneDb(ZoneConfig{})
+	wt.AssertNoErr(t, err)
+	err = zone.Start()
+	wt.AssertNoErr(t, err)
+	defer zone.Stop()
+
 	for _, addr := range []string{addr1, addr2} {
 		ip, _, _ := net.ParseCIDR(addr)
 		err := zone.AddRecord(id, name, ip)
 		wt.AssertNoErr(t, err)
 	}
 
-	_, err := zone.LookupName(name)
+	_, err = zone.LookupName(name)
 	wt.AssertNoErr(t, err)
 
-	err = zone.DeleteRecordsFor(id)
+	count := zone.DeleteRecords(id, "", net.IP{})
+	wt.AssertEqualInt(t, count, 2, "wildcard delete failed")
 	_, err = zone.LookupName(name)
 	wt.AssertErrorType(t, err, (*LookupError)(nil), "after deleting records for ident")
 }

net/if.go

@@ -6,12 +6,16 @@ import (
 	"time"
 )
 
+const (
+	sleepTime = 100 * time.Millisecond
+)
+
 func EnsureInterface(ifaceName string, wait int) (iface *net.Interface, err error) {
 	if iface, err = findInterface(ifaceName); err == nil || wait == 0 {
 		return
 	}
-	for ; err != nil && wait > 0; wait-- {
-		time.Sleep(1 * time.Second)
+	for i := time.Duration(wait) * time.Second / sleepTime; err != nil && i > 0; i-- {
+		time.Sleep(sleepTime)
 		iface, err = findInterface(ifaceName)
 	}
 	return

net/route.go

@@ -0,0 +1,28 @@
+package net
+
+import (
+	"fmt"
+	"net"
+
+	"github.com/vishvananda/netlink"
+)
+
+// A network is considered free if it does not overlap any existing
+// routes on this host. This is the same approach taken by Docker.
+func CheckNetworkFree(subnet *net.IPNet) error {
+	routes, err := netlink.RouteList(nil, netlink.FAMILY_V4)
+	if err != nil {
+		return err
+	}
+	for _, route := range routes {
+		if route.Dst != nil && overlaps(route.Dst, subnet) {
+			return fmt.Errorf("network %s would overlap with route %s", subnet, route.Dst)
+		}
+	}
+	return nil
+}
+
+// Two networks overlap if the start-point of one is inside the other.
+func overlaps(n1, n2 *net.IPNet) bool {
+	return n1.Contains(n2.IP) || n2.Contains(n1.IP)
+}

net/route_test.go

@@ -0,0 +1,25 @@
+package net
+
+import (
+	"net"
+	"testing"
+
+	wt "github.com/weaveworks/weave/testing"
+)
+
+func TestOverlap(t *testing.T) {
+	_, subnet1, _ := net.ParseCIDR("10.0.3.0/24")
+	_, subnet2, _ := net.ParseCIDR("10.0.4.0/24")
+	_, subnet3, _ := net.ParseCIDR("10.0.3.128/25")
+	_, subnet4, _ := net.ParseCIDR("10.0.3.192/25")
+	_, universe, _ := net.ParseCIDR("10.0.0.0/8")
+	wt.AssertEquals(t, overlaps(subnet1, subnet2), false)
+	wt.AssertEquals(t, overlaps(subnet2, subnet1), false)
+	wt.AssertEquals(t, overlaps(subnet1, subnet1), true)
+	wt.AssertEquals(t, overlaps(subnet1, subnet3), true)
+	wt.AssertEquals(t, overlaps(subnet1, subnet4), true)
+	wt.AssertEquals(t, overlaps(subnet2, subnet4), false)
+	wt.AssertEquals(t, overlaps(subnet4, subnet2), false)
+	wt.AssertEquals(t, overlaps(universe, subnet1), true)
+	wt.AssertEquals(t, overlaps(subnet1, universe), true)
+}

prog/netcheck/netcheck.go

@@ -0,0 +1,31 @@
+/* netcheck: check whether a given network overlaps with any existing routes */
+package main
+
+import (
+	"fmt"
+	"net"
+	"os"
+
+	weavenet "github.com/weaveworks/weave/net"
+)
+
+func fatal(err error) {
+	fmt.Println(err)
+	os.Exit(1)
+}
+
+func main() {
+	if len(os.Args) <= 1 {
+		os.Exit(0)
+	}
+
+	ipRangeStr := os.Args[1]
+	_, ipnet, err := net.ParseCIDR(ipRangeStr)
+	if err != nil {
+		fatal(err)
+	}
+	if err := weavenet.CheckNetworkFree(ipnet); err != nil {
+		fatal(err)
+	}
+	os.Exit(0)
+}

prog/weavedns/main.go

@@ -0,0 +1,186 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"net"
+	"os"
+	"strconv"
+
+	"github.com/miekg/dns"
+	. "github.com/weaveworks/weave/common"
+	"github.com/weaveworks/weave/common/docker"
+	weavedns "github.com/weaveworks/weave/nameserver"
+	weavenet "github.com/weaveworks/weave/net"
+)
+
+var version = "(unreleased version)"
+
+func main() {
+	var (
+		justVersion     bool
+		dockerCli       *docker.Client
+		ifaceName       string
+		apiPath         string
+		domain          string
+		dnsPort         int
+		httpIfaceName   string
+		httpPort        int
+		wait            int
+		ttl             int
+		negTTL          int
+		timeout         int
+		udpbuf          int
+		fallback        string
+		refreshInterval int
+		relevantTime    int
+		maxAnswers      int
+		cacheLen        int
+		cacheDisabled   bool
+		watch           bool
+		debug           bool
+		err             error
+	)
+
+	flag.BoolVar(&justVersion, "version", false, "print version and exit")
+	flag.StringVar(&ifaceName, "iface", "", "name of interface to use for multicast")
+	flag.StringVar(&apiPath, "api", "unix:///var/run/docker.sock", "path to Docker API socket")
+	flag.StringVar(&domain, "domain", weavedns.DefaultLocalDomain, "local domain (ie, 'weave.local.')")
+	flag.IntVar(&wait, "wait", 0, "number of seconds to wait for interface to be created and come up")
+	flag.IntVar(&dnsPort, "dnsport", weavedns.DefaultServerPort, "port to listen to DNS requests")
+	flag.StringVar(&httpIfaceName, "httpiface", "", "interface on which to listen for HTTP requests (defaults to empty string which listens on all interfaces)")
+	flag.IntVar(&httpPort, "httpport", weavedns.DefaultHTTPPort, "port to listen to HTTP requests")
+	flag.IntVar(&cacheLen, "cache", weavedns.DefaultCacheLen, "cache length")
+	flag.IntVar(&ttl, "ttl", weavedns.DefaultLocalTTL, "TTL (in secs) for responses for local names")
+	flag.BoolVar(&watch, "watch", true, "watch the docker socket for container events")
+	flag.BoolVar(&debug, "debug", false, "output debugging info to stderr")
+	// advanced options
+	flag.IntVar(&negTTL, "neg-ttl", 0, "negative TTL (in secs) for unanswered queries for local names (0=same value as -ttl")
+	flag.IntVar(&refreshInterval, "refresh", weavedns.DefaultRefreshInterval, "refresh interval (in secs) for local names (0=disable)")
+	flag.IntVar(&maxAnswers, "max-answers", weavedns.DefaultMaxAnswers, "maximum number of answers returned to clients (0=unlimited)")
+	flag.IntVar(&relevantTime, "relevant", weavedns.DefaultRelevantTime, "life time for info in the absence of queries (in secs)")
+	flag.IntVar(&udpbuf, "udpbuf", weavedns.DefaultUDPBuflen, "UDP buffer length")
+	flag.IntVar(&timeout, "timeout", weavedns.DefaultTimeout, "timeout for resolutions (in millisecs)")
+	flag.BoolVar(&cacheDisabled, "no-cache", false, "disable the cache")
+	flag.StringVar(&fallback, "fallback", "", "force a fallback server (ie, '8.8.8.8:53') (instead of /etc/resolv.conf values)")
+	flag.Parse()
+
+	if justVersion {
+		fmt.Printf("weave DNS %s\n", version)
+		os.Exit(0)
+	}
+
+	InitDefaultLogging(debug)
+	Info.Printf("[main] WeaveDNS version %s\n", version) // first thing in log: the version
+
+	var iface *net.Interface
+	if ifaceName != "" {
+		var err error
+		Info.Println("[main] Waiting for mDNS interface", ifaceName, "to come up")
+		iface, err = weavenet.EnsureInterface(ifaceName, wait)
+		if err != nil {
+			Error.Fatal(err)
+		} else {
+			Info.Println("[main] Interface", ifaceName, "is up")
+		}
+	}
+
+	var httpIP string
+	if httpIfaceName == "" {
+		httpIP = "0.0.0.0"
+	} else {
+		Info.Println("[main] Waiting for HTTP interface", httpIfaceName, "to come up")
+		httpIface, err := weavenet.EnsureInterface(httpIfaceName, wait)
+		if err != nil {
+			Error.Fatal(err)
+		}
+		Info.Println("[main] Interface", httpIfaceName, "is up")
+
+		addrs, err := httpIface.Addrs()
+		if err != nil {
+			Error.Fatal(err)
+		}
+
+		if len(addrs) == 0 {
+			Error.Fatal("[main] No addresses on HTTP interface")
+		}
+
+		ip, _, err := net.ParseCIDR(addrs[0].String())
+		if err != nil {
+			Error.Fatal(err)
+		}
+
+		httpIP = ip.String()
+
+	}
+	httpAddr := net.JoinHostPort(httpIP, strconv.Itoa(httpPort))
+
+	zoneConfig := weavedns.ZoneConfig{
+		Domain:          domain,
+		Iface:           iface,
+		LocalTTL:        ttl,
+		RefreshInterval: refreshInterval,
+		RelevantTime:    relevantTime,
+	}
+	zone, err := weavedns.NewZoneDb(zoneConfig)
+	if err != nil {
+		Error.Fatal("[main] Unable to initialize the Zone database", err)
+	}
+	if err := zone.Start(); err != nil {
+		Error.Fatal("[main] Unable to start the Zone database", err)
+	}
+	defer zone.Stop()
+
+	dockerCli, err = docker.NewClient(apiPath)
+	if err != nil {
+		Error.Fatal("[main] Unable to start docker client: ", err)
+	}
+
+	if watch {
+		err := dockerCli.AddObserver(zone)
+		if err != nil {
+			Error.Fatal("[main] Unable to start watcher", err)
+		}
+	}
+
+	srvConfig := weavedns.DNSServerConfig{
+		Zone:             zone,
+		Port:             dnsPort,
+		CacheLen:         cacheLen,
+		LocalTTL:         ttl,
+		CacheNegLocalTTL: negTTL,
+		MaxAnswers:       maxAnswers,
+		Timeout:          timeout,
+		UDPBufLen:        udpbuf,
+		CacheDisabled:    cacheDisabled,
+	}
+
+	if len(fallback) > 0 {
+		fallbackHost, fallbackPort, err := net.SplitHostPort(fallback)
+		if err != nil {
+			Error.Fatal("[main] Could not parse fallback host and port", err)
+		}
+		srvConfig.UpstreamCfg = &dns.ClientConfig{Servers: []string{fallbackHost}, Port: fallbackPort}
+		Debug.Printf("[main] DNS fallback at %s:%s", fallbackHost, fallbackPort)
+	}
+
+	srv, err := weavedns.NewDNSServer(srvConfig)
+	if err != nil {
+		Error.Fatal("[main] Failed to initialize the WeaveDNS server", err)
+	}
+
+	httpListener, err := net.Listen("tcp", httpAddr)
+	if err != nil {
+		Error.Fatal("[main] Unable to create http listener: ", err)
+	}
+	Info.Println("[main] HTTP API listening on", httpAddr)
+
+	go SignalHandlerLoop(srv)
+	go weavedns.ServeHTTP(httpListener, version, srv, dockerCli)
+
+	err = srv.Start()
+	if err != nil {
+		Error.Fatal("[main] Failed to start the WeaveDNS server: ", err)
+	}
+	srv.ActivateAndServe()
+}

prog/weaveexec/Dockerfile

@@ -16,9 +16,10 @@ WORKDIR /home/weave
 ADD ./weave /home/weave/
 ADD ./sigproxy /home/weave/
 ADD ./weaveproxy /home/weave/
-ADD ./weavewait /home/weavewait/weavewait
+ADD ./netcheck /usr/bin/
+ADD ./weavewait /w/w
 ADD ./docker.tgz /
 
-VOLUME /home/weavewait
+VOLUME /w
 
 ENTRYPOINT ["/home/weave/sigproxy", "/home/weave/weave"]

prog/weaveproxy/main.go

@@ -0,0 +1,78 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"code.google.com/p/getopt"
+	. "github.com/weaveworks/weave/common"
+	"github.com/weaveworks/weave/proxy"
+)
+
+var (
+	version           = "(unreleased version)"
+	defaultDockerAddr = "unix:///var/run/docker.sock"
+	defaultListenAddr = "tcp://0.0.0.0:12375"
+)
+
+func main() {
+	var (
+		debug       bool
+		justVersion bool
+		c           = proxy.Config{
+			DockerAddr: defaultDockerAddr,
+			ListenAddr: defaultListenAddr,
+		}
+	)
+
+	c.Version = version
+	getopt.BoolVarLong(&debug, "debug", 'd', "log debugging information")
+	getopt.BoolVarLong(&justVersion, "version", 0, "print version and exit")
+	getopt.StringVar(&c.ListenAddr, 'H', fmt.Sprintf("address on which to listen (default %s)", defaultListenAddr))
+	getopt.StringVar(&c.DockerAddr, 'D', fmt.Sprintf("docker daemon URL to proxy (default %s)", defaultDockerAddr))
+	getopt.BoolVarLong(&c.NoDefaultIPAM, "no-default-ipam", 0, "do not automatically allocate addresses for containers without a WEAVE_CIDR")
+	getopt.StringVarLong(&c.TLSConfig.CACert, "tlscacert", 0, "Trust certs signed only by this CA")
+	getopt.StringVarLong(&c.TLSConfig.Cert, "tlscert", 0, "Path to TLS certificate file")
+	getopt.BoolVarLong(&c.TLSConfig.Enabled, "tls", 0, "Use TLS; implied by --tlsverify")
+	getopt.StringVarLong(&c.TLSConfig.Key, "tlskey", 0, "Path to TLS key file")
+	getopt.BoolVarLong(&c.TLSConfig.Verify, "tlsverify", 0, "Use TLS and verify the remote")
+	getopt.BoolVarLong(&c.WithDNS, "with-dns", 'w', "instruct created containers to always use weaveDNS as their nameserver")
+	getopt.BoolVarLong(&c.WithoutDNS, "without-dns", 0, "instruct created containers to never use weaveDNS as their nameserver")
+	getopt.Parse()
+
+	if justVersion {
+		fmt.Printf("weave proxy %s\n", version)
+		os.Exit(0)
+	}
+
+	if c.WithDNS && c.WithoutDNS {
+		Error.Fatalf("Cannot use both '--with-dns' and '--without-dns' flags")
+	}
+
+	if debug {
+		InitDefaultLogging(true)
+	}
+
+	Info.Println("weave proxy", version)
+	Info.Println("Command line arguments:", strings.Join(os.Args[1:], " "))
+
+	protoAddrParts := strings.SplitN(c.ListenAddr, "://", 2)
+	if len(protoAddrParts) == 2 {
+		if protoAddrParts[0] != "tcp" {
+			Error.Fatalf("Invalid protocol format: %q", protoAddrParts[0])
+		}
+		c.ListenAddr = protoAddrParts[1]
+	} else {
+		c.ListenAddr = protoAddrParts[0]
+	}
+
+	p, err := proxy.NewProxy(c)
+	if err != nil {
+		Error.Fatalf("Could not start proxy: %s", err)
+	}
+
+	if err := p.ListenAndServe(); err != nil {
+		Error.Fatalf("Could not listen on %s: %s", p.ListenAddr, err)
+	}
+}

prog/weaver/main.go

@@ -1,23 +1,25 @@
 package main
 
 import (
-	"code.google.com/p/gopacket/layers"
 	"crypto/sha256"
 	"flag"
 	"fmt"
-	"github.com/davecheney/profile"
-	"github.com/gorilla/mux"
-	. "github.com/weaveworks/weave/common"
-	"github.com/weaveworks/weave/common/updater"
-	"github.com/weaveworks/weave/ipam"
-	weavenet "github.com/weaveworks/weave/net"
-	weave "github.com/weaveworks/weave/router"
 	"log"
 	"net"
 	"net/http"
 	"os"
 	"runtime"
 	"strings"
+
+	"code.google.com/p/gopacket/layers"
+	"github.com/davecheney/profile"
+	"github.com/gorilla/mux"
+	. "github.com/weaveworks/weave/common"
+	"github.com/weaveworks/weave/common/docker"
+	"github.com/weaveworks/weave/ipam"
+	"github.com/weaveworks/weave/ipam/address"
+	weavenet "github.com/weaveworks/weave/net"
+	weave "github.com/weaveworks/weave/router"
 )
 
 var version = "(unreleased version)"
@@ -36,22 +38,24 @@ func main() {
 	runtime.GOMAXPROCS(procs)
 
 	var (
-		config      weave.RouterConfig
-		justVersion bool
-		ifaceName   string
-		routerName  string
-		nickName    string
-		password    string
-		wait        int
-		debug       bool
-		pktdebug    bool
-		prof        string
-		peers       []string
-		bufSzMB     int
-		httpAddr    string
-		iprangeCIDR string
-		peerCount   int
-		apiPath     string
+		config       weave.Config
+		justVersion  bool
+		ifaceName    string
+		routerName   string
+		nickName     string
+		password     string
+		wait         int
+		debug        bool
+		pktdebug     bool
+		prof         string
+		bufSzMB      int
+		noDiscovery  bool
+		httpAddr     string
+		iprangeCIDR  string
+		ipsubnetCIDR string
+		peerCount    int
+		apiPath      string
+		peers        []string
 	)
 
 	flag.BoolVar(&justVersion, "version", false, "print version and exit")
@@ -65,9 +69,11 @@ func main() {
 	flag.BoolVar(&pktdebug, "pktdebug", false, "enable per-packet debug logging")
 	flag.StringVar(&prof, "profile", "", "enable profiling and write profiles to given path")
 	flag.IntVar(&config.ConnLimit, "connlimit", 30, "connection limit (0 for unlimited)")
+	flag.BoolVar(&noDiscovery, "nodiscovery", false, "disable peer discovery")
 	flag.IntVar(&bufSzMB, "bufsz", 8, "capture buffer size in MB")
 	flag.StringVar(&httpAddr, "httpaddr", fmt.Sprintf(":%d", weave.HTTPPort), "address to bind HTTP interface to (disabled if blank, absolute path indicates unix domain socket)")
-	flag.StringVar(&iprangeCIDR, "iprange", "", "IP address range to allocate within, in CIDR notation")
+	flag.StringVar(&iprangeCIDR, "iprange", "", "IP address range reserved for automatic allocation, in CIDR notation")
+	flag.StringVar(&ipsubnetCIDR, "ipsubnet", "", "subnet to allocate within by default, in CIDR notation")
 	flag.IntVar(&peerCount, "initpeercount", 0, "number of peers in network (for IP address allocation)")
 	flag.StringVar(&apiPath, "api", "unix:///var/run/docker.sock", "Path to Docker API socket")
 	flag.Parse()
@@ -112,7 +118,6 @@ func main() {
 	if password == "" {
 		password = os.Getenv("WEAVE_PASSWORD")
 	}
-
 	if password == "" {
 		log.Println("Communication between peers is unencrypted.")
 	} else {
@@ -129,32 +134,50 @@ func main() {
 
 	config.BufSz = bufSzMB * 1024 * 1024
 	config.LogFrame = logFrameFunc(pktdebug)
+	config.PeerDiscovery = !noDiscovery
 
 	router := weave.NewRouter(config, name, nickName)
 	log.Println("Our name is", router.Ourself)
 
 	var allocator *ipam.Allocator
+	var defaultSubnet address.CIDR
+	var dockerCli *docker.Client
 	if iprangeCIDR != "" {
-		allocator = createAllocator(router, apiPath, iprangeCIDR, determineQuorum(peerCount, peers))
+		allocator, defaultSubnet = createAllocator(router, iprangeCIDR, ipsubnetCIDR, determineQuorum(peerCount, peers))
+		dockerCli, err = docker.NewClient(apiPath)
+		if err != nil {
+			Error.Fatal("Unable to start docker client: ", err)
+		}
+		if err = dockerCli.AddObserver(allocator); err != nil {
+			Error.Fatal("Unable to start watcher", err)
+		}
 	} else if peerCount > 0 {
 		log.Fatal("-initpeercount flag specified without -iprange")
-	} else {
-		router.NewGossip("IPallocation", &ipam.DummyAllocator{})
 	}
 
 	router.Start()
-	initiateConnections(router, peers)
+	if errors := router.ConnectionMaker.InitiateConnections(peers, false); len(errors) > 0 {
+		log.Fatal(errorMessages(errors))
+	}
 
 	// The weave script always waits for a status call to succeed,
 	// so there is no point in doing "weave launch -httpaddr ''".
 	// This is here to support stand-alone use of weaver.
 	if httpAddr != "" {
-		go handleHTTP(router, httpAddr, allocator)
+		go handleHTTP(router, httpAddr, allocator, defaultSubnet, dockerCli)
 	}
 
 	SignalHandlerLoop(router)
 }
 
+func errorMessages(errors []error) string {
+	var result []string
+	for _, err := range errors {
+		result = append(result, err.Error())
+	}
+	return strings.Join(result, "\n")
+}
+
 func options() map[string]string {
 	options := make(map[string]string)
 	flag.Visit(func(f *flag.Flag) {
@@ -181,26 +204,32 @@ func logFrameFunc(debug bool) weave.LogFrameFunc {
 	}
 }
 
-func initiateConnections(router *weave.Router, peers []string) {
-	for _, peer := range peers {
-		if err := router.ConnectionMaker.InitiateConnection(peer); err != nil {
-			log.Fatal(err)
-		}
-	}
-}
-
-func createAllocator(router *weave.Router, apiPath string, iprangeCIDR string, quorum uint) *ipam.Allocator {
-	allocator, err := ipam.NewAllocator(router.Ourself.Peer.Name, router.Ourself.Peer.UID, router.Ourself.Peer.NickName, iprangeCIDR, quorum)
+func parseAndCheckCIDR(cidrStr string) address.CIDR {
+	_, cidr, err := address.ParseCIDR(cidrStr)
 	if err != nil {
 		log.Fatal(err)
 	}
+	if cidr.Size() < ipam.MinSubnetSize {
+		log.Fatalf("Allocation range smaller than minimum size %d: %s", ipam.MinSubnetSize, cidrStr)
+	}
+	return cidr
+}
+
+func createAllocator(router *weave.Router, ipRangeStr string, defaultSubnetStr string, quorum uint) (*ipam.Allocator, address.CIDR) {
+	ipRange := parseAndCheckCIDR(ipRangeStr)
+	defaultSubnet := ipRange
+	if defaultSubnetStr != "" {
+		defaultSubnet = parseAndCheckCIDR(defaultSubnetStr)
+		if !ipRange.Range().Overlaps(defaultSubnet.Range()) {
+			log.Fatalf("Default subnet %s out of bounds: %s", defaultSubnet, ipRange)
+		}
+	}
+	allocator := ipam.NewAllocator(router.Ourself.Peer.Name, router.Ourself.Peer.UID, router.Ourself.Peer.NickName, ipRange.Range(), quorum)
+
 	allocator.SetInterfaces(router.NewGossip("IPallocation", allocator))
 	allocator.Start()
-	err = updater.Start(apiPath, allocator)
-	if err != nil {
-		log.Fatal("Unable to start watcher", err)
-	}
-	return allocator
+
+	return allocator, defaultSubnet
 }
 
 // Pick a quorum size heuristically based on the number of peer
@@ -223,40 +252,41 @@ func determineQuorum(initPeerCountFlag int, peers []string) uint {
 	return quorum
 }
 
-func handleHTTP(router *weave.Router, httpAddr string, allocator *ipam.Allocator) {
-	encryption := "off"
-	if router.UsingPassword() {
-		encryption = "on"
-	}
-
+func handleHTTP(router *weave.Router, httpAddr string, allocator *ipam.Allocator, defaultSubnet address.CIDR, docker *docker.Client) {
 	muxRouter := mux.NewRouter()
 
 	if allocator != nil {
-		allocator.HandleHTTP(muxRouter)
+		allocator.HandleHTTP(muxRouter, defaultSubnet, docker)
 	}
 
 	muxRouter.Methods("GET").Path("/status").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		fmt.Fprintln(w, "weave router", version)
-		fmt.Fprintln(w, "Encryption", encryption)
 		fmt.Fprintln(w, router.Status())
 		if allocator != nil {
 			fmt.Fprintln(w, allocator.String())
+			fmt.Fprintln(w, "Allocator default subnet:", defaultSubnet)
 		}
 	})
 
 	muxRouter.Methods("GET").Path("/status-json").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		json, _ := router.StatusJSON(version, encryption)
+		json, _ := router.StatusJSON(version)
 		w.Write(json)
 	})
 
 	muxRouter.Methods("POST").Path("/connect").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if err := router.ConnectionMaker.InitiateConnection(r.FormValue("peer")); err != nil {
-			http.Error(w, fmt.Sprint("invalid peer address: ", err), http.StatusBadRequest)
+		if err := r.ParseForm(); err != nil {
+			http.Error(w, fmt.Sprint("unable to parse form: ", err), http.StatusBadRequest)
+		}
+		if errors := router.ConnectionMaker.InitiateConnections(r.Form["peer"], r.FormValue("replace") == "true"); len(errors) > 0 {
+			http.Error(w, errorMessages(errors), http.StatusBadRequest)
 		}
 	})
 
 	muxRouter.Methods("POST").Path("/forget").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		router.ConnectionMaker.ForgetConnection(r.FormValue("peer"))
+		if err := r.ParseForm(); err != nil {
+			http.Error(w, fmt.Sprint("unable to parse form: ", err), http.StatusBadRequest)
+		}
+		router.ConnectionMaker.ForgetConnections(r.Form["peer"])
 	})
 
 	http.Handle("/", muxRouter)

prog/weavewait/main.go

@@ -0,0 +1,47 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"os/signal"
+	"syscall"
+	"time"
+
+	"github.com/weaveworks/weave/net"
+)
+
+func main() {
+	if len(os.Args) <= 1 {
+		os.Exit(0)
+	}
+
+	args := os.Args[1:]
+	signalWait := 20
+	if args[0] == "-s" {
+		signalWait = 0
+		args = args[1:]
+	}
+	interfaceWait := 20 - signalWait
+
+	usr2 := make(chan os.Signal)
+	signal.Notify(usr2, syscall.SIGUSR2)
+	select {
+	case <-usr2:
+	case <-time.After(time.Duration(signalWait) * time.Second):
+	}
+	_, err := net.EnsureInterface("ethwe", interfaceWait)
+	checkErr(err)
+
+	binary, err := exec.LookPath(args[0])
+	checkErr(err)
+
+	checkErr(syscall.Exec(binary, args, os.Environ()))
+}
+
+func checkErr(err error) {
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+}

proxy/common.go

@@ -8,7 +8,6 @@ import (
 	"net/http"
 	"os/exec"
 	"regexp"
-	"strings"
 
 	"github.com/fsouza/go-dockerclient"
 	. "github.com/weaveworks/weave/common"
@@ -16,7 +15,7 @@ import (
 
 var (
 	containerIDRegexp   = regexp.MustCompile("^(/v[0-9\\.]*)?/containers/([^/]*)/.*")
-	weaveWaitEntrypoint = []string{"/home/weavewait/weavewait"}
+	weaveWaitEntrypoint = []string{"/w/w"}
 )
 
 func callWeave(args ...string) ([]byte, error) {
@@ -28,15 +27,6 @@ func callWeave(args ...string) ([]byte, error) {
 	return out, err
 }
 
-func weaveCIDRsFromConfig(config *docker.Config) ([]string, bool) {
-	for _, e := range config.Env {
-		if strings.HasPrefix(e, "WEAVE_CIDR=") {
-			return strings.Fields(e[11:]), true
-		}
-	}
-	return nil, false
-}
-
 func marshalRequestBody(r *http.Request, body interface{}) error {
 	newBody, err := json.Marshal(body)
 	if err != nil {

proxy/create_container_interceptor.go

@@ -3,20 +3,17 @@ package proxy
 import (
 	"bytes"
 	"encoding/json"
+	"fmt"
 	"io/ioutil"
 	"net/http"
 	"strings"
 
 	"github.com/fsouza/go-dockerclient"
 	. "github.com/weaveworks/weave/common"
+	"github.com/weaveworks/weave/nameserver"
 )
 
-type createContainerInterceptor struct {
-	client         *docker.Client
-	withDNS        bool
-	dockerBridgeIP string
-	withIPAM       bool
-}
+type createContainerInterceptor struct{ proxy *Proxy }
 
 type createContainerRequestBody struct {
 	*docker.Config
@@ -36,7 +33,7 @@ func (i *createContainerInterceptor) InterceptRequest(r *http.Request) error {
 		return err
 	}
 
-	if cidrs, ok := weaveCIDRsFromConfig(container.Config); ok || i.withIPAM {
+	if cidrs, ok := i.proxy.weaveCIDRsFromConfig(container.Config); ok {
 		Info.Printf("Creating container with WEAVE_CIDR \"%s\"", strings.Join(cidrs, " "))
 		if container.HostConfig == nil {
 			container.HostConfig = &docker.HostConfig{}
@@ -48,7 +45,7 @@ func (i *createContainerInterceptor) InterceptRequest(r *http.Request) error {
 		if err := i.setWeaveWaitEntrypoint(container.Config); err != nil {
 			return err
 		}
-		if err := i.setWeaveDNS(&container); err != nil {
+		if err := i.setWeaveDNS(&container, r); err != nil {
 			return err
 		}
 	}
@@ -65,7 +62,7 @@ func (i *createContainerInterceptor) InterceptRequest(r *http.Request) error {
 
 func (i *createContainerInterceptor) setWeaveWaitEntrypoint(container *docker.Config) error {
 	if len(container.Entrypoint) == 0 {
-		image, err := i.client.InspectImage(container.Image)
+		image, err := i.proxy.client.InspectImage(container.Image)
 		if err != nil {
 			return err
 		}
@@ -83,26 +80,60 @@ func (i *createContainerInterceptor) setWeaveWaitEntrypoint(container *docker.Co
 	return nil
 }
 
-func configHasEntrypoint(c *docker.Config) bool {
-	return c != nil && len(c.Entrypoint) > 0
-}
-func configHasCmd(c *docker.Config) bool {
-	return c != nil && len(c.Cmd) > 0
-}
-
-func (i *createContainerInterceptor) setWeaveDNS(container *createContainerRequestBody) error {
-	if !i.withDNS {
+func (i *createContainerInterceptor) setWeaveDNS(container *createContainerRequestBody, r *http.Request) error {
+	if i.proxy.WithoutDNS {
 		return nil
 	}
 
-	container.HostConfig.DNS = append(container.HostConfig.DNS, i.dockerBridgeIP)
+	dnsDomain, dnsRunning := i.getDNSDomain()
+	if !(dnsRunning || i.proxy.WithDNS) {
+		return nil
+	}
+
+	container.HostConfig.DNS = append(container.HostConfig.DNS, i.proxy.dockerBridgeIP)
+
+	name := r.URL.Query().Get("name")
+	if container.Hostname == "" && name != "" {
+		container.Hostname = name
+		// Strip trailing period because it's unusual to see it used on the end of a host name
+		container.Domainname = strings.TrimSuffix(dnsDomain, ".")
+	}
 
 	if len(container.HostConfig.DNSSearch) == 0 {
-		container.HostConfig.DNSSearch = []string{"."}
+		if container.Hostname == "" {
+			container.HostConfig.DNSSearch = []string{dnsDomain}
+		} else {
+			container.HostConfig.DNSSearch = []string{"."}
+		}
 	}
+
 	return nil
 }
 
+func (i *createContainerInterceptor) getDNSDomain() (domain string, running bool) {
+	domain = nameserver.DefaultLocalDomain
+	dnsContainer, err := i.proxy.client.InspectContainer("weavedns")
+	if err != nil ||
+		dnsContainer.NetworkSettings == nil ||
+		dnsContainer.NetworkSettings.IPAddress == "" {
+		return
+	}
+
+	url := fmt.Sprintf("http://%s:%d/domain", dnsContainer.NetworkSettings.IPAddress, nameserver.DefaultHTTPPort)
+	resp, err := http.Get(url)
+	if err != nil || resp.StatusCode != http.StatusOK {
+		return
+	}
+	defer resp.Body.Close()
+
+	b, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return
+	}
+
+	return string(b), true
+}
+
 func (i *createContainerInterceptor) InterceptResponse(r *http.Response) error {
 	return nil
 }

proxy/create_exec_interceptor.go

@@ -10,16 +10,7 @@ import (
 	. "github.com/weaveworks/weave/common"
 )
 
-type createExecInterceptor struct {
-	client   *docker.Client
-	withIPAM bool
-}
-
-type createExecRequestBody struct {
-	*docker.Config
-	HostConfig *docker.HostConfig `json:"HostConfig,omitempty" yaml:"HostConfig,omitempty"`
-	MacAddress string             `json:"MacAddress,omitempty" yaml:"MacAddress,omitempty"`
-}
+type createExecInterceptor struct{ proxy *Proxy }
 
 func (i *createExecInterceptor) InterceptRequest(r *http.Request) error {
 	body, err := ioutil.ReadAll(r.Body)
@@ -33,14 +24,15 @@ func (i *createExecInterceptor) InterceptRequest(r *http.Request) error {
 		return err
 	}
 
-	container, err := inspectContainerInPath(i.client, r.URL.Path)
+	container, err := inspectContainerInPath(i.proxy.client, r.URL.Path)
 	if err != nil {
 		return err
 	}
 
-	if cidrs, ok := weaveCIDRsFromConfig(container.Config); ok || i.withIPAM {
+	if cidrs, ok := i.proxy.weaveCIDRsFromConfig(container.Config); ok {
 		Info.Printf("Exec in container %s with WEAVE_CIDR \"%s\"", container.ID, strings.Join(cidrs, " "))
-		options.Cmd = append(weaveWaitEntrypoint, options.Cmd...)
+		cmd := append(weaveWaitEntrypoint, "-s")
+		options.Cmd = append(cmd, options.Cmd...)
 	}
 
 	if err := marshalRequestBody(r, options); err != nil {

proxy/proxy.go

@@ -1,6 +1,7 @@
 package proxy
 
 import (
+	"crypto/tls"
 	"net"
 	"net/http"
 	"net/url"
@@ -11,6 +12,12 @@ import (
 	. "github.com/weaveworks/weave/common"
 )
 
+const (
+	defaultCaFile   = "ca.pem"
+	defaultKeyFile  = "key.pem"
+	defaultCertFile = "cert.pem"
+)
+
 var (
 	containerCreateRegexp = regexp.MustCompile("^(/v[0-9\\.]*)?/containers/create$")
 	containerStartRegexp  = regexp.MustCompile("^(/v[0-9\\.]*)?/containers/[^/]*/(re)?start$")
@@ -18,28 +25,25 @@ var (
 )
 
 type Proxy struct {
-	Dial           func() (net.Conn, error)
+	Config
+
+	dial           func() (net.Conn, error)
 	client         *docker.Client
-	withDNS        bool
 	dockerBridgeIP string
-	withIPAM       bool
 }
 
-func NewProxy(targetURL string, withDNS, withIPAM bool) (*Proxy, error) {
-	u, err := url.Parse(targetURL)
-	if err != nil {
-		return nil, err
-	}
+type Config struct {
+	DockerAddr    string
+	ListenAddr    string
+	NoDefaultIPAM bool
+	TLSConfig     TLSConfig
+	Version       string
+	WithDNS       bool
+	WithoutDNS    bool
+}
 
-	var dockerBridgeIP []byte
-	if withDNS {
-		dockerBridgeIP, err = callWeave("docker-bridge-ip")
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	client, err := docker.NewClient(targetURL)
+func NewProxy(c Config) (*Proxy, error) {
+	u, err := url.Parse(c.DockerAddr)
 	if err != nil {
 		return nil, err
 	}
@@ -52,34 +56,73 @@ func NewProxy(targetURL string, withDNS, withIPAM bool) (*Proxy, error) {
 		targetAddr = u.Path
 	}
 
-	return &Proxy{
-		Dial: func() (net.Conn, error) {
+	p := &Proxy{
+		Config: c,
+		dial: func() (net.Conn, error) {
 			return net.Dial(u.Scheme, targetAddr)
 		},
-		client:         client,
-		withDNS:        withDNS,
-		dockerBridgeIP: string(dockerBridgeIP),
-		withIPAM:       withIPAM,
-	}, nil
+	}
+
+	if !p.WithoutDNS {
+		dockerBridgeIP, err := callWeave("docker-bridge-ip")
+		if err != nil {
+			return nil, err
+		}
+		p.dockerBridgeIP = string(dockerBridgeIP)
+	}
+
+	if err := p.TLSConfig.loadCerts(); err != nil {
+		Error.Fatalf("Could not configure tls for proxy: %s", err)
+	}
+
+	p.client, err = docker.NewClient(p.DockerAddr)
+	if err != nil {
+		return nil, err
+	}
+
+	return p, nil
 }
 
 func (proxy *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	Info.Printf("%s %s", r.Method, r.URL)
 	path := r.URL.Path
+	var i interceptor
 	switch {
 	case containerCreateRegexp.MatchString(path):
-		proxy.serveWithInterceptor(&createContainerInterceptor{proxy.client, proxy.withDNS, proxy.dockerBridgeIP, proxy.withIPAM}, w, r)
+		i = &createContainerInterceptor{proxy}
 	case containerStartRegexp.MatchString(path):
-		proxy.serveWithInterceptor(&startContainerInterceptor{proxy.client, proxy.withDNS, proxy.withIPAM}, w, r)
+		i = &startContainerInterceptor{proxy}
 	case execCreateRegexp.MatchString(path):
-		proxy.serveWithInterceptor(&createExecInterceptor{proxy.client, proxy.withIPAM}, w, r)
-	case strings.HasPrefix(path, "/weave"):
-		w.WriteHeader(http.StatusOK)
+		i = &createExecInterceptor{proxy}
 	default:
-		proxy.serveWithInterceptor(&nullInterceptor{}, w, r)
+		i = &nullInterceptor{}
 	}
+	proxy.Intercept(i, w, r)
 }
 
-func (proxy *Proxy) serveWithInterceptor(i interceptor, w http.ResponseWriter, r *http.Request) {
-	newClient(proxy.Dial, i).ServeHTTP(w, r)
+func (proxy *Proxy) ListenAndServe() error {
+	listener, err := net.Listen("tcp", proxy.ListenAddr)
+	if err != nil {
+		return err
+	}
+
+	if proxy.TLSConfig.enabled() {
+		listener = tls.NewListener(listener, proxy.TLSConfig.Config)
+	}
+
+	Info.Println("proxy listening on", proxy.ListenAddr)
+
+	return (&http.Server{Handler: proxy}).Serve(listener)
+}
+
+func (proxy *Proxy) weaveCIDRsFromConfig(config *docker.Config) ([]string, bool) {
+	for _, e := range config.Env {
+		if strings.HasPrefix(e, "WEAVE_CIDR=") {
+			if e[11:] == "none" {
+				return nil, false
+			}
+			return strings.Fields(e[11:]), true
+		}
+	}
+	return nil, !proxy.NoDefaultIPAM
 }

proxy/proxy_intercept.go

@@ -14,20 +14,8 @@ import (
 	. "github.com/weaveworks/weave/common"
 )
 
-type client struct {
-	interceptor
-	Dial func() (net.Conn, error)
-}
-
-func newClient(dial func() (net.Conn, error), i interceptor) *client {
-	return &client{
-		interceptor: i,
-		Dial:        dial,
-	}
-}
-
-func (c *client) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	if err := c.InterceptRequest(r); err != nil {
+func (proxy *Proxy) Intercept(i interceptor, w http.ResponseWriter, r *http.Request) {
+	if err := i.InterceptRequest(r); err != nil {
 		_, isNoSuchContainer := err.(*docker.NoSuchContainer)
 		switch {
 		case isNoSuchContainer:
@@ -41,7 +29,7 @@ func (c *client) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	conn, err := c.Dial()
+	conn, err := proxy.dial()
 	if err != nil {
 		http.Error(w, "Could not connect to target", http.StatusInternalServerError)
 		Warning.Print(err)
@@ -56,7 +44,7 @@ func (c *client) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		Warning.Print("Error forwarding request: ", err)
 		return
 	}
-	err = c.InterceptResponse(resp)
+	err = i.InterceptResponse(resp)
 	if err != nil {
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 		Warning.Print("Error intercepting response: ", err)

proxy/start_container_interceptor.go

@@ -9,35 +9,31 @@ import (
 	. "github.com/weaveworks/weave/common"
 )
 
-type startContainerInterceptor struct {
-	client   *docker.Client
-	withDNS  bool
-	withIPAM bool
-}
+type startContainerInterceptor struct{ proxy *Proxy }
 
 func (i *startContainerInterceptor) InterceptRequest(r *http.Request) error {
 	return nil
 }
 
 func (i *startContainerInterceptor) InterceptResponse(r *http.Response) error {
-	container, err := inspectContainerInPath(i.client, r.Request.URL.Path)
+	container, err := inspectContainerInPath(i.proxy.client, r.Request.URL.Path)
 	if err != nil {
 		return err
 	}
 
-	cidrs, ok := weaveCIDRsFromConfig(container.Config)
-	if !ok && !i.withIPAM {
+	cidrs, ok := i.proxy.weaveCIDRsFromConfig(container.Config)
+	if !ok {
 		Debug.Print("No Weave CIDR, ignoring")
 		return nil
 	}
 	Info.Printf("Attaching container %s with WEAVE_CIDR \"%s\" to weave network", container.ID, strings.Join(cidrs, " "))
 	args := []string{"attach"}
 	args = append(args, cidrs...)
-	args = append(args, container.ID)
+	args = append(args, "--or-die", container.ID)
 	if output, err := callWeave(args...); err != nil {
-		i.client.KillContainer(docker.KillContainerOptions{ID: container.ID})
 		Warning.Printf("Attaching container %s to weave network failed: %s", container.ID, string(output))
 		return errors.New(string(output))
 	}
-	return nil
+
+	return i.proxy.client.KillContainer(docker.KillContainerOptions{ID: container.ID, Signal: docker.SIGUSR2})
 }

proxy/tls.go

@@ -0,0 +1,76 @@
+package proxy
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/docker/docker/pkg/homedir"
+)
+
+type TLSConfig struct {
+	Enabled, Verify   bool
+	Cert, Key, CACert string
+	*tls.Config
+}
+
+func (c *TLSConfig) enabled() bool {
+	if c == nil {
+		return false
+	}
+	return c.Enabled || c.Verify
+}
+
+func (c *TLSConfig) loadCerts() error {
+	if !c.enabled() {
+		return nil
+	}
+
+	dockerCertPath := os.Getenv("DOCKER_CERT_PATH")
+	if dockerCertPath == "" {
+		dockerCertPath = filepath.Join(homedir.Get(), ".docker")
+	}
+
+	if c.CACert == "" {
+		c.CACert = filepath.Join(dockerCertPath, defaultCaFile)
+	}
+	if c.Cert == "" {
+		c.Cert = filepath.Join(dockerCertPath, defaultCertFile)
+	}
+	if c.Key == "" {
+		c.Key = filepath.Join(dockerCertPath, defaultKeyFile)
+	}
+
+	tlsConfig := &tls.Config{
+		NextProtos: []string{"http/1.1"},
+		// Avoid fallback on insecure SSL protocols
+		MinVersion: tls.VersionTLS10,
+	}
+
+	if c.Verify {
+		certPool := x509.NewCertPool()
+		file, err := ioutil.ReadFile(c.CACert)
+		if err != nil {
+			return fmt.Errorf("Couldn't read CA certificate: %v", err)
+		}
+		certPool.AppendCertsFromPEM(file)
+		tlsConfig.ClientAuth = tls.RequireAndVerifyClientCert
+		tlsConfig.ClientCAs = certPool
+	}
+
+	_, errCert := os.Stat(c.Cert)
+	_, errKey := os.Stat(c.Key)
+	if errCert == nil && errKey == nil {
+		cert, err := tls.LoadX509KeyPair(c.Cert, c.Key)
+		if err != nil {
+			return fmt.Errorf("Couldn't load X509 key pair: %q. Make sure the key is encrypted", err)
+		}
+		tlsConfig.Certificates = []tls.Certificate{cert}
+	}
+
+	c.Config = tlsConfig
+	return nil
+}

router/connection.go

@@ -41,6 +41,7 @@ type LocalConnection struct {
 	sync.RWMutex
 	RemoteConnection
 	TCPConn           *net.TCPConn
+	version           byte
 	tcpSender         TCPSender
 	remoteUDPAddr     *net.UDPAddr
 	receivedHeartbeat bool
@@ -262,10 +263,7 @@ func (conn *LocalConnection) SetEstablished() {
 		conn.fragTest = time.NewTicker(FragTestInterval)
 		// avoid initial waits for timers to fire
 		conn.Forward(true, conn.heartbeatFrame, nil)
-		conn.setStackFrag(false)
-		if err := conn.sendSimpleProtocolMsg(ProtocolStartFragmentationTest); err != nil {
-			return err
-		}
+		conn.performFragTest()
 		return nil
 	})
 }
@@ -287,15 +285,13 @@ func (conn *LocalConnection) run(actionChan <-chan ConnectionAction, finished ch
 	defer func() { conn.shutdown(err) }()
 	defer close(finished)
 
-	tcpConn := conn.TCPConn
-	tcpConn.SetLinger(0)
-	enc := gob.NewEncoder(tcpConn)
-	dec := gob.NewDecoder(tcpConn)
+	conn.TCPConn.SetLinger(0)
 
-	if err = conn.handshake(enc, dec, acceptNewPeer); err != nil {
+	_, dec, err := conn.handshake(acceptNewPeer)
+	if err != nil {
 		return
 	}
-	conn.Log("completed handshake")
+	conn.Log("completed handshake; using protocol version", conn.version)
 
 	// The ordering of the following is very important. [1]
 
@@ -386,8 +382,7 @@ func (conn *LocalConnection) actorLoop(actionChan <-chan ConnectionAction) (err
 		case <-tickerChan(conn.heartbeat):
 			conn.Forward(true, conn.heartbeatFrame, nil)
 		case <-tickerChan(conn.fragTest):
-			conn.setStackFrag(false)
-			err = conn.sendSimpleProtocolMsg(ProtocolStartFragmentationTest)
+			conn.performFragTest()
 		}
 	}
 	return
@@ -473,12 +468,6 @@ func (conn *LocalConnection) handleProtocolMsg(tag ProtocolTag, payload []byte)
 		// We can now consider the connection as established from our
 		// end.
 		conn.SetEstablished()
-	case ProtocolStartFragmentationTest:
-		conn.Forward(false, &ForwardedFrame{
-			srcPeer: conn.local,
-			dstPeer: conn.remote,
-			frame:   FragTest},
-			nil)
 	case ProtocolFragmentationReceived:
 		conn.setStackFrag(true)
 	case ProtocolPMTUVerified:
@@ -504,6 +493,15 @@ func (conn *LocalConnection) sendFastHeartbeats() error {
 	return err
 }
 
+func (conn *LocalConnection) performFragTest() {
+	conn.setStackFrag(false)
+	conn.Forward(false, &ForwardedFrame{
+		srcPeer: conn.local,
+		dstPeer: conn.remote,
+		frame:   FragTest},
+		nil)
+}
+
 func tickerChan(ticker *time.Ticker) <-chan time.Time {
 	if ticker != nil {
 		return ticker.C

router/connection_maker.go

@@ -10,17 +10,20 @@ import (
 )
 
 const (
-	InitialInterval = 5 * time.Second
-	MaxInterval     = 10 * time.Minute
+	InitialInterval = 2 * time.Second
+	MaxInterval     = 6 * time.Minute
 )
 
+type peerAddrs map[string]*net.TCPAddr
+
 type ConnectionMaker struct {
-	ourself      *LocalPeer
-	peers        *Peers
-	port         int
-	targets      map[string]*Target
-	cmdLinePeers map[string]*net.TCPAddr
-	actionChan   chan<- ConnectionMakerAction
+	ourself     *LocalPeer
+	peers       *Peers
+	port        int
+	discovery   bool
+	targets     map[string]*Target
+	directPeers peerAddrs
+	actionChan  chan<- ConnectionMakerAction
 }
 
 // Information about an address where we may find a peer
@@ -28,18 +31,19 @@ type Target struct {
 	attempting  bool          // are we currently attempting to connect there?
 	lastError   error         // reason for disconnection last time
 	tryAfter    time.Time     // next time to try this address
-	tryInterval time.Duration // backoff time on next failure
+	tryInterval time.Duration // retry delay on next failure
 }
 
 type ConnectionMakerAction func() bool
 
-func NewConnectionMaker(ourself *LocalPeer, peers *Peers, port int) *ConnectionMaker {
+func NewConnectionMaker(ourself *LocalPeer, peers *Peers, port int, discovery bool) *ConnectionMaker {
 	return &ConnectionMaker{
-		ourself:      ourself,
-		peers:        peers,
-		port:         port,
-		cmdLinePeers: make(map[string]*net.TCPAddr),
-		targets:      make(map[string]*Target)}
+		ourself:     ourself,
+		peers:       peers,
+		port:        port,
+		discovery:   discovery,
+		directPeers: peerAddrs{},
+		targets:     make(map[string]*Target)}
 }
 
 func (cm *ConnectionMaker) Start() {
@@ -48,30 +52,42 @@ func (cm *ConnectionMaker) Start() {
 	go cm.queryLoop(actionChan)
 }
 
-func (cm *ConnectionMaker) InitiateConnection(peer string) error {
-	host, port, err := net.SplitHostPort(peer)
-	if err != nil {
-		host = peer
-		port = "0" // we use that as an indication that "no port was supplied"
-	}
-	addr, err := net.ResolveTCPAddr("tcp4", fmt.Sprintf("%s:%s", host, port))
-	if err != nil {
-		return err
+func (cm *ConnectionMaker) InitiateConnections(peers []string, replace bool) []error {
+	errors := []error{}
+	addrs := peerAddrs{}
+	for _, peer := range peers {
+		host, port, err := net.SplitHostPort(peer)
+		if err != nil {
+			host = peer
+			port = "0" // we use that as an indication that "no port was supplied"
+		}
+		if addr, err := net.ResolveTCPAddr("tcp4", fmt.Sprintf("%s:%s", host, port)); err != nil {
+			errors = append(errors, err)
+		} else {
+			addrs[peer] = addr
+		}
 	}
 	cm.actionChan <- func() bool {
-		cm.cmdLinePeers[peer] = addr
-		// curtail any existing reconnect interval
-		if target, found := cm.targets[addr.String()]; found {
-			target.tryAfter, target.tryInterval = tryImmediately()
+		if replace {
+			cm.directPeers = peerAddrs{}
+		}
+		for peer, addr := range addrs {
+			cm.directPeers[peer] = addr
+			// curtail any existing reconnect interval
+			if target, found := cm.targets[addr.String()]; found {
+				target.tryNow()
+			}
 		}
 		return true
 	}
-	return nil
+	return errors
 }
 
-func (cm *ConnectionMaker) ForgetConnection(peer string) {
+func (cm *ConnectionMaker) ForgetConnections(peers []string) {
 	cm.actionChan <- func() bool {
-		delete(cm.cmdLinePeers, peer)
+		for _, peer := range peers {
+			delete(cm.directPeers, peer)
+		}
 		return false
 	}
 }
@@ -81,7 +97,7 @@ func (cm *ConnectionMaker) ConnectionTerminated(address string, err error) {
 		if target, found := cm.targets[address]; found {
 			target.attempting = false
 			target.lastError = err
-			target.tryAfter, target.tryInterval = tryAfter(target.tryInterval)
+			target.retry()
 		}
 		return true
 	}
@@ -91,33 +107,58 @@ func (cm *ConnectionMaker) Refresh() {
 	cm.actionChan <- func() bool { return true }
 }
 
-func (cm *ConnectionMaker) String() string {
+func (cm *ConnectionMaker) Status() ConnectionMakerStatus {
 	// We need to Refresh first in order to clear out any 'attempting'
 	// connections from cm.targets that have been established since
 	// the last run of cm.checkStateAndAttemptConnections. These
 	// entries are harmless but do represent stale state that we do
 	// not want to report.
 	cm.Refresh()
-	resultChan := make(chan string, 0)
+	resultChan := make(chan ConnectionMakerStatus, 0)
 	cm.actionChan <- func() bool {
-		var buf bytes.Buffer
-		for address, target := range cm.targets {
-			fmt.Fprintf(&buf, "->[%s]", address)
-			if target.lastError != nil {
-				fmt.Fprintf(&buf, " (%s)", target.lastError)
-			}
-			if target.attempting {
-				fmt.Fprintf(&buf, " trying since %v\n", target.tryAfter)
-			} else {
-				fmt.Fprintf(&buf, " next try at %v\n", target.tryAfter)
-			}
+		status := ConnectionMakerStatus{
+			DirectPeers: []string{},
+			Reconnects:  make(map[string]Target),
 		}
-		resultChan <- buf.String()
+		for peer := range cm.directPeers {
+			status.DirectPeers = append(status.DirectPeers, peer)
+		}
+
+		for address, target := range cm.targets {
+			status.Reconnects[address] = *target
+		}
+		resultChan <- status
 		return false
 	}
 	return <-resultChan
 }
 
+type ConnectionMakerStatus struct {
+	DirectPeers []string
+	Reconnects  map[string]Target
+}
+
+func (status ConnectionMakerStatus) String() string {
+	var buf bytes.Buffer
+	fmt.Fprint(&buf, "Direct Peers:")
+	for _, peer := range status.DirectPeers {
+		fmt.Fprintf(&buf, " %s", peer)
+	}
+	fmt.Fprintln(&buf, "\nReconnects:")
+	for address, target := range status.Reconnects {
+		fmt.Fprintf(&buf, "->[%s]", address)
+		if target.lastError != nil {
+			fmt.Fprintf(&buf, " (%s)", target.lastError)
+		}
+		if target.attempting {
+			fmt.Fprintf(&buf, " trying since %v\n", target.tryAfter)
+		} else {
+			fmt.Fprintf(&buf, " next try at %v\n", target.tryAfter)
+		}
+	}
+	return buf.String()
+}
+
 func (cm *ConnectionMaker) queryLoop(actionChan <-chan ConnectionMakerAction) {
 	timer := time.NewTimer(MaxDuration)
 	run := func() { timer.Reset(cm.checkStateAndAttemptConnections()) }
@@ -135,8 +176,8 @@ func (cm *ConnectionMaker) queryLoop(actionChan <-chan ConnectionMakerAction) {
 
 func (cm *ConnectionMaker) checkStateAndAttemptConnections() time.Duration {
 	var (
-		validTarget   = make(map[string]struct{})
-		cmdLineTarget = make(map[string]struct{})
+		validTarget  = make(map[string]struct{})
+		directTarget = make(map[string]struct{})
 	)
 	// Copy the set of things we are connected to, so we can access
 	// them without locking.  Also clear out any entries in cm.targets
@@ -152,12 +193,12 @@ func (cm *ConnectionMaker) checkStateAndAttemptConnections() time.Duration {
 			return
 		}
 		target := &Target{}
-		target.tryAfter, target.tryInterval = tryImmediately()
+		target.tryNow()
 		cm.targets[address] = target
 	}
 
-	// Add command-line targets that are not connected
-	for _, addr := range cm.cmdLinePeers {
+	// Add direct targets that are not connected
+	for _, addr := range cm.directPeers {
 		completeAddr := *addr
 		attempt := true
 		if completeAddr.Port == 0 {
@@ -170,7 +211,7 @@ func (cm *ConnectionMaker) checkStateAndAttemptConnections() time.Duration {
 			}
 		}
 		address := completeAddr.String()
-		cmdLineTarget[address] = void
+		directTarget[address] = void
 		if attempt {
 			addTarget(address)
 		}
@@ -178,9 +219,11 @@ func (cm *ConnectionMaker) checkStateAndAttemptConnections() time.Duration {
 
 	// Add targets for peers that someone else is connected to, but we
 	// aren't
-	cm.addPeerTargets(ourConnectedPeers, addTarget)
+	if cm.discovery {
+		cm.addPeerTargets(ourConnectedPeers, addTarget)
+	}
 
-	return cm.connectToTargets(validTarget, cmdLineTarget)
+	return cm.connectToTargets(validTarget, directTarget)
 }
 
 func (cm *ConnectionMaker) ourConnections() (PeerNameSet, map[string]struct{}, map[string]struct{}) {
@@ -233,7 +276,7 @@ func (cm *ConnectionMaker) addPeerTargets(ourConnectedPeers PeerNameSet, addTarg
 	})
 }
 
-func (cm *ConnectionMaker) connectToTargets(validTarget map[string]struct{}, cmdLineTarget map[string]struct{}) time.Duration {
+func (cm *ConnectionMaker) connectToTargets(validTarget map[string]struct{}, directTarget map[string]struct{}) time.Duration {
 	now := time.Now() // make sure we catch items just added
 	after := MaxDuration
 	for address, target := range cm.targets {
@@ -247,7 +290,7 @@ func (cm *ConnectionMaker) connectToTargets(validTarget map[string]struct{}, cmd
 		switch duration := target.tryAfter.Sub(now); {
 		case duration <= 0:
 			target.attempting = true
-			_, isCmdLineTarget := cmdLineTarget[address]
+			_, isCmdLineTarget := directTarget[address]
 			go cm.attemptConnection(address, isCmdLineTarget)
 		case duration < after:
 			after = duration
@@ -264,15 +307,17 @@ func (cm *ConnectionMaker) attemptConnection(address string, acceptNewPeer bool)
 	}
 }
 
-func tryImmediately() (time.Time, time.Duration) {
-	interval := time.Duration(rand.Int63n(int64(InitialInterval)))
-	return time.Now(), interval
+func (t *Target) tryNow() {
+	t.tryAfter = time.Now()
+	t.tryInterval = InitialInterval
 }
 
-func tryAfter(interval time.Duration) (time.Time, time.Duration) {
-	interval += time.Duration(rand.Int63n(int64(interval)))
-	if interval > MaxInterval {
-		interval = MaxInterval
+// The delay at the nth retry is a random value in the range
+// [i-i/2,i+i/2], where i = InitialInterval * 1.5^(n-1).
+func (t *Target) retry() {
+	t.tryAfter = time.Now().Add(t.tryInterval/2 + time.Duration(rand.Int63n(int64(t.tryInterval))))
+	t.tryInterval = t.tryInterval * 3 / 2
+	if t.tryInterval > MaxInterval {
+		t.tryInterval = MaxInterval
 	}
-	return time.Now().Add(interval), interval
 }

router/consts.go

@@ -19,10 +19,12 @@ const (
 	FastHeartbeat       = 500 * time.Millisecond
 	SlowHeartbeat       = 10 * time.Second
 	FragTestInterval    = 5 * time.Minute
+	GossipInterval      = 30 * time.Second
 	PMTUVerifyAttempts  = 8
 	PMTUVerifyTimeout   = 10 * time.Millisecond // gets doubled with every attempt
 	MaxDuration         = time.Duration(math.MaxInt64)
 	MaxMissedHeartbeats = 6
+	HeaderTimeout       = 10 * time.Second
 	HeartbeatTimeout    = MaxMissedHeartbeats * SlowHeartbeat
 )
 

router/crypto.go

@@ -305,10 +305,12 @@ func (di *NaClDecryptorInstance) advanceState(seqNo uint64) (int, *bit.Set) {
 
 // The lowest 64 bits of the nonce contain the message sequence
 // number. The top most bit indicates the connection polarity at the
-// sender - '1' for outbound. The remaining 127 bits are zero. The
-// polarity is needed so that the two ends of a connection do not use
-// the same nonces. This is a requirement of the NaCl Security Model;
-// see http://nacl.cr.yp.to/box.html.
+// sender - '1' for outbound; the next indicates protocol type - '1'
+// for TCP. The remaining 126 bits are zero. The polarity is needed so
+// that the two ends of a connection do not use the same nonces; the
+// protocol type so that the TCP and UDP sender nonces are disjoint.
+// This is a requirement of the NaCl Security Model; see
+// http://nacl.cr.yp.to/box.html.
 type TCPCryptoState struct {
 	sessionKey *[32]byte
 	nonce      [24]byte
@@ -320,6 +322,8 @@ func NewTCPCryptoState(sessionKey *[32]byte, outbound bool) *TCPCryptoState {
 	if outbound {
 		s.nonce[0] |= (1 << 7)
 	}
+	// Ensure that TCP and UDP sequences are disjoint
+	s.nonce[0] |= (1 << 6)
 	return s
 }
 

router/gossip.go

@@ -1,18 +1,11 @@
 package router
 
 import (
-	"bytes"
-	"encoding/gob"
 	"fmt"
-	"log"
-	"sync"
-	"time"
 )
 
-const GossipInterval = 30 * time.Second
-
 type GossipData interface {
-	Encode() []byte
+	Encode() [][]byte
 	Merge(GossipData)
 }
 
@@ -41,6 +34,9 @@ type Gossiper interface {
 type GossipSender struct {
 	send func(GossipData)
 	cell chan GossipData
+	// for testing
+	sent    bool
+	flushch chan chan bool
 }
 
 func NewGossipSender(send func(GossipData)) *GossipSender {
@@ -49,15 +45,30 @@ func NewGossipSender(send func(GossipData)) *GossipSender {
 
 func (sender *GossipSender) Start() {
 	sender.cell = make(chan GossipData, 1)
+	sender.flushch = make(chan chan bool)
 	go sender.run()
 }
 
 func (sender *GossipSender) run() {
 	for {
-		if pending := <-sender.cell; pending == nil { // receive zero value when chan is closed
-			break
-		} else {
+		select {
+		case pending := <-sender.cell:
+			if pending == nil { // receive zero value when chan is closed
+				return
+			}
 			sender.send(pending)
+			sender.sent = true
+		case ch := <-sender.flushch:
+			// send anything pending, then reply back whether we sent
+			// anything since previous flush
+			select {
+			case pending := <-sender.cell:
+				sender.send(pending)
+				sender.sent = true
+			default:
+			}
+			ch <- sender.sent
+			sender.sent = false
 		}
 	}
 }
@@ -77,36 +88,49 @@ func (sender *GossipSender) Stop() {
 	close(sender.cell)
 }
 
-type connectionSenders map[Connection]*GossipSender
-type peerSenders map[PeerName]*GossipSender
-
-type GossipChannel struct {
-	sync.Mutex
-	ourself      *LocalPeer
-	routes       *Routes
-	name         string
-	hash         uint32
-	gossiper     Gossiper
-	senders      connectionSenders
-	broadcasters peerSenders
-}
+type GossipChannels map[string]*GossipChannel
 
 func (router *Router) NewGossip(channelName string, g Gossiper) Gossip {
-	channelHash := hash(channelName)
-	channel := &GossipChannel{
-		ourself:      router.Ourself,
-		routes:       router.Routes,
-		name:         channelName,
-		hash:         channelHash,
-		gossiper:     g,
-		senders:      make(connectionSenders),
-		broadcasters: make(peerSenders)}
-	router.GossipChannels[channelHash] = channel
+	channel := NewGossipChannel(channelName, router.Ourself, router.Routes, g)
+	router.gossipLock.Lock()
+	defer router.gossipLock.Unlock()
+	if _, found := router.gossipChannels[channelName]; found {
+		checkFatal(fmt.Errorf("[gossip] duplicate channel %s", channelName))
+	}
+	router.gossipChannels[channelName] = channel
 	return channel
 }
 
+func (router *Router) gossipChannel(channelName string) *GossipChannel {
+	router.gossipLock.RLock()
+	channel, found := router.gossipChannels[channelName]
+	router.gossipLock.RUnlock()
+	if found {
+		return channel
+	}
+	router.gossipLock.Lock()
+	defer router.gossipLock.Unlock()
+	if channel, found = router.gossipChannels[channelName]; found {
+		return channel
+	}
+	channel = NewGossipChannel(channelName, router.Ourself, router.Routes, &surrogateGossiper)
+	channel.log("created surrogate channel")
+	router.gossipChannels[channelName] = channel
+	return channel
+}
+
+func (router *Router) gossipChannelSet() map[*GossipChannel]struct{} {
+	channels := make(map[*GossipChannel]struct{})
+	router.gossipLock.RLock()
+	defer router.gossipLock.RUnlock()
+	for _, channel := range router.gossipChannels {
+		channels[channel] = void
+	}
+	return channels
+}
+
 func (router *Router) SendAllGossip() {
-	for _, channel := range router.GossipChannels {
+	for channel := range router.gossipChannelSet() {
 		if gossip := channel.gossiper.Gossip(); gossip != nil {
 			channel.Send(router.Ourself.Name, gossip)
 		}
@@ -114,228 +138,32 @@ func (router *Router) SendAllGossip() {
 }
 
 func (router *Router) SendAllGossipDown(conn Connection) {
-	for _, channel := range router.GossipChannels {
+	for channel := range router.gossipChannelSet() {
 		if gossip := channel.gossiper.Gossip(); gossip != nil {
 			channel.SendDown(conn, channel.gossiper.Gossip())
 		}
 	}
 }
 
-func (router *Router) handleGossip(tag ProtocolTag, payload []byte) error {
-	decoder := gob.NewDecoder(bytes.NewReader(payload))
-	var channelHash uint32
-	if err := decoder.Decode(&channelHash); err != nil {
-		return err
-	}
-	channel, found := router.GossipChannels[channelHash]
-	if !found {
-		return fmt.Errorf("[gossip] received unknown channel with hash %v", channelHash)
-	}
-	var srcName PeerName
-	if err := decoder.Decode(&srcName); err != nil {
-		return err
-	}
-	switch tag {
-	case ProtocolGossipUnicast:
-		return channel.deliverUnicast(srcName, payload, decoder)
-	case ProtocolGossipBroadcast:
-		return channel.deliverBroadcast(srcName, payload, decoder)
-	case ProtocolGossip:
-		return channel.deliver(srcName, payload, decoder)
-	}
-	return nil
-}
-
-func (c *GossipChannel) deliverUnicast(srcName PeerName, origPayload []byte, dec *gob.Decoder) error {
-	var destName PeerName
-	if err := dec.Decode(&destName); err != nil {
-		return err
-	}
-	if c.ourself.Name != destName {
-		return c.relayUnicast(destName, origPayload)
-	}
-	var payload []byte
-	if err := dec.Decode(&payload); err != nil {
-		return err
-	}
-	return c.gossiper.OnGossipUnicast(srcName, payload)
-}
-
-func (c *GossipChannel) deliverBroadcast(srcName PeerName, _ []byte, dec *gob.Decoder) error {
-	var payload []byte
-	if err := dec.Decode(&payload); err != nil {
-		return err
-	}
-	data, err := c.gossiper.OnGossipBroadcast(payload)
-	if err != nil || data == nil {
-		return err
-	}
-	return c.relayBroadcast(srcName, data)
-}
-
-func (c *GossipChannel) deliver(srcName PeerName, _ []byte, dec *gob.Decoder) error {
-	var payload []byte
-	if err := dec.Decode(&payload); err != nil {
-		return err
-	}
-	if data, err := c.gossiper.OnGossip(payload); err != nil {
-		return err
-	} else if data != nil {
-		c.Send(srcName, data)
-	}
-	return nil
-}
-
-func (c *GossipChannel) Send(srcName PeerName, data GossipData) {
-	// do this outside the lock below so we avoid lock nesting
-	c.routes.EnsureRecalculated()
-	selectedConnections := make(ConnectionSet)
-	for name := range c.routes.RandomNeighbours(srcName) {
-		if conn, found := c.ourself.ConnectionTo(name); found {
-			selectedConnections[conn] = void
-		}
-	}
-	if len(selectedConnections) == 0 {
-		return
-	}
-	connections := c.ourself.Connections()
-	c.Lock()
-	defer c.Unlock()
-	// GC - randomly (courtesy of go's map iterator) pick some
-	// existing entries and stop&remove them if the associated
-	// connection is no longer active.  We stop as soon as we
-	// encounter a valid entry; the idea being that when there is
-	// little or no garbage then this executes close to O(1)[1],
-	// whereas when there is lots of garbage we remove it quickly.
-	//
-	// [1] TODO Unfortunately, due to the desire to avoid nested
-	// locks, instead of simply invoking Peer.ConnectionTo(name)
-	// below, we have that Peer.Connections() invocation above. That
-	// is O(n_our_connections) at best.
-	for conn, sender := range c.senders {
-		if _, found := connections[conn]; !found {
-			delete(c.senders, conn)
-			sender.Stop()
-		} else {
-			break
-		}
-	}
-	for conn := range selectedConnections {
-		c.sendDown(conn, data)
-	}
-}
-
-func (c *GossipChannel) SendDown(conn Connection, data GossipData) {
-	c.Lock()
-	c.sendDown(conn, data)
-	c.Unlock()
-}
-
-func (c *GossipChannel) sendDown(conn Connection, data GossipData) {
-	sender, found := c.senders[conn]
-	if !found {
-		sender = NewGossipSender(func(pending GossipData) {
-			protocolMsg := ProtocolMsg{ProtocolGossip, GobEncode(c.hash, c.ourself.Name, pending.Encode())}
-			conn.(ProtocolSender).SendProtocolMsg(protocolMsg)
-		})
-		c.senders[conn] = sender
-		sender.Start()
-	}
-	sender.Send(data)
-}
-
-func (c *GossipChannel) GossipUnicast(dstPeerName PeerName, msg []byte) error {
-	return c.relayUnicast(dstPeerName, GobEncode(c.hash, c.ourself.Name, dstPeerName, msg))
-}
-
-func (c *GossipChannel) GossipBroadcast(update GossipData) error {
-	return c.relayBroadcast(c.ourself.Name, update)
-}
-
-func (c *GossipChannel) relayUnicast(dstPeerName PeerName, buf []byte) error {
-	if relayPeerName, found := c.routes.UnicastAll(dstPeerName); !found {
-		c.log("unknown relay destination:", dstPeerName)
-	} else if conn, found := c.ourself.ConnectionTo(relayPeerName); !found {
-		c.log("unable to find connection to relay peer", relayPeerName)
-	} else {
-		conn.(ProtocolSender).SendProtocolMsg(ProtocolMsg{ProtocolGossipUnicast, buf})
-	}
-	return nil
-}
-
-func (c *GossipChannel) relayBroadcast(srcName PeerName, update GossipData) error {
-	names := c.routes.PeerNames() // do this outside the lock so they don't nest
-	c.Lock()
-	defer c.Unlock()
-	// GC - randomly (courtesy of go's map iterator) pick some
-	// existing broadcasters and stop&remove them if their source peer
-	// is unknown. We stop as soon as we encounter a valid entry; the
-	// idea being that when there is little or no garbage then this
-	// executes close to O(1)[1], whereas when there is lots of
-	// garbage we remove it quickly.
-	//
-	// [1] TODO Unfortunately, due to the desire to avoid nested
-	// locks, instead of simply invoking Peers.Fetch(name) below, we
-	// have that Peers.Names() invocation above. That is O(n_peers) at
-	// best.
-	for name, broadcaster := range c.broadcasters {
-		if _, found := names[name]; !found {
-			delete(c.broadcasters, name)
-			broadcaster.Stop()
-		} else {
-			break
-		}
-	}
-	broadcaster, found := c.broadcasters[srcName]
-	if !found {
-		broadcaster = NewGossipSender(func(pending GossipData) { c.sendBroadcast(srcName, pending) })
-		c.broadcasters[srcName] = broadcaster
-		broadcaster.Start()
-	}
-	broadcaster.Send(update)
-	return nil
-}
-
-func (c *GossipChannel) sendBroadcast(srcName PeerName, update GossipData) {
-	c.routes.EnsureRecalculated()
-	nextHops := c.routes.BroadcastAll(srcName)
-	if len(nextHops) == 0 {
-		return
-	}
-	protocolMsg := ProtocolMsg{ProtocolGossipBroadcast, GobEncode(c.hash, srcName, update.Encode())}
-	// FIXME a single blocked connection can stall us
-	for _, conn := range c.ourself.ConnectionsTo(nextHops) {
-		conn.(ProtocolSender).SendProtocolMsg(protocolMsg)
-	}
-}
-
-func (c *GossipChannel) log(args ...interface{}) {
-	log.Println(append(append([]interface{}{}, "[gossip "+c.name+"]:"), args...)...)
-}
-
 // for testing
 
-// FIXME this doesn't actually guarantee everything has been sent
-// since a GossipSender may be in the process of sending and there is
-// no easy way for us to know when that has completed.
-func (router *Router) sendPendingGossip() {
-	for _, channel := range router.GossipChannels {
+func (router *Router) sendPendingGossip() bool {
+	sentSomething := false
+	for channel := range router.gossipChannelSet() {
+		channel.Lock()
 		for _, sender := range channel.senders {
-			sender.flush()
+			sentSomething = sender.flush() || sentSomething
 		}
 		for _, sender := range channel.broadcasters {
-			sender.flush()
+			sentSomething = sender.flush() || sentSomething
 		}
+		channel.Unlock()
 	}
+	return sentSomething
 }
 
-func (sender *GossipSender) flush() {
-	for {
-		select {
-		case pending := <-sender.cell:
-			sender.send(pending)
-		default:
-			return
-		}
-	}
+func (sender *GossipSender) flush() bool {
+	ch := make(chan bool)
+	sender.flushch <- ch
+	return <-ch
 }

router/gossip_channel.go

@@ -0,0 +1,225 @@
+package router
+
+import (
+	"bytes"
+	"encoding/gob"
+	"log"
+	"sync"
+)
+
+type GossipChannel struct {
+	sync.Mutex
+	name         string
+	ourself      *LocalPeer
+	routes       *Routes
+	gossiper     Gossiper
+	senders      connectionSenders
+	broadcasters peerSenders
+}
+
+type connectionSenders map[Connection]*GossipSender
+type peerSenders map[PeerName]*GossipSender
+
+func NewGossipChannel(channelName string, ourself *LocalPeer, routes *Routes, g Gossiper) *GossipChannel {
+	return &GossipChannel{
+		name:         channelName,
+		ourself:      ourself,
+		routes:       routes,
+		gossiper:     g,
+		senders:      make(connectionSenders),
+		broadcasters: make(peerSenders)}
+}
+
+func (router *Router) handleGossip(tag ProtocolTag, payload []byte) error {
+	decoder := gob.NewDecoder(bytes.NewReader(payload))
+	var channelName string
+	if err := decoder.Decode(&channelName); err != nil {
+		return err
+	}
+	channel := router.gossipChannel(channelName)
+	var srcName PeerName
+	if err := decoder.Decode(&srcName); err != nil {
+		return err
+	}
+	switch tag {
+	case ProtocolGossipUnicast:
+		return channel.deliverUnicast(srcName, payload, decoder)
+	case ProtocolGossipBroadcast:
+		return channel.deliverBroadcast(srcName, payload, decoder)
+	case ProtocolGossip:
+		return channel.deliver(srcName, payload, decoder)
+	}
+	return nil
+}
+
+func (c *GossipChannel) deliverUnicast(srcName PeerName, origPayload []byte, dec *gob.Decoder) error {
+	var destName PeerName
+	if err := dec.Decode(&destName); err != nil {
+		return err
+	}
+	if c.ourself.Name != destName {
+		return c.relayUnicast(destName, origPayload)
+	}
+	var payload []byte
+	if err := dec.Decode(&payload); err != nil {
+		return err
+	}
+	return c.gossiper.OnGossipUnicast(srcName, payload)
+}
+
+func (c *GossipChannel) deliverBroadcast(srcName PeerName, _ []byte, dec *gob.Decoder) error {
+	var payload []byte
+	if err := dec.Decode(&payload); err != nil {
+		return err
+	}
+	data, err := c.gossiper.OnGossipBroadcast(payload)
+	if err != nil || data == nil {
+		return err
+	}
+	return c.relayBroadcast(srcName, data)
+}
+
+func (c *GossipChannel) deliver(srcName PeerName, _ []byte, dec *gob.Decoder) error {
+	var payload []byte
+	if err := dec.Decode(&payload); err != nil {
+		return err
+	}
+	if data, err := c.gossiper.OnGossip(payload); err != nil {
+		return err
+	} else if data != nil {
+		c.Send(srcName, data)
+	}
+	return nil
+}
+
+func (c *GossipChannel) Send(srcName PeerName, data GossipData) {
+	// do this outside the lock below so we avoid lock nesting
+	c.routes.EnsureRecalculated()
+	selectedConnections := make(ConnectionSet)
+	for name := range c.routes.RandomNeighbours(srcName) {
+		if conn, found := c.ourself.ConnectionTo(name); found {
+			selectedConnections[conn] = void
+		}
+	}
+	if len(selectedConnections) == 0 {
+		return
+	}
+	connections := c.ourself.Connections()
+	c.Lock()
+	defer c.Unlock()
+	// GC - randomly (courtesy of go's map iterator) pick some
+	// existing entries and stop&remove them if the associated
+	// connection is no longer active.  We stop as soon as we
+	// encounter a valid entry; the idea being that when there is
+	// little or no garbage then this executes close to O(1)[1],
+	// whereas when there is lots of garbage we remove it quickly.
+	//
+	// [1] TODO Unfortunately, due to the desire to avoid nested
+	// locks, instead of simply invoking Peer.ConnectionTo(name)
+	// below, we have that Peer.Connections() invocation above. That
+	// is O(n_our_connections) at best.
+	for conn, sender := range c.senders {
+		if _, found := connections[conn]; !found {
+			delete(c.senders, conn)
+			sender.Stop()
+		} else {
+			break
+		}
+	}
+	for conn := range selectedConnections {
+		c.sendDown(conn, data)
+	}
+}
+
+func (c *GossipChannel) SendDown(conn Connection, data GossipData) {
+	c.Lock()
+	c.sendDown(conn, data)
+	c.Unlock()
+}
+
+func (c *GossipChannel) sendDown(conn Connection, data GossipData) {
+	sender, found := c.senders[conn]
+	if !found {
+		sender = NewGossipSender(func(pending GossipData) {
+			for _, msg := range pending.Encode() {
+				protocolMsg := ProtocolMsg{ProtocolGossip, GobEncode(c.name, c.ourself.Name, msg)}
+				conn.(ProtocolSender).SendProtocolMsg(protocolMsg)
+			}
+		})
+		c.senders[conn] = sender
+		sender.Start()
+	}
+	sender.Send(data)
+}
+
+func (c *GossipChannel) GossipUnicast(dstPeerName PeerName, msg []byte) error {
+	return c.relayUnicast(dstPeerName, GobEncode(c.name, c.ourself.Name, dstPeerName, msg))
+}
+
+func (c *GossipChannel) GossipBroadcast(update GossipData) error {
+	return c.relayBroadcast(c.ourself.Name, update)
+}
+
+func (c *GossipChannel) relayUnicast(dstPeerName PeerName, buf []byte) error {
+	if relayPeerName, found := c.routes.UnicastAll(dstPeerName); !found {
+		c.log("unknown relay destination:", dstPeerName)
+	} else if conn, found := c.ourself.ConnectionTo(relayPeerName); !found {
+		c.log("unable to find connection to relay peer", relayPeerName)
+	} else {
+		conn.(ProtocolSender).SendProtocolMsg(ProtocolMsg{ProtocolGossipUnicast, buf})
+	}
+	return nil
+}
+
+func (c *GossipChannel) relayBroadcast(srcName PeerName, update GossipData) error {
+	names := c.routes.PeerNames() // do this outside the lock so they don't nest
+	c.Lock()
+	defer c.Unlock()
+	// GC - randomly (courtesy of go's map iterator) pick some
+	// existing broadcasters and stop&remove them if their source peer
+	// is unknown. We stop as soon as we encounter a valid entry; the
+	// idea being that when there is little or no garbage then this
+	// executes close to O(1)[1], whereas when there is lots of
+	// garbage we remove it quickly.
+	//
+	// [1] TODO Unfortunately, due to the desire to avoid nested
+	// locks, instead of simply invoking Peers.Fetch(name) below, we
+	// have that Peers.Names() invocation above. That is O(n_peers) at
+	// best.
+	for name, broadcaster := range c.broadcasters {
+		if _, found := names[name]; !found {
+			delete(c.broadcasters, name)
+			broadcaster.Stop()
+		} else {
+			break
+		}
+	}
+	broadcaster, found := c.broadcasters[srcName]
+	if !found {
+		broadcaster = NewGossipSender(func(pending GossipData) { c.sendBroadcast(srcName, pending) })
+		c.broadcasters[srcName] = broadcaster
+		broadcaster.Start()
+	}
+	broadcaster.Send(update)
+	return nil
+}
+
+func (c *GossipChannel) sendBroadcast(srcName PeerName, update GossipData) {
+	c.routes.EnsureRecalculated()
+	nextHops := c.routes.BroadcastAll(srcName)
+	if len(nextHops) == 0 {
+		return
+	}
+	connections := c.ourself.ConnectionsTo(nextHops)
+	for _, msg := range update.Encode() {
+		protocolMsg := ProtocolMsg{ProtocolGossipBroadcast, GobEncode(c.name, srcName, msg)}
+		// FIXME a single blocked connection can stall us
+		for _, conn := range connections {
+			conn.(ProtocolSender).SendProtocolMsg(protocolMsg)
+		}
+	}
+}
+
+func (c *GossipChannel) log(args ...interface{}) {
+	log.Println(append(append([]interface{}{}, "[gossip "+c.name+"]:"), args...)...)
+}

router/gossip_test.go

@@ -1,13 +1,15 @@
 package router
 
 import (
-	wt "github.com/weaveworks/weave/testing"
+	"sync"
 	"testing"
 	"time"
+
+	wt "github.com/weaveworks/weave/testing"
 )
 
-// TODO test gossip unicast and broadcast; atm we only test topology
-// gossip, which does not employ unicast or broadcast.
+// TODO test gossip unicast; atm we only test topology gossip and
+// surrogates, neither of which employ unicast.
 
 type mockChannelConnection struct {
 	RemoteConnection
@@ -16,8 +18,9 @@ type mockChannelConnection struct {
 
 // Construct a "passive" Router, i.e. without any goroutines, except
 // for Routes and GossipSenders.
-func NewTestRouter(name PeerName) *Router {
-	router := NewRouter(RouterConfig{}, name, "")
+func NewTestRouter(name string) *Router {
+	peerName, _ := PeerNameFromString(name)
+	router := NewRouter(Config{}, peerName, "")
 	// need to create a dummy channel otherwise tests hang on nil
 	// channels when the Router invoked ConnectionMaker.Refresh
 	router.ConnectionMaker.actionChan = make(chan ConnectionMakerAction, ChannelSize)
@@ -29,7 +32,16 @@ func (conn *mockChannelConnection) SendProtocolMsg(protocolMsg ProtocolMsg) {
 	if err := conn.dest.handleGossip(protocolMsg.tag, protocolMsg.msg); err != nil {
 		panic(err)
 	}
-	conn.dest.sendPendingGossip()
+}
+
+func sendPendingGossip(routers ...*Router) {
+	// Loop until all routers report they didn't send anything
+	for sentSomething := true; sentSomething; {
+		sentSomething = false
+		for _, router := range routers {
+			sentSomething = router.sendPendingGossip() || sentSomething
+		}
+	}
 }
 
 func (router *Router) AddTestChannelConnection(r *Router) {
@@ -45,7 +57,6 @@ func (router *Router) AddTestChannelConnection(r *Router) {
 	conn := &mockChannelConnection{RemoteConnection{router.Ourself.Peer, toPeer, "", false, true}, r}
 	router.Ourself.handleAddConnection(conn)
 	router.Ourself.handleConnectionEstablished(conn)
-	router.sendPendingGossip()
 }
 
 func (router *Router) DeleteTestChannelConnection(r *Router) {
@@ -60,11 +71,10 @@ func (router *Router) DeleteTestChannelConnection(r *Router) {
 
 	conn, _ := router.Ourself.ConnectionTo(toName)
 	router.Ourself.handleDeleteConnection(conn)
-	router.sendPendingGossip()
 }
 
 func TestGossipTopology(t *testing.T) {
-	wt.RunWithTimeout(t, 1*time.Second, func() {
+	wt.RunWithTimeout(t, 5*time.Second, func() {
 		implTestGossipTopology(t)
 	})
 }
@@ -86,17 +96,16 @@ func (router *Router) tp(routers ...*Router) *Peer {
 
 // Check that the topology of router matches the peers and all of their connections
 func checkTopology(t *testing.T, router *Router, wantedPeers ...*Peer) {
+	router.Peers.RLock()
 	checkTopologyPeers(t, true, router.Peers.allPeers(), wantedPeers...)
+	router.Peers.RUnlock()
 }
 
 func implTestGossipTopology(t *testing.T) {
 	// Create some peers that will talk to each other
-	peer1Name, _ := PeerNameFromString("01:00:00:01:00:00")
-	peer2Name, _ := PeerNameFromString("02:00:00:02:00:00")
-	peer3Name, _ := PeerNameFromString("03:00:00:03:00:00")
-	r1 := NewTestRouter(peer1Name)
-	r2 := NewTestRouter(peer2Name)
-	r3 := NewTestRouter(peer3Name)
+	r1 := NewTestRouter("01:00:00:01:00:00")
+	r2 := NewTestRouter("02:00:00:02:00:00")
+	r3 := NewTestRouter("03:00:00:03:00:00")
 
 	// Check state when they have no connections
 	checkTopology(t, r1, r1.tp())
@@ -104,14 +113,17 @@ func implTestGossipTopology(t *testing.T) {
 
 	// Now try adding some connections
 	r1.AddTestChannelConnection(r2)
+	sendPendingGossip(r1, r2)
 	checkTopology(t, r1, r1.tp(r2), r2.tp())
 	checkTopology(t, r2, r1.tp(r2), r2.tp())
 	r2.AddTestChannelConnection(r1)
+	sendPendingGossip(r1, r2)
 	checkTopology(t, r1, r1.tp(r2), r2.tp(r1))
 	checkTopology(t, r2, r1.tp(r2), r2.tp(r1))
 
 	// Currently, the connection from 2 to 3 is one-way only
 	r2.AddTestChannelConnection(r3)
+	sendPendingGossip(r1, r2, r3)
 	checkTopology(t, r1, r1.tp(r2), r2.tp(r1, r3), r3.tp())
 	checkTopology(t, r2, r1.tp(r2), r2.tp(r1, r3), r3.tp())
 	// When r2 gossiped to r3, 1 was unreachable from r3 so it got removed from the
@@ -120,23 +132,27 @@ func implTestGossipTopology(t *testing.T) {
 
 	// Add a connection from 3 to 1 and now r1 is reachable.
 	r3.AddTestChannelConnection(r1)
+	sendPendingGossip(r1, r2, r3)
 	checkTopology(t, r1, r1.tp(r2), r2.tp(r1, r3), r3.tp(r1))
 	checkTopology(t, r2, r1.tp(r2), r2.tp(r1, r3), r3.tp(r1))
 	checkTopology(t, r3, r1.tp(), r2.tp(r1, r3), r3.tp(r1))
 
 	r1.AddTestChannelConnection(r3)
+	sendPendingGossip(r1, r2, r3)
 	checkTopology(t, r1, r1.tp(r2, r3), r2.tp(r1, r3), r3.tp(r1))
 	checkTopology(t, r2, r1.tp(r2, r3), r2.tp(r1, r3), r3.tp(r1))
 	checkTopology(t, r3, r1.tp(r2, r3), r2.tp(r1, r3), r3.tp(r1))
 
 	// Drop the connection from 2 to 3
 	r2.DeleteTestChannelConnection(r3)
+	sendPendingGossip(r1, r2, r3)
 	checkTopology(t, r1, r1.tp(r2, r3), r2.tp(r1), r3.tp(r1))
 	checkTopology(t, r2, r1.tp(r2, r3), r2.tp(r1))
 	checkTopology(t, r3, r1.tp(r2, r3), r2.tp(r1), r3.tp(r1))
 
 	// Drop the connection from 1 to 3
 	r1.DeleteTestChannelConnection(r3)
+	sendPendingGossip(r1, r2, r3)
 	checkTopology(t, r1, r1.tp(r2), r2.tp(r1), r3.tp(r1))
 
 	checkTopology(t, r1, r1.tp(r2), r2.tp(r1), r3.tp(r1))
@@ -146,5 +162,104 @@ func implTestGossipTopology(t *testing.T) {
 
 	// On a timer, r3 will gossip to r1
 	r3.SendAllGossip()
+	sendPendingGossip(r1, r2, r3)
 	checkTopology(t, r1, r1.tp(r2), r2.tp(r1), r3.tp(r1))
 }
+
+func TestGossipSurrogate(t *testing.T) {
+	// create the topology r1 <-> r2 <-> r3
+	r1 := NewTestRouter("01:00:00:01:00:00")
+	r2 := NewTestRouter("02:00:00:02:00:00")
+	r3 := NewTestRouter("03:00:00:03:00:00")
+	r1.AddTestChannelConnection(r2)
+	r2.AddTestChannelConnection(r1)
+	r3.AddTestChannelConnection(r2)
+	r2.AddTestChannelConnection(r3)
+	sendPendingGossip(r1, r2, r3)
+	checkTopology(t, r1, r1.tp(r2), r2.tp(r1, r3), r3.tp(r2))
+	checkTopology(t, r2, r1.tp(r2), r2.tp(r1, r3), r3.tp(r2))
+	checkTopology(t, r3, r1.tp(r2), r2.tp(r1, r3), r3.tp(r2))
+
+	// create a gossiper at either end, but not the middle
+	g1 := newTestGossiper()
+	g3 := newTestGossiper()
+	s1 := r1.NewGossip("Test", g1)
+	s3 := r3.NewGossip("Test", g3)
+
+	// broadcast a message from each end, check it reaches the other
+	broadcast(s1, 1)
+	broadcast(s3, 2)
+	sendPendingGossip(r1, r2, r3)
+	g1.checkHas(t, 2)
+	g3.checkHas(t, 1)
+
+	// check that each end gets their message back through periodic
+	// gossip
+	r1.SendAllGossip()
+	r3.SendAllGossip()
+	sendPendingGossip(r1, r2, r3)
+	g1.checkHas(t, 1, 2)
+	g3.checkHas(t, 1, 2)
+}
+
+type testGossiper struct {
+	sync.RWMutex
+	state map[byte]struct{}
+}
+
+func newTestGossiper() *testGossiper {
+	return &testGossiper{state: make(map[byte]struct{})}
+}
+
+func (g *testGossiper) OnGossipUnicast(sender PeerName, msg []byte) error {
+	return nil
+}
+
+func (g *testGossiper) OnGossipBroadcast(update []byte) (GossipData, error) {
+	g.Lock()
+	defer g.Unlock()
+	for _, v := range update {
+		g.state[v] = void
+	}
+	return NewSurrogateGossipData(update), nil
+}
+
+func (g *testGossiper) Gossip() GossipData {
+	g.RLock()
+	defer g.RUnlock()
+	state := make([]byte, len(g.state))
+	for v := range g.state {
+		state = append(state, v)
+	}
+	return NewSurrogateGossipData(state)
+}
+
+func (g *testGossiper) OnGossip(update []byte) (GossipData, error) {
+	g.Lock()
+	defer g.Unlock()
+	var delta []byte
+	for _, v := range update {
+		if _, found := g.state[v]; !found {
+			delta = append(delta, v)
+			g.state[v] = void
+		}
+	}
+	if len(delta) > 0 {
+		return NewSurrogateGossipData(delta), nil
+	}
+	return nil, nil
+}
+
+func (g *testGossiper) checkHas(t *testing.T, vs ...byte) {
+	g.RLock()
+	defer g.RUnlock()
+	for _, v := range vs {
+		if _, found := g.state[v]; !found {
+			wt.Fatalf(t, "%d is missing", v)
+		}
+	}
+}
+
+func broadcast(s Gossip, v byte) {
+	s.GossipBroadcast(NewSurrogateGossipData([]byte{v}))
+}

router/handshake.go

@@ -5,6 +5,7 @@ import (
 	"encoding/hex"
 	"fmt"
 	"strconv"
+	"time"
 )
 
 type FieldValidator struct {
@@ -44,58 +45,68 @@ func (fv *FieldValidator) Err() error {
 	return fv.err
 }
 
-func (conn *LocalConnection) handshake(enc *gob.Encoder, dec *gob.Decoder, acceptNewPeer bool) error {
+func (conn *LocalConnection) handshake(acceptNewPeer bool) (*gob.Encoder, *gob.Decoder, error) {
 	// We do not need to worry about locking in here as at this point
 	// the connection is not reachable by any go-routine other than
 	// ourself. Only when we add this connection to the conn.local
 	// peer will it be visible from multiple go-routines.
 
+	conn.TCPConn.SetDeadline(time.Now().Add(HeaderTimeout))
+	version, err := exchangeProtocolHeader(conn.TCPConn, conn.TCPConn)
+	if err != nil {
+		return nil, nil, err
+	}
+	conn.version = version
+	conn.TCPConn.SetDeadline(time.Time{})
 	conn.extendReadDeadline()
 
 	localConnID := randUint64()
 	usingPassword := conn.Router.UsingPassword()
-	fv, private, err := conn.handshakeSendRecv(localConnID, usingPassword, enc, dec)
+	enc := gob.NewEncoder(conn.TCPConn)
+	dec := gob.NewDecoder(conn.TCPConn)
+	fv, private, err := conn.exchangeHandshake(localConnID, usingPassword, enc, dec)
 	if err != nil {
-		return err
+		return nil, nil, err
 	}
+
 	nameStr, _ := fv.Value("Name")
 	nickNameStr, _ := fv.Value("NickName")
 	uidStr, _ := fv.Value("UID")
 	remoteConnIDStr, _ := fv.Value("ConnID")
 	if err := fv.Err(); err != nil {
-		return err
+		return nil, nil, err
 	}
 
 	name, err := PeerNameFromString(nameStr)
 	if err != nil {
-		return err
+		return nil, nil, err
 	}
 	if !acceptNewPeer {
 		if _, found := conn.Router.Peers.Fetch(name); !found {
-			return fmt.Errorf("Found unknown remote name: %s at %s", name, conn.remoteTCPAddr)
+			return nil, nil, fmt.Errorf("Found unknown remote name: %s at %s", name, conn.remoteTCPAddr)
 		}
 	}
 	if existingConn, found := conn.Router.Ourself.ConnectionTo(name); found && existingConn.Established() {
-		return fmt.Errorf("Already have connection to %s at %s", existingConn.Remote(), existingConn.RemoteTCPAddr())
+		return nil, nil, fmt.Errorf("Already have connection to %s at %s", existingConn.Remote(), existingConn.RemoteTCPAddr())
 	}
 	uid, err := ParsePeerUID(uidStr)
 	if err != nil {
-		return err
+		return nil, nil, err
 	}
 	remoteConnID, err := strconv.ParseUint(remoteConnIDStr, 10, 64)
 	if err != nil {
-		return err
+		return nil, nil, err
 	}
 	conn.uid = localConnID ^ remoteConnID
 
 	remotePublicStr, rpErr := fv.Value("PublicKey")
 	if usingPassword {
 		if rpErr != nil {
-			return rpErr
+			return nil, nil, rpErr
 		}
 		remotePublicSlice, rpErr := hex.DecodeString(remotePublicStr)
 		if rpErr != nil {
-			return rpErr
+			return nil, nil, rpErr
 		}
 		remotePublic := [32]byte{}
 		for idx, elem := range remotePublicSlice {
@@ -106,27 +117,23 @@ func (conn *LocalConnection) handshake(enc *gob.Encoder, dec *gob.Decoder, accep
 		conn.Decryptor = NewNaClDecryptor(conn.SessionKey, conn.outbound)
 	} else {
 		if rpErr == nil {
-			return fmt.Errorf("Remote network is encrypted. Password required.")
+			return nil, nil, fmt.Errorf("Remote network is encrypted. Password required.")
 		}
 		conn.tcpSender = NewSimpleTCPSender(enc)
 		conn.Decryptor = NewNonDecryptor()
 	}
 
-	return conn.setRemote(NewPeer(name, nickNameStr, uid, 0))
+	return enc, dec, conn.setRemote(NewPeer(name, nickNameStr, uid, 0))
 }
 
-func (conn *LocalConnection) handshakeSendRecv(localConnID uint64, usingPassword bool, enc *gob.Encoder, dec *gob.Decoder) (*FieldValidator, *[32]byte, error) {
-	versionStr := fmt.Sprint(ProtocolVersion)
+func (conn *LocalConnection) exchangeHandshake(localConnID uint64, usingPassword bool, enc *gob.Encoder, dec *gob.Decoder) (*FieldValidator, *[32]byte, error) {
 	handshakeSend := map[string]string{
-		"Protocol":        Protocol,
-		"ProtocolVersion": versionStr,
 		"PeerNameFlavour": PeerNameFlavour,
 		"Name":            conn.local.Name.String(),
 		"NickName":        conn.local.NickName,
 		"UID":             fmt.Sprint(conn.local.UID),
 		"ConnID":          fmt.Sprint(localConnID)}
 	handshakeRecv := map[string]string{}
-
 	var public, private *[32]byte
 	var err error
 	if usingPassword {
@@ -137,14 +144,10 @@ func (conn *LocalConnection) handshakeSendRecv(localConnID uint64, usingPassword
 		handshakeSend["PublicKey"] = hex.EncodeToString(public[:])
 	}
 	enc.Encode(handshakeSend)
-
-	err = dec.Decode(&handshakeRecv)
-	if err != nil {
+	if err = dec.Decode(&handshakeRecv); err != nil {
 		return nil, nil, err
 	}
 	fv := NewFieldValidator(handshakeRecv)
-	fv.CheckEqual("Protocol", Protocol)
-	fv.CheckEqual("ProtocolVersion", versionStr)
 	fv.CheckEqual("PeerNameFlavour", PeerNameFlavour)
 	return fv, private, nil
 }

router/json.go

@@ -6,18 +6,19 @@ import (
 	"time"
 )
 
-func (router *Router) StatusJSON(version, encryption string) ([]byte, error) {
+func (router *Router) StatusJSON(version string) ([]byte, error) {
 	return json.Marshal(struct {
-		Version    string
-		Encryption string
-		Name       string
-		NickName   string
-		Interface  string
-		Macs       *MacCache
-		Peers      *Peers
-		Routes     *Routes
-	}{version, encryption, router.Ourself.Name.String(), router.Ourself.NickName, fmt.Sprintf("%v", router.Iface), router.Macs, router.Peers, router.Routes})
-	// leaving out ConectionMaker due to async complexities
+		Version       string
+		Encryption    bool
+		PeerDiscovery bool
+		Name          string
+		NickName      string
+		Interface     string
+		Macs          *MacCache
+		Peers         *Peers
+		Routes        *Routes
+		ConnectionMakerStatus
+	}{version, router.UsingPassword(), router.PeerDiscovery, router.Ourself.Name.String(), router.Ourself.NickName, fmt.Sprintf("%v", router.Iface), router.Macs, router.Peers, router.Routes, router.ConnectionMaker.Status()})
 }
 
 func (cache *MacCache) MarshalJSON() ([]byte, error) {
@@ -96,3 +97,20 @@ func (conn *RemoteConnection) MarshalJSON() ([]byte, error) {
 func (name PeerName) MarshalJSON() ([]byte, error) {
 	return json.Marshal(name.String())
 }
+
+func (target Target) MarshalJSON() ([]byte, error) {
+	t := struct {
+		Attempting  bool      `json:"Attempting,omitempty"`
+		TryAfter    time.Time `json:"TryAfter,omitempty"`
+		TryInterval string    `json:"TryInterval,omitempty"`
+		LastError   string    `json:"LastError,omitempty"`
+	}{
+		Attempting:  target.attempting,
+		TryAfter:    target.tryAfter,
+		TryInterval: target.tryInterval.String(),
+	}
+	if target.lastError != nil {
+		t.LastError = target.lastError.Error()
+	}
+	return json.Marshal(t)
+}

router/pcap.go

@@ -70,9 +70,9 @@ func newPcapIO(ifName string, promisc bool, snaplen int, bufSz int) (handle *Pca
 	return &PcapIO{handle: active}, nil
 }
 
-func (pi *PcapIO) ReadPacket() (data []byte, err error) {
+func (pio *PcapIO) ReadPacket() (data []byte, err error) {
 	for {
-		data, _, err = pi.handle.ZeroCopyReadPacketData()
+		data, _, err = pio.handle.ZeroCopyReadPacketData()
 		if err == nil || err != pcap.NextErrorTimeoutExpired {
 			break
 		}
@@ -80,6 +80,6 @@ func (pi *PcapIO) ReadPacket() (data []byte, err error) {
 	return
 }
 
-func (po *PcapIO) WritePacket(data []byte) error {
-	return po.handle.WritePacketData(data)
+func (pio *PcapIO) WritePacket(data []byte) error {
+	return pio.handle.WritePacketData(data)
 }

router/peers.go

@@ -132,15 +132,13 @@ func (peers *Peers) EncodePeers(names PeerNameSet) []byte {
 	enc := gob.NewEncoder(buf)
 	peers.RLock()
 	defer peers.RUnlock()
-	peerList := make([]*Peer, 0, len(names))
 	for name := range names {
 		if peer, found := peers.table[name]; found {
 			if peer == peers.ourself.Peer {
-				peers.ourself.Peer.Encode(enc)
+				peers.ourself.Encode(enc)
 			} else {
 				peer.Encode(enc)
 			}
-			peerList = append(peerList, peer)
 		}
 	}
 	return buf.Bytes()
@@ -208,9 +206,7 @@ func (peers *Peers) decodeUpdate(update []byte) (newPeers map[PeerName]*Peer, de
 	decodedUpdate = []*Peer{}
 	decodedConns = [][]ConnectionSummary{}
 
-	updateBuf := new(bytes.Buffer)
-	updateBuf.Write(update)
-	decoder := gob.NewDecoder(updateBuf)
+	decoder := gob.NewDecoder(bytes.NewReader(update))
 
 	for {
 		peerSummary, connSummaries, decErr := decodePeer(decoder)
@@ -295,7 +291,6 @@ func (peer *Peer) Encode(enc *gob.Encoder) {
 			conn.Remote().NameByte,
 			conn.RemoteTCPAddr(),
 			conn.Outbound(),
-			// DANGER holding rlock on peer, going to take rlock on conn
 			conn.Established(),
 		})
 	}

router/protocol.go

@@ -1,17 +1,65 @@
 package router
 
-const (
-	Protocol        = "weave"
-	ProtocolVersion = 17
+import (
+	"bytes"
+	"fmt"
+	"io"
 )
 
+const (
+	Protocol                = "weave"
+	ProtocolVersion    byte = 1
+	ProtocolMinVersion byte = ProtocolVersion
+)
+
+var (
+	ProtocolBytes     = []byte(Protocol)
+	ProtocolLen       = len(ProtocolBytes)
+	ProtocolHeader    = append(ProtocolBytes, ProtocolMinVersion, ProtocolVersion)
+	ProtocolHeaderLen = len(ProtocolHeader)
+)
+
+func exchangeProtocolHeader(w io.Writer, r io.Reader) (byte, error) {
+	if n, err := w.Write(ProtocolHeader); err != nil {
+		return 0, err
+	} else if n != ProtocolHeaderLen {
+		return 0, fmt.Errorf("failed to send complete protocol header")
+	}
+	header := make([]byte, ProtocolHeaderLen)
+	if n, err := io.ReadFull(r, header); err != nil && n == 0 {
+		return 0, fmt.Errorf("failed to receive remote protocol header: %s", err)
+	} else if err != nil {
+		return 0, fmt.Errorf("received incomplete remote protocol header (%d octets instead of %d): %v; error: %s",
+			n, ProtocolHeaderLen, header[:n], err)
+	}
+	if !bytes.Equal(ProtocolBytes, header[:ProtocolLen]) {
+		return 0, fmt.Errorf("remote protocol header not recognised: %v", header[:ProtocolHeaderLen])
+	}
+	var (
+		remoteMinVersion = header[ProtocolLen]
+		remoteVersion    = header[ProtocolLen+1]
+	)
+	minVersion := ProtocolMinVersion
+	if remoteMinVersion > minVersion {
+		minVersion = remoteMinVersion
+	}
+	maxVersion := ProtocolVersion
+	if remoteVersion < maxVersion {
+		maxVersion = remoteVersion
+	}
+	if minVersion > maxVersion {
+		return 0, fmt.Errorf("remote version [%d,%d] is incompatible with our version [%d,%d]",
+			remoteMinVersion, remoteVersion, ProtocolMinVersion, ProtocolVersion)
+	}
+	return maxVersion, nil
+}
+
 type ProtocolTag byte
 
 const (
 	ProtocolHeartbeat ProtocolTag = iota
 	ProtocolConnectionEstablished
 	ProtocolFragmentationReceived
-	ProtocolStartFragmentationTest
 	ProtocolPMTUVerified
 	ProtocolGossip
 	ProtocolGossipUnicast

router/router.go

@@ -2,40 +2,52 @@ package router
 
 import (
 	"bytes"
-	"code.google.com/p/gopacket/layers"
 	"encoding/binary"
 	"fmt"
 	"io"
 	"log"
 	"net"
+	"sync"
 	"syscall"
 	"time"
+
+	"code.google.com/p/gopacket/layers"
+	. "github.com/weaveworks/weave/common"
 )
 
-const macMaxAge = 10 * time.Minute // [1]
+const (
+	macMaxAge      = 10 * time.Minute     // [1]
+	tcpAcceptDelay = 1 * time.Millisecond // [2]
+)
 
 // [1] should be greater than typical ARP cache expiries, i.e. > 3/2 *
 // /proc/sys/net/ipv4_neigh/*/base_reachable_time_ms on Linux
 
+// [2] time to wait between accepting tcp connections. This guards
+// against brute-force attacks on the password when encryption is in
+// use. It is also a basic DoS defence.
+
 type LogFrameFunc func(string, []byte, *layers.Ethernet)
 
-type RouterConfig struct {
-	Port      int
-	Iface     *net.Interface
-	Password  []byte
-	ConnLimit int
-	BufSz     int
-	LogFrame  LogFrameFunc
+type Config struct {
+	Port          int
+	Iface         *net.Interface
+	Password      []byte
+	ConnLimit     int
+	PeerDiscovery bool
+	BufSz         int
+	LogFrame      LogFrameFunc
 }
 
 type Router struct {
-	RouterConfig
+	Config
 	Ourself         *LocalPeer
 	Macs            *MacCache
 	Peers           *Peers
 	Routes          *Routes
 	ConnectionMaker *ConnectionMaker
-	GossipChannels  map[uint32]*GossipChannel
+	gossipLock      sync.RWMutex
+	gossipChannels  GossipChannels
 	TopologyGossip  Gossip
 	UDPListener     *net.UDPConn
 }
@@ -53,8 +65,8 @@ type PacketSourceSink interface {
 	PacketSink
 }
 
-func NewRouter(config RouterConfig, name PeerName, nickName string) *Router {
-	router := &Router{RouterConfig: config, GossipChannels: make(map[uint32]*GossipChannel)}
+func NewRouter(config Config, name PeerName, nickName string) *Router {
+	router := &Router{Config: config, gossipChannels: make(GossipChannels)}
 	onMacExpiry := func(mac net.HardwareAddr, peer *Peer) {
 		log.Println("Expired MAC", mac, "at", peer)
 	}
@@ -67,7 +79,7 @@ func NewRouter(config RouterConfig, name PeerName, nickName string) *Router {
 	router.Peers = NewPeers(router.Ourself, onPeerGC)
 	router.Peers.FetchWithDefault(router.Ourself.Peer)
 	router.Routes = NewRoutes(router.Ourself, router.Peers)
-	router.ConnectionMaker = NewConnectionMaker(router.Ourself, router.Peers, router.Port)
+	router.ConnectionMaker = NewConnectionMaker(router.Ourself, router.Peers, router.Port, router.PeerDiscovery)
 	router.TopologyGossip = router.NewGossip("topology", router)
 	return router
 }
@@ -106,11 +118,13 @@ func (router *Router) UsingPassword() bool {
 func (router *Router) Status() string {
 	var buf bytes.Buffer
 	fmt.Fprintln(&buf, "Our name is", router.Ourself)
+	fmt.Fprintln(&buf, "Encryption", OnOff(router.UsingPassword()))
+	fmt.Fprintln(&buf, "Peer discovery", OnOff(router.PeerDiscovery))
 	fmt.Fprintln(&buf, "Sniffing traffic on", router.Iface)
 	fmt.Fprintf(&buf, "MACs:\n%s", router.Macs)
 	fmt.Fprintf(&buf, "Peers:\n%s", router.Peers)
 	fmt.Fprintf(&buf, "Routes:\n%s", router.Routes)
-	fmt.Fprintf(&buf, "Reconnects:\n%s", router.ConnectionMaker)
+	fmt.Fprint(&buf, router.ConnectionMaker.Status())
 	return buf.String()
 }
 
@@ -198,6 +212,7 @@ func (router *Router) listenTCP(localPort int) {
 				continue
 			}
 			router.acceptTCP(tcpConn)
+			time.Sleep(tcpAcceptDelay)
 		}
 	}()
 }
@@ -379,8 +394,8 @@ func (d *TopologyGossipData) Merge(other GossipData) {
 	}
 }
 
-func (d *TopologyGossipData) Encode() []byte {
-	return d.peers.EncodePeers(d.update)
+func (d *TopologyGossipData) Encode() [][]byte {
+	return [][]byte{d.peers.EncodePeers(d.update)}
 }
 
 func (router *Router) OnGossipUnicast(sender PeerName, msg []byte) error {

router/surrogate_gossiper.go

@@ -0,0 +1,41 @@
+package router
+
+type SurrogateGossipData struct {
+	messages [][]byte
+}
+
+func NewSurrogateGossipData(msg []byte) *SurrogateGossipData {
+	return &SurrogateGossipData{messages: [][]byte{msg}}
+}
+
+func (d *SurrogateGossipData) Encode() [][]byte {
+	return d.messages
+}
+
+func (d *SurrogateGossipData) Merge(other GossipData) {
+	d.messages = append(d.messages, other.(*SurrogateGossipData).messages...)
+}
+
+// SurrogateGossiper ignores unicasts and relays broadcasts and gossips.
+
+type SurrogateGossiper struct{}
+
+func (*SurrogateGossiper) OnGossipUnicast(sender PeerName, msg []byte) error {
+	return nil
+}
+
+func (*SurrogateGossiper) OnGossipBroadcast(update []byte) (GossipData, error) {
+	return NewSurrogateGossipData(update), nil
+}
+
+func (*SurrogateGossiper) Gossip() GossipData {
+	return nil
+}
+
+func (*SurrogateGossiper) OnGossip(update []byte) (GossipData, error) {
+	return NewSurrogateGossipData(update), nil
+}
+
+var (
+	surrogateGossiper SurrogateGossiper
+)

router/utils.go

@@ -5,7 +5,6 @@ import (
 	"crypto/rand"
 	"encoding/gob"
 	"fmt"
-	"hash/fnv"
 	"log"
 	"net"
 )
@@ -74,12 +73,6 @@ func randUint64() (r uint64) {
 	return
 }
 
-func hash(s string) uint32 {
-	h := fnv.New32a()
-	h.Write([]byte(s))
-	return h.Sum32()
-}
-
 func GobEncode(items ...interface{}) []byte {
 	buf := new(bytes.Buffer)
 	enc := gob.NewEncoder(buf)

site/Gemfile.lock

@@ -0,0 +1,70 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    blankslate (2.1.2.4)
+    celluloid (0.16.0)
+      timers (~> 4.0.0)
+    classifier-reborn (2.0.3)
+      fast-stemmer (~> 1.0)
+    coffee-script (2.3.0)
+      coffee-script-source
+      execjs
+    coffee-script-source (1.9.1)
+    colorator (0.1)
+    execjs (2.3.0)
+    fast-stemmer (1.0.2)
+    ffi (1.9.6)
+    hitimes (1.2.2)
+    jekyll (2.5.3)
+      classifier-reborn (~> 2.0)
+      colorator (~> 0.1)
+      jekyll-coffeescript (~> 1.0)
+      jekyll-gist (~> 1.0)
+      jekyll-paginate (~> 1.0)
+      jekyll-sass-converter (~> 1.0)
+      jekyll-watch (~> 1.1)
+      kramdown (~> 1.3)
+      liquid (~> 2.6.1)
+      mercenary (~> 0.3.3)
+      pygments.rb (~> 0.6.0)
+      redcarpet (~> 3.1)
+      safe_yaml (~> 1.0)
+      toml (~> 0.1.0)
+    jekyll-coffeescript (1.0.1)
+      coffee-script (~> 2.2)
+    jekyll-gist (1.1.0)
+    jekyll-paginate (1.1.0)
+    jekyll-sass-converter (1.3.0)
+      sass (~> 3.2)
+    jekyll-watch (1.2.1)
+      listen (~> 2.7)
+    kramdown (1.5.0)
+    liquid (2.6.2)
+    listen (2.8.5)
+      celluloid (>= 0.15.2)
+      rb-fsevent (>= 0.9.3)
+      rb-inotify (>= 0.9)
+    mercenary (0.3.5)
+    parslet (1.5.0)
+      blankslate (~> 2.0)
+    posix-spawn (0.3.10)
+    pygments.rb (0.6.2)
+      posix-spawn (~> 0.3.6)
+      yajl-ruby (~> 1.2.0)
+    rb-fsevent (0.9.4)
+    rb-inotify (0.9.5)
+      ffi (>= 0.5.0)
+    redcarpet (3.2.2)
+    safe_yaml (1.0.4)
+    sass (3.4.13)
+    timers (4.0.1)
+      hitimes
+    toml (0.1.2)
+      parslet (~> 1.5.0)
+    yajl-ruby (1.2.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  jekyll

site/_config.yml

@@ -8,3 +8,6 @@ exclude:
   - 'Gemfile.lock'
   - 'params.json'
   - 'build-and-publish.sh'
+
+google:
+  analytics: "UA-57286487-1"

site/_includes/footer.html

@@ -0,0 +1,60 @@
+  <!-- FOOTER -->
+<div class="section footer">
+  <div class="container">
+    <div class="row clearfix">
+      <div class="col-md-3 col-md-offset-2 column">
+        <b>Company</b><br>
+        <a href="http://weave.works/about.html">About Weave</a><br>
+        <!-- <a href="#">Meet the Team</a><br> -->
+        <!-- <a href="#">Careers</a><br> -->
+        <a href="http://blog.weave.works">Blog</a><br>
+        <br>
+        <b>Community</b><br>
+        <a href="http://weave.works/help/index.html#partners">Partners</a><br>
+        <a href="http://github.com/weaveworks">Code</a><br>
+      </div>
+      <div class="col-md-3 column">
+        <b>Documentation</b><br>
+        <a href="http://docs.weave.works">Docs</a><br>
+        <a href="http://weave.works/guides/">Getting Started Guides</a><br>
+        <a href="http://weave.works/help/index.html">Help and Support</a><br>
+        <a href="http://blog.weave.works">Blog</a><br>
+      </div>
+      <div class="col-md-3 column">
+        <b>Get in Touch</b><br>
+        <a href="mailto:help@weave.works">help@weave.works</a><br>
+        <a href="http://twitter.com/weaveworks">Twitter</a><br>
+        <a href="http://github.com/weaveworks">Github</a><br>
+        <a href="http://slideshare.net/weaveworks">Slideshare</a><br>
+        <a href="https://www.linkedin.com/company/weaveworks">Linkedin</a><br>
+        <a href="https://plus.google.com/+WeaveWorksInc/">GooglePlus</a><br>
+      </div>
+    </div>
+    <div class="row clearfix weavefooter">
+      <br>
+      <br>
+      <div class="col-md-4 col-md-offset-2">
+        <p class="copyright">&copy; Weaveworks 2014 &dash; 2015</p>
+      </div>
+    </div>
+  </div>
+</div>
+
+<!-- HTML5 shim, for IE6-8 support of HTML5 elements -->
+<!--[if lt IE 9]>
+  <script src="http://weave.works/js/html5shiv.js"></script>
+<![endif]-->
+
+<script type="text/javascript">
+    WebFontConfig = { fontdeck: { id: '56326' } };
+
+    (function() {
+      var wf = document.createElement('script');
+      wf.src = ('https:' == document.location.protocol ? 'https' : 'http') +
+      '://ajax.googleapis.com/ajax/libs/webfont/1/webfont.js';
+      wf.type = 'text/javascript';
+      wf.async = 'true';
+      var s = document.getElementsByTagName('script')[0];
+      s.parentNode.insertBefore(wf, s);
+    })();
+</script>

site/_includes/header-navbaronly.html

@@ -0,0 +1,23 @@
+  <div class="section header navbaronly">
+    <nav class="navbar navbar-inverse">
+      <div class="container">
+          <div><!-- class="navbar-header"> -->
+            <a class="navbar-brand" href="http://weave.works/"><img src="http://weave.works/img/weave-logo.png"></a>
+          </div>
+          <ul class="nav navbar-nav navbar-right">
+            <li>
+              <a href="http://docs.weave.works">Docs</a>
+            </li>
+            <li>
+              <a href="http://github.com/weaveworks">Code</a>
+            </li>
+            <li>
+              <a href="http://blog.weave.works">Blog</a>
+            </li>
+            <li>
+              <a href="http://weave.works/help/index.html">Help &amp; Support</a>
+            </li>
+          </ul>
+      </div>
+    </nav>
+  </div>

site/_includes/tracking.html

@@ -0,0 +1,12 @@
+{% if site.google.analytics %}
+<script>
+  (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
+  (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
+  m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
+  })(window,document,'script','//www.google-analytics.com/analytics.js','ga');
+
+  ga('create', '{{ site.google.analytics }}', 'auto');
+  ga('send', 'pageview');
+
+</script>
+{% endif %}

site/_layouts/default.html

@@ -1,43 +1,46 @@
 <!DOCTYPE html>
 <html>
-
   <head>
     <meta charset='utf-8' />
     <meta http-equiv="X-UA-Compatible" content="chrome=1" />
-    <meta name="description" content="weave : The Docker Network" />
-
-    <link rel="stylesheet" type="text/css" media="screen" href="stylesheets/stylesheet.css">
 
     <title>weave {% if page.title %} | {{ page.title }} {% endif %}</title>
-  </head>
+
+    <meta name="description" content="Weave creates a virtual network that connects Docker containers deployed across multiple hosts and enables their automatic discovery."> 
+    <meta name="keywords" content="docker, networking, microservices, dns, software defined networking, linux, ubuntu, kubernetes, docker deployment, docker management, docker integration, docker mesos">
+    <link rel="author" href="https://plus.google.com/+WeaveWorksInc"/>
+    <link rel="canonical" href="{{ page.url | replace:'index.html','' | prepend: site.url }}">
+
+    <link href="{{ site.url }}/atom.xml" type="application/atom+xml" rel="alternate" title="{{ site.title }} Atom Feed">
+
+    <!-- Mobile Specific Metas
+      –––––––––––––––––––––––––––––––––––––––––––––––––– -->
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+
+     <!-- CSS
+       –––––––––––––––––––––––––––––––––––––––––––––––––– -->
+    <link rel="stylesheet" type="text/css" href="http://weave.works/less/custom.css">
+
+     <!-- Favicon
+       –––––––––––––––––––––––––––––––––––––––––––––––––– -->
+    <link rel="shortcut icon" type="image/png" href="http://weave.works/img/favicon.png">
+
+    {% include tracking.html %}
+
+</head>
 
   <body>
-    <!-- HEADER -->
-    <div id="header_wrap" class="outer">
-        <header class="inner">
-          <a id="forkme_banner" href="https://github.com/weaveworks/weave">View on GitHub</a>
 
-          <h1 id="project_title">weave</h1>
-          <h2 id="project_tagline">The Docker Network</h2>
+   <div class="content">
+       {% include header-navbaronly.html %}
+       <div class="container">
+           <div class="col-md-10 col-md-offset-1">
+               {{ content }}
+           </div>
+       </div>
+   </div>
 
-        </header>
-    </div>
+   {% include footer.html %}
 
-    <!-- MAIN CONTENT -->
-    <div id="main_content_wrap" class="outer">
-      <section id="main_content" class="inner">
-        {{ content }}
-      </section>
-    </div>
-
-    <!-- FOOTER  -->
-    <div id="footer_wrap" class="outer">
-      <footer class="inner">
-        
-      </footer>
-    </div>
-
-    
-
-  </body>
+  </body> 
 </html>

site/boot2docker.md

@@ -3,7 +3,7 @@ title: Installing Weave with Boot2Docker
 layout: default
 ---
 
-## Installation with Boot2Docker
+# Installing Weave with Boot2Docker
 
 If you are running Docker inside the Boot2Docker VM, e.g. because you
 are on a Mac, then the following changes are required to the weave

site/building.md

@@ -3,6 +3,8 @@ title: Building Weave
 layout: default
 ---
 
+# Building Weave
+
 You only need to build weave if you want to work on the weave codebase
 (or you just enjoy building software).
 

site/features.md

@@ -1,14 +1,15 @@
 ---
-title: Weave features
+title: Weave Features
 layout: default
 ---
 
-# Weave features
+# Weave Features
 
 Weave has a few more features beyond those illustrated by the [basic
 example](https://github.com/weaveworks/weave#example):
 
  * [Virtual ethernet switch](#virtual-ethernet-switch)
+ * [Manual addressing](#manual-addressing)
  * [Application isolation](#application-isolation)
  * [Dynamic network attachment](#dynamic-network-attachment)
  * [Security](#security)
@@ -24,7 +25,7 @@ example](https://github.com/weaveworks/weave#example):
  * [Fault tolerance](#fault-tolerance)
  * [Automatic IP address management](#ipam)
  * [Automatic discovery with WeaveDNS](#dns)
- * [Starting containers with Docker](#proxy)
+ * [Working with the Docker CLI and API](#proxy)
 
 ### <a name="virtual-ethernet-switch"></a>Virtual Ethernet Switch
 
@@ -35,18 +36,18 @@ connected.
 Containers can easily access services from each other; e.g. in the
 container on `$HOST1` we can start a netcat "service" with
 
-    root@28841bd02eff:/# nc -lk -p 4422
+    root@a1:/# nc -lk -p 4422
 
 and then connect to it from the container on `$HOST2` with
 
-    root@f76829496120:/# echo 'Hello, world.' | nc 10.2.1.1 4422
+    root@a2:/# echo 'Hello, world.' | nc a1 4422
 
 Note that *any* protocol is supported. Doesn't even have to be over
 TCP/IP, e.g. a netcat UDP service would be run with
 
-    root@28841bd02eff:/# nc -lu -p 5533
+    root@a1:/# nc -lu -p 5533
 
-    root@f76829496120:/# echo 'Hello, world.' | nc -u 10.2.1.1 5533
+    root@a2:/# echo 'Hello, world.' | nc -u a1 5533
 
 We can deploy the entire arsenal of standard network tools and
 applications, developed over decades, to configure, secure, monitor,
@@ -55,47 +56,104 @@ now re-use the same tools and techniques when deploying applications
 as containers as we would have done when deploying them 'on metal' in
 our data centre.
 
+### <a name="manual-addressing"></a>Manual addressing
+
+The examples so far have leveraged weave's [address allocation](#ipam)
+and [discovery](#dns) features, meaning that we've not had to concern
+ourselves with IP addresses at all. It is, however, possible to
+specify address and network explicitly using [CIDR
+notation](http://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing#CIDR_notation)
+- let's see how the first example would have looked:
+
+On $HOST1:
+
+    host1$ C=$(weave run 10.2.1.1/24 -ti ubuntu)
+
+And $HOST2:
+
+    host2$ C=$(weave run 10.2.1.2/24 -ti ubuntu)
+
+Then on $HOST1...
+
+    host1$ docker attach $C
+    root@28841bd02eff:/# ping -c 1 -q 10.2.1.2
+    PING 10.2.1.2 (10.2.1.2): 48 data bytes
+    --- 10.2.1.2 ping statistics ---
+    1 packets transmitted, 1 packets received, 0% packet loss
+    round-trip min/avg/max/stddev = 1.048/1.048/1.048/0.000 ms
+
+Similarly, on $HOST2...
+
+    host2$ docker attach $C
+    root@f76829496120:/# ping -c 1 -q 10.2.1.1
+    PING 10.2.1.1 (10.2.1.1): 48 data bytes
+    --- 10.2.1.1 ping statistics ---
+    1 packets transmitted, 1 packets received, 0% packet loss
+    round-trip min/avg/max/stddev = 1.034/1.034/1.034/0.000 ms
+
+The IP addresses and netmasks can be anything you like, but make sure
+they don't conflict with any IP ranges in use on the hosts (including
+those delegated to IPAM) or IP addresses of external services the
+hosts or containers need to connect to. The same IP range must be used
+everywhere, and the individual IP addresses must, of course, be
+unique.
+
 ### <a name="application-isolation"></a>Application isolation
 
 A single weave network can host multiple, isolated applications, with
 each application's containers being able to communicate with each
 other but not containers of other applications.
 
-To accomplish that, we assign each application a different subnet. So,
-in the above example, if we wanted to add another application similar
-to, but isolated from, our first, we'd launch the containers with...
+To accomplish that, we assign each application a different subnet.
+Let's begin by configuring weave's allocator to manage multiple
+subnets:
 
-    host1$ D=$(weave run 10.2.2.1/24 -t -i ubuntu)
-    host2$ D=$(weave run 10.2.2.2/24 -t -i ubuntu)
+    host1$ weave launch -iprange 10.2.0.0/16 -ipsubnet 10.2.1.0/24
+    host1$ weave launch-dns
+    host2$ weave launch -iprange 10.2.0.0/16 -ipsubnet 10.2.1.0/24 $HOST1
+    host2$ weave launch-dns
+
+This delegates the entire 10.2.0.0/16 subnet to weave, and instructs
+it to allocate from 10.2.1.0/24 within that if no specific subnet is
+specified. Now we can launch some containers in the default subnet:
+
+    host1$ weave run --name a1 -ti ubuntu
+    host2$ weave run --name a2 -ti ubuntu
+
+And some more containers in a different subnet:
+
+    host1$ weave run net:10.2.2.0/24 --name b1 -ti ubuntu
+    host2$ weave run net:10.2.2.0.24 --name b2 -ti ubuntu
 
 A quick 'ping' test in the containers confirms that they can talk to
 each other but not the containers of our first application...
 
-    host1$ docker attach $D
-    
-    root@da50502598d5:/# ping -c 1 -q 10.2.2.2
-    PING 10.2.2.2 (10.2.2.2): 48 data bytes
-    --- 10.2.2.2 ping statistics ---
-    1 packets transmitted, 1 packets received, 0% packet loss
-    round-trip min/avg/max/stddev = 0.562/0.562/0.562/0.000 ms
-    
-    root@da50502598d5:/# ping -c 1 -q 10.2.1.1
-    PING 10.2.1.1 (10.2.1.1) 56(84) bytes of data.
-    --- 10.2.1.1 ping statistics ---
+    host1:~$ docker attach b1
+
+    root@b1:/# ping -c 1 -q b2
+    PING b2.weave.local (10.2.2.128) 56(84) bytes of data.
+    --- b2.weave.local ping statistics ---
+    1 packets transmitted, 1 received, 0% packet loss, time 0ms
+    rtt min/avg/max/mdev = 1.338/1.338/1.338/0.000 ms
+
+    root@b1:/# ping -c 1 -q a1
+    PING a1.weave.local (10.2.1.2) 56(84) bytes of data.
+    --- a1.weave.local ping statistics ---
     1 packets transmitted, 0 received, 100% packet loss, time 0ms
-    
-    root@da50502598d5:/# ping -c 1 -q 10.2.1.2
-    PING 10.2.1.2 (10.2.1.2) 56(84) bytes of data.
-    --- 10.2.1.2 ping statistics ---
+
+    root@b1:/# ping -c 1 -q a2
+    PING a2.weave.local (10.2.1.130) 56(84) bytes of data.
+    --- a2.weave.local ping statistics ---
     1 packets transmitted, 0 received, 100% packet loss, time 0ms
 
+
 This isolation-through-subnets scheme is an example of carrying over a
 well-known technique from the 'on metal' days to containers.
 
 If desired, a container can be attached to multiple subnets when it is
 started:
 
-    host1$ weave run 10.2.2.1/24 10.2.3.1/24 -t -i ubuntu
+    host1$ weave run net:default net:10.2.2.0/24 -ti ubuntu
 
 NB: By default docker permits communication between containers on the
 same host, via their docker-assigned IP addresses. For complete
@@ -117,29 +175,29 @@ to the weave network. To illustrate, we can achieve the same effect as
 the first example with
 
     host1$ C=$(docker run -d -t -i ubuntu)
-    host1$ weave attach 10.2.1.1/24 $C
+    host1$ weave attach $C
 
 There is a matching `weave detach` command:
 
-    host1$ weave detach 10.2.1.1/24 $C
+    host1$ weave detach $C
 
 You can detach a container from one application network and attach it
 to another:
 
-    host1$ weave detach 10.2.1.1/24 $C
-    host1$ weave attach 10.2.2.1/24 $C
+    host1$ weave detach net:default $C
+    host1$ weave attach net:10.2.2.0/24 $C
 
 or attach a container to multiple application networks, effectively
 sharing it between applications:
 
-    host1$ weave attach 10.2.1.1/24 $C
-    host1$ weave attach 10.2.2.1/24 $C
+    host1$ weave attach net:default
+    host1$ weave attach net:10.2.2.0/24
 
 Finally, multiple addresses can be attached or detached with a single
 invocation:
 
-    host1$ weave attach 10.2.1.1/24 10.2.2.1/24 10.2.3.1/24 $C
-    host1$ weave detach 10.2.1.1/24 10.2.2.1/24 10.2.3.1/24 $C
+    host1$ weave attach net:default net:10.2.2.0/24 net:10.2.3.0/24 $C
+    host1$ weave detach net:default net:10.2.2.0/24 net:10.2.3.0/24 $C
 
 ### <a name="security"></a>Security
 
@@ -157,10 +215,13 @@ or
 _NOTE: The command line option takes precedence over the environment
 variable._
 
-The same password must be specified for all weave peers; it is a
-component in the creation of ephemeral session keys for connections
-between peers. See the [crypto documentation](how-it-works.html#crypto)
-for more details.
+The password needs to be reasonably strong to guard against online
+dictionary attacks. We recommend at least 50 bits of entropy. An easy
+way to generate a random password which satsifies this requirement is
+
+    < /dev/urandom tr -dc A-Za-z0-9 | head -c9 ; echo
+
+The same password must be specified for all weave peers.
 
 ### <a name="host-network-integration"></a>Host network integration
 
@@ -171,31 +232,38 @@ anywhere.
 Let's say that in our example we want `$HOST2` to have access to the
 application containers. On `$HOST2` we run
 
-    host2$ weave expose 10.2.1.102/24
+    host2$ weave expose
 
-choosing an unused IP address in the application subnet. (There is a
-corresponding 'hide' command to revert this step.)
+(There is a corresponding 'hide' command to revert this step.)
 
-Now
+Now,  after finding allocated IPs via `weave ps`
 
-    host2$ ping 10.2.1.2
+    host2$ weave ps weave:expose
+    weave:expose 02:80:5c:02:f1:b2 10.2.1.132/24
+
+    host1$ weave ps a1
+    a1 1e:88:d7:5b:77:68 10.2.1.2/24
+
+this
+
+    host2$ ping 10.2.1.132
 
 will work. And, more interestingly,
 
-    host2$ ping 10.2.1.1
+    host2$ ping 10.2.1.2
 
 will work too, which is talking to a container that resides on `$HOST1`.
 
 Multiple subnet addresses can be exposed or hidden with a single
 invocation:
 
-    host2$ weave expose 10.2.1.102/24 10.2.2.102/24
-    host2$ weave hide   10.2.1.102/24 10.2.2.102/24
+    host2$ weave expose net:default net:10.2.2.0/24
+    host2$ weave hide   net:default net:10.2.2.0/24
 
 Finally, exposed addresses can be added to weaveDNS by supplying a
 fully-qualified domain name:
 
-    host2$ weave expose 10.2.1.102/24 -h exposed.weave.local
+    host2$ weave expose -h exposed.weave.local
 
 ### <a name="service-export"></a>Service export
 
@@ -210,18 +278,21 @@ a container on `$HOST1`, accessible to the outside world via `$HOST2`.
 First we need to expose the application network to `$HOST2`, as
 explained [above](#host-network-integration), i.e.
 
-    host2$ weave expose 10.2.1.102/24
+    host2$ weave expose
 
 Then we add a NAT rule to route from the outside world to the
 destination container service.
 
+    host1$ weave ps a1
+    a1 1e:88:d7:5b:77:68 10.2.1.2/24
+
     host2$ iptables -t nat -A PREROUTING -p tcp -i eth0 --dport 2211 \
-           -j DNAT --to-destination 10.2.1.1:4422
+           -j DNAT --to-destination 10.2.1.2:4422
 
 Here we are assuming that the "outside world" is connecting to `$HOST2`
 via 'eth0'. We want TCP traffic to port 2211 on the external IPs to be
 routed to our 'nc' service, which is running on port 4422 in the
-container with IP 10.2.1.1.
+container a1.
 
 With the above in place, we can connect to our 'nc' service from
 anywhere with
@@ -250,12 +321,15 @@ First we need to expose the application network to the host, as
 explained [above](#host-network-integration), this time on `$HOST1`,
 i.e.
 
-    host1$ weave expose 10.2.1.101/24
+    host1$ weave expose -h host1.weave.local
 
 Then we add a NAT rule to route from the above IP to the destination
 service.
 
-    host1$ iptables -t nat -A PREROUTING -p tcp -d 10.2.1.101 --dport 3322 \
+    host1$ weave ps weave:expose
+    weave:expose 66:46:f5:ac:7b:c9 10.2.1.3/24
+
+    host1$ iptables -t nat -A PREROUTING -p tcp -d 10.2.1.3 --dport 3322 \
            -j DNAT --to-destination $HOST3:2211
 
 This allows any application container to reach the service by
@@ -266,7 +340,7 @@ service on port 2211, e.g.
 
 then we can connect to it from our application container on `$HOST2` with
 
-    root@f76829496120:/# echo 'Hello, world.' | nc 10.2.1.101 3322
+    root@a2:/# echo 'Hello, world.' | nc host1 3322
 
 The same command will work from any application container.
 
@@ -275,7 +349,7 @@ The same command will work from any application container.
 Importing a service provides a degree of indirection that allows late
 and dynamic binding, similar to what can be achieved with a proxy. In
 our example, application containers are unaware that the service they
-are accessing at `10.2.1.101:3322` is in fact residing on
+are accessing at `10.2.1.3:3322` is in fact residing on
 `$HOST3:2211`. We can point application containers at another service
 location by changing the above NAT rule, without having to alter the
 applications themselves.
@@ -297,14 +371,17 @@ In our example above, the netcat service on `$HOST3` is imported into
 weave via `$HOST1`. We can export it on `$HOST2` by first exposing the
 application network with
 
-    host2$ weave expose 10.2.1.102/24
+    host2$ weave expose
 
 and then adding a NAT rule which routes traffic from the `$HOST2`
 network (i.e. anything which can connect to `$HOST2`) to the service
 endpoint in the weave network
 
+    host1$ weave ps weave:expose
+    weave:expose 66:46:f5:ac:7b:c9 10.2.1.3/24
+
     host2$ iptables -t nat -A PREROUTING -p tcp -i eth0 --dport 4433 \
-           -j DNAT --to-destination 10.2.1.101:3322
+           -j DNAT --to-destination 10.2.1.3:3322
 
 Now any host on the same network as `$HOST2` can access the service with
 
@@ -368,6 +445,16 @@ This will prevent the peer from trying to reconnect to that host once
 connectivity to it is lost, and thus can be used to administratively
 remove decommissioned peers from the network.
 
+Hosts can also be bulk-replaced. All existing hosts will be forgotten,
+and the new hosts will be added, when one runs
+
+    host# weave connect --replace $NEW_HOST1 $NEW_HOST2
+
+For complete control over the peer topology, automatic discovery can
+be disabled with the `-nodiscovery` option to `weave launch`. In this
+mode, weave will only connect to the addresses specified at launch
+time and with `weave connect`.
+
 ### <a name="container-mobility"></a>Container mobility
 
 Containers can be moved between hosts without requiring any
@@ -406,7 +493,7 @@ WeaveDNS is a distributed DNS service for weave networks, enabling
 containers to address each other by name rather than IP address. Find
 out more about WeaveDNS from its [documentation](weavedns.html).
 
-### <a name="proxy"></a>Starting containers with Docker
+### <a name="proxy"></a>Working with the Docker CLI and API
 
 Instead of the `weave` command-line utility, you may prefer to use the
 standard Docker command-line interface, or the Docker remote API, for

site/how-it-works.md

@@ -1,9 +1,9 @@
 ---
-title: How Weave works
+title: How Weave Works
 layout: default
 ---
 
-## How does it work?
+# How Weave Works
 
  * [Overview](#overview)
  * [Encapsulation](#encapsulation)
@@ -48,6 +48,7 @@ peer. Weave can route packets in partially connected networks with
 changing topology. For example, in this network, peer 1 is connected
 directly to 2 and 3, but if 1 needs to send a packet to 4 or 5 it must
 first send it to peer 3:
+
 ![Partially connected Weave Network](images/top-diag1.png "Partially connected Weave Network")
 
 ### <a name="encapsulation"></a>Encapsulation
@@ -274,15 +275,19 @@ local peer in the usual Diffie-Hellman way, resulting in both peers
 arriving at the same shared key. To this is appended the supplied
 password, and the result is hashed through SHA256, to form the final
 ephemeral session key. Thus the supplied password is never exchanged
-directly, and is thoroughly mixed into the shared secret. The shared
-key formed by Diffie-Hellman is 256 bits long, appending the password
-to this obviously makes it longer by an unknown amount, and the use of
-SHA256 reduces this back to 256 bits, to form the final ephemeral
-session key. This late combination with the password eliminates "Man
-In The Middle" attacks: sniffing the public key exchange between the
-two peers and faking their responses will not grant an attacker
-knowledge of the password, and so an attacker would not be able to
-form valid ephemeral session keys.
+directly, and is thoroughly mixed into the shared secret. Furthermore,
+the rate at which TCP connections are accepted is limited by weave to
+1kHz, which twarts online dictionary attacks on reasonably strong
+passwords.
+
+The shared key formed by Diffie-Hellman is 256 bits long, appending
+the password to this obviously makes it longer by an unknown amount,
+and the use of SHA256 reduces this back to 256 bits, to form the final
+ephemeral session key. This late combination with the password
+eliminates "Man In The Middle" attacks: sniffing the public key
+exchange between the two peers and faking their responses will not
+grant an attacker knowledge of the password, and so an attacker would
+not be able to form valid ephemeral session keys.
 
 The same ephemeral session key is used for both TCP and UDP traffic
 between two peers.

site/index.md

@@ -3,13 +3,14 @@ title: weave, the Docker network
 layout: default
 ---
 
-# weave documentation
+# Weave Documentation
 
  * [Introduction, Installation and Example](https://github.com/weaveworks/weave#readme)
  * [Getting Started Guides](http://weave.works/guides/)
  * [Features](features.html)
  * [Automatic IP Address Management](ipam.html)
  * [Automatic Discovery with WeaveDNS](weavedns.html)
+ * [Working with the Docker CLI and API](proxy.html)
  * [Troubleshooting](troubleshooting.html)
  * [Building](building.html)
  * [How it works](how-it-works.html)

site/ipam.md

@@ -5,14 +5,21 @@ layout: default
 
 # Automatic IP Address Management
 
-Weave can automatically assign unique IP addresses to each container
-across the network. To make this work, weave must be told on startup
-what range of addresses to allocate from, for example:
+Weave can automatically assign containers an IP address that is unique
+across the network.
 
-    host1# weave launch -iprange 10.2.3.0/24
+ * [Usage](#usage)
+ * [Initialisation](#initialisation)
+ * [Choosing an allocation range](#range)
+ * [Automatic allocation across multiple subnets](#subnets)
+ * [Mixing automatic and manual allocation](#manual)
+ * [Stopping and removing peers](#stop)
+ * [Troubleshooting](#troubleshooting)
 
-The `run`, `start`, `attach`, `expose` and `hide` commands will then
-fetch an address automatically if none is specified, i.e.:
+## <a name="usage"></a>Usage
+
+Containers are automatically allocated an IP address when none is
+specified when the container is started, e.g.
 
     host1# C=$(weave run -ti ubuntu)
 
@@ -20,65 +27,156 @@ You can see which address was allocated with
 [`weave ps`](troubleshooting.html#list-attached-containers):
 
     host1# weave ps $C
-    a7aff7249393 7a:51:d1:09:21:78 10.2.3.1/24
+    a7aff7249393 7a:51:d1:09:21:78 10.128.0.1/10
 
-The `-iprange` parameter is given in [CIDR
-notation](http://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing)
-format - in this example "/24" means the first 24 bits of the address
-form the network address and the allocator is to allocate container
-addresses that all start 10.2.3. The ".0" and ".-1" addresses in a
-subnet are not used, as required by [RFC
-1122](https://tools.ietf.org/html/rfc1122#page-29).
+Weave detects when a container has exited and releases its
+automatically allocated addresses so they can be re-used.
 
-Once you have given a range of addresses to the IP allocator, you must
-not use any addresses in that range for anything else.  If, in our
-example, you subsequently did `weave run 10.2.3.x/24`, you run the
-risk that that address will be allocated to another container, which
-will make network traffic delivery intermittent or non-existent for
-the containers that share the same IP address. No diagnostic message
-is output by weave if you break this rule.
+Automatic IP address assignment is available for the `run`, `start`,
+`attach`, `detach`, `expose`, `hide` and `launch-dns` commands.
 
-Weave will automatically learn when a container has exited
-and hence can release its IP address.
+## <a name="initialisation"></a>Initialisation
 
-Weave shares the IP range across all peers, dynamically according to
-their needs.  If a group of peers becomes isolated from the rest (a
-partition), they can continue to work with the IP ranges they had
-before isolation, and can be re-connected to the rest of the network
-and carry on. Note that you must specify the same range with
-`-iprange` on each host, and you cannot mix weaves started with and
-without -iprange.
-
-### Initialisation
-
-Just once, when you start up the whole network, weave needs a majority
-of peers to agree in order to avoid isolated groups starting off
-inconsistently. Therefore, you must either supply the list of all
-peers in the network to `weave launch` or add the `-initpeercount`
-flag to specify how many peers there will be.  It isn't a problem to
-over-estimate by a bit, but if you supply a number that is too small
-then multiple independent groups may form.
+Just once, when the first automatic IP address allocation is requested
+in the whole network, weave needs a majority of peers to be present in
+order to avoid formation of isolated groups, which could lead to
+inconsistency, i.e. the same IP address being allocated to two
+different containers. Therefore, you must either supply the list of
+all peers in the network to `weave launch` or add the `-initpeercount`
+flag to specify how many peers there will be.
 
 To illustrate, suppose you have three hosts, accessible to each other
 as `$HOST1`, `$HOST2` and `$HOST3`. You can start weave on those three
 hosts with these three commands:
 
-    host1$ weave launch -iprange 10.3.0.0/16 $HOST2 $HOST3
+    host1$ weave launch $HOST2 $HOST3
 
-    host2$ weave launch -iprange 10.3.0.0/16 $HOST1 $HOST3
+    host2$ weave launch $HOST1 $HOST3
 
-    host3$ weave launch -iprange 10.3.0.0/16 $HOST1 $HOST2
+    host3$ weave launch $HOST1 $HOST2
 
 Or, if it is not convenient to name all the other hosts at launch
 time, you can give the number of peers like this:
 
-    host1$ weave launch -iprange 10.3.0.0/16 -initpeercount 3
+    host1$ weave launch -initpeercount 3
 
-    host2$ weave launch -iprange 10.3.0.0/16 -initpeercount 3 $HOST3
+    host2$ weave launch -initpeercount 3 $HOST3
 
-    host3$ weave launch -iprange 10.3.0.0/16 -initpeercount 3 $HOST2
+    host3$ weave launch -initpeercount 3 $HOST2
 
-### Stopping and removing peers
+### More on `-initpeercount`
+
+TL;DR: it isn't a problem to over-estimate by a bit, but if you supply
+a number that is too small then multiple independent groups may form.
+
+Weave uses the estimate of the number of peers at initialization to
+compute a majority or quorum number - specifically floor(n/2) + 1. So,
+if the actual number of peers is less than half the number stated then
+they will keep waiting for someone else to join to reach a quorum.  On
+the other hand, if the actual number is more than twice the quorum
+number then you could have two sets of peers each reach a quorum and
+initialize independent data structures. You'd have to be quite unlucky
+for that to happen in practice, as they have to go through the whole
+agreement process without learning about each other, but it's
+definitely possible.
+
+The quorum number is only used once at start-up (specifically, the
+first time someone tries to allocate or claim an IP address), so once
+a set of peers is initialized you can add more and they will join on
+to the data structure used by the existing set.  The one thing you
+have to watch is if the early ones get restarted, you must restart
+them with the current number of peers - if they use the smaller number
+that was correct when they first started then they could form an
+independent set again.
+
+To illustrate this last point, the following sequence of operations
+would be safe wrt weave's startup quorum:
+
+    host1$ weave launch
+    ...time passes...
+    host2$ weave launch $HOST1
+    ...time passes...
+    host3$ weave launch $HOST1 $HOST2
+    ...time passes...
+    ...host1 is rebooted...
+    host1$ weave launch $HOST2 $HOST3
+
+## <a name="range"></a>Choosing an allocation range
+
+By default, weave will allocate IP addresses in the 10.128.0.0/10
+range. This can be overridden with the `-iprange` option, e.g.
+
+    host1# weave launch -iprange 10.2.0.0/16
+
+and must be the same on every host.
+
+The range parameter is written in
+[CIDR notation](http://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing) -
+in this example "/16" means the first 16 bits of the address form the
+network address and the allocator is to allocate container addresses
+that all start 10.2. The ".0" and ".-1" addresses in a subnet are not
+used, as required by
+[RFC 1122](https://tools.ietf.org/html/rfc1122#page-29).
+
+Weave shares the IP address range across all peers, dynamically
+according to their needs.  If a group of peers becomes isolated from
+the rest (a partition), they can continue to work with the address
+ranges they had before isolation, and can subsequently be re-connected
+to the rest of the network without any conflicts arising.
+
+Once you have given a range of addresses to the IP allocator, you must
+not use any addresses in that range for anything else.  If, in our
+example, you subsequently executed `weave run 10.2.3.1/24 -ti ubuntu`,
+you run the risk that the IP allocator will assign the same address to
+another container, which will make network traffic delivery
+intermittent or non-existent for the containers that share the same IP
+address.
+
+## <a name="subnets"></a>Automatic allocation across multiple subnets
+
+When
+[running containers on different subnets](features.html#application-isolation),
+you may wish to request the allocation of an address from a particular
+subnet. This is done by specifying the subnet with `net:<subnet>`, in
+CIDR notation, e.g.
+
+    host1# C=$(weave run net:10.2.7.0/24 -ti ubuntu)
+
+You can ask for multiple addresses in different subnets and add in
+manually-assigned addresses (outside the automatic allocation range),
+for instance:
+
+    host1# C=$(weave run net:10.2.7.0/24 net:10.2.8.0/24 ip:10.3.9.1/24 -ti ubuntu)
+
+When working with multiple subnets in this way, it is usually
+desirable to constrain the default subnet - i.e. the one chosen by the
+allocator when no subnet is supplied - so that it does not overlap
+with others. One can specify that with `-ipsubnet`:
+
+    host1# weave launch -iprange 10.2.0.0/16 -ipsubnet 10.2.3.0/24
+
+`-iprange` should cover the entire range that you will ever use for
+allocation, and `-ipsubnet` is the subnet that will be used when you
+don't explicitly specify one.
+
+When specifying addresses, the default subnet can be denoted
+symbolically with `net:default`.
+
+## <a name="manual"></a>Mixing automatic and manual allocation
+
+If you want to start containers with a mixture of
+automatically-allocated addresses and manually-chosen addresses, *and
+have the containers communicate with each other*, you can choose a
+`-iprange` that is smaller than `-ipsubnet`, For example, if you
+launch weave with:
+
+    host1# weave launch -iprange 10.9.0.0/17 -ipsubnet 10.9.0.0/16
+
+then you can run all containers in the 10.9.0.0/16 subnet, with
+automatic allocation using the lower half, leaving the upper half free
+for manual allocation.
+
+## <a name="stop"></a>Stopping and removing peers
 
 You may wish to `weave stop` and re-launch to change some config or to
 upgrade to a new version; provided the underlying protocol hasn't
@@ -117,12 +215,12 @@ reports on the current status of the weave router and IP allocator:
 ````
 weave router git-8f675f15c0b5
 ...
-Allocator subnet 10.2.1.0/24
-  Free IPs: ~98.0%, 62 local, ~189 remote
+Allocator universe 10.2.0.0/16
 Owned Ranges:
   10.2.1.1 -> 96:e9:e2:2e:2d:bc (host1) (v3)
   10.2.1.128 -> ea:84:25:9b:31:2e (host2) (v3)
   10.2.1.192 -> ea:6c:21:09:cf:f0 (host3) (v9)
+Allocator default subnet: 10.2.1.0/24
 ````
 
 The first section covers the router; see the [troubleshooting
@@ -134,6 +232,3 @@ lists which address ranges have been assigned to which peer. Each
 range begins at the address shown and ends just before the next
 address, or wraps around at the end of the subnet. The 'v' number
 denotes how many times that entry has been updated.
-
-The 'Free IPs' information may be out of date with respect to changes
-happening elsewhere in the network.

site/javascripts/main.js

@@ -1 +0,0 @@
-console.log('This would be the main JS file.');