add 0.9.0 release change log

LGTM.  Fixes #380

.gitignore

@@ -27,5 +27,7 @@ _testmain.go
 *~
 
 # Project specific
+weaver/weaver
 weavedns/weavedns
+tools/bin
 releases

CHANGELOG.md

@@ -1,4 +1,47 @@
-== Release 0.7.0
+## Release 0.9.0
 
-This is the first release assigned a version number. See the README at
+- Improve WeaveDNS to the point where it can act as the name server
-https://github.com/zettio/weave#readme for more information.
+  for containers in nearly all situations.
+
+- Diagnose and report peer connectivity more comprehensively.
+
+- Adapt to changes in topology - adding & removing of weave peers,
+  disruption of connectivity - more rapidly.
+
+- Cope with delays delays in downloading/running docker
+  images/containers required for weave operation.
+
+See the
+[complete change log](https://github.com/zettio/weave/issues?q=milestone%3A0.9.0)
+for more details.
+
+## Release 0.8.0
+
+- Align script and image version. When the `weave` script has a
+  version number, i.e. it is part of an official release, it runs
+  docker images matching that version. Thus the script and image
+  versions are automatically aligned. Unversioned/unreleased `weave`
+  scripts run the 'latest'-tagged image versions.
+
+- Eliminate dependency on ethtool and conntrack. Instead of requiring
+  these to be installed on the host, weave invokes them via a
+  `weavetools` docker image that contains minimally packaged versions
+  of these utilities.
+
+- New `weave setup` command. This downloads all docker images used by
+  weave. Invoking this is strictly optional; its main purpose is to
+  facilitate automated installation of weave and preventing delays in
+  subsequent weave command execution due to image downloading.
+
+## Release 0.7.0
+
+This is the first release assigned a version number.
+
+When downloading weave you now have the following choices...
+
+1. a specific version, e.g. https://github.com/zettio/weave/releases/download/v0.7.0/weave
+2. latest released version: https://github.com/zettio/weave/releases/download/latest_release/weave
+3. most recent 'master' commit: https://raw.githubusercontent.com/zettio/weave/master/weave
+
+Previously the only documented download location was (3). We recommend
+that any automated scripts using that be changed to either (1) or (2).

Makefile

@@ -1,21 +1,30 @@
+PUBLISH=publish_weave publish_weavedns publish_weavetools
+
 .DEFAULT: all
-.PHONY: all publish clean tests
+.PHONY: all update tests publish $(PUBLISH) clean
 
 # If you can use docker without being root, you can do "make SUDO="
 SUDO=sudo
 
+DOCKERHUB_USER=zettio
 WEAVE_VERSION=git-$(shell git rev-parse --short=12 HEAD)
 WEAVER_EXE=weaver/weaver
 WEAVEDNS_EXE=weavedns/weavedns
-WEAVER_IMAGE=zettio/weave
+WEAVETOOLS_EXES=tools/bin
-WEAVEDNS_IMAGE=zettio/weavedns
+WEAVER_IMAGE=$(DOCKERHUB_USER)/weave
+WEAVEDNS_IMAGE=$(DOCKERHUB_USER)/weavedns
+WEAVETOOLS_IMAGE=$(DOCKERHUB_USER)/weavetools
 WEAVER_EXPORT=/var/tmp/weave.tar
 WEAVEDNS_EXPORT=/var/tmp/weavedns.tar
+WEAVETOOLS_EXPORT=/var/tmp/weavetools.tar
 
-all: $(WEAVER_EXPORT) $(WEAVEDNS_EXPORT)
+all: $(WEAVER_EXPORT) $(WEAVEDNS_EXPORT) $(WEAVETOOLS_EXPORT)
 
-$(WEAVER_EXE) $(WEAVEDNS_EXE):
+update:
-	go get -tags netgo ./$(shell dirname $@)
+	go get -u -f -v -tags -netgo ./$(dir $(WEAVER_EXE)) ./$(dir $(WEAVEDNS_EXE))
+
+$(WEAVER_EXE) $(WEAVEDNS_EXE): common/*.go
+	go get -tags netgo ./$(@D)
 	go build -ldflags "-extldflags \"-static\" -X main.version $(WEAVE_VERSION)" -tags netgo -o $@ ./$(shell dirname $@)
 	@strings $@ | grep cgo_stub\\\.go >/dev/null || { \
 		rm $@; \
@@ -29,6 +38,9 @@ $(WEAVER_EXE) $(WEAVEDNS_EXE):
 $(WEAVER_EXE): router/*.go weaver/main.go
 $(WEAVEDNS_EXE): nameserver/*.go weavedns/main.go
 
+$(WEAVETOOLS_EXES): tools/build.sh
+	$(SUDO) docker run --rm -v $(realpath $(<D)):/home/weave ubuntu sh /home/weave/build.sh
+
 $(WEAVER_EXPORT): weaver/Dockerfile $(WEAVER_EXE)
 	$(SUDO) docker build -t $(WEAVER_IMAGE) weaver
 	$(SUDO) docker save $(WEAVER_IMAGE):latest > $@
@@ -37,10 +49,23 @@ $(WEAVEDNS_EXPORT): weavedns/Dockerfile $(WEAVEDNS_EXE)
 	$(SUDO) docker build -t $(WEAVEDNS_IMAGE) weavedns
 	$(SUDO) docker save $(WEAVEDNS_IMAGE):latest > $@
 
+$(WEAVETOOLS_EXPORT): tools/Dockerfile $(WEAVETOOLS_EXES)
+	$(SUDO) docker build -t $(WEAVETOOLS_IMAGE) tools
+	$(SUDO) docker save $(WEAVETOOLS_IMAGE):latest > $@
+
 # Add more directories in here as more tests are created
 tests:
+	cd router; go test -tags netgo
 	cd nameserver; go test -tags netgo
 
+$(PUBLISH): publish_%:
+	$(SUDO) docker tag  $(DOCKERHUB_USER)/$* $(DOCKERHUB_USER)/$*:$(WEAVE_VERSION)
+	$(SUDO) docker push $(DOCKERHUB_USER)/$*:$(WEAVE_VERSION)
+	$(SUDO) docker push $(DOCKERHUB_USER)/$*:latest
+
+publish: $(PUBLISH)
+
 clean:
-	-$(SUDO) docker rmi $(WEAVER_IMAGE) $(WEAVEDNS_IMAGE)
+	-$(SUDO) docker rmi $(WEAVER_IMAGE) $(WEAVEDNS_IMAGE) $(WEAVETOOLS_IMAGE)
-	rm -f $(WEAVER_EXE) $(WEAVEDNS_EXE) $(WEAVER_EXPORT) $(WEAVEDNS_EXPORT)
+	rm -f $(WEAVER_EXE) $(WEAVEDNS_EXE) $(WEAVER_EXPORT) $(WEAVEDNS_EXPORT) $(WEAVETOOLS_EXPORT)
+	$(SUDO) rm -rf $(WEAVETOOLS_EXES)

README.md

@@ -27,26 +27,12 @@ capabilities, so these can continue to be used by containers.
 
 ## Installation
 
-To run weave on a host, you need to install...
+Ensure you are running Linux (kernel 3.5 or later) and have Docker
+(version 0.9.1 or later) installed. Then install weave with
 
-1. Linux and Docker. We've tested with Docker versions 0.9.1 through
+    sudo wget -O /usr/local/bin/weave \
-   1.3.1, but other versions should work too. Linux kernels after 3.5
+      https://github.com/zettio/weave/releases/download/latest_release/weave
-   are known to work; the newer the better.
+    sudo chmod a+x /usr/local/bin/weave
-2. weave. Install this with
-
-        sudo wget -O /usr/local/bin/weave \
-          https://raw.githubusercontent.com/zettio/weave/master/weave
-        sudo chmod a+x /usr/local/bin/weave
-
-3. (recommended) ethtool. On many systems this is installed already;
-   if not then grab it via your favourite package manager. On some
-   systems, weave application container networking may not operate
-   correctly unless ethtool is available.
-
-4. (optional) conntrack. Install this via your favourite package
-   manager. Without conntrack, the weave network may not re-establish
-   itself fully when individual weave instances are stopped (with
-   `weave stop`) and restarted quickly (typically within ~3 minutes).
 
 ## Quick Start Screencast
 
@@ -66,7 +52,13 @@ On `$HOST1` run (as root)
     host1# C=$(weave run 10.0.1.1/24 -t -i ubuntu)
 
 The first line starts the weave router, in a container. This needs to
-be done once on each host.
+be done once on each host. The required docker image for the weave
+router container is downloaded automatically. There is also a `weave
+setup` command for downloading this and other images required for
+weave operation; this is a strictly optional step which is especially
+useful for automated installation of weave and ensures that any
+subsequent weave commands do not encounter delays due to image
+downloading.
 
 The second line runs our application container. We give it an IP
 address and network, in
@@ -136,7 +128,7 @@ to each other.
  * [Troubleshooting](http://zettio.github.io/weave/troubleshooting.html)
  * [Building](http://zettio.github.io/weave/building.html)
  * [How it works](http://zettio.github.io/weave/how-it-works.html)
- * [Preview of weaveDNS](http://weaveblog.com/2014/11/04/have-you-met-weavedns/)
+ * [WeaveDNS README](https://github.com/zettio/weave/tree/master/weavedns#readme)
 
 ## Contact Us
 
@@ -146,7 +138,7 @@ help@weave.works. When reporting a bug, please include which version of
 weave you are running, as shown by `weave version`.
 
 Follow weave on Twitter:
-[@weavenetwork](https://twitter.com/weavenetwork).
+[@weaveworks](https://twitter.com/weaveworks).
 
 Read the Weave blog:
 [Weaveblog](http://weaveblog.com/).

Vagrantfile

@@ -1,7 +1,7 @@
 VAGRANTFILE_API_VERSION = "2"
 
 vm_ip = "172.16.0.3" # arbitrary private IP
-pkgs = "lxc-docker golang ethtool libpcap-dev git mercurial"
+pkgs = "lxc-docker build-essential ethtool libpcap-dev git mercurial"
 
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
   config.vm.box = "phusion/ubuntu-14.04-amd64"
@@ -14,14 +14,19 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
   config.vm.provision :shell, :inline => "sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 36A1D7869245C8950F966E92D8576A8BA88D21E9"
   config.vm.provision :shell, :inline => "echo deb https://get.docker.io/ubuntu docker main > /etc/apt/sources.list.d/docker.list"
 
-  pkg_cmd = "apt-get update -qq; " \
+  pkg_cmd = "export DEBIAN_FRONTEND=noninteractive; " \
-  "apt-get install -q -y --force-yes --no-install-recommends "
+    "apt-get update -qq; " \
-  pkg_cmd << pkgs
+    "apt-get install -qq --no-install-recommends #{pkgs}"
+
+  install_go_toolchain = "curl -s https://storage.googleapis.com/golang/go1.4.linux-amd64.tar.gz | tar xz -C /usr/local"
+
   config.vm.provision :shell, :inline => pkg_cmd
+  config.vm.provision :shell, :inline => install_go_toolchain
   config.vm.provision :shell, :inline => "usermod -a -G docker vagrant; "
   config.vm.provision :shell, :inline => "echo export GOPATH=/home/vagrant >> /home/vagrant/.bashrc"
+  config.vm.provision :shell, :inline => "echo export PATH=/usr/local/go/bin:${PATH} >> /home/vagrant/.bashrc"
   config.vm.provision :shell, :inline => "chown -R vagrant:vagrant /home/vagrant/src"
-  config.vm.provision :shell, :inline => "sudo go install -a -tags netgo std"
+  config.vm.provision :shell, :inline => "/usr/local/go/bin/go clean -i net; /usr/local/go/bin/go install -tags netgo std"
 
 end
 

bin/release

@@ -2,11 +2,13 @@
 
 set -e
 
+SUDO=${SUDO-sudo}
+GITHUB_USER=${GITHUB_USER:-zettio}
+DOCKERHUB_USER=${DOCKERHUB_USER:-zettio}
+RELEASE_NAME=${RELEASE_NAME:-"Weave"}
+RELEASE_DESCRIPTION=${RELEASE_DESCRIPTION:-"Weaving Containers into Applications"}
+
 PWD=`pwd`
-GITHUB_USER=zettio
-DOCKERHUB_USER=zettio
-RELEASE_NAME="Weave"
-RELEASE_DESCRIPTION="Weaving Containers into Applications"
 
 setup() {
     ## Get the new version number from the most recent tag
@@ -37,7 +39,7 @@ build() {
     cd $RELEASE_DIR
 
     ## Check that the top changelog entry is this version
-    if ! latest_changelog=$(grep -o "== Release [0-9].*" -m1 ./CHANGELOG.md) || \
+    if ! latest_changelog=$(grep -o "## Release [0-9].*" -m1 ./CHANGELOG.md) || \
        ! [ `echo "$latest_changelog" | grep -o '[0-9][^ ]*'` == "$VERSION" ]; then
         echo -e "\u2757 Latest changelog entry \"$latest_changelog\" does not match the release version $VERSION" >&2
         exit 1
@@ -49,10 +51,7 @@ build() {
     ## Inject the version numbers and build the distributables
     ## (library versions?)
     sed -i "/SCRIPT_VERSION=/ c\SCRIPT_VERSION=\"$VERSION\"" ./weave
-    make \
+    make SUDO=$SUDO WEAVE_VERSION=$VERSION DOCKERHUB_USER=$DOCKERHUB_USER
-        WEAVE_VERSION=$VERSION \
-        WEAVER_IMAGE=$DOCKERHUB_USER/weave \
-        WEAVEDNS_IMAGE=$DOCKERHUB_USER/weavedns
 
     if make tests; then
         echo -e '\u2713 Tests pass'
@@ -121,14 +120,8 @@ publish() {
 
     echo '** Sanity checks OK for publishing tag' $LATEST_TAG as $DOCKERHUB_USER/weave:$VERSION
 
-    echo == Tagging $DOCKERHUB_USER/weave and $DOCKERHUB_USER/weavedns as $VERSION
+    echo "== Tagging and publishing images on docker hub as user $DOCKERHUB_USER"
-    docker tag $DOCKERHUB_USER/weave $DOCKERHUB_USER/weave:$VERSION
+    make SUDO=$SUDO WEAVE_VERSION=$VERSION DOCKERHUB_USER=$DOCKERHUB_USER publish
-    docker tag $DOCKERHUB_USER/weavedns $DOCKERHUB_USER/weavedns:$VERSION
-
-    docker push $DOCKERHUB_USER/weave:$VERSION
-    echo "** Pushed $DOCKERHUB_USER/weave:$VERSION to docker hub"
-    docker push $DOCKERHUB_USER/weavedns:$VERSION
-    echo "** Pushed $DOCKERHUB_USER/weavedns:$VERSION to docker hub"
 
     echo "== Creating GitHub release $RELEASE_NAME $VERSION"
     # This cannot be done as a draft because of a bug in
@@ -173,7 +166,7 @@ publish() {
         --file "./weave"
 
     echo "** Published $RELEASE_NAME as latest_release at"
-    echo -e "\thttps://github.com/$GITHUB_USER/weave/releases/latest_release"    
+    echo -e "\thttps://github.com/$GITHUB_USER/weave/releases/latest_release"
 }
 
 usage() {

common/init.go

@@ -0,0 +1,10 @@
+package common
+
+import (
+	"io/ioutil"
+	"os"
+)
+
+func init() {
+	InitLogging(ioutil.Discard, os.Stdout, os.Stdout, os.Stderr)
+}

common/logging.go

@@ -1,4 +1,4 @@
-package nameserver
+package common
 
 import (
 	"io"
@@ -32,6 +32,10 @@ func InitLogging(debugHandle io.Writer,
 	Error = log.New(errorHandle, "ERROR: ", standardLogFlags)
 }
 
-func init() {
+func InitDefaultLogging(debug bool) {
-	InitLogging(ioutil.Discard, os.Stdout, os.Stdout, os.Stderr)
+	debugOut := ioutil.Discard
+	if debug {
+		debugOut = os.Stderr
+	}
+	InitLogging(debugOut, os.Stdout, os.Stdout, os.Stderr)
 }

docs/architecture.txt

@@ -83,13 +83,17 @@ Router:
     local peer. It will tell the local connection (communicating to
     the actor thread) about the UDP address of the remote peer. The
     local connection will then start its forwarder threads as
-    described in 6a, and start sending slow heartbeats. The local connection
+    described in 6a, and start sending fast heartbeats. We send to the
-    is marked as established which means it is included in network updates
+    remote peer via TCP a ConnectionEstablished message. The remote
-    broadcast to our peers. We also send to the remote peer via TCP a
+    peer receives this (on the TCP receiver process), tells the
-    ConnectionEstablished message. The remote peer receives this (on
+    connection actor process, which then replaces the fast heartbeater
-    the TCP receiver process), tells the connection actor process,
+    with a slow heartbeater and marks the connection as established
-    which then replaces the fast heartbeater with a slow heartbeater
+    (which means it is included in network updates broadcast to our
-    and similarly marks the connection as established.
+    peers).
+  6c. When the connection initiator receives the fast heartbeat from
+    the remote peer it sends to the remote peer via a TCP a
+    ConnectionEstablished message. This is handled by the remote peer
+    as described in 6b.
   7. Whenever a connection is established or terminated, the local
     peer's version is incremented. Whenever this happens, the peer
     generates a network update message which is broadcast to its

nameserver/http.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"fmt"
 	"github.com/miekg/dns"
+	. "github.com/zettio/weave/common"
 	"io"
 	"log"
 	"net"
@@ -23,7 +24,7 @@ func parseUrl(url string) (identifier string, ipaddr string, err error) {
 func httpErrorAndLog(level *log.Logger, w http.ResponseWriter, msg string,
 	status int, logmsg string, logargs ...interface{}) {
 	http.Error(w, msg, status)
-	level.Printf(logmsg, logargs...)
+	level.Printf("[http] "+logmsg, logargs...)
 }
 
 func ListenHttp(domain string, db Zone, port int) {
@@ -59,7 +60,7 @@ func ListenHttp(domain string, db Zone, port int) {
 			}
 
 			if dns.IsSubDomain(domain, name) {
-				Info.Printf("Adding %s -> %s", name, ipStr)
+				Info.Printf("[http] Adding %s -> %s", name, ipStr)
 				if err = db.AddRecord(ident, name, ip); err != nil {
 					if _, ok := err.(DuplicateError); !ok {
 						httpErrorAndLog(
@@ -69,7 +70,7 @@ func ListenHttp(domain string, db Zone, port int) {
 					} // oh, I already know this. whatever.
 				}
 			} else {
-				Info.Printf("Ignoring name %s, not in %s", name, domain)
+				Info.Printf("[http] Ignoring name %s, not in %s", name, domain)
 			}
 
 		case "DELETE":
@@ -84,7 +85,7 @@ func ListenHttp(domain string, db Zone, port int) {
 				reqError("Invalid IP in request", "Invalid IP in request: %s", ipStr)
 				return
 			}
-			Info.Printf("Deleting %s (%s)", ident, ipStr)
+			Info.Printf("[http] Deleting %s (%s)", ident, ipStr)
 			if err = db.DeleteRecord(ident, ip); err != nil {
 				if _, ok := err.(LookupError); !ok {
 					httpErrorAndLog(
@@ -102,6 +103,6 @@ func ListenHttp(domain string, db Zone, port int) {
 
 	address := fmt.Sprintf(":%d", port)
 	if err := http.ListenAndServe(address, nil); err != nil {
-		Error.Fatal("Unable to create http listener: ", err)
+		Error.Fatal("[http] Unable to create http listener: ", err)
 	}
 }

nameserver/http_test.go

@@ -2,6 +2,7 @@ package nameserver
 
 import (
 	"fmt"
+	wt "github.com/zettio/weave/testing"
 	"math/rand"
 	"net"
 	"net/http"
@@ -38,12 +39,12 @@ func TestHttp(t *testing.T) {
 	addrUrl := fmt.Sprintf("http://localhost:%d/name/%s/%s", port, containerID, addrParts[0])
 	resp, err := genForm("PUT", addrUrl,
 		url.Values{"fqdn": {successTestName}, "local_ip": {dockerIP}, "routing_prefix": {addrParts[1]}})
-	assertNoErr(t, err)
+	wt.AssertNoErr(t, err)
-	assertStatus(t, resp.StatusCode, http.StatusOK, "http response")
+	wt.AssertStatus(t, resp.StatusCode, http.StatusOK, "http response")
 
 	// Check that the address is now there.
-	foundIP, err := zone.LookupLocal(successTestName)
+	foundIP, err := zone.LookupName(successTestName)
-	assertNoErr(t, err)
+	wt.AssertNoErr(t, err)
 	ip, _, _ := net.ParseCIDR(testAddr1)
 	if !foundIP.Equal(ip) {
 		t.Fatal("Unexpected result for", successTestName, foundIP)
@@ -52,34 +53,34 @@ func TestHttp(t *testing.T) {
 	// Adding exactly the same address should be OK
 	resp, err = genForm("PUT", addrUrl,
 		url.Values{"fqdn": {successTestName}, "local_ip": {dockerIP}, "routing_prefix": {addrParts[1]}})
-	assertNoErr(t, err)
+	wt.AssertNoErr(t, err)
-	assertStatus(t, resp.StatusCode, http.StatusOK, "http success response for duplicate add")
+	wt.AssertStatus(t, resp.StatusCode, http.StatusOK, "http success response for duplicate add")
 
 	// Now try adding the same address again with a different ident --
 	// again should be fine
 	otherUrl := fmt.Sprintf("http://localhost:%d/name/%s/%s", port, "other", addrParts[0])
 	resp, err = genForm("PUT", otherUrl,
 		url.Values{"fqdn": {successTestName}, "local_ip": {dockerIP}, "routing_prefix": {addrParts[1]}})
-	assertNoErr(t, err)
+	wt.AssertNoErr(t, err)
-	assertStatus(t, resp.StatusCode, http.StatusOK, "http response")
+	wt.AssertStatus(t, resp.StatusCode, http.StatusOK, "http response")
 
 	// Delete the address
 	resp, err = genForm("DELETE", addrUrl, nil)
-	assertNoErr(t, err)
+	wt.AssertNoErr(t, err)
-	assertStatus(t, resp.StatusCode, http.StatusOK, "http response")
+	wt.AssertStatus(t, resp.StatusCode, http.StatusOK, "http response")
 
 	// Check that the address is still resolvable.
-	_, err = zone.LookupLocal(successTestName)
+	_, err = zone.LookupName(successTestName)
-	assertNoErr(t, err)
+	wt.AssertNoErr(t, err)
 
 	// Delete the address record mentioning the other container
 	resp, err = genForm("DELETE", otherUrl, nil)
-	assertNoErr(t, err)
+	wt.AssertNoErr(t, err)
-	assertStatus(t, resp.StatusCode, http.StatusOK, "http response")
+	wt.AssertStatus(t, resp.StatusCode, http.StatusOK, "http response")
 
 	// Check that the address is gone
-	_, err = zone.LookupLocal(successTestName)
+	_, err = zone.LookupName(successTestName)
-	assertErrorType(t, err, (*LookupError)(nil), "fully-removed address")
+	wt.AssertErrorType(t, err, (*LookupError)(nil), "fully-removed address")
 
 	// Would like to shut down the http server at the end of this test
 	// but it's complicated.

nameserver/mdns_client.go

@@ -26,7 +26,7 @@ var (
 	}
 )
 
-type ResponseA struct {
+type Response struct {
 	Name string
 	Addr net.IP
 	Err  error
@@ -34,7 +34,7 @@ type ResponseA struct {
 
 type responseInfo struct {
 	timeout time.Time // if no answer by this time, give up
-	ch      chan<- *ResponseA
+	ch      chan<- *Response
 }
 
 // Represents one query that we have sent for one name.
@@ -60,7 +60,7 @@ type MDNSClient struct {
 type mDNSQueryInfo struct {
 	name       string
 	querytype  uint16
-	responseCh chan<- *ResponseA
+	responseCh chan<- *Response
 }
 
 func NewMDNSClient() (*MDNSClient, error) {
@@ -104,9 +104,9 @@ func LinkLocalMulticastListener(ifi *net.Interface) (net.PacketConn, error) {
 // ACTOR client API
 
 const (
-	CSendQuery       = iota
+	CSendQuery = iota
-	CShutdown        = iota
+	CShutdown
-	CMessageReceived = iota
+	CMessageReceived
 )
 
 type MDNSInteraction struct {
@@ -121,7 +121,7 @@ func (c *MDNSClient) Shutdown() {
 }
 
 // Async
-func (c *MDNSClient) SendQuery(name string, querytype uint16, responseCh chan<- *ResponseA) {
+func (c *MDNSClient) SendQuery(name string, querytype uint16, responseCh chan<- *Response) {
 	c.queryChan <- &MDNSInteraction{
 		code:    CSendQuery,
 		payload: mDNSQueryInfo{name, querytype, responseCh},
@@ -209,7 +209,7 @@ func (c *MDNSClient) handleSendQuery(q mDNSQueryInfo) {
 		m.RecursionDesired = false
 		buf, err := m.Pack()
 		if err != nil {
-			q.responseCh <- &ResponseA{Err: err}
+			q.responseCh <- &Response{Err: err}
 			close(q.responseCh)
 			return
 		}
@@ -218,7 +218,7 @@ func (c *MDNSClient) handleSendQuery(q mDNSQueryInfo) {
 			id:   m.Id,
 		}
 		if _, err = c.conn.WriteTo(buf, c.addr); err != nil {
-			q.responseCh <- &ResponseA{Err: err}
+			q.responseCh <- &Response{Err: err}
 			close(q.responseCh)
 			return
 		}
@@ -235,19 +235,29 @@ func (c *MDNSClient) handleSendQuery(q mDNSQueryInfo) {
 
 func (c *MDNSClient) handleResponse(r *dns.Msg) {
 	for _, answer := range r.Answer {
+		var name string
+		var res *Response
+
 		switch rr := answer.(type) {
 		case *dns.A:
-			name := rr.Hdr.Name
+			name = rr.Hdr.Name
-			if query, found := c.inflight[name]; found {
+			res = &Response{Addr: rr.A}
-				for _, resp := range query.responseInfos {
+		case *dns.PTR:
-					resp.ch <- &ResponseA{Name: rr.Hdr.Name, Addr: rr.A}
+			name = rr.Hdr.Name
-					close(resp.ch)
+			res = &Response{Name: rr.Ptr}
-				}
+		default:
-				delete(c.inflight, name)
+			return
-			} else {
+		}
-				// We've received a response that didn't match a query
+
-				// Do we want to cache it?
+		if query, found := c.inflight[name]; found {
+			for _, resp := range query.responseInfos {
+				resp.ch <- res
+				close(resp.ch)
 			}
+			delete(c.inflight, name)
+		} else {
+			// We've received a response that didn't match a query
+			// Do we want to cache it?
 		}
 	}
 }

nameserver/mdns_client_test.go

@@ -2,6 +2,7 @@ package nameserver
 
 import (
 	"github.com/miekg/dns"
+	wt "github.com/zettio/weave/testing"
 	"log"
 	"net"
 	"testing"
@@ -12,9 +13,27 @@ var (
 	successTestName = "test1.weave."
 	failTestName    = "test2.weave."
 	testAddr        = net.ParseIP("9.8.7.6")
+	testInAddr      = "6.7.8.9.in-addr.arpa."
 )
 
 func minimalServer(w dns.ResponseWriter, req *dns.Msg) {
+
+	sendAnswer := func(m *dns.Msg) {
+		buf, err := m.Pack()
+		checkFatal(err)
+		if buf == nil {
+			log.Fatal("Nil buffer")
+		}
+		//log.Println("minimalServer sending:", buf)
+		// This is a bit of a kludge - per the RFC we should send responses from 5353, but that doesn't seem to work
+		sendconn, err := net.DialUDP("udp", nil, ipv4Addr)
+		checkFatal(err)
+
+		_, err = sendconn.Write(buf)
+		sendconn.Close()
+		checkFatal(err)
+	}
+
 	//log.Println("minimalServer received:", req)
 	if len(req.Answer) > 0 {
 		return // Only interested in questions.
@@ -22,31 +41,24 @@ func minimalServer(w dns.ResponseWriter, req *dns.Msg) {
 	if len(req.Question) != 1 {
 		return // We only handle single-question messages
 	}
-	if req.Question[0].Name != successTestName {
+
-		return // This is not the DNS record you are looking for
+	if req.Question[0].Qtype == dns.TypeA && req.Question[0].Name == successTestName {
+		m := new(dns.Msg)
+		m.SetReply(req)
+		hdr := dns.RR_Header{Name: m.Question[0].Name, Rrtype: dns.TypeA,
+			Class: dns.ClassINET, Ttl: 3600}
+		a := &dns.A{hdr, testAddr}
+		m.Answer = append(m.Answer, a)
+		sendAnswer(m)
+	} else if req.Question[0].Qtype == dns.TypePTR && req.Question[0].Name == testInAddr {
+		m := new(dns.Msg)
+		m.SetReply(req)
+		hdr := dns.RR_Header{Name: m.Question[0].Name, Rrtype: dns.TypePTR,
+			Class: dns.ClassINET, Ttl: 3600}
+		a := &dns.PTR{hdr, successTestName}
+		m.Answer = append(m.Answer, a)
+		sendAnswer(m)
 	}
-
-	m := new(dns.Msg)
-	m.SetReply(req)
-
-	hdr := dns.RR_Header{Name: m.Question[0].Name, Rrtype: dns.TypeA,
-		Class: dns.ClassINET, Ttl: 3600}
-	a := &dns.A{hdr, testAddr}
-	m.Answer = append(m.Answer, a)
-
-	buf, err := m.Pack()
-	checkFatal(err)
-	if buf == nil {
-		log.Fatal("Nil buffer")
-	}
-	//log.Println("minimalServer sending:", buf)
-	// This is a bit of a kludge - per the RFC we should send responses from 5353, but that doesn't seem to work
-	sendconn, err := net.DialUDP("udp", nil, ipv4Addr)
-	checkFatal(err)
-
-	_, err = sendconn.Write(buf)
-	sendconn.Close()
-	checkFatal(err)
 }
 
 func RunLocalMulticastServer() (*dns.Server, error) {
@@ -61,9 +73,9 @@ func RunLocalMulticastServer() (*dns.Server, error) {
 
 func setup(t *testing.T) (*MDNSClient, *dns.Server, error) {
 	mdnsClient, err := NewMDNSClient()
-	assertNoErr(t, err)
+	wt.AssertNoErr(t, err)
 	err = mdnsClient.Start(nil)
-	assertNoErr(t, err)
+	wt.AssertNoErr(t, err)
 
 	server, err := RunLocalMulticastServer()
 	if err != nil {
@@ -75,19 +87,19 @@ func setup(t *testing.T) (*MDNSClient, *dns.Server, error) {
 type testContext struct {
 	receivedAddr  net.IP
 	receivedCount int
-	channel       chan *ResponseA
+	channel       chan *Response
 }
 
 func newTestContext() *testContext {
-	return &testContext{channel: make(chan *ResponseA, 4)}
+	return &testContext{channel: make(chan *Response, 4)}
 }
 
-func (c *testContext) checkResponse(t *testing.T, channelOk bool, resp *ResponseA) {
+func (c *testContext) checkResponse(t *testing.T, channelOk bool, resp *Response) {
 	if !channelOk {
 		c.channel = nil
 		return
 	}
-	assertNoErr(t, resp.Err)
+	wt.AssertNoErr(t, resp.Err)
 	log.Printf("Got address response %s addr %s", resp.Name, resp.Addr)
 	c.receivedAddr = resp.Addr
 	c.receivedCount++
@@ -151,3 +163,24 @@ outerloop:
 		t.Fatal("Unexpected result for", successTestName, context1.receivedAddr, context2.receivedAddr, context1.receivedCount, context2.receivedCount)
 	}
 }
+
+func TestAsLookup(t *testing.T) {
+	mdnsClient, server, _ := setup(t)
+	defer mdnsClient.Shutdown()
+	defer server.Shutdown()
+
+	ip, err := mdnsClient.LookupName(successTestName)
+	wt.AssertNoErr(t, err)
+	if !testAddr.Equal(ip) {
+		t.Fatalf("Returned address incorrect %s", ip)
+	}
+
+	ip, err = mdnsClient.LookupName("foo.example.com.")
+	wt.AssertErrorType(t, err, (*LookupError)(nil), "unknown hostname")
+
+	name, err := mdnsClient.LookupInaddr(testInAddr)
+	wt.AssertNoErr(t, err)
+	if !(successTestName == name) {
+		t.Fatalf("Expected name %s, got %s", successTestName, name)
+	}
+}

nameserver/mdns_lookup.go

@@ -0,0 +1,40 @@
+package nameserver
+
+import (
+	"github.com/miekg/dns"
+	. "github.com/zettio/weave/common"
+	"net"
+)
+
+func mdnsLookup(client *MDNSClient, name string, qtype uint16) (*Response, error) {
+	channel := make(chan *Response)
+	client.SendQuery(name, qtype, channel)
+	for resp := range channel {
+		if err := resp.Err; err != nil {
+			Debug.Printf("[mdns] Error for query type %s name %s: %s",
+				dns.TypeToString[qtype], name, err)
+			return nil, err
+		} else {
+			Debug.Printf("[mdns] Got response name for query type %s name %s: name %s, addr %s",
+				dns.TypeToString[qtype], name, resp.Name, resp.Addr)
+			return resp, nil
+		}
+	}
+	return nil, LookupError(name)
+}
+
+func (client *MDNSClient) LookupName(name string) (net.IP, error) {
+	if r, e := mdnsLookup(client, name, dns.TypeA); r != nil {
+		return r.Addr, nil
+	} else {
+		return nil, e
+	}
+}
+
+func (client *MDNSClient) LookupInaddr(inaddr string) (string, error) {
+	if r, e := mdnsLookup(client, inaddr, dns.TypePTR); r != nil {
+		return r.Name, nil
+	} else {
+		return "", e
+	}
+}

nameserver/mdns_server.go

@@ -2,6 +2,7 @@ package nameserver
 
 import (
 	"github.com/miekg/dns"
+	. "github.com/zettio/weave/common"
 	"net"
 )
 
@@ -49,24 +50,60 @@ func (s *MDNSServer) Start(ifi *net.Interface) error {
 		return err
 	}
 
-	handleMDNS := func(w dns.ResponseWriter, r *dns.Msg) {
+	handleLocal := s.makeHandler(dns.TypeA,
-		// Ignore answers to other questions
+		func(zone Lookup, r *dns.Msg, q *dns.Question) *dns.Msg {
-		if len(r.Answer) == 0 && len(r.Question) > 0 {
+			if ip, err := zone.LookupName(q.Name); err == nil {
-			q := r.Question[0]
+				return makeAddressReply(r, q, []net.IP{ip})
-			if ip, err := s.zone.LookupLocal(q.Name); err == nil {
-				m := makeAddressReply(r, &q, []net.IP{ip})
-				if err = s.sendResponse(m); err != nil {
-					Warning.Printf("Error writing to %s", w)
-				}
-			} else if s.addrIsLocal(w.RemoteAddr()) {
-				// ignore this - it's our own query received via multicast
 			} else {
-				Debug.Printf("Failed MDNS lookup for %s", q.Name)
+				return nil
+			}
+		})
+
+	handleReverse := s.makeHandler(dns.TypePTR,
+		func(zone Lookup, r *dns.Msg, q *dns.Question) *dns.Msg {
+			if name, err := zone.LookupInaddr(q.Name); err == nil {
+				return makePTRReply(r, q, []string{name})
+			} else {
+				return nil
+			}
+		})
+
+	mux := dns.NewServeMux()
+	mux.HandleFunc(LOCAL_DOMAIN, handleLocal)
+	mux.HandleFunc(RDNS_DOMAIN, handleReverse)
+
+	go dns.ActivateAndServe(nil, conn, mux)
+	return err
+}
+
+type LookupFunc func(Lookup, *dns.Msg, *dns.Question) *dns.Msg
+
+func (s *MDNSServer) makeHandler(qtype uint16, lookup LookupFunc) dns.HandlerFunc {
+	return func(_ dns.ResponseWriter, r *dns.Msg) {
+		// Handle only questions, ignore answers. We might also ignore
+		// questions that arise locally (i.e., that come from an IP we
+		// think is local), but in the interest of avoiding
+		// complication, and easier testing, this is elided on the
+		// assumption that the client wouldn't ask if it already knew
+		// the answer, and if it does ask, it'll be happy to get an
+		// answer.
+		if len(r.Answer) == 0 && len(r.Question) > 0 {
+			q := &r.Question[0]
+			if q.Qtype == qtype {
+				if m := lookup(s.zone, r, q); m != nil {
+					Debug.Printf("[mdns msgid %d] Found local answer to mDNS query %s",
+						r.MsgHdr.Id, q.Name)
+					if err := s.sendResponse(m); err != nil {
+						Warning.Printf("[mdns msgid %d] Error writing to %s",
+							r.MsgHdr.Id, s.sendconn)
+					}
+				} else {
+					Debug.Printf("[mdns msgid %d] No local answer for mDNS query %s",
+						r.MsgHdr.Id, q.Name)
+				}
 			}
 		}
 	}
-	go dns.ActivateAndServe(nil, conn, dns.HandlerFunc(handleMDNS))
-	return err
 }
 
 func (s *MDNSServer) sendResponse(m *dns.Msg) error {

nameserver/mdns_server_test.go

@@ -2,6 +2,8 @@ package nameserver
 
 import (
 	"github.com/miekg/dns"
+	"github.com/zettio/weave/common"
+	wt "github.com/zettio/weave/testing"
 	"log"
 	"net"
 	"testing"
@@ -10,7 +12,9 @@ import (
 
 var (
 	containerID = "deadbeef"
+	testName    = "test.weave.local."
 	testAddr1   = "10.0.2.1/24"
+	testInAddr1 = "1.2.0.10.in-addr.arpa."
 )
 
 func sendQuery(name string, querytype uint16) error {
@@ -30,22 +34,43 @@ func sendQuery(name string, querytype uint16) error {
 }
 
 func TestServerSimpleQuery(t *testing.T) {
+	// The ff can be handy for debugging (obvs)
+	common.InitDefaultLogging(true)
+
 	log.Println("TestServerSimpleQuery starting")
 	var zone = new(ZoneDb)
 	ip, _, _ := net.ParseCIDR(testAddr1)
-	zone.AddRecord(containerID, "test.weave.", ip)
+	zone.AddRecord(containerID, testName, ip)
 
 	mdnsServer, err := NewMDNSServer(zone)
-	assertNoErr(t, err)
+	wt.AssertNoErr(t, err)
 	err = mdnsServer.Start(nil)
-	assertNoErr(t, err)
+	wt.AssertNoErr(t, err)
 
 	var receivedAddr net.IP
+	var receivedName string
+	var recvChan chan interface{}
 	receivedCount := 0
 
+	reset := func() {
+		receivedAddr = nil
+		receivedName = ""
+		receivedCount = 0
+		recvChan = make(chan interface{})
+	}
+
+	wait := func() {
+		select {
+		case <-recvChan:
+			return
+		case <-time.After(100 * time.Millisecond):
+			return
+		}
+	}
+
 	// Implement a minimal listener for responses
 	multicast, err := LinkLocalMulticastListener(nil)
-	assertNoErr(t, err)
+	wt.AssertNoErr(t, err)
 
 	handleMDNS := func(w dns.ResponseWriter, r *dns.Msg) {
 		// Only handle responses here
@@ -55,8 +80,12 @@ func TestServerSimpleQuery(t *testing.T) {
 				case *dns.A:
 					receivedAddr = rr.A
 					receivedCount++
+				case *dns.PTR:
+					receivedName = rr.Ptr
+					receivedCount++
 				}
 			}
+			recvChan <- "ok"
 		}
 	}
 
@@ -66,22 +95,32 @@ func TestServerSimpleQuery(t *testing.T) {
 
 	time.Sleep(100 * time.Millisecond) // Allow for server to get going
 
-	sendQuery("test.weave.", dns.TypeA)
+	reset()
-
+	sendQuery(testName, dns.TypeA)
-	time.Sleep(time.Second)
+	wait()
 
 	if receivedCount != 1 {
-		t.Fatal("Unexpected result count for test.weave", receivedCount)
+		t.Fatalf("Unexpected result count %d for %s", receivedCount, testName)
 	}
 	if !receivedAddr.Equal(ip) {
-		t.Fatal("Unexpected result for test.weave", receivedAddr)
+		t.Fatalf("Unexpected result %s for %s", receivedAddr, testName)
 	}
 
-	receivedCount = 0
+	reset()
-
 	sendQuery("testfail.weave.", dns.TypeA)
+	wait()
 
 	if receivedCount != 0 {
-		t.Fatal("Unexpected result count for testfail.weave", receivedCount)
+		t.Fatalf("Unexpected result count %d for testfail.weave", receivedCount)
+	}
+
+	reset()
+	sendQuery(testInAddr1, dns.TypePTR)
+	wait()
+
+	if receivedCount != 1 {
+		t.Fatalf("Expected an answer to %s, got %d answers", testInAddr1, receivedCount)
+	} else if !(testName == receivedName) {
+		t.Fatalf("Expected answer %s to query for %s, got %s", testName, testInAddr1, receivedName)
 	}
 }

nameserver/server.go

@@ -3,17 +3,14 @@ package nameserver
 import (
 	"fmt"
 	"github.com/miekg/dns"
+	. "github.com/zettio/weave/common"
 	"net"
 )
 
 const (
 	LOCAL_DOMAIN = "weave.local."
-	RDNS_DOMAIN  = "in-addr.arpa."
 )
 
-// +1 to also exclude a dot
-var rdnsDomainLen = len(RDNS_DOMAIN) + 1
-
 func checkFatal(e error) {
 	if e != nil {
 		Error.Fatal(e)
@@ -34,93 +31,80 @@ func makeDNSFailResponse(r *dns.Msg) *dns.Msg {
 	return m
 }
 
-func queryHandler(zone Zone, mdnsClient *MDNSClient) dns.HandlerFunc {
+func queryHandler(lookups []Lookup) dns.HandlerFunc {
 	return func(w dns.ResponseWriter, r *dns.Msg) {
 		q := r.Question[0]
-		Debug.Printf("Local query: %+v", q)
+		Debug.Printf("Query: %+v", q)
 		if q.Qtype == dns.TypeA {
-			if ip, err := zone.LookupLocal(q.Name); err == nil {
+			for _, lookup := range lookups {
-				m := makeAddressReply(r, &q, []net.IP{ip})
+				if ip, err := lookup.LookupName(q.Name); err == nil {
-				w.WriteMsg(m)
+					m := makeAddressReply(r, &q, []net.IP{ip})
-			} else {
-				Debug.Printf("Failed lookup for %s; sending mDNS query", q.Name)
-				// We don't know the answer; see if someone else does
-				channel := make(chan *ResponseA)
-				replies := make([]net.IP, 0)
-				go func() {
-					for resp := range channel {
-						Debug.Printf("Got address response %s to query %s addr %s", resp.Name, q.Name, resp.Addr)
-						replies = append(replies, resp.Addr)
-					}
-					var responseMsg *dns.Msg
-					if len(replies) > 0 {
-						responseMsg = makeAddressReply(r, &q, replies)
-					} else {
-						responseMsg = makeDNSFailResponse(r)
-					}
-					w.WriteMsg(responseMsg)
-				}()
-				mdnsClient.SendQuery(q.Name, dns.TypeA, channel)
-			}
-		} else {
-			Warning.Printf("Local query not handled: %+v", q)
-			m := makeDNSFailResponse(r)
-			w.WriteMsg(m)
-		}
-		return
-	}
-}
-
-func rdnsHandler(zone Zone, mdnsClient *MDNSClient) dns.HandlerFunc {
-	return func(w dns.ResponseWriter, r *dns.Msg) {
-		q := r.Question[0]
-		Debug.Printf("Local rdns query: %+v", q)
-		if q.Qtype == dns.TypePTR {
-			if ip := net.ParseIP(q.Name[:len(q.Name)-rdnsDomainLen]); ip != nil {
-				ip4 := ip.To4()
-				revIP := []byte{ip4[3], ip4[2], ip4[1], ip4[0]}
-				Debug.Printf("Looking for address: %+v", revIP)
-				if name, err := zone.ReverseLookupLocal(revIP); err == nil {
-					Debug.Printf("Found name: %s", name)
-					m := makePTRReply(r, &q, []string{name})
 					w.WriteMsg(m)
-				} else {
+					return
-					Debug.Printf("Failed lookup for %s; sending mDNS query", q.Name)
-					// We don't know the answer; see if someone else does
-					// TODO
 				}
 			}
 		}
+		Info.Printf("[dns msgid %d] No results for type %s query %s",
+			r.MsgHdr.Id, dns.TypeToString[q.Qtype], q.Name)
+		w.WriteMsg(makeDNSFailResponse(r))
 	}
 }
 
-/* When we receive a request for a name outside of our '.weave' domain, call
+func rdnsHandler(lookups []Lookup) dns.HandlerFunc {
-   the underlying lookup mechanism and return the answer(s) it gives.
+	fallback := notUsHandler()
-   Unfortunately, this means that TTLs from a real DNS server are lost - FIXME.
-*/
-func notUsHandler() dns.HandlerFunc {
 	return func(w dns.ResponseWriter, r *dns.Msg) {
 		q := r.Question[0]
-		Debug.Printf("Non-local query: %+v", q)
+		Debug.Printf("Reverse query: %+v", q)
-		var responseMsg *dns.Msg
+		if q.Qtype == dns.TypePTR {
-		if q.Qtype == dns.TypeA || q.Qtype == dns.TypeAAAA {
+			for _, lookup := range lookups {
-			if addrs, err := net.LookupIP(q.Name); err == nil {
+				if name, err := lookup.LookupInaddr(q.Name); err == nil {
-				responseMsg = makeAddressReply(r, &q, addrs)
+					m := makePTRReply(r, &q, []string{name})
-			} else {
+					w.WriteMsg(m)
-				responseMsg = makeDNSFailResponse(r)
+					return
-				Debug.Print("Failed fallback lookup ", err)
+				}
 			}
+			fallback(w, r)
+			return
 		} else {
-			Warning.Printf("Non-local query not handled: %+v", q)
+			Warning.Printf("[dns msgid %d] Unexpected reverse query type %s: %+v",
-			responseMsg = makeDNSFailResponse(r)
+				r.MsgHdr.Id, dns.TypeToString[q.Qtype], q)
 		}
-		w.WriteMsg(responseMsg)
 	}
 }
 
-func StartServer(zone Zone, iface *net.Interface, dnsPort int, httpPort int, wait int) error {
+/* When we receive a request for a name outside of our '.weave.local.'
-	go ListenHttp(LOCAL_DOMAIN, zone, httpPort)
+   domain, ask the configured DNS server as a fallback.
+*/
+func notUsHandler() dns.HandlerFunc {
+	config, err := dns.ClientConfigFromFile("/etc/resolv.conf")
+	checkFatal(err)
+	return func(w dns.ResponseWriter, r *dns.Msg) {
+		q := r.Question[0]
+		Debug.Printf("[dns msgid %d] Fallback query: %+v", r.MsgHdr.Id, q)
+		for _, server := range config.Servers {
+			reply, err := dns.Exchange(r, fmt.Sprintf("%s:%s", server, config.Port))
+			if err != nil {
+				Debug.Printf("[dns msgid %d] Network error trying %s (%s)",
+					r.MsgHdr.Id, server, err)
+				continue
+			}
+			if reply != nil && reply.Rcode != dns.RcodeSuccess {
+				Debug.Printf("[dns msgid %d] Failure reported by %s for query %s",
+					r.MsgHdr.Id, server, q.Name)
+				continue
+			}
+			Debug.Printf("[dns msgid %d] Given answer by %s for query %s",
+				r.MsgHdr.Id, server, q.Name)
+			w.WriteMsg(reply)
+			return
+		}
+		Warning.Printf("[dns msgid %d] Failed lookup for external name %s",
+			r.MsgHdr.Id, q.Name)
+		w.WriteMsg(makeDNSFailResponse(r))
+	}
+}
 
+func StartServer(zone Zone, iface *net.Interface, dnsPort int, wait int) error {
 	mdnsClient, err := NewMDNSClient()
 	checkFatal(err)
 
@@ -133,8 +117,8 @@ func StartServer(zone Zone, iface *net.Interface, dnsPort int, httpPort int, wai
 	checkFatal(err)
 
 	LocalServeMux := dns.NewServeMux()
-	LocalServeMux.HandleFunc(LOCAL_DOMAIN, queryHandler(zone, mdnsClient))
+	LocalServeMux.HandleFunc(LOCAL_DOMAIN, queryHandler([]Lookup{zone, mdnsClient}))
-	LocalServeMux.HandleFunc(RDNS_DOMAIN, rdnsHandler(zone, mdnsClient))
+	LocalServeMux.HandleFunc(RDNS_DOMAIN, rdnsHandler([]Lookup{zone, mdnsClient}))
 	LocalServeMux.HandleFunc(".", notUsHandler())
 
 	mdnsServer, err := NewMDNSServer(zone)

nameserver/server_test.go

@@ -0,0 +1,91 @@
+package nameserver
+
+import (
+	"fmt"
+	"github.com/miekg/dns"
+	"github.com/zettio/weave/common"
+	wt "github.com/zettio/weave/testing"
+	"net"
+	"testing"
+	"time"
+)
+
+func TestDNSServer(t *testing.T) {
+	const (
+		port            = 17625
+		successTestName = "test1.weave.local."
+		failTestName    = "test2.weave.local."
+		nonLocalName    = "weave.works."
+		testAddr1       = "10.0.2.1"
+		testRDNSsuccess = "1.2.0.10.in-addr.arpa."
+		testRDNSfail    = "4.3.2.1.in-addr.arpa."
+	)
+	dnsAddr := fmt.Sprintf("localhost:%d", port)
+	testCIDR1 := testAddr1 + "/24"
+
+	common.InitDefaultLogging(true)
+	var zone = new(ZoneDb)
+	ip, _, _ := net.ParseCIDR(testCIDR1)
+	zone.AddRecord(containerID, successTestName, ip)
+
+	go StartServer(zone, nil, port, 0)
+	time.Sleep(100 * time.Millisecond) // Allow sever goroutine to start
+
+	c := new(dns.Client)
+	m := new(dns.Msg)
+	m.SetQuestion(successTestName, dns.TypeA)
+	m.RecursionDesired = true
+	r, _, err := c.Exchange(m, dnsAddr)
+	wt.AssertNoErr(t, err)
+	wt.AssertStatus(t, r.Rcode, dns.RcodeSuccess, "DNS response code")
+	wt.AssertEqualInt(t, len(r.Answer), 1, "Number of answers")
+	wt.AssertType(t, r.Answer[0], (*dns.A)(nil), "DNS record")
+	wt.AssertEqualString(t, r.Answer[0].(*dns.A).A.String(), testAddr1, "IP address")
+
+	m.SetQuestion(failTestName, dns.TypeA)
+	r, _, err = c.Exchange(m, dnsAddr)
+	wt.AssertNoErr(t, err)
+	wt.AssertStatus(t, r.Rcode, dns.RcodeNameError, "DNS response code")
+	wt.AssertEqualInt(t, len(r.Answer), 0, "Number of answers")
+
+	m.SetQuestion(testRDNSsuccess, dns.TypePTR)
+	r, _, err = c.Exchange(m, dnsAddr)
+	wt.AssertNoErr(t, err)
+	wt.AssertStatus(t, r.Rcode, dns.RcodeSuccess, "DNS response code")
+	wt.AssertEqualInt(t, len(r.Answer), 1, "Number of answers")
+	wt.AssertType(t, r.Answer[0], (*dns.PTR)(nil), "DNS record")
+	wt.AssertEqualString(t, r.Answer[0].(*dns.PTR).Ptr, successTestName, "IP address")
+	m.SetQuestion(testRDNSfail, dns.TypePTR)
+	r, _, err = c.Exchange(m, dnsAddr)
+	wt.AssertNoErr(t, err)
+	wt.AssertStatus(t, r.Rcode, dns.RcodeNameError, "DNS response code")
+	wt.AssertEqualInt(t, len(r.Answer), 0, "Number of answers")
+
+	// This should fail because we don't handle MX records
+	m.SetQuestion(successTestName, dns.TypeMX)
+	r, _, err = c.Exchange(m, dnsAddr)
+	wt.AssertNoErr(t, err)
+	wt.AssertStatus(t, r.Rcode, dns.RcodeNameError, "DNS response code")
+	wt.AssertEqualInt(t, len(r.Answer), 0, "Number of answers")
+
+	// This non-local query for an MX record should succeed by being
+	// passed on to the configured (/etc/resolv.conf) DNS server.
+	m.SetQuestion(nonLocalName, dns.TypeMX)
+	r, _, err = c.Exchange(m, dnsAddr)
+	wt.AssertNoErr(t, err)
+	wt.AssertStatus(t, r.Rcode, dns.RcodeSuccess, "DNS response code")
+	if !(len(r.Answer) > 0) {
+		t.Fatal("Number of answers > 0")
+	}
+
+	m.SetQuestion("8.8.8.8.in-addr.arpa.", dns.TypePTR)
+	r, _, err = c.Exchange(m, dnsAddr)
+	wt.AssertNoErr(t, err)
+	wt.AssertStatus(t, r.Rcode, dns.RcodeSuccess, "DNS success response code")
+	if !(len(r.Answer) > 0) {
+		t.Fatal("Number of answers > 0")
+	}
+
+	// Not testing MDNS functionality of server here (yet), since it
+	// needs two servers, each listening on its own address
+}

nameserver/testing_utils.go

@@ -1,38 +0,0 @@
-package nameserver
-
-import (
-	"reflect"
-	"testing"
-)
-
-func assertNoErr(t *testing.T, err error) {
-	if err != nil {
-		t.Fatal(err)
-	}
-}
-
-func assertStatus(t *testing.T, got int, wanted int, desc string) {
-	if got != wanted {
-		t.Fatalf("Expected %s %d but got %d", desc, wanted, got)
-	}
-}
-
-func assertErrorInterface(t *testing.T, got interface{}, wanted interface{}, desc string) {
-	if got == nil {
-		t.Fatalf("Expected %s but got nil (%s)", reflect.TypeOf(wanted).Elem(), desc)
-	}
-	gotT, wantedT := reflect.TypeOf(got), reflect.TypeOf(wanted).Elem()
-	if !gotT.Implements(wantedT) {
-		t.Fatalf("Expected %s but got %s (%s)", wantedT, gotT, desc)
-	}
-}
-
-func assertErrorType(t *testing.T, got interface{}, wanted interface{}, desc string) {
-	if got == nil {
-		t.Fatalf("Expected %s but got nil (%s)", reflect.TypeOf(wanted).Elem(), desc)
-	}
-	gotT, wantedT := reflect.TypeOf(got), reflect.TypeOf(wanted).Elem()
-	if gotT != wantedT {
-		t.Fatalf("Expected %s but got %s (%s)", wantedT, gotT, desc)
-	}
-}

nameserver/updater.go

@@ -2,11 +2,13 @@ package nameserver
 
 import (
 	"github.com/fsouza/go-dockerclient"
+	. "github.com/zettio/weave/common"
 )
 
 func checkError(err error, apiPath string) {
 	if err != nil {
-		Error.Fatalf("Unable to connect to Docker API on %s: %s", apiPath, err)
+		Error.Fatalf("[updater] Unable to connect to Docker API on %s: %s",
+			apiPath, err)
 	}
 }
 
@@ -21,7 +23,7 @@ func StartUpdater(apiPath string, zone Zone) error {
 	err = client.AddEventListener(events)
 	checkError(err, apiPath)
 
-	Info.Printf("Using Docker API on %s: %v", apiPath, env)
+	Info.Printf("[updater] Using Docker API on %s: %v", apiPath, env)
 
 	go func() {
 		for event := range events {
@@ -35,7 +37,7 @@ func handleEvent(zone Zone, event *docker.APIEvents, client *docker.Client) erro
 	switch event.Status {
 	case "die":
 		id := event.ID
-		Info.Printf("Container %s down. Removing records", id)
+		Info.Printf("[updater] Container %s down. Removing records", id)
 		zone.DeleteRecordsFor(id)
 	}
 	return nil

nameserver/zone.go

@@ -2,16 +2,28 @@ package nameserver
 
 import (
 	"github.com/miekg/dns"
+	. "github.com/zettio/weave/common"
 	"net"
 	"sync"
 )
 
+const (
+	RDNS_DOMAIN = "in-addr.arpa."
+)
+
+// +1 to also exclude a dot
+var rdnsDomainLen = len(RDNS_DOMAIN) + 1
+
+type Lookup interface {
+	LookupName(name string) (net.IP, error)
+	LookupInaddr(inaddr string) (string, error)
+}
+
 type Zone interface {
 	AddRecord(ident string, name string, ip net.IP) error
 	DeleteRecord(ident string, ip net.IP) error
 	DeleteRecordsFor(ident string) error
-	LookupLocal(name string) (net.IP, error)
+	Lookup
-	ReverseLookupLocal(ip net.IP) (string, error)
 }
 
 type Record struct {
@@ -49,7 +61,7 @@ func (zone *ZoneDb) indexOf(match func(Record) bool) int {
 	return -1
 }
 
-func (zone *ZoneDb) LookupLocal(name string) (net.IP, error) {
+func (zone *ZoneDb) LookupName(name string) (net.IP, error) {
 	zone.mx.RLock()
 	defer zone.mx.RUnlock()
 	for _, r := range zone.recs {
@@ -60,15 +72,23 @@ func (zone *ZoneDb) LookupLocal(name string) (net.IP, error) {
 	return nil, LookupError(name)
 }
 
-func (zone *ZoneDb) ReverseLookupLocal(ip net.IP) (string, error) {
+func (zone *ZoneDb) LookupInaddr(inaddr string) (string, error) {
-	zone.mx.RLock()
+	if revIP := net.ParseIP(inaddr[:len(inaddr)-rdnsDomainLen]); revIP != nil {
-	defer zone.mx.RUnlock()
+		revIP4 := revIP.To4()
-	for _, r := range zone.recs {
+		ip := []byte{revIP4[3], revIP4[2], revIP4[1], revIP4[0]}
-		if r.IP.Equal(ip) {
+		Debug.Printf("[zonedb] Looking for address: %+v", ip)
-			return r.Name, nil
+		zone.mx.RLock()
+		defer zone.mx.RUnlock()
+		for _, r := range zone.recs {
+			if r.IP.Equal(ip) {
+				return r.Name, nil
+			}
 		}
+		return "", LookupError(inaddr)
+	} else {
+		Warning.Printf("[zonedb] Asked to reverse lookup %s", inaddr)
+		return "", LookupError(inaddr)
 	}
-	return "", LookupError(ip.String())
 }
 
 func (zone *ZoneDb) AddRecord(ident string, name string, ip net.IP) error {

nameserver/zone_test.go

@@ -1,6 +1,7 @@
 package nameserver
 
 import (
+	wt "github.com/zettio/weave/testing"
 	"net"
 	"testing"
 )
@@ -17,51 +18,51 @@ func TestZone(t *testing.T) {
 
 	ip, _, _ := net.ParseCIDR(testAddr1)
 	err := zone.AddRecord(containerID, successTestName, ip)
-	assertNoErr(t, err)
+	wt.AssertNoErr(t, err)
 
 	// Add a few more records to make the job harder
 	err = zone.AddRecord("abcdef0123", "adummy.weave.", net.ParseIP("10.0.0.1"))
-	assertNoErr(t, err)
+	wt.AssertNoErr(t, err)
 	err = zone.AddRecord("0123abcdef", "zdummy.weave.", net.ParseIP("10.0.0.2"))
-	assertNoErr(t, err)
+	wt.AssertNoErr(t, err)
 
 	// Check that the address is now there.
-	foundIP, err := zone.LookupLocal(successTestName)
+	foundIP, err := zone.LookupName(successTestName)
-	assertNoErr(t, err)
+	wt.AssertNoErr(t, err)
 
 	if !foundIP.Equal(ip) {
 		t.Fatal("Unexpected result for", successTestName, foundIP)
 	}
 
 	// See if we can find the address by IP.
-	foundName, err := zone.ReverseLookupLocal(ip)
+	foundName, err := zone.LookupInaddr("1.2.0.10.in-addr.arpa.")
-	assertNoErr(t, err)
+	wt.AssertNoErr(t, err)
 
 	if foundName != successTestName {
 		t.Fatal("Unexpected result for", ip, foundName)
 	}
 
 	err = zone.AddRecord(containerID, successTestName, ip)
-	assertErrorType(t, err, (*DuplicateError)(nil), "duplicate add")
+	wt.AssertErrorType(t, err, (*DuplicateError)(nil), "duplicate add")
 
 	err = zone.AddRecord(otherContainerID, successTestName, ip)
 	// Delete the record for the original container
 	err = zone.DeleteRecord(containerID, ip)
-	assertNoErr(t, err)
+	wt.AssertNoErr(t, err)
 
-	_, err = zone.LookupLocal(successTestName)
+	_, err = zone.LookupName(successTestName)
-	assertNoErr(t, err)
+	wt.AssertNoErr(t, err)
 
 	err = zone.DeleteRecord(otherContainerID, ip)
-	assertNoErr(t, err)
+	wt.AssertNoErr(t, err)
 
 	// Check that the address is not there now.
-	_, err = zone.LookupLocal(successTestName)
+	_, err = zone.LookupName(successTestName)
-	assertErrorType(t, err, (*LookupError)(nil), "after deleting record")
+	wt.AssertErrorType(t, err, (*LookupError)(nil), "after deleting record")
 
 	// Delete a record that isn't there
 	err = zone.DeleteRecord(containerID, net.ParseIP("0.0.0.0"))
-	assertErrorType(t, err, (*LookupError)(nil), "when deleting record that doesn't exist")
+	wt.AssertErrorType(t, err, (*LookupError)(nil), "when deleting record that doesn't exist")
 }
 
 func TestDeleteFor(t *testing.T) {
@@ -75,13 +76,13 @@ func TestDeleteFor(t *testing.T) {
 	for _, addr := range []string{addr1, addr2} {
 		ip, _, _ := net.ParseCIDR(addr)
 		err := zone.AddRecord(id, name, ip)
-		assertNoErr(t, err)
+		wt.AssertNoErr(t, err)
 	}
 
-	_, err := zone.LookupLocal(name)
+	_, err := zone.LookupName(name)
-	assertNoErr(t, err)
+	wt.AssertNoErr(t, err)
 
 	err = zone.DeleteRecordsFor(id)
-	_, err = zone.LookupLocal(name)
+	_, err = zone.LookupName(name)
-	assertErrorType(t, err, (*LookupError)(nil), "after deleting records for ident")
+	wt.AssertErrorType(t, err, (*LookupError)(nil), "after deleting records for ident")
 }

router/connection.go

@@ -8,14 +8,70 @@ import (
 	"log"
 	"net"
 	"strconv"
+	"sync"
 	"time"
 )
 
-func NewRemoteConnection(from, to *Peer, tcpAddr string) *RemoteConnection {
+type Connection interface {
+	Local() *Peer
+	Remote() *Peer
+	BreakTie(Connection) ConnectionTieBreak
+	RemoteTCPAddr() string
+	Established() bool
+	Shutdown(error)
+}
+
+type ConnectionTieBreak int
+
+const (
+	TieBreakWon ConnectionTieBreak = iota
+	TieBreakLost
+	TieBreakTied
+)
+
+type RemoteConnection struct {
+	local         *Peer
+	remote        *Peer
+	remoteTCPAddr string
+	established   bool
+}
+
+type LocalConnection struct {
+	sync.RWMutex
+	RemoteConnection
+	TCPConn            *net.TCPConn
+	tcpSender          TCPSender
+	remoteUDPAddr      *net.UDPAddr
+	receivedHeartbeat  bool
+	stackFrag          bool
+	effectivePMTU      int
+	SessionKey         *[32]byte
+	establishedTimeout *time.Timer
+	heartbeatFrame     *ForwardedFrame
+	heartbeat          *time.Ticker
+	fragTest           *time.Ticker
+	forwardChan        chan<- *ForwardedFrame
+	forwardChanDF      chan<- *ForwardedFrame
+	stopForward        chan<- interface{}
+	stopForwardDF      chan<- interface{}
+	verifyPMTU         chan<- int
+	Decryptor          Decryptor
+	Router             *Router
+	uid                uint64
+	queryChan          chan<- *ConnectionInteraction
+}
+
+type ConnectionInteraction struct {
+	Interaction
+	payload interface{}
+}
+
+func NewRemoteConnection(from, to *Peer, tcpAddr string, established bool) *RemoteConnection {
 	return &RemoteConnection{
 		local:         from,
 		remote:        to,
-		remoteTCPAddr: tcpAddr}
+		remoteTCPAddr: tcpAddr,
+		established:   established}
 }
 
 func (conn *RemoteConnection) Local() *Peer {
@@ -26,15 +82,19 @@ func (conn *RemoteConnection) Remote() *Peer {
 	return conn.remote
 }
 
+func (conn *RemoteConnection) BreakTie(Connection) ConnectionTieBreak {
+	return TieBreakTied
+}
+
 func (conn *RemoteConnection) RemoteTCPAddr() string {
 	return conn.remoteTCPAddr
 }
 
-func (conn *RemoteConnection) Shutdown() {
+func (conn *RemoteConnection) Established() bool {
+	return conn.established
 }
 
-func (conn *RemoteConnection) Established() bool {
+func (conn *RemoteConnection) Shutdown(error) {
-	return true
 }
 
 func (conn *RemoteConnection) String() string {
@@ -49,29 +109,38 @@ func (conn *RemoteConnection) String() string {
 	return fmt.Sprint("Connection ", from, "->", to)
 }
 
-// Async. Does not return anything. If the connection is successful,
+func NewLocalConnection(connRemote *RemoteConnection, tcpConn *net.TCPConn, udpAddr *net.UDPAddr, router *Router) *LocalConnection {
-// it will end up in the local peer's connections map.
-func NewLocalConnection(connRemote *RemoteConnection, acceptNewPeer bool, tcpConn *net.TCPConn, udpAddr *net.UDPAddr, router *Router) {
 	if connRemote.local != router.Ourself.Peer {
 		log.Fatal("Attempt to create local connection from a peer which is not ourself")
 	}
-
-	queryChan := make(chan *ConnectionInteraction, ChannelSize)
 	// NB, we're taking a copy of connRemote here.
-	connLocal := &LocalConnection{
+	return &LocalConnection{
 		RemoteConnection: *connRemote,
 		Router:           router,
 		TCPConn:          tcpConn,
 		remoteUDPAddr:    udpAddr,
-		effectivePMTU:    DefaultPMTU,
+		effectivePMTU:    DefaultPMTU}
-		queryChan:        queryChan}
-	go connLocal.queryLoop(queryChan, acceptNewPeer)
 }
 
-func (conn *LocalConnection) Established() bool {
+// Async. Does not return anything. If the connection is successful,
-	conn.RLock()
+// it will end up in the local peer's connections map.
-	defer conn.RUnlock()
+func (conn *LocalConnection) Start(acceptNewPeer bool) {
-	return conn.established
+	queryChan := make(chan *ConnectionInteraction, ChannelSize)
+	conn.queryChan = queryChan
+	go conn.run(queryChan, acceptNewPeer)
+}
+
+func (conn *LocalConnection) BreakTie(dupConn Connection) ConnectionTieBreak {
+	dupConnLocal := dupConn.(*LocalConnection)
+	// conn.uid is used as the tie breaker here, in the knowledge that
+	// both sides will make the same decision.
+	if conn.uid < dupConnLocal.uid {
+		return TieBreakWon
+	} else if dupConnLocal.uid < conn.uid {
+		return TieBreakLost
+	} else {
+		return TieBreakTied
+	}
 }
 
 // Read by the forwarder processes when in the UDP senders
@@ -81,17 +150,10 @@ func (conn *LocalConnection) RemoteUDPAddr() *net.UDPAddr {
 	return conn.remoteUDPAddr
 }
 
-// Called by the forwarder processes in a few places (including
+func (conn *LocalConnection) Established() bool {
-// crypto), but the connection TCP receiver process, and by the local
+	conn.RLock()
-// peer actor process. Do not call this from the connection's actor
+	defer conn.RUnlock()
-// process itself.
+	return conn.established
-func (conn *LocalConnection) CheckFatal(err error) error {
-	if err == nil {
-		return nil
-	}
-	conn.log("error:", err)
-	conn.Shutdown()
-	return err
 }
 
 // Called by forwarder processes, read in Forward (by sniffer and udp
@@ -115,32 +177,36 @@ func (conn *LocalConnection) setStackFrag(frag bool) {
 }
 
 func (conn *LocalConnection) log(args ...interface{}) {
-	v := append([]interface{}{}, fmt.Sprintf("->[%s]:", conn.remote.Name))
+	log.Println(append(append([]interface{}{}, fmt.Sprintf("->[%s]:", conn.remote.Name)), args...)...)
-	v = append(v, args...)
-	log.Println(v...)
 }
 
 // ACTOR client API
 
 const (
-	CSendTCP          = iota
+	CSendProtocolMsg = iota
-	CSetEstablished   = iota
+	CSetEstablished
-	CSetRemoteUDPAddr = iota
+	CReceivedHeartbeat
-	CShutdown         = iota
+	CShutdown
 )
 
 // Async
-func (conn *LocalConnection) Shutdown() {
+func (conn *LocalConnection) Shutdown(err error) {
-	conn.queryChan <- &ConnectionInteraction{Interaction: Interaction{code: CShutdown}}
+	conn.queryChan <- &ConnectionInteraction{
+		Interaction: Interaction{code: CShutdown},
+		payload:     err}
 }
 
 // Async
-func (conn *LocalConnection) SetRemoteUDPAddr(remoteUDPAddr *net.UDPAddr) {
+//
-	if remoteUDPAddr == nil {
+// Heartbeating serves two purposes: a) keeping NAT paths alive, and
+// b) updating a remote peer's knowledge of our address, in the event
+// it changes (e.g. because NAT paths expired).
+func (conn *LocalConnection) ReceivedHeartbeat(remoteUDPAddr *net.UDPAddr, connUID uint64) {
+	if remoteUDPAddr == nil || connUID != conn.uid {
 		return
 	}
 	conn.queryChan <- &ConnectionInteraction{
-		Interaction: Interaction{code: CSetRemoteUDPAddr},
+		Interaction: Interaction{code: CReceivedHeartbeat},
 		payload:     remoteUDPAddr}
 }
 
@@ -150,34 +216,76 @@ func (conn *LocalConnection) SetEstablished() {
 }
 
 // Async
-func (conn *LocalConnection) SendTCP(msg []byte) {
+func (conn *LocalConnection) SendProtocolMsg(m ProtocolMsg) {
 	conn.queryChan <- &ConnectionInteraction{
-		Interaction: Interaction{code: CSendTCP},
+		Interaction: Interaction{code: CSendProtocolMsg},
-		payload:     msg}
+		payload:     m}
 }
 
 // ACTOR server
 
-func (conn *LocalConnection) queryLoop(queryChan <-chan *ConnectionInteraction, acceptNewPeer bool) {
+func (conn *LocalConnection) run(queryChan <-chan *ConnectionInteraction, acceptNewPeer bool) {
-	err := conn.handshake(acceptNewPeer)
+	defer conn.handleShutdown()
-	if err != nil {
+
-		log.Printf("->[%s] encountered error during handshake: %v\n", conn.remoteTCPAddr, err)
+	tcpConn := conn.TCPConn
-		conn.handleShutdown()
+	tcpConn.SetLinger(0)
+	enc := gob.NewEncoder(tcpConn)
+	dec := gob.NewDecoder(tcpConn)
+
+	if err := conn.handshake(enc, dec, acceptNewPeer); err != nil {
+		log.Printf("->[%s] connection shutting down due to error during handshake: %v\n", conn.remoteTCPAddr, err)
 		return
 	}
-	conn.Router.Ourself.AddConnection(conn)
+	log.Printf("->[%s] completed handshake with %s\n", conn.remoteTCPAddr, conn.remote.Name)
+
+	// We invoke AddConnection in the same goroutine that subsequently
+	// becomes the tcp receive loop, rather than outside, because a)
+	// the ordering relative to the receive loop is the only one that
+	// matters [1], b) it prevents unnecessary delays in entering the
+	// main connection loop, and c) it guards against potential
+	// deadlocks.
+	go func() {
+		conn.Router.Ourself.AddConnection(conn)
+		conn.receiveTCP(dec)
+	}()
+
+	heartbeatFrameBytes := make([]byte, EthernetOverhead+8)
+	binary.BigEndian.PutUint64(heartbeatFrameBytes[EthernetOverhead:], conn.uid)
+	conn.heartbeatFrame = &ForwardedFrame{
+		srcPeer: conn.local,
+		dstPeer: conn.remote,
+		frame:   heartbeatFrameBytes}
+
 	if conn.remoteUDPAddr != nil {
-		if err = conn.ensureForwarders(); err == nil {
+		if err := conn.sendFastHeartbeats(); err != nil {
-			conn.heartbeat = time.NewTicker(FastHeartbeat)
+			conn.log("connection shutting down due to error:", err)
-			conn.forwardHeartbeatFrame() // avoid initial wait
+			return
 		}
 	}
+
+	conn.establishedTimeout = time.NewTimer(EstablishedTimeout)
+
+	if err := conn.queryLoop(queryChan); err != nil {
+		conn.log("connection shutting down due to error:", err)
+	} else {
+		conn.log("connection shutting down")
+	}
+}
+
+// [1] In the absence of any indirect connectivity to the remote peer,
+// the first we hear about it (and any peers reachable from it) is
+// through topology gossip it sends us on the connection. We must
+// ensure that the connection has been added to Ourself prior to
+// processing any such gossip, otherwise we risk immediately gc'ing
+// part of that newly received portion of the topology (though not the
+// remote peer itself, since that will have a positive ref count),
+// leaving behind dangling references to peers. Therefore we invoke
+// AddConnection, which is *synchronous*, before entering the tcp
+// receive loop.
+
+func (conn *LocalConnection) queryLoop(queryChan <-chan *ConnectionInteraction) (err error) {
 	terminate := false
-	for !terminate {
+	for !terminate && err == nil {
-		if err != nil {
-			conn.log("error:", err)
-			break
-		}
 		select {
 		case query, ok := <-queryChan:
 			if !ok {
@@ -185,95 +293,97 @@ func (conn *LocalConnection) queryLoop(queryChan <-chan *ConnectionInteraction,
 			}
 			switch query.code {
 			case CShutdown:
+				err = query.payload.(error)
 				terminate = true
+			case CReceivedHeartbeat:
+				err = conn.handleReceivedHeartbeat(query.payload.(*net.UDPAddr))
 			case CSetEstablished:
+				conn.establishedTimeout.Stop()
 				err = conn.handleSetEstablished()
-			case CSetRemoteUDPAddr:
+			case CSendProtocolMsg:
-				err = conn.handleSetRemoteUDPAddr(query.payload.(*net.UDPAddr))
+				err = conn.handleSendProtocolMsg(query.payload.(ProtocolMsg))
-			case CSendTCP:
+			}
-				err = conn.handleSendTCP(query.payload.([]byte))
+		case <-conn.establishedTimeout.C:
+			if !conn.established {
+				err = fmt.Errorf("failed to establish UDP connectivity")
 			}
 		case <-tickerChan(conn.heartbeat):
-			conn.forwardHeartbeatFrame()
+			conn.Forward(true, conn.heartbeatFrame, nil)
-		case <-tickerChan(conn.fetchAll):
-			err = conn.handleSendTCP(ProtocolFetchAllByte)
 		case <-tickerChan(conn.fragTest):
 			conn.setStackFrag(false)
-			err = conn.handleSendTCP(ProtocolStartFragmentationTestByte)
+			err = conn.handleSendSimpleProtocolMsg(ProtocolStartFragmentationTest)
-
 		}
 	}
-	conn.handleShutdown()
+	return
 }
 
-func (conn *LocalConnection) handleSetRemoteUDPAddr(remoteUDPAddr *net.UDPAddr) error {
+// Handlers
-	if remoteUDPAddr == nil {
+//
-		return nil
+// NB: The conn.* fields are only written by the connection actor
-	}
+// process, which is the caller of the handlers. Hence we do not need
+// locks for reading, and only need write locks for fields read by
+// other processes.
+
+func (conn *LocalConnection) handleReceivedHeartbeat(remoteUDPAddr *net.UDPAddr) error {
+	oldRemoteUDPAddr := conn.remoteUDPAddr
+	old := conn.receivedHeartbeat
 	conn.Lock()
-	old := conn.remoteUDPAddr
 	conn.remoteUDPAddr = remoteUDPAddr
+	conn.receivedHeartbeat = true
 	conn.Unlock()
-	if old == nil {
+	if !old {
-		if err := conn.handleSendTCP(ProtocolConnectionEstablishedByte); err != nil {
+		if err := conn.handleSendSimpleProtocolMsg(ProtocolConnectionEstablished); err != nil {
 			return err
 		}
-		return conn.handleSetEstablished()
+	}
-	} else if old.String() != remoteUDPAddr.String() {
+	if oldRemoteUDPAddr == nil {
+		return conn.sendFastHeartbeats()
+	} else if oldRemoteUDPAddr.String() != remoteUDPAddr.String() {
 		log.Println("Peer", conn.remote.Name, "moved from", old, "to", remoteUDPAddr)
 	}
 	return nil
 }
 
 func (conn *LocalConnection) handleSetEstablished() error {
-	conn.Lock()
+	stopTicker(conn.heartbeat)
 	old := conn.established
+	conn.Lock()
 	conn.established = true
 	conn.Unlock()
-	if !old {
+	if old {
-		conn.Router.Ourself.ConnectionEstablished(conn)
+		return nil
-		if err := conn.ensureForwarders(); err != nil {
+	}
-			return err
+	conn.Router.Ourself.ConnectionEstablished(conn)
-		}
+	if err := conn.ensureForwarders(); err != nil {
-		stopTicker(conn.heartbeat)
+		return err
-		conn.heartbeat = time.NewTicker(SlowHeartbeat)
+	}
-		conn.fetchAll = time.NewTicker(FetchAllInterval)
+	// Send a large frame down the DF channel in order to prompt
-		conn.fragTest = time.NewTicker(FragTestInterval)
+	// PMTU discovery to start.
-		conn.forwardHeartbeatFrame() // avoid initial wait
+	conn.Forward(true, &ForwardedFrame{
-		// Send a large frame down the DF channel in order to prompt
+		srcPeer: conn.local,
-		// PMTU discovery to start.
+		dstPeer: conn.remote,
-		conn.Forward(true, &ForwardedFrame{
+		frame:   PMTUDiscovery},
-			srcPeer: conn.local,
+		nil)
-			dstPeer: conn.remote,
+	conn.heartbeat = time.NewTicker(SlowHeartbeat)
-			frame:   PMTUDiscovery},
+	conn.fragTest = time.NewTicker(FragTestInterval)
-			nil)
+	// avoid initial waits for timers to fire
-		conn.setStackFrag(false)
+	conn.Forward(true, conn.heartbeatFrame, nil)
-		return conn.handleSendTCP(ProtocolStartFragmentationTestByte)
+	conn.setStackFrag(false)
+	if err := conn.handleSendSimpleProtocolMsg(ProtocolStartFragmentationTest); err != nil {
+		return err
 	}
 	return nil
 }
 
-func (conn *LocalConnection) handleSendTCP(msg []byte) error {
+func (conn *LocalConnection) handleSendSimpleProtocolMsg(tag ProtocolTag) error {
-	return conn.tcpSender.Send(msg)
+	return conn.handleSendProtocolMsg(ProtocolMsg{tag: tag})
+}
+
+func (conn *LocalConnection) handleSendProtocolMsg(m ProtocolMsg) error {
+	return conn.tcpSender.Send(Concat([]byte{byte(m.tag)}, m.msg))
 }
 
 func (conn *LocalConnection) handleShutdown() {
-	if conn.remote != nil {
-		conn.log("connection shutting down")
-	}
-
-	// Whilst some of these elements may have been written to whilst
-	// holding locks, they were only written to by the connection
-	// actor process. handleShutdown is only called by the connection
-	// actor process. So there is no need to take locks to read these
-	// (or write elements which are only read by the same
-	// process). Taking locks is only done for elements which are read
-	// by other processes.
-	if conn.shutdown {
-		return
-	}
-	conn.shutdown = true
-
 	if conn.TCPConn != nil {
 		checkWarn(conn.TCPConn.Close())
 	}
@@ -283,8 +393,11 @@ func (conn *LocalConnection) handleShutdown() {
 		conn.Router.Ourself.DeleteConnection(conn)
 	}
 
+	if conn.establishedTimeout != nil {
+		conn.establishedTimeout.Stop()
+	}
+
 	stopTicker(conn.heartbeat)
-	stopTicker(conn.fetchAll)
 	stopTicker(conn.fragTest)
 
 	// blank out the forwardChan so that the router processes don't
@@ -294,19 +407,17 @@ func (conn *LocalConnection) handleShutdown() {
 	conn.Router.ConnectionMaker.ConnectionTerminated(conn.remoteTCPAddr)
 }
 
-func (conn *LocalConnection) handshake(acceptNewPeer bool) error {
+// Helpers
-	// We do not need to worry about locking in here as at this point,
+
+func (conn *LocalConnection) handshake(enc *gob.Encoder, dec *gob.Decoder, acceptNewPeer bool) error {
+	// We do not need to worry about locking in here as at this point
 	// the connection is not reachable by any go-routine other than
 	// ourself. Only when we add this connection to the conn.local
 	// peer will it be visible from multiple go-routines.
-	tcpConn := conn.TCPConn
+
-	tcpConn.SetLinger(0)
 	conn.extendReadDeadline()
 
-	enc := gob.NewEncoder(tcpConn)
-
 	localConnID := randUint64()
-
 	versionStr := fmt.Sprint(ProtocolVersion)
 	handshakeSend := map[string]string{
 		"Protocol":        Protocol,
@@ -329,7 +440,6 @@ func (conn *LocalConnection) handshake(acceptNewPeer bool) error {
 	}
 	enc.Encode(handshakeSend)
 
-	dec := gob.NewDecoder(tcpConn)
 	err = dec.Decode(&handshakeRecv)
 	if err != nil {
 		return err
@@ -380,7 +490,7 @@ func (conn *LocalConnection) handshake(acceptNewPeer bool) error {
 	if err != nil {
 		return err
 	}
-	conn.UID = localConnID ^ remoteConnID
+	conn.uid = localConnID ^ remoteConnID
 
 	if usingPassword {
 		remotePublicStr, rpErr := checkHandshakeStringField("PublicKey", "", handshakeRecv)
@@ -408,17 +518,16 @@ func (conn *LocalConnection) handshake(acceptNewPeer bool) error {
 
 	toPeer := NewPeer(name, uid, 0)
 	toPeer = conn.Router.Peers.FetchWithDefault(toPeer)
-	if toPeer == nil {
+	switch toPeer {
+	case nil:
 		return fmt.Errorf("Connection appears to be with different version of a peer we already know of")
-	} else if toPeer == conn.local {
+	case conn.local:
-		// have to do assigment here to ensure Shutdown releases ref count
+		conn.remote = toPeer // have to do assigment here to ensure Shutdown releases ref count
-		conn.remote = toPeer
 		return fmt.Errorf("Cannot connect to ourself")
+	default:
+		conn.remote = toPeer
+		return nil
 	}
-	conn.remote = toPeer
-
-	go conn.receiveTCP(dec, usingPassword)
-	return nil
 }
 
 func checkHandshakeStringField(fieldName string, expectedValue string, handshake map[string]string) (string, error) {
@@ -432,8 +541,9 @@ func checkHandshakeStringField(fieldName string, expectedValue string, handshake
 	return val, nil
 }
 
-func (conn *LocalConnection) receiveTCP(decoder *gob.Decoder, usingPassword bool) {
+func (conn *LocalConnection) receiveTCP(decoder *gob.Decoder) {
 	defer conn.Decryptor.Shutdown()
+	usingPassword := conn.SessionKey != nil
 	var receiver TCPReceiver
 	if usingPassword {
 		receiver = NewEncryptedTCPReceiver(conn)
@@ -444,94 +554,70 @@ func (conn *LocalConnection) receiveTCP(decoder *gob.Decoder, usingPassword bool
 	for {
 		var msg []byte
 		conn.extendReadDeadline()
-		if conn.CheckFatal(decoder.Decode(&msg)) != nil {
+		if err = decoder.Decode(&msg); err != nil {
-			return
+			break
 		}
 		msg, err = receiver.Decode(msg)
 		if err != nil {
-			checkWarn(err)
+			break
-			// remote peer may be using wrong password. Or there could
+		}
-			// be some sort of injection attack going on. Just ignore
+		if len(msg) < 1 {
-			// the traffic rather than shutting down.
+			conn.log("ignoring blank msg")
 			continue
 		}
-		if msg[0] == ProtocolConnectionEstablished {
+		if err = conn.handleProtocolMsg(ProtocolTag(msg[0]), msg[1:]); err != nil {
-			// We initiated the connection. We sent fast heartbeats to
+			break
-			// the remote side, which has now received at least one of
-			// them and thus has informed us via TCP that it considers
-			// the connection is now up. We now do a fetchAll on it.
-			conn.SetEstablished()
-			conn.SendTCP(ProtocolFetchAllByte)
-		} else if msg[0] == ProtocolStartFragmentationTest {
-			conn.Forward(false, &ForwardedFrame{
-				srcPeer: conn.local,
-				dstPeer: conn.remote,
-				frame:   FragTest},
-				nil)
-		} else if msg[0] == ProtocolFragmentationReceived {
-			conn.setStackFrag(true)
-		} else if usingPassword && msg[0] == ProtocolNonce {
-			conn.Decryptor.ReceiveNonce(msg[1:])
-		} else if msg[0] == ProtocolFetchAll {
-			// There are exactly two messages that relate to topology
-			// updates.
-			//
-			// 1. FetchAll. This carries no payload. The receiver
-			// responds with the entire topology model as the receiver
-			// has it.
-			//
-			// 2. Update. This carries a topology payload. The
-			// receiver merges it with its own topology model. If the
-			// payload is a subset of the receiver's topology, no
-			// further action is taken. Otherwise, the receiver sends
-			// out to all its connections an "improved" update:
-			//  - elements which the original payload added to the
-			//    receiver are included
-			//  - elements which the original payload updated in the
-			//    receiver are included
-			//  - elements which are equal between the receiver and
-			//    the payload are not included
-			//  - elements where the payload was older than the
-			//    receiver's version are updated
-			conn.SendTCP(Concat(ProtocolUpdateByte, conn.Router.Peers.EncodeAllPeers()))
-		} else if msg[0] == ProtocolUpdate {
-			newUpdate, err := conn.Router.Peers.ApplyUpdate(msg[1:])
-			if _, ok := err.(UnknownPeersError); err != nil && ok {
-				// That update contained a peer we didn't know about;
-				// request full update
-				conn.SendTCP(ProtocolFetchAllByte)
-				continue
-			}
-			if conn.CheckFatal(err) != nil {
-				return
-			}
-			if len(newUpdate) != 0 {
-				conn.Router.ConnectionMaker.Refresh()
-				conn.Router.Routes.Recalculate()
-				conn.Router.Ourself.BroadcastTCP(Concat(ProtocolUpdateByte, newUpdate))
-			}
-		} else if msg[0] == ProtocolPMTUVerified {
-			conn.verifyPMTU <- int(binary.BigEndian.Uint16(msg[1:]))
-		} else {
-			conn.log("received unknown msg:\n", msg)
 		}
 	}
+	conn.Shutdown(err)
+}
+
+func (conn *LocalConnection) handleProtocolMsg(tag ProtocolTag, payload []byte) error {
+	switch tag {
+	case ProtocolConnectionEstablished:
+		// We sent fast heartbeats to the remote peer, which has now
+		// received at least one of them and told us via this message.
+		// We can now consider the connection as established from our
+		// end.
+		conn.SetEstablished()
+	case ProtocolStartFragmentationTest:
+		conn.Forward(false, &ForwardedFrame{
+			srcPeer: conn.local,
+			dstPeer: conn.remote,
+			frame:   FragTest},
+			nil)
+	case ProtocolFragmentationReceived:
+		conn.setStackFrag(true)
+	case ProtocolNonce:
+		if conn.SessionKey == nil {
+			return fmt.Errorf("unexpected nonce on unencrypted connection")
+		}
+		conn.Decryptor.ReceiveNonce(payload)
+	case ProtocolPMTUVerified:
+		conn.verifyPMTU <- int(binary.BigEndian.Uint16(payload))
+	case ProtocolGossipUnicast:
+		return conn.Router.handleGossip(payload, deliverGossipUnicast)
+	case ProtocolGossipBroadcast:
+		return conn.Router.handleGossip(payload, deliverGossipBroadcast)
+	case ProtocolGossip:
+		return conn.Router.handleGossip(payload, deliverGossip)
+	default:
+		conn.log("ignoring unknown protocol tag:", tag)
+	}
+	return nil
 }
 
 func (conn *LocalConnection) extendReadDeadline() {
 	conn.TCPConn.SetReadDeadline(time.Now().Add(ReadTimeout))
 }
 
-// Heartbeating serves two purposes: a) keeping NAT paths alive, and
+func (conn *LocalConnection) sendFastHeartbeats() error {
-// b) updating a remote peer's knowledge of our address, in the event
+	err := conn.ensureForwarders()
-// it changes (e.g. because NAT paths expired).
+	if err == nil {
-// Called only by connection actor process.
+		conn.heartbeat = time.NewTicker(FastHeartbeat)
-func (conn *LocalConnection) forwardHeartbeatFrame() {
+		conn.Forward(true, conn.heartbeatFrame, nil) // avoid initial wait
-	heartbeatFrame := &ForwardedFrame{
+	}
-		srcPeer: conn.local,
+	return err
-		dstPeer: conn.remote,
-		frame:   []byte{}}
-	conn.Forward(true, heartbeatFrame, nil)
 }
 
 func tickerChan(ticker *time.Ticker) <-chan time.Time {

router/connection_maker.go

@@ -15,10 +15,10 @@ const (
 )
 
 const (
-	CMInitiate   = iota
+	CMInitiate = iota
-	CMTerminated = iota
+	CMTerminated
-	CMRefresh    = iota
+	CMRefresh
-	CMStatus     = iota
+	CMStatus
 )
 
 type ConnectionMaker struct {
@@ -41,16 +41,18 @@ type ConnectionMakerInteraction struct {
 	address string
 }
 
-func StartConnectionMaker(ourself *LocalPeer, peers *Peers) *ConnectionMaker {
+func NewConnectionMaker(ourself *LocalPeer, peers *Peers) *ConnectionMaker {
-	queryChan := make(chan *ConnectionMakerInteraction, ChannelSize)
+	return &ConnectionMaker{
-	state := &ConnectionMaker{
 		ourself:        ourself,
 		peers:          peers,
-		queryChan:      queryChan,
 		cmdLineAddress: make(map[string]bool),
 		targets:        make(map[string]*Target)}
-	go state.queryLoop(queryChan)
+}
-	return state
+
+func (cm *ConnectionMaker) Start() {
+	queryChan := make(chan *ConnectionMakerInteraction, ChannelSize)
+	cm.queryChan = queryChan
+	go cm.queryLoop(queryChan)
 }
 
 func (cm *ConnectionMaker) InitiateConnection(address string) {
@@ -198,9 +200,9 @@ func (cm *ConnectionMaker) status() string {
 }
 
 func (cm *ConnectionMaker) attemptConnection(address string, acceptNewPeer bool) {
-	log.Println("Attempting connection to", address)
+	log.Printf("->[%s] attempting connection\n", address)
 	if err := cm.ourself.CreateConnection(address, acceptNewPeer); err != nil {
-		log.Println(err)
+		log.Printf("->[%s] error during connection attempt: %v\n", address, err)
 		cm.ConnectionTerminated(address)
 	}
 }

router/consts.go

@@ -6,8 +6,6 @@ import (
 )
 
 const (
-	Protocol           = "weave"
-	ProtocolVersion    = 8
 	EthernetOverhead   = 14
 	UDPOverhead        = 28 // 20 bytes for IPv4, 8 bytes for UDP
 	Port               = 6783
@@ -20,32 +18,15 @@ const (
 	PMTUDiscoverySize  = 60000
 	FastHeartbeat      = 500 * time.Millisecond
 	SlowHeartbeat      = 10 * time.Second
-	FetchAllInterval   = 30 * time.Second
 	FragTestInterval   = 5 * time.Minute
-	ReadTimeout        = 2 * FetchAllInterval
+	EstablishedTimeout = 30 * time.Second
+	ReadTimeout        = 1 * time.Minute
 	PMTUVerifyAttempts = 8
 	PMTUVerifyTimeout  = 10 * time.Millisecond // gets doubled with every attempt
 	MaxDuration        = time.Duration(math.MaxInt64)
 )
 
-const (
-	ProtocolConnectionEstablished  = iota
-	ProtocolFragmentationReceived  = iota
-	ProtocolStartFragmentationTest = iota
-	ProtocolNonce                  = iota
-	ProtocolFetchAll               = iota
-	ProtocolUpdate                 = iota
-	ProtocolPMTUVerified           = iota
-)
-
 var (
-	FragTest                           = make([]byte, FragTestSize)
+	FragTest      = make([]byte, FragTestSize)
-	PMTUDiscovery                      = make([]byte, PMTUDiscoverySize)
+	PMTUDiscovery = make([]byte, PMTUDiscoverySize)
-	ProtocolConnectionEstablishedByte  = []byte{ProtocolConnectionEstablished}
-	ProtocolFragmentationReceivedByte  = []byte{ProtocolFragmentationReceived}
-	ProtocolStartFragmentationTestByte = []byte{ProtocolStartFragmentationTest}
-	ProtocolNonceByte                  = []byte{ProtocolNonce}
-	ProtocolFetchAllByte               = []byte{ProtocolFetchAll}
-	ProtocolUpdateByte                 = []byte{ProtocolUpdate}
-	ProtocolPMTUVerifiedByte           = []byte{ProtocolPMTUVerified}
 )

router/crypto.go

@@ -11,6 +11,8 @@ import (
 	"encoding/gob"
 	"fmt"
 	"log"
+	"net"
+	"sync"
 )
 
 func GenerateKeyPair() (publicKey, privateKey *[32]byte, err error) {
@@ -83,7 +85,8 @@ func EncodeNonce(df bool) (*[24]byte, []byte, error) {
 		flags = flags | 1
 	}
 	SetNonceLow15Bits(&nonce, flags)
-	return &nonce, Concat(ProtocolNonceByte, nonce[:]), nil
+	// NB: need to make a copy since callers may modify the array
+	return &nonce, Concat(nonce[:]), nil
 }
 
 func DecodeNonce(msg []byte) (bool, *[24]byte) {
@@ -99,6 +102,34 @@ func DecodeNonce(msg []byte) (bool, *[24]byte) {
 
 // Frame Encryptors
 
+type Encryptor interface {
+	FrameOverhead() int
+	PacketOverhead() int
+	IsEmpty() bool
+	Bytes() []byte
+	AppendFrame(*ForwardedFrame)
+	TotalLen() int
+}
+
+type NonEncryptor struct {
+	buf       []byte
+	bufTail   []byte
+	buffered  int
+	prefixLen int
+}
+
+type NaClEncryptor struct {
+	NonEncryptor
+	buf       []byte
+	offset    uint16
+	nonce     *[24]byte
+	nonceChan chan *[24]byte
+	flags     uint16
+	prefixLen int
+	conn      *LocalConnection
+	df        bool
+}
+
 func NewNonEncryptor(prefix []byte) *NonEncryptor {
 	buf := make([]byte, MaxUDPPacketSize)
 	prefixLen := copy(buf, prefix)
@@ -173,10 +204,11 @@ func (ne *NaClEncryptor) Bytes() []byte {
 	nonce := ne.nonce
 	if nonce == nil {
 		freshNonce, encodedNonce, err := EncodeNonce(ne.df)
-		if err = ne.conn.CheckFatal(err); err != nil {
+		if err != nil {
+			ne.conn.Shutdown(err)
 			return []byte{}
 		}
-		ne.conn.SendTCP(encodedNonce)
+		ne.conn.SendProtocolMsg(ProtocolMsg{ProtocolNonce, encodedNonce})
 		ne.nonce = freshNonce
 		nonce = freshNonce
 	}
@@ -191,11 +223,12 @@ func (ne *NaClEncryptor) Bytes() []byte {
 		ne.nonce = <-ne.nonceChan
 	} else if offset == 1<<14 { // half way through range, send new nonce
 		nonce, encodedNonce, err := EncodeNonce(ne.df)
-		if err = ne.conn.CheckFatal(err); err != nil {
+		if err != nil {
+			ne.conn.Shutdown(err)
 			return []byte{}
 		}
 		ne.nonceChan <- nonce
-		ne.conn.SendTCP(encodedNonce)
+		ne.conn.SendProtocolMsg(ProtocolMsg{ProtocolNonce, encodedNonce})
 	}
 	ne.offset = offset
 
@@ -212,6 +245,38 @@ func (ne *NaClEncryptor) TotalLen() int {
 
 // Frame Decryptors
 
+type FrameConsumer func(*LocalConnection, *net.UDPAddr, []byte, []byte, uint16, []byte) error
+
+type Decryptor interface {
+	IterateFrames(FrameConsumer, *UDPPacket) error
+	ReceiveNonce([]byte)
+	Shutdown()
+}
+
+type NonDecryptor struct {
+	conn *LocalConnection
+}
+
+type NaClDecryptor struct {
+	NonDecryptor
+	instance   *NaClDecryptorInstance
+	instanceDF *NaClDecryptorInstance
+}
+
+type NaClDecryptorInstance struct {
+	nonce               *[24]byte
+	previousNonce       *[24]byte
+	usedOffsets         *bit.Set
+	previousUsedOffsets *bit.Set
+	highestOffsetSeen   uint16
+	nonceChan           chan *[24]byte
+}
+
+type PacketDecodingError struct {
+	Fatal bool
+	Desc  string
+}
+
 func NewNonDecryptor(conn *LocalConnection) *NonDecryptor {
 	return &NonDecryptor{conn: conn}
 }
@@ -320,8 +385,9 @@ func (nd *NaClDecryptor) decrypt(buf []byte) ([]byte, error) {
 		usedOffsets = decState.usedOffsets
 	} else {
 		highestOffsetSeen := decState.highestOffsetSeen
-		if offsetNoFlags < (1<<13) && highestOffsetSeen > ((1<<14)+(1<<13)) &&
+		switch {
-			(highestOffsetSeen-offsetNoFlags) > ((1<<14)+(1<<13)) {
+		case offsetNoFlags < (1<<13) && highestOffsetSeen > ((1<<14)+(1<<13)) &&
+			(highestOffsetSeen-offsetNoFlags) > ((1<<14)+(1<<13)):
 			// offset is in the first quarter, highestOffsetSeen is in
 			// the top quarter and under a quarter behind us. We
 			// interpret this as we need to move to the next nonce
@@ -335,29 +401,29 @@ func (nd *NaClDecryptor) decrypt(buf []byte) ([]byte, error) {
 			decState.highestOffsetSeen = offsetNoFlags
 			nonce = decState.nonce
 			usedOffsets = decState.usedOffsets
-		} else if offsetNoFlags > highestOffsetSeen &&
+		case offsetNoFlags > highestOffsetSeen &&
-			(offsetNoFlags-highestOffsetSeen) < (1<<13) {
+			(offsetNoFlags-highestOffsetSeen) < (1<<13):
 			// offset is under a quarter above highestOffsetSeen. This
 			// is ok - maybe some packet loss
 			decState.highestOffsetSeen = offsetNoFlags
 			nonce = decState.nonce
 			usedOffsets = decState.usedOffsets
-		} else if offsetNoFlags <= highestOffsetSeen &&
+		case offsetNoFlags <= highestOffsetSeen &&
-			(highestOffsetSeen-offsetNoFlags) < (1<<13) {
+			(highestOffsetSeen-offsetNoFlags) < (1<<13):
 			// offset is within a quarter of the highest we've
 			// seen. This is ok - just assuming some out-of-order
 			// delivery.
 			nonce = decState.nonce
 			usedOffsets = decState.usedOffsets
-		} else if highestOffsetSeen < (1<<13) && offsetNoFlags > ((1<<14)+(1<<13)) &&
+		case highestOffsetSeen < (1<<13) && offsetNoFlags > ((1<<14)+(1<<13)) &&
-			(offsetNoFlags-highestOffsetSeen) > ((1<<14)+(1<<13)) {
+			(offsetNoFlags-highestOffsetSeen) > ((1<<14)+(1<<13)):
 			// offset is in the last quarter, highestOffsetSeen is in
 			// the first quarter, and offset is under a quarter behind
 			// us. This is ok - as above, just some out of order. But
 			// here it means we're dealing with the previous nonce
 			nonce = decState.previousNonce
 			usedOffsets = decState.previousUsedOffsets
-		} else {
+		default:
 			return nil, fmt.Errorf("Unexpected offset when decrypting UDP packet")
 		}
 	}
@@ -377,6 +443,28 @@ func (nd *NaClDecryptor) decrypt(buf []byte) ([]byte, error) {
 
 // TCP Senders
 
+type TCPSender interface {
+	Send([]byte) error
+}
+
+type SimpleTCPSender struct {
+	encoder *gob.Encoder
+}
+
+type EncryptedTCPSender struct {
+	sync.RWMutex
+	outerEncoder *gob.Encoder
+	innerEncoder *gob.Encoder
+	buffer       *bytes.Buffer
+	conn         *LocalConnection
+	msgCount     int
+}
+
+type EncryptedTCPMessage struct {
+	Number int
+	Body   []byte
+}
+
 func NewSimpleTCPSender(encoder *gob.Encoder) *SimpleTCPSender {
 	return &SimpleTCPSender{encoder: encoder}
 }
@@ -416,6 +504,20 @@ func (sender *EncryptedTCPSender) Send(msg []byte) error {
 
 // TCP Receivers
 
+type TCPReceiver interface {
+	Decode([]byte) ([]byte, error)
+}
+
+type SimpleTCPReceiver struct {
+}
+
+type EncryptedTCPReceiver struct {
+	conn     *LocalConnection
+	decoder  *gob.Decoder
+	buffer   *bytes.Buffer
+	msgCount int
+}
+
 func NewSimpleTCPReceiver() *SimpleTCPReceiver {
 	return &SimpleTCPReceiver{}
 }

router/ethernet_decoder.go

@@ -8,6 +8,13 @@ import (
 	"net"
 )
 
+type EthernetDecoder struct {
+	eth     layers.Ethernet
+	ip      layers.IPv4
+	decoded []gopacket.LayerType
+	parser  *gopacket.DecodingLayerParser
+}
+
 func NewEthernetDecoder() *EthernetDecoder {
 	dec := &EthernetDecoder{}
 	dec.parser = gopacket.NewDecodingLayerParser(layers.LayerTypeEthernet, &dec.eth, &dec.ip)
@@ -75,7 +82,7 @@ func (dec *EthernetDecoder) DropFrame() bool {
 	return bytes.Equal(stpMACPrefix, dec.eth.DstMAC[:len(stpMACPrefix)])
 }
 
-func (dec *EthernetDecoder) IsPMTUVerify() bool {
+func (dec *EthernetDecoder) IsSpecial() bool {
-	return bytes.Equal(zeroMAC, dec.eth.SrcMAC) &&
+	return dec.eth.Length == 0 && dec.eth.EthernetType == layers.EthernetTypeLLC &&
-		bytes.Equal(zeroMAC, dec.eth.DstMAC)
+		bytes.Equal(zeroMAC, dec.eth.SrcMAC) && bytes.Equal(zeroMAC, dec.eth.DstMAC)
 }

router/forwarder.go

@@ -3,12 +3,20 @@ package router
 import (
 	"code.google.com/p/gopacket"
 	"code.google.com/p/gopacket/layers"
-	"log"
-	"net"
 	"syscall"
 	"time"
 )
 
+type ForwardedFrame struct {
+	srcPeer *Peer
+	dstPeer *Peer
+	frame   []byte
+}
+
+type FrameTooBigError struct {
+	EPMTU int // effective pmtu, i.e. what we tell packet senders
+}
+
 func (conn *LocalConnection) ensureForwarders() error {
 	if conn.forwardChan != nil || conn.forwardChanDF != nil {
 		return nil
@@ -52,8 +60,8 @@ func (conn *LocalConnection) ensureForwarders() error {
 	conn.effectivePMTU = forwarder.unverifiedPMTU
 	conn.Unlock()
 
-	go forwarder.Run()
+	forwarder.Start()
-	go forwarderDF.Run()
+	forwarderDF.Start()
 
 	return nil
 }
@@ -193,6 +201,22 @@ func fragment(eth layers.Ethernet, ip layers.IPv4, pmtu int, frame *ForwardedFra
 
 // Forwarder
 
+type Forwarder struct {
+	conn            *LocalConnection
+	ch              <-chan *ForwardedFrame
+	stop            <-chan interface{}
+	verifyPMTUTick  <-chan time.Time
+	verifyPMTU      <-chan int
+	pmtuVerifyCount uint
+	enc             Encryptor
+	udpSender       UDPSender
+	maxPayload      int
+	pmtuVerified    bool
+	highestGoodPMTU int
+	unverifiedPMTU  int
+	lowestBadPMTU   int
+}
+
 func NewForwarder(conn *LocalConnection, ch <-chan *ForwardedFrame, stop <-chan interface{}, verifyPMTU <-chan int, enc Encryptor, udpSender UDPSender, pmtu int) *Forwarder {
 	fwd := &Forwarder{
 		conn:       conn,
@@ -206,7 +230,11 @@ func NewForwarder(conn *LocalConnection, ch <-chan *ForwardedFrame, stop <-chan
 	return fwd
 }
 
-func (fwd *Forwarder) Run() {
+func (fwd *Forwarder) Start() {
+	go fwd.run()
+}
+
+func (fwd *Forwarder) run() {
 	defer fwd.udpSender.Shutdown()
 	var flushed, ok bool
 	var frame *ForwardedFrame
@@ -321,7 +349,7 @@ func (fwd *Forwarder) flush() {
 		} else if PosixError(err) == syscall.ENOBUFS {
 			// TODO handle this better
 		} else {
-			fwd.conn.CheckFatal(err)
+			fwd.conn.Shutdown(err)
 		}
 	}
 }
@@ -342,112 +370,3 @@ func (fwd *Forwarder) drain() {
 func (fwd *Forwarder) logDrop(frame *ForwardedFrame) {
 	fwd.conn.log("Dropping too big frame during forwarding: frame len:", len(frame.frame), "; effective PMTU:", fwd.maxPayload+UDPOverhead-fwd.effectiveOverhead())
 }
-
-// UDP Senders
-
-func NewSimpleUDPSender(conn *LocalConnection) *SimpleUDPSender {
-	return &SimpleUDPSender{udpConn: conn.Router.UDPListener, conn: conn}
-}
-
-func (sender *SimpleUDPSender) Send(msg []byte) error {
-	_, err := sender.udpConn.WriteToUDP(msg, sender.conn.RemoteUDPAddr())
-	return err
-}
-
-func (sender *SimpleUDPSender) Shutdown() error {
-	return nil
-}
-
-func NewRawUDPSender(conn *LocalConnection) (*RawUDPSender, error) {
-	ipSocket, err := dialIP(conn)
-	if err != nil {
-		return nil, err
-	}
-	udpHeader := &layers.UDP{SrcPort: layers.UDPPort(Port)}
-	ipBuf := gopacket.NewSerializeBuffer()
-	opts := gopacket.SerializeOptions{
-		FixLengths: true,
-		// UDP header is calculated with a phantom IP
-		// header. Yes, it's totally nuts. Thankfully, for UDP
-		// over IPv4, the checksum is optional. It's not
-		// optional for IPv6, but we'll ignore that for
-		// now. TODO
-		ComputeChecksums: false}
-
-	return &RawUDPSender{
-		ipBuf:     ipBuf,
-		opts:      opts,
-		udpHeader: udpHeader,
-		socket:    ipSocket,
-		conn:      conn}, nil
-}
-
-func (sender *RawUDPSender) Send(msg []byte) error {
-	payload := gopacket.Payload(msg)
-	sender.udpHeader.DstPort = layers.UDPPort(sender.conn.RemoteUDPAddr().Port)
-
-	err := gopacket.SerializeLayers(sender.ipBuf, sender.opts, sender.udpHeader, &payload)
-	if err != nil {
-		return err
-	}
-	packet := sender.ipBuf.Bytes()
-	_, err = sender.socket.Write(packet)
-	if err == nil || PosixError(err) != syscall.EMSGSIZE {
-		return err
-	}
-	f, err := sender.socket.File()
-	if err != nil {
-		return err
-	}
-	defer f.Close()
-	fd := int(f.Fd())
-	log.Println("EMSGSIZE on send, expecting PMTU update (IP packet was",
-		len(packet), "bytes, payload was", len(msg), "bytes)")
-	pmtu, err := syscall.GetsockoptInt(fd, syscall.IPPROTO_IP, syscall.IP_MTU)
-	if err != nil {
-		return err
-	}
-	return MsgTooBigError{PMTU: pmtu}
-}
-
-func (sender *RawUDPSender) Shutdown() error {
-	defer func() { sender.socket = nil }()
-	return sender.socket.Close()
-}
-
-func dialIP(conn *LocalConnection) (*net.IPConn, error) {
-	ipLocalAddr, err := ipAddr(conn.TCPConn.LocalAddr())
-	if err != nil {
-		return nil, err
-	}
-	ipRemoteAddr, err := ipAddr(conn.TCPConn.RemoteAddr())
-	if err != nil {
-		return nil, err
-	}
-	ipSocket, err := net.DialIP("ip4:UDP", ipLocalAddr, ipRemoteAddr)
-	if err != nil {
-		return nil, err
-	}
-	f, err := ipSocket.File()
-	if err != nil {
-		return nil, err
-	}
-	defer f.Close()
-	fd := int(f.Fd())
-	// This Makes sure all packets we send out have DF set on them.
-	err = syscall.SetsockoptInt(fd, syscall.IPPROTO_IP, syscall.IP_MTU_DISCOVER, syscall.IP_PMTUDISC_DO)
-	if err != nil {
-		return nil, err
-	}
-	return ipSocket, nil
-}
-
-func ipAddr(addr net.Addr) (*net.IPAddr, error) {
-	host, _, err := net.SplitHostPort(addr.String())
-	if err != nil {
-		return nil, err
-	}
-	return &net.IPAddr{
-		IP:   net.ParseIP(host),
-		Zone: ""}, nil
-}

router/gossip.go

@@ -0,0 +1,162 @@
+package router
+
+import (
+	"bytes"
+	"encoding/gob"
+	"fmt"
+	"log"
+	"time"
+)
+
+const GossipInterval = 30 * time.Second
+
+type Gossip interface {
+	// specific message from one peer to another
+	// intermediate peers relay it using unicast topology.
+	GossipUnicast(dstPeerName PeerName, buf []byte) error
+	// send a message to every peer, relayed using broadcast topology.
+	GossipBroadcast(buf []byte) error
+}
+
+type Gossiper interface {
+	OnGossipUnicast(sender PeerName, msg []byte) error
+	OnGossipBroadcast(msg []byte) error
+	// Return state of everything we know; gets called periodically
+	Gossip() []byte
+	// merge in state and return "everything new I've just learnt",
+	// or nil if nothing in the received message was new
+	OnGossip(buf []byte) ([]byte, error)
+}
+
+type GossipChannel struct {
+	ourself  *LocalPeer
+	name     string
+	hash     uint32
+	gossiper Gossiper
+}
+
+func (router *Router) NewGossip(channelName string, g Gossiper) Gossip {
+	channelHash := hash(channelName)
+	channel := &GossipChannel{router.Ourself, channelName, channelHash, g}
+	router.GossipChannels[channelHash] = channel
+	return channel
+}
+
+func (router *Router) SendAllGossip() {
+	for _, channel := range router.GossipChannels {
+		channel.SendGossipMsg(channel.gossiper.Gossip())
+	}
+}
+
+func (router *Router) SendAllGossipDown(conn Connection) {
+	for _, channel := range router.GossipChannels {
+		protocolMsg := channel.gossipMsg(channel.gossiper.Gossip())
+		conn.(ProtocolSender).SendProtocolMsg(protocolMsg)
+	}
+}
+
+func (router *Router) handleGossip(payload []byte, onok func(*GossipChannel, PeerName, []byte, *gob.Decoder) error) error {
+	decoder := gob.NewDecoder(bytes.NewReader(payload))
+	var channelHash uint32
+	if err := decoder.Decode(&channelHash); err != nil {
+		return err
+	}
+	channel, found := router.GossipChannels[channelHash]
+	if !found {
+		return fmt.Errorf("[gossip] received unknown channel with hash %v", channelHash)
+	}
+	var srcName PeerName
+	if err := decoder.Decode(&srcName); err != nil {
+		return err
+	}
+	if err := onok(channel, srcName, payload, decoder); err != nil {
+		return err
+	}
+	return nil
+}
+
+func deliverGossipUnicast(channel *GossipChannel, srcName PeerName, origPayload []byte, dec *gob.Decoder) error {
+	var destName PeerName
+	if err := dec.Decode(&destName); err != nil {
+		return err
+	}
+	if channel.ourself.Name == destName {
+		var payload []byte
+		if err := dec.Decode(&payload); err != nil {
+			return err
+		}
+		return channel.gossiper.OnGossipUnicast(srcName, payload)
+	} else {
+		return channel.relayGossipUnicast(destName, origPayload)
+	}
+}
+
+func deliverGossipBroadcast(channel *GossipChannel, srcName PeerName, origPayload []byte, dec *gob.Decoder) error {
+	var payload []byte
+	if err := dec.Decode(&payload); err != nil {
+		return err
+	}
+	if err := channel.gossiper.OnGossipBroadcast(payload); err != nil {
+		return err
+	}
+	return channel.relayGossipBroadcast(srcName, origPayload)
+}
+
+func deliverGossip(channel *GossipChannel, srcName PeerName, _ []byte, dec *gob.Decoder) error {
+	var payload []byte
+	if err := dec.Decode(&payload); err != nil {
+		return err
+	}
+	if newBuf, err := channel.gossiper.OnGossip(payload); err != nil {
+		return err
+	} else if newBuf != nil {
+		channel.SendGossipMsg(newBuf)
+	}
+	return nil
+}
+
+func (c *GossipChannel) SendGossipMsg(buf []byte) {
+	protocolMsg := c.gossipMsg(buf)
+	c.ourself.ForEachConnection(func(_ PeerName, conn Connection) {
+		conn.(ProtocolSender).SendProtocolMsg(protocolMsg)
+	})
+}
+
+func (c *GossipChannel) gossipMsg(buf []byte) ProtocolMsg {
+	return ProtocolMsg{ProtocolGossip, GobEncode(c.hash, c.ourself.Name, buf)}
+}
+
+func (c *GossipChannel) GossipUnicast(dstPeerName PeerName, buf []byte) error {
+	return c.relayGossipUnicast(dstPeerName, GobEncode(c.hash, c.ourself.Name, dstPeerName, buf))
+}
+
+func (c *GossipChannel) GossipBroadcast(buf []byte) error {
+	return c.relayGossipBroadcast(c.ourself.Name, GobEncode(c.hash, c.ourself.Name, buf))
+}
+
+func (c *GossipChannel) relayGossipUnicast(dstPeerName PeerName, msg []byte) error {
+	if relayPeerName, found := c.ourself.Router.Routes.Unicast(dstPeerName); !found {
+		c.log("unknown relay destination:", dstPeerName)
+	} else if conn, found := c.ourself.ConnectionTo(relayPeerName); !found {
+		c.log("unable to find connection to relay peer", relayPeerName)
+	} else {
+		conn.(ProtocolSender).SendProtocolMsg(ProtocolMsg{ProtocolGossipUnicast, msg})
+	}
+	return nil
+}
+
+func (c *GossipChannel) relayGossipBroadcast(srcName PeerName, msg []byte) error {
+	if srcPeer, found := c.ourself.Router.Peers.Fetch(srcName); !found {
+		c.log("unable to relay broadcast from unknown peer", srcName)
+	} else {
+		protocolMsg := ProtocolMsg{ProtocolGossipBroadcast, msg}
+		for _, conn := range c.ourself.NextBroadcastHops(srcPeer) {
+			conn.SendProtocolMsg(protocolMsg)
+		}
+	}
+	return nil
+}
+
+func (c *GossipChannel) log(args ...interface{}) {
+	log.Println(append(append([]interface{}{}, "[gossip "+c.name+"]:"), args...)...)
+}

router/gossip_test.go

@@ -0,0 +1,146 @@
+package router
+
+import (
+	wt "github.com/zettio/weave/testing"
+	"testing"
+	"time"
+)
+
+// TODO test gossip unicast and broadcast; atm we only test topology
+// gossip, which does not employ unicast or broadcast.
+
+type mockChannelConnection struct {
+	RemoteConnection
+	dest *Router
+}
+
+// Construct a "passive" Router, i.e. without any goroutines.
+//
+// We need to create some dummy channels otherwise tests hang on nil
+// channels when Router.OnGossip() calls async methods.
+func NewTestRouter(name PeerName) *Router {
+	router := NewRouter(nil, name, nil, 10, 1024, nil)
+	router.ConnectionMaker.queryChan = make(chan *ConnectionMakerInteraction, ChannelSize)
+	router.Routes.queryChan = make(chan *Interaction, ChannelSize)
+	return router
+}
+
+func (conn *mockChannelConnection) SendProtocolMsg(protocolMsg ProtocolMsg) {
+	if err := conn.dest.handleGossip(protocolMsg.msg, deliverGossip); err != nil {
+		panic(err)
+	}
+}
+
+func (router *Router) AddTestChannelConnection(r *Router) {
+	fromName := router.Ourself.Peer.Name
+	toName := r.Ourself.Peer.Name
+
+	fromPeer := NewPeer(fromName, router.Ourself.Peer.UID, 0)
+	toPeer := NewPeer(toName, r.Ourself.Peer.UID, 0)
+
+	r.Peers.FetchWithDefault(fromPeer)    // Has side-effect of incrementing refcount
+	router.Peers.FetchWithDefault(toPeer) //
+
+	conn := &mockChannelConnection{RemoteConnection{router.Ourself.Peer, toPeer, "", false}, r}
+	router.Ourself.handleAddConnection(conn)
+	router.Ourself.handleConnectionEstablished(conn)
+}
+
+func (router *Router) DeleteTestChannelConnection(r *Router) {
+	fromName := router.Ourself.Peer.Name
+	toName := r.Ourself.Peer.Name
+
+	fromPeer, _ := r.Peers.Fetch(fromName)
+	toPeer, _ := router.Peers.Fetch(toName)
+
+	fromPeer.DecrementLocalRefCount()
+	toPeer.DecrementLocalRefCount()
+
+	conn, _ := router.Ourself.ConnectionTo(toName)
+	router.Ourself.handleDeleteConnection(conn)
+}
+
+func TestGossipTopology(t *testing.T) {
+	wt.RunWithTimeout(t, 1*time.Second, func() {
+		implTestGossipTopology(t)
+	})
+}
+
+// Create a Peer representing the receiver router, with connections to
+// the routers supplied as arguments, carrying across all UID and
+// version information.
+func (router *Router) tp(routers ...*Router) *Peer {
+	peer := NewPeer(router.Ourself.Peer.Name, router.Ourself.Peer.UID, 0)
+	connections := make(map[PeerName]Connection)
+	for _, r := range routers {
+		p := NewPeer(r.Ourself.Peer.Name, r.Ourself.Peer.UID, r.Ourself.Peer.version)
+		connections[r.Ourself.Peer.Name] = newMockConnection(peer, p)
+	}
+	peer.SetVersionAndConnections(router.Ourself.Peer.version, connections)
+	return peer
+}
+
+// Check that the topology of router matches the peers and all of their connections
+func checkTopology(t *testing.T, router *Router, wantedPeers ...*Peer) {
+	checkTopologyPeers(t, true, router.Peers.allPeers(), wantedPeers...)
+}
+
+func implTestGossipTopology(t *testing.T) {
+	// Create some peers that will talk to each other
+	peer1Name, _ := PeerNameFromString("01:00:00:01:00:00")
+	peer2Name, _ := PeerNameFromString("02:00:00:02:00:00")
+	peer3Name, _ := PeerNameFromString("03:00:00:03:00:00")
+	r1 := NewTestRouter(peer1Name)
+	r2 := NewTestRouter(peer2Name)
+	r3 := NewTestRouter(peer3Name)
+
+	// Check state when they have no connections
+	checkTopology(t, r1, r1.tp())
+	checkTopology(t, r2, r2.tp())
+
+	// Now try adding some connections
+	r1.AddTestChannelConnection(r2)
+	checkTopology(t, r1, r1.tp(r2), r2.tp())
+	checkTopology(t, r2, r1.tp(r2), r2.tp())
+	r2.AddTestChannelConnection(r1)
+	checkTopology(t, r1, r1.tp(r2), r2.tp(r1))
+	checkTopology(t, r2, r1.tp(r2), r2.tp(r1))
+
+	// Currently, the connection from 2 to 3 is one-way only
+	r2.AddTestChannelConnection(r3)
+	checkTopology(t, r1, r1.tp(r2), r2.tp(r1, r3), r3.tp())
+	checkTopology(t, r2, r1.tp(r2), r2.tp(r1, r3), r3.tp())
+	// When r2 gossiped to r3, 1 was unreachable from r3 so it got removed from the
+	// list of peers, but remains referenced in the connection from 1 to 3.
+	checkTopology(t, r3, r2.tp(r1, r3), r3.tp())
+
+	// Add a connection from 3 to 1 and now r1 is reachable.
+	r3.AddTestChannelConnection(r1)
+	checkTopology(t, r1, r1.tp(r2), r2.tp(r1, r3), r3.tp(r1))
+	checkTopology(t, r2, r1.tp(r2), r2.tp(r1, r3), r3.tp(r1))
+	checkTopology(t, r3, r1.tp(), r2.tp(r1, r3), r3.tp(r1))
+
+	r1.AddTestChannelConnection(r3)
+	checkTopology(t, r1, r1.tp(r2, r3), r2.tp(r1, r3), r3.tp(r1))
+	checkTopology(t, r2, r1.tp(r2, r3), r2.tp(r1, r3), r3.tp(r1))
+	checkTopology(t, r3, r1.tp(r2, r3), r2.tp(r1, r3), r3.tp(r1))
+
+	// Drop the connection from 2 to 3
+	r2.DeleteTestChannelConnection(r3)
+	checkTopology(t, r1, r1.tp(r2, r3), r2.tp(r1), r3.tp(r1))
+	checkTopology(t, r2, r1.tp(r2, r3), r2.tp(r1))
+	checkTopology(t, r3, r1.tp(r2, r3), r2.tp(r1), r3.tp(r1))
+
+	// Drop the connection from 1 to 3
+	r1.DeleteTestChannelConnection(r3)
+	checkTopology(t, r1, r1.tp(r2), r2.tp(r1), r3.tp(r1))
+
+	checkTopology(t, r1, r1.tp(r2), r2.tp(r1), r3.tp(r1))
+	checkTopology(t, r2, r1.tp(r2), r2.tp(r1))
+	// r3 still thinks r1 has a connection to it
+	checkTopology(t, r3, r1.tp(r2, r3), r2.tp(r1), r3.tp(r1))
+
+	// On a timer, r3 will gossip to r1
+	r3.SendAllGossip()
+	checkTopology(t, r1, r1.tp(r2), r2.tp(r1), r3.tp(r1))
+}

router/local_peer.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"log"
 	"net"
+	"time"
 )
 
 type LocalPeer struct {
@@ -17,12 +18,14 @@ type PeerInteraction struct {
 	payload interface{}
 }
 
-func StartLocalPeer(name PeerName, router *Router) *LocalPeer {
+func NewLocalPeer(name PeerName, router *Router) *LocalPeer {
-	peer := &LocalPeer{Peer: NewPeer(name, 0, 0), Router: router}
+	return &LocalPeer{Peer: NewPeer(name, 0, 0), Router: router}
+}
+
+func (peer *LocalPeer) Start() {
 	queryChan := make(chan *PeerInteraction, ChannelSize)
 	peer.queryChan = queryChan
 	go peer.queryLoop(queryChan)
-	return peer
 }
 
 func (peer *LocalPeer) Forward(dstPeer *Peer, df bool, frame []byte, dec *EthernetDecoder) error {
@@ -55,26 +58,8 @@ func (peer *LocalPeer) Relay(srcPeer, dstPeer *Peer, df bool, frame []byte, dec
 }
 
 func (peer *LocalPeer) RelayBroadcast(srcPeer *Peer, df bool, frame []byte, dec *EthernetDecoder) error {
-	nextHops := peer.Router.Routes.Broadcast(srcPeer.Name)
+	for _, conn := range peer.NextBroadcastHops(srcPeer) {
-	if len(nextHops) == 0 {
+		err := conn.Forward(df, &ForwardedFrame{
-		return nil
-	}
-	// We must not hold a read lock on peer during the conn.Forward
-	// below, since that is a potentially blocking operation (e.g. if
-	// the channel is full).
-	nextConns := make([]*LocalConnection, 0, len(nextHops))
-	peer.RLock()
-	for _, hopName := range nextHops {
-		conn, found := peer.connections[hopName]
-		// Again, !found could just be due to a race.
-		if found {
-			nextConns = append(nextConns, conn.(*LocalConnection))
-		}
-	}
-	peer.RUnlock()
-	var err error
-	for _, conn := range nextConns {
-		err = conn.Forward(df, &ForwardedFrame{
 			srcPeer: srcPeer,
 			dstPeer: conn.Remote(),
 			frame:   frame},
@@ -86,6 +71,24 @@ func (peer *LocalPeer) RelayBroadcast(srcPeer *Peer, df bool, frame []byte, dec
 	return nil
 }
 
+func (peer *LocalPeer) NextBroadcastHops(srcPeer *Peer) []*LocalConnection {
+	nextHops := peer.Router.Routes.Broadcast(srcPeer.Name)
+	if len(nextHops) == 0 {
+		return nil
+	}
+	nextConns := make([]*LocalConnection, 0, len(nextHops))
+	peer.RLock()
+	defer peer.RUnlock()
+	for _, hopName := range nextHops {
+		conn, found := peer.connections[hopName]
+		// Again, !found could just be due to a race.
+		if found {
+			nextConns = append(nextConns, conn.(*LocalConnection))
+		}
+	}
+	return nextConns
+}
+
 func (peer *LocalPeer) CreateConnection(peerAddr string, acceptNewPeer bool) error {
 	if err := peer.checkConnectionLimit(); err != nil {
 		return err
@@ -105,25 +108,34 @@ func (peer *LocalPeer) CreateConnection(peerAddr string, acceptNewPeer bool) err
 	if err != nil {
 		return err
 	}
-	connRemote := NewRemoteConnection(peer.Peer, nil, tcpConn.RemoteAddr().String())
+	connRemote := NewRemoteConnection(peer.Peer, nil, tcpConn.RemoteAddr().String(), false)
-	NewLocalConnection(connRemote, acceptNewPeer, tcpConn, udpAddr, peer.Router)
+	connLocal := NewLocalConnection(connRemote, tcpConn, udpAddr, peer.Router)
+	connLocal.Start(acceptNewPeer)
 	return nil
 }
 
 // ACTOR client API
 
 const (
-	PAddConnection         = iota
+	PAddConnection = iota
-	PBroadcastTCP          = iota
+	PConnectionEstablished
-	PDeleteConnection      = iota
+	PDeleteConnection
-	PConnectionEstablished = iota
+	PSendProtocolMsg
 )
 
-// Async: rely on the peer to shut us down if we shouldn't be adding
+// Sync.
-// ourselves, so therefore this can be async
 func (peer *LocalPeer) AddConnection(conn *LocalConnection) {
+	resultChan := make(chan interface{})
 	peer.queryChan <- &PeerInteraction{
-		Interaction: Interaction{code: PAddConnection},
+		Interaction: Interaction{code: PAddConnection, resultChan: resultChan},
+		payload:     conn}
+	<-resultChan
+}
+
+// Async.
+func (peer *LocalPeer) ConnectionEstablished(conn *LocalConnection) {
+	peer.queryChan <- &PeerInteraction{
+		Interaction: Interaction{code: PConnectionEstablished},
 		payload:     conn}
 }
 
@@ -137,42 +149,50 @@ func (peer *LocalPeer) DeleteConnection(conn *LocalConnection) {
 }
 
 // Async.
-func (peer *LocalPeer) ConnectionEstablished(conn *LocalConnection) {
+func (peer *LocalPeer) SendProtocolMsg(m ProtocolMsg) {
 	peer.queryChan <- &PeerInteraction{
-		Interaction: Interaction{code: PConnectionEstablished},
+		Interaction: Interaction{code: PSendProtocolMsg},
-		payload:     conn}
+		payload:     m}
-}
-
-// Async.
-func (peer *LocalPeer) BroadcastTCP(msg []byte) {
-	peer.queryChan <- &PeerInteraction{
-		Interaction: Interaction{code: PBroadcastTCP},
-		payload:     msg}
 }
 
 // ACTOR server
 
 func (peer *LocalPeer) queryLoop(queryChan <-chan *PeerInteraction) {
+	gossipTimer := time.Tick(GossipInterval)
 	for {
-		query, ok := <-queryChan
+		select {
-		if !ok {
+		case query, ok := <-queryChan:
-			return
+			if !ok {
-		}
+				return
-		switch query.code {
+			}
-		case PAddConnection:
+			switch query.code {
-			peer.handleAddConnection(query.payload.(*LocalConnection))
+			case PAddConnection:
-		case PDeleteConnection:
+				conn := query.payload.(*LocalConnection)
-			peer.handleDeleteConnection(query.payload.(*LocalConnection))
+				if peer.handleAddConnection(conn) {
-			query.resultChan <- nil
+					conn.log("connection added")
-		case PConnectionEstablished:
+				}
-			peer.handleConnectionEstablished(query.payload.(*LocalConnection))
+				query.resultChan <- nil
-		case PBroadcastTCP:
+			case PConnectionEstablished:
-			peer.handleBroadcastTCP(query.payload.([]byte))
+				conn := query.payload.(*LocalConnection)
+				if peer.handleConnectionEstablished(conn) {
+					conn.log("connection fully established")
+				}
+			case PDeleteConnection:
+				conn := query.payload.(*LocalConnection)
+				if peer.handleDeleteConnection(conn) {
+					conn.log("connection deleted")
+				}
+				query.resultChan <- nil
+			case PSendProtocolMsg:
+				peer.handleSendProtocolMsg(query.payload.(ProtocolMsg))
+			}
+		case <-gossipTimer:
+			peer.Router.SendAllGossip()
 		}
 	}
 }
 
-func (peer *LocalPeer) handleAddConnection(conn *LocalConnection) {
+func (peer *LocalPeer) handleAddConnection(conn Connection) bool {
 	if peer.Peer != conn.Local() {
 		log.Fatal("Attempt made to add connection to peer where peer is not the source of connection")
 	}
@@ -184,35 +204,47 @@ func (peer *LocalPeer) handleAddConnection(conn *LocalConnection) {
 	// deliberately non symmetrical
 	if dupConn, found := peer.connections[toName]; found {
 		if dupConn == conn {
-			return
+			return false
 		}
-		// conn.UID is used as the tie breaker here, in the
+		switch conn.BreakTie(dupConn) {
-		// knowledge that both sides will make the same decision.
+		case TieBreakWon:
-		dupConnLocal := dupConn.(*LocalConnection)
+			dupConn.Shutdown(dupErr)
-		if conn.UID == dupConnLocal.UID {
+			peer.handleDeleteConnection(dupConn)
+		case TieBreakLost:
+			conn.Shutdown(dupErr)
+			return false
+		case TieBreakTied:
 			// oh good grief. Sod it, just kill both of them.
-			conn.CheckFatal(dupErr)
+			conn.Shutdown(dupErr)
-			dupConnLocal.CheckFatal(dupErr)
+			dupConn.Shutdown(dupErr)
-			peer.handleDeleteConnection(dupConnLocal)
+			peer.handleDeleteConnection(dupConn)
-			return
+			return false
-		} else if conn.UID < dupConnLocal.UID {
-			dupConnLocal.CheckFatal(dupErr)
-			peer.handleDeleteConnection(dupConnLocal)
-		} else {
-			conn.CheckFatal(dupErr)
-			return
 		}
 	}
 	if err := peer.checkConnectionLimit(); err != nil {
-		conn.CheckFatal(err)
+		conn.Shutdown(err)
-		return
+		return false
 	}
-	peer.Lock()
+	peer.addConnection(conn)
-	peer.connections[toName] = conn
+	peer.Router.SendAllGossipDown(conn)
-	peer.Unlock()
+	peer.broadcastPeerUpdate(conn.Remote())
+	return true
 }
 
-func (peer *LocalPeer) handleDeleteConnection(conn *LocalConnection) {
+func (peer *LocalPeer) handleConnectionEstablished(conn Connection) bool {
+	if peer.Peer != conn.Local() {
+		log.Fatal("Peer informed of active connection where peer is not the source of connection")
+	}
+	if dupConn, found := peer.connections[conn.Remote().Name]; !found || conn != dupConn {
+		conn.Shutdown(fmt.Errorf("Cannot set unknown connection active"))
+		return false
+	}
+	peer.connectionEstablished(conn)
+	peer.broadcastPeerUpdate()
+	return true
+}
+
+func (peer *LocalPeer) handleDeleteConnection(conn Connection) bool {
 	if peer.Peer != conn.Local() {
 		log.Fatal("Attempt made to delete connection from peer where peer is not the source of connection")
 	}
@@ -221,50 +253,32 @@ func (peer *LocalPeer) handleDeleteConnection(conn *LocalConnection) {
 	}
 	toName := conn.Remote().Name
 	if connFound, found := peer.connections[toName]; !found || connFound != conn {
-		return
+		return false
-	}
-	peer.Lock()
-	delete(peer.connections, toName)
-	peer.Unlock()
-	broadcast := false
-	if conn.Established() {
-		peer.Lock()
-		peer.version += 1
-		peer.Unlock()
-		broadcast = true
 	}
+	peer.deleteConnection(conn)
 	// Must do garbage collection first to ensure we don't send out an
 	// update with unreachable peers (can cause looping)
 	peer.Router.Peers.GarbageCollect()
-	if broadcast {
+	peer.broadcastPeerUpdate()
-		peer.broadcastPeerUpdate()
+	return true
-	}
 }
 
-func (peer *LocalPeer) handleConnectionEstablished(conn *LocalConnection) {
+func (peer *LocalPeer) handleSendProtocolMsg(m ProtocolMsg) {
-	if peer.Peer != conn.Local() {
-		log.Fatal("Peer informed of active connection where peer is not the source of connection")
-	}
-	if dupConn, found := peer.connections[conn.Remote().Name]; !found || conn != dupConn {
-		conn.CheckFatal(fmt.Errorf("Cannot set unknown connection active"))
-		return
-	}
-	peer.Lock()
-	peer.version += 1
-	peer.Unlock()
-	log.Println("Peer", peer.Name, "established active connection to remote peer", conn.Remote().Name, "at", conn.RemoteTCPAddr())
-	peer.broadcastPeerUpdate(conn.Remote())
-}
-
-func (peer *LocalPeer) handleBroadcastTCP(msg []byte) {
 	peer.ForEachConnection(func(_ PeerName, conn Connection) {
-		conn.(*LocalConnection).SendTCP(msg)
+		conn.(ProtocolSender).SendProtocolMsg(m)
 	})
 }
 
 func (peer *LocalPeer) broadcastPeerUpdate(peers ...*Peer) {
 	peer.Router.Routes.Recalculate()
-	peer.handleBroadcastTCP(Concat(ProtocolUpdateByte, EncodePeers(append(peers, peer.Peer)...)))
+	// TODO We should just be invoking TopologyGossip.GossipBroadcast
+	// here, but route calculation is asynchronous and in this
+	// particular case would likely result in the broadcast not
+	// reaching all peers. So instead we slightly break the Gossip
+	// abstraction (hence the cast) and send a regular update. This is
+	// less efficient though since it will almost certainly reach
+	// peers more than once.
+	peer.Router.TopologyGossip.(*GossipChannel).SendGossipMsg(EncodePeers(append(peers, peer.Peer)...))
 }
 
 func (peer *LocalPeer) checkConnectionLimit() error {

router/mac_cache.go

@@ -22,12 +22,14 @@ type MacCache struct {
 }
 
 func NewMacCache(maxAge time.Duration, onExpiry func(net.HardwareAddr, *Peer)) *MacCache {
-	res := &MacCache{
+	return &MacCache{
 		table:    make(map[uint64]*MacCacheEntry),
 		maxAge:   maxAge,
 		onExpiry: onExpiry}
-	res.setExpiryTimer()
+}
-	return res
+
+func (cache *MacCache) Start() {
+	cache.setExpiryTimer()
 }
 
 func (cache *MacCache) Enter(mac net.HardwareAddr, peer *Peer) bool {

router/mocks_test.go

@@ -0,0 +1,113 @@
+// No mocks are tested by this file.
+//
+// It supplies some mock implementations to other unit tests, and is
+// named "...test.go" so it is only compiled under `go test`.
+
+package router
+
+import (
+	wt "github.com/zettio/weave/testing"
+	"testing"
+)
+
+// Add to peers a connection from peers.ourself to p
+func (peers *Peers) AddTestConnection(p *Peer) {
+	toPeer := NewPeer(p.Name, p.UID, 0)
+	peers.FetchWithDefault(toPeer) // Has side-effect of incrementing refcount
+	conn := newMockConnection(peers.ourself, toPeer)
+	peers.ourself.addConnection(conn)
+	peers.ourself.connectionEstablished(conn)
+}
+
+// Add to peers a connection from p1 to p2
+func (peers *Peers) AddTestRemoteConnection(p1, p2 *Peer) {
+	fromName := p1.Name
+	fromPeer := NewPeer(fromName, p1.UID, 0)
+	fromPeer = peers.FetchWithDefault(fromPeer)
+	toName := p2.Name
+	toPeer := NewPeer(toName, p2.UID, 0)
+	toPeer = peers.FetchWithDefault(toPeer)
+	peers.ourself.addConnection(&RemoteConnection{fromPeer, toPeer, "", false})
+}
+
+func (peers *Peers) DeleteTestConnection(p *Peer) {
+	toName := p.Name
+	toPeer, _ := peers.Fetch(toName)
+	toPeer.DecrementLocalRefCount()
+	conn, _ := peers.ourself.ConnectionTo(toName)
+	peers.ourself.deleteConnection(conn)
+}
+
+// mockConnection used in testing is very similar to a
+// RemoteConnection, without the RemoteTCPAddr(). We are making it a
+// separate type in order to distinguish what is created by the test
+// from what is created by the real code.
+func newMockConnection(from, to *Peer) Connection {
+	type mockConnection struct{ RemoteConnection }
+	return &mockConnection{RemoteConnection{from, to, "", false}}
+}
+
+func checkEqualConns(t *testing.T, ourName PeerName, got, wanted map[PeerName]Connection) {
+	checkConns := make(map[PeerName]bool)
+	for _, conn := range wanted {
+		checkConns[conn.Remote().Name] = true
+	}
+	for _, conn := range got {
+		remoteName := conn.Remote().Name
+		if _, found := checkConns[remoteName]; found {
+			delete(checkConns, remoteName)
+		} else {
+			wt.Fatalf(t, "Unexpected connection from %s to %s", ourName, remoteName)
+		}
+	}
+	if len(checkConns) > 0 {
+		wt.Fatalf(t, "Expected connections not found: from %s to %v", ourName, checkConns)
+	}
+}
+
+// Get all the peers from a Peers in a slice
+func (peers *Peers) allPeers() []*Peer {
+	res := make([]*Peer, 0)
+	for _, peer := range peers.table {
+		res = append(res, peer)
+	}
+	return res
+}
+
+func (peers *Peers) allPeersExcept(excludeName PeerName) []*Peer {
+	res := peers.allPeers()
+	for i, peer := range res {
+		if peer.Name == excludeName {
+			return append(res[:i], res[i+1:]...)
+		}
+	}
+	return res
+}
+
+// Check that the peers slice matches the wanted peers
+func checkPeerArray(t *testing.T, peers []*Peer, wantedPeers ...*Peer) {
+	checkTopologyPeers(t, false, peers, wantedPeers...)
+}
+
+// Check that the peers slice matches the wanted peers and optionally
+// all of their connections
+func checkTopologyPeers(t *testing.T, checkConns bool, peers []*Peer, wantedPeers ...*Peer) {
+	check := make(map[PeerName]*Peer)
+	for _, peer := range wantedPeers {
+		check[peer.Name] = peer
+	}
+	for _, peer := range peers {
+		name := peer.Name
+		if wantedPeer, found := check[name]; found {
+			if checkConns {
+				checkEqualConns(t, name, peer.connections, wantedPeer.connections)
+			}
+			delete(check, name)
+		} else {
+			wt.Fatalf(t, "Unexpected peer: %s", name)
+		}
+	}
+	if len(check) > 0 {
+		wt.Fatalf(t, "Expected peers not found: %v", check)
+	}
+}

router/peer.go

@@ -86,6 +86,26 @@ func (peer *Peer) SetVersionAndConnections(version uint64, connections map[PeerN
 	peer.connections = connections
 }
 
+func (peer *Peer) addConnection(conn Connection) {
+	peer.Lock()
+	defer peer.Unlock()
+	peer.connections[conn.Remote().Name] = conn
+	peer.version += 1
+}
+
+func (peer *Peer) deleteConnection(conn Connection) {
+	peer.Lock()
+	defer peer.Unlock()
+	delete(peer.connections, conn.Remote().Name)
+	peer.version += 1
+}
+
+func (peer *Peer) connectionEstablished(conn Connection) {
+	peer.Lock()
+	defer peer.Unlock()
+	peer.version += 1
+}
+
 // Calculate the routing table from this peer to all peers reachable
 // from it, returning a "next hop" map of PeerNameX -> PeerNameY,
 // which says "in order to send a message to X, the peer should send
@@ -99,9 +119,9 @@ func (peer *Peer) SetVersionAndConnections(version uint64, connections map[PeerN
 // same result. This is important since otherwise we risk message loss
 // or routing cycles.
 //
-// When the 'symmetric' flag is set, only symmetric connections are
+// When the 'establishedAndSymmetric' flag is set, only connections
-// considered, i.e. where both sides indicate they have a connection
+// that are marked as 'established' and are symmetric (i.e. where both
-// to the other.
+// sides indicate they have a connection to the other) are considered.
 //
 // When a non-nil stopAt peer is supplied, the widening stops when it
 // reaches that peer. The boolean return indicates whether that has
@@ -111,7 +131,7 @@ func (peer *Peer) SetVersionAndConnections(version uint64, connections map[PeerN
 // traversal. This prevents the connectivity graph from changing
 // underneath us in ways that would invalidate the result. Thus the
 // answer returned may be out of date, but never inconsistent.
-func (peer *Peer) Routes(stopAt *Peer, symmetric bool) (bool, map[PeerName]PeerName) {
+func (peer *Peer) Routes(stopAt *Peer, establishedAndSymmetric bool) (bool, map[PeerName]PeerName) {
 	peer.RLock()
 	defer peer.RUnlock()
 	routes := make(map[PeerName]PeerName)
@@ -127,12 +147,17 @@ func (peer *Peer) Routes(stopAt *Peer, symmetric bool) (bool, map[PeerName]PeerN
 			}
 			curName := curPeer.Name
 			for remoteName, conn := range curPeer.connections {
+				if establishedAndSymmetric && !conn.Established() {
+					continue
+				}
 				if _, found := routes[remoteName]; found {
 					continue
 				}
 				remote := conn.Remote()
 				remote.RLock()
-				if _, found := remote.connections[curName]; !symmetric || found {
+				// DANGER holding rlock on remote, possibly taking
+				// rlock on remoteConn
+				if remoteConn, found := remote.connections[curName]; !establishedAndSymmetric || (found && remoteConn.Established()) {
 					defer remote.RUnlock()
 					nextWorklist = append(nextWorklist, remote)
 					// We now know how to get to remoteName: the same

router/peers.go

@@ -11,15 +11,21 @@ import (
 type Peers struct {
 	sync.RWMutex
 	ourself *Peer
-	macs    *MacCache
 	table   map[PeerName]*Peer
 	onGC    func(*Peer)
 }
 
-func NewPeers(ourself *Peer, macs *MacCache, onGC func(*Peer)) *Peers {
+type UnknownPeerError struct {
+	Name PeerName
+}
+
+type NameCollisionError struct {
+	Name PeerName
+}
+
+func NewPeers(ourself *Peer, onGC func(*Peer)) *Peers {
 	return &Peers{
 		ourself: ourself,
-		macs:    macs,
 		table:   make(map[PeerName]*Peer),
 		onGC:    onGC}
 }
@@ -57,6 +63,12 @@ func (peers *Peers) ForEach(fun func(PeerName, *Peer)) {
 	}
 }
 
+// Merge an incoming update with our own topology.
+//
+// We add peers hitherto unknown to us, and update peers for which the
+// update contains a more recent version than known to us. The return
+// value is an "improved" update containing just these new/updated
+// elements.
 func (peers *Peers) ApplyUpdate(update []byte) ([]byte, error) {
 	peers.Lock()
 
@@ -96,7 +108,7 @@ func EncodePeers(peers ...*Peer) []byte {
 	buf := new(bytes.Buffer)
 	enc := gob.NewEncoder(buf)
 	for _, peer := range peers {
-		peer.encodePeer(enc)
+		peer.encode(enc)
 	}
 	return buf.Bytes()
 }
@@ -112,7 +124,11 @@ func (peers *Peers) String() string {
 	peers.ForEach(func(name PeerName, peer *Peer) {
 		buf.WriteString(fmt.Sprint(peer, "\n"))
 		peer.ForEachConnection(func(remoteName PeerName, conn Connection) {
-			buf.WriteString(fmt.Sprintf("   -> %v [%v]\n", remoteName, conn.RemoteTCPAddr()))
+			established := ""
+			if !conn.Established() {
+				established = " (unestablished)"
+			}
+			buf.WriteString(fmt.Sprintf("   -> %v [%v%s]\n", remoteName, conn.RemoteTCPAddr(), established))
 		})
 	})
 	return buf.String()
@@ -132,12 +148,11 @@ func (peers *Peers) fetchAlias(peer *Peer) (*Peer, bool) {
 
 func (peers *Peers) garbageCollect() []*Peer {
 	removed := []*Peer{}
+	_, reached := peers.ourself.Routes(nil, false)
 	for name, peer := range peers.table {
-		found, _ := peers.ourself.Routes(peer, false)
+		if _, found := reached[peer.Name]; !found && !peer.IsLocallyReferenced() {
-		if !found && !peer.IsLocallyReferenced() {
-			peers.onGC(peer)
 			delete(peers.table, name)
-			peers.macs.Delete(peer)
+			peers.onGC(peer)
 			removed = append(removed, peer)
 		}
 	}
@@ -148,7 +163,7 @@ func encodePeersMap(peers map[PeerName]*Peer) []byte {
 	buf := new(bytes.Buffer)
 	enc := gob.NewEncoder(buf)
 	for _, peer := range peers {
-		peer.encodePeer(enc)
+		peer.encode(enc)
 	}
 	return buf.Bytes()
 }
@@ -183,9 +198,8 @@ func (peers *Peers) decodeUpdate(update []byte) (newPeers map[PeerName]*Peer, de
 		}
 	}
 
-	unknownPeers := false
 	for _, connsBuf := range decodedConns {
-		connsIterator(connsBuf, func(remoteNameByte []byte, _ string) {
+		decErr := connsIterator(connsBuf, func(remoteNameByte []byte, _ string, _ bool) {
 			remoteName := PeerNameFromBin(remoteNameByte)
 			if _, found := newPeers[remoteName]; found {
 				return
@@ -195,12 +209,14 @@ func (peers *Peers) decodeUpdate(update []byte) (newPeers map[PeerName]*Peer, de
 			}
 			// Update refers to a peer which we have no knowledge
 			// of. Thus we can't apply the update. Abort.
-			unknownPeers = true
+			err = UnknownPeerError{remoteName}
 		})
-	}
+		if decErr != nil && decErr != io.EOF {
-	if unknownPeers {
+			err = decErr
-		err = UnknownPeersError{}
+		}
-		return
+		if err != nil {
+			return
+		}
 	}
 	return
 }
@@ -212,23 +228,12 @@ func (peers *Peers) applyUpdate(decodedUpdate []*Peer, decodedConns [][]byte) ma
 		name := newPeer.Name
 		// guaranteed to find peer in the peers.table
 		peer := peers.table[name]
-		if peer != newPeer {
+		if peer != newPeer &&
-			if peer.Version() > newPeer.Version() {
+			(peer == peers.ourself || peer.Version() >= newPeer.Version()) {
-				// we know more about this one than the update. If
+			// Nobody but us updates us. And if we know more about a
-				// peer is ourself, this is slightly racey (further
+			// peer than what's in the the update, we ignore the
-				// changes could occur to ourself in the mean
+			// latter.
-				// time). But it doesn't matter as we know that we're
+			continue
-				// already > newPeer.Version() and that's not going to
-				// change.
-				newUpdate[name] = peer
-				continue
-			} else if peer == peers.ourself {
-				// nobody but us updates us
-				continue
-			} else if peer.Version() == newPeer.Version() {
-				// implication is that connections are equal too
-				continue
-			}
 		}
 		// If we're here, either it was a new peer, or the update has
 		// more info about the peer than we do. Either case, we need
@@ -247,7 +252,7 @@ func (peers *Peers) applyUpdate(decodedUpdate []*Peer, decodedConns [][]byte) ma
 	return newUpdate
 }
 
-func (peer *Peer) encodePeer(enc *gob.Encoder) {
+func (peer *Peer) encode(enc *gob.Encoder) {
 	peer.RLock()
 	defer peer.RUnlock()
 
@@ -258,60 +263,62 @@ func (peer *Peer) encodePeer(enc *gob.Encoder) {
 	connsBuf := new(bytes.Buffer)
 	connsEnc := gob.NewEncoder(connsBuf)
 	for _, conn := range peer.connections {
-		// DANGER holding rlock on peer, going to take rlock on conn
-		if !conn.Established() {
-			continue
-		}
 		checkFatal(connsEnc.Encode(conn.Remote().NameByte))
 		checkFatal(connsEnc.Encode(conn.RemoteTCPAddr()))
+		// DANGER holding rlock on peer, going to take rlock on conn
+		checkFatal(connsEnc.Encode(conn.Established()))
 	}
 	checkFatal(enc.Encode(connsBuf.Bytes()))
 }
 
 func decodePeerNoConns(dec *gob.Decoder) (nameByte []byte, uid uint64, version uint64, conns []byte, err error) {
-	err = dec.Decode(&nameByte)
+	if err = dec.Decode(&nameByte); err != nil {
-	if err != nil {
 		return
 	}
-	err = dec.Decode(&uid)
+	if err = dec.Decode(&uid); err != nil {
-	if err != nil {
 		return
 	}
-	err = dec.Decode(&version)
+	if err = dec.Decode(&version); err != nil {
-	if err != nil {
 		return
 	}
-	err = dec.Decode(&conns)
+	if err = dec.Decode(&conns); err != nil {
-	if err == io.EOF {
+		return
-		err = nil
 	}
 	return
 }
 
-func connsIterator(input []byte, fun func([]byte, string)) {
+func connsIterator(input []byte, fun func([]byte, string, bool)) error {
 	buf := new(bytes.Buffer)
 	buf.Write(input)
 	dec := gob.NewDecoder(buf)
 	for {
 		var nameByte []byte
-		err := dec.Decode(&nameByte)
+		if err := dec.Decode(&nameByte); err != nil {
-		if err == io.EOF {
+			return err
-			return
 		}
-		checkFatal(err)
 		var foundAt string
-		checkFatal(dec.Decode(&foundAt))
+		if err := dec.Decode(&foundAt); err != nil {
-		fun(nameByte, string(foundAt))
+			return err
+		}
+		var established bool
+		if err := dec.Decode(&established); err != nil {
+			return err
+		}
+		fun(nameByte, foundAt, established)
 	}
 }
 
 func readConnsMap(peer *Peer, buf []byte, table map[PeerName]*Peer) map[PeerName]Connection {
 	conns := make(map[PeerName]Connection)
-	connsIterator(buf, func(nameByte []byte, remoteTCPAddr string) {
+	if err := connsIterator(buf, func(nameByte []byte, remoteTCPAddr string, established bool) {
 		name := PeerNameFromBin(nameByte)
 		remotePeer := table[name]
-		conn := NewRemoteConnection(peer, remotePeer, remoteTCPAddr)
+		conn := NewRemoteConnection(peer, remotePeer, remoteTCPAddr, established)
 		conns[name] = conn
-	})
+	}); err != io.EOF {
+		// this should never happen since we've already successfully
+		// decoded the same data in decodeUpdate
+		checkFatal(err)
+	}
 	return conns
 }

router/peers_test.go

@@ -0,0 +1,113 @@
+package router
+
+import (
+	"fmt"
+	wt "github.com/zettio/weave/testing"
+	"math/rand"
+	"testing"
+)
+
+// TODO we should also test:
+//
+// - applying an incremental update, including the case where that
+//   leads to an UnknownPeerError
+//
+// - the "improved update" calculation
+//
+// - non-gc of peers that are only referenced locally
+
+func newNode(name PeerName) (*Peer, *Peers) {
+	peer := NewPeer(name, 0, 0)
+	peers := NewPeers(peer, func(*Peer) {})
+	peers.FetchWithDefault(peer)
+	return peer, peers
+}
+
+// Check that ApplyUpdate copies the whole topology from peers
+func checkApplyUpdate(t *testing.T, peers *Peers) {
+	dummyName, _ := PeerNameFromString("99:00:00:01:00:00")
+	// We need a new node outside of the network, with a connection
+	// into it.
+	_, testBedPeers := newNode(dummyName)
+	testBedPeers.AddTestConnection(peers.ourself)
+	testBedPeers.ApplyUpdate(peers.EncodeAllPeers())
+
+	checkTopologyPeers(t, true, testBedPeers.allPeersExcept(dummyName), peers.allPeers()...)
+}
+
+func TestPeersEncoding(t *testing.T) {
+	const numNodes = 20
+	const numIters = 1000
+	var peer [numNodes]*Peer
+	var ps [numNodes]*Peers
+	for i := 0; i < numNodes; i++ {
+		name, _ := PeerNameFromString(fmt.Sprintf("%02d:00:00:01:00:00", i))
+		peer[i], ps[i] = newNode(name)
+	}
+
+	conns := make([]struct{ from, to int }, 0)
+	for i := 0; i < numIters; i++ {
+		oper := rand.Intn(2)
+		switch oper {
+		case 0:
+			from, to := rand.Intn(numNodes), rand.Intn(numNodes)
+			if from != to {
+				if _, found := peer[from].ConnectionTo(peer[to].Name); !found {
+					ps[from].AddTestConnection(peer[to])
+					conns = append(conns, struct{ from, to int }{from, to})
+					checkApplyUpdate(t, ps[from])
+				}
+			}
+		case 1:
+			if len(conns) > 0 {
+				n := rand.Intn(len(conns))
+				c := conns[n]
+				ps[c.from].DeleteTestConnection(peer[c.to])
+				ps[c.from].GarbageCollect()
+				checkApplyUpdate(t, ps[c.from])
+				conns = append(conns[:n], conns[n+1:]...)
+			}
+		}
+	}
+}
+
+func TestPeersGarbageCollection(t *testing.T) {
+	const (
+		peer1NameString = "01:00:00:01:00:00"
+		peer2NameString = "02:00:00:02:00:00"
+		peer3NameString = "03:00:00:03:00:00"
+	)
+	var (
+		peer1Name, _ = PeerNameFromString(peer1NameString)
+		peer2Name, _ = PeerNameFromString(peer2NameString)
+		peer3Name, _ = PeerNameFromString(peer3NameString)
+	)
+
+	// Create some peers with some connections to each other
+	p1, ps1 := newNode(peer1Name)
+	p2, ps2 := newNode(peer2Name)
+	p3, ps3 := newNode(peer3Name)
+	ps1.AddTestConnection(p2)
+	ps2.AddTestRemoteConnection(p1, p2)
+	ps2.AddTestConnection(p1)
+	ps2.AddTestConnection(p3)
+	ps3.AddTestConnection(p1)
+	ps1.AddTestConnection(p3)
+	ps2.AddTestRemoteConnection(p1, p3)
+	ps2.AddTestRemoteConnection(p3, p1)
+
+	// Every peer is referenced, so nothing should be dropped
+	wt.AssertEmpty(t, ps1.GarbageCollect(), "peers removed")
+	wt.AssertEmpty(t, ps2.GarbageCollect(), "peers removed")
+	wt.AssertEmpty(t, ps3.GarbageCollect(), "peers removed")
+
+	// Drop the connection from 2 to 3, and 3 isn't garbage-collected
+	// because 1 has a connection to 3
+	ps2.DeleteTestConnection(p3)
+	wt.AssertEmpty(t, ps2.GarbageCollect(), "peers removed")
+
+	// Drop the connection from 1 to 3, and 3 will get removed by
+	// garbage-collection
+	ps1.DeleteTestConnection(p3)
+	checkPeerArray(t, ps1.GarbageCollect(), p3)
+}

router/protocol.go

@@ -0,0 +1,28 @@
+package router
+
+const (
+	Protocol        = "weave"
+	ProtocolVersion = 12
+)
+
+type ProtocolTag byte
+
+const (
+	ProtocolConnectionEstablished ProtocolTag = iota
+	ProtocolFragmentationReceived
+	ProtocolStartFragmentationTest
+	ProtocolNonce
+	ProtocolPMTUVerified
+	ProtocolGossip
+	ProtocolGossipUnicast
+	ProtocolGossipBroadcast
+)
+
+type ProtocolMsg struct {
+	tag ProtocolTag
+	msg []byte
+}
+
+type ProtocolSender interface {
+	SendProtocolMsg(m ProtocolMsg)
+}

router/router.go

@@ -17,47 +17,81 @@ const macMaxAge = 10 * time.Minute // [1]
 // [1] should be greater than typical ARP cache expiries, i.e. > 3/2 *
 // /proc/sys/net/ipv4_neigh/*/base_reachable_time_ms on Linux
 
+type Router struct {
+	Iface           *net.Interface
+	Ourself         *LocalPeer
+	Macs            *MacCache
+	Peers           *Peers
+	Routes          *Routes
+	ConnectionMaker *ConnectionMaker
+	GossipChannels  map[uint32]*GossipChannel
+	TopologyGossip  Gossip
+	UDPListener     *net.UDPConn
+	Password        *[]byte
+	ConnLimit       int
+	BufSz           int
+	LogFrame        func(string, []byte, *layers.Ethernet)
+}
+
+type PacketSource interface {
+	ReadPacket() ([]byte, error)
+}
+
+type PacketSink interface {
+	WritePacket([]byte) error
+}
+
+type PacketSourceSink interface {
+	PacketSource
+	PacketSink
+}
+
 func NewRouter(iface *net.Interface, name PeerName, password []byte, connLimit int, bufSz int, logFrame func(string, []byte, *layers.Ethernet)) *Router {
+	router := &Router{
+		Iface:          iface,
+		GossipChannels: make(map[uint32]*GossipChannel),
+		ConnLimit:      connLimit,
+		BufSz:          bufSz,
+		LogFrame:       logFrame}
+	if len(password) > 0 {
+		router.Password = &password
+	}
 	onMacExpiry := func(mac net.HardwareAddr, peer *Peer) {
 		log.Println("Expired MAC", mac, "at", peer.Name)
 	}
 	onPeerGC := func(peer *Peer) {
-		log.Println("Removing unreachable", peer)
+		router.Macs.Delete(peer)
+		log.Println("Removed unreachable", peer)
 	}
-	router := &Router{
+	router.Ourself = NewLocalPeer(name, router)
-		Iface:     iface,
+	router.Macs = NewMacCache(macMaxAge, onMacExpiry)
-		Macs:      NewMacCache(macMaxAge, onMacExpiry),
+	router.Peers = NewPeers(router.Ourself.Peer, onPeerGC)
-		ConnLimit: connLimit,
-		BufSz:     bufSz,
-		LogFrame:  logFrame}
-	if len(password) > 0 {
-		router.Password = &password
-	}
-	router.Ourself = StartLocalPeer(name, router)
-	router.Peers = NewPeers(router.Ourself.Peer, router.Macs, onPeerGC)
 	router.Peers.FetchWithDefault(router.Ourself.Peer)
-	log.Println("Our name is", router.Ourself.Name)
+	router.Routes = NewRoutes(router.Ourself.Peer, router.Peers)
-
+	router.ConnectionMaker = NewConnectionMaker(router.Ourself, router.Peers)
+	router.TopologyGossip = router.NewGossip("topology", router)
 	return router
 }
 
-func (router *Router) UsingPassword() bool {
-	return router.Password != nil
-}
-
 func (router *Router) Start() {
 	// we need two pcap handles since they aren't thread-safe
 	pio, err := NewPcapIO(router.Iface.Name, router.BufSz)
 	checkFatal(err)
 	po, err := NewPcapO(router.Iface.Name)
 	checkFatal(err)
-	router.Routes = StartRoutes(router.Ourself.Peer, router.Peers)
+	router.Ourself.Start()
-	router.ConnectionMaker = StartConnectionMaker(router.Ourself, router.Peers)
+	router.Macs.Start()
+	router.Routes.Start()
+	router.ConnectionMaker.Start()
 	router.UDPListener = router.listenUDP(Port, po)
 	router.listenTCP(Port)
 	router.sniff(pio)
 }
 
+func (router *Router) UsingPassword() bool {
+	return router.Password != nil
+}
+
 func (router *Router) Status() string {
 	var buf bytes.Buffer
 	buf.WriteString(fmt.Sprintln("Our name is", router.Ourself.Name))
@@ -156,8 +190,11 @@ func (router *Router) acceptTCP(tcpConn *net.TCPConn) {
 	// someone else is dialing us, so our udp sender is the conn
 	// on Port and we wait for them to send us something on UDP to
 	// start.
-	connRemote := NewRemoteConnection(router.Ourself.Peer, nil, tcpConn.RemoteAddr().String())
+	remoteAddrStr := tcpConn.RemoteAddr().String()
-	NewLocalConnection(connRemote, true, tcpConn, nil, router)
+	log.Printf("->[%s] connection accepted\n", remoteAddrStr)
+	connRemote := NewRemoteConnection(router.Ourself.Peer, nil, remoteAddrStr, false)
+	connLocal := NewLocalConnection(connRemote, tcpConn, nil, router)
+	connLocal.Start(true)
 }
 
 func (router *Router) listenUDP(localPort int, po PacketSink) *net.UDPConn {
@@ -176,6 +213,16 @@ func (router *Router) listenUDP(localPort int, po PacketSink) *net.UDPConn {
 	return conn
 }
 
+type UDPPacket struct {
+	Name   PeerName
+	Packet []byte
+	Sender *net.UDPAddr
+}
+
+func (packet UDPPacket) String() string {
+	return fmt.Sprintf("UDP Packet\n name: %s\n sender: %v\n payload: % X", packet.Name, packet.Sender, packet.Packet)
+}
+
 func (router *Router) udpReader(conn *net.UDPConn, po PacketSink) {
 	defer conn.Close()
 	dec := NewEthernetDecoder()
@@ -210,7 +257,7 @@ func (router *Router) udpReader(conn *net.UDPConn, po PacketSink) {
 		err = relayConn.Decryptor.IterateFrames(handleUDPPacket, udpPacket)
 		if pde, ok := err.(PacketDecodingError); ok {
 			if pde.Fatal {
-				relayConn.CheckFatal(pde)
+				relayConn.Shutdown(pde)
 			} else {
 				relayConn.log(pde.Error())
 			}
@@ -245,14 +292,41 @@ func (router *Router) handleUDPPacketFunc(dec *EthernetDecoder, po PacketSink) F
 
 		dec.DecodeLayers(frame)
 		decodedLen := len(dec.decoded)
+		if decodedLen == 0 {
+			return nil
+		}
+		// Handle special frames produced internally (rather than
+		// captured/forwarded) by the remote router.
+		//
+		// We really shouldn't be decoding these above, since they are
+		// not genuine Ethernet frames. However, it is actually more
+		// efficient to do so, as we want to optimise for the common
+		// (i.e. non-special) frames. These always need decoding, and
+		// detecting special frames is cheaper post decoding than pre.
+		if decodedLen == 1 && dec.IsSpecial() {
+			if srcPeer != relayConn.Remote() || dstPeer != router.Ourself.Peer {
+				// A special frame not originating from the remote, or
+				// not for us? How odd; let's just drop it.
+				return nil
+			}
+			switch {
+			case frameLen == EthernetOverhead+8:
+				relayConn.ReceivedHeartbeat(sender, binary.BigEndian.Uint64(frame[EthernetOverhead:]))
+			case frameLen == FragTestSize && bytes.Equal(frame, FragTest):
+				relayConn.SendProtocolMsg(ProtocolMsg{ProtocolFragmentationReceived, nil})
+			case frameLen == PMTUDiscoverySize && bytes.Equal(frame, PMTUDiscovery):
+			default:
+				frameLenBytes := []byte{0, 0}
+				binary.BigEndian.PutUint16(frameLenBytes, uint16(frameLen-EthernetOverhead))
+				relayConn.SendProtocolMsg(ProtocolMsg{ProtocolPMTUVerified, frameLenBytes})
+			}
+			return nil
+		}
+
 		df := decodedLen == 2 && (dec.ip.Flags&layers.IPv4DontFragment != 0)
-		srcMac := dec.eth.SrcMAC
 
 		if dstPeer != router.Ourself.Peer {
 			// it's not for us, we're just relaying it
-			if decodedLen == 0 {
-				return nil
-			}
 			if df {
 				router.LogFrame("Relaying DF", frame, &dec.eth)
 			} else {
@@ -261,36 +335,16 @@ func (router *Router) handleUDPPacketFunc(dec *EthernetDecoder, po PacketSink) F
 			return checkFrameTooBig(router.Ourself.Relay(srcPeer, dstPeer, df, frame, dec), srcPeer)
 		}
 
-		if relayConn.Remote().Name == srcPeer.Name {
+		srcMac := dec.eth.SrcMAC
-			if frameLen == 0 {
+		dstMac := dec.eth.DstMAC
-				relayConn.SetRemoteUDPAddr(sender)
-				return nil
-			} else if frameLen == FragTestSize && bytes.Equal(frame, FragTest) {
-				relayConn.SendTCP(ProtocolFragmentationReceivedByte)
-				return nil
-			} else if frameLen == PMTUDiscoverySize && bytes.Equal(frame, PMTUDiscovery) {
-				return nil
-			}
-		}
-
-		if decodedLen == 0 {
-			return nil
-		}
-
-		if dec.IsPMTUVerify() && relayConn.Remote().Name == srcPeer.Name {
-			frameLenBytes := []byte{0, 0}
-			binary.BigEndian.PutUint16(frameLenBytes, uint16(frameLen-EthernetOverhead))
-			relayConn.SendTCP(Concat(ProtocolPMTUVerifiedByte, frameLenBytes))
-			return nil
-		}
 
 		if router.Macs.Enter(srcMac, srcPeer) {
-			log.Println("Discovered remote MAC", srcMac, "at", srcPeer.Name)
+			log.Println("Discovered remote MAC", srcMac, "at", srcName)
 		}
 		router.LogFrame("Injecting", frame, &dec.eth)
 		checkWarn(po.WritePacket(frame))
 
-		dstPeer, found = router.Macs.Lookup(dec.eth.DstMAC)
+		dstPeer, found = router.Macs.Lookup(dstMac)
 		if !found || dstPeer != router.Ourself.Peer {
 			return checkFrameTooBig(router.Ourself.RelayBroadcast(srcPeer, df, frame, dec), srcPeer)
 		}
@@ -298,3 +352,41 @@ func (router *Router) handleUDPPacketFunc(dec *EthernetDecoder, po PacketSink) F
 		return nil
 	}
 }
+
+// Gossiper methods - the Router is the topology Gossiper
+
+func (router *Router) OnGossipUnicast(sender PeerName, msg []byte) error {
+	return fmt.Errorf("unexpected topology gossip unicast: %v", msg)
+}
+
+func (router *Router) OnGossipBroadcast(msg []byte) error {
+	return fmt.Errorf("unexpected topology gossip broadcast: %v", msg)
+}
+
+// Return state of everything we know; intended to be called periodically
+func (router *Router) Gossip() []byte {
+	return router.Peers.EncodeAllPeers()
+}
+
+// merge in state and return "everything new I've just learnt",
+// or nil if nothing in the received message was new
+func (router *Router) OnGossip(buf []byte) ([]byte, error) {
+	newUpdate, err := router.Peers.ApplyUpdate(buf)
+	if _, ok := err.(UnknownPeerError); err != nil && ok {
+		// That update contained a reference to a peer which wasn't
+		// itself included in the update, and we didn't know about
+		// already. We ignore this; eventually we should receive an
+		// update containing a complete topology.
+		log.Println("Topology gossip:", err)
+		return nil, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+	if len(newUpdate) == 0 {
+		return nil, nil
+	}
+	router.ConnectionMaker.Refresh()
+	router.Routes.Recalculate()
+	return newUpdate, nil
+}

router/routes.go

@@ -16,6 +16,23 @@ type Routes struct {
 	queryChan chan<- *Interaction
 }
 
+func NewRoutes(ourself *Peer, peers *Peers) *Routes {
+	routes := &Routes{
+		ourself:   ourself,
+		peers:     peers,
+		unicast:   make(map[PeerName]PeerName),
+		broadcast: make(map[PeerName][]PeerName)}
+	routes.unicast[ourself.Name] = UnknownPeerName
+	routes.broadcast[ourself.Name] = []PeerName{}
+	return routes
+}
+
+func (routes *Routes) Start() {
+	queryChan := make(chan *Interaction, ChannelSize)
+	routes.queryChan = queryChan
+	go routes.queryLoop(queryChan)
+}
+
 func (routes *Routes) Unicast(name PeerName) (PeerName, bool) {
 	routes.RLock()
 	defer routes.RUnlock()
@@ -48,20 +65,6 @@ func (routes *Routes) String() string {
 	return buf.String()
 }
 
-func StartRoutes(ourself *Peer, peers *Peers) *Routes {
-	queryChan := make(chan *Interaction, ChannelSize)
-	state := &Routes{
-		ourself:   ourself,
-		peers:     peers,
-		unicast:   make(map[PeerName]PeerName),
-		broadcast: make(map[PeerName][]PeerName),
-		queryChan: queryChan}
-	state.unicast[ourself.Name] = UnknownPeerName
-	state.broadcast[ourself.Name] = []PeerName{}
-	go state.queryLoop(queryChan)
-	return state
-}
-
 // ACTOR client API
 
 const (

router/types.go

@@ -1,269 +0,0 @@
-package router
-
-import (
-	"bytes"
-	"code.google.com/p/go-bit/bit"
-	"code.google.com/p/gopacket"
-	"code.google.com/p/gopacket/layers"
-	"encoding/gob"
-	"net"
-	"sync"
-	"time"
-)
-
-type Router struct {
-	Iface           *net.Interface
-	Ourself         *LocalPeer
-	Macs            *MacCache
-	Peers           *Peers
-	Routes          *Routes
-	ConnectionMaker *ConnectionMaker
-	UDPListener     *net.UDPConn
-	Password        *[]byte
-	ConnLimit       int
-	BufSz           int
-	LogFrame        func(string, []byte, *layers.Ethernet)
-}
-
-type Connection interface {
-	Local() *Peer
-	Remote() *Peer
-	RemoteTCPAddr() string
-	Established() bool
-	Shutdown()
-}
-
-type RemoteConnection struct {
-	local         *Peer
-	remote        *Peer
-	remoteTCPAddr string
-}
-
-type LocalConnection struct {
-	sync.RWMutex
-	RemoteConnection
-	TCPConn       *net.TCPConn
-	tcpSender     TCPSender
-	remoteUDPAddr *net.UDPAddr
-	established   bool
-	stackFrag     bool
-	effectivePMTU int
-	SessionKey    *[32]byte
-	heartbeat     *time.Ticker
-	fetchAll      *time.Ticker
-	fragTest      *time.Ticker
-	forwardChan   chan<- *ForwardedFrame
-	forwardChanDF chan<- *ForwardedFrame
-	stopForward   chan<- interface{}
-	stopForwardDF chan<- interface{}
-	verifyPMTU    chan<- int
-	Decryptor     Decryptor
-	Router        *Router
-	UID           uint64
-	shutdown      bool
-	queryChan     chan<- *ConnectionInteraction
-}
-
-type ConnectionInteraction struct {
-	Interaction
-	payload interface{}
-}
-
-type Forwarder struct {
-	conn            *LocalConnection
-	ch              <-chan *ForwardedFrame
-	stop            <-chan interface{}
-	verifyPMTUTick  <-chan time.Time
-	verifyPMTU      <-chan int
-	pmtuVerifyCount uint
-	enc             Encryptor
-	udpSender       UDPSender
-	maxPayload      int
-	pmtuVerified    bool
-	highestGoodPMTU int
-	unverifiedPMTU  int
-	lowestBadPMTU   int
-}
-
-type UDPPacket struct {
-	Name   PeerName
-	Packet []byte
-	Sender *net.UDPAddr
-}
-
-type EthernetDecoder struct {
-	eth     layers.Ethernet
-	ip      layers.IPv4
-	decoded []gopacket.LayerType
-	parser  *gopacket.DecodingLayerParser
-}
-
-type MsgTooBigError struct {
-	PMTU int // actual pmtu, i.e. what the kernel told us
-}
-
-type FrameTooBigError struct {
-	EPMTU int // effective pmtu, i.e. what we tell packet senders
-}
-
-type UnknownPeersError struct {
-}
-
-type NameCollisionError struct {
-	Name PeerName
-}
-
-type PacketDecodingError struct {
-	Fatal bool
-	Desc  string
-}
-
-type LocalAddress struct {
-	ip      net.IP
-	network *net.IPNet
-}
-
-type ForwardedFrame struct {
-	srcPeer *Peer
-	dstPeer *Peer
-	frame   []byte
-}
-
-type Interaction struct {
-	code       int
-	resultChan chan<- interface{}
-}
-
-// UDPSender interface and implementations
-
-type UDPSender interface {
-	Send([]byte) error
-	Shutdown() error
-}
-
-type SimpleUDPSender struct {
-	conn    *LocalConnection
-	udpConn *net.UDPConn
-}
-
-type RawUDPSender struct {
-	ipBuf     gopacket.SerializeBuffer
-	opts      gopacket.SerializeOptions
-	udpHeader *layers.UDP
-	socket    *net.IPConn
-	conn      *LocalConnection
-}
-
-// TCPSender interface and implementations
-
-type TCPSender interface {
-	Send([]byte) error
-}
-
-type SimpleTCPSender struct {
-	encoder *gob.Encoder
-}
-
-type EncryptedTCPSender struct {
-	sync.RWMutex
-	outerEncoder *gob.Encoder
-	innerEncoder *gob.Encoder
-	buffer       *bytes.Buffer
-	conn         *LocalConnection
-	msgCount     int
-}
-
-type EncryptedTCPMessage struct {
-	Number int
-	Body   []byte
-}
-
-// TCPReceiver interface and implementations
-
-type TCPReceiver interface {
-	Decode([]byte) ([]byte, error)
-}
-
-type SimpleTCPReceiver struct {
-}
-
-type EncryptedTCPReceiver struct {
-	conn     *LocalConnection
-	decoder  *gob.Decoder
-	buffer   *bytes.Buffer
-	msgCount int
-}
-
-// Encryptor interface and implementations
-
-type Encryptor interface {
-	FrameOverhead() int
-	PacketOverhead() int
-	IsEmpty() bool
-	Bytes() []byte
-	AppendFrame(*ForwardedFrame)
-	TotalLen() int
-}
-
-type NonEncryptor struct {
-	buf       []byte
-	bufTail   []byte
-	buffered  int
-	prefixLen int
-}
-
-type NaClEncryptor struct {
-	NonEncryptor
-	buf       []byte
-	offset    uint16
-	nonce     *[24]byte
-	nonceChan chan *[24]byte
-	flags     uint16
-	prefixLen int
-	conn      *LocalConnection
-	df        bool
-}
-
-// Decryptor interface and implementations
-
-type FrameConsumer func(*LocalConnection, *net.UDPAddr, []byte, []byte, uint16, []byte) error
-
-type Decryptor interface {
-	IterateFrames(FrameConsumer, *UDPPacket) error
-	ReceiveNonce([]byte)
-	Shutdown()
-}
-
-type NonDecryptor struct {
-	conn *LocalConnection
-}
-
-type NaClDecryptor struct {
-	NonDecryptor
-	instance   *NaClDecryptorInstance
-	instanceDF *NaClDecryptorInstance
-}
-
-type NaClDecryptorInstance struct {
-	nonce               *[24]byte
-	previousNonce       *[24]byte
-	usedOffsets         *bit.Set
-	previousUsedOffsets *bit.Set
-	highestOffsetSeen   uint16
-	nonceChan           chan *[24]byte
-}
-
-// Packet capture/inject interfaces
-
-type PacketSource interface {
-	ReadPacket() ([]byte, error)
-}
-
-type PacketSink interface {
-	WritePacket([]byte) error
-}
-
-type PacketSourceSink interface {
-	PacketSource
-	PacketSink
-}

router/udp_sender.go

@@ -0,0 +1,138 @@
+package router
+
+import (
+	"code.google.com/p/gopacket"
+	"code.google.com/p/gopacket/layers"
+	"log"
+	"net"
+	"syscall"
+)
+
+type UDPSender interface {
+	Send([]byte) error
+	Shutdown() error
+}
+
+type SimpleUDPSender struct {
+	conn    *LocalConnection
+	udpConn *net.UDPConn
+}
+
+type RawUDPSender struct {
+	ipBuf     gopacket.SerializeBuffer
+	opts      gopacket.SerializeOptions
+	udpHeader *layers.UDP
+	socket    *net.IPConn
+	conn      *LocalConnection
+}
+
+type MsgTooBigError struct {
+	PMTU int // actual pmtu, i.e. what the kernel told us
+}
+
+func NewSimpleUDPSender(conn *LocalConnection) *SimpleUDPSender {
+	return &SimpleUDPSender{udpConn: conn.Router.UDPListener, conn: conn}
+}
+
+func (sender *SimpleUDPSender) Send(msg []byte) error {
+	_, err := sender.udpConn.WriteToUDP(msg, sender.conn.RemoteUDPAddr())
+	return err
+}
+
+func (sender *SimpleUDPSender) Shutdown() error {
+	return nil
+}
+
+func NewRawUDPSender(conn *LocalConnection) (*RawUDPSender, error) {
+	ipSocket, err := dialIP(conn)
+	if err != nil {
+		return nil, err
+	}
+	udpHeader := &layers.UDP{SrcPort: layers.UDPPort(Port)}
+	ipBuf := gopacket.NewSerializeBuffer()
+	opts := gopacket.SerializeOptions{
+		FixLengths: true,
+		// UDP header is calculated with a phantom IP
+		// header. Yes, it's totally nuts. Thankfully, for UDP
+		// over IPv4, the checksum is optional. It's not
+		// optional for IPv6, but we'll ignore that for
+		// now. TODO
+		ComputeChecksums: false}
+
+	return &RawUDPSender{
+		ipBuf:     ipBuf,
+		opts:      opts,
+		udpHeader: udpHeader,
+		socket:    ipSocket,
+		conn:      conn}, nil
+}
+
+func (sender *RawUDPSender) Send(msg []byte) error {
+	payload := gopacket.Payload(msg)
+	sender.udpHeader.DstPort = layers.UDPPort(sender.conn.RemoteUDPAddr().Port)
+
+	err := gopacket.SerializeLayers(sender.ipBuf, sender.opts, sender.udpHeader, &payload)
+	if err != nil {
+		return err
+	}
+	packet := sender.ipBuf.Bytes()
+	_, err = sender.socket.Write(packet)
+	if err == nil || PosixError(err) != syscall.EMSGSIZE {
+		return err
+	}
+	f, err := sender.socket.File()
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	fd := int(f.Fd())
+	log.Println("EMSGSIZE on send, expecting PMTU update (IP packet was",
+		len(packet), "bytes, payload was", len(msg), "bytes)")
+	pmtu, err := syscall.GetsockoptInt(fd, syscall.IPPROTO_IP, syscall.IP_MTU)
+	if err != nil {
+		return err
+	}
+	return MsgTooBigError{PMTU: pmtu}
+}
+
+func (sender *RawUDPSender) Shutdown() error {
+	defer func() { sender.socket = nil }()
+	return sender.socket.Close()
+}
+
+func dialIP(conn *LocalConnection) (*net.IPConn, error) {
+	ipLocalAddr, err := ipAddr(conn.TCPConn.LocalAddr())
+	if err != nil {
+		return nil, err
+	}
+	ipRemoteAddr, err := ipAddr(conn.TCPConn.RemoteAddr())
+	if err != nil {
+		return nil, err
+	}
+	ipSocket, err := net.DialIP("ip4:UDP", ipLocalAddr, ipRemoteAddr)
+	if err != nil {
+		return nil, err
+	}
+	f, err := ipSocket.File()
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	fd := int(f.Fd())
+	// This Makes sure all packets we send out have DF set on them.
+	err = syscall.SetsockoptInt(fd, syscall.IPPROTO_IP, syscall.IP_MTU_DISCOVER, syscall.IP_PMTUDISC_DO)
+	if err != nil {
+		return nil, err
+	}
+	return ipSocket, nil
+}
+
+func ipAddr(addr net.Addr) (*net.IPAddr, error) {
+	host, _, err := net.SplitHostPort(addr.String())
+	if err != nil {
+		return nil, err
+	}
+	return &net.IPAddr{
+		IP:   net.ParseIP(host),
+		Zone: ""}, nil
+}

router/utils.go

@@ -1,12 +1,20 @@
 package router
 
 import (
+	"bytes"
 	"crypto/rand"
+	"encoding/gob"
 	"fmt"
+	"hash/fnv"
 	"log"
 	"net"
 )
 
+type Interaction struct {
+	code       int
+	resultChan chan<- interface{}
+}
+
 func checkFatal(e error) {
 	if e != nil {
 		log.Fatal(e)
@@ -38,8 +46,8 @@ func (ftbe FrameTooBigError) Error() string {
 	return fmt.Sprint("Frame too big error. Effective PMTU is ", ftbe.EPMTU)
 }
 
-func (upe UnknownPeersError) Error() string {
+func (upe UnknownPeerError) Error() string {
-	return fmt.Sprint("Reference to unknown peers")
+	return fmt.Sprint("Reference to unknown peer ", upe.Name)
 }
 
 func (nce NameCollisionError) Error() string {
@@ -50,10 +58,6 @@ func (pde PacketDecodingError) Error() string {
 	return fmt.Sprint("Failed to decode packet: ", pde.Desc)
 }
 
-func (packet UDPPacket) String() string {
-	return fmt.Sprintf("UDP Packet\n name: %s\n sender: %v\n payload: % X", packet.Name, packet.Sender, packet.Packet)
-}
-
 func Concat(elems ...[]byte) []byte {
 	res := []byte{}
 	for _, e := range elems {
@@ -73,6 +77,21 @@ func randUint64() (r uint64) {
 	return
 }
 
+func hash(s string) uint32 {
+	h := fnv.New32a()
+	h.Write([]byte(s))
+	return h.Sum32()
+}
+
+func GobEncode(items ...interface{}) []byte {
+	buf := new(bytes.Buffer)
+	enc := gob.NewEncoder(buf)
+	for _, i := range items {
+		checkFatal(enc.Encode(i))
+	}
+	return buf.Bytes()
+}
+
 func macint(mac net.HardwareAddr) (r uint64) {
 	for _, b := range mac {
 		r <<= 8

site/features.md

@@ -95,7 +95,9 @@ same host, via their docker-assigned IP addresses. For complete
 isolation between application containers, that feature needs to be
 disabled by
 [setting `--icc=false`](https://docs.docker.com/articles/networking/#between-containers)
-in the docker daemon configuration.
+in the docker daemon configuration. Furthermore, containers should be
+prevented from capturing and injecting raw network packets - this can
+be accomplished by starting them with the `--cap-drop net_raw` option.
 
 ### <a name="dynamic-network-attachment"></a>Dynamic network attachment
 
@@ -129,23 +131,23 @@ sharing it between applications:
 ### <a name="security"></a>Security
 
 In order to connect containers across untrusted networks, weave peers
-can be told to encrypt traffic by supplying a `-password` or
+can be told to encrypt traffic by supplying a `-password` option or
-`-passwordfile` option when launching weave, e.g.
+`WEAVE_PASSWORD` environment variable when launching weave, e.g.
 
     host1# weave launch -password wEaVe
 
 or
 
-    host1# weave launch -passwordfile /path/to/file
+    host1# export WEAVE_PASSWORD=wEaVe
+    host1# weave launch
 
-where the file must contain a single line like this:
+_NOTE: The command line option takes precedence over the environment
-
+variable._
-    WEAVE_PASSWORD=wEaVe
 
 The same password must be specified for all weave peers; it is a
 component in the creation of ephemeral session keys for connections
-between peers. See the
+between peers. See the [crypto documentation](how-it-works.html#crypto)
-[crypto documentation](how-it-works.html#crypto) for more details.
+for more details.
 
 ### <a name="host-network-integration"></a>Host network integration
 

site/how-it-works.md

@@ -12,9 +12,9 @@ different each time it is run.  These are opaque identifiers as far as
 the router is concerned, although the name defaults to a MAC address.
 
 Weave routers establish TCP connections to each other, over which they
-perform a protocol handshake and subsequently exchange topology
+perform a protocol handshake and subsequently exchange
-information (see below). These connections are encrypted if so
+[topology](#topology) information. These connections are encrypted if
-configured. Peers also establish UDP "connections", possibly
+so configured. Peers also establish UDP "connections", possibly
 encrypted, which carry encapsulated network packets. These
 "connections" are duplex and can traverse firewalls.
 
@@ -35,9 +35,11 @@ packet to peers.
 Weave routers learn which peer host a particular MAC address resides
 on. They combine this knowledge with topology information in order to
 make routing decisions and thus avoid forwarding every packet to every
-peer. The topology information captures which peers are connected to
+peer. Weave can route packets in partially connected networks with
-which other peers; weave can route packets in partially connected
+changing topology. For example, in this network, peer 1 is connected
-networks with changing topology.
+directly to 2 and 3, but if 1 needs to send a packet to 4 or 5 it must
+first send it to peer 3:
+![Partially connected Weave Network](images/top-diag1.png "Partially connected Weave Network")
 
 ### <a name="encapsulation"></a>Encapsulation
 
@@ -78,7 +80,7 @@ The name of the sending peer enables the receiving peer to identify
 who sent this UDP packet. This is followed by the meta data and
 payload for one or more captured frames. The router performs batching:
 if it captures several frames very quickly that all need forwarding to
-the same peer, it will fit as many of them as possible into a single
+the same peer, it fits as many of them as possible into a single
 UDP packet.
 
 The meta data for each frame contains the names of the capturing and
@@ -88,13 +90,117 @@ receiving peers to build up their mappings of which client MAC
 addresses are local to which peers. The destination peer name enables
 the receiving peer to identify whether this frame is destined for
 itself or whether it should be forwarded on to some other peer,
-accomodating multi-hop routing. This will work even when the receiving
+accommodating multi-hop routing. This works even when the receiving
 intermediate peer has no knowledge of the destination MAC: only the
 original capturing peer needs to determine the destination peer from
 the MAC. This way weave peers never need to exchange the MAC addresses
 of clients and need not take any special action for ARP traffic and
 MAC discovery.
 
+### <a name="topology"></a>Topology
+
+The topology information captures which peers are connected to which
+other peers. Weave peers communicate their knowledge of the topology
+(and changes to it) to their neighbours, who then pass this
+information on to their own neighbours, and so on, until the entire
+network knows about any change.
+
+Topology is communicated over the TCP links between peers, using a
+Gossip mechanism.  Topology messages are sent by a peer...
+
+- when a connection has been added; the entire topology is sent to the
+  remote peer, and an incremental update, containing information on
+  just the two peers at the ends of the connection, is sent to all
+  neighbours,
+- when a connection has been marked as 'established', indicating that
+  the remote peer can receive UDP traffic from the peer; an update
+  containing just information about the local peer is sent to all
+  neighbours,
+- when a connection has been torn down; an update containing just
+  information about the local peer is sent to all neighbours,
+- periodically, on a timer, in case someone has missed an update.
+
+The receiver of a topology update merges that update with its own
+topology model, adding peers hitherto unknown to it, and updating
+peers for which the update contains a more recent version than known
+to it. If there were any such new/updated peers, then an improved
+update containing them is sent out on all connections.
+
+If the update mentions a peer that the receiver does not know, then
+the entire update is ignored.
+
+#### Message details
+Every gossip message is structured as follows:
+
+    +-----------------------------------+
+    | 1-byte message type - Gossip      |
+    +-----------------------------------+
+    | 4-byte Gossip channel - Topology  |
+    +-----------------------------------+
+    | Peer Name of source               |
+    +-----------------------------------+
+    | Gossip payload (topology update)  |
+    +-----------------------------------+
+
+The topology update payload is laid out like this:
+
+    +-----------------------------------+
+    | Peer 1: Name                      |
+    +-----------------------------------+
+    | Peer 1: UID                       |
+    +-----------------------------------+
+    | Peer 1: Version number            |
+    +-----------------------------------+
+    | Peer 1: List of connections       |
+    +-----------------------------------+
+    |                ...                |
+    +-----------------------------------+
+    | Peer N: Name                      |
+    +-----------------------------------+
+    | Peer N: UID                       |
+    +-----------------------------------+
+    | Peer N: Version number            |
+    +-----------------------------------+
+    | Peer N: List of connections       |
+    +-----------------------------------+
+
+Each List of connections is encapsulated as a byte buffer, within
+which the structure is:
+
+    +-----------------------------------+
+    | Connection 1: Remote Peer Name    |
+    +-----------------------------------+
+    | Connection 1: Remote IP address   |
+    +-----------------------------------+
+    | Connection 2: Remote Peer Name    |
+    +-----------------------------------+
+    | Connection 2: Remote IP address   |
+    +-----------------------------------+
+    |                ...                |
+    +-----------------------------------+
+    | Connection N: Remote Peer Name    |
+    +-----------------------------------+
+    | Connection N: Remote IP address   |
+    +-----------------------------------+
+
+#### Removal of peers
+If a peer, after receiving a topology update, sees that another peer
+no longer has any connections within the network, it drops all
+knowledge of that second peer.
+
+#### Out-of-date topology
+The peer-to-peer gossiping of updates is not instantaneous, so it is
+very possible for a node elsewhere in the network to have an
+out-of-date view.
+
+If the destination peer for a packet is still reachable, then
+out-of-date topology can result in it taking a less efficient route.
+
+If the out-of-date topology makes it look as if the destination peer
+is not reachable, then the packet is dropped.  For most protocols
+(e.g. TCP), the transmission will be retried a short time later, by
+which time the topology should have updated.
+
 ### <a name="crypto"></a>Crypto
 
 Weave can be configured to encrypt both the data passing over the TCP
@@ -210,7 +316,7 @@ This is very similar to the [non-crypto encapsulation](#encapsulation).
 All of the frames are encrypted with the same ephemeral session key
 and all must be decrypted by the receiving peer. Frames which are to
 be forwarded on to some further peer will be re-encrypted with the
-relevant ephemeral session keys for the onwards connections. Thus all
+relevant ephemeral session keys for the onward connections. Thus all
 traffic is fully decrypted on every peer it passes through.
 Encryption is again done with the NaCl `secretbox.Seal` function.
 

site/index.md

@@ -9,6 +9,8 @@ layout: default
  * [Features](features.html)
  * [Troubleshooting](troubleshooting.html)
  * [Building](building.html)
- * [Using with Boot2Docker](boot2docker.html)
  * [How it works](how-it-works.html)
  * [Contact information](https://github.com/zettio/weave#contact-us)
+ * *Installation Notes*
+   * [Using with boot2docker](boot2docker.html)
+   * [Using with systemd](systemd.html)

site/systemd.md

@@ -0,0 +1,70 @@
+---
+title: Using Weave with systemd
+layout: default
+---
+
+Having installed `weave` as per [readme][], you might wish to configure the
+init daemon to start it on boot. Most recent Linux distribution releases are
+shipping with [systemd][]. The information below should provide you with some
+initial guidance on getting a weave service configured on systemd-based OS.
+
+## Weave Service Unit and Configuration
+
+A regular service unit definition for weave is shown below and you should
+normally place it in `/etc/systemd/system/weave.service`.
+
+    [Unit]
+    Description=Weave Network
+    Documentation=http://zettio.github.io/weave/
+    Requires=docker.service
+    After=docker.service
+    [Service]
+    EnvironmentFile=-/etc/sysconfig/weave
+    ExecStartPre=/usr/local/bin/weave launch $PEERS
+    ExecStart=/usr/bin/docker logs -f weave
+    SuccessExitStatus=2
+    ExecStop=/usr/local/bin/weave stop
+    [Install]
+    WantedBy=multi-user.target
+
+
+To specify the addresses or names of other weave hosts to join the network
+you can create the `/etc/sysconfig/weave` environment file which would be of
+the following format:
+
+    PEERS="HOST1 HOST2 .. HOSTn"
+
+You can also use the [connect][] command to add participating hosts dynamically.
+
+Additionally, if you want to enable [encryption][] you can specify a password with
+`WEAVE_PASSWORD="MakeSureThisIsSecure"` in the `/etc/sysconfig/weave` environment
+file, and it will get picked up by weave on launch.
+
+You now should be able to launch weave with
+
+    sudo systemctl start weave
+
+To ensure weave launches after reboot, you need run
+
+    sudo systemctl enable weave
+
+For more information on systemd, please refer to the documentation supplied
+by your distribution of Linux.
+
+## SELinux Tweaks
+
+If your OS has SELinux enabled and you wish to run weave as a systemd unit,
+then you should follow the instructions below. These instructions apply to
+CentOS and RHEL as of 7.0. On Fedora 21, there is no need to do this.
+
+Once you have installed `weave` in `/usr/local/bin`, set its execution
+context with the commands shown below. You will need to have the
+`policycoreutils-python` package installed.
+
+    sudo semanage fcontext -a -t unconfined_exec_t -f f /usr/local/bin/weave
+    sudo restorecon /usr/local/bin/weave
+
+[readme]: https://github.com/zettio/weave/blob/master/README.md#installation
+[connect]: http://zettio.github.io/weave/features.html#dynamic-topologies
+[systemd]: http://www.freedesktop.org/wiki/Software/systemd/
+[encryption]: http://zettio.github.io/weave/how-it-works.html#crypto

site/top-diag1.dot

@@ -0,0 +1,9 @@
+graph weave {
+rankdir=LR;
+"peer 1"  -- "peer 2" ;
+"peer 1"  -- "peer 3" ;
+"peer 2"  -- "peer 3" ;
+"peer 3"  -- "peer 4" ;
+"peer 4"  -- "peer 5" ;
+"peer 3"  -- "peer 5" ;
+}

site/troubleshooting.md

@@ -57,6 +57,7 @@ This produces output like:
 
 ````
 weave router 0.7.0
+Encryption off
 Our name is 7a:f4:56:87:76:3b
 Sniffing traffic on &{39 65535 ethwe ae:e3:07:9c:8c:d4 up|broadcast|multicast}
 MACs:

test/.gitignore

@@ -1,3 +1,4 @@
 insecure_private_key
 assert.sh
 .vagrant
+.ssh_known_hosts

test/110_encryption_test.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+. ./config.sh
+
+C1=10.1.1.4
+C2=10.1.1.7
+
+start_suite "Ping over encrypted cross-host weave network"
+
+for HOST in $HOST1 $HOST2; do
+    run_on $HOST sudo $WEAVE stop || true
+    run_on $HOST sudo $WEAVE stop-dns || true
+    docker_on $HOST rm -f c1 c2 || true
+done
+
+run_on $HOST1 sudo $WEAVE launch -password notverysecure
+run_on $HOST2 sudo $WEAVE launch -password notverysecure $HOST1
+
+run_on $HOST2 sudo $WEAVE run $C2/24 -t --name=c2 ubuntu
+run_on $HOST1 sudo $WEAVE run $C1/24 -t --name=c1 ubuntu
+ok=$(docker -H tcp://$HOST1:2375 exec -i c1 sh -c "ping -q -c 4 $C2 >&2 && echo ok")
+assert "echo $ok" "ok"
+
+end_suite

test/200_dns_test.sh

@@ -5,7 +5,7 @@
 C1=10.2.0.78
 C2=10.2.0.34
 
-start_suite "Resolve names"
+start_suite "Resolve names on a single host"
 
 run_on $HOST1 sudo $WEAVE stop || true
 run_on $HOST1 sudo $WEAVE stop-dns || true
@@ -14,9 +14,12 @@ run_on $HOST1 sudo $WEAVE launch-dns 10.0.0.2/8
 docker_on $HOST1 rm -f c1 c2 || true
 
 run_on $HOST1 sudo $WEAVE run $C2/24 -t --name=c2 -h seetwo.weave.local ubuntu
-run_on $HOST1 sudo $WEAVE run --with-dns $C1/24 -t --name=c1 ubuntu
+run_on $HOST1 sudo $WEAVE run --with-dns $C1/24 -t --name=c1 aanand/docker-dnsutils /bin/sh
 
-ok=$(docker -H tcp://$HOST1:2375 exec -i c1 sh -c "ping -q -c4 seetwo.weave.local >&2 && echo ok")
+ok=$(docker -H tcp://$HOST1:2375 exec -i c1 sh -c "dig +short seetwo.weave.local")
-assert "echo $ok" "ok"
+assert "echo $ok" "$C2"
+
+ok=$(docker -H tcp://$HOST1:2375 exec -i c1 sh -c "dig +short -x $C2")
+assert "echo $ok" "seetwo.weave.local."
 
 end_suite

test/210_dns_multicast_test.sh

@@ -0,0 +1,34 @@
+#! /bin/bash
+
+. ./config.sh
+
+C1=10.2.3.78
+C2=10.2.3.34
+
+start_suite "Resolve names across hosts"
+
+for host in $HOST1 $HOST2; do
+    run_on $host sudo $WEAVE stop || true
+    run_on $host sudo $WEAVE stop-dns || true
+    docker_on $host rm -f c1 c2 || true
+done
+
+run_on $HOST1 sudo $WEAVE launch
+run_on $HOST2 sudo $WEAVE launch $HOST1
+
+run_on $HOST1 sudo $WEAVE launch-dns 10.3.3.2/24 -debug
+run_on $HOST2 sudo $WEAVE launch-dns 10.3.3.3/24 -debug
+
+run_on $HOST2 sudo $WEAVE run $C2/24 -t --name=c2 -h seetwo.weave.local ubuntu
+run_on $HOST1 sudo $WEAVE run --with-dns $C1/24 --name=c1 -t aanand/docker-dnsutils /bin/sh
+
+ok=$(docker -H tcp://$HOST1:2375 exec -i c1 dig +short seetwo.weave.local)
+assert "echo $ok" "$C2"
+
+ok=$(docker -H tcp://$HOST1:2375 exec -i c1 dig +short -x $C2)
+assert "echo $ok" "seetwo.weave.local."
+
+ok=$(docker -H tcp://$HOST1:2375 exec -i c1 dig +short -x 8.8.8.8)
+assert "test -n \"$ok\" && echo pass" "pass"
+
+end_suite

test/300_fallback_dns_test.sh

@@ -0,0 +1,19 @@
+#! /bin/bash
+
+. ./config.sh
+
+start_suite "Resolve a non-weave address"
+
+run_on $HOST1 sudo $WEAVE stop-dns || true
+run_on $HOST1 sudo $WEAVE launch-dns 10.2.6.2 -debug
+docker_on $HOST1 rm -f c1 || true
+
+run_on $HOST1 sudo $WEAVE run --with-dns 10.1.1.5/24 --name=c1 -t aanand/docker-dnsutils /bin/sh
+
+ok=$(docker -H tcp://$HOST1:2375 exec -i c1 dig +short -t MX weave.works)
+assert "test -n \"$ok\" && echo pass" "pass"
+
+ok=$(docker -H tcp://$HOST1:2375 exec -i c1 dig +short -x 8.8.8.8)
+assert "test -n \"$ok\" && echo pass" "pass"
+
+end_suite

test/Vagrantfile

@@ -1,14 +1,17 @@
 VAGRANTFILE_API_VERSION = "2"
 
-host1_ip = "192.168.48.19" # arbitrary private IP
+# these ought to match what is in config.sh
-host2_ip = "192.168.48.15" # arbitrary private IP
+n_machines = 2
+ip_prefix = "192.168.48"
+ip_suffix_base = 10
 
-def configure_docker(host, ip)
+def configure_docker(host, hostname, ip)
-  pkgs = "lxc-docker ethtool conntrack"
+  pkgs = "lxc-docker"
 
   host.vm.box = "phusion/ubuntu-14.04-amd64"
   host.vm.box_url = "https://oss-binaries.phusionpassenger.com/vagrant/boxes/latest/ubuntu-14.04-amd64-vbox.box"
 
+  host.vm.hostname = hostname
   host.vm.network "private_network", ip: ip
 
   # host.vm.synced_folder "./", "/home/vagrant/src/github.com/zettio/weave"
@@ -30,12 +33,11 @@ end
 
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
 
-  config.vm.define "host1" do |host1|
+  (1..n_machines).each do |i|
-    configure_docker(host1, host1_ip)
+    config.vm.define "host#{i}" do |host|
-  end
+      ip_suffix = ip_suffix_base + i
-
+      configure_docker(host, "host#{i}", "#{ip_prefix}.#{ip_suffix}")
-  config.vm.define "host2" do |host2|
+    end
-    configure_docker(host2, host2_ip)
   end
 
 end

test/config.sh

@@ -2,11 +2,24 @@
 
 set -e
 
+# these ought to match what is in Vagrantfile
+N_MACHINES=${N_MACHINES:-2}
+IP_PREFIX=${IP_PREFIX:-192.168.48}
+IP_SUFFIX_BASE=${IP_SUFFIX_BASE:-10}
+
+HOSTS=
+for i in $(seq 1 $N_MACHINES); do
+    IP="${IP_PREFIX}.$((${IP_SUFFIX_BASE}+$i))"
+    HOSTS="$HOSTS $IP"
+done
+
+# these are used by the tests
+HOST1="${IP_PREFIX}.$((${IP_SUFFIX_BASE}+1))"
+HOST2="${IP_PREFIX}.$((${IP_SUFFIX_BASE}+2))"
+
 . ./assert.sh
 
-SSH=${SSH:-ssh -l vagrant -i ./insecure_private_key}
+SSH=${SSH:-ssh -l vagrant -i ./insecure_private_key -o UserKnownHostsFile=./.ssh_known_hosts -o CheckHostIP=no -o StrictHostKeyChecking=no}
-HOST1=${HOST1:-192.168.48.19}
-HOST2=${HOST2:-192.168.48.15}
 
 remote() {
     rem=$1

test/setup.sh

@@ -8,9 +8,10 @@ curl -sS https://raw.githubusercontent.com/lehmannro/assert.sh/master/assert.sh
 . ./config.sh
 
 echo Copying weave images and script to hosts
-for HOST in $HOST1 $HOST2; do
+for HOST in $HOSTS; do
     docker_on $HOST load -i /var/tmp/weave.tar
     docker_on $HOST load -i /var/tmp/weavedns.tar
+    docker_on $HOST load -i /var/tmp/weavetools.tar
     run_on $HOST mkdir -p `dirname $WEAVE`
     cat ../weave         | run_on $HOST sh -c "cat > $WEAVE"
     cat ../bin/docker-ns | run_on $HOST sh -c "cat > $DOCKER_NS"

testing/util.go

@@ -0,0 +1,96 @@
+package testing
+
+import (
+	"reflect"
+	"runtime"
+	"testing"
+	"time"
+)
+
+func AssertNoErr(t *testing.T, err error) {
+	if err != nil {
+		Fatalf(t, "Unexpected error: %s", err)
+	}
+}
+
+func AssertEqualuint64(t *testing.T, got, wanted uint64, desc string) {
+	if got != wanted {
+		Fatalf(t, "Expected %s %d but got %d", desc, wanted, got)
+	}
+}
+
+func AssertEqualInt(t *testing.T, got, wanted int, desc string) {
+	if got != wanted {
+		Fatalf(t, "Expected %s %d but got %d", desc, wanted, got)
+	}
+}
+
+func AssertEqualString(t *testing.T, got, wanted string, desc string) {
+	if got != wanted {
+		Fatalf(t, "Expected %s '%s' but got '%s'", desc, wanted, got)
+	}
+}
+
+func AssertStatus(t *testing.T, got int, wanted int, desc string) {
+	if got != wanted {
+		Fatalf(t, "Expected %s %d but got %d", desc, wanted, got)
+	}
+}
+
+func AssertErrorInterface(t *testing.T, got interface{}, wanted interface{}, desc string) {
+	gotT, wantedT := reflect.TypeOf(got), reflect.TypeOf(wanted).Elem()
+	if !gotT.Implements(wantedT) {
+		Fatalf(t, "Expected %s but got %s (%s)", wantedT.String(), gotT.String(), desc)
+	}
+}
+
+func AssertErrorType(t *testing.T, got interface{}, wanted interface{}, desc string) {
+	gotT, wantedT := reflect.TypeOf(got), reflect.TypeOf(wanted).Elem()
+	if gotT != wantedT {
+		Fatalf(t, "Expected %s but got %s (%s)", wantedT.String(), gotT.String(), desc)
+	}
+}
+
+func AssertType(t *testing.T, got interface{}, wanted interface{}, desc string) {
+	gotT, wantedT := reflect.TypeOf(got), reflect.TypeOf(wanted)
+	if gotT != wantedT {
+		Fatalf(t, "Expected %s but got %s (%s)", wantedT.String(), gotT.String(), desc)
+	}
+}
+
+func AssertEmpty(t *testing.T, array interface{}, desc string) {
+	if reflect.ValueOf(array).Len() != 0 {
+		Fatalf(t, "Expected empty %s but got %s", desc, array)
+	}
+}
+
+// Like testing.Fatalf, but adds the stack trace of the current call
+func Fatalf(t *testing.T, format string, args ...interface{}) {
+	t.Fatalf(format+"\n%s", append(args, StackTrace())...)
+}
+
+func StackTrace() string {
+	return stackTrace(false)
+}
+
+func stackTrace(all bool) string {
+	buf := make([]byte, 1<<20)
+	stacklen := runtime.Stack(buf, all)
+	return string(buf[:stacklen])
+}
+
+// Borrowed from net/http tests:
+// goTimeout runs f, failing t if f takes more than d to complete.
+func RunWithTimeout(t *testing.T, d time.Duration, f func()) {
+	ch := make(chan bool, 2)
+	timer := time.AfterFunc(d, func() {
+		t.Errorf("Timeout expired after %v: stacks:\n%s", d, stackTrace(true))
+		ch <- true
+	})
+	defer timer.Stop()
+	go func() {
+		defer func() { ch <- true }()
+		f()
+	}()
+	<-ch
+}

tools/Dockerfile

@@ -0,0 +1,3 @@
+FROM scratch
+MAINTAINER Weaveworks Inc <help@weave.works>
+ADD ./bin /bin

tools/build.sh

@@ -0,0 +1,60 @@
+#!/bin/sh
+set -e
+
+apt-get -y update
+apt-get -y install curl make pkg-config gcc bison flex
+
+BASEDIR=$(dirname $0)
+mkdir -p "$BASEDIR"/bin
+
+copy_exe() {
+    if ldd $1 >/dev/null 2>&1 ; then
+        echo "ERROR: $1 was not linked statically." 2>&1
+        exit 1
+    fi
+    cp $1 "$BASEDIR"/bin/
+}
+
+# ethtool
+
+ETHTOOL=ethtool-3.16
+
+rm -rf $ETHTOOL
+
+curl -s -S https://www.kernel.org/pub/software/network/ethtool/$ETHTOOL.tar.gz | tar xvz
+(cd $ETHTOOL; ./configure LDFLAGS=-static && make)
+copy_exe $ETHTOOL/ethtool
+
+# conntrack
+
+PACKAGES="libmnl-1.0.3 libnfnetlink-1.0.1 libnetfilter_cttimeout-1.0.0 libnetfilter_cthelper-1.0.0 libnetfilter_queue-1.0.2 libnetfilter_conntrack-1.0.4"
+CONNTRACK=conntrack-tools-1.4.2
+
+rm -rf $PACKAGES $CONNTRACK
+
+fetch() {
+    curl -s -S http://www.netfilter.org/projects/${1%-*}/files/$1.tar.bz2 | tar xj
+}
+
+for PACKAGE in $PACKAGES; do
+    fetch $PACKAGE
+    (cd $PACKAGE; ./configure --disable-shared && make install)
+done
+
+fetch $CONNTRACK
+(cd $CONNTRACK; ./configure --disable-shared && make && rm -f src/conntrack && make LDFLAGS=-all-static)
+copy_exe $CONNTRACK/src/conntrack
+
+# curl
+
+CURL=curl-7.40.0
+
+rm -rf $CURL
+
+curl -s -S  http://curl.haxx.se/download/$CURL.tar.gz | tar xvz
+(cd $CURL; ./configure --without-ssl --disable-shared && make && rm src/curl && make LDFLAGS=-all-static)
+copy_exe $CURL/src/curl
+
+#
+
+touch "$BASEDIR"/bin

weave

@@ -5,7 +5,8 @@ SCRIPT_VERSION="(unreleased version)"
 
 usage() {
     echo "Usage:"
-    echo "weave launch     [-password <password> | -passwordfile <path>] <peer> ..."
+    echo "weave setup"
+    echo "weave launch     [-password <password>] <peer> ..."
     echo "weave launch-dns <cidr>"
     echo "weave connect    <peer>"
     echo "weave run        [--with-dns] <cidr> <docker run args> ..."
@@ -33,8 +34,20 @@ usage() {
 
 [ $# -gt 0 ] || usage
 
-IMAGE=zettio/weave
+if [ "$SCRIPT_VERSION" = "(unreleased version)" ] ; then
-DNS_IMAGE=zettio/weavedns
+    IMAGE_VERSION=latest
+else
+    IMAGE_VERSION=$SCRIPT_VERSION
+fi
+
+IMAGE_VERSION=${VERSION:-$IMAGE_VERSION}
+
+BASE_IMAGE=zettio/weave
+BASE_DNS_IMAGE=zettio/weavedns
+BASE_TOOLS_IMAGE=zettio/weavetools
+IMAGE=$BASE_IMAGE:$IMAGE_VERSION
+DNS_IMAGE=$BASE_DNS_IMAGE:$IMAGE_VERSION
+TOOLS_IMAGE=$BASE_TOOLS_IMAGE:$IMAGE_VERSION
 CONTAINER_NAME=weave
 DNS_CONTAINER_NAME=weavedns
 BRIDGE=weave
@@ -44,6 +57,7 @@ PORT=6783
 HTTP_PORT=6784
 DNS_HTTP_PORT=6785
 DOCKER_BRIDGE=${DOCKER_BRIDGE:-docker0}
+PROCFS=${PROCFS:-/proc}
 
 COMMAND=$1
 
@@ -55,11 +69,11 @@ command_exists () {
     command -v $1 >/dev/null 2>&1
 }
 
-command_exists_or_warn () {
+run_tool() {
-    if ! command_exists $1 ; then
+    TOOL_NET=$1
-        echo "Cannot find $1; please install it. Continuing without it." >&2
+    TOOL_COMMAND=$2
-        return 1
+    shift 2
-    fi
+    docker run --rm --privileged --net=$TOOL_NET $TOOLS_IMAGE /bin/$TOOL_COMMAND "$@"
 }
 
 is_cidr() {
@@ -118,8 +132,6 @@ create_bridge() {
         ip link add name v${CONTAINER_IFNAME}du mtu $MTU type dummy
         ip link set dev v${CONTAINER_IFNAME}du master $BRIDGE
         ip link del dev v${CONTAINER_IFNAME}du
-        # disable offloading
-        if command_exists_or_warn ethtool; then ethtool -K $BRIDGE tx off >/dev/null; fi
         # Work around the situation where there are no rules allowing traffic
         # across our bridge. E.g. ufw
         add_iptables_rule filter FORWARD -i $BRIDGE -o $BRIDGE -j ACCEPT
@@ -127,6 +139,14 @@ create_bridge() {
         run_iptables -t nat -N WEAVE >/dev/null 2>&1 || true
         add_iptables_rule nat POSTROUTING -j WEAVE
     }
+    if [ ! "$1" = "--without-ethtool" ] ; then
+        # disable offloading - we do this even when $BRIDGE already exists
+        # simply because it has the desirable side-effect of ensuring we
+        # have the $TOOLS_IMAGE and thus do not incur downloading-induced
+        # delays in more time-sensitive sections elsewhere.
+        run_tool host ethtool -K $BRIDGE tx off >/dev/null
+    fi
+
     ip link set dev $BRIDGE up
 }
 
@@ -153,7 +173,7 @@ docker_bridge_ip() {
 # If an error is caused by container dying, swallow output from error.
 with_container_netns () {
     CONTAINER="$1"
-    CONTAINER_PID=$(docker inspect --format='{{ .State.Pid }}' $CONTAINER)
+    CONTAINER_PID=$(docker inspect --format='{{.State.Pid}}' $CONTAINER)
 
     if [ "$CONTAINER_PID" = 0 ] ; then
         echo "Container $CONTAINER not running." >&2
@@ -168,7 +188,7 @@ with_container_netns () {
     NETNS=$CONTAINER_PID
     [ ! -d /var/run/netns ] && mkdir -p /var/run/netns
     rm -f /var/run/netns/$NETNS
-    ln -s /proc/$CONTAINER_PID/ns/net /var/run/netns/$NETNS
+    ln -s $PROCFS/$CONTAINER_PID/ns/net /var/run/netns/$NETNS
 
     LOCAL_IFNAME="v${CONTAINER_IFNAME}pl${CONTAINER_PID}"
     GUEST_IFNAME="v${CONTAINER_IFNAME}pg${CONTAINER_PID}"
@@ -181,7 +201,7 @@ with_container_netns () {
     shift 1
     if ! "$@" >$IP_TMPOUT 2>$IP_TMPERR ; then
         STATUS=1
-        if [ "$(docker inspect --format='{{ .State.Pid }}' $CONTAINER)" != "$CONTAINER_PID" ] ; then
+        if [ "$(docker inspect --format='{{.State.Pid}}' $CONTAINER)" != "$CONTAINER_PID" ] ; then
             echo "Container $CONTAINER died" >&2
         else
             echo "Failure during network configuration for container $CONTAINER:" >&2
@@ -196,7 +216,7 @@ with_container_netns () {
 }
 
 connect_container_to_bridge() {
-    if [ "$(readlink /proc/$CONTAINER_PID/ns/net)" = "$(readlink /proc/$$/ns/net)" ] ; then
+    if [ "$(readlink $PROCFS/$CONTAINER_PID/ns/net)" = "$(readlink $PROCFS/$$/ns/net)" ] ; then
         echo "Container is running in the host network namespace, and therefore cannot be" >&2
         echo "connected to weave. Perhaps the container was started with --net=host." >&2
         return 1
@@ -205,7 +225,7 @@ connect_container_to_bridge() {
         return 1
     fi
 
-    if { if command_exists_or_warn ethtool; then ethtool -K $GUEST_IFNAME tx off >/dev/null; fi } &&
+    if run_tool host ethtool -K $GUEST_IFNAME tx off >/dev/null &&
         ip link set $LOCAL_IFNAME master $BRIDGE &&
         ip link set $LOCAL_IFNAME up &&
         ip link set $GUEST_IFNAME netns $NETNS ; then
@@ -224,7 +244,7 @@ launch() {
         return 1
     fi
     connect_container_to_bridge &&
-        { if command_exists_or_warn ethtool ; then ip netns exec $NETNS ethtool -K eth0 tx off >/dev/null; fi } &&
+        run_tool container:$CONTAINER ethtool -K eth0 tx off >/dev/null &&
         ip netns exec $NETNS ip link set $CONTAINER_IFNAME up
 }
 
@@ -277,7 +297,7 @@ detach() {
 
 # Call url $4 with http verb $3 on container $1 at port $2
 http_call() {
-    if ! status=$(docker inspect --format='{{ .State.Running }} {{ .NetworkSettings.IPAddress }}' $1 2>/dev/null); then
+    if ! status=$(docker inspect --format='{{.State.Running}} {{.NetworkSettings.IPAddress}}' $1 2>/dev/null); then
         echo "$1 container is not present; have you launched it?" >&2
         return 1
     fi
@@ -304,32 +324,27 @@ http_call_ip() {
     http_verb="$3"
     url="$4"
     shift 4
-    if command_exists curl ; then
+    run_tool host curl -s -X $http_verb "$@" http://$ip:$port$url
-        curl -s -X $http_verb "$@" http://$ip:$port$url
-    else
-        echo "Cannot find curl, which is required for this operation." >&2
-        exit 1
-    fi
 }
 
 # Perform operation $1 on container ID $2 to local DNS database at address $3
 # This function is only called where we know $2 is a valid container name
 tell_dns() {
-    if ! status=$(docker inspect --format='{{ .State.Running }}' $DNS_CONTAINER_NAME 2>/dev/null) || [ "$status" != "true" ] ; then
+    if ! status=$(docker inspect --format='{{.State.Running}}' $DNS_CONTAINER_NAME 2>/dev/null) || [ "$status" != "true" ] ; then
         # weavedns not running - silently return
         return
     fi
     # get the long form of the container ID
-    CONTAINER=$(docker inspect --format='{{ .Id }}' $2 2>/dev/null)
+    CONTAINER=$(docker inspect --format='{{.Id}}' $2 2>/dev/null)
     # extract IP address and routing prefix from CIDR
     WEAVE_ADDR_IP=$(echo $3 | sed -e 's/\([^/]*\)\/\(.*\)/\1/')
-    MORE_ARGS=$(docker inspect --format='--data-urlencode fqdn={{ .Config.Hostname }}.{{ .Config.Domainname }}.' $CONTAINER 2>/dev/null) && true
+    MORE_ARGS=$(docker inspect --format='--data-urlencode fqdn={{.Config.Hostname}}.{{.Config.Domainname}}.' $CONTAINER 2>/dev/null) && true
     http_call $DNS_CONTAINER_NAME $DNS_HTTP_PORT $1 /name/$CONTAINER/$WEAVE_ADDR_IP $MORE_ARGS || true
 }
 
 # Tell the newly-started weaveDNS about existing weave IPs
 populate_dns() {
-    DNS_IP=$(docker inspect --format='{{ .NetworkSettings.IPAddress }}' $DNS_CONTAINER_NAME)
+    DNS_IP=$(docker inspect --format='{{.NetworkSettings.IPAddress}}' $DNS_CONTAINER_NAME)
     WAIT_TIME=1
     while ! http_call_ip $DNS_IP $DNS_HTTP_PORT GET /status >/dev/null && [ $WAIT_TIME -lt 4 ]; do
         sleep $WAIT_TIME
@@ -341,7 +356,7 @@ populate_dns() {
     fi
 
     for CONTAINER in $(docker ps -q --no-trunc); do
-        MORE_ARGS=$(docker inspect --format='--data-urlencode fqdn={{ .Config.Hostname }}.{{ .Config.Domainname }}.' $CONTAINER 2>/dev/null) && true
+        MORE_ARGS=$(docker inspect --format='--data-urlencode fqdn={{.Config.Hostname}}.{{.Config.Domainname}}.' $CONTAINER 2>/dev/null) && true
         if CONTAINER_ADDRS=$(with_container_netns $CONTAINER container_weave_addrs 2>&1) ; then
             CONTAINER_IPS=$(echo "$CONTAINER_ADDRS" | grep -o 'inet .*' | sed -e 's/inet \([^/]*\)\/\(.*\)/\1/')
             for IP in $CONTAINER_IPS; do
@@ -353,14 +368,21 @@ populate_dns() {
 
 # Check that a container named $1 with image $2 is not running
 check_not_running() {
-    case $(docker inspect --format='{{ .State.Running }} {{ .Config.Image }}' $1 2>/dev/null) in
+    case $(docker inspect --format='{{.State.Running}} {{.Config.Image}}' $1 2>/dev/null) in
         "true $2")
             echo "$1 is already running." >&2
             exit 1
             ;;
+        "true $2:"*)
+            echo "$1 is already running." >&2
+            exit 1
+            ;;
         "false $2")
             docker rm $1 >/dev/null
             ;;
+        "false $2:"*)
+            docker rm $1 >/dev/null
+            ;;
         true*)
             echo "Found another running container named '$1'. Aborting." >&2
             exit 1
@@ -376,7 +398,7 @@ container_weave_addrs() {
     ip netns exec $NETNS ip addr show dev $CONTAINER_IFNAME
 }
 
-uname -s -r | sed -n 's|^\([^ ]*\) \([0-9]\+\)\.\([0-9]\+\).*|\1 \2 \3|p' | {
+uname -s -r | sed -n -e 's|^\([^ ]*\) \([0-9][0-9]*\)\.\([0-9][0-9]*\).*|\1 \2 \3|p' | {
     if ! read sys maj min ; then
       echo "ERROR: Unable to parse operating system version $(uname -s -r)" >&2
       exit 1
@@ -393,8 +415,8 @@ uname -s -r | sed -n 's|^\([^ ]*\) \([0-9]\+\)\.\([0-9]\+\).*|\1 \2 \3|p' | {
 }
 
 ask_version() {
-    if ! DOCKERIMAGE=$(docker inspect --format='{{ .Image }}' $1 2>/dev/null) ; then
+    if ! DOCKERIMAGE=$(docker inspect --format='{{.Image}}' $1 2>/dev/null) ; then
-            if ! DOCKERIMAGE=$(docker inspect --format='{{ .Id }}' $2 2>/dev/null) ; then
+            if ! DOCKERIMAGE=$(docker inspect --format='{{.Id}}' $2 2>/dev/null) ; then
                 echo "Unable to find $2 image." >&2
             fi
     fi
@@ -412,7 +434,7 @@ if ! ip netns list >/dev/null 2>&1 ; then
     exit 1
 fi
 
-if ! DOCKER_VERSION=$(docker -v | sed -n 's|^Docker version \([0-9]\+\.[0-9]\+.[0-9]\+\).*|\1|p') || [ -z "$DOCKER_VERSION" ] ; then
+if ! DOCKER_VERSION=$(docker -v | sed -n -e 's|^Docker version \([0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*\).*|\1|p') || [ -z "$DOCKER_VERSION" ] ; then
     echo "ERROR: Unable to parse docker version" >&2
     exit 1
 fi
@@ -428,8 +450,16 @@ DOCKER_VERSION_MINOR=$(echo "$DOCKER_VERSION" | cut -d. -f 2)
 DOCKER_VERSION_PATCH=$(echo "$DOCKER_VERSION" | cut -d. -f 3)
 
 case "$COMMAND" in
+    setup)
+        for img in $IMAGE $DNS_IMAGE $TOOLS_IMAGE ; do
+            docker pull $img
+        done
+        ;;
+    create-bridge)
+        create_bridge --without-ethtool
+        ;;
     launch)
-        check_not_running $CONTAINER_NAME $IMAGE
+        check_not_running $CONTAINER_NAME $BASE_IMAGE
         create_bridge
         # We set the router name to the bridge mac since that is
         # stable across re-creations of the containers.
@@ -445,19 +475,15 @@ case "$COMMAND" in
         fi
         if [ "$1" = "-password" ] ; then
             [ $# -gt 1 ] || usage
-            PASSWORD=$2
+            WEAVE_PASSWORD="$2"
-            shift 2
+            export WEAVE_PASSWORD
-        elif [ "$1" = "-passwordfile" ] ; then
-            [ $# -gt 1 ] || usage
-            PASSWORDFILE=$2
             shift 2
         fi
         # Set WEAVE_DOCKER_ARGS in the environment in order to supply
         # additional parameters, such as resource limits, to docker
         # when launching the weave container.
         CONTAINER=$(docker run --privileged -d --name=$CONTAINER_NAME \
-            -p $PORT:$PORT/tcp -p $PORT:$PORT/udp \
+            -p $PORT:$PORT/tcp -p $PORT:$PORT/udp -e WEAVE_PASSWORD \
-            ${PASSWORD:+-e WEAVE_PASSWORD="$PASSWORD"} ${PASSWORDFILE:+--env-file="$PASSWORDFILE"} \
             $WEAVE_DOCKER_ARGS $IMAGE -name $MACADDR -iface $CONTAINER_IFNAME "$@")
         with_container_netns $CONTAINER launch >/dev/null
         echo $CONTAINER
@@ -466,7 +492,7 @@ case "$COMMAND" in
         [ $# -gt 0 ] || usage
         CIDR=$1
         shift 1
-        check_not_running $DNS_CONTAINER_NAME $DNS_IMAGE
+        check_not_running $DNS_CONTAINER_NAME $BASE_DNS_IMAGE
         create_bridge
         docker_bridge_ip
         DNS_CONTAINER=$(docker run --privileged -d --name=$DNS_CONTAINER_NAME \
@@ -498,6 +524,16 @@ case "$COMMAND" in
         echo weave script $SCRIPT_VERSION
         ask_version $CONTAINER_NAME $IMAGE
         ask_version $DNS_CONTAINER_NAME $DNS_IMAGE
+        if ! TOOLS_IMAGE_ID=$(docker inspect --format='{{.Id}}' $TOOLS_IMAGE 2>/dev/null) ; then
+            echo "Unable to find $TOOLS_IMAGE image." >&2
+        else
+            TOOLS_VERSION=$(docker images --no-trunc | grep $TOOLS_IMAGE_ID | grep -v latest | tr -s ' ' | cut -d ' ' -f 2 | tr "\n" ' ')
+            if [ -n "$TOOLS_VERSION" ] ; then
+                echo "weave tools $TOOLS_VERSION"
+            else
+                echo "weave tools (unreleased version)"
+            fi
+        fi
         ;;
     run)
         [ $# -gt 0 ] || usage
@@ -556,7 +592,7 @@ case "$COMMAND" in
     expose)
         [ $# -eq 1 ] || usage
         validate_cidr $1
-        create_bridge
+        create_bridge --without-ethtool
         if ! ip addr show dev $BRIDGE | grep -qF $1
         then
             ip addr add dev $BRIDGE $1
@@ -567,7 +603,7 @@ case "$COMMAND" in
     hide)
         [ $# -eq 1 ] || usage
         validate_cidr $1
-        create_bridge
+        create_bridge --without-ethtool
         if ip addr show dev $BRIDGE | grep -qF $1
         then
             ip addr del dev $BRIDGE $1
@@ -581,9 +617,7 @@ case "$COMMAND" in
             echo "Weave is not running." >&2
         fi
         docker rm -f $CONTAINER_NAME >/dev/null 2>&1 || true
-        if command_exists_or_warn conntrack; then
+        run_tool host conntrack -D -p udp --dport $PORT >/dev/null 2>&1 || true
-            conntrack -D -p udp --dport $PORT >/dev/null 2>&1 || true
-        fi
         ;;
     stop-dns)
         [ $# -eq 0 ] || usage
@@ -598,9 +632,7 @@ case "$COMMAND" in
         docker kill  $DNS_CONTAINER_NAME >/dev/null 2>&1 || true
         docker rm -f $CONTAINER_NAME     >/dev/null 2>&1 || true
         docker rm -f $DNS_CONTAINER_NAME >/dev/null 2>&1 || true
-        if command_exists_or_warn conntrack; then
+        run_tool host conntrack -D -p udp --dport $PORT >/dev/null 2>&1 || true
-            conntrack -D -p udp --dport $PORT >/dev/null 2>&1 || true
-        fi
         destroy_bridge
         for LOCAL_IFNAME in $(ip link show | grep v${CONTAINER_IFNAME}pl | cut -d ' ' -f 2 | tr -d ':') ; do
             ip link del $LOCAL_IFNAME

weavedns/Dockerfile

@@ -3,4 +3,4 @@ MAINTAINER Weaveworks Inc <help@weave.works>
 WORKDIR /home/weave
 ADD ./weavedns /home/weave/
 EXPOSE 53/udp
-ENTRYPOINT ["/home/weave/weavedns", "-wait", "5"]
+ENTRYPOINT ["/home/weave/weavedns", "-wait", "20"]

weavedns/README.md

@@ -1,17 +1,20 @@
 # Weave DNS server
 
-The Weave DNS server answers name queries in a Weave network. It is
+The Weave DNS server answers name queries in a Weave network. This
-run per-host, to be supplied as the nameserver for containers on that
+provides a simple way for containers to find each other: just give
-host. It is then told about hostnames for the local containers. For
+them hostnames and tell other containers to connect to those names.
-other names it will ask the other weave hosts, or fall back to using
+Unlike Docker 'links', this requires no code changes and works across
-the host's configured name server.
+hosts.
 
 ## Using weaveDNS
 
-The weave script command `launch-dns` starts the DNS container, and
+WeaveDNS is deployed as a set of containers that communicate with each
-then you use the `--with-dns` option on containers you wish to use it.
+other over the weave network. One such container needs to be started
-Subsquently, giving any container a hostname in the domain
+on every weave host, by invoking the weave script command
-`.weave.local` will register it in DNS. For example:
+`launch-dns`. Application containers are then instructed to use
+WeaveDNS as their nameserver by supplying the `--with-dns` option when
+starting them. Giving any container a hostname in the `.weave.local`
+domain registers it in weaveDNS.  For example:
 
 ```bash
 $ weave launch
@@ -24,25 +27,43 @@ $ docker attach $shell1
 ...
 ```
 
-The weave IP address supplied to `weave launch-dns` must not be used
+Each weaveDNS container started with `launch-dns` needs to be given
-by any other container, and the supplied network must be the same for
+its own, unique, IP address, in a subnet that is a) common to all
-all DNS containers, and be disjoint from all application networks.
+weaveDNS containers, b) disjoint from the application subnets, and c)
+not in use on any of the hosts. In our example the weaveDNS address is
+in subnet 10.1.254.0/24 and the application containers are in
+subnet 10.1.1.0/24.
 
-The DNS container can be stopped with `stop-dns`.
+WeaveDNS containers can be stopped with `stop-dns`.
 
+## How it works
+
+The weaveDNS container running on every host acts as the nameserver
+for containers on that host. It is told about hostnames for local
+containers by the `weave run` command. If a hostname is in the
+`.weave.local` domain then weaveDNS records the association of that
+name with the container's weave IP address(es).
+
+When weaveDNS is queried for a name in the `.weave.local` domain, it
+first checks its own records. If the name is not found there, it asks
+the weaveDNS servers on the other hosts in the weave network.
+
+When weaveDNS is queried for a name in a domain other than
+`.weave.local`, it queries the host's configured nameserver,
+which is the standard behaviour for Docker containers.
 
 ## Domain search paths
 
 If you don't supply a domain search path (with `--dns-search=`),
-`weave run ...` will tell a container to look for "bare" hostnames,
+`weave run ...` tells a container to look for "bare" hostnames, like
-like `pingme`, in its own domain. That's why you can just say `ping
+`pingme`, in its own domain. That's why you can just invoke `ping
 pingme` above -- since the hostname is `ubuntu.weave.local`, it will
 look for `pingme.weave.local`.
 
 If you want to supply other entries for the domain search path,
 e.g. if you want containers in different sub-domains to resolve
-hostnames across all sub-domains plus some external domains, you will
+hostnames across all sub-domains plus some external domains, you need
-need *also* to supply the `weave.local` domain to retain the above
+*also* to supply the `weave.local` domain to retain the above
 behaviour.
 
 ```bash
@@ -54,10 +75,10 @@ weave run --with-dns 10.1.1.4/24 -ti \
 
 ## Doing things more manually
 
-If you use the `--with-dns` option, `weave run` will automatically
+If you use the `--with-dns` option, `weave run` automatically supplies
-supply the DNS server address to the new container. And both
+the DNS server address to the new container. And both `weave run` and
-`weave run` and `weave attach` will register the hostname of the given
+`weave attach` register the hostname of the given container against
-container against the given weave network IP address.
+the given weave network IP address.
 
 In some circumstances, you may not want to use the `weave`
 command. You can still take advantage of a running weaveDNS, with some
@@ -69,7 +90,7 @@ So that containers can connect to a stable and always routable IP
 address, weaveDNS publishes its port 53 to the Docker bridge device,
 which is assumed to be `docker0`.
 
-Some configurations will use a different Docker bridge device. To
+Some configurations may use a different Docker bridge device. To
 supply a different bridge device, use the environment variable
 `DOCKER_BRIDGE`, e.g.,
 
@@ -102,10 +123,10 @@ $ docker_ip=$(ip -4 addr show dev docker0 | grep -o 'inet [0-9.]*' | cut -d ' '
 
 ### Supplying the domain search path
 
-By default, Docker will provide containers with a `/etc/resolv.conf`
+By default, Docker provides containers with a `/etc/resolv.conf` that
-that matches that for the host. In some circumstances, this may
+matches that for the host. In some circumstances, this may include a
-include a DNS search path, which will break the nice "bare names
+DNS search path, which will break the nice "bare names resolve"
-resolve" property above.
+property above.
 
 Therefore, when starting containers with `docker run` instead of
 `weave run`, you will usually want to supply a domain search path so
@@ -121,13 +142,15 @@ its hostname, you can register it using the HTTP API:
 
 ```bash
 $ docker start $shell2
-$ shell2_ip=$(docker inspect --format='{{ .NetworkSettings.IPAddress }}' $shell2)
+$ curl -X PUT "http://$dns_ip:6785/name/$shell2/10.1.1.27" -d fqdn=shell2.weave.local
-$ curl -X PUT "http://$dns_ip:6785/name/$shell2/10.1.1.27" -d local_ip=$shell2_ip -d fqdn=shell2.weave.local
 ```
 
 ### Registering multiple containers with the same name
 
-This is supported; in the initial implementation weaveDNS will pick one address to return when you ask for the name.  Since weave-dns will remove any container that dies, this is a simple way to implement redundancy.  In the current implementation it does not attempt to do load-balancing.
+This is supported; weaveDNS picks one address to return when you ask
+for the name. Since weaveDNS removes any container that dies, this is
+a simple way to implement redundancy.  In the current implementation
+it does not attempt to do load-balancing.
 
 ### Replacing one container with another at the same name
 
@@ -135,9 +158,9 @@ If you would like to deploy a new version of a service, keep the old one running
 
 ### Not watching docker events
 
-By default, the server will watch docker events and remove entries for
+By default, weaveDNS watchs docker events and removes entries for any
-any containers that die. You can tell it not to, by adding
+containers that die. You can tell it not to, by adding `--watch=false`
-`--watch=false` to the container args:
+to the container args:
 
 ```bash
 $ weave launch-dns 10.1.254.1/24 --watch=false

weavedns/main.go

@@ -3,10 +3,10 @@ package main
 import (
 	"flag"
 	"fmt"
+	. "github.com/zettio/weave/common"
 	weavedns "github.com/zettio/weave/nameserver"
 	weavenet "github.com/zettio/weave/net"
 	"io"
-	"io/ioutil"
 	"net"
 	"os"
 )
@@ -40,35 +40,32 @@ func main() {
 		os.Exit(0)
 	}
 
-	debugOut := ioutil.Discard
+	InitDefaultLogging(debug)
-	if debug {
-		debugOut = os.Stderr
-	}
-	weavedns.InitLogging(debugOut, os.Stdout, os.Stdout, os.Stderr)
 
 	var zone = new(weavedns.ZoneDb)
 
 	if watch {
 		err := weavedns.StartUpdater(apiPath, zone)
 		if err != nil {
-			weavedns.Error.Fatal("Unable to start watcher", err)
+			Error.Fatal("Unable to start watcher", err)
 		}
 	}
 
 	var iface *net.Interface = nil
 	if ifaceName != "" {
 		var err error
-		weavedns.Info.Println("Waiting for interface", ifaceName, "to come up")
+		Info.Println("Waiting for interface", ifaceName, "to come up")
 		iface, err = weavenet.EnsureInterface(ifaceName, wait)
 		if err != nil {
-			weavedns.Error.Fatal(err)
+			Error.Fatal(err)
 		} else {
-			weavedns.Info.Println("Interface", ifaceName, "is up")
+			Info.Println("Interface", ifaceName, "is up")
 		}
 	}
 
-	err := weavedns.StartServer(zone, iface, dnsPort, httpPort, wait)
+	go weavedns.ListenHttp(weavedns.LOCAL_DOMAIN, zone, httpPort)
+	err := weavedns.StartServer(zone, iface, dnsPort, wait)
 	if err != nil {
-		weavedns.Error.Fatal("Failed to start server", err)
+		Error.Fatal("Failed to start server", err)
 	}
 }

weaver/.gitignore

@@ -1 +0,0 @@
-weaver

weaver/Dockerfile

@@ -2,4 +2,4 @@ FROM scratch
 MAINTAINER Weaveworks Inc <help@weave.works>
 WORKDIR /home/weave
 ADD ./weaver /home/weave/
-ENTRYPOINT ["/home/weave/weaver", "-wait", "5"]
+ENTRYPOINT ["/home/weave/weaver", "-wait", "20"]

weaver/main.go

@@ -122,6 +122,7 @@ func main() {
 	}
 
 	router := weave.NewRouter(iface, ourName, []byte(password), connLimit, bufSz*1024*1024, logFrame)
+	log.Println("Our name is", router.Ourself.Name)
 	router.Start()
 	for _, peer := range peers {
 		if addr, err := net.ResolveTCPAddr("tcp4", weave.NormalisePeerAddr(peer)); err == nil {
@@ -135,8 +136,13 @@ func main() {
 }
 
 func handleHttp(router *weave.Router) {
+	encryption := "off"
+	if router.Password != nil && len(*router.Password) > 0 {
+		encryption = "on"
+	}
 	http.HandleFunc("/status", func(w http.ResponseWriter, r *http.Request) {
 		io.WriteString(w, fmt.Sprintln("weave router", version))
+		io.WriteString(w, fmt.Sprintln("Encryption", encryption))
 		io.WriteString(w, router.Status())
 	})
 	http.HandleFunc("/connect", func(w http.ResponseWriter, r *http.Request) {
@@ -162,8 +168,8 @@ func handleSignals(router *weave.Router) {
 		sig := <-sigs
 		switch sig {
 		case syscall.SIGQUIT:
-			runtime.Stack(buf, true)
+			stacklen := runtime.Stack(buf, true)
-			log.Printf("=== received SIGQUIT ===\n*** goroutine dump...\n%s\n*** end\n", buf)
+			log.Printf("=== received SIGQUIT ===\n*** goroutine dump...\n%s\n*** end\n", buf[:stacklen])
 		case syscall.SIGUSR1:
 			log.Printf("=== received SIGUSR1 ===\n*** status...\n%s\n*** end\n", router.Status())
 		}