add 0.9.0 release change log

LGTM.  Fixes #380

CHANGELOG.md

@@ -1,3 +1,20 @@
+## Release 0.9.0
+
+- Improve WeaveDNS to the point where it can act as the name server
+  for containers in nearly all situations.
+
+- Diagnose and report peer connectivity more comprehensively.
+
+- Adapt to changes in topology - adding & removing of weave peers,
+  disruption of connectivity - more rapidly.
+
+- Cope with delays delays in downloading/running docker
+  images/containers required for weave operation.
+
+See the
+[complete change log](https://github.com/zettio/weave/issues?q=milestone%3A0.9.0)
+for more details.
+
 ## Release 0.8.0
 
 - Align script and image version. When the `weave` script has a

Makefile

@@ -1,16 +1,19 @@
+PUBLISH=publish_weave publish_weavedns publish_weavetools
+
 .DEFAULT: all
-.PHONY: all update tests clean
+.PHONY: all update tests publish $(PUBLISH) clean
 
 # If you can use docker without being root, you can do "make SUDO="
 SUDO=sudo
 
+DOCKERHUB_USER=zettio
 WEAVE_VERSION=git-$(shell git rev-parse --short=12 HEAD)
 WEAVER_EXE=weaver/weaver
 WEAVEDNS_EXE=weavedns/weavedns
 WEAVETOOLS_EXES=tools/bin
-WEAVER_IMAGE=zettio/weave
-WEAVEDNS_IMAGE=zettio/weavedns
-WEAVETOOLS_IMAGE=zettio/weavetools
+WEAVER_IMAGE=$(DOCKERHUB_USER)/weave
+WEAVEDNS_IMAGE=$(DOCKERHUB_USER)/weavedns
+WEAVETOOLS_IMAGE=$(DOCKERHUB_USER)/weavetools
 WEAVER_EXPORT=/var/tmp/weave.tar
 WEAVEDNS_EXPORT=/var/tmp/weavedns.tar
 WEAVETOOLS_EXPORT=/var/tmp/weavetools.tar
@@ -20,7 +23,7 @@ all: $(WEAVER_EXPORT) $(WEAVEDNS_EXPORT) $(WEAVETOOLS_EXPORT)
 update:
 	go get -u -f -v -tags -netgo ./$(dir $(WEAVER_EXE)) ./$(dir $(WEAVEDNS_EXE))
 
-$(WEAVER_EXE) $(WEAVEDNS_EXE):
+$(WEAVER_EXE) $(WEAVEDNS_EXE): common/*.go
 	go get -tags netgo ./$(@D)
 	go build -ldflags "-extldflags \"-static\" -X main.version $(WEAVE_VERSION)" -tags netgo -o $@ ./$(shell dirname $@)
 	@strings $@ | grep cgo_stub\\\.go >/dev/null || { \
@@ -52,8 +55,16 @@ $(WEAVETOOLS_EXPORT): tools/Dockerfile $(WEAVETOOLS_EXES)
 
 # Add more directories in here as more tests are created
 tests:
+	cd router; go test -tags netgo
 	cd nameserver; go test -tags netgo
 
+$(PUBLISH): publish_%:
+	$(SUDO) docker tag  $(DOCKERHUB_USER)/$* $(DOCKERHUB_USER)/$*:$(WEAVE_VERSION)
+	$(SUDO) docker push $(DOCKERHUB_USER)/$*:$(WEAVE_VERSION)
+	$(SUDO) docker push $(DOCKERHUB_USER)/$*:latest
+
+publish: $(PUBLISH)
+
 clean:
 	-$(SUDO) docker rmi $(WEAVER_IMAGE) $(WEAVEDNS_IMAGE) $(WEAVETOOLS_IMAGE)
 	rm -f $(WEAVER_EXE) $(WEAVEDNS_EXE) $(WEAVER_EXPORT) $(WEAVEDNS_EXPORT) $(WEAVETOOLS_EXPORT)

README.md

@@ -128,7 +128,7 @@ to each other.
  * [Troubleshooting](http://zettio.github.io/weave/troubleshooting.html)
  * [Building](http://zettio.github.io/weave/building.html)
  * [How it works](http://zettio.github.io/weave/how-it-works.html)
- * [Preview of weaveDNS](http://weaveblog.com/2014/11/04/have-you-met-weavedns/)
+ * [WeaveDNS README](https://github.com/zettio/weave/tree/master/weavedns#readme)
 
 ## Contact Us
 
@@ -138,7 +138,7 @@ help@weave.works. When reporting a bug, please include which version of
 weave you are running, as shown by `weave version`.
 
 Follow weave on Twitter:
-[@weavenetwork](https://twitter.com/weavenetwork).
+[@weaveworks](https://twitter.com/weaveworks).
 
 Read the Weave blog:
 [Weaveblog](http://weaveblog.com/).

bin/release

@@ -2,11 +2,13 @@
 
 set -e
 
+SUDO=${SUDO-sudo}
+GITHUB_USER=${GITHUB_USER:-zettio}
+DOCKERHUB_USER=${DOCKERHUB_USER:-zettio}
+RELEASE_NAME=${RELEASE_NAME:-"Weave"}
+RELEASE_DESCRIPTION=${RELEASE_DESCRIPTION:-"Weaving Containers into Applications"}
+
 PWD=`pwd`
-GITHUB_USER=zettio
-DOCKERHUB_USER=zettio
-RELEASE_NAME="Weave"
-RELEASE_DESCRIPTION="Weaving Containers into Applications"
 
 setup() {
     ## Get the new version number from the most recent tag
@@ -49,10 +51,7 @@ build() {
     ## Inject the version numbers and build the distributables
     ## (library versions?)
     sed -i "/SCRIPT_VERSION=/ c\SCRIPT_VERSION=\"$VERSION\"" ./weave
-    make \
-        WEAVE_VERSION=$VERSION \
-        WEAVER_IMAGE=$DOCKERHUB_USER/weave \
-        WEAVEDNS_IMAGE=$DOCKERHUB_USER/weavedns
+    make SUDO=$SUDO WEAVE_VERSION=$VERSION DOCKERHUB_USER=$DOCKERHUB_USER
 
     if make tests; then
         echo -e '\u2713 Tests pass'
@@ -121,13 +120,8 @@ publish() {
 
     echo '** Sanity checks OK for publishing tag' $LATEST_TAG as $DOCKERHUB_USER/weave:$VERSION
 
-    for IMAGE in weave weavedns weavetools ; do
-        echo == Tagging $DOCKERHUB_USER/$IMAGE as $VERSION
-        docker tag  $DOCKERHUB_USER/$IMAGE $DOCKERHUB_USER/$IMAGE:$VERSION
-        docker push $DOCKERHUB_USER/$IMAGE:$VERSION
-        docker push $DOCKERHUB_USER/$IMAGE:latest
-        echo "** Pushed $DOCKERHUB_USER/$IMAGE:$VERSION to docker hub"
-    done
+    echo "== Tagging and publishing images on docker hub as user $DOCKERHUB_USER"
+    make SUDO=$SUDO WEAVE_VERSION=$VERSION DOCKERHUB_USER=$DOCKERHUB_USER publish
 
     echo "== Creating GitHub release $RELEASE_NAME $VERSION"
     # This cannot be done as a draft because of a bug in

common/testing_utils.go

@@ -1,51 +0,0 @@
-package common
-
-import (
-	"fmt"
-	"reflect"
-	"runtime"
-	"strings"
-	"testing"
-)
-
-func CallSite() string {
-	_, file, line, ok := runtime.Caller(2)
-	if ok {
-		// Truncate file name at last file name separator.
-		if index := strings.LastIndex(file, "/"); index >= 0 {
-			file = file[index+1:]
-		} else if index = strings.LastIndex(file, "\\"); index >= 0 {
-			file = file[index+1:]
-		}
-	} else {
-		file = "???"
-		line = 1
-	}
-	return fmt.Sprintf("%s:%d: ", file, line)
-}
-
-func AssertNoErr(t *testing.T, err error) {
-	if err != nil {
-		t.Fatal(CallSite(), err)
-	}
-}
-
-func AssertStatus(t *testing.T, got int, wanted int, desc string) {
-	if got != wanted {
-		t.Fatalf("%s: Expected %s %d but got %d", CallSite(), desc, wanted, got)
-	}
-}
-
-func AssertErrorInterface(t *testing.T, got interface{}, wanted interface{}, desc string) {
-	gotT, wantedT := reflect.TypeOf(got), reflect.TypeOf(wanted).Elem()
-	if !gotT.Implements(wantedT) {
-		t.Fatalf("%s: Expected %s but got %s (%s)", CallSite(), wantedT.String(), gotT.String(), desc)
-	}
-}
-
-func AssertErrorType(t *testing.T, got interface{}, wanted interface{}, desc string) {
-	gotT, wantedT := reflect.TypeOf(got), reflect.TypeOf(wanted).Elem()
-	if gotT != wantedT {
-		t.Fatalf("%s: Expected %s but got %s (%s)", CallSite(), wantedT.String(), gotT.String(), desc)
-	}
-}

docs/architecture.txt

@@ -83,13 +83,17 @@ Router:
     local peer. It will tell the local connection (communicating to
     the actor thread) about the UDP address of the remote peer. The
     local connection will then start its forwarder threads as
-    described in 6a, and start sending slow heartbeats. The local connection
-    is marked as established which means it is included in network updates
-    broadcast to our peers. We also send to the remote peer via TCP a
-    ConnectionEstablished message. The remote peer receives this (on
-    the TCP receiver process), tells the connection actor process,
-    which then replaces the fast heartbeater with a slow heartbeater
-    and similarly marks the connection as established.
+    described in 6a, and start sending fast heartbeats. We send to the
+    remote peer via TCP a ConnectionEstablished message. The remote
+    peer receives this (on the TCP receiver process), tells the
+    connection actor process, which then replaces the fast heartbeater
+    with a slow heartbeater and marks the connection as established
+    (which means it is included in network updates broadcast to our
+    peers).
+  6c. When the connection initiator receives the fast heartbeat from
+    the remote peer it sends to the remote peer via a TCP a
+    ConnectionEstablished message. This is handled by the remote peer
+    as described in 6b.
   7. Whenever a connection is established or terminated, the local
     peer's version is incremented. Whenever this happens, the peer
     generates a network update message which is broadcast to its

nameserver/http.go

@@ -24,7 +24,7 @@ func parseUrl(url string) (identifier string, ipaddr string, err error) {
 func httpErrorAndLog(level *log.Logger, w http.ResponseWriter, msg string,
 	status int, logmsg string, logargs ...interface{}) {
 	http.Error(w, msg, status)
-	level.Printf(logmsg, logargs...)
+	level.Printf("[http] "+logmsg, logargs...)
 }
 
 func ListenHttp(domain string, db Zone, port int) {
@@ -60,7 +60,7 @@ func ListenHttp(domain string, db Zone, port int) {
 			}
 
 			if dns.IsSubDomain(domain, name) {
-				Info.Printf("Adding %s -> %s", name, ipStr)
+				Info.Printf("[http] Adding %s -> %s", name, ipStr)
 				if err = db.AddRecord(ident, name, ip); err != nil {
 					if _, ok := err.(DuplicateError); !ok {
 						httpErrorAndLog(
@@ -70,7 +70,7 @@ func ListenHttp(domain string, db Zone, port int) {
 					} // oh, I already know this. whatever.
 				}
 			} else {
-				Info.Printf("Ignoring name %s, not in %s", name, domain)
+				Info.Printf("[http] Ignoring name %s, not in %s", name, domain)
 			}
 
 		case "DELETE":
@@ -85,7 +85,7 @@ func ListenHttp(domain string, db Zone, port int) {
 				reqError("Invalid IP in request", "Invalid IP in request: %s", ipStr)
 				return
 			}
-			Info.Printf("Deleting %s (%s)", ident, ipStr)
+			Info.Printf("[http] Deleting %s (%s)", ident, ipStr)
 			if err = db.DeleteRecord(ident, ip); err != nil {
 				if _, ok := err.(LookupError); !ok {
 					httpErrorAndLog(
@@ -103,6 +103,6 @@ func ListenHttp(domain string, db Zone, port int) {
 
 	address := fmt.Sprintf(":%d", port)
 	if err := http.ListenAndServe(address, nil); err != nil {
-		Error.Fatal("Unable to create http listener: ", err)
+		Error.Fatal("[http] Unable to create http listener: ", err)
 	}
 }

nameserver/http_test.go

@@ -2,7 +2,7 @@ package nameserver
 
 import (
 	"fmt"
-	wt "github.com/zettio/weave/common"
+	wt "github.com/zettio/weave/testing"
 	"math/rand"
 	"net"
 	"net/http"
@@ -43,7 +43,7 @@ func TestHttp(t *testing.T) {
 	wt.AssertStatus(t, resp.StatusCode, http.StatusOK, "http response")
 
 	// Check that the address is now there.
-	foundIP, err := zone.LookupLocal(successTestName)
+	foundIP, err := zone.LookupName(successTestName)
 	wt.AssertNoErr(t, err)
 	ip, _, _ := net.ParseCIDR(testAddr1)
 	if !foundIP.Equal(ip) {
@@ -70,7 +70,7 @@ func TestHttp(t *testing.T) {
 	wt.AssertStatus(t, resp.StatusCode, http.StatusOK, "http response")
 
 	// Check that the address is still resolvable.
-	_, err = zone.LookupLocal(successTestName)
+	_, err = zone.LookupName(successTestName)
 	wt.AssertNoErr(t, err)
 
 	// Delete the address record mentioning the other container
@@ -79,7 +79,7 @@ func TestHttp(t *testing.T) {
 	wt.AssertStatus(t, resp.StatusCode, http.StatusOK, "http response")
 
 	// Check that the address is gone
-	_, err = zone.LookupLocal(successTestName)
+	_, err = zone.LookupName(successTestName)
 	wt.AssertErrorType(t, err, (*LookupError)(nil), "fully-removed address")
 
 	// Would like to shut down the http server at the end of this test

nameserver/mdns_client.go

@@ -26,7 +26,7 @@ var (
 	}
 )
 
-type ResponseA struct {
+type Response struct {
 	Name string
 	Addr net.IP
 	Err  error
@@ -34,7 +34,7 @@ type ResponseA struct {
 
 type responseInfo struct {
 	timeout time.Time // if no answer by this time, give up
-	ch      chan<- *ResponseA
+	ch      chan<- *Response
 }
 
 // Represents one query that we have sent for one name.
@@ -60,7 +60,7 @@ type MDNSClient struct {
 type mDNSQueryInfo struct {
 	name       string
 	querytype  uint16
-	responseCh chan<- *ResponseA
+	responseCh chan<- *Response
 }
 
 func NewMDNSClient() (*MDNSClient, error) {
@@ -104,9 +104,9 @@ func LinkLocalMulticastListener(ifi *net.Interface) (net.PacketConn, error) {
 // ACTOR client API
 
 const (
-	CSendQuery       = iota
-	CShutdown        = iota
-	CMessageReceived = iota
+	CSendQuery = iota
+	CShutdown
+	CMessageReceived
 )
 
 type MDNSInteraction struct {
@@ -121,7 +121,7 @@ func (c *MDNSClient) Shutdown() {
 }
 
 // Async
-func (c *MDNSClient) SendQuery(name string, querytype uint16, responseCh chan<- *ResponseA) {
+func (c *MDNSClient) SendQuery(name string, querytype uint16, responseCh chan<- *Response) {
 	c.queryChan <- &MDNSInteraction{
 		code:    CSendQuery,
 		payload: mDNSQueryInfo{name, querytype, responseCh},
@@ -209,7 +209,7 @@ func (c *MDNSClient) handleSendQuery(q mDNSQueryInfo) {
 		m.RecursionDesired = false
 		buf, err := m.Pack()
 		if err != nil {
-			q.responseCh <- &ResponseA{Err: err}
+			q.responseCh <- &Response{Err: err}
 			close(q.responseCh)
 			return
 		}
@@ -218,7 +218,7 @@ func (c *MDNSClient) handleSendQuery(q mDNSQueryInfo) {
 			id:   m.Id,
 		}
 		if _, err = c.conn.WriteTo(buf, c.addr); err != nil {
-			q.responseCh <- &ResponseA{Err: err}
+			q.responseCh <- &Response{Err: err}
 			close(q.responseCh)
 			return
 		}
@@ -235,19 +235,29 @@ func (c *MDNSClient) handleSendQuery(q mDNSQueryInfo) {
 
 func (c *MDNSClient) handleResponse(r *dns.Msg) {
 	for _, answer := range r.Answer {
+		var name string
+		var res *Response
+
 		switch rr := answer.(type) {
 		case *dns.A:
-			name := rr.Hdr.Name
-			if query, found := c.inflight[name]; found {
-				for _, resp := range query.responseInfos {
-					resp.ch <- &ResponseA{Name: rr.Hdr.Name, Addr: rr.A}
-					close(resp.ch)
-				}
-				delete(c.inflight, name)
-			} else {
-				// We've received a response that didn't match a query
-				// Do we want to cache it?
+			name = rr.Hdr.Name
+			res = &Response{Addr: rr.A}
+		case *dns.PTR:
+			name = rr.Hdr.Name
+			res = &Response{Name: rr.Ptr}
+		default:
+			return
+		}
+
+		if query, found := c.inflight[name]; found {
+			for _, resp := range query.responseInfos {
+				resp.ch <- res
+				close(resp.ch)
 			}
+			delete(c.inflight, name)
+		} else {
+			// We've received a response that didn't match a query
+			// Do we want to cache it?
 		}
 	}
 }

nameserver/mdns_client_test.go

@@ -2,7 +2,7 @@ package nameserver
 
 import (
 	"github.com/miekg/dns"
-	wt "github.com/zettio/weave/common"
+	wt "github.com/zettio/weave/testing"
 	"log"
 	"net"
 	"testing"
@@ -13,9 +13,27 @@ var (
 	successTestName = "test1.weave."
 	failTestName    = "test2.weave."
 	testAddr        = net.ParseIP("9.8.7.6")
+	testInAddr      = "6.7.8.9.in-addr.arpa."
 )
 
 func minimalServer(w dns.ResponseWriter, req *dns.Msg) {
+
+	sendAnswer := func(m *dns.Msg) {
+		buf, err := m.Pack()
+		checkFatal(err)
+		if buf == nil {
+			log.Fatal("Nil buffer")
+		}
+		//log.Println("minimalServer sending:", buf)
+		// This is a bit of a kludge - per the RFC we should send responses from 5353, but that doesn't seem to work
+		sendconn, err := net.DialUDP("udp", nil, ipv4Addr)
+		checkFatal(err)
+
+		_, err = sendconn.Write(buf)
+		sendconn.Close()
+		checkFatal(err)
+	}
+
 	//log.Println("minimalServer received:", req)
 	if len(req.Answer) > 0 {
 		return // Only interested in questions.
@@ -23,31 +41,24 @@ func minimalServer(w dns.ResponseWriter, req *dns.Msg) {
 	if len(req.Question) != 1 {
 		return // We only handle single-question messages
 	}
-	if req.Question[0].Name != successTestName {
-		return // This is not the DNS record you are looking for
+
+	if req.Question[0].Qtype == dns.TypeA && req.Question[0].Name == successTestName {
+		m := new(dns.Msg)
+		m.SetReply(req)
+		hdr := dns.RR_Header{Name: m.Question[0].Name, Rrtype: dns.TypeA,
+			Class: dns.ClassINET, Ttl: 3600}
+		a := &dns.A{hdr, testAddr}
+		m.Answer = append(m.Answer, a)
+		sendAnswer(m)
+	} else if req.Question[0].Qtype == dns.TypePTR && req.Question[0].Name == testInAddr {
+		m := new(dns.Msg)
+		m.SetReply(req)
+		hdr := dns.RR_Header{Name: m.Question[0].Name, Rrtype: dns.TypePTR,
+			Class: dns.ClassINET, Ttl: 3600}
+		a := &dns.PTR{hdr, successTestName}
+		m.Answer = append(m.Answer, a)
+		sendAnswer(m)
 	}
-
-	m := new(dns.Msg)
-	m.SetReply(req)
-
-	hdr := dns.RR_Header{Name: m.Question[0].Name, Rrtype: dns.TypeA,
-		Class: dns.ClassINET, Ttl: 3600}
-	a := &dns.A{hdr, testAddr}
-	m.Answer = append(m.Answer, a)
-
-	buf, err := m.Pack()
-	checkFatal(err)
-	if buf == nil {
-		log.Fatal("Nil buffer")
-	}
-	//log.Println("minimalServer sending:", buf)
-	// This is a bit of a kludge - per the RFC we should send responses from 5353, but that doesn't seem to work
-	sendconn, err := net.DialUDP("udp", nil, ipv4Addr)
-	checkFatal(err)
-
-	_, err = sendconn.Write(buf)
-	sendconn.Close()
-	checkFatal(err)
 }
 
 func RunLocalMulticastServer() (*dns.Server, error) {
@@ -76,14 +87,14 @@ func setup(t *testing.T) (*MDNSClient, *dns.Server, error) {
 type testContext struct {
 	receivedAddr  net.IP
 	receivedCount int
-	channel       chan *ResponseA
+	channel       chan *Response
 }
 
 func newTestContext() *testContext {
-	return &testContext{channel: make(chan *ResponseA, 4)}
+	return &testContext{channel: make(chan *Response, 4)}
 }
 
-func (c *testContext) checkResponse(t *testing.T, channelOk bool, resp *ResponseA) {
+func (c *testContext) checkResponse(t *testing.T, channelOk bool, resp *Response) {
 	if !channelOk {
 		c.channel = nil
 		return
@@ -152,3 +163,24 @@ outerloop:
 		t.Fatal("Unexpected result for", successTestName, context1.receivedAddr, context2.receivedAddr, context1.receivedCount, context2.receivedCount)
 	}
 }
+
+func TestAsLookup(t *testing.T) {
+	mdnsClient, server, _ := setup(t)
+	defer mdnsClient.Shutdown()
+	defer server.Shutdown()
+
+	ip, err := mdnsClient.LookupName(successTestName)
+	wt.AssertNoErr(t, err)
+	if !testAddr.Equal(ip) {
+		t.Fatalf("Returned address incorrect %s", ip)
+	}
+
+	ip, err = mdnsClient.LookupName("foo.example.com.")
+	wt.AssertErrorType(t, err, (*LookupError)(nil), "unknown hostname")
+
+	name, err := mdnsClient.LookupInaddr(testInAddr)
+	wt.AssertNoErr(t, err)
+	if !(successTestName == name) {
+		t.Fatalf("Expected name %s, got %s", successTestName, name)
+	}
+}

nameserver/mdns_lookup.go

@@ -0,0 +1,40 @@
+package nameserver
+
+import (
+	"github.com/miekg/dns"
+	. "github.com/zettio/weave/common"
+	"net"
+)
+
+func mdnsLookup(client *MDNSClient, name string, qtype uint16) (*Response, error) {
+	channel := make(chan *Response)
+	client.SendQuery(name, qtype, channel)
+	for resp := range channel {
+		if err := resp.Err; err != nil {
+			Debug.Printf("[mdns] Error for query type %s name %s: %s",
+				dns.TypeToString[qtype], name, err)
+			return nil, err
+		} else {
+			Debug.Printf("[mdns] Got response name for query type %s name %s: name %s, addr %s",
+				dns.TypeToString[qtype], name, resp.Name, resp.Addr)
+			return resp, nil
+		}
+	}
+	return nil, LookupError(name)
+}
+
+func (client *MDNSClient) LookupName(name string) (net.IP, error) {
+	if r, e := mdnsLookup(client, name, dns.TypeA); r != nil {
+		return r.Addr, nil
+	} else {
+		return nil, e
+	}
+}
+
+func (client *MDNSClient) LookupInaddr(inaddr string) (string, error) {
+	if r, e := mdnsLookup(client, inaddr, dns.TypePTR); r != nil {
+		return r.Name, nil
+	} else {
+		return "", e
+	}
+}

nameserver/mdns_server.go

@@ -50,24 +50,60 @@ func (s *MDNSServer) Start(ifi *net.Interface) error {
 		return err
 	}
 
-	handleMDNS := func(w dns.ResponseWriter, r *dns.Msg) {
-		// Ignore answers to other questions
-		if len(r.Answer) == 0 && len(r.Question) > 0 {
-			q := r.Question[0]
-			if ip, err := s.zone.LookupLocal(q.Name); err == nil {
-				m := makeAddressReply(r, &q, []net.IP{ip})
-				if err = s.sendResponse(m); err != nil {
-					Warning.Printf("Error writing to %s", w)
-				}
-			} else if s.addrIsLocal(w.RemoteAddr()) {
-				// ignore this - it's our own query received via multicast
+	handleLocal := s.makeHandler(dns.TypeA,
+		func(zone Lookup, r *dns.Msg, q *dns.Question) *dns.Msg {
+			if ip, err := zone.LookupName(q.Name); err == nil {
+				return makeAddressReply(r, q, []net.IP{ip})
 			} else {
-				Debug.Printf("Failed MDNS lookup for %s", q.Name)
+				return nil
+			}
+		})
+
+	handleReverse := s.makeHandler(dns.TypePTR,
+		func(zone Lookup, r *dns.Msg, q *dns.Question) *dns.Msg {
+			if name, err := zone.LookupInaddr(q.Name); err == nil {
+				return makePTRReply(r, q, []string{name})
+			} else {
+				return nil
+			}
+		})
+
+	mux := dns.NewServeMux()
+	mux.HandleFunc(LOCAL_DOMAIN, handleLocal)
+	mux.HandleFunc(RDNS_DOMAIN, handleReverse)
+
+	go dns.ActivateAndServe(nil, conn, mux)
+	return err
+}
+
+type LookupFunc func(Lookup, *dns.Msg, *dns.Question) *dns.Msg
+
+func (s *MDNSServer) makeHandler(qtype uint16, lookup LookupFunc) dns.HandlerFunc {
+	return func(_ dns.ResponseWriter, r *dns.Msg) {
+		// Handle only questions, ignore answers. We might also ignore
+		// questions that arise locally (i.e., that come from an IP we
+		// think is local), but in the interest of avoiding
+		// complication, and easier testing, this is elided on the
+		// assumption that the client wouldn't ask if it already knew
+		// the answer, and if it does ask, it'll be happy to get an
+		// answer.
+		if len(r.Answer) == 0 && len(r.Question) > 0 {
+			q := &r.Question[0]
+			if q.Qtype == qtype {
+				if m := lookup(s.zone, r, q); m != nil {
+					Debug.Printf("[mdns msgid %d] Found local answer to mDNS query %s",
+						r.MsgHdr.Id, q.Name)
+					if err := s.sendResponse(m); err != nil {
+						Warning.Printf("[mdns msgid %d] Error writing to %s",
+							r.MsgHdr.Id, s.sendconn)
+					}
+				} else {
+					Debug.Printf("[mdns msgid %d] No local answer for mDNS query %s",
+						r.MsgHdr.Id, q.Name)
+				}
 			}
 		}
 	}
-	go dns.ActivateAndServe(nil, conn, dns.HandlerFunc(handleMDNS))
-	return err
 }
 
 func (s *MDNSServer) sendResponse(m *dns.Msg) error {

nameserver/mdns_server_test.go

@@ -2,7 +2,8 @@ package nameserver
 
 import (
 	"github.com/miekg/dns"
-	wt "github.com/zettio/weave/common"
+	"github.com/zettio/weave/common"
+	wt "github.com/zettio/weave/testing"
 	"log"
 	"net"
 	"testing"
@@ -11,7 +12,9 @@ import (
 
 var (
 	containerID = "deadbeef"
+	testName    = "test.weave.local."
 	testAddr1   = "10.0.2.1/24"
+	testInAddr1 = "1.2.0.10.in-addr.arpa."
 )
 
 func sendQuery(name string, querytype uint16) error {
@@ -31,10 +34,13 @@ func sendQuery(name string, querytype uint16) error {
 }
 
 func TestServerSimpleQuery(t *testing.T) {
+	// The ff can be handy for debugging (obvs)
+	common.InitDefaultLogging(true)
+
 	log.Println("TestServerSimpleQuery starting")
 	var zone = new(ZoneDb)
 	ip, _, _ := net.ParseCIDR(testAddr1)
-	zone.AddRecord(containerID, "test.weave.", ip)
+	zone.AddRecord(containerID, testName, ip)
 
 	mdnsServer, err := NewMDNSServer(zone)
 	wt.AssertNoErr(t, err)
@@ -42,8 +48,26 @@ func TestServerSimpleQuery(t *testing.T) {
 	wt.AssertNoErr(t, err)
 
 	var receivedAddr net.IP
+	var receivedName string
+	var recvChan chan interface{}
 	receivedCount := 0
 
+	reset := func() {
+		receivedAddr = nil
+		receivedName = ""
+		receivedCount = 0
+		recvChan = make(chan interface{})
+	}
+
+	wait := func() {
+		select {
+		case <-recvChan:
+			return
+		case <-time.After(100 * time.Millisecond):
+			return
+		}
+	}
+
 	// Implement a minimal listener for responses
 	multicast, err := LinkLocalMulticastListener(nil)
 	wt.AssertNoErr(t, err)
@@ -56,8 +80,12 @@ func TestServerSimpleQuery(t *testing.T) {
 				case *dns.A:
 					receivedAddr = rr.A
 					receivedCount++
+				case *dns.PTR:
+					receivedName = rr.Ptr
+					receivedCount++
 				}
 			}
+			recvChan <- "ok"
 		}
 	}
 
@@ -67,22 +95,32 @@ func TestServerSimpleQuery(t *testing.T) {
 
 	time.Sleep(100 * time.Millisecond) // Allow for server to get going
 
-	sendQuery("test.weave.", dns.TypeA)
-
-	time.Sleep(time.Second)
+	reset()
+	sendQuery(testName, dns.TypeA)
+	wait()
 
 	if receivedCount != 1 {
-		t.Fatal("Unexpected result count for test.weave", receivedCount)
+		t.Fatalf("Unexpected result count %d for %s", receivedCount, testName)
 	}
 	if !receivedAddr.Equal(ip) {
-		t.Fatal("Unexpected result for test.weave", receivedAddr)
+		t.Fatalf("Unexpected result %s for %s", receivedAddr, testName)
 	}
 
-	receivedCount = 0
-
+	reset()
 	sendQuery("testfail.weave.", dns.TypeA)
+	wait()
 
 	if receivedCount != 0 {
-		t.Fatal("Unexpected result count for testfail.weave", receivedCount)
+		t.Fatalf("Unexpected result count %d for testfail.weave", receivedCount)
+	}
+
+	reset()
+	sendQuery(testInAddr1, dns.TypePTR)
+	wait()
+
+	if receivedCount != 1 {
+		t.Fatalf("Expected an answer to %s, got %d answers", testInAddr1, receivedCount)
+	} else if !(testName == receivedName) {
+		t.Fatalf("Expected answer %s to query for %s, got %s", testName, testInAddr1, receivedName)
 	}
 }

nameserver/server.go

@@ -9,12 +9,8 @@ import (
 
 const (
 	LOCAL_DOMAIN = "weave.local."
-	RDNS_DOMAIN  = "in-addr.arpa."
 )
 
-// +1 to also exclude a dot
-var rdnsDomainLen = len(RDNS_DOMAIN) + 1
-
 func checkFatal(e error) {
 	if e != nil {
 		Error.Fatal(e)
@@ -35,93 +31,80 @@ func makeDNSFailResponse(r *dns.Msg) *dns.Msg {
 	return m
 }
 
-func queryHandler(zone Zone, mdnsClient *MDNSClient) dns.HandlerFunc {
+func queryHandler(lookups []Lookup) dns.HandlerFunc {
 	return func(w dns.ResponseWriter, r *dns.Msg) {
 		q := r.Question[0]
-		Debug.Printf("Local query: %+v", q)
+		Debug.Printf("Query: %+v", q)
 		if q.Qtype == dns.TypeA {
-			if ip, err := zone.LookupLocal(q.Name); err == nil {
-				m := makeAddressReply(r, &q, []net.IP{ip})
-				w.WriteMsg(m)
-			} else {
-				Debug.Printf("Failed lookup for %s; sending mDNS query", q.Name)
-				// We don't know the answer; see if someone else does
-				channel := make(chan *ResponseA)
-				replies := make([]net.IP, 0)
-				go func() {
-					for resp := range channel {
-						Debug.Printf("Got address response %s to query %s addr %s", resp.Name, q.Name, resp.Addr)
-						replies = append(replies, resp.Addr)
-					}
-					var responseMsg *dns.Msg
-					if len(replies) > 0 {
-						responseMsg = makeAddressReply(r, &q, replies)
-					} else {
-						responseMsg = makeDNSFailResponse(r)
-					}
-					w.WriteMsg(responseMsg)
-				}()
-				mdnsClient.SendQuery(q.Name, dns.TypeA, channel)
-			}
-		} else {
-			Warning.Printf("Local query not handled: %+v", q)
-			m := makeDNSFailResponse(r)
-			w.WriteMsg(m)
-		}
-		return
-	}
-}
-
-func rdnsHandler(zone Zone, mdnsClient *MDNSClient) dns.HandlerFunc {
-	return func(w dns.ResponseWriter, r *dns.Msg) {
-		q := r.Question[0]
-		Debug.Printf("Local rdns query: %+v", q)
-		if q.Qtype == dns.TypePTR {
-			if ip := net.ParseIP(q.Name[:len(q.Name)-rdnsDomainLen]); ip != nil {
-				ip4 := ip.To4()
-				revIP := []byte{ip4[3], ip4[2], ip4[1], ip4[0]}
-				Debug.Printf("Looking for address: %+v", revIP)
-				if name, err := zone.ReverseLookupLocal(revIP); err == nil {
-					Debug.Printf("Found name: %s", name)
-					m := makePTRReply(r, &q, []string{name})
+			for _, lookup := range lookups {
+				if ip, err := lookup.LookupName(q.Name); err == nil {
+					m := makeAddressReply(r, &q, []net.IP{ip})
 					w.WriteMsg(m)
-				} else {
-					Debug.Printf("Failed lookup for %s; sending mDNS query", q.Name)
-					// We don't know the answer; see if someone else does
-					// TODO
+					return
 				}
 			}
 		}
+		Info.Printf("[dns msgid %d] No results for type %s query %s",
+			r.MsgHdr.Id, dns.TypeToString[q.Qtype], q.Name)
+		w.WriteMsg(makeDNSFailResponse(r))
 	}
 }
 
-/* When we receive a request for a name outside of our '.weave' domain, call
-   the underlying lookup mechanism and return the answer(s) it gives.
-   Unfortunately, this means that TTLs from a real DNS server are lost - FIXME.
-*/
-func notUsHandler() dns.HandlerFunc {
+func rdnsHandler(lookups []Lookup) dns.HandlerFunc {
+	fallback := notUsHandler()
 	return func(w dns.ResponseWriter, r *dns.Msg) {
 		q := r.Question[0]
-		Debug.Printf("Non-local query: %+v", q)
-		var responseMsg *dns.Msg
-		if q.Qtype == dns.TypeA || q.Qtype == dns.TypeAAAA {
-			if addrs, err := net.LookupIP(q.Name); err == nil {
-				responseMsg = makeAddressReply(r, &q, addrs)
-			} else {
-				responseMsg = makeDNSFailResponse(r)
-				Debug.Print("Failed fallback lookup ", err)
+		Debug.Printf("Reverse query: %+v", q)
+		if q.Qtype == dns.TypePTR {
+			for _, lookup := range lookups {
+				if name, err := lookup.LookupInaddr(q.Name); err == nil {
+					m := makePTRReply(r, &q, []string{name})
+					w.WriteMsg(m)
+					return
+				}
 			}
+			fallback(w, r)
+			return
 		} else {
-			Warning.Printf("Non-local query not handled: %+v", q)
-			responseMsg = makeDNSFailResponse(r)
+			Warning.Printf("[dns msgid %d] Unexpected reverse query type %s: %+v",
+				r.MsgHdr.Id, dns.TypeToString[q.Qtype], q)
 		}
-		w.WriteMsg(responseMsg)
 	}
 }
 
-func StartServer(zone Zone, iface *net.Interface, dnsPort int, httpPort int, wait int) error {
-	go ListenHttp(LOCAL_DOMAIN, zone, httpPort)
+/* When we receive a request for a name outside of our '.weave.local.'
+   domain, ask the configured DNS server as a fallback.
+*/
+func notUsHandler() dns.HandlerFunc {
+	config, err := dns.ClientConfigFromFile("/etc/resolv.conf")
+	checkFatal(err)
+	return func(w dns.ResponseWriter, r *dns.Msg) {
+		q := r.Question[0]
+		Debug.Printf("[dns msgid %d] Fallback query: %+v", r.MsgHdr.Id, q)
+		for _, server := range config.Servers {
+			reply, err := dns.Exchange(r, fmt.Sprintf("%s:%s", server, config.Port))
+			if err != nil {
+				Debug.Printf("[dns msgid %d] Network error trying %s (%s)",
+					r.MsgHdr.Id, server, err)
+				continue
+			}
+			if reply != nil && reply.Rcode != dns.RcodeSuccess {
+				Debug.Printf("[dns msgid %d] Failure reported by %s for query %s",
+					r.MsgHdr.Id, server, q.Name)
+				continue
+			}
+			Debug.Printf("[dns msgid %d] Given answer by %s for query %s",
+				r.MsgHdr.Id, server, q.Name)
+			w.WriteMsg(reply)
+			return
+		}
+		Warning.Printf("[dns msgid %d] Failed lookup for external name %s",
+			r.MsgHdr.Id, q.Name)
+		w.WriteMsg(makeDNSFailResponse(r))
+	}
+}
 
+func StartServer(zone Zone, iface *net.Interface, dnsPort int, wait int) error {
 	mdnsClient, err := NewMDNSClient()
 	checkFatal(err)
 
@@ -134,8 +117,8 @@ func StartServer(zone Zone, iface *net.Interface, dnsPort int, httpPort int, wai
 	checkFatal(err)
 
 	LocalServeMux := dns.NewServeMux()
-	LocalServeMux.HandleFunc(LOCAL_DOMAIN, queryHandler(zone, mdnsClient))
-	LocalServeMux.HandleFunc(RDNS_DOMAIN, rdnsHandler(zone, mdnsClient))
+	LocalServeMux.HandleFunc(LOCAL_DOMAIN, queryHandler([]Lookup{zone, mdnsClient}))
+	LocalServeMux.HandleFunc(RDNS_DOMAIN, rdnsHandler([]Lookup{zone, mdnsClient}))
 	LocalServeMux.HandleFunc(".", notUsHandler())
 
 	mdnsServer, err := NewMDNSServer(zone)

nameserver/server_test.go

@@ -0,0 +1,91 @@
+package nameserver
+
+import (
+	"fmt"
+	"github.com/miekg/dns"
+	"github.com/zettio/weave/common"
+	wt "github.com/zettio/weave/testing"
+	"net"
+	"testing"
+	"time"
+)
+
+func TestDNSServer(t *testing.T) {
+	const (
+		port            = 17625
+		successTestName = "test1.weave.local."
+		failTestName    = "test2.weave.local."
+		nonLocalName    = "weave.works."
+		testAddr1       = "10.0.2.1"
+		testRDNSsuccess = "1.2.0.10.in-addr.arpa."
+		testRDNSfail    = "4.3.2.1.in-addr.arpa."
+	)
+	dnsAddr := fmt.Sprintf("localhost:%d", port)
+	testCIDR1 := testAddr1 + "/24"
+
+	common.InitDefaultLogging(true)
+	var zone = new(ZoneDb)
+	ip, _, _ := net.ParseCIDR(testCIDR1)
+	zone.AddRecord(containerID, successTestName, ip)
+
+	go StartServer(zone, nil, port, 0)
+	time.Sleep(100 * time.Millisecond) // Allow sever goroutine to start
+
+	c := new(dns.Client)
+	m := new(dns.Msg)
+	m.SetQuestion(successTestName, dns.TypeA)
+	m.RecursionDesired = true
+	r, _, err := c.Exchange(m, dnsAddr)
+	wt.AssertNoErr(t, err)
+	wt.AssertStatus(t, r.Rcode, dns.RcodeSuccess, "DNS response code")
+	wt.AssertEqualInt(t, len(r.Answer), 1, "Number of answers")
+	wt.AssertType(t, r.Answer[0], (*dns.A)(nil), "DNS record")
+	wt.AssertEqualString(t, r.Answer[0].(*dns.A).A.String(), testAddr1, "IP address")
+
+	m.SetQuestion(failTestName, dns.TypeA)
+	r, _, err = c.Exchange(m, dnsAddr)
+	wt.AssertNoErr(t, err)
+	wt.AssertStatus(t, r.Rcode, dns.RcodeNameError, "DNS response code")
+	wt.AssertEqualInt(t, len(r.Answer), 0, "Number of answers")
+
+	m.SetQuestion(testRDNSsuccess, dns.TypePTR)
+	r, _, err = c.Exchange(m, dnsAddr)
+	wt.AssertNoErr(t, err)
+	wt.AssertStatus(t, r.Rcode, dns.RcodeSuccess, "DNS response code")
+	wt.AssertEqualInt(t, len(r.Answer), 1, "Number of answers")
+	wt.AssertType(t, r.Answer[0], (*dns.PTR)(nil), "DNS record")
+	wt.AssertEqualString(t, r.Answer[0].(*dns.PTR).Ptr, successTestName, "IP address")
+	m.SetQuestion(testRDNSfail, dns.TypePTR)
+	r, _, err = c.Exchange(m, dnsAddr)
+	wt.AssertNoErr(t, err)
+	wt.AssertStatus(t, r.Rcode, dns.RcodeNameError, "DNS response code")
+	wt.AssertEqualInt(t, len(r.Answer), 0, "Number of answers")
+
+	// This should fail because we don't handle MX records
+	m.SetQuestion(successTestName, dns.TypeMX)
+	r, _, err = c.Exchange(m, dnsAddr)
+	wt.AssertNoErr(t, err)
+	wt.AssertStatus(t, r.Rcode, dns.RcodeNameError, "DNS response code")
+	wt.AssertEqualInt(t, len(r.Answer), 0, "Number of answers")
+
+	// This non-local query for an MX record should succeed by being
+	// passed on to the configured (/etc/resolv.conf) DNS server.
+	m.SetQuestion(nonLocalName, dns.TypeMX)
+	r, _, err = c.Exchange(m, dnsAddr)
+	wt.AssertNoErr(t, err)
+	wt.AssertStatus(t, r.Rcode, dns.RcodeSuccess, "DNS response code")
+	if !(len(r.Answer) > 0) {
+		t.Fatal("Number of answers > 0")
+	}
+
+	m.SetQuestion("8.8.8.8.in-addr.arpa.", dns.TypePTR)
+	r, _, err = c.Exchange(m, dnsAddr)
+	wt.AssertNoErr(t, err)
+	wt.AssertStatus(t, r.Rcode, dns.RcodeSuccess, "DNS success response code")
+	if !(len(r.Answer) > 0) {
+		t.Fatal("Number of answers > 0")
+	}
+
+	// Not testing MDNS functionality of server here (yet), since it
+	// needs two servers, each listening on its own address
+}

nameserver/updater.go

@@ -7,7 +7,8 @@ import (
 
 func checkError(err error, apiPath string) {
 	if err != nil {
-		Error.Fatalf("Unable to connect to Docker API on %s: %s", apiPath, err)
+		Error.Fatalf("[updater] Unable to connect to Docker API on %s: %s",
+			apiPath, err)
 	}
 }
 
@@ -22,7 +23,7 @@ func StartUpdater(apiPath string, zone Zone) error {
 	err = client.AddEventListener(events)
 	checkError(err, apiPath)
 
-	Info.Printf("Using Docker API on %s: %v", apiPath, env)
+	Info.Printf("[updater] Using Docker API on %s: %v", apiPath, env)
 
 	go func() {
 		for event := range events {
@@ -36,7 +37,7 @@ func handleEvent(zone Zone, event *docker.APIEvents, client *docker.Client) erro
 	switch event.Status {
 	case "die":
 		id := event.ID
-		Info.Printf("Container %s down. Removing records", id)
+		Info.Printf("[updater] Container %s down. Removing records", id)
 		zone.DeleteRecordsFor(id)
 	}
 	return nil

nameserver/zone.go

@@ -2,16 +2,28 @@ package nameserver
 
 import (
 	"github.com/miekg/dns"
+	. "github.com/zettio/weave/common"
 	"net"
 	"sync"
 )
 
+const (
+	RDNS_DOMAIN = "in-addr.arpa."
+)
+
+// +1 to also exclude a dot
+var rdnsDomainLen = len(RDNS_DOMAIN) + 1
+
+type Lookup interface {
+	LookupName(name string) (net.IP, error)
+	LookupInaddr(inaddr string) (string, error)
+}
+
 type Zone interface {
 	AddRecord(ident string, name string, ip net.IP) error
 	DeleteRecord(ident string, ip net.IP) error
 	DeleteRecordsFor(ident string) error
-	LookupLocal(name string) (net.IP, error)
-	ReverseLookupLocal(ip net.IP) (string, error)
+	Lookup
 }
 
 type Record struct {
@@ -49,7 +61,7 @@ func (zone *ZoneDb) indexOf(match func(Record) bool) int {
 	return -1
 }
 
-func (zone *ZoneDb) LookupLocal(name string) (net.IP, error) {
+func (zone *ZoneDb) LookupName(name string) (net.IP, error) {
 	zone.mx.RLock()
 	defer zone.mx.RUnlock()
 	for _, r := range zone.recs {
@@ -60,15 +72,23 @@ func (zone *ZoneDb) LookupLocal(name string) (net.IP, error) {
 	return nil, LookupError(name)
 }
 
-func (zone *ZoneDb) ReverseLookupLocal(ip net.IP) (string, error) {
-	zone.mx.RLock()
-	defer zone.mx.RUnlock()
-	for _, r := range zone.recs {
-		if r.IP.Equal(ip) {
-			return r.Name, nil
+func (zone *ZoneDb) LookupInaddr(inaddr string) (string, error) {
+	if revIP := net.ParseIP(inaddr[:len(inaddr)-rdnsDomainLen]); revIP != nil {
+		revIP4 := revIP.To4()
+		ip := []byte{revIP4[3], revIP4[2], revIP4[1], revIP4[0]}
+		Debug.Printf("[zonedb] Looking for address: %+v", ip)
+		zone.mx.RLock()
+		defer zone.mx.RUnlock()
+		for _, r := range zone.recs {
+			if r.IP.Equal(ip) {
+				return r.Name, nil
+			}
 		}
+		return "", LookupError(inaddr)
+	} else {
+		Warning.Printf("[zonedb] Asked to reverse lookup %s", inaddr)
+		return "", LookupError(inaddr)
 	}
-	return "", LookupError(ip.String())
 }
 
 func (zone *ZoneDb) AddRecord(ident string, name string, ip net.IP) error {

nameserver/zone_test.go

@@ -1,7 +1,7 @@
 package nameserver
 
 import (
-	wt "github.com/zettio/weave/common"
+	wt "github.com/zettio/weave/testing"
 	"net"
 	"testing"
 )
@@ -27,7 +27,7 @@ func TestZone(t *testing.T) {
 	wt.AssertNoErr(t, err)
 
 	// Check that the address is now there.
-	foundIP, err := zone.LookupLocal(successTestName)
+	foundIP, err := zone.LookupName(successTestName)
 	wt.AssertNoErr(t, err)
 
 	if !foundIP.Equal(ip) {
@@ -35,7 +35,7 @@ func TestZone(t *testing.T) {
 	}
 
 	// See if we can find the address by IP.
-	foundName, err := zone.ReverseLookupLocal(ip)
+	foundName, err := zone.LookupInaddr("1.2.0.10.in-addr.arpa.")
 	wt.AssertNoErr(t, err)
 
 	if foundName != successTestName {
@@ -50,14 +50,14 @@ func TestZone(t *testing.T) {
 	err = zone.DeleteRecord(containerID, ip)
 	wt.AssertNoErr(t, err)
 
-	_, err = zone.LookupLocal(successTestName)
+	_, err = zone.LookupName(successTestName)
 	wt.AssertNoErr(t, err)
 
 	err = zone.DeleteRecord(otherContainerID, ip)
 	wt.AssertNoErr(t, err)
 
 	// Check that the address is not there now.
-	_, err = zone.LookupLocal(successTestName)
+	_, err = zone.LookupName(successTestName)
 	wt.AssertErrorType(t, err, (*LookupError)(nil), "after deleting record")
 
 	// Delete a record that isn't there
@@ -79,10 +79,10 @@ func TestDeleteFor(t *testing.T) {
 		wt.AssertNoErr(t, err)
 	}
 
-	_, err := zone.LookupLocal(name)
+	_, err := zone.LookupName(name)
 	wt.AssertNoErr(t, err)
 
 	err = zone.DeleteRecordsFor(id)
-	_, err = zone.LookupLocal(name)
+	_, err = zone.LookupName(name)
 	wt.AssertErrorType(t, err, (*LookupError)(nil), "after deleting records for ident")
 }

router/connection.go

@@ -8,14 +8,70 @@ import (
 	"log"
 	"net"
 	"strconv"
+	"sync"
 	"time"
 )
 
-func NewRemoteConnection(from, to *Peer, tcpAddr string) *RemoteConnection {
+type Connection interface {
+	Local() *Peer
+	Remote() *Peer
+	BreakTie(Connection) ConnectionTieBreak
+	RemoteTCPAddr() string
+	Established() bool
+	Shutdown(error)
+}
+
+type ConnectionTieBreak int
+
+const (
+	TieBreakWon ConnectionTieBreak = iota
+	TieBreakLost
+	TieBreakTied
+)
+
+type RemoteConnection struct {
+	local         *Peer
+	remote        *Peer
+	remoteTCPAddr string
+	established   bool
+}
+
+type LocalConnection struct {
+	sync.RWMutex
+	RemoteConnection
+	TCPConn            *net.TCPConn
+	tcpSender          TCPSender
+	remoteUDPAddr      *net.UDPAddr
+	receivedHeartbeat  bool
+	stackFrag          bool
+	effectivePMTU      int
+	SessionKey         *[32]byte
+	establishedTimeout *time.Timer
+	heartbeatFrame     *ForwardedFrame
+	heartbeat          *time.Ticker
+	fragTest           *time.Ticker
+	forwardChan        chan<- *ForwardedFrame
+	forwardChanDF      chan<- *ForwardedFrame
+	stopForward        chan<- interface{}
+	stopForwardDF      chan<- interface{}
+	verifyPMTU         chan<- int
+	Decryptor          Decryptor
+	Router             *Router
+	uid                uint64
+	queryChan          chan<- *ConnectionInteraction
+}
+
+type ConnectionInteraction struct {
+	Interaction
+	payload interface{}
+}
+
+func NewRemoteConnection(from, to *Peer, tcpAddr string, established bool) *RemoteConnection {
 	return &RemoteConnection{
 		local:         from,
 		remote:        to,
-		remoteTCPAddr: tcpAddr}
+		remoteTCPAddr: tcpAddr,
+		established:   established}
 }
 
 func (conn *RemoteConnection) Local() *Peer {
@@ -26,15 +82,19 @@ func (conn *RemoteConnection) Remote() *Peer {
 	return conn.remote
 }
 
+func (conn *RemoteConnection) BreakTie(Connection) ConnectionTieBreak {
+	return TieBreakTied
+}
+
 func (conn *RemoteConnection) RemoteTCPAddr() string {
 	return conn.remoteTCPAddr
 }
 
-func (conn *RemoteConnection) Shutdown() {
+func (conn *RemoteConnection) Established() bool {
+	return conn.established
 }
 
-func (conn *RemoteConnection) Established() bool {
-	return true
+func (conn *RemoteConnection) Shutdown(error) {
 }
 
 func (conn *RemoteConnection) String() string {
@@ -49,29 +109,38 @@ func (conn *RemoteConnection) String() string {
 	return fmt.Sprint("Connection ", from, "->", to)
 }
 
-// Async. Does not return anything. If the connection is successful,
-// it will end up in the local peer's connections map.
-func NewLocalConnection(connRemote *RemoteConnection, acceptNewPeer bool, tcpConn *net.TCPConn, udpAddr *net.UDPAddr, router *Router) {
+func NewLocalConnection(connRemote *RemoteConnection, tcpConn *net.TCPConn, udpAddr *net.UDPAddr, router *Router) *LocalConnection {
 	if connRemote.local != router.Ourself.Peer {
 		log.Fatal("Attempt to create local connection from a peer which is not ourself")
 	}
-
-	queryChan := make(chan *ConnectionInteraction, ChannelSize)
 	// NB, we're taking a copy of connRemote here.
-	connLocal := &LocalConnection{
+	return &LocalConnection{
 		RemoteConnection: *connRemote,
 		Router:           router,
 		TCPConn:          tcpConn,
 		remoteUDPAddr:    udpAddr,
-		effectivePMTU:    DefaultPMTU,
-		queryChan:        queryChan}
-	go connLocal.queryLoop(queryChan, acceptNewPeer)
+		effectivePMTU:    DefaultPMTU}
 }
 
-func (conn *LocalConnection) Established() bool {
-	conn.RLock()
-	defer conn.RUnlock()
-	return conn.established
+// Async. Does not return anything. If the connection is successful,
+// it will end up in the local peer's connections map.
+func (conn *LocalConnection) Start(acceptNewPeer bool) {
+	queryChan := make(chan *ConnectionInteraction, ChannelSize)
+	conn.queryChan = queryChan
+	go conn.run(queryChan, acceptNewPeer)
+}
+
+func (conn *LocalConnection) BreakTie(dupConn Connection) ConnectionTieBreak {
+	dupConnLocal := dupConn.(*LocalConnection)
+	// conn.uid is used as the tie breaker here, in the knowledge that
+	// both sides will make the same decision.
+	if conn.uid < dupConnLocal.uid {
+		return TieBreakWon
+	} else if dupConnLocal.uid < conn.uid {
+		return TieBreakLost
+	} else {
+		return TieBreakTied
+	}
 }
 
 // Read by the forwarder processes when in the UDP senders
@@ -81,17 +150,10 @@ func (conn *LocalConnection) RemoteUDPAddr() *net.UDPAddr {
 	return conn.remoteUDPAddr
 }
 
-// Called by the forwarder processes in a few places (including
-// crypto), but the connection TCP receiver process, and by the local
-// peer actor process. Do not call this from the connection's actor
-// process itself.
-func (conn *LocalConnection) CheckFatal(err error) error {
-	if err == nil {
-		return nil
-	}
-	conn.log("error:", err)
-	conn.Shutdown()
-	return err
+func (conn *LocalConnection) Established() bool {
+	conn.RLock()
+	defer conn.RUnlock()
+	return conn.established
 }
 
 // Called by forwarder processes, read in Forward (by sniffer and udp
@@ -115,32 +177,36 @@ func (conn *LocalConnection) setStackFrag(frag bool) {
 }
 
 func (conn *LocalConnection) log(args ...interface{}) {
-	v := append([]interface{}{}, fmt.Sprintf("->[%s]:", conn.remote.Name))
-	v = append(v, args...)
-	log.Println(v...)
+	log.Println(append(append([]interface{}{}, fmt.Sprintf("->[%s]:", conn.remote.Name)), args...)...)
 }
 
 // ACTOR client API
 
 const (
-	CSendTCP          = iota
-	CSetEstablished   = iota
-	CSetRemoteUDPAddr = iota
-	CShutdown         = iota
+	CSendProtocolMsg = iota
+	CSetEstablished
+	CReceivedHeartbeat
+	CShutdown
 )
 
 // Async
-func (conn *LocalConnection) Shutdown() {
-	conn.queryChan <- &ConnectionInteraction{Interaction: Interaction{code: CShutdown}}
+func (conn *LocalConnection) Shutdown(err error) {
+	conn.queryChan <- &ConnectionInteraction{
+		Interaction: Interaction{code: CShutdown},
+		payload:     err}
 }
 
 // Async
-func (conn *LocalConnection) SetRemoteUDPAddr(remoteUDPAddr *net.UDPAddr) {
-	if remoteUDPAddr == nil {
+//
+// Heartbeating serves two purposes: a) keeping NAT paths alive, and
+// b) updating a remote peer's knowledge of our address, in the event
+// it changes (e.g. because NAT paths expired).
+func (conn *LocalConnection) ReceivedHeartbeat(remoteUDPAddr *net.UDPAddr, connUID uint64) {
+	if remoteUDPAddr == nil || connUID != conn.uid {
 		return
 	}
 	conn.queryChan <- &ConnectionInteraction{
-		Interaction: Interaction{code: CSetRemoteUDPAddr},
+		Interaction: Interaction{code: CReceivedHeartbeat},
 		payload:     remoteUDPAddr}
 }
 
@@ -150,34 +216,76 @@ func (conn *LocalConnection) SetEstablished() {
 }
 
 // Async
-func (conn *LocalConnection) SendTCP(msg []byte) {
+func (conn *LocalConnection) SendProtocolMsg(m ProtocolMsg) {
 	conn.queryChan <- &ConnectionInteraction{
-		Interaction: Interaction{code: CSendTCP},
-		payload:     msg}
+		Interaction: Interaction{code: CSendProtocolMsg},
+		payload:     m}
 }
 
 // ACTOR server
 
-func (conn *LocalConnection) queryLoop(queryChan <-chan *ConnectionInteraction, acceptNewPeer bool) {
-	err := conn.handshake(acceptNewPeer)
-	if err != nil {
-		log.Printf("->[%s] encountered error during handshake: %v\n", conn.remoteTCPAddr, err)
-		conn.handleShutdown()
+func (conn *LocalConnection) run(queryChan <-chan *ConnectionInteraction, acceptNewPeer bool) {
+	defer conn.handleShutdown()
+
+	tcpConn := conn.TCPConn
+	tcpConn.SetLinger(0)
+	enc := gob.NewEncoder(tcpConn)
+	dec := gob.NewDecoder(tcpConn)
+
+	if err := conn.handshake(enc, dec, acceptNewPeer); err != nil {
+		log.Printf("->[%s] connection shutting down due to error during handshake: %v\n", conn.remoteTCPAddr, err)
 		return
 	}
-	conn.Router.Ourself.AddConnection(conn)
+	log.Printf("->[%s] completed handshake with %s\n", conn.remoteTCPAddr, conn.remote.Name)
+
+	// We invoke AddConnection in the same goroutine that subsequently
+	// becomes the tcp receive loop, rather than outside, because a)
+	// the ordering relative to the receive loop is the only one that
+	// matters [1], b) it prevents unnecessary delays in entering the
+	// main connection loop, and c) it guards against potential
+	// deadlocks.
+	go func() {
+		conn.Router.Ourself.AddConnection(conn)
+		conn.receiveTCP(dec)
+	}()
+
+	heartbeatFrameBytes := make([]byte, EthernetOverhead+8)
+	binary.BigEndian.PutUint64(heartbeatFrameBytes[EthernetOverhead:], conn.uid)
+	conn.heartbeatFrame = &ForwardedFrame{
+		srcPeer: conn.local,
+		dstPeer: conn.remote,
+		frame:   heartbeatFrameBytes}
+
 	if conn.remoteUDPAddr != nil {
-		if err = conn.ensureForwarders(); err == nil {
-			conn.heartbeat = time.NewTicker(FastHeartbeat)
-			conn.forwardHeartbeatFrame() // avoid initial wait
+		if err := conn.sendFastHeartbeats(); err != nil {
+			conn.log("connection shutting down due to error:", err)
+			return
 		}
 	}
+
+	conn.establishedTimeout = time.NewTimer(EstablishedTimeout)
+
+	if err := conn.queryLoop(queryChan); err != nil {
+		conn.log("connection shutting down due to error:", err)
+	} else {
+		conn.log("connection shutting down")
+	}
+}
+
+// [1] In the absence of any indirect connectivity to the remote peer,
+// the first we hear about it (and any peers reachable from it) is
+// through topology gossip it sends us on the connection. We must
+// ensure that the connection has been added to Ourself prior to
+// processing any such gossip, otherwise we risk immediately gc'ing
+// part of that newly received portion of the topology (though not the
+// remote peer itself, since that will have a positive ref count),
+// leaving behind dangling references to peers. Therefore we invoke
+// AddConnection, which is *synchronous*, before entering the tcp
+// receive loop.
+
+func (conn *LocalConnection) queryLoop(queryChan <-chan *ConnectionInteraction) (err error) {
 	terminate := false
-	for !terminate {
-		if err != nil {
-			conn.log("error:", err)
-			break
-		}
+	for !terminate && err == nil {
 		select {
 		case query, ok := <-queryChan:
 			if !ok {
@@ -185,95 +293,97 @@ func (conn *LocalConnection) queryLoop(queryChan <-chan *ConnectionInteraction,
 			}
 			switch query.code {
 			case CShutdown:
+				err = query.payload.(error)
 				terminate = true
+			case CReceivedHeartbeat:
+				err = conn.handleReceivedHeartbeat(query.payload.(*net.UDPAddr))
 			case CSetEstablished:
+				conn.establishedTimeout.Stop()
 				err = conn.handleSetEstablished()
-			case CSetRemoteUDPAddr:
-				err = conn.handleSetRemoteUDPAddr(query.payload.(*net.UDPAddr))
-			case CSendTCP:
-				err = conn.handleSendTCP(query.payload.([]byte))
+			case CSendProtocolMsg:
+				err = conn.handleSendProtocolMsg(query.payload.(ProtocolMsg))
+			}
+		case <-conn.establishedTimeout.C:
+			if !conn.established {
+				err = fmt.Errorf("failed to establish UDP connectivity")
 			}
 		case <-tickerChan(conn.heartbeat):
-			conn.forwardHeartbeatFrame()
-		case <-tickerChan(conn.fetchAll):
-			err = conn.handleSendTCP(ProtocolFetchAllByte)
+			conn.Forward(true, conn.heartbeatFrame, nil)
 		case <-tickerChan(conn.fragTest):
 			conn.setStackFrag(false)
-			err = conn.handleSendTCP(ProtocolStartFragmentationTestByte)
-
+			err = conn.handleSendSimpleProtocolMsg(ProtocolStartFragmentationTest)
 		}
 	}
-	conn.handleShutdown()
+	return
 }
 
-func (conn *LocalConnection) handleSetRemoteUDPAddr(remoteUDPAddr *net.UDPAddr) error {
-	if remoteUDPAddr == nil {
-		return nil
-	}
+// Handlers
+//
+// NB: The conn.* fields are only written by the connection actor
+// process, which is the caller of the handlers. Hence we do not need
+// locks for reading, and only need write locks for fields read by
+// other processes.
+
+func (conn *LocalConnection) handleReceivedHeartbeat(remoteUDPAddr *net.UDPAddr) error {
+	oldRemoteUDPAddr := conn.remoteUDPAddr
+	old := conn.receivedHeartbeat
 	conn.Lock()
-	old := conn.remoteUDPAddr
 	conn.remoteUDPAddr = remoteUDPAddr
+	conn.receivedHeartbeat = true
 	conn.Unlock()
-	if old == nil {
-		if err := conn.handleSendTCP(ProtocolConnectionEstablishedByte); err != nil {
+	if !old {
+		if err := conn.handleSendSimpleProtocolMsg(ProtocolConnectionEstablished); err != nil {
 			return err
 		}
-		return conn.handleSetEstablished()
-	} else if old.String() != remoteUDPAddr.String() {
+	}
+	if oldRemoteUDPAddr == nil {
+		return conn.sendFastHeartbeats()
+	} else if oldRemoteUDPAddr.String() != remoteUDPAddr.String() {
 		log.Println("Peer", conn.remote.Name, "moved from", old, "to", remoteUDPAddr)
 	}
 	return nil
 }
 
 func (conn *LocalConnection) handleSetEstablished() error {
-	conn.Lock()
+	stopTicker(conn.heartbeat)
 	old := conn.established
+	conn.Lock()
 	conn.established = true
 	conn.Unlock()
-	if !old {
-		conn.Router.Ourself.ConnectionEstablished(conn)
-		if err := conn.ensureForwarders(); err != nil {
-			return err
-		}
-		stopTicker(conn.heartbeat)
-		conn.heartbeat = time.NewTicker(SlowHeartbeat)
-		conn.fetchAll = time.NewTicker(FetchAllInterval)
-		conn.fragTest = time.NewTicker(FragTestInterval)
-		conn.forwardHeartbeatFrame() // avoid initial wait
-		// Send a large frame down the DF channel in order to prompt
-		// PMTU discovery to start.
-		conn.Forward(true, &ForwardedFrame{
-			srcPeer: conn.local,
-			dstPeer: conn.remote,
-			frame:   PMTUDiscovery},
-			nil)
-		conn.setStackFrag(false)
-		return conn.handleSendTCP(ProtocolStartFragmentationTestByte)
+	if old {
+		return nil
+	}
+	conn.Router.Ourself.ConnectionEstablished(conn)
+	if err := conn.ensureForwarders(); err != nil {
+		return err
+	}
+	// Send a large frame down the DF channel in order to prompt
+	// PMTU discovery to start.
+	conn.Forward(true, &ForwardedFrame{
+		srcPeer: conn.local,
+		dstPeer: conn.remote,
+		frame:   PMTUDiscovery},
+		nil)
+	conn.heartbeat = time.NewTicker(SlowHeartbeat)
+	conn.fragTest = time.NewTicker(FragTestInterval)
+	// avoid initial waits for timers to fire
+	conn.Forward(true, conn.heartbeatFrame, nil)
+	conn.setStackFrag(false)
+	if err := conn.handleSendSimpleProtocolMsg(ProtocolStartFragmentationTest); err != nil {
+		return err
 	}
 	return nil
 }
 
-func (conn *LocalConnection) handleSendTCP(msg []byte) error {
-	return conn.tcpSender.Send(msg)
+func (conn *LocalConnection) handleSendSimpleProtocolMsg(tag ProtocolTag) error {
+	return conn.handleSendProtocolMsg(ProtocolMsg{tag: tag})
+}
+
+func (conn *LocalConnection) handleSendProtocolMsg(m ProtocolMsg) error {
+	return conn.tcpSender.Send(Concat([]byte{byte(m.tag)}, m.msg))
 }
 
 func (conn *LocalConnection) handleShutdown() {
-	if conn.remote != nil {
-		conn.log("connection shutting down")
-	}
-
-	// Whilst some of these elements may have been written to whilst
-	// holding locks, they were only written to by the connection
-	// actor process. handleShutdown is only called by the connection
-	// actor process. So there is no need to take locks to read these
-	// (or write elements which are only read by the same
-	// process). Taking locks is only done for elements which are read
-	// by other processes.
-	if conn.shutdown {
-		return
-	}
-	conn.shutdown = true
-
 	if conn.TCPConn != nil {
 		checkWarn(conn.TCPConn.Close())
 	}
@@ -283,8 +393,11 @@ func (conn *LocalConnection) handleShutdown() {
 		conn.Router.Ourself.DeleteConnection(conn)
 	}
 
+	if conn.establishedTimeout != nil {
+		conn.establishedTimeout.Stop()
+	}
+
 	stopTicker(conn.heartbeat)
-	stopTicker(conn.fetchAll)
 	stopTicker(conn.fragTest)
 
 	// blank out the forwardChan so that the router processes don't
@@ -294,19 +407,17 @@ func (conn *LocalConnection) handleShutdown() {
 	conn.Router.ConnectionMaker.ConnectionTerminated(conn.remoteTCPAddr)
 }
 
-func (conn *LocalConnection) handshake(acceptNewPeer bool) error {
-	// We do not need to worry about locking in here as at this point,
+// Helpers
+
+func (conn *LocalConnection) handshake(enc *gob.Encoder, dec *gob.Decoder, acceptNewPeer bool) error {
+	// We do not need to worry about locking in here as at this point
 	// the connection is not reachable by any go-routine other than
 	// ourself. Only when we add this connection to the conn.local
 	// peer will it be visible from multiple go-routines.
-	tcpConn := conn.TCPConn
-	tcpConn.SetLinger(0)
+
 	conn.extendReadDeadline()
 
-	enc := gob.NewEncoder(tcpConn)
-
 	localConnID := randUint64()
-
 	versionStr := fmt.Sprint(ProtocolVersion)
 	handshakeSend := map[string]string{
 		"Protocol":        Protocol,
@@ -329,7 +440,6 @@ func (conn *LocalConnection) handshake(acceptNewPeer bool) error {
 	}
 	enc.Encode(handshakeSend)
 
-	dec := gob.NewDecoder(tcpConn)
 	err = dec.Decode(&handshakeRecv)
 	if err != nil {
 		return err
@@ -380,7 +490,7 @@ func (conn *LocalConnection) handshake(acceptNewPeer bool) error {
 	if err != nil {
 		return err
 	}
-	conn.UID = localConnID ^ remoteConnID
+	conn.uid = localConnID ^ remoteConnID
 
 	if usingPassword {
 		remotePublicStr, rpErr := checkHandshakeStringField("PublicKey", "", handshakeRecv)
@@ -408,17 +518,16 @@ func (conn *LocalConnection) handshake(acceptNewPeer bool) error {
 
 	toPeer := NewPeer(name, uid, 0)
 	toPeer = conn.Router.Peers.FetchWithDefault(toPeer)
-	if toPeer == nil {
+	switch toPeer {
+	case nil:
 		return fmt.Errorf("Connection appears to be with different version of a peer we already know of")
-	} else if toPeer == conn.local {
-		// have to do assigment here to ensure Shutdown releases ref count
-		conn.remote = toPeer
+	case conn.local:
+		conn.remote = toPeer // have to do assigment here to ensure Shutdown releases ref count
 		return fmt.Errorf("Cannot connect to ourself")
+	default:
+		conn.remote = toPeer
+		return nil
 	}
-	conn.remote = toPeer
-
-	go conn.receiveTCP(dec, usingPassword)
-	return nil
 }
 
 func checkHandshakeStringField(fieldName string, expectedValue string, handshake map[string]string) (string, error) {
@@ -432,8 +541,9 @@ func checkHandshakeStringField(fieldName string, expectedValue string, handshake
 	return val, nil
 }
 
-func (conn *LocalConnection) receiveTCP(decoder *gob.Decoder, usingPassword bool) {
+func (conn *LocalConnection) receiveTCP(decoder *gob.Decoder) {
 	defer conn.Decryptor.Shutdown()
+	usingPassword := conn.SessionKey != nil
 	var receiver TCPReceiver
 	if usingPassword {
 		receiver = NewEncryptedTCPReceiver(conn)
@@ -444,94 +554,70 @@ func (conn *LocalConnection) receiveTCP(decoder *gob.Decoder, usingPassword bool
 	for {
 		var msg []byte
 		conn.extendReadDeadline()
-		if conn.CheckFatal(decoder.Decode(&msg)) != nil {
-			return
+		if err = decoder.Decode(&msg); err != nil {
+			break
 		}
 		msg, err = receiver.Decode(msg)
 		if err != nil {
-			checkWarn(err)
-			// remote peer may be using wrong password. Or there could
-			// be some sort of injection attack going on. Just ignore
-			// the traffic rather than shutting down.
+			break
+		}
+		if len(msg) < 1 {
+			conn.log("ignoring blank msg")
 			continue
 		}
-		if msg[0] == ProtocolConnectionEstablished {
-			// We initiated the connection. We sent fast heartbeats to
-			// the remote side, which has now received at least one of
-			// them and thus has informed us via TCP that it considers
-			// the connection is now up. We now do a fetchAll on it.
-			conn.SetEstablished()
-			conn.SendTCP(ProtocolFetchAllByte)
-		} else if msg[0] == ProtocolStartFragmentationTest {
-			conn.Forward(false, &ForwardedFrame{
-				srcPeer: conn.local,
-				dstPeer: conn.remote,
-				frame:   FragTest},
-				nil)
-		} else if msg[0] == ProtocolFragmentationReceived {
-			conn.setStackFrag(true)
-		} else if usingPassword && msg[0] == ProtocolNonce {
-			conn.Decryptor.ReceiveNonce(msg[1:])
-		} else if msg[0] == ProtocolFetchAll {
-			// There are exactly two messages that relate to topology
-			// updates.
-			//
-			// 1. FetchAll. This carries no payload. The receiver
-			// responds with the entire topology model as the receiver
-			// has it.
-			//
-			// 2. Update. This carries a topology payload. The
-			// receiver merges it with its own topology model. If the
-			// payload is a subset of the receiver's topology, no
-			// further action is taken. Otherwise, the receiver sends
-			// out to all its connections an "improved" update:
-			//  - elements which the original payload added to the
-			//    receiver are included
-			//  - elements which the original payload updated in the
-			//    receiver are included
-			//  - elements which are equal between the receiver and
-			//    the payload are not included
-			//  - elements where the payload was older than the
-			//    receiver's version are updated
-			conn.SendTCP(Concat(ProtocolUpdateByte, conn.Router.Peers.EncodeAllPeers()))
-		} else if msg[0] == ProtocolUpdate {
-			newUpdate, err := conn.Router.Peers.ApplyUpdate(msg[1:])
-			if _, ok := err.(UnknownPeersError); err != nil && ok {
-				// That update contained a peer we didn't know about;
-				// request full update
-				conn.SendTCP(ProtocolFetchAllByte)
-				continue
-			}
-			if conn.CheckFatal(err) != nil {
-				return
-			}
-			if len(newUpdate) != 0 {
-				conn.Router.ConnectionMaker.Refresh()
-				conn.Router.Routes.Recalculate()
-				conn.Router.Ourself.BroadcastTCP(Concat(ProtocolUpdateByte, newUpdate))
-			}
-		} else if msg[0] == ProtocolPMTUVerified {
-			conn.verifyPMTU <- int(binary.BigEndian.Uint16(msg[1:]))
-		} else {
-			conn.log("received unknown msg:\n", msg)
+		if err = conn.handleProtocolMsg(ProtocolTag(msg[0]), msg[1:]); err != nil {
+			break
 		}
 	}
+	conn.Shutdown(err)
+}
+
+func (conn *LocalConnection) handleProtocolMsg(tag ProtocolTag, payload []byte) error {
+	switch tag {
+	case ProtocolConnectionEstablished:
+		// We sent fast heartbeats to the remote peer, which has now
+		// received at least one of them and told us via this message.
+		// We can now consider the connection as established from our
+		// end.
+		conn.SetEstablished()
+	case ProtocolStartFragmentationTest:
+		conn.Forward(false, &ForwardedFrame{
+			srcPeer: conn.local,
+			dstPeer: conn.remote,
+			frame:   FragTest},
+			nil)
+	case ProtocolFragmentationReceived:
+		conn.setStackFrag(true)
+	case ProtocolNonce:
+		if conn.SessionKey == nil {
+			return fmt.Errorf("unexpected nonce on unencrypted connection")
+		}
+		conn.Decryptor.ReceiveNonce(payload)
+	case ProtocolPMTUVerified:
+		conn.verifyPMTU <- int(binary.BigEndian.Uint16(payload))
+	case ProtocolGossipUnicast:
+		return conn.Router.handleGossip(payload, deliverGossipUnicast)
+	case ProtocolGossipBroadcast:
+		return conn.Router.handleGossip(payload, deliverGossipBroadcast)
+	case ProtocolGossip:
+		return conn.Router.handleGossip(payload, deliverGossip)
+	default:
+		conn.log("ignoring unknown protocol tag:", tag)
+	}
+	return nil
 }
 
 func (conn *LocalConnection) extendReadDeadline() {
 	conn.TCPConn.SetReadDeadline(time.Now().Add(ReadTimeout))
 }
 
-// Heartbeating serves two purposes: a) keeping NAT paths alive, and
-// b) updating a remote peer's knowledge of our address, in the event
-// it changes (e.g. because NAT paths expired).
-// Called only by connection actor process.
-func (conn *LocalConnection) forwardHeartbeatFrame() {
-	heartbeatFrame := &ForwardedFrame{
-		srcPeer: conn.local,
-		dstPeer: conn.remote,
-		frame:   []byte{}}
-	conn.Forward(true, heartbeatFrame, nil)
+func (conn *LocalConnection) sendFastHeartbeats() error {
+	err := conn.ensureForwarders()
+	if err == nil {
+		conn.heartbeat = time.NewTicker(FastHeartbeat)
+		conn.Forward(true, conn.heartbeatFrame, nil) // avoid initial wait
+	}
+	return err
 }
 
 func tickerChan(ticker *time.Ticker) <-chan time.Time {

router/connection_maker.go

@@ -15,10 +15,10 @@ const (
 )
 
 const (
-	CMInitiate   = iota
-	CMTerminated = iota
-	CMRefresh    = iota
-	CMStatus     = iota
+	CMInitiate = iota
+	CMTerminated
+	CMRefresh
+	CMStatus
 )
 
 type ConnectionMaker struct {
@@ -41,16 +41,18 @@ type ConnectionMakerInteraction struct {
 	address string
 }
 
-func StartConnectionMaker(ourself *LocalPeer, peers *Peers) *ConnectionMaker {
-	queryChan := make(chan *ConnectionMakerInteraction, ChannelSize)
-	state := &ConnectionMaker{
+func NewConnectionMaker(ourself *LocalPeer, peers *Peers) *ConnectionMaker {
+	return &ConnectionMaker{
 		ourself:        ourself,
 		peers:          peers,
-		queryChan:      queryChan,
 		cmdLineAddress: make(map[string]bool),
 		targets:        make(map[string]*Target)}
-	go state.queryLoop(queryChan)
-	return state
+}
+
+func (cm *ConnectionMaker) Start() {
+	queryChan := make(chan *ConnectionMakerInteraction, ChannelSize)
+	cm.queryChan = queryChan
+	go cm.queryLoop(queryChan)
 }
 
 func (cm *ConnectionMaker) InitiateConnection(address string) {
@@ -198,9 +200,9 @@ func (cm *ConnectionMaker) status() string {
 }
 
 func (cm *ConnectionMaker) attemptConnection(address string, acceptNewPeer bool) {
-	log.Println("Attempting connection to", address)
+	log.Printf("->[%s] attempting connection\n", address)
 	if err := cm.ourself.CreateConnection(address, acceptNewPeer); err != nil {
-		log.Println(err)
+		log.Printf("->[%s] error during connection attempt: %v\n", address, err)
 		cm.ConnectionTerminated(address)
 	}
 }

router/consts.go

@@ -6,8 +6,6 @@ import (
 )
 
 const (
-	Protocol           = "weave"
-	ProtocolVersion    = 8
 	EthernetOverhead   = 14
 	UDPOverhead        = 28 // 20 bytes for IPv4, 8 bytes for UDP
 	Port               = 6783
@@ -20,32 +18,15 @@ const (
 	PMTUDiscoverySize  = 60000
 	FastHeartbeat      = 500 * time.Millisecond
 	SlowHeartbeat      = 10 * time.Second
-	FetchAllInterval   = 30 * time.Second
 	FragTestInterval   = 5 * time.Minute
-	ReadTimeout        = 2 * FetchAllInterval
+	EstablishedTimeout = 30 * time.Second
+	ReadTimeout        = 1 * time.Minute
 	PMTUVerifyAttempts = 8
 	PMTUVerifyTimeout  = 10 * time.Millisecond // gets doubled with every attempt
 	MaxDuration        = time.Duration(math.MaxInt64)
 )
 
-const (
-	ProtocolConnectionEstablished  = iota
-	ProtocolFragmentationReceived  = iota
-	ProtocolStartFragmentationTest = iota
-	ProtocolNonce                  = iota
-	ProtocolFetchAll               = iota
-	ProtocolUpdate                 = iota
-	ProtocolPMTUVerified           = iota
-)
-
 var (
-	FragTest                           = make([]byte, FragTestSize)
-	PMTUDiscovery                      = make([]byte, PMTUDiscoverySize)
-	ProtocolConnectionEstablishedByte  = []byte{ProtocolConnectionEstablished}
-	ProtocolFragmentationReceivedByte  = []byte{ProtocolFragmentationReceived}
-	ProtocolStartFragmentationTestByte = []byte{ProtocolStartFragmentationTest}
-	ProtocolNonceByte                  = []byte{ProtocolNonce}
-	ProtocolFetchAllByte               = []byte{ProtocolFetchAll}
-	ProtocolUpdateByte                 = []byte{ProtocolUpdate}
-	ProtocolPMTUVerifiedByte           = []byte{ProtocolPMTUVerified}
+	FragTest      = make([]byte, FragTestSize)
+	PMTUDiscovery = make([]byte, PMTUDiscoverySize)
 )

router/crypto.go

@@ -11,6 +11,8 @@ import (
 	"encoding/gob"
 	"fmt"
 	"log"
+	"net"
+	"sync"
 )
 
 func GenerateKeyPair() (publicKey, privateKey *[32]byte, err error) {
@@ -83,7 +85,8 @@ func EncodeNonce(df bool) (*[24]byte, []byte, error) {
 		flags = flags | 1
 	}
 	SetNonceLow15Bits(&nonce, flags)
-	return &nonce, Concat(ProtocolNonceByte, nonce[:]), nil
+	// NB: need to make a copy since callers may modify the array
+	return &nonce, Concat(nonce[:]), nil
 }
 
 func DecodeNonce(msg []byte) (bool, *[24]byte) {
@@ -99,6 +102,34 @@ func DecodeNonce(msg []byte) (bool, *[24]byte) {
 
 // Frame Encryptors
 
+type Encryptor interface {
+	FrameOverhead() int
+	PacketOverhead() int
+	IsEmpty() bool
+	Bytes() []byte
+	AppendFrame(*ForwardedFrame)
+	TotalLen() int
+}
+
+type NonEncryptor struct {
+	buf       []byte
+	bufTail   []byte
+	buffered  int
+	prefixLen int
+}
+
+type NaClEncryptor struct {
+	NonEncryptor
+	buf       []byte
+	offset    uint16
+	nonce     *[24]byte
+	nonceChan chan *[24]byte
+	flags     uint16
+	prefixLen int
+	conn      *LocalConnection
+	df        bool
+}
+
 func NewNonEncryptor(prefix []byte) *NonEncryptor {
 	buf := make([]byte, MaxUDPPacketSize)
 	prefixLen := copy(buf, prefix)
@@ -173,10 +204,11 @@ func (ne *NaClEncryptor) Bytes() []byte {
 	nonce := ne.nonce
 	if nonce == nil {
 		freshNonce, encodedNonce, err := EncodeNonce(ne.df)
-		if err = ne.conn.CheckFatal(err); err != nil {
+		if err != nil {
+			ne.conn.Shutdown(err)
 			return []byte{}
 		}
-		ne.conn.SendTCP(encodedNonce)
+		ne.conn.SendProtocolMsg(ProtocolMsg{ProtocolNonce, encodedNonce})
 		ne.nonce = freshNonce
 		nonce = freshNonce
 	}
@@ -191,11 +223,12 @@ func (ne *NaClEncryptor) Bytes() []byte {
 		ne.nonce = <-ne.nonceChan
 	} else if offset == 1<<14 { // half way through range, send new nonce
 		nonce, encodedNonce, err := EncodeNonce(ne.df)
-		if err = ne.conn.CheckFatal(err); err != nil {
+		if err != nil {
+			ne.conn.Shutdown(err)
 			return []byte{}
 		}
 		ne.nonceChan <- nonce
-		ne.conn.SendTCP(encodedNonce)
+		ne.conn.SendProtocolMsg(ProtocolMsg{ProtocolNonce, encodedNonce})
 	}
 	ne.offset = offset
 
@@ -212,6 +245,38 @@ func (ne *NaClEncryptor) TotalLen() int {
 
 // Frame Decryptors
 
+type FrameConsumer func(*LocalConnection, *net.UDPAddr, []byte, []byte, uint16, []byte) error
+
+type Decryptor interface {
+	IterateFrames(FrameConsumer, *UDPPacket) error
+	ReceiveNonce([]byte)
+	Shutdown()
+}
+
+type NonDecryptor struct {
+	conn *LocalConnection
+}
+
+type NaClDecryptor struct {
+	NonDecryptor
+	instance   *NaClDecryptorInstance
+	instanceDF *NaClDecryptorInstance
+}
+
+type NaClDecryptorInstance struct {
+	nonce               *[24]byte
+	previousNonce       *[24]byte
+	usedOffsets         *bit.Set
+	previousUsedOffsets *bit.Set
+	highestOffsetSeen   uint16
+	nonceChan           chan *[24]byte
+}
+
+type PacketDecodingError struct {
+	Fatal bool
+	Desc  string
+}
+
 func NewNonDecryptor(conn *LocalConnection) *NonDecryptor {
 	return &NonDecryptor{conn: conn}
 }
@@ -320,8 +385,9 @@ func (nd *NaClDecryptor) decrypt(buf []byte) ([]byte, error) {
 		usedOffsets = decState.usedOffsets
 	} else {
 		highestOffsetSeen := decState.highestOffsetSeen
-		if offsetNoFlags < (1<<13) && highestOffsetSeen > ((1<<14)+(1<<13)) &&
-			(highestOffsetSeen-offsetNoFlags) > ((1<<14)+(1<<13)) {
+		switch {
+		case offsetNoFlags < (1<<13) && highestOffsetSeen > ((1<<14)+(1<<13)) &&
+			(highestOffsetSeen-offsetNoFlags) > ((1<<14)+(1<<13)):
 			// offset is in the first quarter, highestOffsetSeen is in
 			// the top quarter and under a quarter behind us. We
 			// interpret this as we need to move to the next nonce
@@ -335,29 +401,29 @@ func (nd *NaClDecryptor) decrypt(buf []byte) ([]byte, error) {
 			decState.highestOffsetSeen = offsetNoFlags
 			nonce = decState.nonce
 			usedOffsets = decState.usedOffsets
-		} else if offsetNoFlags > highestOffsetSeen &&
-			(offsetNoFlags-highestOffsetSeen) < (1<<13) {
+		case offsetNoFlags > highestOffsetSeen &&
+			(offsetNoFlags-highestOffsetSeen) < (1<<13):
 			// offset is under a quarter above highestOffsetSeen. This
 			// is ok - maybe some packet loss
 			decState.highestOffsetSeen = offsetNoFlags
 			nonce = decState.nonce
 			usedOffsets = decState.usedOffsets
-		} else if offsetNoFlags <= highestOffsetSeen &&
-			(highestOffsetSeen-offsetNoFlags) < (1<<13) {
+		case offsetNoFlags <= highestOffsetSeen &&
+			(highestOffsetSeen-offsetNoFlags) < (1<<13):
 			// offset is within a quarter of the highest we've
 			// seen. This is ok - just assuming some out-of-order
 			// delivery.
 			nonce = decState.nonce
 			usedOffsets = decState.usedOffsets
-		} else if highestOffsetSeen < (1<<13) && offsetNoFlags > ((1<<14)+(1<<13)) &&
-			(offsetNoFlags-highestOffsetSeen) > ((1<<14)+(1<<13)) {
+		case highestOffsetSeen < (1<<13) && offsetNoFlags > ((1<<14)+(1<<13)) &&
+			(offsetNoFlags-highestOffsetSeen) > ((1<<14)+(1<<13)):
 			// offset is in the last quarter, highestOffsetSeen is in
 			// the first quarter, and offset is under a quarter behind
 			// us. This is ok - as above, just some out of order. But
 			// here it means we're dealing with the previous nonce
 			nonce = decState.previousNonce
 			usedOffsets = decState.previousUsedOffsets
-		} else {
+		default:
 			return nil, fmt.Errorf("Unexpected offset when decrypting UDP packet")
 		}
 	}
@@ -377,6 +443,28 @@ func (nd *NaClDecryptor) decrypt(buf []byte) ([]byte, error) {
 
 // TCP Senders
 
+type TCPSender interface {
+	Send([]byte) error
+}
+
+type SimpleTCPSender struct {
+	encoder *gob.Encoder
+}
+
+type EncryptedTCPSender struct {
+	sync.RWMutex
+	outerEncoder *gob.Encoder
+	innerEncoder *gob.Encoder
+	buffer       *bytes.Buffer
+	conn         *LocalConnection
+	msgCount     int
+}
+
+type EncryptedTCPMessage struct {
+	Number int
+	Body   []byte
+}
+
 func NewSimpleTCPSender(encoder *gob.Encoder) *SimpleTCPSender {
 	return &SimpleTCPSender{encoder: encoder}
 }
@@ -416,6 +504,20 @@ func (sender *EncryptedTCPSender) Send(msg []byte) error {
 
 // TCP Receivers
 
+type TCPReceiver interface {
+	Decode([]byte) ([]byte, error)
+}
+
+type SimpleTCPReceiver struct {
+}
+
+type EncryptedTCPReceiver struct {
+	conn     *LocalConnection
+	decoder  *gob.Decoder
+	buffer   *bytes.Buffer
+	msgCount int
+}
+
 func NewSimpleTCPReceiver() *SimpleTCPReceiver {
 	return &SimpleTCPReceiver{}
 }

router/ethernet_decoder.go

@@ -8,6 +8,13 @@ import (
 	"net"
 )
 
+type EthernetDecoder struct {
+	eth     layers.Ethernet
+	ip      layers.IPv4
+	decoded []gopacket.LayerType
+	parser  *gopacket.DecodingLayerParser
+}
+
 func NewEthernetDecoder() *EthernetDecoder {
 	dec := &EthernetDecoder{}
 	dec.parser = gopacket.NewDecodingLayerParser(layers.LayerTypeEthernet, &dec.eth, &dec.ip)
@@ -75,7 +82,7 @@ func (dec *EthernetDecoder) DropFrame() bool {
 	return bytes.Equal(stpMACPrefix, dec.eth.DstMAC[:len(stpMACPrefix)])
 }
 
-func (dec *EthernetDecoder) IsPMTUVerify() bool {
-	return bytes.Equal(zeroMAC, dec.eth.SrcMAC) &&
-		bytes.Equal(zeroMAC, dec.eth.DstMAC)
+func (dec *EthernetDecoder) IsSpecial() bool {
+	return dec.eth.Length == 0 && dec.eth.EthernetType == layers.EthernetTypeLLC &&
+		bytes.Equal(zeroMAC, dec.eth.SrcMAC) && bytes.Equal(zeroMAC, dec.eth.DstMAC)
 }

router/forwarder.go

@@ -3,12 +3,20 @@ package router
 import (
 	"code.google.com/p/gopacket"
 	"code.google.com/p/gopacket/layers"
-	"log"
-	"net"
 	"syscall"
 	"time"
 )
 
+type ForwardedFrame struct {
+	srcPeer *Peer
+	dstPeer *Peer
+	frame   []byte
+}
+
+type FrameTooBigError struct {
+	EPMTU int // effective pmtu, i.e. what we tell packet senders
+}
+
 func (conn *LocalConnection) ensureForwarders() error {
 	if conn.forwardChan != nil || conn.forwardChanDF != nil {
 		return nil
@@ -52,8 +60,8 @@ func (conn *LocalConnection) ensureForwarders() error {
 	conn.effectivePMTU = forwarder.unverifiedPMTU
 	conn.Unlock()
 
-	go forwarder.Run()
-	go forwarderDF.Run()
+	forwarder.Start()
+	forwarderDF.Start()
 
 	return nil
 }
@@ -193,6 +201,22 @@ func fragment(eth layers.Ethernet, ip layers.IPv4, pmtu int, frame *ForwardedFra
 
 // Forwarder
 
+type Forwarder struct {
+	conn            *LocalConnection
+	ch              <-chan *ForwardedFrame
+	stop            <-chan interface{}
+	verifyPMTUTick  <-chan time.Time
+	verifyPMTU      <-chan int
+	pmtuVerifyCount uint
+	enc             Encryptor
+	udpSender       UDPSender
+	maxPayload      int
+	pmtuVerified    bool
+	highestGoodPMTU int
+	unverifiedPMTU  int
+	lowestBadPMTU   int
+}
+
 func NewForwarder(conn *LocalConnection, ch <-chan *ForwardedFrame, stop <-chan interface{}, verifyPMTU <-chan int, enc Encryptor, udpSender UDPSender, pmtu int) *Forwarder {
 	fwd := &Forwarder{
 		conn:       conn,
@@ -206,7 +230,11 @@ func NewForwarder(conn *LocalConnection, ch <-chan *ForwardedFrame, stop <-chan
 	return fwd
 }
 
-func (fwd *Forwarder) Run() {
+func (fwd *Forwarder) Start() {
+	go fwd.run()
+}
+
+func (fwd *Forwarder) run() {
 	defer fwd.udpSender.Shutdown()
 	var flushed, ok bool
 	var frame *ForwardedFrame
@@ -321,7 +349,7 @@ func (fwd *Forwarder) flush() {
 		} else if PosixError(err) == syscall.ENOBUFS {
 			// TODO handle this better
 		} else {
-			fwd.conn.CheckFatal(err)
+			fwd.conn.Shutdown(err)
 		}
 	}
 }
@@ -342,112 +370,3 @@ func (fwd *Forwarder) drain() {
 func (fwd *Forwarder) logDrop(frame *ForwardedFrame) {
 	fwd.conn.log("Dropping too big frame during forwarding: frame len:", len(frame.frame), "; effective PMTU:", fwd.maxPayload+UDPOverhead-fwd.effectiveOverhead())
 }
-
-// UDP Senders
-
-func NewSimpleUDPSender(conn *LocalConnection) *SimpleUDPSender {
-	return &SimpleUDPSender{udpConn: conn.Router.UDPListener, conn: conn}
-}
-
-func (sender *SimpleUDPSender) Send(msg []byte) error {
-	_, err := sender.udpConn.WriteToUDP(msg, sender.conn.RemoteUDPAddr())
-	return err
-}
-
-func (sender *SimpleUDPSender) Shutdown() error {
-	return nil
-}
-
-func NewRawUDPSender(conn *LocalConnection) (*RawUDPSender, error) {
-	ipSocket, err := dialIP(conn)
-	if err != nil {
-		return nil, err
-	}
-	udpHeader := &layers.UDP{SrcPort: layers.UDPPort(Port)}
-	ipBuf := gopacket.NewSerializeBuffer()
-	opts := gopacket.SerializeOptions{
-		FixLengths: true,
-		// UDP header is calculated with a phantom IP
-		// header. Yes, it's totally nuts. Thankfully, for UDP
-		// over IPv4, the checksum is optional. It's not
-		// optional for IPv6, but we'll ignore that for
-		// now. TODO
-		ComputeChecksums: false}
-
-	return &RawUDPSender{
-		ipBuf:     ipBuf,
-		opts:      opts,
-		udpHeader: udpHeader,
-		socket:    ipSocket,
-		conn:      conn}, nil
-}
-
-func (sender *RawUDPSender) Send(msg []byte) error {
-	payload := gopacket.Payload(msg)
-	sender.udpHeader.DstPort = layers.UDPPort(sender.conn.RemoteUDPAddr().Port)
-
-	err := gopacket.SerializeLayers(sender.ipBuf, sender.opts, sender.udpHeader, &payload)
-	if err != nil {
-		return err
-	}
-	packet := sender.ipBuf.Bytes()
-	_, err = sender.socket.Write(packet)
-	if err == nil || PosixError(err) != syscall.EMSGSIZE {
-		return err
-	}
-	f, err := sender.socket.File()
-	if err != nil {
-		return err
-	}
-	defer f.Close()
-	fd := int(f.Fd())
-	log.Println("EMSGSIZE on send, expecting PMTU update (IP packet was",
-		len(packet), "bytes, payload was", len(msg), "bytes)")
-	pmtu, err := syscall.GetsockoptInt(fd, syscall.IPPROTO_IP, syscall.IP_MTU)
-	if err != nil {
-		return err
-	}
-	return MsgTooBigError{PMTU: pmtu}
-}
-
-func (sender *RawUDPSender) Shutdown() error {
-	defer func() { sender.socket = nil }()
-	return sender.socket.Close()
-}
-
-func dialIP(conn *LocalConnection) (*net.IPConn, error) {
-	ipLocalAddr, err := ipAddr(conn.TCPConn.LocalAddr())
-	if err != nil {
-		return nil, err
-	}
-	ipRemoteAddr, err := ipAddr(conn.TCPConn.RemoteAddr())
-	if err != nil {
-		return nil, err
-	}
-	ipSocket, err := net.DialIP("ip4:UDP", ipLocalAddr, ipRemoteAddr)
-	if err != nil {
-		return nil, err
-	}
-	f, err := ipSocket.File()
-	if err != nil {
-		return nil, err
-	}
-	defer f.Close()
-	fd := int(f.Fd())
-	// This Makes sure all packets we send out have DF set on them.
-	err = syscall.SetsockoptInt(fd, syscall.IPPROTO_IP, syscall.IP_MTU_DISCOVER, syscall.IP_PMTUDISC_DO)
-	if err != nil {
-		return nil, err
-	}
-	return ipSocket, nil
-}
-
-func ipAddr(addr net.Addr) (*net.IPAddr, error) {
-	host, _, err := net.SplitHostPort(addr.String())
-	if err != nil {
-		return nil, err
-	}
-	return &net.IPAddr{
-		IP:   net.ParseIP(host),
-		Zone: ""}, nil
-}

router/gossip.go

@@ -0,0 +1,162 @@
+package router
+
+import (
+	"bytes"
+	"encoding/gob"
+	"fmt"
+	"log"
+	"time"
+)
+
+const GossipInterval = 30 * time.Second
+
+type Gossip interface {
+	// specific message from one peer to another
+	// intermediate peers relay it using unicast topology.
+	GossipUnicast(dstPeerName PeerName, buf []byte) error
+	// send a message to every peer, relayed using broadcast topology.
+	GossipBroadcast(buf []byte) error
+}
+
+type Gossiper interface {
+	OnGossipUnicast(sender PeerName, msg []byte) error
+	OnGossipBroadcast(msg []byte) error
+	// Return state of everything we know; gets called periodically
+	Gossip() []byte
+	// merge in state and return "everything new I've just learnt",
+	// or nil if nothing in the received message was new
+	OnGossip(buf []byte) ([]byte, error)
+}
+
+type GossipChannel struct {
+	ourself  *LocalPeer
+	name     string
+	hash     uint32
+	gossiper Gossiper
+}
+
+func (router *Router) NewGossip(channelName string, g Gossiper) Gossip {
+	channelHash := hash(channelName)
+	channel := &GossipChannel{router.Ourself, channelName, channelHash, g}
+	router.GossipChannels[channelHash] = channel
+	return channel
+}
+
+func (router *Router) SendAllGossip() {
+	for _, channel := range router.GossipChannels {
+		channel.SendGossipMsg(channel.gossiper.Gossip())
+	}
+}
+
+func (router *Router) SendAllGossipDown(conn Connection) {
+	for _, channel := range router.GossipChannels {
+		protocolMsg := channel.gossipMsg(channel.gossiper.Gossip())
+		conn.(ProtocolSender).SendProtocolMsg(protocolMsg)
+	}
+}
+
+func (router *Router) handleGossip(payload []byte, onok func(*GossipChannel, PeerName, []byte, *gob.Decoder) error) error {
+	decoder := gob.NewDecoder(bytes.NewReader(payload))
+	var channelHash uint32
+	if err := decoder.Decode(&channelHash); err != nil {
+		return err
+	}
+	channel, found := router.GossipChannels[channelHash]
+	if !found {
+		return fmt.Errorf("[gossip] received unknown channel with hash %v", channelHash)
+	}
+	var srcName PeerName
+	if err := decoder.Decode(&srcName); err != nil {
+		return err
+	}
+	if err := onok(channel, srcName, payload, decoder); err != nil {
+		return err
+	}
+	return nil
+}
+
+func deliverGossipUnicast(channel *GossipChannel, srcName PeerName, origPayload []byte, dec *gob.Decoder) error {
+	var destName PeerName
+	if err := dec.Decode(&destName); err != nil {
+		return err
+	}
+	if channel.ourself.Name == destName {
+		var payload []byte
+		if err := dec.Decode(&payload); err != nil {
+			return err
+		}
+		return channel.gossiper.OnGossipUnicast(srcName, payload)
+	} else {
+		return channel.relayGossipUnicast(destName, origPayload)
+	}
+}
+
+func deliverGossipBroadcast(channel *GossipChannel, srcName PeerName, origPayload []byte, dec *gob.Decoder) error {
+	var payload []byte
+	if err := dec.Decode(&payload); err != nil {
+		return err
+	}
+	if err := channel.gossiper.OnGossipBroadcast(payload); err != nil {
+		return err
+	}
+	return channel.relayGossipBroadcast(srcName, origPayload)
+}
+
+func deliverGossip(channel *GossipChannel, srcName PeerName, _ []byte, dec *gob.Decoder) error {
+	var payload []byte
+	if err := dec.Decode(&payload); err != nil {
+		return err
+	}
+	if newBuf, err := channel.gossiper.OnGossip(payload); err != nil {
+		return err
+	} else if newBuf != nil {
+		channel.SendGossipMsg(newBuf)
+	}
+	return nil
+}
+
+func (c *GossipChannel) SendGossipMsg(buf []byte) {
+	protocolMsg := c.gossipMsg(buf)
+	c.ourself.ForEachConnection(func(_ PeerName, conn Connection) {
+		conn.(ProtocolSender).SendProtocolMsg(protocolMsg)
+	})
+}
+
+func (c *GossipChannel) gossipMsg(buf []byte) ProtocolMsg {
+	return ProtocolMsg{ProtocolGossip, GobEncode(c.hash, c.ourself.Name, buf)}
+}
+
+func (c *GossipChannel) GossipUnicast(dstPeerName PeerName, buf []byte) error {
+	return c.relayGossipUnicast(dstPeerName, GobEncode(c.hash, c.ourself.Name, dstPeerName, buf))
+}
+
+func (c *GossipChannel) GossipBroadcast(buf []byte) error {
+	return c.relayGossipBroadcast(c.ourself.Name, GobEncode(c.hash, c.ourself.Name, buf))
+}
+
+func (c *GossipChannel) relayGossipUnicast(dstPeerName PeerName, msg []byte) error {
+	if relayPeerName, found := c.ourself.Router.Routes.Unicast(dstPeerName); !found {
+		c.log("unknown relay destination:", dstPeerName)
+	} else if conn, found := c.ourself.ConnectionTo(relayPeerName); !found {
+		c.log("unable to find connection to relay peer", relayPeerName)
+	} else {
+		conn.(ProtocolSender).SendProtocolMsg(ProtocolMsg{ProtocolGossipUnicast, msg})
+	}
+	return nil
+}
+
+func (c *GossipChannel) relayGossipBroadcast(srcName PeerName, msg []byte) error {
+	if srcPeer, found := c.ourself.Router.Peers.Fetch(srcName); !found {
+		c.log("unable to relay broadcast from unknown peer", srcName)
+	} else {
+		protocolMsg := ProtocolMsg{ProtocolGossipBroadcast, msg}
+		for _, conn := range c.ourself.NextBroadcastHops(srcPeer) {
+			conn.SendProtocolMsg(protocolMsg)
+		}
+	}
+	return nil
+}
+
+func (c *GossipChannel) log(args ...interface{}) {
+	log.Println(append(append([]interface{}{}, "[gossip "+c.name+"]:"), args...)...)
+}

router/gossip_test.go

@@ -0,0 +1,146 @@
+package router
+
+import (
+	wt "github.com/zettio/weave/testing"
+	"testing"
+	"time"
+)
+
+// TODO test gossip unicast and broadcast; atm we only test topology
+// gossip, which does not employ unicast or broadcast.
+
+type mockChannelConnection struct {
+	RemoteConnection
+	dest *Router
+}
+
+// Construct a "passive" Router, i.e. without any goroutines.
+//
+// We need to create some dummy channels otherwise tests hang on nil
+// channels when Router.OnGossip() calls async methods.
+func NewTestRouter(name PeerName) *Router {
+	router := NewRouter(nil, name, nil, 10, 1024, nil)
+	router.ConnectionMaker.queryChan = make(chan *ConnectionMakerInteraction, ChannelSize)
+	router.Routes.queryChan = make(chan *Interaction, ChannelSize)
+	return router
+}
+
+func (conn *mockChannelConnection) SendProtocolMsg(protocolMsg ProtocolMsg) {
+	if err := conn.dest.handleGossip(protocolMsg.msg, deliverGossip); err != nil {
+		panic(err)
+	}
+}
+
+func (router *Router) AddTestChannelConnection(r *Router) {
+	fromName := router.Ourself.Peer.Name
+	toName := r.Ourself.Peer.Name
+
+	fromPeer := NewPeer(fromName, router.Ourself.Peer.UID, 0)
+	toPeer := NewPeer(toName, r.Ourself.Peer.UID, 0)
+
+	r.Peers.FetchWithDefault(fromPeer)    // Has side-effect of incrementing refcount
+	router.Peers.FetchWithDefault(toPeer) //
+
+	conn := &mockChannelConnection{RemoteConnection{router.Ourself.Peer, toPeer, "", false}, r}
+	router.Ourself.handleAddConnection(conn)
+	router.Ourself.handleConnectionEstablished(conn)
+}
+
+func (router *Router) DeleteTestChannelConnection(r *Router) {
+	fromName := router.Ourself.Peer.Name
+	toName := r.Ourself.Peer.Name
+
+	fromPeer, _ := r.Peers.Fetch(fromName)
+	toPeer, _ := router.Peers.Fetch(toName)
+
+	fromPeer.DecrementLocalRefCount()
+	toPeer.DecrementLocalRefCount()
+
+	conn, _ := router.Ourself.ConnectionTo(toName)
+	router.Ourself.handleDeleteConnection(conn)
+}
+
+func TestGossipTopology(t *testing.T) {
+	wt.RunWithTimeout(t, 1*time.Second, func() {
+		implTestGossipTopology(t)
+	})
+}
+
+// Create a Peer representing the receiver router, with connections to
+// the routers supplied as arguments, carrying across all UID and
+// version information.
+func (router *Router) tp(routers ...*Router) *Peer {
+	peer := NewPeer(router.Ourself.Peer.Name, router.Ourself.Peer.UID, 0)
+	connections := make(map[PeerName]Connection)
+	for _, r := range routers {
+		p := NewPeer(r.Ourself.Peer.Name, r.Ourself.Peer.UID, r.Ourself.Peer.version)
+		connections[r.Ourself.Peer.Name] = newMockConnection(peer, p)
+	}
+	peer.SetVersionAndConnections(router.Ourself.Peer.version, connections)
+	return peer
+}
+
+// Check that the topology of router matches the peers and all of their connections
+func checkTopology(t *testing.T, router *Router, wantedPeers ...*Peer) {
+	checkTopologyPeers(t, true, router.Peers.allPeers(), wantedPeers...)
+}
+
+func implTestGossipTopology(t *testing.T) {
+	// Create some peers that will talk to each other
+	peer1Name, _ := PeerNameFromString("01:00:00:01:00:00")
+	peer2Name, _ := PeerNameFromString("02:00:00:02:00:00")
+	peer3Name, _ := PeerNameFromString("03:00:00:03:00:00")
+	r1 := NewTestRouter(peer1Name)
+	r2 := NewTestRouter(peer2Name)
+	r3 := NewTestRouter(peer3Name)
+
+	// Check state when they have no connections
+	checkTopology(t, r1, r1.tp())
+	checkTopology(t, r2, r2.tp())
+
+	// Now try adding some connections
+	r1.AddTestChannelConnection(r2)
+	checkTopology(t, r1, r1.tp(r2), r2.tp())
+	checkTopology(t, r2, r1.tp(r2), r2.tp())
+	r2.AddTestChannelConnection(r1)
+	checkTopology(t, r1, r1.tp(r2), r2.tp(r1))
+	checkTopology(t, r2, r1.tp(r2), r2.tp(r1))
+
+	// Currently, the connection from 2 to 3 is one-way only
+	r2.AddTestChannelConnection(r3)
+	checkTopology(t, r1, r1.tp(r2), r2.tp(r1, r3), r3.tp())
+	checkTopology(t, r2, r1.tp(r2), r2.tp(r1, r3), r3.tp())
+	// When r2 gossiped to r3, 1 was unreachable from r3 so it got removed from the
+	// list of peers, but remains referenced in the connection from 1 to 3.
+	checkTopology(t, r3, r2.tp(r1, r3), r3.tp())
+
+	// Add a connection from 3 to 1 and now r1 is reachable.
+	r3.AddTestChannelConnection(r1)
+	checkTopology(t, r1, r1.tp(r2), r2.tp(r1, r3), r3.tp(r1))
+	checkTopology(t, r2, r1.tp(r2), r2.tp(r1, r3), r3.tp(r1))
+	checkTopology(t, r3, r1.tp(), r2.tp(r1, r3), r3.tp(r1))
+
+	r1.AddTestChannelConnection(r3)
+	checkTopology(t, r1, r1.tp(r2, r3), r2.tp(r1, r3), r3.tp(r1))
+	checkTopology(t, r2, r1.tp(r2, r3), r2.tp(r1, r3), r3.tp(r1))
+	checkTopology(t, r3, r1.tp(r2, r3), r2.tp(r1, r3), r3.tp(r1))
+
+	// Drop the connection from 2 to 3
+	r2.DeleteTestChannelConnection(r3)
+	checkTopology(t, r1, r1.tp(r2, r3), r2.tp(r1), r3.tp(r1))
+	checkTopology(t, r2, r1.tp(r2, r3), r2.tp(r1))
+	checkTopology(t, r3, r1.tp(r2, r3), r2.tp(r1), r3.tp(r1))
+
+	// Drop the connection from 1 to 3
+	r1.DeleteTestChannelConnection(r3)
+	checkTopology(t, r1, r1.tp(r2), r2.tp(r1), r3.tp(r1))
+
+	checkTopology(t, r1, r1.tp(r2), r2.tp(r1), r3.tp(r1))
+	checkTopology(t, r2, r1.tp(r2), r2.tp(r1))
+	// r3 still thinks r1 has a connection to it
+	checkTopology(t, r3, r1.tp(r2, r3), r2.tp(r1), r3.tp(r1))
+
+	// On a timer, r3 will gossip to r1
+	r3.SendAllGossip()
+	checkTopology(t, r1, r1.tp(r2), r2.tp(r1), r3.tp(r1))
+}

router/local_peer.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"log"
 	"net"
+	"time"
 )
 
 type LocalPeer struct {
@@ -17,12 +18,14 @@ type PeerInteraction struct {
 	payload interface{}
 }
 
-func StartLocalPeer(name PeerName, router *Router) *LocalPeer {
-	peer := &LocalPeer{Peer: NewPeer(name, 0, 0), Router: router}
+func NewLocalPeer(name PeerName, router *Router) *LocalPeer {
+	return &LocalPeer{Peer: NewPeer(name, 0, 0), Router: router}
+}
+
+func (peer *LocalPeer) Start() {
 	queryChan := make(chan *PeerInteraction, ChannelSize)
 	peer.queryChan = queryChan
 	go peer.queryLoop(queryChan)
-	return peer
 }
 
 func (peer *LocalPeer) Forward(dstPeer *Peer, df bool, frame []byte, dec *EthernetDecoder) error {
@@ -55,26 +58,8 @@ func (peer *LocalPeer) Relay(srcPeer, dstPeer *Peer, df bool, frame []byte, dec
 }
 
 func (peer *LocalPeer) RelayBroadcast(srcPeer *Peer, df bool, frame []byte, dec *EthernetDecoder) error {
-	nextHops := peer.Router.Routes.Broadcast(srcPeer.Name)
-	if len(nextHops) == 0 {
-		return nil
-	}
-	// We must not hold a read lock on peer during the conn.Forward
-	// below, since that is a potentially blocking operation (e.g. if
-	// the channel is full).
-	nextConns := make([]*LocalConnection, 0, len(nextHops))
-	peer.RLock()
-	for _, hopName := range nextHops {
-		conn, found := peer.connections[hopName]
-		// Again, !found could just be due to a race.
-		if found {
-			nextConns = append(nextConns, conn.(*LocalConnection))
-		}
-	}
-	peer.RUnlock()
-	var err error
-	for _, conn := range nextConns {
-		err = conn.Forward(df, &ForwardedFrame{
+	for _, conn := range peer.NextBroadcastHops(srcPeer) {
+		err := conn.Forward(df, &ForwardedFrame{
 			srcPeer: srcPeer,
 			dstPeer: conn.Remote(),
 			frame:   frame},
@@ -86,6 +71,24 @@ func (peer *LocalPeer) RelayBroadcast(srcPeer *Peer, df bool, frame []byte, dec
 	return nil
 }
 
+func (peer *LocalPeer) NextBroadcastHops(srcPeer *Peer) []*LocalConnection {
+	nextHops := peer.Router.Routes.Broadcast(srcPeer.Name)
+	if len(nextHops) == 0 {
+		return nil
+	}
+	nextConns := make([]*LocalConnection, 0, len(nextHops))
+	peer.RLock()
+	defer peer.RUnlock()
+	for _, hopName := range nextHops {
+		conn, found := peer.connections[hopName]
+		// Again, !found could just be due to a race.
+		if found {
+			nextConns = append(nextConns, conn.(*LocalConnection))
+		}
+	}
+	return nextConns
+}
+
 func (peer *LocalPeer) CreateConnection(peerAddr string, acceptNewPeer bool) error {
 	if err := peer.checkConnectionLimit(); err != nil {
 		return err
@@ -105,25 +108,34 @@ func (peer *LocalPeer) CreateConnection(peerAddr string, acceptNewPeer bool) err
 	if err != nil {
 		return err
 	}
-	connRemote := NewRemoteConnection(peer.Peer, nil, tcpConn.RemoteAddr().String())
-	NewLocalConnection(connRemote, acceptNewPeer, tcpConn, udpAddr, peer.Router)
+	connRemote := NewRemoteConnection(peer.Peer, nil, tcpConn.RemoteAddr().String(), false)
+	connLocal := NewLocalConnection(connRemote, tcpConn, udpAddr, peer.Router)
+	connLocal.Start(acceptNewPeer)
 	return nil
 }
 
 // ACTOR client API
 
 const (
-	PAddConnection         = iota
-	PBroadcastTCP          = iota
-	PDeleteConnection      = iota
-	PConnectionEstablished = iota
+	PAddConnection = iota
+	PConnectionEstablished
+	PDeleteConnection
+	PSendProtocolMsg
 )
 
-// Async: rely on the peer to shut us down if we shouldn't be adding
-// ourselves, so therefore this can be async
+// Sync.
 func (peer *LocalPeer) AddConnection(conn *LocalConnection) {
+	resultChan := make(chan interface{})
 	peer.queryChan <- &PeerInteraction{
-		Interaction: Interaction{code: PAddConnection},
+		Interaction: Interaction{code: PAddConnection, resultChan: resultChan},
+		payload:     conn}
+	<-resultChan
+}
+
+// Async.
+func (peer *LocalPeer) ConnectionEstablished(conn *LocalConnection) {
+	peer.queryChan <- &PeerInteraction{
+		Interaction: Interaction{code: PConnectionEstablished},
 		payload:     conn}
 }
 
@@ -137,42 +149,50 @@ func (peer *LocalPeer) DeleteConnection(conn *LocalConnection) {
 }
 
 // Async.
-func (peer *LocalPeer) ConnectionEstablished(conn *LocalConnection) {
+func (peer *LocalPeer) SendProtocolMsg(m ProtocolMsg) {
 	peer.queryChan <- &PeerInteraction{
-		Interaction: Interaction{code: PConnectionEstablished},
-		payload:     conn}
-}
-
-// Async.
-func (peer *LocalPeer) BroadcastTCP(msg []byte) {
-	peer.queryChan <- &PeerInteraction{
-		Interaction: Interaction{code: PBroadcastTCP},
-		payload:     msg}
+		Interaction: Interaction{code: PSendProtocolMsg},
+		payload:     m}
 }
 
 // ACTOR server
 
 func (peer *LocalPeer) queryLoop(queryChan <-chan *PeerInteraction) {
+	gossipTimer := time.Tick(GossipInterval)
 	for {
-		query, ok := <-queryChan
-		if !ok {
-			return
-		}
-		switch query.code {
-		case PAddConnection:
-			peer.handleAddConnection(query.payload.(*LocalConnection))
-		case PDeleteConnection:
-			peer.handleDeleteConnection(query.payload.(*LocalConnection))
-			query.resultChan <- nil
-		case PConnectionEstablished:
-			peer.handleConnectionEstablished(query.payload.(*LocalConnection))
-		case PBroadcastTCP:
-			peer.handleBroadcastTCP(query.payload.([]byte))
+		select {
+		case query, ok := <-queryChan:
+			if !ok {
+				return
+			}
+			switch query.code {
+			case PAddConnection:
+				conn := query.payload.(*LocalConnection)
+				if peer.handleAddConnection(conn) {
+					conn.log("connection added")
+				}
+				query.resultChan <- nil
+			case PConnectionEstablished:
+				conn := query.payload.(*LocalConnection)
+				if peer.handleConnectionEstablished(conn) {
+					conn.log("connection fully established")
+				}
+			case PDeleteConnection:
+				conn := query.payload.(*LocalConnection)
+				if peer.handleDeleteConnection(conn) {
+					conn.log("connection deleted")
+				}
+				query.resultChan <- nil
+			case PSendProtocolMsg:
+				peer.handleSendProtocolMsg(query.payload.(ProtocolMsg))
+			}
+		case <-gossipTimer:
+			peer.Router.SendAllGossip()
 		}
 	}
 }
 
-func (peer *LocalPeer) handleAddConnection(conn *LocalConnection) {
+func (peer *LocalPeer) handleAddConnection(conn Connection) bool {
 	if peer.Peer != conn.Local() {
 		log.Fatal("Attempt made to add connection to peer where peer is not the source of connection")
 	}
@@ -184,35 +204,47 @@ func (peer *LocalPeer) handleAddConnection(conn *LocalConnection) {
 	// deliberately non symmetrical
 	if dupConn, found := peer.connections[toName]; found {
 		if dupConn == conn {
-			return
+			return false
 		}
-		// conn.UID is used as the tie breaker here, in the
-		// knowledge that both sides will make the same decision.
-		dupConnLocal := dupConn.(*LocalConnection)
-		if conn.UID == dupConnLocal.UID {
+		switch conn.BreakTie(dupConn) {
+		case TieBreakWon:
+			dupConn.Shutdown(dupErr)
+			peer.handleDeleteConnection(dupConn)
+		case TieBreakLost:
+			conn.Shutdown(dupErr)
+			return false
+		case TieBreakTied:
 			// oh good grief. Sod it, just kill both of them.
-			conn.CheckFatal(dupErr)
-			dupConnLocal.CheckFatal(dupErr)
-			peer.handleDeleteConnection(dupConnLocal)
-			return
-		} else if conn.UID < dupConnLocal.UID {
-			dupConnLocal.CheckFatal(dupErr)
-			peer.handleDeleteConnection(dupConnLocal)
-		} else {
-			conn.CheckFatal(dupErr)
-			return
+			conn.Shutdown(dupErr)
+			dupConn.Shutdown(dupErr)
+			peer.handleDeleteConnection(dupConn)
+			return false
 		}
 	}
 	if err := peer.checkConnectionLimit(); err != nil {
-		conn.CheckFatal(err)
-		return
+		conn.Shutdown(err)
+		return false
 	}
-	peer.Lock()
-	peer.connections[toName] = conn
-	peer.Unlock()
+	peer.addConnection(conn)
+	peer.Router.SendAllGossipDown(conn)
+	peer.broadcastPeerUpdate(conn.Remote())
+	return true
 }
 
-func (peer *LocalPeer) handleDeleteConnection(conn *LocalConnection) {
+func (peer *LocalPeer) handleConnectionEstablished(conn Connection) bool {
+	if peer.Peer != conn.Local() {
+		log.Fatal("Peer informed of active connection where peer is not the source of connection")
+	}
+	if dupConn, found := peer.connections[conn.Remote().Name]; !found || conn != dupConn {
+		conn.Shutdown(fmt.Errorf("Cannot set unknown connection active"))
+		return false
+	}
+	peer.connectionEstablished(conn)
+	peer.broadcastPeerUpdate()
+	return true
+}
+
+func (peer *LocalPeer) handleDeleteConnection(conn Connection) bool {
 	if peer.Peer != conn.Local() {
 		log.Fatal("Attempt made to delete connection from peer where peer is not the source of connection")
 	}
@@ -221,50 +253,32 @@ func (peer *LocalPeer) handleDeleteConnection(conn *LocalConnection) {
 	}
 	toName := conn.Remote().Name
 	if connFound, found := peer.connections[toName]; !found || connFound != conn {
-		return
-	}
-	peer.Lock()
-	delete(peer.connections, toName)
-	peer.Unlock()
-	broadcast := false
-	if conn.Established() {
-		peer.Lock()
-		peer.version += 1
-		peer.Unlock()
-		broadcast = true
+		return false
 	}
+	peer.deleteConnection(conn)
 	// Must do garbage collection first to ensure we don't send out an
 	// update with unreachable peers (can cause looping)
 	peer.Router.Peers.GarbageCollect()
-	if broadcast {
-		peer.broadcastPeerUpdate()
-	}
+	peer.broadcastPeerUpdate()
+	return true
 }
 
-func (peer *LocalPeer) handleConnectionEstablished(conn *LocalConnection) {
-	if peer.Peer != conn.Local() {
-		log.Fatal("Peer informed of active connection where peer is not the source of connection")
-	}
-	if dupConn, found := peer.connections[conn.Remote().Name]; !found || conn != dupConn {
-		conn.CheckFatal(fmt.Errorf("Cannot set unknown connection active"))
-		return
-	}
-	peer.Lock()
-	peer.version += 1
-	peer.Unlock()
-	log.Println("Peer", peer.Name, "established active connection to remote peer", conn.Remote().Name, "at", conn.RemoteTCPAddr())
-	peer.broadcastPeerUpdate(conn.Remote())
-}
-
-func (peer *LocalPeer) handleBroadcastTCP(msg []byte) {
+func (peer *LocalPeer) handleSendProtocolMsg(m ProtocolMsg) {
 	peer.ForEachConnection(func(_ PeerName, conn Connection) {
-		conn.(*LocalConnection).SendTCP(msg)
+		conn.(ProtocolSender).SendProtocolMsg(m)
 	})
 }
 
 func (peer *LocalPeer) broadcastPeerUpdate(peers ...*Peer) {
 	peer.Router.Routes.Recalculate()
-	peer.handleBroadcastTCP(Concat(ProtocolUpdateByte, EncodePeers(append(peers, peer.Peer)...)))
+	// TODO We should just be invoking TopologyGossip.GossipBroadcast
+	// here, but route calculation is asynchronous and in this
+	// particular case would likely result in the broadcast not
+	// reaching all peers. So instead we slightly break the Gossip
+	// abstraction (hence the cast) and send a regular update. This is
+	// less efficient though since it will almost certainly reach
+	// peers more than once.
+	peer.Router.TopologyGossip.(*GossipChannel).SendGossipMsg(EncodePeers(append(peers, peer.Peer)...))
 }
 
 func (peer *LocalPeer) checkConnectionLimit() error {

router/mac_cache.go

@@ -22,12 +22,14 @@ type MacCache struct {
 }
 
 func NewMacCache(maxAge time.Duration, onExpiry func(net.HardwareAddr, *Peer)) *MacCache {
-	res := &MacCache{
+	return &MacCache{
 		table:    make(map[uint64]*MacCacheEntry),
 		maxAge:   maxAge,
 		onExpiry: onExpiry}
-	res.setExpiryTimer()
-	return res
+}
+
+func (cache *MacCache) Start() {
+	cache.setExpiryTimer()
 }
 
 func (cache *MacCache) Enter(mac net.HardwareAddr, peer *Peer) bool {

router/mocks_test.go

@@ -0,0 +1,113 @@
+// No mocks are tested by this file.
+//
+// It supplies some mock implementations to other unit tests, and is
+// named "...test.go" so it is only compiled under `go test`.
+
+package router
+
+import (
+	wt "github.com/zettio/weave/testing"
+	"testing"
+)
+
+// Add to peers a connection from peers.ourself to p
+func (peers *Peers) AddTestConnection(p *Peer) {
+	toPeer := NewPeer(p.Name, p.UID, 0)
+	peers.FetchWithDefault(toPeer) // Has side-effect of incrementing refcount
+	conn := newMockConnection(peers.ourself, toPeer)
+	peers.ourself.addConnection(conn)
+	peers.ourself.connectionEstablished(conn)
+}
+
+// Add to peers a connection from p1 to p2
+func (peers *Peers) AddTestRemoteConnection(p1, p2 *Peer) {
+	fromName := p1.Name
+	fromPeer := NewPeer(fromName, p1.UID, 0)
+	fromPeer = peers.FetchWithDefault(fromPeer)
+	toName := p2.Name
+	toPeer := NewPeer(toName, p2.UID, 0)
+	toPeer = peers.FetchWithDefault(toPeer)
+	peers.ourself.addConnection(&RemoteConnection{fromPeer, toPeer, "", false})
+}
+
+func (peers *Peers) DeleteTestConnection(p *Peer) {
+	toName := p.Name
+	toPeer, _ := peers.Fetch(toName)
+	toPeer.DecrementLocalRefCount()
+	conn, _ := peers.ourself.ConnectionTo(toName)
+	peers.ourself.deleteConnection(conn)
+}
+
+// mockConnection used in testing is very similar to a
+// RemoteConnection, without the RemoteTCPAddr(). We are making it a
+// separate type in order to distinguish what is created by the test
+// from what is created by the real code.
+func newMockConnection(from, to *Peer) Connection {
+	type mockConnection struct{ RemoteConnection }
+	return &mockConnection{RemoteConnection{from, to, "", false}}
+}
+
+func checkEqualConns(t *testing.T, ourName PeerName, got, wanted map[PeerName]Connection) {
+	checkConns := make(map[PeerName]bool)
+	for _, conn := range wanted {
+		checkConns[conn.Remote().Name] = true
+	}
+	for _, conn := range got {
+		remoteName := conn.Remote().Name
+		if _, found := checkConns[remoteName]; found {
+			delete(checkConns, remoteName)
+		} else {
+			wt.Fatalf(t, "Unexpected connection from %s to %s", ourName, remoteName)
+		}
+	}
+	if len(checkConns) > 0 {
+		wt.Fatalf(t, "Expected connections not found: from %s to %v", ourName, checkConns)
+	}
+}
+
+// Get all the peers from a Peers in a slice
+func (peers *Peers) allPeers() []*Peer {
+	res := make([]*Peer, 0)
+	for _, peer := range peers.table {
+		res = append(res, peer)
+	}
+	return res
+}
+
+func (peers *Peers) allPeersExcept(excludeName PeerName) []*Peer {
+	res := peers.allPeers()
+	for i, peer := range res {
+		if peer.Name == excludeName {
+			return append(res[:i], res[i+1:]...)
+		}
+	}
+	return res
+}
+
+// Check that the peers slice matches the wanted peers
+func checkPeerArray(t *testing.T, peers []*Peer, wantedPeers ...*Peer) {
+	checkTopologyPeers(t, false, peers, wantedPeers...)
+}
+
+// Check that the peers slice matches the wanted peers and optionally
+// all of their connections
+func checkTopologyPeers(t *testing.T, checkConns bool, peers []*Peer, wantedPeers ...*Peer) {
+	check := make(map[PeerName]*Peer)
+	for _, peer := range wantedPeers {
+		check[peer.Name] = peer
+	}
+	for _, peer := range peers {
+		name := peer.Name
+		if wantedPeer, found := check[name]; found {
+			if checkConns {
+				checkEqualConns(t, name, peer.connections, wantedPeer.connections)
+			}
+			delete(check, name)
+		} else {
+			wt.Fatalf(t, "Unexpected peer: %s", name)
+		}
+	}
+	if len(check) > 0 {
+		wt.Fatalf(t, "Expected peers not found: %v", check)
+	}
+}

router/peer.go

@@ -86,6 +86,26 @@ func (peer *Peer) SetVersionAndConnections(version uint64, connections map[PeerN
 	peer.connections = connections
 }
 
+func (peer *Peer) addConnection(conn Connection) {
+	peer.Lock()
+	defer peer.Unlock()
+	peer.connections[conn.Remote().Name] = conn
+	peer.version += 1
+}
+
+func (peer *Peer) deleteConnection(conn Connection) {
+	peer.Lock()
+	defer peer.Unlock()
+	delete(peer.connections, conn.Remote().Name)
+	peer.version += 1
+}
+
+func (peer *Peer) connectionEstablished(conn Connection) {
+	peer.Lock()
+	defer peer.Unlock()
+	peer.version += 1
+}
+
 // Calculate the routing table from this peer to all peers reachable
 // from it, returning a "next hop" map of PeerNameX -> PeerNameY,
 // which says "in order to send a message to X, the peer should send
@@ -99,9 +119,9 @@ func (peer *Peer) SetVersionAndConnections(version uint64, connections map[PeerN
 // same result. This is important since otherwise we risk message loss
 // or routing cycles.
 //
-// When the 'symmetric' flag is set, only symmetric connections are
-// considered, i.e. where both sides indicate they have a connection
-// to the other.
+// When the 'establishedAndSymmetric' flag is set, only connections
+// that are marked as 'established' and are symmetric (i.e. where both
+// sides indicate they have a connection to the other) are considered.
 //
 // When a non-nil stopAt peer is supplied, the widening stops when it
 // reaches that peer. The boolean return indicates whether that has
@@ -111,7 +131,7 @@ func (peer *Peer) SetVersionAndConnections(version uint64, connections map[PeerN
 // traversal. This prevents the connectivity graph from changing
 // underneath us in ways that would invalidate the result. Thus the
 // answer returned may be out of date, but never inconsistent.
-func (peer *Peer) Routes(stopAt *Peer, symmetric bool) (bool, map[PeerName]PeerName) {
+func (peer *Peer) Routes(stopAt *Peer, establishedAndSymmetric bool) (bool, map[PeerName]PeerName) {
 	peer.RLock()
 	defer peer.RUnlock()
 	routes := make(map[PeerName]PeerName)
@@ -127,12 +147,17 @@ func (peer *Peer) Routes(stopAt *Peer, symmetric bool) (bool, map[PeerName]PeerN
 			}
 			curName := curPeer.Name
 			for remoteName, conn := range curPeer.connections {
+				if establishedAndSymmetric && !conn.Established() {
+					continue
+				}
 				if _, found := routes[remoteName]; found {
 					continue
 				}
 				remote := conn.Remote()
 				remote.RLock()
-				if _, found := remote.connections[curName]; !symmetric || found {
+				// DANGER holding rlock on remote, possibly taking
+				// rlock on remoteConn
+				if remoteConn, found := remote.connections[curName]; !establishedAndSymmetric || (found && remoteConn.Established()) {
 					defer remote.RUnlock()
 					nextWorklist = append(nextWorklist, remote)
 					// We now know how to get to remoteName: the same

router/peers.go

@@ -11,15 +11,21 @@ import (
 type Peers struct {
 	sync.RWMutex
 	ourself *Peer
-	macs    *MacCache
 	table   map[PeerName]*Peer
 	onGC    func(*Peer)
 }
 
-func NewPeers(ourself *Peer, macs *MacCache, onGC func(*Peer)) *Peers {
+type UnknownPeerError struct {
+	Name PeerName
+}
+
+type NameCollisionError struct {
+	Name PeerName
+}
+
+func NewPeers(ourself *Peer, onGC func(*Peer)) *Peers {
 	return &Peers{
 		ourself: ourself,
-		macs:    macs,
 		table:   make(map[PeerName]*Peer),
 		onGC:    onGC}
 }
@@ -57,6 +63,12 @@ func (peers *Peers) ForEach(fun func(PeerName, *Peer)) {
 	}
 }
 
+// Merge an incoming update with our own topology.
+//
+// We add peers hitherto unknown to us, and update peers for which the
+// update contains a more recent version than known to us. The return
+// value is an "improved" update containing just these new/updated
+// elements.
 func (peers *Peers) ApplyUpdate(update []byte) ([]byte, error) {
 	peers.Lock()
 
@@ -96,7 +108,7 @@ func EncodePeers(peers ...*Peer) []byte {
 	buf := new(bytes.Buffer)
 	enc := gob.NewEncoder(buf)
 	for _, peer := range peers {
-		peer.encodePeer(enc)
+		peer.encode(enc)
 	}
 	return buf.Bytes()
 }
@@ -112,7 +124,11 @@ func (peers *Peers) String() string {
 	peers.ForEach(func(name PeerName, peer *Peer) {
 		buf.WriteString(fmt.Sprint(peer, "\n"))
 		peer.ForEachConnection(func(remoteName PeerName, conn Connection) {
-			buf.WriteString(fmt.Sprintf("   -> %v [%v]\n", remoteName, conn.RemoteTCPAddr()))
+			established := ""
+			if !conn.Established() {
+				established = " (unestablished)"
+			}
+			buf.WriteString(fmt.Sprintf("   -> %v [%v%s]\n", remoteName, conn.RemoteTCPAddr(), established))
 		})
 	})
 	return buf.String()
@@ -132,12 +148,11 @@ func (peers *Peers) fetchAlias(peer *Peer) (*Peer, bool) {
 
 func (peers *Peers) garbageCollect() []*Peer {
 	removed := []*Peer{}
+	_, reached := peers.ourself.Routes(nil, false)
 	for name, peer := range peers.table {
-		found, _ := peers.ourself.Routes(peer, false)
-		if !found && !peer.IsLocallyReferenced() {
-			peers.onGC(peer)
+		if _, found := reached[peer.Name]; !found && !peer.IsLocallyReferenced() {
 			delete(peers.table, name)
-			peers.macs.Delete(peer)
+			peers.onGC(peer)
 			removed = append(removed, peer)
 		}
 	}
@@ -148,7 +163,7 @@ func encodePeersMap(peers map[PeerName]*Peer) []byte {
 	buf := new(bytes.Buffer)
 	enc := gob.NewEncoder(buf)
 	for _, peer := range peers {
-		peer.encodePeer(enc)
+		peer.encode(enc)
 	}
 	return buf.Bytes()
 }
@@ -183,9 +198,8 @@ func (peers *Peers) decodeUpdate(update []byte) (newPeers map[PeerName]*Peer, de
 		}
 	}
 
-	unknownPeers := false
 	for _, connsBuf := range decodedConns {
-		connsIterator(connsBuf, func(remoteNameByte []byte, _ string) {
+		decErr := connsIterator(connsBuf, func(remoteNameByte []byte, _ string, _ bool) {
 			remoteName := PeerNameFromBin(remoteNameByte)
 			if _, found := newPeers[remoteName]; found {
 				return
@@ -195,12 +209,14 @@ func (peers *Peers) decodeUpdate(update []byte) (newPeers map[PeerName]*Peer, de
 			}
 			// Update refers to a peer which we have no knowledge
 			// of. Thus we can't apply the update. Abort.
-			unknownPeers = true
+			err = UnknownPeerError{remoteName}
 		})
-	}
-	if unknownPeers {
-		err = UnknownPeersError{}
-		return
+		if decErr != nil && decErr != io.EOF {
+			err = decErr
+		}
+		if err != nil {
+			return
+		}
 	}
 	return
 }
@@ -212,23 +228,12 @@ func (peers *Peers) applyUpdate(decodedUpdate []*Peer, decodedConns [][]byte) ma
 		name := newPeer.Name
 		// guaranteed to find peer in the peers.table
 		peer := peers.table[name]
-		if peer != newPeer {
-			if peer.Version() > newPeer.Version() {
-				// we know more about this one than the update. If
-				// peer is ourself, this is slightly racey (further
-				// changes could occur to ourself in the mean
-				// time). But it doesn't matter as we know that we're
-				// already > newPeer.Version() and that's not going to
-				// change.
-				newUpdate[name] = peer
-				continue
-			} else if peer == peers.ourself {
-				// nobody but us updates us
-				continue
-			} else if peer.Version() == newPeer.Version() {
-				// implication is that connections are equal too
-				continue
-			}
+		if peer != newPeer &&
+			(peer == peers.ourself || peer.Version() >= newPeer.Version()) {
+			// Nobody but us updates us. And if we know more about a
+			// peer than what's in the the update, we ignore the
+			// latter.
+			continue
 		}
 		// If we're here, either it was a new peer, or the update has
 		// more info about the peer than we do. Either case, we need
@@ -247,7 +252,7 @@ func (peers *Peers) applyUpdate(decodedUpdate []*Peer, decodedConns [][]byte) ma
 	return newUpdate
 }
 
-func (peer *Peer) encodePeer(enc *gob.Encoder) {
+func (peer *Peer) encode(enc *gob.Encoder) {
 	peer.RLock()
 	defer peer.RUnlock()
 
@@ -258,60 +263,62 @@ func (peer *Peer) encodePeer(enc *gob.Encoder) {
 	connsBuf := new(bytes.Buffer)
 	connsEnc := gob.NewEncoder(connsBuf)
 	for _, conn := range peer.connections {
-		// DANGER holding rlock on peer, going to take rlock on conn
-		if !conn.Established() {
-			continue
-		}
 		checkFatal(connsEnc.Encode(conn.Remote().NameByte))
 		checkFatal(connsEnc.Encode(conn.RemoteTCPAddr()))
+		// DANGER holding rlock on peer, going to take rlock on conn
+		checkFatal(connsEnc.Encode(conn.Established()))
 	}
 	checkFatal(enc.Encode(connsBuf.Bytes()))
 }
 
 func decodePeerNoConns(dec *gob.Decoder) (nameByte []byte, uid uint64, version uint64, conns []byte, err error) {
-	err = dec.Decode(&nameByte)
-	if err != nil {
+	if err = dec.Decode(&nameByte); err != nil {
 		return
 	}
-	err = dec.Decode(&uid)
-	if err != nil {
+	if err = dec.Decode(&uid); err != nil {
 		return
 	}
-	err = dec.Decode(&version)
-	if err != nil {
+	if err = dec.Decode(&version); err != nil {
 		return
 	}
-	err = dec.Decode(&conns)
-	if err == io.EOF {
-		err = nil
+	if err = dec.Decode(&conns); err != nil {
+		return
 	}
 	return
 }
 
-func connsIterator(input []byte, fun func([]byte, string)) {
+func connsIterator(input []byte, fun func([]byte, string, bool)) error {
 	buf := new(bytes.Buffer)
 	buf.Write(input)
 	dec := gob.NewDecoder(buf)
 	for {
 		var nameByte []byte
-		err := dec.Decode(&nameByte)
-		if err == io.EOF {
-			return
+		if err := dec.Decode(&nameByte); err != nil {
+			return err
 		}
-		checkFatal(err)
 		var foundAt string
-		checkFatal(dec.Decode(&foundAt))
-		fun(nameByte, string(foundAt))
+		if err := dec.Decode(&foundAt); err != nil {
+			return err
+		}
+		var established bool
+		if err := dec.Decode(&established); err != nil {
+			return err
+		}
+		fun(nameByte, foundAt, established)
 	}
 }
 
 func readConnsMap(peer *Peer, buf []byte, table map[PeerName]*Peer) map[PeerName]Connection {
 	conns := make(map[PeerName]Connection)
-	connsIterator(buf, func(nameByte []byte, remoteTCPAddr string) {
+	if err := connsIterator(buf, func(nameByte []byte, remoteTCPAddr string, established bool) {
 		name := PeerNameFromBin(nameByte)
 		remotePeer := table[name]
-		conn := NewRemoteConnection(peer, remotePeer, remoteTCPAddr)
+		conn := NewRemoteConnection(peer, remotePeer, remoteTCPAddr, established)
 		conns[name] = conn
-	})
+	}); err != io.EOF {
+		// this should never happen since we've already successfully
+		// decoded the same data in decodeUpdate
+		checkFatal(err)
+	}
 	return conns
 }

router/peers_test.go

@@ -0,0 +1,113 @@
+package router
+
+import (
+	"fmt"
+	wt "github.com/zettio/weave/testing"
+	"math/rand"
+	"testing"
+)
+
+// TODO we should also test:
+//
+// - applying an incremental update, including the case where that
+//   leads to an UnknownPeerError
+//
+// - the "improved update" calculation
+//
+// - non-gc of peers that are only referenced locally
+
+func newNode(name PeerName) (*Peer, *Peers) {
+	peer := NewPeer(name, 0, 0)
+	peers := NewPeers(peer, func(*Peer) {})
+	peers.FetchWithDefault(peer)
+	return peer, peers
+}
+
+// Check that ApplyUpdate copies the whole topology from peers
+func checkApplyUpdate(t *testing.T, peers *Peers) {
+	dummyName, _ := PeerNameFromString("99:00:00:01:00:00")
+	// We need a new node outside of the network, with a connection
+	// into it.
+	_, testBedPeers := newNode(dummyName)
+	testBedPeers.AddTestConnection(peers.ourself)
+	testBedPeers.ApplyUpdate(peers.EncodeAllPeers())
+
+	checkTopologyPeers(t, true, testBedPeers.allPeersExcept(dummyName), peers.allPeers()...)
+}
+
+func TestPeersEncoding(t *testing.T) {
+	const numNodes = 20
+	const numIters = 1000
+	var peer [numNodes]*Peer
+	var ps [numNodes]*Peers
+	for i := 0; i < numNodes; i++ {
+		name, _ := PeerNameFromString(fmt.Sprintf("%02d:00:00:01:00:00", i))
+		peer[i], ps[i] = newNode(name)
+	}
+
+	conns := make([]struct{ from, to int }, 0)
+	for i := 0; i < numIters; i++ {
+		oper := rand.Intn(2)
+		switch oper {
+		case 0:
+			from, to := rand.Intn(numNodes), rand.Intn(numNodes)
+			if from != to {
+				if _, found := peer[from].ConnectionTo(peer[to].Name); !found {
+					ps[from].AddTestConnection(peer[to])
+					conns = append(conns, struct{ from, to int }{from, to})
+					checkApplyUpdate(t, ps[from])
+				}
+			}
+		case 1:
+			if len(conns) > 0 {
+				n := rand.Intn(len(conns))
+				c := conns[n]
+				ps[c.from].DeleteTestConnection(peer[c.to])
+				ps[c.from].GarbageCollect()
+				checkApplyUpdate(t, ps[c.from])
+				conns = append(conns[:n], conns[n+1:]...)
+			}
+		}
+	}
+}
+
+func TestPeersGarbageCollection(t *testing.T) {
+	const (
+		peer1NameString = "01:00:00:01:00:00"
+		peer2NameString = "02:00:00:02:00:00"
+		peer3NameString = "03:00:00:03:00:00"
+	)
+	var (
+		peer1Name, _ = PeerNameFromString(peer1NameString)
+		peer2Name, _ = PeerNameFromString(peer2NameString)
+		peer3Name, _ = PeerNameFromString(peer3NameString)
+	)
+
+	// Create some peers with some connections to each other
+	p1, ps1 := newNode(peer1Name)
+	p2, ps2 := newNode(peer2Name)
+	p3, ps3 := newNode(peer3Name)
+	ps1.AddTestConnection(p2)
+	ps2.AddTestRemoteConnection(p1, p2)
+	ps2.AddTestConnection(p1)
+	ps2.AddTestConnection(p3)
+	ps3.AddTestConnection(p1)
+	ps1.AddTestConnection(p3)
+	ps2.AddTestRemoteConnection(p1, p3)
+	ps2.AddTestRemoteConnection(p3, p1)
+
+	// Every peer is referenced, so nothing should be dropped
+	wt.AssertEmpty(t, ps1.GarbageCollect(), "peers removed")
+	wt.AssertEmpty(t, ps2.GarbageCollect(), "peers removed")
+	wt.AssertEmpty(t, ps3.GarbageCollect(), "peers removed")
+
+	// Drop the connection from 2 to 3, and 3 isn't garbage-collected
+	// because 1 has a connection to 3
+	ps2.DeleteTestConnection(p3)
+	wt.AssertEmpty(t, ps2.GarbageCollect(), "peers removed")
+
+	// Drop the connection from 1 to 3, and 3 will get removed by
+	// garbage-collection
+	ps1.DeleteTestConnection(p3)
+	checkPeerArray(t, ps1.GarbageCollect(), p3)
+}

router/protocol.go

@@ -0,0 +1,28 @@
+package router
+
+const (
+	Protocol        = "weave"
+	ProtocolVersion = 12
+)
+
+type ProtocolTag byte
+
+const (
+	ProtocolConnectionEstablished ProtocolTag = iota
+	ProtocolFragmentationReceived
+	ProtocolStartFragmentationTest
+	ProtocolNonce
+	ProtocolPMTUVerified
+	ProtocolGossip
+	ProtocolGossipUnicast
+	ProtocolGossipBroadcast
+)
+
+type ProtocolMsg struct {
+	tag ProtocolTag
+	msg []byte
+}
+
+type ProtocolSender interface {
+	SendProtocolMsg(m ProtocolMsg)
+}

router/router.go

@@ -17,47 +17,81 @@ const macMaxAge = 10 * time.Minute // [1]
 // [1] should be greater than typical ARP cache expiries, i.e. > 3/2 *
 // /proc/sys/net/ipv4_neigh/*/base_reachable_time_ms on Linux
 
+type Router struct {
+	Iface           *net.Interface
+	Ourself         *LocalPeer
+	Macs            *MacCache
+	Peers           *Peers
+	Routes          *Routes
+	ConnectionMaker *ConnectionMaker
+	GossipChannels  map[uint32]*GossipChannel
+	TopologyGossip  Gossip
+	UDPListener     *net.UDPConn
+	Password        *[]byte
+	ConnLimit       int
+	BufSz           int
+	LogFrame        func(string, []byte, *layers.Ethernet)
+}
+
+type PacketSource interface {
+	ReadPacket() ([]byte, error)
+}
+
+type PacketSink interface {
+	WritePacket([]byte) error
+}
+
+type PacketSourceSink interface {
+	PacketSource
+	PacketSink
+}
+
 func NewRouter(iface *net.Interface, name PeerName, password []byte, connLimit int, bufSz int, logFrame func(string, []byte, *layers.Ethernet)) *Router {
+	router := &Router{
+		Iface:          iface,
+		GossipChannels: make(map[uint32]*GossipChannel),
+		ConnLimit:      connLimit,
+		BufSz:          bufSz,
+		LogFrame:       logFrame}
+	if len(password) > 0 {
+		router.Password = &password
+	}
 	onMacExpiry := func(mac net.HardwareAddr, peer *Peer) {
 		log.Println("Expired MAC", mac, "at", peer.Name)
 	}
 	onPeerGC := func(peer *Peer) {
-		log.Println("Removing unreachable", peer)
+		router.Macs.Delete(peer)
+		log.Println("Removed unreachable", peer)
 	}
-	router := &Router{
-		Iface:     iface,
-		Macs:      NewMacCache(macMaxAge, onMacExpiry),
-		ConnLimit: connLimit,
-		BufSz:     bufSz,
-		LogFrame:  logFrame}
-	if len(password) > 0 {
-		router.Password = &password
-	}
-	router.Ourself = StartLocalPeer(name, router)
-	router.Peers = NewPeers(router.Ourself.Peer, router.Macs, onPeerGC)
+	router.Ourself = NewLocalPeer(name, router)
+	router.Macs = NewMacCache(macMaxAge, onMacExpiry)
+	router.Peers = NewPeers(router.Ourself.Peer, onPeerGC)
 	router.Peers.FetchWithDefault(router.Ourself.Peer)
-	log.Println("Our name is", router.Ourself.Name)
-
+	router.Routes = NewRoutes(router.Ourself.Peer, router.Peers)
+	router.ConnectionMaker = NewConnectionMaker(router.Ourself, router.Peers)
+	router.TopologyGossip = router.NewGossip("topology", router)
 	return router
 }
 
-func (router *Router) UsingPassword() bool {
-	return router.Password != nil
-}
-
 func (router *Router) Start() {
 	// we need two pcap handles since they aren't thread-safe
 	pio, err := NewPcapIO(router.Iface.Name, router.BufSz)
 	checkFatal(err)
 	po, err := NewPcapO(router.Iface.Name)
 	checkFatal(err)
-	router.Routes = StartRoutes(router.Ourself.Peer, router.Peers)
-	router.ConnectionMaker = StartConnectionMaker(router.Ourself, router.Peers)
+	router.Ourself.Start()
+	router.Macs.Start()
+	router.Routes.Start()
+	router.ConnectionMaker.Start()
 	router.UDPListener = router.listenUDP(Port, po)
 	router.listenTCP(Port)
 	router.sniff(pio)
 }
 
+func (router *Router) UsingPassword() bool {
+	return router.Password != nil
+}
+
 func (router *Router) Status() string {
 	var buf bytes.Buffer
 	buf.WriteString(fmt.Sprintln("Our name is", router.Ourself.Name))
@@ -156,8 +190,11 @@ func (router *Router) acceptTCP(tcpConn *net.TCPConn) {
 	// someone else is dialing us, so our udp sender is the conn
 	// on Port and we wait for them to send us something on UDP to
 	// start.
-	connRemote := NewRemoteConnection(router.Ourself.Peer, nil, tcpConn.RemoteAddr().String())
-	NewLocalConnection(connRemote, true, tcpConn, nil, router)
+	remoteAddrStr := tcpConn.RemoteAddr().String()
+	log.Printf("->[%s] connection accepted\n", remoteAddrStr)
+	connRemote := NewRemoteConnection(router.Ourself.Peer, nil, remoteAddrStr, false)
+	connLocal := NewLocalConnection(connRemote, tcpConn, nil, router)
+	connLocal.Start(true)
 }
 
 func (router *Router) listenUDP(localPort int, po PacketSink) *net.UDPConn {
@@ -176,6 +213,16 @@ func (router *Router) listenUDP(localPort int, po PacketSink) *net.UDPConn {
 	return conn
 }
 
+type UDPPacket struct {
+	Name   PeerName
+	Packet []byte
+	Sender *net.UDPAddr
+}
+
+func (packet UDPPacket) String() string {
+	return fmt.Sprintf("UDP Packet\n name: %s\n sender: %v\n payload: % X", packet.Name, packet.Sender, packet.Packet)
+}
+
 func (router *Router) udpReader(conn *net.UDPConn, po PacketSink) {
 	defer conn.Close()
 	dec := NewEthernetDecoder()
@@ -210,7 +257,7 @@ func (router *Router) udpReader(conn *net.UDPConn, po PacketSink) {
 		err = relayConn.Decryptor.IterateFrames(handleUDPPacket, udpPacket)
 		if pde, ok := err.(PacketDecodingError); ok {
 			if pde.Fatal {
-				relayConn.CheckFatal(pde)
+				relayConn.Shutdown(pde)
 			} else {
 				relayConn.log(pde.Error())
 			}
@@ -245,14 +292,41 @@ func (router *Router) handleUDPPacketFunc(dec *EthernetDecoder, po PacketSink) F
 
 		dec.DecodeLayers(frame)
 		decodedLen := len(dec.decoded)
+		if decodedLen == 0 {
+			return nil
+		}
+		// Handle special frames produced internally (rather than
+		// captured/forwarded) by the remote router.
+		//
+		// We really shouldn't be decoding these above, since they are
+		// not genuine Ethernet frames. However, it is actually more
+		// efficient to do so, as we want to optimise for the common
+		// (i.e. non-special) frames. These always need decoding, and
+		// detecting special frames is cheaper post decoding than pre.
+		if decodedLen == 1 && dec.IsSpecial() {
+			if srcPeer != relayConn.Remote() || dstPeer != router.Ourself.Peer {
+				// A special frame not originating from the remote, or
+				// not for us? How odd; let's just drop it.
+				return nil
+			}
+			switch {
+			case frameLen == EthernetOverhead+8:
+				relayConn.ReceivedHeartbeat(sender, binary.BigEndian.Uint64(frame[EthernetOverhead:]))
+			case frameLen == FragTestSize && bytes.Equal(frame, FragTest):
+				relayConn.SendProtocolMsg(ProtocolMsg{ProtocolFragmentationReceived, nil})
+			case frameLen == PMTUDiscoverySize && bytes.Equal(frame, PMTUDiscovery):
+			default:
+				frameLenBytes := []byte{0, 0}
+				binary.BigEndian.PutUint16(frameLenBytes, uint16(frameLen-EthernetOverhead))
+				relayConn.SendProtocolMsg(ProtocolMsg{ProtocolPMTUVerified, frameLenBytes})
+			}
+			return nil
+		}
+
 		df := decodedLen == 2 && (dec.ip.Flags&layers.IPv4DontFragment != 0)
-		srcMac := dec.eth.SrcMAC
 
 		if dstPeer != router.Ourself.Peer {
 			// it's not for us, we're just relaying it
-			if decodedLen == 0 {
-				return nil
-			}
 			if df {
 				router.LogFrame("Relaying DF", frame, &dec.eth)
 			} else {
@@ -261,36 +335,16 @@ func (router *Router) handleUDPPacketFunc(dec *EthernetDecoder, po PacketSink) F
 			return checkFrameTooBig(router.Ourself.Relay(srcPeer, dstPeer, df, frame, dec), srcPeer)
 		}
 
-		if relayConn.Remote().Name == srcPeer.Name {
-			if frameLen == 0 {
-				relayConn.SetRemoteUDPAddr(sender)
-				return nil
-			} else if frameLen == FragTestSize && bytes.Equal(frame, FragTest) {
-				relayConn.SendTCP(ProtocolFragmentationReceivedByte)
-				return nil
-			} else if frameLen == PMTUDiscoverySize && bytes.Equal(frame, PMTUDiscovery) {
-				return nil
-			}
-		}
-
-		if decodedLen == 0 {
-			return nil
-		}
-
-		if dec.IsPMTUVerify() && relayConn.Remote().Name == srcPeer.Name {
-			frameLenBytes := []byte{0, 0}
-			binary.BigEndian.PutUint16(frameLenBytes, uint16(frameLen-EthernetOverhead))
-			relayConn.SendTCP(Concat(ProtocolPMTUVerifiedByte, frameLenBytes))
-			return nil
-		}
+		srcMac := dec.eth.SrcMAC
+		dstMac := dec.eth.DstMAC
 
 		if router.Macs.Enter(srcMac, srcPeer) {
-			log.Println("Discovered remote MAC", srcMac, "at", srcPeer.Name)
+			log.Println("Discovered remote MAC", srcMac, "at", srcName)
 		}
 		router.LogFrame("Injecting", frame, &dec.eth)
 		checkWarn(po.WritePacket(frame))
 
-		dstPeer, found = router.Macs.Lookup(dec.eth.DstMAC)
+		dstPeer, found = router.Macs.Lookup(dstMac)
 		if !found || dstPeer != router.Ourself.Peer {
 			return checkFrameTooBig(router.Ourself.RelayBroadcast(srcPeer, df, frame, dec), srcPeer)
 		}
@@ -298,3 +352,41 @@ func (router *Router) handleUDPPacketFunc(dec *EthernetDecoder, po PacketSink) F
 		return nil
 	}
 }
+
+// Gossiper methods - the Router is the topology Gossiper
+
+func (router *Router) OnGossipUnicast(sender PeerName, msg []byte) error {
+	return fmt.Errorf("unexpected topology gossip unicast: %v", msg)
+}
+
+func (router *Router) OnGossipBroadcast(msg []byte) error {
+	return fmt.Errorf("unexpected topology gossip broadcast: %v", msg)
+}
+
+// Return state of everything we know; intended to be called periodically
+func (router *Router) Gossip() []byte {
+	return router.Peers.EncodeAllPeers()
+}
+
+// merge in state and return "everything new I've just learnt",
+// or nil if nothing in the received message was new
+func (router *Router) OnGossip(buf []byte) ([]byte, error) {
+	newUpdate, err := router.Peers.ApplyUpdate(buf)
+	if _, ok := err.(UnknownPeerError); err != nil && ok {
+		// That update contained a reference to a peer which wasn't
+		// itself included in the update, and we didn't know about
+		// already. We ignore this; eventually we should receive an
+		// update containing a complete topology.
+		log.Println("Topology gossip:", err)
+		return nil, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+	if len(newUpdate) == 0 {
+		return nil, nil
+	}
+	router.ConnectionMaker.Refresh()
+	router.Routes.Recalculate()
+	return newUpdate, nil
+}

router/routes.go

@@ -16,6 +16,23 @@ type Routes struct {
 	queryChan chan<- *Interaction
 }
 
+func NewRoutes(ourself *Peer, peers *Peers) *Routes {
+	routes := &Routes{
+		ourself:   ourself,
+		peers:     peers,
+		unicast:   make(map[PeerName]PeerName),
+		broadcast: make(map[PeerName][]PeerName)}
+	routes.unicast[ourself.Name] = UnknownPeerName
+	routes.broadcast[ourself.Name] = []PeerName{}
+	return routes
+}
+
+func (routes *Routes) Start() {
+	queryChan := make(chan *Interaction, ChannelSize)
+	routes.queryChan = queryChan
+	go routes.queryLoop(queryChan)
+}
+
 func (routes *Routes) Unicast(name PeerName) (PeerName, bool) {
 	routes.RLock()
 	defer routes.RUnlock()
@@ -48,20 +65,6 @@ func (routes *Routes) String() string {
 	return buf.String()
 }
 
-func StartRoutes(ourself *Peer, peers *Peers) *Routes {
-	queryChan := make(chan *Interaction, ChannelSize)
-	state := &Routes{
-		ourself:   ourself,
-		peers:     peers,
-		unicast:   make(map[PeerName]PeerName),
-		broadcast: make(map[PeerName][]PeerName),
-		queryChan: queryChan}
-	state.unicast[ourself.Name] = UnknownPeerName
-	state.broadcast[ourself.Name] = []PeerName{}
-	go state.queryLoop(queryChan)
-	return state
-}
-
 // ACTOR client API
 
 const (

router/types.go

@@ -1,269 +0,0 @@
-package router
-
-import (
-	"bytes"
-	"code.google.com/p/go-bit/bit"
-	"code.google.com/p/gopacket"
-	"code.google.com/p/gopacket/layers"
-	"encoding/gob"
-	"net"
-	"sync"
-	"time"
-)
-
-type Router struct {
-	Iface           *net.Interface
-	Ourself         *LocalPeer
-	Macs            *MacCache
-	Peers           *Peers
-	Routes          *Routes
-	ConnectionMaker *ConnectionMaker
-	UDPListener     *net.UDPConn
-	Password        *[]byte
-	ConnLimit       int
-	BufSz           int
-	LogFrame        func(string, []byte, *layers.Ethernet)
-}
-
-type Connection interface {
-	Local() *Peer
-	Remote() *Peer
-	RemoteTCPAddr() string
-	Established() bool
-	Shutdown()
-}
-
-type RemoteConnection struct {
-	local         *Peer
-	remote        *Peer
-	remoteTCPAddr string
-}
-
-type LocalConnection struct {
-	sync.RWMutex
-	RemoteConnection
-	TCPConn       *net.TCPConn
-	tcpSender     TCPSender
-	remoteUDPAddr *net.UDPAddr
-	established   bool
-	stackFrag     bool
-	effectivePMTU int
-	SessionKey    *[32]byte
-	heartbeat     *time.Ticker
-	fetchAll      *time.Ticker
-	fragTest      *time.Ticker
-	forwardChan   chan<- *ForwardedFrame
-	forwardChanDF chan<- *ForwardedFrame
-	stopForward   chan<- interface{}
-	stopForwardDF chan<- interface{}
-	verifyPMTU    chan<- int
-	Decryptor     Decryptor
-	Router        *Router
-	UID           uint64
-	shutdown      bool
-	queryChan     chan<- *ConnectionInteraction
-}
-
-type ConnectionInteraction struct {
-	Interaction
-	payload interface{}
-}
-
-type Forwarder struct {
-	conn            *LocalConnection
-	ch              <-chan *ForwardedFrame
-	stop            <-chan interface{}
-	verifyPMTUTick  <-chan time.Time
-	verifyPMTU      <-chan int
-	pmtuVerifyCount uint
-	enc             Encryptor
-	udpSender       UDPSender
-	maxPayload      int
-	pmtuVerified    bool
-	highestGoodPMTU int
-	unverifiedPMTU  int
-	lowestBadPMTU   int
-}
-
-type UDPPacket struct {
-	Name   PeerName
-	Packet []byte
-	Sender *net.UDPAddr
-}
-
-type EthernetDecoder struct {
-	eth     layers.Ethernet
-	ip      layers.IPv4
-	decoded []gopacket.LayerType
-	parser  *gopacket.DecodingLayerParser
-}
-
-type MsgTooBigError struct {
-	PMTU int // actual pmtu, i.e. what the kernel told us
-}
-
-type FrameTooBigError struct {
-	EPMTU int // effective pmtu, i.e. what we tell packet senders
-}
-
-type UnknownPeersError struct {
-}
-
-type NameCollisionError struct {
-	Name PeerName
-}
-
-type PacketDecodingError struct {
-	Fatal bool
-	Desc  string
-}
-
-type LocalAddress struct {
-	ip      net.IP
-	network *net.IPNet
-}
-
-type ForwardedFrame struct {
-	srcPeer *Peer
-	dstPeer *Peer
-	frame   []byte
-}
-
-type Interaction struct {
-	code       int
-	resultChan chan<- interface{}
-}
-
-// UDPSender interface and implementations
-
-type UDPSender interface {
-	Send([]byte) error
-	Shutdown() error
-}
-
-type SimpleUDPSender struct {
-	conn    *LocalConnection
-	udpConn *net.UDPConn
-}
-
-type RawUDPSender struct {
-	ipBuf     gopacket.SerializeBuffer
-	opts      gopacket.SerializeOptions
-	udpHeader *layers.UDP
-	socket    *net.IPConn
-	conn      *LocalConnection
-}
-
-// TCPSender interface and implementations
-
-type TCPSender interface {
-	Send([]byte) error
-}
-
-type SimpleTCPSender struct {
-	encoder *gob.Encoder
-}
-
-type EncryptedTCPSender struct {
-	sync.RWMutex
-	outerEncoder *gob.Encoder
-	innerEncoder *gob.Encoder
-	buffer       *bytes.Buffer
-	conn         *LocalConnection
-	msgCount     int
-}
-
-type EncryptedTCPMessage struct {
-	Number int
-	Body   []byte
-}
-
-// TCPReceiver interface and implementations
-
-type TCPReceiver interface {
-	Decode([]byte) ([]byte, error)
-}
-
-type SimpleTCPReceiver struct {
-}
-
-type EncryptedTCPReceiver struct {
-	conn     *LocalConnection
-	decoder  *gob.Decoder
-	buffer   *bytes.Buffer
-	msgCount int
-}
-
-// Encryptor interface and implementations
-
-type Encryptor interface {
-	FrameOverhead() int
-	PacketOverhead() int
-	IsEmpty() bool
-	Bytes() []byte
-	AppendFrame(*ForwardedFrame)
-	TotalLen() int
-}
-
-type NonEncryptor struct {
-	buf       []byte
-	bufTail   []byte
-	buffered  int
-	prefixLen int
-}
-
-type NaClEncryptor struct {
-	NonEncryptor
-	buf       []byte
-	offset    uint16
-	nonce     *[24]byte
-	nonceChan chan *[24]byte
-	flags     uint16
-	prefixLen int
-	conn      *LocalConnection
-	df        bool
-}
-
-// Decryptor interface and implementations
-
-type FrameConsumer func(*LocalConnection, *net.UDPAddr, []byte, []byte, uint16, []byte) error
-
-type Decryptor interface {
-	IterateFrames(FrameConsumer, *UDPPacket) error
-	ReceiveNonce([]byte)
-	Shutdown()
-}
-
-type NonDecryptor struct {
-	conn *LocalConnection
-}
-
-type NaClDecryptor struct {
-	NonDecryptor
-	instance   *NaClDecryptorInstance
-	instanceDF *NaClDecryptorInstance
-}
-
-type NaClDecryptorInstance struct {
-	nonce               *[24]byte
-	previousNonce       *[24]byte
-	usedOffsets         *bit.Set
-	previousUsedOffsets *bit.Set
-	highestOffsetSeen   uint16
-	nonceChan           chan *[24]byte
-}
-
-// Packet capture/inject interfaces
-
-type PacketSource interface {
-	ReadPacket() ([]byte, error)
-}
-
-type PacketSink interface {
-	WritePacket([]byte) error
-}
-
-type PacketSourceSink interface {
-	PacketSource
-	PacketSink
-}

router/udp_sender.go

@@ -0,0 +1,138 @@
+package router
+
+import (
+	"code.google.com/p/gopacket"
+	"code.google.com/p/gopacket/layers"
+	"log"
+	"net"
+	"syscall"
+)
+
+type UDPSender interface {
+	Send([]byte) error
+	Shutdown() error
+}
+
+type SimpleUDPSender struct {
+	conn    *LocalConnection
+	udpConn *net.UDPConn
+}
+
+type RawUDPSender struct {
+	ipBuf     gopacket.SerializeBuffer
+	opts      gopacket.SerializeOptions
+	udpHeader *layers.UDP
+	socket    *net.IPConn
+	conn      *LocalConnection
+}
+
+type MsgTooBigError struct {
+	PMTU int // actual pmtu, i.e. what the kernel told us
+}
+
+func NewSimpleUDPSender(conn *LocalConnection) *SimpleUDPSender {
+	return &SimpleUDPSender{udpConn: conn.Router.UDPListener, conn: conn}
+}
+
+func (sender *SimpleUDPSender) Send(msg []byte) error {
+	_, err := sender.udpConn.WriteToUDP(msg, sender.conn.RemoteUDPAddr())
+	return err
+}
+
+func (sender *SimpleUDPSender) Shutdown() error {
+	return nil
+}
+
+func NewRawUDPSender(conn *LocalConnection) (*RawUDPSender, error) {
+	ipSocket, err := dialIP(conn)
+	if err != nil {
+		return nil, err
+	}
+	udpHeader := &layers.UDP{SrcPort: layers.UDPPort(Port)}
+	ipBuf := gopacket.NewSerializeBuffer()
+	opts := gopacket.SerializeOptions{
+		FixLengths: true,
+		// UDP header is calculated with a phantom IP
+		// header. Yes, it's totally nuts. Thankfully, for UDP
+		// over IPv4, the checksum is optional. It's not
+		// optional for IPv6, but we'll ignore that for
+		// now. TODO
+		ComputeChecksums: false}
+
+	return &RawUDPSender{
+		ipBuf:     ipBuf,
+		opts:      opts,
+		udpHeader: udpHeader,
+		socket:    ipSocket,
+		conn:      conn}, nil
+}
+
+func (sender *RawUDPSender) Send(msg []byte) error {
+	payload := gopacket.Payload(msg)
+	sender.udpHeader.DstPort = layers.UDPPort(sender.conn.RemoteUDPAddr().Port)
+
+	err := gopacket.SerializeLayers(sender.ipBuf, sender.opts, sender.udpHeader, &payload)
+	if err != nil {
+		return err
+	}
+	packet := sender.ipBuf.Bytes()
+	_, err = sender.socket.Write(packet)
+	if err == nil || PosixError(err) != syscall.EMSGSIZE {
+		return err
+	}
+	f, err := sender.socket.File()
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	fd := int(f.Fd())
+	log.Println("EMSGSIZE on send, expecting PMTU update (IP packet was",
+		len(packet), "bytes, payload was", len(msg), "bytes)")
+	pmtu, err := syscall.GetsockoptInt(fd, syscall.IPPROTO_IP, syscall.IP_MTU)
+	if err != nil {
+		return err
+	}
+	return MsgTooBigError{PMTU: pmtu}
+}
+
+func (sender *RawUDPSender) Shutdown() error {
+	defer func() { sender.socket = nil }()
+	return sender.socket.Close()
+}
+
+func dialIP(conn *LocalConnection) (*net.IPConn, error) {
+	ipLocalAddr, err := ipAddr(conn.TCPConn.LocalAddr())
+	if err != nil {
+		return nil, err
+	}
+	ipRemoteAddr, err := ipAddr(conn.TCPConn.RemoteAddr())
+	if err != nil {
+		return nil, err
+	}
+	ipSocket, err := net.DialIP("ip4:UDP", ipLocalAddr, ipRemoteAddr)
+	if err != nil {
+		return nil, err
+	}
+	f, err := ipSocket.File()
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	fd := int(f.Fd())
+	// This Makes sure all packets we send out have DF set on them.
+	err = syscall.SetsockoptInt(fd, syscall.IPPROTO_IP, syscall.IP_MTU_DISCOVER, syscall.IP_PMTUDISC_DO)
+	if err != nil {
+		return nil, err
+	}
+	return ipSocket, nil
+}
+
+func ipAddr(addr net.Addr) (*net.IPAddr, error) {
+	host, _, err := net.SplitHostPort(addr.String())
+	if err != nil {
+		return nil, err
+	}
+	return &net.IPAddr{
+		IP:   net.ParseIP(host),
+		Zone: ""}, nil
+}

router/utils.go

@@ -1,12 +1,20 @@
 package router
 
 import (
+	"bytes"
 	"crypto/rand"
+	"encoding/gob"
 	"fmt"
+	"hash/fnv"
 	"log"
 	"net"
 )
 
+type Interaction struct {
+	code       int
+	resultChan chan<- interface{}
+}
+
 func checkFatal(e error) {
 	if e != nil {
 		log.Fatal(e)
@@ -38,8 +46,8 @@ func (ftbe FrameTooBigError) Error() string {
 	return fmt.Sprint("Frame too big error. Effective PMTU is ", ftbe.EPMTU)
 }
 
-func (upe UnknownPeersError) Error() string {
-	return fmt.Sprint("Reference to unknown peers")
+func (upe UnknownPeerError) Error() string {
+	return fmt.Sprint("Reference to unknown peer ", upe.Name)
 }
 
 func (nce NameCollisionError) Error() string {
@@ -50,10 +58,6 @@ func (pde PacketDecodingError) Error() string {
 	return fmt.Sprint("Failed to decode packet: ", pde.Desc)
 }
 
-func (packet UDPPacket) String() string {
-	return fmt.Sprintf("UDP Packet\n name: %s\n sender: %v\n payload: % X", packet.Name, packet.Sender, packet.Packet)
-}
-
 func Concat(elems ...[]byte) []byte {
 	res := []byte{}
 	for _, e := range elems {
@@ -73,6 +77,21 @@ func randUint64() (r uint64) {
 	return
 }
 
+func hash(s string) uint32 {
+	h := fnv.New32a()
+	h.Write([]byte(s))
+	return h.Sum32()
+}
+
+func GobEncode(items ...interface{}) []byte {
+	buf := new(bytes.Buffer)
+	enc := gob.NewEncoder(buf)
+	for _, i := range items {
+		checkFatal(enc.Encode(i))
+	}
+	return buf.Bytes()
+}
+
 func macint(mac net.HardwareAddr) (r uint64) {
 	for _, b := range mac {
 		r <<= 8

site/features.md

@@ -95,7 +95,9 @@ same host, via their docker-assigned IP addresses. For complete
 isolation between application containers, that feature needs to be
 disabled by
 [setting `--icc=false`](https://docs.docker.com/articles/networking/#between-containers)
-in the docker daemon configuration.
+in the docker daemon configuration. Furthermore, containers should be
+prevented from capturing and injecting raw network packets - this can
+be accomplished by starting them with the `--cap-drop net_raw` option.
 
 ### <a name="dynamic-network-attachment"></a>Dynamic network attachment
 
@@ -129,23 +131,23 @@ sharing it between applications:
 ### <a name="security"></a>Security
 
 In order to connect containers across untrusted networks, weave peers
-can be told to encrypt traffic by supplying a `-password` or
-`-passwordfile` option when launching weave, e.g.
+can be told to encrypt traffic by supplying a `-password` option or
+`WEAVE_PASSWORD` environment variable when launching weave, e.g.
 
     host1# weave launch -password wEaVe
 
 or
 
-    host1# weave launch -passwordfile /path/to/file
+    host1# export WEAVE_PASSWORD=wEaVe
+    host1# weave launch
 
-where the file must contain a single line like this:
-
-    WEAVE_PASSWORD=wEaVe
+_NOTE: The command line option takes precedence over the environment
+variable._
 
 The same password must be specified for all weave peers; it is a
 component in the creation of ephemeral session keys for connections
-between peers. See the
-[crypto documentation](how-it-works.html#crypto) for more details.
+between peers. See the [crypto documentation](how-it-works.html#crypto)
+for more details.
 
 ### <a name="host-network-integration"></a>Host network integration
 

site/how-it-works.md

@@ -12,9 +12,9 @@ different each time it is run.  These are opaque identifiers as far as
 the router is concerned, although the name defaults to a MAC address.
 
 Weave routers establish TCP connections to each other, over which they
-perform a protocol handshake and subsequently exchange topology
-information (see below). These connections are encrypted if so
-configured. Peers also establish UDP "connections", possibly
+perform a protocol handshake and subsequently exchange
+[topology](#topology) information. These connections are encrypted if
+so configured. Peers also establish UDP "connections", possibly
 encrypted, which carry encapsulated network packets. These
 "connections" are duplex and can traverse firewalls.
 
@@ -35,9 +35,11 @@ packet to peers.
 Weave routers learn which peer host a particular MAC address resides
 on. They combine this knowledge with topology information in order to
 make routing decisions and thus avoid forwarding every packet to every
-peer. The topology information captures which peers are connected to
-which other peers; weave can route packets in partially connected
-networks with changing topology.
+peer. Weave can route packets in partially connected networks with
+changing topology. For example, in this network, peer 1 is connected
+directly to 2 and 3, but if 1 needs to send a packet to 4 or 5 it must
+first send it to peer 3:
+![Partially connected Weave Network](images/top-diag1.png "Partially connected Weave Network")
 
 ### <a name="encapsulation"></a>Encapsulation
 
@@ -78,7 +80,7 @@ The name of the sending peer enables the receiving peer to identify
 who sent this UDP packet. This is followed by the meta data and
 payload for one or more captured frames. The router performs batching:
 if it captures several frames very quickly that all need forwarding to
-the same peer, it will fit as many of them as possible into a single
+the same peer, it fits as many of them as possible into a single
 UDP packet.
 
 The meta data for each frame contains the names of the capturing and
@@ -88,13 +90,117 @@ receiving peers to build up their mappings of which client MAC
 addresses are local to which peers. The destination peer name enables
 the receiving peer to identify whether this frame is destined for
 itself or whether it should be forwarded on to some other peer,
-accomodating multi-hop routing. This will work even when the receiving
+accommodating multi-hop routing. This works even when the receiving
 intermediate peer has no knowledge of the destination MAC: only the
 original capturing peer needs to determine the destination peer from
 the MAC. This way weave peers never need to exchange the MAC addresses
 of clients and need not take any special action for ARP traffic and
 MAC discovery.
 
+### <a name="topology"></a>Topology
+
+The topology information captures which peers are connected to which
+other peers. Weave peers communicate their knowledge of the topology
+(and changes to it) to their neighbours, who then pass this
+information on to their own neighbours, and so on, until the entire
+network knows about any change.
+
+Topology is communicated over the TCP links between peers, using a
+Gossip mechanism.  Topology messages are sent by a peer...
+
+- when a connection has been added; the entire topology is sent to the
+  remote peer, and an incremental update, containing information on
+  just the two peers at the ends of the connection, is sent to all
+  neighbours,
+- when a connection has been marked as 'established', indicating that
+  the remote peer can receive UDP traffic from the peer; an update
+  containing just information about the local peer is sent to all
+  neighbours,
+- when a connection has been torn down; an update containing just
+  information about the local peer is sent to all neighbours,
+- periodically, on a timer, in case someone has missed an update.
+
+The receiver of a topology update merges that update with its own
+topology model, adding peers hitherto unknown to it, and updating
+peers for which the update contains a more recent version than known
+to it. If there were any such new/updated peers, then an improved
+update containing them is sent out on all connections.
+
+If the update mentions a peer that the receiver does not know, then
+the entire update is ignored.
+
+#### Message details
+Every gossip message is structured as follows:
+
+    +-----------------------------------+
+    | 1-byte message type - Gossip      |
+    +-----------------------------------+
+    | 4-byte Gossip channel - Topology  |
+    +-----------------------------------+
+    | Peer Name of source               |
+    +-----------------------------------+
+    | Gossip payload (topology update)  |
+    +-----------------------------------+
+
+The topology update payload is laid out like this:
+
+    +-----------------------------------+
+    | Peer 1: Name                      |
+    +-----------------------------------+
+    | Peer 1: UID                       |
+    +-----------------------------------+
+    | Peer 1: Version number            |
+    +-----------------------------------+
+    | Peer 1: List of connections       |
+    +-----------------------------------+
+    |                ...                |
+    +-----------------------------------+
+    | Peer N: Name                      |
+    +-----------------------------------+
+    | Peer N: UID                       |
+    +-----------------------------------+
+    | Peer N: Version number            |
+    +-----------------------------------+
+    | Peer N: List of connections       |
+    +-----------------------------------+
+
+Each List of connections is encapsulated as a byte buffer, within
+which the structure is:
+
+    +-----------------------------------+
+    | Connection 1: Remote Peer Name    |
+    +-----------------------------------+
+    | Connection 1: Remote IP address   |
+    +-----------------------------------+
+    | Connection 2: Remote Peer Name    |
+    +-----------------------------------+
+    | Connection 2: Remote IP address   |
+    +-----------------------------------+
+    |                ...                |
+    +-----------------------------------+
+    | Connection N: Remote Peer Name    |
+    +-----------------------------------+
+    | Connection N: Remote IP address   |
+    +-----------------------------------+
+
+#### Removal of peers
+If a peer, after receiving a topology update, sees that another peer
+no longer has any connections within the network, it drops all
+knowledge of that second peer.
+
+#### Out-of-date topology
+The peer-to-peer gossiping of updates is not instantaneous, so it is
+very possible for a node elsewhere in the network to have an
+out-of-date view.
+
+If the destination peer for a packet is still reachable, then
+out-of-date topology can result in it taking a less efficient route.
+
+If the out-of-date topology makes it look as if the destination peer
+is not reachable, then the packet is dropped.  For most protocols
+(e.g. TCP), the transmission will be retried a short time later, by
+which time the topology should have updated.
+
 ### <a name="crypto"></a>Crypto
 
 Weave can be configured to encrypt both the data passing over the TCP
@@ -210,7 +316,7 @@ This is very similar to the [non-crypto encapsulation](#encapsulation).
 All of the frames are encrypted with the same ephemeral session key
 and all must be decrypted by the receiving peer. Frames which are to
 be forwarded on to some further peer will be re-encrypted with the
-relevant ephemeral session keys for the onwards connections. Thus all
+relevant ephemeral session keys for the onward connections. Thus all
 traffic is fully decrypted on every peer it passes through.
 Encryption is again done with the NaCl `secretbox.Seal` function.
 

site/index.md

@@ -9,6 +9,8 @@ layout: default
  * [Features](features.html)
  * [Troubleshooting](troubleshooting.html)
  * [Building](building.html)
- * [Using with Boot2Docker](boot2docker.html)
  * [How it works](how-it-works.html)
  * [Contact information](https://github.com/zettio/weave#contact-us)
+ * *Installation Notes*
+   * [Using with boot2docker](boot2docker.html)
+   * [Using with systemd](systemd.html)

site/systemd.md

@@ -0,0 +1,70 @@
+---
+title: Using Weave with systemd
+layout: default
+---
+
+Having installed `weave` as per [readme][], you might wish to configure the
+init daemon to start it on boot. Most recent Linux distribution releases are
+shipping with [systemd][]. The information below should provide you with some
+initial guidance on getting a weave service configured on systemd-based OS.
+
+## Weave Service Unit and Configuration
+
+A regular service unit definition for weave is shown below and you should
+normally place it in `/etc/systemd/system/weave.service`.
+
+    [Unit]
+    Description=Weave Network
+    Documentation=http://zettio.github.io/weave/
+    Requires=docker.service
+    After=docker.service
+    [Service]
+    EnvironmentFile=-/etc/sysconfig/weave
+    ExecStartPre=/usr/local/bin/weave launch $PEERS
+    ExecStart=/usr/bin/docker logs -f weave
+    SuccessExitStatus=2
+    ExecStop=/usr/local/bin/weave stop
+    [Install]
+    WantedBy=multi-user.target
+
+
+To specify the addresses or names of other weave hosts to join the network
+you can create the `/etc/sysconfig/weave` environment file which would be of
+the following format:
+
+    PEERS="HOST1 HOST2 .. HOSTn"
+
+You can also use the [connect][] command to add participating hosts dynamically.
+
+Additionally, if you want to enable [encryption][] you can specify a password with
+`WEAVE_PASSWORD="MakeSureThisIsSecure"` in the `/etc/sysconfig/weave` environment
+file, and it will get picked up by weave on launch.
+
+You now should be able to launch weave with
+
+    sudo systemctl start weave
+
+To ensure weave launches after reboot, you need run
+
+    sudo systemctl enable weave
+
+For more information on systemd, please refer to the documentation supplied
+by your distribution of Linux.
+
+## SELinux Tweaks
+
+If your OS has SELinux enabled and you wish to run weave as a systemd unit,
+then you should follow the instructions below. These instructions apply to
+CentOS and RHEL as of 7.0. On Fedora 21, there is no need to do this.
+
+Once you have installed `weave` in `/usr/local/bin`, set its execution
+context with the commands shown below. You will need to have the
+`policycoreutils-python` package installed.
+
+    sudo semanage fcontext -a -t unconfined_exec_t -f f /usr/local/bin/weave
+    sudo restorecon /usr/local/bin/weave
+
+[readme]: https://github.com/zettio/weave/blob/master/README.md#installation
+[connect]: http://zettio.github.io/weave/features.html#dynamic-topologies
+[systemd]: http://www.freedesktop.org/wiki/Software/systemd/
+[encryption]: http://zettio.github.io/weave/how-it-works.html#crypto

site/top-diag1.dot

@@ -0,0 +1,9 @@
+graph weave {
+rankdir=LR;
+"peer 1"  -- "peer 2" ;
+"peer 1"  -- "peer 3" ;
+"peer 2"  -- "peer 3" ;
+"peer 3"  -- "peer 4" ;
+"peer 4"  -- "peer 5" ;
+"peer 3"  -- "peer 5" ;
+}

site/troubleshooting.md

@@ -57,6 +57,7 @@ This produces output like:
 
 ````
 weave router 0.7.0
+Encryption off
 Our name is 7a:f4:56:87:76:3b
 Sniffing traffic on &{39 65535 ethwe ae:e3:07:9c:8c:d4 up|broadcast|multicast}
 MACs:

test/110_encryption_test.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+. ./config.sh
+
+C1=10.1.1.4
+C2=10.1.1.7
+
+start_suite "Ping over encrypted cross-host weave network"
+
+for HOST in $HOST1 $HOST2; do
+    run_on $HOST sudo $WEAVE stop || true
+    run_on $HOST sudo $WEAVE stop-dns || true
+    docker_on $HOST rm -f c1 c2 || true
+done
+
+run_on $HOST1 sudo $WEAVE launch -password notverysecure
+run_on $HOST2 sudo $WEAVE launch -password notverysecure $HOST1
+
+run_on $HOST2 sudo $WEAVE run $C2/24 -t --name=c2 ubuntu
+run_on $HOST1 sudo $WEAVE run $C1/24 -t --name=c1 ubuntu
+ok=$(docker -H tcp://$HOST1:2375 exec -i c1 sh -c "ping -q -c 4 $C2 >&2 && echo ok")
+assert "echo $ok" "ok"
+
+end_suite

test/200_dns_test.sh

@@ -5,7 +5,7 @@
 C1=10.2.0.78
 C2=10.2.0.34
 
-start_suite "Resolve names"
+start_suite "Resolve names on a single host"
 
 run_on $HOST1 sudo $WEAVE stop || true
 run_on $HOST1 sudo $WEAVE stop-dns || true
@@ -14,9 +14,12 @@ run_on $HOST1 sudo $WEAVE launch-dns 10.0.0.2/8
 docker_on $HOST1 rm -f c1 c2 || true
 
 run_on $HOST1 sudo $WEAVE run $C2/24 -t --name=c2 -h seetwo.weave.local ubuntu
-run_on $HOST1 sudo $WEAVE run --with-dns $C1/24 -t --name=c1 ubuntu
+run_on $HOST1 sudo $WEAVE run --with-dns $C1/24 -t --name=c1 aanand/docker-dnsutils /bin/sh
 
-ok=$(docker -H tcp://$HOST1:2375 exec -i c1 sh -c "ping -q -c4 seetwo.weave.local >&2 && echo ok")
-assert "echo $ok" "ok"
+ok=$(docker -H tcp://$HOST1:2375 exec -i c1 sh -c "dig +short seetwo.weave.local")
+assert "echo $ok" "$C2"
+
+ok=$(docker -H tcp://$HOST1:2375 exec -i c1 sh -c "dig +short -x $C2")
+assert "echo $ok" "seetwo.weave.local."
 
 end_suite

test/210_dns_multicast_test.sh

@@ -0,0 +1,34 @@
+#! /bin/bash
+
+. ./config.sh
+
+C1=10.2.3.78
+C2=10.2.3.34
+
+start_suite "Resolve names across hosts"
+
+for host in $HOST1 $HOST2; do
+    run_on $host sudo $WEAVE stop || true
+    run_on $host sudo $WEAVE stop-dns || true
+    docker_on $host rm -f c1 c2 || true
+done
+
+run_on $HOST1 sudo $WEAVE launch
+run_on $HOST2 sudo $WEAVE launch $HOST1
+
+run_on $HOST1 sudo $WEAVE launch-dns 10.3.3.2/24 -debug
+run_on $HOST2 sudo $WEAVE launch-dns 10.3.3.3/24 -debug
+
+run_on $HOST2 sudo $WEAVE run $C2/24 -t --name=c2 -h seetwo.weave.local ubuntu
+run_on $HOST1 sudo $WEAVE run --with-dns $C1/24 --name=c1 -t aanand/docker-dnsutils /bin/sh
+
+ok=$(docker -H tcp://$HOST1:2375 exec -i c1 dig +short seetwo.weave.local)
+assert "echo $ok" "$C2"
+
+ok=$(docker -H tcp://$HOST1:2375 exec -i c1 dig +short -x $C2)
+assert "echo $ok" "seetwo.weave.local."
+
+ok=$(docker -H tcp://$HOST1:2375 exec -i c1 dig +short -x 8.8.8.8)
+assert "test -n \"$ok\" && echo pass" "pass"
+
+end_suite

test/300_fallback_dns_test.sh

@@ -0,0 +1,19 @@
+#! /bin/bash
+
+. ./config.sh
+
+start_suite "Resolve a non-weave address"
+
+run_on $HOST1 sudo $WEAVE stop-dns || true
+run_on $HOST1 sudo $WEAVE launch-dns 10.2.6.2 -debug
+docker_on $HOST1 rm -f c1 || true
+
+run_on $HOST1 sudo $WEAVE run --with-dns 10.1.1.5/24 --name=c1 -t aanand/docker-dnsutils /bin/sh
+
+ok=$(docker -H tcp://$HOST1:2375 exec -i c1 dig +short -t MX weave.works)
+assert "test -n \"$ok\" && echo pass" "pass"
+
+ok=$(docker -H tcp://$HOST1:2375 exec -i c1 dig +short -x 8.8.8.8)
+assert "test -n \"$ok\" && echo pass" "pass"
+
+end_suite

test/Vagrantfile

@@ -1,14 +1,17 @@
 VAGRANTFILE_API_VERSION = "2"
 
-host1_ip = "192.168.48.19" # arbitrary private IP
-host2_ip = "192.168.48.15" # arbitrary private IP
+# these ought to match what is in config.sh
+n_machines = 2
+ip_prefix = "192.168.48"
+ip_suffix_base = 10
 
-def configure_docker(host, ip)
-  pkgs = "lxc-docker ethtool conntrack"
+def configure_docker(host, hostname, ip)
+  pkgs = "lxc-docker"
 
   host.vm.box = "phusion/ubuntu-14.04-amd64"
   host.vm.box_url = "https://oss-binaries.phusionpassenger.com/vagrant/boxes/latest/ubuntu-14.04-amd64-vbox.box"
 
+  host.vm.hostname = hostname
   host.vm.network "private_network", ip: ip
 
   # host.vm.synced_folder "./", "/home/vagrant/src/github.com/zettio/weave"
@@ -30,12 +33,11 @@ end
 
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
 
-  config.vm.define "host1" do |host1|
-    configure_docker(host1, host1_ip)
-  end
-
-  config.vm.define "host2" do |host2|
-    configure_docker(host2, host2_ip)
+  (1..n_machines).each do |i|
+    config.vm.define "host#{i}" do |host|
+      ip_suffix = ip_suffix_base + i
+      configure_docker(host, "host#{i}", "#{ip_prefix}.#{ip_suffix}")
+    end
   end
 
 end

test/config.sh

@@ -2,11 +2,24 @@
 
 set -e
 
+# these ought to match what is in Vagrantfile
+N_MACHINES=${N_MACHINES:-2}
+IP_PREFIX=${IP_PREFIX:-192.168.48}
+IP_SUFFIX_BASE=${IP_SUFFIX_BASE:-10}
+
+HOSTS=
+for i in $(seq 1 $N_MACHINES); do
+    IP="${IP_PREFIX}.$((${IP_SUFFIX_BASE}+$i))"
+    HOSTS="$HOSTS $IP"
+done
+
+# these are used by the tests
+HOST1="${IP_PREFIX}.$((${IP_SUFFIX_BASE}+1))"
+HOST2="${IP_PREFIX}.$((${IP_SUFFIX_BASE}+2))"
+
 . ./assert.sh
 
 SSH=${SSH:-ssh -l vagrant -i ./insecure_private_key -o UserKnownHostsFile=./.ssh_known_hosts -o CheckHostIP=no -o StrictHostKeyChecking=no}
-HOST1=${HOST1:-192.168.48.19}
-HOST2=${HOST2:-192.168.48.15}
 
 remote() {
     rem=$1

test/setup.sh

@@ -8,9 +8,10 @@ curl -sS https://raw.githubusercontent.com/lehmannro/assert.sh/master/assert.sh
 . ./config.sh
 
 echo Copying weave images and script to hosts
-for HOST in $HOST1 $HOST2; do
+for HOST in $HOSTS; do
     docker_on $HOST load -i /var/tmp/weave.tar
     docker_on $HOST load -i /var/tmp/weavedns.tar
+    docker_on $HOST load -i /var/tmp/weavetools.tar
     run_on $HOST mkdir -p `dirname $WEAVE`
     cat ../weave         | run_on $HOST sh -c "cat > $WEAVE"
     cat ../bin/docker-ns | run_on $HOST sh -c "cat > $DOCKER_NS"

testing/util.go

@@ -0,0 +1,96 @@
+package testing
+
+import (
+	"reflect"
+	"runtime"
+	"testing"
+	"time"
+)
+
+func AssertNoErr(t *testing.T, err error) {
+	if err != nil {
+		Fatalf(t, "Unexpected error: %s", err)
+	}
+}
+
+func AssertEqualuint64(t *testing.T, got, wanted uint64, desc string) {
+	if got != wanted {
+		Fatalf(t, "Expected %s %d but got %d", desc, wanted, got)
+	}
+}
+
+func AssertEqualInt(t *testing.T, got, wanted int, desc string) {
+	if got != wanted {
+		Fatalf(t, "Expected %s %d but got %d", desc, wanted, got)
+	}
+}
+
+func AssertEqualString(t *testing.T, got, wanted string, desc string) {
+	if got != wanted {
+		Fatalf(t, "Expected %s '%s' but got '%s'", desc, wanted, got)
+	}
+}
+
+func AssertStatus(t *testing.T, got int, wanted int, desc string) {
+	if got != wanted {
+		Fatalf(t, "Expected %s %d but got %d", desc, wanted, got)
+	}
+}
+
+func AssertErrorInterface(t *testing.T, got interface{}, wanted interface{}, desc string) {
+	gotT, wantedT := reflect.TypeOf(got), reflect.TypeOf(wanted).Elem()
+	if !gotT.Implements(wantedT) {
+		Fatalf(t, "Expected %s but got %s (%s)", wantedT.String(), gotT.String(), desc)
+	}
+}
+
+func AssertErrorType(t *testing.T, got interface{}, wanted interface{}, desc string) {
+	gotT, wantedT := reflect.TypeOf(got), reflect.TypeOf(wanted).Elem()
+	if gotT != wantedT {
+		Fatalf(t, "Expected %s but got %s (%s)", wantedT.String(), gotT.String(), desc)
+	}
+}
+
+func AssertType(t *testing.T, got interface{}, wanted interface{}, desc string) {
+	gotT, wantedT := reflect.TypeOf(got), reflect.TypeOf(wanted)
+	if gotT != wantedT {
+		Fatalf(t, "Expected %s but got %s (%s)", wantedT.String(), gotT.String(), desc)
+	}
+}
+
+func AssertEmpty(t *testing.T, array interface{}, desc string) {
+	if reflect.ValueOf(array).Len() != 0 {
+		Fatalf(t, "Expected empty %s but got %s", desc, array)
+	}
+}
+
+// Like testing.Fatalf, but adds the stack trace of the current call
+func Fatalf(t *testing.T, format string, args ...interface{}) {
+	t.Fatalf(format+"\n%s", append(args, StackTrace())...)
+}
+
+func StackTrace() string {
+	return stackTrace(false)
+}
+
+func stackTrace(all bool) string {
+	buf := make([]byte, 1<<20)
+	stacklen := runtime.Stack(buf, all)
+	return string(buf[:stacklen])
+}
+
+// Borrowed from net/http tests:
+// goTimeout runs f, failing t if f takes more than d to complete.
+func RunWithTimeout(t *testing.T, d time.Duration, f func()) {
+	ch := make(chan bool, 2)
+	timer := time.AfterFunc(d, func() {
+		t.Errorf("Timeout expired after %v: stacks:\n%s", d, stackTrace(true))
+		ch <- true
+	})
+	defer timer.Stop()
+	go func() {
+		defer func() { ch <- true }()
+		f()
+	}()
+	<-ch
+}

tools/build.sh

@@ -7,6 +7,14 @@ apt-get -y install curl make pkg-config gcc bison flex
 BASEDIR=$(dirname $0)
 mkdir -p "$BASEDIR"/bin
 
+copy_exe() {
+    if ldd $1 >/dev/null 2>&1 ; then
+        echo "ERROR: $1 was not linked statically." 2>&1
+        exit 1
+    fi
+    cp $1 "$BASEDIR"/bin/
+}
+
 # ethtool
 
 ETHTOOL=ethtool-3.16
@@ -14,8 +22,8 @@ ETHTOOL=ethtool-3.16
 rm -rf $ETHTOOL
 
 curl -s -S https://www.kernel.org/pub/software/network/ethtool/$ETHTOOL.tar.gz | tar xvz
-(cd $ETHTOOL; ./configure LDFLAGS=-static; make)
-cp $ETHTOOL/ethtool "$BASEDIR"/bin/
+(cd $ETHTOOL; ./configure LDFLAGS=-static && make)
+copy_exe $ETHTOOL/ethtool
 
 # conntrack
 
@@ -30,11 +38,23 @@ fetch() {
 
 for PACKAGE in $PACKAGES; do
     fetch $PACKAGE
-    (cd $PACKAGE; ./configure && make LDFLAGS=-static install)
+    (cd $PACKAGE; ./configure --disable-shared && make install)
 done
 
 fetch $CONNTRACK
-(cd $CONNTRACK; ./configure && make LDFLAGS=-static && rm -f src/conntrack && make LDFLAGS=-all-static)
-cp $CONNTRACK/src/conntrack "$BASEDIR"/bin/
+(cd $CONNTRACK; ./configure --disable-shared && make && rm -f src/conntrack && make LDFLAGS=-all-static)
+copy_exe $CONNTRACK/src/conntrack
+
+# curl
+
+CURL=curl-7.40.0
+
+rm -rf $CURL
+
+curl -s -S  http://curl.haxx.se/download/$CURL.tar.gz | tar xvz
+(cd $CURL; ./configure --without-ssl --disable-shared && make && rm src/curl && make LDFLAGS=-all-static)
+copy_exe $CURL/src/curl
+
+#
 
 touch "$BASEDIR"/bin

weave

@@ -6,7 +6,7 @@ SCRIPT_VERSION="(unreleased version)"
 usage() {
     echo "Usage:"
     echo "weave setup"
-    echo "weave launch     [-password <password> | -passwordfile <path>] <peer> ..."
+    echo "weave launch     [-password <password>] <peer> ..."
     echo "weave launch-dns <cidr>"
     echo "weave connect    <peer>"
     echo "weave run        [--with-dns] <cidr> <docker run args> ..."
@@ -57,6 +57,7 @@ PORT=6783
 HTTP_PORT=6784
 DNS_HTTP_PORT=6785
 DOCKER_BRIDGE=${DOCKER_BRIDGE:-docker0}
+PROCFS=${PROCFS:-/proc}
 
 COMMAND=$1
 
@@ -131,8 +132,6 @@ create_bridge() {
         ip link add name v${CONTAINER_IFNAME}du mtu $MTU type dummy
         ip link set dev v${CONTAINER_IFNAME}du master $BRIDGE
         ip link del dev v${CONTAINER_IFNAME}du
-        # disable offloading
-        run_tool host ethtool -K $BRIDGE tx off >/dev/null
         # Work around the situation where there are no rules allowing traffic
         # across our bridge. E.g. ufw
         add_iptables_rule filter FORWARD -i $BRIDGE -o $BRIDGE -j ACCEPT
@@ -140,6 +139,14 @@ create_bridge() {
         run_iptables -t nat -N WEAVE >/dev/null 2>&1 || true
         add_iptables_rule nat POSTROUTING -j WEAVE
     }
+    if [ ! "$1" = "--without-ethtool" ] ; then
+        # disable offloading - we do this even when $BRIDGE already exists
+        # simply because it has the desirable side-effect of ensuring we
+        # have the $TOOLS_IMAGE and thus do not incur downloading-induced
+        # delays in more time-sensitive sections elsewhere.
+        run_tool host ethtool -K $BRIDGE tx off >/dev/null
+    fi
+
     ip link set dev $BRIDGE up
 }
 
@@ -166,7 +173,7 @@ docker_bridge_ip() {
 # If an error is caused by container dying, swallow output from error.
 with_container_netns () {
     CONTAINER="$1"
-    CONTAINER_PID=$(docker inspect --format='{{ .State.Pid }}' $CONTAINER)
+    CONTAINER_PID=$(docker inspect --format='{{.State.Pid}}' $CONTAINER)
 
     if [ "$CONTAINER_PID" = 0 ] ; then
         echo "Container $CONTAINER not running." >&2
@@ -181,7 +188,7 @@ with_container_netns () {
     NETNS=$CONTAINER_PID
     [ ! -d /var/run/netns ] && mkdir -p /var/run/netns
     rm -f /var/run/netns/$NETNS
-    ln -s /proc/$CONTAINER_PID/ns/net /var/run/netns/$NETNS
+    ln -s $PROCFS/$CONTAINER_PID/ns/net /var/run/netns/$NETNS
 
     LOCAL_IFNAME="v${CONTAINER_IFNAME}pl${CONTAINER_PID}"
     GUEST_IFNAME="v${CONTAINER_IFNAME}pg${CONTAINER_PID}"
@@ -194,7 +201,7 @@ with_container_netns () {
     shift 1
     if ! "$@" >$IP_TMPOUT 2>$IP_TMPERR ; then
         STATUS=1
-        if [ "$(docker inspect --format='{{ .State.Pid }}' $CONTAINER)" != "$CONTAINER_PID" ] ; then
+        if [ "$(docker inspect --format='{{.State.Pid}}' $CONTAINER)" != "$CONTAINER_PID" ] ; then
             echo "Container $CONTAINER died" >&2
         else
             echo "Failure during network configuration for container $CONTAINER:" >&2
@@ -209,7 +216,7 @@ with_container_netns () {
 }
 
 connect_container_to_bridge() {
-    if [ "$(readlink /proc/$CONTAINER_PID/ns/net)" = "$(readlink /proc/$$/ns/net)" ] ; then
+    if [ "$(readlink $PROCFS/$CONTAINER_PID/ns/net)" = "$(readlink $PROCFS/$$/ns/net)" ] ; then
         echo "Container is running in the host network namespace, and therefore cannot be" >&2
         echo "connected to weave. Perhaps the container was started with --net=host." >&2
         return 1
@@ -290,7 +297,7 @@ detach() {
 
 # Call url $4 with http verb $3 on container $1 at port $2
 http_call() {
-    if ! status=$(docker inspect --format='{{ .State.Running }} {{ .NetworkSettings.IPAddress }}' $1 2>/dev/null); then
+    if ! status=$(docker inspect --format='{{.State.Running}} {{.NetworkSettings.IPAddress}}' $1 2>/dev/null); then
         echo "$1 container is not present; have you launched it?" >&2
         return 1
     fi
@@ -317,32 +324,27 @@ http_call_ip() {
     http_verb="$3"
     url="$4"
     shift 4
-    if command_exists curl ; then
-        curl -s -X $http_verb "$@" http://$ip:$port$url
-    else
-        echo "Cannot find curl, which is required for this operation." >&2
-        exit 1
-    fi
+    run_tool host curl -s -X $http_verb "$@" http://$ip:$port$url
 }
 
 # Perform operation $1 on container ID $2 to local DNS database at address $3
 # This function is only called where we know $2 is a valid container name
 tell_dns() {
-    if ! status=$(docker inspect --format='{{ .State.Running }}' $DNS_CONTAINER_NAME 2>/dev/null) || [ "$status" != "true" ] ; then
+    if ! status=$(docker inspect --format='{{.State.Running}}' $DNS_CONTAINER_NAME 2>/dev/null) || [ "$status" != "true" ] ; then
         # weavedns not running - silently return
         return
     fi
     # get the long form of the container ID
-    CONTAINER=$(docker inspect --format='{{ .Id }}' $2 2>/dev/null)
+    CONTAINER=$(docker inspect --format='{{.Id}}' $2 2>/dev/null)
     # extract IP address and routing prefix from CIDR
     WEAVE_ADDR_IP=$(echo $3 | sed -e 's/\([^/]*\)\/\(.*\)/\1/')
-    MORE_ARGS=$(docker inspect --format='--data-urlencode fqdn={{ .Config.Hostname }}.{{ .Config.Domainname }}.' $CONTAINER 2>/dev/null) && true
+    MORE_ARGS=$(docker inspect --format='--data-urlencode fqdn={{.Config.Hostname}}.{{.Config.Domainname}}.' $CONTAINER 2>/dev/null) && true
     http_call $DNS_CONTAINER_NAME $DNS_HTTP_PORT $1 /name/$CONTAINER/$WEAVE_ADDR_IP $MORE_ARGS || true
 }
 
 # Tell the newly-started weaveDNS about existing weave IPs
 populate_dns() {
-    DNS_IP=$(docker inspect --format='{{ .NetworkSettings.IPAddress }}' $DNS_CONTAINER_NAME)
+    DNS_IP=$(docker inspect --format='{{.NetworkSettings.IPAddress}}' $DNS_CONTAINER_NAME)
     WAIT_TIME=1
     while ! http_call_ip $DNS_IP $DNS_HTTP_PORT GET /status >/dev/null && [ $WAIT_TIME -lt 4 ]; do
         sleep $WAIT_TIME
@@ -354,7 +356,7 @@ populate_dns() {
     fi
 
     for CONTAINER in $(docker ps -q --no-trunc); do
-        MORE_ARGS=$(docker inspect --format='--data-urlencode fqdn={{ .Config.Hostname }}.{{ .Config.Domainname }}.' $CONTAINER 2>/dev/null) && true
+        MORE_ARGS=$(docker inspect --format='--data-urlencode fqdn={{.Config.Hostname}}.{{.Config.Domainname}}.' $CONTAINER 2>/dev/null) && true
         if CONTAINER_ADDRS=$(with_container_netns $CONTAINER container_weave_addrs 2>&1) ; then
             CONTAINER_IPS=$(echo "$CONTAINER_ADDRS" | grep -o 'inet .*' | sed -e 's/inet \([^/]*\)\/\(.*\)/\1/')
             for IP in $CONTAINER_IPS; do
@@ -366,7 +368,7 @@ populate_dns() {
 
 # Check that a container named $1 with image $2 is not running
 check_not_running() {
-    case $(docker inspect --format='{{ .State.Running }} {{ .Config.Image }}' $1 2>/dev/null) in
+    case $(docker inspect --format='{{.State.Running}} {{.Config.Image}}' $1 2>/dev/null) in
         "true $2")
             echo "$1 is already running." >&2
             exit 1
@@ -396,7 +398,7 @@ container_weave_addrs() {
     ip netns exec $NETNS ip addr show dev $CONTAINER_IFNAME
 }
 
-uname -s -r | sed -n 's|^\([^ ]*\) \([0-9]\+\)\.\([0-9]\+\).*|\1 \2 \3|p' | {
+uname -s -r | sed -n -e 's|^\([^ ]*\) \([0-9][0-9]*\)\.\([0-9][0-9]*\).*|\1 \2 \3|p' | {
     if ! read sys maj min ; then
       echo "ERROR: Unable to parse operating system version $(uname -s -r)" >&2
       exit 1
@@ -413,8 +415,8 @@ uname -s -r | sed -n 's|^\([^ ]*\) \([0-9]\+\)\.\([0-9]\+\).*|\1 \2 \3|p' | {
 }
 
 ask_version() {
-    if ! DOCKERIMAGE=$(docker inspect --format='{{ .Image }}' $1 2>/dev/null) ; then
-            if ! DOCKERIMAGE=$(docker inspect --format='{{ .Id }}' $2 2>/dev/null) ; then
+    if ! DOCKERIMAGE=$(docker inspect --format='{{.Image}}' $1 2>/dev/null) ; then
+            if ! DOCKERIMAGE=$(docker inspect --format='{{.Id}}' $2 2>/dev/null) ; then
                 echo "Unable to find $2 image." >&2
             fi
     fi
@@ -432,7 +434,7 @@ if ! ip netns list >/dev/null 2>&1 ; then
     exit 1
 fi
 
-if ! DOCKER_VERSION=$(docker -v | sed -n 's|^Docker version \([0-9]\+\.[0-9]\+.[0-9]\+\).*|\1|p') || [ -z "$DOCKER_VERSION" ] ; then
+if ! DOCKER_VERSION=$(docker -v | sed -n -e 's|^Docker version \([0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*\).*|\1|p') || [ -z "$DOCKER_VERSION" ] ; then
     echo "ERROR: Unable to parse docker version" >&2
     exit 1
 fi
@@ -453,6 +455,9 @@ case "$COMMAND" in
             docker pull $img
         done
         ;;
+    create-bridge)
+        create_bridge --without-ethtool
+        ;;
     launch)
         check_not_running $CONTAINER_NAME $BASE_IMAGE
         create_bridge
@@ -470,19 +475,15 @@ case "$COMMAND" in
         fi
         if [ "$1" = "-password" ] ; then
             [ $# -gt 1 ] || usage
-            PASSWORD=$2
-            shift 2
-        elif [ "$1" = "-passwordfile" ] ; then
-            [ $# -gt 1 ] || usage
-            PASSWORDFILE=$2
+            WEAVE_PASSWORD="$2"
+            export WEAVE_PASSWORD
             shift 2
         fi
         # Set WEAVE_DOCKER_ARGS in the environment in order to supply
         # additional parameters, such as resource limits, to docker
         # when launching the weave container.
         CONTAINER=$(docker run --privileged -d --name=$CONTAINER_NAME \
-            -p $PORT:$PORT/tcp -p $PORT:$PORT/udp \
-            ${PASSWORD:+-e WEAVE_PASSWORD="$PASSWORD"} ${PASSWORDFILE:+--env-file="$PASSWORDFILE"} \
+            -p $PORT:$PORT/tcp -p $PORT:$PORT/udp -e WEAVE_PASSWORD \
             $WEAVE_DOCKER_ARGS $IMAGE -name $MACADDR -iface $CONTAINER_IFNAME "$@")
         with_container_netns $CONTAINER launch >/dev/null
         echo $CONTAINER
@@ -523,7 +524,7 @@ case "$COMMAND" in
         echo weave script $SCRIPT_VERSION
         ask_version $CONTAINER_NAME $IMAGE
         ask_version $DNS_CONTAINER_NAME $DNS_IMAGE
-        if ! TOOLS_IMAGE_ID=$(docker inspect --format='{{ .Id }}' $TOOLS_IMAGE 2>/dev/null) ; then
+        if ! TOOLS_IMAGE_ID=$(docker inspect --format='{{.Id}}' $TOOLS_IMAGE 2>/dev/null) ; then
             echo "Unable to find $TOOLS_IMAGE image." >&2
         else
             TOOLS_VERSION=$(docker images --no-trunc | grep $TOOLS_IMAGE_ID | grep -v latest | tr -s ' ' | cut -d ' ' -f 2 | tr "\n" ' ')
@@ -591,7 +592,7 @@ case "$COMMAND" in
     expose)
         [ $# -eq 1 ] || usage
         validate_cidr $1
-        create_bridge
+        create_bridge --without-ethtool
         if ! ip addr show dev $BRIDGE | grep -qF $1
         then
             ip addr add dev $BRIDGE $1
@@ -602,7 +603,7 @@ case "$COMMAND" in
     hide)
         [ $# -eq 1 ] || usage
         validate_cidr $1
-        create_bridge
+        create_bridge --without-ethtool
         if ip addr show dev $BRIDGE | grep -qF $1
         then
             ip addr del dev $BRIDGE $1

weavedns/Dockerfile

@@ -3,4 +3,4 @@ MAINTAINER Weaveworks Inc <help@weave.works>
 WORKDIR /home/weave
 ADD ./weavedns /home/weave/
 EXPOSE 53/udp
-ENTRYPOINT ["/home/weave/weavedns", "-wait", "5"]
+ENTRYPOINT ["/home/weave/weavedns", "-wait", "20"]

weavedns/README.md

@@ -1,17 +1,20 @@
 # Weave DNS server
 
-The Weave DNS server answers name queries in a Weave network. It is
-run per-host, to be supplied as the nameserver for containers on that
-host. It is then told about hostnames for the local containers. For
-other names it will ask the other weave hosts, or fall back to using
-the host's configured name server.
+The Weave DNS server answers name queries in a Weave network. This
+provides a simple way for containers to find each other: just give
+them hostnames and tell other containers to connect to those names.
+Unlike Docker 'links', this requires no code changes and works across
+hosts.
 
 ## Using weaveDNS
 
-The weave script command `launch-dns` starts the DNS container, and
-then you use the `--with-dns` option on containers you wish to use it.
-Subsquently, giving any container a hostname in the domain
-`.weave.local` will register it in DNS. For example:
+WeaveDNS is deployed as a set of containers that communicate with each
+other over the weave network. One such container needs to be started
+on every weave host, by invoking the weave script command
+`launch-dns`. Application containers are then instructed to use
+WeaveDNS as their nameserver by supplying the `--with-dns` option when
+starting them. Giving any container a hostname in the `.weave.local`
+domain registers it in weaveDNS.  For example:
 
 ```bash
 $ weave launch
@@ -24,25 +27,43 @@ $ docker attach $shell1
 ...
 ```
 
-The weave IP address supplied to `weave launch-dns` must not be used
-by any other container, and the supplied network must be the same for
-all DNS containers, and be disjoint from all application networks.
+Each weaveDNS container started with `launch-dns` needs to be given
+its own, unique, IP address, in a subnet that is a) common to all
+weaveDNS containers, b) disjoint from the application subnets, and c)
+not in use on any of the hosts. In our example the weaveDNS address is
+in subnet 10.1.254.0/24 and the application containers are in
+subnet 10.1.1.0/24.
 
-The DNS container can be stopped with `stop-dns`.
+WeaveDNS containers can be stopped with `stop-dns`.
 
+## How it works
+
+The weaveDNS container running on every host acts as the nameserver
+for containers on that host. It is told about hostnames for local
+containers by the `weave run` command. If a hostname is in the
+`.weave.local` domain then weaveDNS records the association of that
+name with the container's weave IP address(es).
+
+When weaveDNS is queried for a name in the `.weave.local` domain, it
+first checks its own records. If the name is not found there, it asks
+the weaveDNS servers on the other hosts in the weave network.
+
+When weaveDNS is queried for a name in a domain other than
+`.weave.local`, it queries the host's configured nameserver,
+which is the standard behaviour for Docker containers.
 
 ## Domain search paths
 
 If you don't supply a domain search path (with `--dns-search=`),
-`weave run ...` will tell a container to look for "bare" hostnames,
-like `pingme`, in its own domain. That's why you can just say `ping
+`weave run ...` tells a container to look for "bare" hostnames, like
+`pingme`, in its own domain. That's why you can just invoke `ping
 pingme` above -- since the hostname is `ubuntu.weave.local`, it will
 look for `pingme.weave.local`.
 
 If you want to supply other entries for the domain search path,
 e.g. if you want containers in different sub-domains to resolve
-hostnames across all sub-domains plus some external domains, you will
-need *also* to supply the `weave.local` domain to retain the above
+hostnames across all sub-domains plus some external domains, you need
+*also* to supply the `weave.local` domain to retain the above
 behaviour.
 
 ```bash
@@ -54,10 +75,10 @@ weave run --with-dns 10.1.1.4/24 -ti \
 
 ## Doing things more manually
 
-If you use the `--with-dns` option, `weave run` will automatically
-supply the DNS server address to the new container. And both
-`weave run` and `weave attach` will register the hostname of the given
-container against the given weave network IP address.
+If you use the `--with-dns` option, `weave run` automatically supplies
+the DNS server address to the new container. And both `weave run` and
+`weave attach` register the hostname of the given container against
+the given weave network IP address.
 
 In some circumstances, you may not want to use the `weave`
 command. You can still take advantage of a running weaveDNS, with some
@@ -69,7 +90,7 @@ So that containers can connect to a stable and always routable IP
 address, weaveDNS publishes its port 53 to the Docker bridge device,
 which is assumed to be `docker0`.
 
-Some configurations will use a different Docker bridge device. To
+Some configurations may use a different Docker bridge device. To
 supply a different bridge device, use the environment variable
 `DOCKER_BRIDGE`, e.g.,
 
@@ -102,10 +123,10 @@ $ docker_ip=$(ip -4 addr show dev docker0 | grep -o 'inet [0-9.]*' | cut -d ' '
 
 ### Supplying the domain search path
 
-By default, Docker will provide containers with a `/etc/resolv.conf`
-that matches that for the host. In some circumstances, this may
-include a DNS search path, which will break the nice "bare names
-resolve" property above.
+By default, Docker provides containers with a `/etc/resolv.conf` that
+matches that for the host. In some circumstances, this may include a
+DNS search path, which will break the nice "bare names resolve"
+property above.
 
 Therefore, when starting containers with `docker run` instead of
 `weave run`, you will usually want to supply a domain search path so
@@ -121,13 +142,15 @@ its hostname, you can register it using the HTTP API:
 
 ```bash
 $ docker start $shell2
-$ shell2_ip=$(docker inspect --format='{{ .NetworkSettings.IPAddress }}' $shell2)
-$ curl -X PUT "http://$dns_ip:6785/name/$shell2/10.1.1.27" -d local_ip=$shell2_ip -d fqdn=shell2.weave.local
+$ curl -X PUT "http://$dns_ip:6785/name/$shell2/10.1.1.27" -d fqdn=shell2.weave.local
 ```
 
 ### Registering multiple containers with the same name
 
-This is supported; in the initial implementation weaveDNS will pick one address to return when you ask for the name.  Since weave-dns will remove any container that dies, this is a simple way to implement redundancy.  In the current implementation it does not attempt to do load-balancing.
+This is supported; weaveDNS picks one address to return when you ask
+for the name. Since weaveDNS removes any container that dies, this is
+a simple way to implement redundancy.  In the current implementation
+it does not attempt to do load-balancing.
 
 ### Replacing one container with another at the same name
 
@@ -135,9 +158,9 @@ If you would like to deploy a new version of a service, keep the old one running
 
 ### Not watching docker events
 
-By default, the server will watch docker events and remove entries for
-any containers that die. You can tell it not to, by adding
-`--watch=false` to the container args:
+By default, weaveDNS watchs docker events and removes entries for any
+containers that die. You can tell it not to, by adding `--watch=false`
+to the container args:
 
 ```bash
 $ weave launch-dns 10.1.254.1/24 --watch=false

weavedns/main.go

@@ -63,7 +63,8 @@ func main() {
 		}
 	}
 
-	err := weavedns.StartServer(zone, iface, dnsPort, httpPort, wait)
+	go weavedns.ListenHttp(weavedns.LOCAL_DOMAIN, zone, httpPort)
+	err := weavedns.StartServer(zone, iface, dnsPort, wait)
 	if err != nil {
 		Error.Fatal("Failed to start server", err)
 	}

weaver/Dockerfile

@@ -2,4 +2,4 @@ FROM scratch
 MAINTAINER Weaveworks Inc <help@weave.works>
 WORKDIR /home/weave
 ADD ./weaver /home/weave/
-ENTRYPOINT ["/home/weave/weaver", "-wait", "5"]
+ENTRYPOINT ["/home/weave/weaver", "-wait", "20"]

weaver/main.go

@@ -122,6 +122,7 @@ func main() {
 	}
 
 	router := weave.NewRouter(iface, ourName, []byte(password), connLimit, bufSz*1024*1024, logFrame)
+	log.Println("Our name is", router.Ourself.Name)
 	router.Start()
 	for _, peer := range peers {
 		if addr, err := net.ResolveTCPAddr("tcp4", weave.NormalisePeerAddr(peer)); err == nil {
@@ -135,8 +136,13 @@ func main() {
 }
 
 func handleHttp(router *weave.Router) {
+	encryption := "off"
+	if router.Password != nil && len(*router.Password) > 0 {
+		encryption = "on"
+	}
 	http.HandleFunc("/status", func(w http.ResponseWriter, r *http.Request) {
 		io.WriteString(w, fmt.Sprintln("weave router", version))
+		io.WriteString(w, fmt.Sprintln("Encryption", encryption))
 		io.WriteString(w, router.Status())
 	})
 	http.HandleFunc("/connect", func(w http.ResponseWriter, r *http.Request) {
@@ -162,8 +168,8 @@ func handleSignals(router *weave.Router) {
 		sig := <-sigs
 		switch sig {
 		case syscall.SIGQUIT:
-			runtime.Stack(buf, true)
-			log.Printf("=== received SIGQUIT ===\n*** goroutine dump...\n%s\n*** end\n", buf)
+			stacklen := runtime.Stack(buf, true)
+			log.Printf("=== received SIGQUIT ===\n*** goroutine dump...\n%s\n*** end\n", buf[:stacklen])
 		case syscall.SIGUSR1:
 			log.Printf("=== received SIGUSR1 ===\n*** status...\n%s\n*** end\n", router.Status())
 		}