Add release v1.6.0

Fixes #2355

.ackrc

@@ -0,0 +1,2 @@
+--ignore-dir=vendor
+--ignore-dir=.pkg

.gitmodules

@@ -4,9 +4,6 @@
 [submodule "vendor/github.com/vishvananda/netlink"]
 	path = vendor/github.com/vishvananda/netlink
 	url = https://github.com/vishvananda/netlink.git
-[submodule "vendor/github.com/davecheney/profile"]
-	path = vendor/github.com/davecheney/profile
-	url = https://github.com/davecheney/profile.git
 [submodule "vendor/github.com/gorilla/mux"]
 	path = vendor/github.com/gorilla/mux
 	url = https://github.com/gorilla/mux.git
@@ -70,3 +67,12 @@
 [submodule "vendor/github.com/coreos/go-iptables"]
 	path = vendor/github.com/coreos/go-iptables
 	url = https://github.com/coreos/go-iptables
+[submodule "vendor/github.com/pkg/profile"]
+	path = vendor/github.com/pkg/profile
+	url = https://github.com/pkg/profile
+[submodule "vendor/github.com/j-keck/arping"]
+	path = vendor/github.com/j-keck/arping
+	url = https://github.com/j-keck/arping
+[submodule "vendor/github.com/aws/aws-sdk-go"]
+	path = vendor/github.com/aws/aws-sdk-go
+	url = https://github.com/aws/aws-sdk-go

CHANGELOG.md

@@ -1,3 +1,39 @@
+## Release 1.6.0
+
+## Highlights
+
+* A new [AWS VPC](https://weave.works/docs/net/latest/using-weave/awsvpc/) mode
+  that leverages Amazon Virtual Private Cloud for near-native network
+  performance, as an alternative to the Sleeve and Fast Datapath
+  overlays
+* Docker 1.12 introduced some internal changes that made it
+  incompatible with previous version of Weave Net - this version
+  restores compatability
+* An [operational guide](https://weave.works/docs/net/latest/operational-guide)
+  detailing best practices for deploying and operating Weave Net
+* Changes to the target peer list are remembered across restarts,
+  making it much easier to deploy resilient networks
+* The version checkpoint now transmits network mode (e.g. 'awsvpc')
+  and kernel/docker versions to us to inform and guide our development
+  efforts. See the [installation documentation](https://weave.works/docs/net/latest/installing-weave/)
+  for instructions on disabling the checkpoint feature.
+
+More details in the [change log](https://github.com/weaveworks/weave/issues?q=milestone%3A1.6.0).
+
+## Release 1.5.2
+
+Bug fixes and minor improvements
+
+* Weave Proxy did not flush the initial http header in the Docker event stream, which could cause Docker Swarm to show all nodes pending. #2306/#2311
+* When using the CNI plugin, if a container was removed and quickly replaced by another using the same IP address, other containers might be unable to contact it. Send an address resolution protocol message to update them. #2313
+* Avoid Docker hanging for 1 minute in `weave launch` if the plugin had not shut down cleanly #2286/#2292
+* Print an error message when Weave bridge mode is changed without `weave reset` #2304
+* Eliminate spurious warning message from IP allocator on plugin shutdown #2300/#2319
+* Display error message when address requested in a subnet that is too small (/31 or /32) #2282/#2321
+* Add short wait after `weave reset` to allow updates to reach peers #2280
+* Weave was occasionally unable to claim existing IP address immediately after launch #2275/#2281
+* Refactor some integration tests to run faster and more reliably #2291
+
 ## Release 1.5.1
 
 Bug fixes and minor improvements

CONTRIBUTING.md

@@ -0,0 +1,32 @@
+Thanks for your interest in contributing to this project and for
+taking the time to read this guide.
+
+# Code of conduct
+
+**Be respectful:** disagreements between people are to be expected and are usually the sign of healthy debate and engagement. Disagreements can lead to frustration and even anger for some members. Turning to personal insults, intimidation or threatening behaviour does not improve the situation. Participants should thus take care to ensure all communications / interactions stay professional at all times.
+
+**Be considerate:** remember that the community has members with a diverse background, many of whom have English as a second language. What might appear impolite, may simply be a result of a lack of knowledge of the English language. Bear in mind that actions will have an impact on other community members and the project as a whole, so take potential consequences into account before pursuing a course of action.
+
+**Be forgiving:** humans are fallible and as such prone to make mistakes and inexplicably change their positions at times. Don't assume that other members are acting with malicious intent. Be prepared to forgive people who make mistakes and assist each other in learning from them. Playing a blame game doesn't help anyone.
+
+# Issues
+* Before reporting an issue make sure you search first if anybody has already reported a similar issue and whether or not it has been fixed.
+* Make sure your issue report sufficiently details the problem:
+ * If you set up weave in some way other than that described by our [main instructions](https://github.com/zettio/weave#installation), please mention that.
+ * Include step-by-step instructions to reproduce the issue.
+ * State the OS platform you are using, i.e. `uname -a`
+ * `iptables` rules and firewalls can have a great impact on any Linux network; please note whether you have set up any rules and/or are running a firewall on your machine(s).
+* If you want to tackle any open issue, make sure you let people know you are working on it.
+
+## Pull Requests
+* Please be generous describing your changes.
+* Although it is highly suggested to include tests, they are not a hard requirement in order to get your contributions accepted.
+* Keep pull requests small so core developers can review them quickly.
+* Keep each pull request focused on a specific topic.  If you have two things to change, create two pull requests. This helps reviewers to understand the meat of your contribution.
+
+## Workflow
+See [WorkingOnWeave](https://github.com/weaveworks/weave/wiki/WorkingOnWeave)
+
+# Acknowledgements
+
+Much of this taken from https://github.com/cloudescape/gowsdl/blob/master/CONTRIBUTING.md

Makefile

@@ -4,7 +4,7 @@ PUBLISH=publish_weave publish_weaveexec publish_plugin
 .PHONY: all exes testrunner update tests lint publish $(PUBLISH) clean clean-bin prerequisites build run-smoketests
 
 # If you can use docker without being root, you can do "make SUDO="
-SUDO=sudo -E
+SUDO=$(shell docker info >/dev/null 2>&1 || echo "sudo -E")
 BUILD_IN_CONTAINER=true
 RM=--rm
 RUN_FLAGS=-ti
@@ -32,7 +32,7 @@ WEAVEEXEC_UPTODATE=.weaveexec.uptodate
 PLUGIN_UPTODATE=.plugin.uptodate
 WEAVEDB_UPTODATE=.weavedb.uptodate
 
-IMAGES_UPTODATE=$(WEAVER_UPTODATE) $(WEAVEEXEC_UPTODATE) $(PLUGIN_UPTODATE) $(BUILD_UPTODATE) $(WEAVEDB_UPTODATE)
+IMAGES_UPTODATE=$(WEAVER_UPTODATE) $(WEAVEEXEC_UPTODATE) $(PLUGIN_UPTODATE) $(WEAVEDB_UPTODATE)
 
 WEAVER_IMAGE=$(DOCKERHUB_USER)/weave
 WEAVEEXEC_IMAGE=$(DOCKERHUB_USER)/weaveexec
@@ -60,26 +60,24 @@ BUILD_FLAGS=-i -ldflags "-extldflags \"-static\" -X main.version=$(WEAVE_VERSION
 PACKAGE_BASE=$(shell go list -e ./)
 
 all: $(WEAVE_EXPORT)
-exes: $(EXES)
 testrunner: $(RUNNER_EXE) $(TEST_TLS_EXE)
 
-$(EXES): $(BUILD_UPTODATE)
 $(WEAVER_EXE) $(WEAVEPROXY_EXE) $(WEAVEUTIL_EXE): common/*.go common/*/*.go net/*.go net/*/*.go
 $(WEAVER_EXE): router/*.go ipam/*.go ipam/*/*.go db/*.go nameserver/*.go prog/weaver/*.go
 $(WEAVEPROXY_EXE): proxy/*.go prog/weaveproxy/*.go
-$(WEAVEUTIL_EXE): prog/weaveutil/*.go
+$(WEAVEUTIL_EXE): prog/weaveutil/*.go net/*.go
 $(SIGPROXY_EXE): prog/sigproxy/*.go
-$(PLUGIN_EXE): prog/plugin/*.go plugin/*/*.go api/*.go common/*.go common/docker/*.go
+$(PLUGIN_EXE): prog/plugin/*.go plugin/*/*.go api/*.go common/*.go common/docker/*.go net/*.go
 $(TEST_TLS_EXE): test/tls/*.go
 $(WEAVEWAIT_NOOP_EXE): prog/weavewait/*.go
 $(WEAVEWAIT_EXE): prog/weavewait/*.go net/*.go
 $(WEAVEWAIT_NOMCAST_EXE): prog/weavewait/*.go net/*.go
-tests: $(BUILD_UPTODATE) tools/.git
-lint: $(BUILD_UPTODATE) tools/.git
+tests: tools/.git
+lint: tools/.git
 
 ifeq ($(BUILD_IN_CONTAINER),true)
 
-$(EXES) tests lint:
+exes $(EXES) tests lint: $(BUILD_UPTODATE)
 	git submodule update --init
 	@mkdir -p $(shell pwd)/.pkg
 	$(SUDO) docker run $(RM) $(RUN_FLAGS) \
@@ -90,9 +88,10 @@ $(EXES) tests lint:
 
 else
 
+exes: $(EXES)
+
 $(WEAVER_EXE) $(WEAVEPROXY_EXE) $(PLUGIN_EXE):
 ifeq ($(COVERAGE),true)
-	go build -v $(BUILD_FLAGS) -o $@ ./$(@D)
 	$(eval COVERAGE_MODULES := $(shell (go list ./$(@D); go list -f '{{join .Deps "\n"}}' ./$(@D) | grep "^$(PACKAGE_BASE)/") | grep -v "^$(PACKAGE_BASE)/vendor/" | paste -s -d,))
 	go test -c -o ./$@ $(BUILD_FLAGS) -v -covermode=atomic -coverpkg $(COVERAGE_MODULES) ./$(@D)/
 else
@@ -173,13 +172,17 @@ ifneq ($(UPDATE_LATEST),false)
 endif
 
 publish: $(PUBLISH)
+ifeq ($(PUBLISH_WEAVEDB),true)
+	$(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker push   $(DOCKERHUB_USER)/weavedb:latest
+endif
 
 clean-bin:
-	-$(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker rmi $(IMAGES) $(BUILD_IMAGE)
+	-$(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker rmi $(IMAGES)
 	rm -rf $(EXES) $(IMAGES_UPTODATE) $(WEAVE_EXPORT) .pkg
 
 clean: clean-bin
-	rm -rf test/tls/*.pem test/coverage.* test/coverage
+	-$(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker rmi $(BUILD_IMAGE)
+	rm -rf test/tls/*.pem test/coverage.* test/coverage $(BUILD_UPTODATE)
 
 build:
 	$(SUDO) go clean -i net

README.md

@@ -1,13 +1,17 @@
 # Weave Net - Weaving Containers into Applications
 
-[![Build Status](https://travis-ci.org/weaveworks/weave.svg?branch=master)](https://travis-ci.org/weaveworks/weave) [![Integration Tests](https://circleci.com/gh/weaveworks/weave/tree/master.svg?style=shield&circle-token=4933c7dabb3d0383e62117565cb9d16df7b1a811)](https://circleci.com/gh/weaveworks/weave) [![Coverage Status](https://coveralls.io/repos/weaveworks/weave/badge.svg)](https://coveralls.io/r/weaveworks/weave)
+[![Integration Tests](https://circleci.com/gh/weaveworks/weave/tree/master.svg?style=shield)](https://circleci.com/gh/weaveworks/weave)
+[![Coverage Status](https://coveralls.io/repos/weaveworks/weave/badge.svg)](https://coveralls.io/r/weaveworks/weave)
+[![Go Report Card](https://goreportcard.com/badge/github.com/weaveworks/weave)](https://goreportcard.com/report/github.com/weaveworks/weave)
+[![Slack Status](https://slack.weave.works/badge.svg)](https://slack.weave.works)
+[![Docker Pulls](https://img.shields.io/docker/pulls/weaveworks/weave.svg?maxAge=604800)](https://hub.docker.com/r/weaveworks/weave/)
 
 # About Weaveworks
 
 [Weaveworks](https://www.weave.works) is the company that delivers the most productive way for developers to connect, observe and control
 Docker containers.
 
-This repository contains [Weave Net](https://www.weave.works/products/weave-net/), the first product developed by Weaveworks, with nearly 5 million downloads to date, which enables users to get started with Docker clusters and portable apps in a fraction of the time compared with other solutions.
+This repository contains [Weave Net](https://www.weave.works/products/weave-net/), the first product developed by Weaveworks, and with nearly 5 million downloads to date, it enables you to get started with Docker clusters and portable apps in a fraction of the time compared with other solutions.
 
 Other products from Weaveworks include [Weave Scope](https://www.weave.works/products/weave-scope/), a powerful container monitoring tool that automatically maps Docker containers and their interactions, and [Weave Flux](https://www.weave.works/products/weave-flux/), a microservice router that automates the access of containers as services.
 

bin/release

@@ -165,14 +165,14 @@ publish() {
     setup
     cd $PWD/$RELEASE_DIR
 
-    UPDATE_LATEST=false
+    IS_MAINLINE=false
     if [ "$RELEASE_TYPE" = 'MAINLINE' ] ; then
-        UPDATE_LATEST=true
+        IS_MAINLINE=true
     fi
 
     if [ "$RELEASE_TYPE" = 'PRERELEASE' ] ; then
         echo "== Tagging and pushing images on docker hub as user $DOCKERHUB_USER"
-        make UPDATE_LATEST=$UPDATE_LATEST SUDO=$SUDO WEAVE_VERSION=$VERSION DOCKERHUB_USER=$DOCKERHUB_USER publish
+        make UPDATE_LATEST=$IS_MAINLINE SUDO=$SUDO WEAVE_VERSION=$VERSION DOCKERHUB_USER=$DOCKERHUB_USER publish
         echo "** Docker images tagged and pushed"
 
         echo "== Publishing pre-release on GitHub"
@@ -204,7 +204,7 @@ publish() {
         echo '** Sanity checks OK for publishing tag' $LATEST_TAG as $DOCKERHUB_USER/weave:$VERSION
 
         echo "== Tagging and pushing images on docker hub as user $DOCKERHUB_USER"
-        make UPDATE_LATEST=$UPDATE_LATEST SUDO=$SUDO WEAVE_VERSION=$VERSION DOCKERHUB_USER=$DOCKERHUB_USER publish
+        make PUBLISH_WEAVEDB=$IS_MAINLINE UPDATE_LATEST=$IS_MAINLINE SUDO=$SUDO WEAVE_VERSION=$VERSION DOCKERHUB_USER=$DOCKERHUB_USER publish
         echo "** Docker images tagged and pushed"
 
         echo "== Publishing release on GitHub"

circle.yml

@@ -32,7 +32,7 @@ test:
         parallel: true
     - cd $SRCDIR; COVERDIR=test/coverage make RM= tests:
         parallel: true
-    - cd $SRCDIR; make COVERAGE=true RM= all:
+    - cd $SRCDIR; make COVERAGE=true RM= exes all:
         parallel: true
     - cd $SRCDIR; make testrunner:
         parallel: true
@@ -67,12 +67,12 @@ deployment:
     commands:
       - docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS
       - cd $SRCDIR; make clean-bin
-      - cd $SRCDIR; make publish
-      - go get github.com/weaveworks/wordepress && cd /home/ubuntu/src/github.com/weaveworks/wordepress && git checkout v1.0.0 && cd cmd/wordepress && go get
-      - cd $SRCDIR; bin/publish-site "$WP_DEV_URL" "$WP_DEV_USER" "$WP_DEV_PASSWORD"
-  issues:
-    branch: /.*/
-    owner: weaveworks
-    commands:
-      - go get github.com/weaveworks/wordepress && cd /home/ubuntu/src/github.com/weaveworks/wordepress && git checkout v1.0.0 && cd cmd/wordepress && go get
-      - cd $SRCDIR; bin/publish-site "$WP_DEV_URL" "$WP_DEV_USER" "$WP_DEV_PASSWORD"
+      - cd $SRCDIR; make exes publish
+#      - go get github.com/weaveworks/wordepress && cd /home/ubuntu/src/github.com/weaveworks/wordepress && git checkout v1.0.0 && cd cmd/wordepress && go get
+#      - cd $SRCDIR; bin/publish-site "$WP_DEV_URL" "$WP_DEV_USER" "$WP_DEV_PASSWORD"
+#  issues:
+#    branch: /.*/
+#    owner: weaveworks
+#    commands:
+#      - go get github.com/weaveworks/wordepress && cd /home/ubuntu/src/github.com/weaveworks/wordepress && git checkout v1.0.0 && cd cmd/wordepress && go get
+#      - cd $SRCDIR; bin/publish-site "$WP_DEV_URL" "$WP_DEV_USER" "$WP_DEV_PASSWORD"

common/docker/client.go

@@ -84,6 +84,15 @@ func (c *Client) Info() string {
 	return fmt.Sprintf("Docker API on %s: %v", c.Endpoint(), env)
 }
 
+func (c *Client) DockerVersion() string {
+	if env, err := c.Version(); err == nil {
+		if v, found := env.Map()["Version"]; found {
+			return v
+		}
+	}
+	return "unknown"
+}
+
 // AddObserver adds an observer for docker events
 func (c *Client) AddObserver(ob ContainerObserver) error {
 	go func() {

common/http.go

@@ -0,0 +1,18 @@
+package common
+
+import (
+	"net/http"
+)
+
+type loggingHandler struct {
+	next http.Handler
+}
+
+func (l *loggingHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	Log.Debugf("[http] %s %s", r.Method, r.URL.RequestURI())
+	l.next.ServeHTTP(w, r)
+}
+
+func LoggingHTTPHandler(h http.Handler) http.Handler {
+	return &loggingHandler{next: h}
+}

common/utils.go

@@ -2,12 +2,15 @@ package common
 
 import (
 	"fmt"
-	"github.com/vishvananda/netlink"
-	"github.com/vishvananda/netns"
 	"net"
 	"os"
-	"runtime"
 	"strings"
+
+	"github.com/j-keck/arping"
+	"github.com/vishvananda/netlink"
+	"github.com/vishvananda/netns"
+
+	weavenet "github.com/weaveworks/weave/net"
 )
 
 // Assert test is true, panic otherwise
@@ -25,25 +28,6 @@ func ErrorMessages(errors []error) string {
 	return strings.Join(result, "\n")
 }
 
-func WithNetNS(ns netns.NsHandle, work func() error) error {
-	runtime.LockOSThread()
-	defer runtime.UnlockOSThread()
-
-	oldNs, err := netns.Get()
-	if err == nil {
-		defer oldNs.Close()
-
-		err = netns.Set(ns)
-		if err == nil {
-			defer netns.Set(oldNs)
-
-			err = work()
-		}
-	}
-
-	return err
-}
-
 type NetDev struct {
 	Name  string
 	MAC   net.HardwareAddr
@@ -51,6 +35,7 @@ type NetDev struct {
 }
 
 // Search the network namespace of a process for interfaces matching a predicate
+// Note that the predicate is called while the goroutine is inside the process' netns
 func FindNetDevs(processID int, match func(link netlink.Link) bool) ([]NetDev, error) {
 	var netDevs []NetDev
 
@@ -63,7 +48,7 @@ func FindNetDevs(processID int, match func(link netlink.Link) bool) ([]NetDev, e
 	}
 	defer ns.Close()
 
-	err = WithNetNS(ns, func() error {
+	err = weavenet.WithNetNS(ns, func() error {
 		return forEachLink(func(link netlink.Link) error {
 			if match(link) {
 				netDev, err := linkToNetDev(link)
@@ -154,3 +139,20 @@ func GetBridgeNetDev(bridgeName string) ([]NetDev, error) {
 		return link.Attrs().Name == bridgeName
 	})
 }
+
+// Do post-attach configuration of all veths we have created
+func ConfigureARPforVeths(processID int, prefix string) error {
+	_, err := FindNetDevs(processID, func(link netlink.Link) bool {
+		ifName := link.Attrs().Name
+		if strings.HasPrefix(ifName, prefix) {
+			weavenet.ConfigureARPCache(ifName)
+			if addrs, err := netlink.AddrList(link, netlink.FAMILY_V4); err == nil {
+				for _, addr := range addrs {
+					arping.GratuitousArpOverIfaceByName(addr.IPNet.IP, ifName)
+				}
+			}
+		}
+		return false
+	})
+	return err
+}

docs/release-process.md

@@ -6,8 +6,6 @@
 * Create a [github token for
   github-release](https://help.github.com/articles/creating-an-access-token-for-command-line-use/);
 set and export `$GITHUB_TOKEN` with this value
-* Update all dependencies with `make update`
-* Ensure you are still on the right branch, since Go may switch to `master` as part of `make update`
 
 ## Release Types
 
@@ -133,12 +131,13 @@ Finally, for **Mainline** releases only:
 ### Finish up
 
 * If not on master, merge branch into master and push to GitHub.
-* Merge the branch you are on into `latest_release_doc_updates` and push to GitHub.
 * Close the [milestone](https://github.com/weaveworks/weave/milestones) in GitHub and create the next milestone
 * Update the `#weavenetwork` topic heading on freenode (requires 'chanops' permission)
 * For a mainline release vX.Y.0, create a release branch X.Y from the
   tag and push to GitHub - this will result in X.Y.0 site docs being
   published to https://www.weave.works
+* Add the new version of `weave-net` to the checkpoint system at
+  https://checkpoint-api.weave.works/admin
 
 ## Troubleshooting
 

ipam/allocate.go

@@ -33,7 +33,7 @@ func (g *allocate) Try(alloc *Allocator) bool {
 		return true
 	}
 
-	if !alloc.universe.Overlaps(g.r.Range()) {
+	if !alloc.universe.Range().Overlaps(g.r.Range()) {
 		g.resultChan <- allocateResult{err: fmt.Errorf("range %s out of bounds: %s", g.r, alloc.universe)}
 		return true
 	}

ipam/allocator.go

@@ -14,6 +14,7 @@ import (
 	"github.com/weaveworks/weave/ipam/paxos"
 	"github.com/weaveworks/weave/ipam/ring"
 	"github.com/weaveworks/weave/ipam/space"
+	"github.com/weaveworks/weave/ipam/tracker"
 	"github.com/weaveworks/weave/net/address"
 )
 
@@ -55,7 +56,7 @@ type Allocator struct {
 	stopChan          chan<- struct{}
 	ourName           mesh.PeerName
 	seed              []mesh.PeerName          // optional user supplied ring seed
-	universe          address.Range            // superset of all ranges
+	universe          address.CIDR             // superset of all ranges
 	ring              *ring.Ring               // information on ranges owned by all peers
 	space             space.Space              // more detail on ranges owned by us
 	owned             map[string]ownedData     // who owns what addresses, indexed by container-ID
@@ -80,26 +81,39 @@ type Config struct {
 	OurUID      mesh.PeerUID
 	OurNickname string
 	Seed        []mesh.PeerName
-	Universe    address.Range
+	Universe    address.CIDR
 	IsObserver  bool
 	Quorum      func() uint
 	Db          db.DB
 	IsKnownPeer func(name mesh.PeerName) bool
+	Tracker     tracker.LocalRangeTracker
 }
 
 // NewAllocator creates and initialises a new Allocator
 func NewAllocator(config Config) *Allocator {
 	var participant paxos.Participant
+	var alloc *Allocator
+	var onUpdate ring.OnUpdate
+
 	if config.IsObserver {
 		participant = paxos.NewObserver()
 	} else {
 		participant = paxos.NewNode(config.OurName, config.OurUID, 0)
 	}
-	return &Allocator{
+
+	if config.Tracker != nil {
+		onUpdate = func(prev []address.Range, curr []address.Range, local bool) {
+			if err := config.Tracker.HandleUpdate(prev, curr, local); err != nil {
+				alloc.errorf("HandleUpdate failed: %s", err)
+			}
+		}
+	}
+
+	alloc = &Allocator{
 		ourName:     config.OurName,
 		seed:        config.Seed,
 		universe:    config.Universe,
-		ring:        ring.New(config.Universe.Start, config.Universe.End, config.OurName),
+		ring:        ring.New(config.Universe.Range().Start, config.Universe.Range().End, config.OurName, onUpdate),
 		owned:       make(map[string]ownedData),
 		db:          config.Db,
 		paxos:       participant,
@@ -109,6 +123,21 @@ func NewAllocator(config Config) *Allocator {
 		dead:        make(map[string]time.Time),
 		now:         time.Now,
 	}
+	return alloc
+}
+
+func ParseCIDRSubnet(cidrStr string) (cidr address.CIDR, err error) {
+	cidr, err = address.ParseCIDR(cidrStr)
+	if err != nil {
+		return
+	}
+	if !cidr.IsSubnet() {
+		err = fmt.Errorf("invalid subnet - bits after network prefix are not all zero: %s", cidrStr)
+	}
+	if cidr.Size() < MinSubnetSize {
+		err = fmt.Errorf("invalid subnet - smaller than minimum size %d: %s", MinSubnetSize, cidrStr)
+	}
+	return
 }
 
 // Start runs the allocator goroutine
@@ -332,13 +361,13 @@ func (alloc *Allocator) ContainerStarted(ident string) {
 	}
 }
 
-func (alloc *Allocator) AllContainerIDs(ids []string) {
+func (alloc *Allocator) PruneOwned(ids []string) {
+	idmap := make(map[string]struct{}, len(ids))
+	for _, id := range ids {
+		idmap[id] = struct{}{}
+	}
 	alloc.actionChan <- func() {
-		idmap := make(map[string]struct{}, len(ids))
-		for _, id := range ids {
-			idmap[id] = struct{}{}
-		}
-		alloc.syncOwned(idmap)
+		alloc.pruneOwned(idmap)
 	}
 }
 
@@ -414,12 +443,12 @@ func (alloc *Allocator) Shutdown() {
 		alloc.cancelOps(&alloc.pendingClaims)
 		alloc.cancelOps(&alloc.pendingAllocates)
 		alloc.cancelOps(&alloc.pendingPrimes)
-		if heir := alloc.pickPeerForTransfer(); heir != mesh.UnknownPeerName {
-			alloc.ring.Transfer(alloc.ourName, heir)
+		heir := alloc.pickPeerForTransfer()
+		alloc.ring.Transfer(alloc.ourName, heir)
+		alloc.space.Clear()
+		if heir != mesh.UnknownPeerName {
 			alloc.persistRing()
-			alloc.space.Clear()
 			alloc.gossip.GossipBroadcast(alloc.Gossip())
-			time.Sleep(100 * time.Millisecond)
 		}
 		doneChan <- struct{}{}
 	}
@@ -500,6 +529,22 @@ func (alloc *Allocator) annotatePeernames(names []mesh.PeerName) []string {
 	return res
 }
 
+// PeerGone removes nicknames of peers which are no longer mentioned
+// in the ring. Async.
+//
+// NB: the function is invoked by the gossip library routines and should be
+//     registered manually.
+func (alloc *Allocator) PeerGone(peerName mesh.PeerName) {
+	alloc.debugf("PeerGone: peer %s", peerName)
+
+	alloc.actionChan <- func() {
+		ringPeers := alloc.ring.PeerNames()
+		if _, ok := ringPeers[peerName]; !ok {
+			delete(alloc.nicknames, peerName)
+		}
+	}
+}
+
 func decodeRange(msg []byte) (r address.Range, err error) {
 	decoder := gob.NewDecoder(bytes.NewReader(msg))
 	return r, decoder.Decode(&r)
@@ -512,9 +557,11 @@ func (alloc *Allocator) OnGossipUnicast(sender mesh.PeerName, msg []byte) error
 	alloc.actionChan <- func() {
 		switch msg[0] {
 		case msgSpaceRequest:
-			// some other peer asked us for space
+			alloc.debugln("Peer", sender, "asked me for space")
 			r, err := decodeRange(msg[1:])
-			if err == nil {
+			// If we don't have a ring, just ignore a request for space.
+			// They'll probably ask again later.
+			if err == nil && !alloc.ring.Empty() {
 				alloc.donateSpace(r, sender)
 			}
 			resultChan <- err
@@ -744,7 +791,6 @@ func (alloc *Allocator) update(sender mesh.PeerName, msg []byte) error {
 	reader := bytes.NewReader(msg)
 	decoder := gob.NewDecoder(reader)
 	var data gossipState
-	var err error
 
 	if err := decoder.Decode(&data); err != nil {
 		return err
@@ -759,9 +805,10 @@ func (alloc *Allocator) update(sender mesh.PeerName, msg []byte) error {
 	// If someone sent us a ring, merge it into ours. Note this will move us
 	// out of the awaiting-consensus state if we didn't have a ring already.
 	case data.Ring != nil:
-		switch err = alloc.ring.Merge(*data.Ring); err {
+		updated, err := alloc.ring.Merge(*data.Ring)
+		switch err {
 		case nil:
-			if !alloc.ring.Empty() {
+			if updated {
 				alloc.pruneNicknames()
 				alloc.ringUpdated()
 			}
@@ -823,7 +870,6 @@ func (alloc *Allocator) donateSpace(r address.Range, to mesh.PeerName) {
 	// more.
 	defer alloc.sendRingUpdate(to)
 
-	alloc.debugln("Peer", to, "asked me for space")
 	chunk, ok := alloc.space.Donate(r)
 	if !ok {
 		free := alloc.space.NumFreeAddressesInRange(r)
@@ -866,7 +912,9 @@ func (alloc *Allocator) reportFreeSpace() {
 	for _, r := range ranges {
 		freespace[r.Start] = alloc.space.NumFreeAddressesInRange(r)
 	}
-	alloc.ring.ReportFree(freespace)
+	if alloc.ring.ReportFree(freespace) {
+		alloc.persistRing()
+	}
 }
 
 // Persistent data
@@ -921,12 +969,12 @@ func (alloc *Allocator) loadPersistedData() bool {
 		return false
 	}
 
-	if persistedRing.Range() != alloc.universe {
+	if persistedRing.Range() != alloc.universe.Range() {
 		overwritePersisted("Deleting persisted data for IPAM range %s; our range is %s", persistedRing.Range(), alloc.universe)
 		return false
 	}
 
-	alloc.ring = persistedRing
+	alloc.ring.Restore(persistedRing)
 	alloc.space.UpdateRanges(alloc.ring.OwnedRanges())
 
 	if ownedFound {
@@ -1008,7 +1056,7 @@ func (alloc *Allocator) findOwner(addr address.Address) string {
 }
 
 // For each ID in the 'owned' map, remove the entry if it isn't in the map
-func (alloc *Allocator) syncOwned(ids map[string]struct{}) {
+func (alloc *Allocator) pruneOwned(ids map[string]struct{}) {
 	changed := false
 	for ident, d := range alloc.owned {
 		if !d.IsContainer {
@@ -1036,6 +1084,9 @@ func (alloc *Allocator) fatalf(fmt string, args ...interface{}) {
 func (alloc *Allocator) warnf(fmt string, args ...interface{}) {
 	alloc.logf(common.Log.Warnf, fmt, args...)
 }
+func (alloc *Allocator) errorf(fmt string, args ...interface{}) {
+	common.Log.Errorf("[allocator %s] "+fmt, append([]interface{}{alloc.ourName}, args...)...)
+}
 func (alloc *Allocator) infof(fmt string, args ...interface{}) {
 	alloc.logf(common.Log.Infof, fmt, args...)
 }

ipam/allocator_test.go

@@ -220,7 +220,7 @@ func (alloc *Allocator) pause() func() {
 }
 
 func TestCancel(t *testing.T) {
-	const cidr = "10.0.1.7/26"
+	const cidr = "10.0.4.0/26"
 	router := gossip.NewTestRouter(0.0)
 
 	alloc1, subnet := makeAllocator("01:00:00:02:00:00", cidr, 2)
@@ -281,7 +281,7 @@ func TestCancel(t *testing.T) {
 
 func TestCancelOnDied(t *testing.T) {
 	const (
-		cidr       = "10.0.1.7/26"
+		cidr       = "10.0.4.0/26"
 		container1 = "abcdef"
 	)
 
@@ -333,8 +333,23 @@ func TestGossipShutdown(t *testing.T) {
 	CheckAllExpectedMessagesSent(alloc)
 }
 
+func TestNoFrag(t *testing.T) {
+	const cidr = "10.0.4.0/22"
+	resultChan := make(chan int)
+	for i := 0; i < 100; i++ {
+		allocs, router, subnet := makeNetworkOfAllocators(3, cidr)
+		allocs[1].Allocate("foo", subnet, true, returnFalse)
+		allocs[2].Allocate("bar", subnet, true, returnFalse)
+		allocs[2].actionChan <- func() {
+			resultChan <- len(allocs[2].ring.Entries)
+		}
+		require.True(t, <-resultChan < 6, "excessive ring fragmentation")
+		stopNetworkOfAllocators(allocs, router)
+	}
+}
+
 func TestTransfer(t *testing.T) {
-	const cidr = "10.0.1.7/22"
+	const cidr = "10.0.4.0/22"
 	allocs, router, subnet := makeNetworkOfAllocators(3, cidr)
 	defer stopNetworkOfAllocators(allocs, router)
 	alloc0 := allocs[0]
@@ -374,7 +389,7 @@ func TestTransfer(t *testing.T) {
 }
 
 func TestFakeRouterSimple(t *testing.T) {
-	const cidr = "10.0.1.7/22"
+	const cidr = "10.0.4.0/22"
 	allocs, router, subnet := makeNetworkOfAllocators(2, cidr)
 	defer stopNetworkOfAllocators(allocs, router)
 
@@ -392,7 +407,7 @@ func TestAllocatorFuzz(t *testing.T) {
 		nodes        = 5
 		maxAddresses = 1000
 		concurrency  = 5
-		cidr         = "10.0.1.7/22"
+		cidr         = "10.0.4.0/22"
 	)
 	allocs, router, subnet := makeNetworkOfAllocators(nodes, cidr)
 	defer stopNetworkOfAllocators(allocs, router)

ipam/claim.go

@@ -58,7 +58,7 @@ func (c *claim) Try(alloc *Allocator) bool {
 	case mesh.UnknownPeerName:
 		// If our ring doesn't know, it must be empty.
 		alloc.infof("Claim %s for %s: is in the range %s, but the allocator is not initialized yet; will try later.",
-			c.cidr, c.ident, alloc.universe.AsCIDRString())
+			c.cidr, c.ident, alloc.universe)
 		if c.noErrorOnUnknown {
 			c.sendResult(nil)
 		}

ipam/http.go

@@ -17,15 +17,17 @@ func badRequest(w http.ResponseWriter, err error) {
 }
 
 func parseCIDR(w http.ResponseWriter, cidrStr string, net bool) (address.CIDR, bool) {
-	cidr, err := address.ParseCIDR(cidrStr)
+	var cidr address.CIDR
+	var err error
+	if net {
+		cidr, err = ParseCIDRSubnet(cidrStr)
+	} else {
+		cidr, err = address.ParseCIDR(cidrStr)
+	}
 	if err != nil {
 		badRequest(w, err)
 		return address.CIDR{}, false
 	}
-	if net && !cidr.IsSubnet() {
-		badRequest(w, fmt.Errorf("Invalid subnet %s - bits after network prefix are not all zero", cidrStr))
-		return address.CIDR{}, false
-	}
 	return cidr, true
 }
 
@@ -38,7 +40,7 @@ func writeAddresses(w http.ResponseWriter, cidrs []address.CIDR) {
 	}
 }
 
-func hasBeenCanceled(dockerCli *docker.Client, closedChan <-chan bool, ident string, checkAlive bool) func() bool {
+func hasBeenCancelled(dockerCli *docker.Client, closedChan <-chan bool, ident string, checkAlive bool) func() bool {
 	return func() bool {
 		select {
 		case <-closedChan:
@@ -62,7 +64,7 @@ func cancellationErr(w http.ResponseWriter, err error) bool {
 
 func (alloc *Allocator) handleHTTPAllocate(dockerCli *docker.Client, w http.ResponseWriter, ident string, checkAlive bool, subnet address.CIDR) {
 	addr, err := alloc.Allocate(ident, subnet, checkAlive,
-		hasBeenCanceled(dockerCli, w.(http.CloseNotifier).CloseNotify(), ident, checkAlive))
+		hasBeenCancelled(dockerCli, w.(http.CloseNotifier).CloseNotify(), ident, checkAlive))
 	if err != nil {
 		if !cancellationErr(w, err) {
 			badRequest(w, err)
@@ -74,7 +76,7 @@ func (alloc *Allocator) handleHTTPAllocate(dockerCli *docker.Client, w http.Resp
 
 func (alloc *Allocator) handleHTTPClaim(dockerCli *docker.Client, w http.ResponseWriter, ident string, cidr address.CIDR, checkAlive, noErrorOnUnknown bool) {
 	err := alloc.Claim(ident, cidr, checkAlive, noErrorOnUnknown,
-		hasBeenCanceled(dockerCli, w.(http.CloseNotifier).CloseNotify(), ident, checkAlive))
+		hasBeenCancelled(dockerCli, w.(http.CloseNotifier).CloseNotify(), ident, checkAlive))
 	if err != nil {
 		if !cancellationErr(w, err) {
 			badRequest(w, fmt.Errorf("Unable to claim: %s", err))
@@ -85,7 +87,7 @@ func (alloc *Allocator) handleHTTPClaim(dockerCli *docker.Client, w http.Respons
 }
 
 // HandleHTTP wires up ipams HTTP endpoints to the provided mux.
-func (alloc *Allocator) HandleHTTP(router *mux.Router, defaultSubnet address.CIDR, dockerCli *docker.Client) {
+func (alloc *Allocator) HandleHTTP(router *mux.Router, defaultSubnet address.CIDR, tracker string, dockerCli *docker.Client) {
 	router.Methods("GET").Path("/ipinfo/defaultsubnet").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		fmt.Fprintf(w, "%s", defaultSubnet)
 	})
@@ -172,4 +174,8 @@ func (alloc *Allocator) HandleHTTP(router *mux.Router, defaultSubnet address.CID
 		transferred := alloc.AdminTakeoverRanges(ident)
 		fmt.Fprintf(w, "%d IPs taken over from %s\n", transferred, ident)
 	})
+
+	router.Methods("GET").Path("/ipinfo/tracker").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintf(w, tracker)
+	})
 }

ipam/http_test.go

@@ -43,7 +43,7 @@ func listenHTTP(alloc *Allocator, subnet address.CIDR) int {
 	router.HandleFunc("/status", func(w http.ResponseWriter, r *http.Request) {
 		io.WriteString(w, fmt.Sprintln(alloc))
 	})
-	alloc.HandleHTTP(router, subnet, nil)
+	alloc.HandleHTTP(router, subnet, "", nil)
 
 	httpListener, err := net.Listen("tcp", ":0")
 	if err != nil {

ipam/ring/entry.go

@@ -14,8 +14,7 @@ type entry struct {
 	Token   address.Address // The start of this range
 	Peer    mesh.PeerName   // Who owns this range
 	Version uint32          // Version of this range
-	Free    address.Offset  // Number of free IPs in this range
-	// Note: Free should perhaps be an address.Count, but we can't change the wire protocol
+	Free    address.Count   // Number of free IPs in this range
 }
 
 func (e *entry) Equal(e2 *entry) bool {
@@ -23,7 +22,7 @@ func (e *entry) Equal(e2 *entry) bool {
 		e.Version == e2.Version
 }
 
-func (e *entry) update(peername mesh.PeerName, free address.Offset) {
+func (e *entry) update(peername mesh.PeerName, free address.Count) {
 	e.Peer = peername
 	e.Version++
 	e.Free = free

ipam/ring/ring.go

@@ -18,12 +18,15 @@ import (
 	"github.com/weaveworks/weave/net/address"
 )
 
+type OnUpdate func(prev, curr []address.Range, local bool)
+
 // Ring represents the ring itself
 type Ring struct {
 	Start, End address.Address // [min, max) tokens in this ring.  Due to wrapping, min == max (effectively)
 	Peer       mesh.PeerName   // name of peer owning this ring instance
 	Entries    entries         // list of entries sorted by token
 	Seeds      []mesh.PeerName // peers with which the ring was seeded
+	onUpdate   OnUpdate
 }
 
 func (r *Ring) assertInvariants() {
@@ -40,7 +43,6 @@ var (
 	ErrTokenOutOfRange = errors.New("Token is out of range")
 	ErrDifferentSeeds  = errors.New("Received ring was seeded differently from ours!")
 	ErrDifferentRange  = errors.New("Received range differs from ours!")
-	ErrNotFound        = errors.New("No entries for peer found")
 )
 
 func errInconsistentEntry(mine, theirs *entry) error {
@@ -95,27 +97,45 @@ func (r *Ring) checkEntries(entries entries) error {
 	return nil
 }
 
+func (r *Ring) trackUpdates() func() {
+	return r.trackUpdatesOfPeer(r.Peer)
+}
+
+func (r *Ring) trackUpdatesOfPeer(peer mesh.PeerName) func() {
+	if r.onUpdate == nil {
+		return func() {}
+	}
+	ranges := r.OwnedRangesOfPeer(peer)
+	return func() { r.onUpdate(ranges, r.OwnedRangesOfPeer(peer), peer == r.Peer) }
+}
+
 // New creates an empty ring belonging to peer.
-func New(start, end address.Address, peer mesh.PeerName) *Ring {
+func New(start, end address.Address, peer mesh.PeerName, f OnUpdate) *Ring {
 	common.Assert(start < end)
 
-	ring := &Ring{Start: start, End: end, Peer: peer, Entries: make([]*entry, 0)}
+	ring := &Ring{Start: start, End: end, Peer: peer, Entries: make([]*entry, 0), onUpdate: f}
 	ring.updateExportedVariables()
 	return ring
 }
 
+func (r *Ring) Restore(other *Ring) {
+	onUpdate := r.onUpdate
+	*r = *other
+	r.onUpdate = onUpdate
+}
+
 func (r *Ring) Range() address.Range {
 	return address.Range{Start: r.Start, End: r.End}
 }
 
 // Returns the distance between two tokens on this ring, dealing
 // with ranges which cross the origin
-func (r *Ring) distance(start, end address.Address) address.Offset {
+func (r *Ring) distance(start, end address.Address) address.Count {
 	if end > start {
-		return address.Offset(end - start)
+		return address.Count(end - start)
 	}
 
-	return address.Offset((r.End - start) + (end - r.Start))
+	return address.Count((r.End - start) + (end - r.Start))
 }
 
 // GrantRangeToHost modifies the ring such that range [start, end)
@@ -127,6 +147,7 @@ func (r *Ring) GrantRangeToHost(start, end address.Address, peer mesh.PeerName)
 	//fmt.Printf("%s GrantRangeToHost [%v,%v) -> %s\n", r.Peer, start, end, peer)
 
 	r.assertInvariants()
+	defer r.trackUpdates()()
 	defer r.assertInvariants()
 	defer r.updateExportedVariables()
 
@@ -190,42 +211,70 @@ func (r *Ring) GrantRangeToHost(start, end address.Address, peer mesh.PeerName)
 	r.Entries.insert(entry{Token: end, Peer: r.Peer, Free: endFree})
 }
 
-// Merge the given ring into this ring and return any new ranges added
-func (r *Ring) Merge(gossip Ring) error {
+// Merge the given ring into this ring and indicate whether this ring
+// got updated as a result.
+func (r *Ring) Merge(gossip Ring) (bool, error) {
 	r.assertInvariants()
+	defer r.trackUpdates()()
 	defer r.updateExportedVariables()
 
 	// Don't panic when checking the gossiped in ring.
 	// In this case just return any error found.
 	if err := gossip.checkInvariants(); err != nil {
-		return err
+		return false, err
 	}
 
 	if len(gossip.Seeds) > 0 && len(r.Seeds) > 0 {
 		if len(gossip.Seeds) != len(r.Seeds) {
-			return ErrDifferentSeeds
+			return false, ErrDifferentSeeds
 		}
 		for i, seed := range gossip.Seeds {
 			if seed != r.Seeds[i] {
-				return ErrDifferentSeeds
+				return false, ErrDifferentSeeds
 			}
 		}
 	}
 
 	if r.Start != gossip.Start || r.End != gossip.End {
-		return ErrDifferentRange
+		return false, ErrDifferentRange
 	}
 
-	// Now merge their ring with yours, in a temporary ring.
-	var result entries
-	addToResult := func(e entry) { result = append(result, &e) }
+	result, updated, err := r.Entries.merge(gossip.Entries, r.Peer)
 
+	if err != nil {
+		return false, err
+	}
+
+	if err := r.checkEntries(result); err != nil {
+		return false, fmt.Errorf("Merge of incoming data causes: %s", err)
+	}
+
+	if len(r.Seeds) == 0 {
+		r.Seeds = gossip.Seeds
+	}
+	r.Entries = result
+
+	return updated, nil
+}
+
+// Merge other entries into ours, and complain when that stomps on
+// entries belonging to ourPeer. Returns the merged entries and an
+// indication whether the merge resulted in any changes, i.e. the
+// result differs from the original.
+func (es entries) merge(other entries, ourPeer mesh.PeerName) (result entries, updated bool, err error) {
 	var mine, theirs *entry
 	var previousOwner *mesh.PeerName
-	// i is index into r.Entries; j is index into gossip.Entries
+	addToResult := func(e entry) { result = append(result, &e) }
+	addTheirs := func(e entry) {
+		addToResult(e)
+		updated = true
+		previousOwner = nil
+	}
+
+	// i is index into es; j is index into other
 	var i, j int
-	for i < len(r.Entries) && j < len(gossip.Entries) {
-		mine, theirs = r.Entries[i], gossip.Entries[j]
+	for i < len(es) && j < len(other) {
+		mine, theirs = es[i], other[j]
 		switch {
 		case mine.Token < theirs.Token:
 			addToResult(*mine)
@@ -233,59 +282,52 @@ func (r *Ring) Merge(gossip Ring) error {
 			i++
 		case mine.Token > theirs.Token:
 			// insert, checking that a range owned by us hasn't been split
-			if previousOwner != nil && *previousOwner == r.Peer && theirs.Peer != r.Peer {
-				return errEntryInMyRange(theirs)
+			if previousOwner != nil && *previousOwner == ourPeer && theirs.Peer != ourPeer {
+				err = errEntryInMyRange(theirs)
+				return
 			}
-			addToResult(*theirs)
-			previousOwner = nil
+			addTheirs(*theirs)
 			j++
 		case mine.Token == theirs.Token:
 			// merge
 			switch {
 			case mine.Version >= theirs.Version:
 				if mine.Version == theirs.Version && !mine.Equal(theirs) {
-					return errInconsistentEntry(mine, theirs)
+					err = errInconsistentEntry(mine, theirs)
+					return
 				}
 				addToResult(*mine)
 				previousOwner = &mine.Peer
 			case mine.Version < theirs.Version:
-				if mine.Peer == r.Peer { // We shouldn't receive updates to our own tokens
-					return errNewerVersion(mine, theirs)
+				if mine.Peer == ourPeer { // We shouldn't receive updates to our own tokens
+					err = errNewerVersion(mine, theirs)
+					return
 				}
-				addToResult(*theirs)
-				previousOwner = nil
+				addTheirs(*theirs)
 			}
 			i++
 			j++
 		}
 	}
 
-	// At this point, either i is at the end of r or j is at the end
-	// of gossip, so copy over the remaining entries.
+	// At this point, either i is at the end of es or j is at the end
+	// of other, so copy over the remaining entries.
 
-	for ; i < len(r.Entries); i++ {
-		mine = r.Entries[i]
+	for ; i < len(es); i++ {
+		mine = es[i]
 		addToResult(*mine)
 	}
 
-	for ; j < len(gossip.Entries); j++ {
-		theirs = gossip.Entries[j]
-		if previousOwner != nil && *previousOwner == r.Peer && theirs.Peer != r.Peer {
-			return errEntryInMyRange(theirs)
+	for ; j < len(other); j++ {
+		theirs = other[j]
+		if previousOwner != nil && *previousOwner == ourPeer && theirs.Peer != ourPeer {
+			err = errEntryInMyRange(theirs)
+			return
 		}
-		addToResult(*theirs)
-		previousOwner = nil
+		addTheirs(*theirs)
 	}
 
-	if err := r.checkEntries(result); err != nil {
-		return fmt.Errorf("Merge of incoming data causes: %s", err)
-	}
-
-	if len(r.Seeds) == 0 {
-		r.Seeds = gossip.Seeds
-	}
-	r.Entries = result
-	return nil
+	return
 }
 
 // Empty returns true if the ring has no entries
@@ -318,10 +360,14 @@ func (r *Ring) splitRangesOverZero(ranges []address.Range) []address.Range {
 // ranges are owned by this peer.  Will split ranges which
 // span 0 in the ring.
 func (r *Ring) OwnedRanges() (result []address.Range) {
+	return r.OwnedRangesOfPeer(r.Peer)
+}
+
+func (r *Ring) OwnedRangesOfPeer(peer mesh.PeerName) (result []address.Range) {
 	r.assertInvariants()
 
 	for i, entry := range r.Entries {
-		if entry.Peer == r.Peer {
+		if entry.Peer == peer {
 			nextEntry := r.Entries.entry(i + 1)
 			result = append(result, address.Range{Start: entry.Token, End: nextEntry.Token})
 		}
@@ -350,34 +396,38 @@ func (r *Ring) AllRangeInfo() (result []RangeInfo) {
 }
 
 // ClaimForPeers claims the entire ring for the array of peers passed
-// in.  Only works for empty rings.
+// in.  Only works for empty rings. Each claimed range is CIDR-aligned.
 func (r *Ring) ClaimForPeers(peers []mesh.PeerName) {
 	common.Assert(r.Empty())
+
+	defer r.trackUpdates()()
 	defer r.assertInvariants()
 	defer r.updateExportedVariables()
+	defer func() {
+		e := r.Entries[len(r.Entries)-1]
+		common.Assert(address.Add(e.Token, address.Offset(e.Free)) == r.End)
+	}()
 
-	totalSize := r.distance(r.Start, r.End)
-	share := totalSize/address.Offset(len(peers)) + 1
-	remainder := totalSize % address.Offset(len(peers))
-	pos := r.Start
-
-	for i, peer := range peers {
-		if address.Offset(i) == remainder {
-			share--
-			if share == 0 {
-				break
-			}
-		}
-
-		r.Entries.insert(entry{Token: pos, Peer: peer, Free: share})
-		pos += address.Address(share)
-	}
-
-	common.Assert(pos == r.End)
-
+	r.subdivide(r.Start, r.End, peers)
 	r.Seeds = peers
 }
 
+// subdivide subdivides the [from,to) CIDR for the given peers into
+// CIDR-aligned subranges.
+func (r *Ring) subdivide(from, to address.Address, peers []mesh.PeerName) {
+	share := address.Length(to, from)
+	if share == 0 {
+		return
+	}
+	if share == 1 || len(peers) == 1 {
+		r.Entries.insert(entry{Token: from, Peer: peers[0], Free: share})
+		return
+	}
+	mid := address.Add(from, address.Offset(share/2))
+	r.subdivide(from, mid, peers[:len(peers)/2])
+	r.subdivide(address.Add(mid, address.Offset(share%2)), to, peers[len(peers)/2:])
+}
+
 func (r *Ring) FprintWithNicknames(w io.Writer, m map[mesh.PeerName]string) {
 	for _, entry := range r.Entries {
 		nickname, found := m[entry.Peer]
@@ -399,8 +449,8 @@ func (r *Ring) String() string {
 
 // ReportFree is used by the allocator to tell the ring how many free
 // ips are in a given range, so that ChoosePeersToAskForSpace can make
-// more intelligent decisions.
-func (r *Ring) ReportFree(freespace map[address.Address]address.Count) {
+// more intelligent decisions.  Returns true if any changes made.
+func (r *Ring) ReportFree(freespace map[address.Address]address.Count) (updated bool) {
 	r.assertInvariants()
 	defer r.assertInvariants()
 	defer r.updateExportedVariables()
@@ -432,13 +482,15 @@ func (r *Ring) ReportFree(freespace map[address.Address]address.Count) {
 		maxSize := r.distance(entry.Token, next.Token)
 		common.Assert(free <= address.Count(maxSize))
 
-		if address.Count(entries[i].Free) == free {
-			return
+		if entries[i].Free == free {
+			continue
 		}
 
-		entries[i].Free = address.Offset(free)
+		entries[i].Free = free
 		entries[i].Version++
+		updated = true
 	}
+	return
 }
 
 type weightedPeer struct {
@@ -455,7 +507,7 @@ func (ws weightedPeers) Swap(i, j int)      { ws[i], ws[j] = ws[j], ws[i] }
 // ChoosePeersToAskForSpace returns all peers we can ask for space in
 // the range [start, end), in weighted-random order.  Assumes start<end.
 func (r *Ring) ChoosePeersToAskForSpace(start, end address.Address) []mesh.PeerName {
-	totalSpacePerPeer := make(map[mesh.PeerName]address.Offset)
+	totalSpacePerPeer := make(map[mesh.PeerName]address.Count)
 
 	// iterate through tokens
 	for i, entry := range r.Entries {
@@ -507,6 +559,8 @@ func (r *Ring) PickPeerForTransfer(isValid func(mesh.PeerName) bool) mesh.PeerNa
 // and return ranges indicating the new space we picked up
 func (r *Ring) Transfer(from, to mesh.PeerName) []address.Range {
 	r.assertInvariants()
+	defer r.trackUpdates()()
+	defer r.trackUpdatesOfPeer(from)()
 	defer r.assertInvariants()
 	defer r.updateExportedVariables()
 

ipam/ring/ring_test.go

@@ -19,7 +19,8 @@ var (
 	peer2name, _ = mesh.PeerNameFromString("02:00:00:00:02:00")
 	peer3name, _ = mesh.PeerNameFromString("03:00:00:00:02:00")
 
-	start, end    = ParseIP("10.0.0.0"), ParseIP("10.0.0.255")
+	start, end    = ParseIP("10.0.0.0"), ParseIP("10.0.1.0")
+	dot8          = ParseIP("10.0.0.8")
 	dot10, dot245 = ParseIP("10.0.0.10"), ParseIP("10.0.0.245")
 	dot250        = ParseIP("10.0.0.250")
 	middle        = ParseIP("10.0.0.128")
@@ -30,8 +31,17 @@ func ParseIP(s string) address.Address {
 	return addr
 }
 
+func merge(r1, r2 *Ring) error {
+	_, err := r1.Merge(*r2)
+	return err
+}
+
+func NewRing(start, end address.Address, peer mesh.PeerName) *Ring {
+	return New(start, end, peer, nil)
+}
+
 func TestInvariants(t *testing.T) {
-	ring := New(start, end, peer1name)
+	ring := NewRing(start, end, peer1name)
 
 	// Check ring is sorted
 	ring.Entries = []*entry{{Token: dot245, Peer: peer1name}, {Token: dot10, Peer: peer2name}}
@@ -42,7 +52,7 @@ func TestInvariants(t *testing.T) {
 	require.True(t, ring.checkInvariants() == ErrTokenRepeated, "Expected error")
 
 	// Check tokens are in bounds
-	ring = New(dot10, dot245, peer1name)
+	ring = NewRing(dot10, dot245, peer1name)
 	ring.Entries = []*entry{{Token: start, Peer: peer1name}}
 	require.True(t, ring.checkInvariants() == ErrTokenOutOfRange, "Expected error")
 
@@ -51,7 +61,7 @@ func TestInvariants(t *testing.T) {
 }
 
 func TestInsert(t *testing.T) {
-	ring := New(start, end, peer1name)
+	ring := NewRing(start, end, peer1name)
 	ring.Entries = []*entry{{Token: start, Peer: peer1name, Free: 255}}
 
 	require.Panics(t, func() {
@@ -76,7 +86,7 @@ func TestInsert(t *testing.T) {
 }
 
 func TestBetween(t *testing.T) {
-	ring1 := New(start, end, peer1name)
+	ring1 := NewRing(start, end, peer1name)
 	ring1.Entries = []*entry{{Token: start, Peer: peer1name, Free: 255}}
 
 	// First off, in a ring where everything is owned by the peer
@@ -118,29 +128,29 @@ func TestBetween(t *testing.T) {
 }
 
 func TestGrantSimple(t *testing.T) {
-	ring1 := New(start, end, peer1name)
-	ring2 := New(start, end, peer2name)
+	ring1 := NewRing(start, end, peer1name)
+	ring2 := NewRing(start, end, peer2name)
 
 	// Claim everything for peer1
 	ring1.ClaimItAll()
-	require.Equal(t, entries{{Token: start, Peer: peer1name, Free: 255}}, ring1.Entries)
+	require.Equal(t, entries{{Token: start, Peer: peer1name, Free: 256}}, ring1.Entries)
 
 	// Now grant everything to peer2
 	ring1.GrantRangeToHost(start, end, peer2name)
-	ring2.Entries = []*entry{{Token: start, Peer: peer2name, Free: 255, Version: 1}}
+	ring2.Entries = []*entry{{Token: start, Peer: peer2name, Free: 256, Version: 1}}
 	require.Equal(t, ring2.Entries, ring1.Entries)
 
 	// Now spint back to peer 1
 	ring2.GrantRangeToHost(dot10, end, peer1name)
 	ring1.Entries = []*entry{{Token: start, Peer: peer2name, Free: 10, Version: 2},
-		{Token: dot10, Peer: peer1name, Free: 245}}
+		{Token: dot10, Peer: peer1name, Free: 246}}
 	require.Equal(t, ring2.Entries, ring1.Entries)
 
 	// And spint back to peer 2 again
 	ring1.GrantRangeToHost(dot245, end, peer2name)
 	require.Equal(t, entries{{Token: start, Peer: peer2name, Free: 10, Version: 2},
 		{Token: dot10, Peer: peer1name, Free: 235, Version: 1},
-		{Token: dot245, Peer: peer2name, Free: 10}}, ring1.Entries)
+		{Token: dot245, Peer: peer2name, Free: 11}}, ring1.Entries)
 
 	// Grant range spanning a live token
 	ring1.Entries = []*entry{{Token: start, Peer: peer1name, Free: 10, Version: 2},
@@ -153,19 +163,19 @@ func TestGrantSimple(t *testing.T) {
 }
 
 func TestGrantSplit(t *testing.T) {
-	ring1 := New(start, end, peer1name)
-	ring2 := New(start, end, peer2name)
+	ring1 := NewRing(start, end, peer1name)
+	ring2 := NewRing(start, end, peer2name)
 
 	// Claim everything for peer1
-	ring1.Entries = []*entry{{Token: start, Peer: peer1name, Free: 255}}
-	ring2.Merge(*ring1)
+	ring1.Entries = []*entry{{Token: start, Peer: peer1name, Free: 256}}
+	merge(ring2, ring1)
 	require.Equal(t, ring2.Entries, ring1.Entries)
 
 	// Now grant a split range to peer2
 	ring1.GrantRangeToHost(dot10, dot245, peer2name)
 	require.Equal(t, entries{{Token: start, Peer: peer1name, Free: 10, Version: 1},
 		{Token: dot10, Peer: peer2name, Free: 235},
-		{Token: dot245, Peer: peer1name, Free: 10}}, ring1.Entries)
+		{Token: dot245, Peer: peer1name, Free: 11}}, ring1.Entries)
 
 	ring1.assertInvariants()
 
@@ -176,22 +186,22 @@ func TestGrantSplit(t *testing.T) {
 	require.Equal(t, entries{{Token: start, Peer: peer1name, Free: 10, Version: 2},
 		{Token: dot10, Peer: peer2name, Free: 118, Version: 1},
 		{Token: middle, Peer: peer2name, Free: 117, Version: 1},
-		{Token: dot245, Peer: peer1name, Free: 10, Version: 0}}, ring1.Entries)
+		{Token: dot245, Peer: peer1name, Free: 11, Version: 0}}, ring1.Entries)
 
 	ring1.assertInvariants()
 }
 
 func TestMergeSimple(t *testing.T) {
-	ring1 := New(start, end, peer1name)
-	ring2 := New(start, end, peer2name)
+	ring1 := NewRing(start, end, peer1name)
+	ring2 := NewRing(start, end, peer2name)
 
 	// Claim everything for peer1
 	ring1.ClaimItAll()
 	ring1.GrantRangeToHost(middle, end, peer2name)
-	require.NoError(t, ring2.Merge(*ring1))
+	require.NoError(t, merge(ring2, ring1))
 
 	require.Equal(t, entries{{Token: start, Peer: peer1name, Free: 128, Version: 1},
-		{Token: middle, Peer: peer2name, Free: 127}}, ring1.Entries)
+		{Token: middle, Peer: peer2name, Free: 128}}, ring1.Entries)
 
 	require.Equal(t, ring2.Entries, ring1.Entries)
 
@@ -200,48 +210,48 @@ func TestMergeSimple(t *testing.T) {
 	ring1.GrantRangeToHost(start, middle, peer2name)
 	ring2.GrantRangeToHost(middle, end, peer1name)
 
-	require.NoError(t, ring2.Merge(*ring1))
-	require.NoError(t, ring1.Merge(*ring2))
+	require.NoError(t, merge(ring2, ring1))
+	require.NoError(t, merge(ring1, ring2))
 
 	require.Equal(t, entries{{Token: start, Peer: peer2name, Free: 128, Version: 2},
-		{Token: middle, Peer: peer1name, Free: 127, Version: 1}}, ring1.Entries)
+		{Token: middle, Peer: peer1name, Free: 128, Version: 1}}, ring1.Entries)
 
 	require.Equal(t, ring2.Entries, ring1.Entries)
 }
 
 func TestMergeErrors(t *testing.T) {
 	// Cannot Merge in an invalid ring
-	ring1 := New(start, end, peer1name)
-	ring2 := New(start, end, peer2name)
+	ring1 := NewRing(start, end, peer1name)
+	ring2 := NewRing(start, end, peer2name)
 	ring2.Entries = []*entry{{Token: middle, Peer: peer2name}, {Token: start, Peer: peer2name}}
-	require.True(t, ring1.Merge(*ring2) == ErrNotSorted, "Expected ErrNotSorted")
+	require.True(t, merge(ring1, ring2) == ErrNotSorted, "Expected ErrNotSorted")
 
 	// Should Merge two rings for different ranges
-	ring2 = New(start, middle, peer2name)
+	ring2 = NewRing(start, middle, peer2name)
 	ring2.Entries = []*entry{}
-	require.True(t, ring1.Merge(*ring2) == ErrDifferentRange, "Expected ErrDifferentRange")
+	require.True(t, merge(ring1, ring2) == ErrDifferentRange, "Expected ErrDifferentRange")
 
 	// Cannot Merge newer version of entry I own
-	ring2 = New(start, end, peer2name)
+	ring2 = NewRing(start, end, peer2name)
 	ring1.Entries = []*entry{{Token: start, Peer: peer1name}}
 	ring2.Entries = []*entry{{Token: start, Peer: peer1name, Version: 1}}
-	fmt.Println(ring1.Merge(*ring2))
-	require.Error(t, ring1.Merge(*ring2), "Expected error")
+	fmt.Println(merge(ring1, ring2))
+	require.Error(t, merge(ring1, ring2), "Expected error")
 
 	// Cannot Merge two entries with same version but different hosts
 	ring1.Entries = []*entry{{Token: start, Peer: peer1name}}
 	ring2.Entries = []*entry{{Token: start, Peer: peer2name}}
-	require.Error(t, ring1.Merge(*ring2), "Expected error")
+	require.Error(t, merge(ring1, ring2), "Expected error")
 
 	// Cannot Merge an entry into a range I own
 	ring1.Entries = []*entry{{Token: start, Peer: peer1name}}
 	ring2.Entries = []*entry{{Token: middle, Peer: peer2name}}
-	require.Error(t, ring1.Merge(*ring2), "Expected error")
+	require.Error(t, merge(ring1, ring2), "Expected error")
 }
 
 func TestMergeMore(t *testing.T) {
-	ring1 := New(start, end, peer1name)
-	ring2 := New(start, end, peer2name)
+	ring1 := NewRing(start, end, peer1name)
+	ring2 := NewRing(start, end, peer2name)
 
 	assertRing := func(ring *Ring, entries entries) {
 		require.Equal(t, entries, ring.Entries)
@@ -252,74 +262,74 @@ func TestMergeMore(t *testing.T) {
 
 	// Claim everything for peer1
 	ring1.ClaimItAll()
-	assertRing(ring1, []*entry{{Token: start, Peer: peer1name, Free: 255}})
+	assertRing(ring1, []*entry{{Token: start, Peer: peer1name, Free: 256}})
 	assertRing(ring2, []*entry{})
 
 	// Check the Merge sends it to the other ring
-	require.NoError(t, ring2.Merge(*ring1))
-	assertRing(ring1, []*entry{{Token: start, Peer: peer1name, Free: 255}})
-	assertRing(ring2, []*entry{{Token: start, Peer: peer1name, Free: 255}})
+	require.NoError(t, merge(ring2, ring1))
+	assertRing(ring1, []*entry{{Token: start, Peer: peer1name, Free: 256}})
+	assertRing(ring2, []*entry{{Token: start, Peer: peer1name, Free: 256}})
 
 	// Give everything to peer2
 	ring1.GrantRangeToHost(start, end, peer2name)
-	assertRing(ring1, []*entry{{Token: start, Peer: peer2name, Free: 255, Version: 1}})
-	assertRing(ring2, []*entry{{Token: start, Peer: peer1name, Free: 255}})
+	assertRing(ring1, []*entry{{Token: start, Peer: peer2name, Free: 256, Version: 1}})
+	assertRing(ring2, []*entry{{Token: start, Peer: peer1name, Free: 256}})
 
-	require.NoError(t, ring2.Merge(*ring1))
-	assertRing(ring1, []*entry{{Token: start, Peer: peer2name, Free: 255, Version: 1}})
-	assertRing(ring2, []*entry{{Token: start, Peer: peer2name, Free: 255, Version: 1}})
+	require.NoError(t, merge(ring2, ring1))
+	assertRing(ring1, []*entry{{Token: start, Peer: peer2name, Free: 256, Version: 1}})
+	assertRing(ring2, []*entry{{Token: start, Peer: peer2name, Free: 256, Version: 1}})
 
 	// And carve off some space
 	ring2.GrantRangeToHost(middle, end, peer1name)
 	assertRing(ring2, []*entry{{Token: start, Peer: peer2name, Free: 128, Version: 2},
-		{Token: middle, Peer: peer1name, Free: 127}})
-	assertRing(ring1, []*entry{{Token: start, Peer: peer2name, Free: 255, Version: 1}})
+		{Token: middle, Peer: peer1name, Free: 128}})
+	assertRing(ring1, []*entry{{Token: start, Peer: peer2name, Free: 256, Version: 1}})
 
 	// And Merge back
-	require.NoError(t, ring1.Merge(*ring2))
+	require.NoError(t, merge(ring1, ring2))
 	assertRing(ring1, []*entry{{Token: start, Peer: peer2name, Free: 128, Version: 2},
-		{Token: middle, Peer: peer1name, Free: 127}})
+		{Token: middle, Peer: peer1name, Free: 128}})
 	assertRing(ring2, []*entry{{Token: start, Peer: peer2name, Free: 128, Version: 2},
-		{Token: middle, Peer: peer1name, Free: 127}})
+		{Token: middle, Peer: peer1name, Free: 128}})
 
 	// This should be a no-op
-	require.NoError(t, ring2.Merge(*ring1))
+	require.NoError(t, merge(ring2, ring1))
 	assertRing(ring1, []*entry{{Token: start, Peer: peer2name, Free: 128, Version: 2},
-		{Token: middle, Peer: peer1name, Free: 127}})
+		{Token: middle, Peer: peer1name, Free: 128}})
 	assertRing(ring2, []*entry{{Token: start, Peer: peer2name, Free: 128, Version: 2},
-		{Token: middle, Peer: peer1name, Free: 127}})
+		{Token: middle, Peer: peer1name, Free: 128}})
 }
 
 func TestMergeSplit(t *testing.T) {
-	ring1 := New(start, end, peer1name)
-	ring2 := New(start, end, peer2name)
+	ring1 := NewRing(start, end, peer1name)
+	ring2 := NewRing(start, end, peer2name)
 
 	// Claim everything for peer2
-	ring1.Entries = []*entry{{Token: start, Peer: peer2name, Free: 255}}
-	require.NoError(t, ring2.Merge(*ring1))
+	ring1.Entries = []*entry{{Token: start, Peer: peer2name, Free: 256}}
+	require.NoError(t, merge(ring2, ring1))
 	require.Equal(t, ring2.Entries, ring1.Entries)
 
 	// Now grant a split range to peer1
 	ring2.GrantRangeToHost(dot10, dot245, peer1name)
 	require.Equal(t, entries{{Token: start, Peer: peer2name, Free: 10, Version: 1},
 		{Token: dot10, Peer: peer1name, Free: 235},
-		{Token: dot245, Peer: peer2name, Free: 10}}, ring2.Entries)
+		{Token: dot245, Peer: peer2name, Free: 11}}, ring2.Entries)
 
-	require.NoError(t, ring1.Merge(*ring2))
+	require.NoError(t, merge(ring1, ring2))
 	require.Equal(t, entries{{Token: start, Peer: peer2name, Free: 10, Version: 1},
 		{Token: dot10, Peer: peer1name, Free: 235},
-		{Token: dot245, Peer: peer2name, Free: 10}}, ring1.Entries)
+		{Token: dot245, Peer: peer2name, Free: 11}}, ring1.Entries)
 
 	require.Equal(t, ring2.Entries, ring1.Entries)
 }
 
 func TestMergeSplit2(t *testing.T) {
-	ring1 := New(start, end, peer1name)
-	ring2 := New(start, end, peer2name)
+	ring1 := NewRing(start, end, peer1name)
+	ring2 := NewRing(start, end, peer2name)
 
 	// Claim everything for peer2
 	ring1.Entries = []*entry{{Token: start, Peer: peer2name, Free: 250}, {Token: dot250, Peer: peer2name, Free: 5}}
-	require.NoError(t, ring2.Merge(*ring1))
+	require.NoError(t, merge(ring2, ring1))
 	require.Equal(t, ring2.Entries, ring1.Entries)
 
 	// Now grant a split range to peer1
@@ -328,7 +338,7 @@ func TestMergeSplit2(t *testing.T) {
 		{Token: dot10, Peer: peer1name, Free: 235},
 		{Token: dot245, Peer: peer2name, Free: 5}, {Token: dot250, Peer: peer2name, Free: 5}}, ring2.Entries)
 
-	require.NoError(t, ring1.Merge(*ring2))
+	require.NoError(t, merge(ring1, ring2))
 	require.Equal(t, entries{{Token: start, Peer: peer2name, Free: 10, Version: 1},
 		{Token: dot10, Peer: peer1name, Free: 235},
 		{Token: dot245, Peer: peer2name, Free: 5}, {Token: dot250, Peer: peer2name, Free: 5}}, ring1.Entries)
@@ -338,8 +348,8 @@ func TestMergeSplit2(t *testing.T) {
 
 // A simple test, very similar to above, but using the marshalling to byte[]s
 func TestGossip(t *testing.T) {
-	ring1 := New(start, end, peer1name)
-	ring2 := New(start, end, peer2name)
+	ring1 := NewRing(start, end, peer1name)
+	ring2 := NewRing(start, end, peer2name)
 
 	assertRing := func(ring *Ring, entries entries) {
 		require.Equal(t, entries, ring.Entries)
@@ -350,13 +360,13 @@ func TestGossip(t *testing.T) {
 
 	// Claim everything for peer1
 	ring1.ClaimItAll()
-	assertRing(ring1, []*entry{{Token: start, Peer: peer1name, Free: 255}})
+	assertRing(ring1, []*entry{{Token: start, Peer: peer1name, Free: 256}})
 	assertRing(ring2, []*entry{})
 
 	// Check the Merge sends it to the other ring
-	require.NoError(t, ring2.Merge(*ring1))
-	assertRing(ring1, []*entry{{Token: start, Peer: peer1name, Free: 255}})
-	assertRing(ring2, []*entry{{Token: start, Peer: peer1name, Free: 255}})
+	require.NoError(t, merge(ring2, ring1))
+	assertRing(ring1, []*entry{{Token: start, Peer: peer1name, Free: 256}})
+	assertRing(ring2, []*entry{{Token: start, Peer: peer1name, Free: 256}})
 }
 
 func assertPeersWithSpace(t *testing.T, ring *Ring, start, end address.Address, expected int) []mesh.PeerName {
@@ -366,7 +376,7 @@ func assertPeersWithSpace(t *testing.T, ring *Ring, start, end address.Address,
 }
 
 func TestFindFree(t *testing.T) {
-	ring1 := New(start, end, peer1name)
+	ring1 := NewRing(start, end, peer1name)
 
 	assertPeersWithSpace(t, ring1, start, end, 0)
 
@@ -402,12 +412,12 @@ func TestFindFree(t *testing.T) {
 }
 
 func TestReportFree(t *testing.T) {
-	ring1 := New(start, end, peer1name)
-	ring2 := New(start, end, peer2name)
+	ring1 := NewRing(start, end, peer1name)
+	ring2 := NewRing(start, end, peer2name)
 
 	ring1.ClaimItAll()
 	ring1.GrantRangeToHost(middle, end, peer2name)
-	require.NoError(t, ring2.Merge(*ring1))
+	require.NoError(t, merge(ring2, ring1))
 
 	freespace := make(map[address.Address]address.Count)
 	for _, r := range ring2.OwnedRanges() {
@@ -417,7 +427,7 @@ func TestReportFree(t *testing.T) {
 }
 
 func TestMisc(t *testing.T) {
-	ring := New(start, end, peer1name)
+	ring := NewRing(start, end, peer1name)
 
 	require.True(t, ring.Empty(), "empty")
 
@@ -426,38 +436,38 @@ func TestMisc(t *testing.T) {
 }
 
 func TestEmptyGossip(t *testing.T) {
-	ring1 := New(start, end, peer1name)
-	ring2 := New(start, end, peer2name)
+	ring1 := NewRing(start, end, peer1name)
+	ring2 := NewRing(start, end, peer2name)
 
 	ring1.ClaimItAll()
 	// This used to panic, and it shouldn't
-	require.NoError(t, ring1.Merge(*ring2))
+	require.NoError(t, merge(ring1, ring2))
 }
 
 func TestMergeOldMessage(t *testing.T) {
-	ring1 := New(start, end, peer1name)
-	ring2 := New(start, end, peer2name)
+	ring1 := NewRing(start, end, peer1name)
+	ring2 := NewRing(start, end, peer2name)
 
 	ring1.ClaimItAll()
-	require.NoError(t, ring2.Merge(*ring1))
+	require.NoError(t, merge(ring2, ring1))
 
 	ring1.GrantRangeToHost(middle, end, peer1name)
-	require.NoError(t, ring1.Merge(*ring2))
+	require.NoError(t, merge(ring1, ring2))
 }
 
 func TestSplitRangeAtBeginning(t *testing.T) {
-	ring1 := New(start, end, peer1name)
-	ring2 := New(start, end, peer2name)
+	ring1 := NewRing(start, end, peer1name)
+	ring2 := NewRing(start, end, peer2name)
 
 	ring1.ClaimItAll()
-	require.NoError(t, ring2.Merge(*ring1))
+	require.NoError(t, merge(ring2, ring1))
 
 	ring1.GrantRangeToHost(start, middle, peer2name)
-	require.NoError(t, ring2.Merge(*ring1))
+	require.NoError(t, merge(ring2, ring1))
 }
 
 func TestOwnedRange(t *testing.T) {
-	ring1 := New(start, end, peer1name)
+	ring1 := NewRing(start, end, peer1name)
 	ring1.ClaimItAll()
 
 	require.Equal(t, []address.Range{{Start: start, End: end}}, ring1.OwnedRanges())
@@ -465,8 +475,8 @@ func TestOwnedRange(t *testing.T) {
 	ring1.GrantRangeToHost(middle, end, peer2name)
 	require.Equal(t, []address.Range{{Start: start, End: middle}}, ring1.OwnedRanges())
 
-	ring2 := New(start, end, peer2name)
-	ring2.Merge(*ring1)
+	ring2 := NewRing(start, end, peer2name)
+	merge(ring2, ring1)
 	require.Equal(t, []address.Range{{Start: middle, End: end}}, ring2.OwnedRanges())
 
 	ring2.Entries = []*entry{{Token: middle, Peer: peer2name}}
@@ -480,7 +490,7 @@ func TestOwnedRange(t *testing.T) {
 
 func TestTransfer(t *testing.T) {
 	// First test just checks if we can grant some range to a host, when we transfer it, we get it back
-	ring1 := New(start, end, peer1name)
+	ring1 := NewRing(start, end, peer1name)
 	ring1.ClaimItAll()
 	ring1.GrantRangeToHost(middle, end, peer2name)
 	ring1.Transfer(peer2name, peer1name)
@@ -488,7 +498,7 @@ func TestTransfer(t *testing.T) {
 
 	// Second test is what happens when a token exists at the end of a range but is transferred
 	// - does it get resurrected correctly?
-	ring1 = New(start, end, peer1name)
+	ring1 = NewRing(start, end, peer1name)
 	ring1.ClaimItAll()
 	ring1.GrantRangeToHost(middle, end, peer2name)
 	ring1.Transfer(peer2name, peer1name)
@@ -497,7 +507,7 @@ func TestTransfer(t *testing.T) {
 }
 
 func TestOwner(t *testing.T) {
-	ring1 := New(start, end, peer1name)
+	ring1 := NewRing(start, end, peer1name)
 	require.True(t, ring1.Contains(start), "start should be in ring")
 	require.False(t, ring1.Contains(end), "end should not be in ring")
 
@@ -521,17 +531,24 @@ func makePeerName(i int) mesh.PeerName {
 	return peer
 }
 
+func makePeers(numPeers int) []mesh.PeerName {
+	peers := make([]mesh.PeerName, numPeers)
+	for i := 0; i < numPeers; i++ {
+		peers[i] = makePeerName(i)
+	}
+	return peers
+}
+
 func TestClaimForPeers(t *testing.T) {
 	const numPeers = 12
 	// Different end to usual so we get a number of addresses that a)
 	// is smaller than the max number of peers, and b) is divisible by
 	// some number of peers. This maximises coverage of edge cases.
-	end := dot10
-	peers := make([]mesh.PeerName, numPeers)
+	end := dot8
+	peers := makePeers(numPeers)
 	// Test for a range of peer counts
 	for i := 0; i < numPeers; i++ {
-		peers[i] = makePeerName(i)
-		ring := New(start, end, peers[0])
+		ring := NewRing(start, end, peers[0])
 		ring.ClaimForPeers(peers[:i+1])
 	}
 }
@@ -569,7 +586,7 @@ func TestFuzzRing(t *testing.T) {
 		sort.Sort(addressSlice(tokens))
 
 		peer := peers[rand.Intn(len(peers))]
-		ring := New(start, end, peer)
+		ring := NewRing(start, end, peer)
 		for _, token := range tokens {
 			peer = peers[rand.Intn(len(peers))]
 			ring.Entries = append(ring.Entries, &entry{Token: start + token, Peer: peer})
@@ -586,7 +603,7 @@ func TestFuzzRing(t *testing.T) {
 
 		// Merge them - this might fail, we don't care
 		// We just want to make sure it doesn't panic
-		ring1.Merge(*ring2)
+		merge(ring1, ring2)
 
 		// Check whats left still passes assertions
 		ring1.assertInvariants()
@@ -603,7 +620,7 @@ func TestFuzzRing(t *testing.T) {
 		}
 
 		peer := peers[rand.Intn(len(peers))]
-		ring := New(start, end, peer)
+		ring := NewRing(start, end, peer)
 		for _, token := range tokens {
 			peer = peers[rand.Intn(len(peers))]
 			ring.Entries = append(ring.Entries, &entry{Token: start + token, Peer: peer})
@@ -619,7 +636,7 @@ func TestFuzzRing(t *testing.T) {
 
 		// Merge them - this might fail, we don't care
 		// We just want to make sure it doesn't panic
-		ring1.Merge(*ring2)
+		merge(ring1, ring2)
 
 		// Check whats left still passes assertions
 		ring1.assertInvariants()
@@ -641,7 +658,7 @@ func TestFuzzRingHard(t *testing.T) {
 		common.Log.Debugf("%s: Adding peer", peer)
 		nextPeerID++
 		peers = append(peers, peer)
-		rings = append(rings, New(start, end, peer))
+		rings = append(rings, NewRing(start, end, peer))
 	}
 
 	for i := 0; i < numPeers; i++ {
@@ -680,7 +697,7 @@ func TestFuzzRingHard(t *testing.T) {
 
 		// We need to be in a ~converged ring to rmpeer
 		for _, ring := range rings {
-			require.NoError(t, otherRing.Merge(*ring))
+			require.NoError(t, merge(otherRing, ring))
 		}
 
 		common.Log.Debugf("%s: transferring from peer %s", otherPeername, peername)
@@ -694,7 +711,7 @@ func TestFuzzRingHard(t *testing.T) {
 		// And now tell everyone about the transfer - rmpeer is
 		// not partition safe
 		for i, ring := range rings {
-			require.NoError(t, ring.Merge(*otherRing))
+			require.NoError(t, merge(ring, otherRing))
 			theRanges[i] = ring.OwnedRanges()
 		}
 	}
@@ -723,7 +740,7 @@ func TestFuzzRingHard(t *testing.T) {
 			// Now 'gossip' this to a random host (note, note could be same host as above)
 			otherIndex, _, otherRing := randomPeer(-1)
 			common.Log.Debugf("%s: 'Gossiping' to %s", ring.Peer, otherRing.Peer)
-			require.NoError(t, otherRing.Merge(*ring))
+			require.NoError(t, merge(otherRing, ring))
 
 			theRanges[indexWithRanges] = ring.OwnedRanges()
 			theRanges[otherIndex] = otherRing.OwnedRanges()
@@ -742,7 +759,7 @@ func TestFuzzRingHard(t *testing.T) {
 		ring1 := ringsWithEntries[rand.Intn(len(ringsWithEntries))]
 		ring2index, _, ring2 := randomPeer(-1)
 		common.Log.Debugf("%s: 'Gossiping' to %s", ring1.Peer, ring2.Peer)
-		require.NoError(t, ring2.Merge(*ring1))
+		require.NoError(t, merge(ring2, ring1))
 		theRanges[ring2index] = ring2.OwnedRanges()
 	}
 

ipam/space/space.go

@@ -122,8 +122,11 @@ func (s *Space) biggestFreeRange(r address.Range) (biggest address.Range) {
 	biggestSize := address.Count(0)
 	s.walkFree(r, func(chunk address.Range) bool {
 		if size := chunk.Size(); size >= biggestSize {
-			biggest = chunk
-			biggestSize = size
+			chunk = chunk.BiggestCIDRRange()
+			if size = chunk.Size(); size >= biggestSize {
+				biggest = chunk
+				biggestSize = size
+			}
 		}
 		return false
 	})
@@ -137,9 +140,12 @@ func (s *Space) Donate(r address.Range) (address.Range, bool) {
 		return address.Range{}, false
 	}
 
-	// Donate half of that biggest free range. Note size/2 rounds down, so
-	// the resulting donation size rounds up, and in particular can't be empty.
-	biggest.Start = address.Add(biggest.Start, address.Offset(biggest.Size()/2))
+	// Donate no more than half what we have available
+	if biggest.Size() > s.NumFreeAddressesInRange(r)/2 {
+		// Note size/2 rounds down, so the resulting donation size
+		// rounds up, and in particular can't be empty.
+		biggest.Start = address.Add(biggest.Start, address.Offset(biggest.Size()/2))
+	}
 
 	s.ours = subtract(s.ours, biggest.Start, biggest.End)
 	s.free = subtract(s.free, biggest.Start, biggest.End)

ipam/space/space_test.go

@@ -68,7 +68,7 @@ func TestLowlevel(t *testing.T) {
 	wt.AssertErrorInterface(t, (*error)(nil), s.Free(100), "double free")
 
 	r, ok := s.Donate(address.NewRange(0, 1000))
-	require.True(t, ok && r.Start == 125 && r.Size() == 25, "donate")
+	require.Equal(t, address.NewRange(0xa0, 0x20), r, "donate")
 
 	// test Donate when addresses are scarce
 	s = New()
@@ -78,7 +78,7 @@ func TestLowlevel(t *testing.T) {
 	require.NoError(t, s.Claim(0))
 	require.NoError(t, s.Claim(2))
 	r, ok = s.Donate(address.NewRange(0, 1000))
-	require.True(t, ok && r.Start == 1 && r.End == 2, "donate")
+	require.Equal(t, address.NewRange(1, 1), r, "donate")
 	r, ok = s.Donate(address.NewRange(0, 1000))
 	require.True(t, !ok, "donate should fail")
 }
@@ -124,20 +124,21 @@ func TestSpaceAllocate(t *testing.T) {
 
 func TestSpaceFree(t *testing.T) {
 	const (
-		testAddr1   = "10.0.3.4"
+		testAddr1   = "10.0.3.16"
 		testAddrx   = "10.0.3.19"
 		testAddry   = "10.0.9.19"
 		containerID = "deadbeef"
+		size        = 16
 	)
 
-	entireRange := address.NewRange(ip(testAddr1), 20)
-	space := makeSpace(ip(testAddr1), 20)
+	entireRange := address.NewRange(ip(testAddr1), size)
+	space := makeSpace(ip(testAddr1), size)
 
 	// Check we are prepared to give up the entire space
 	r := space.biggestFreeRange(entireRange)
-	require.True(t, r.Start == ip(testAddr1) && r.Size() == 20, "Wrong space")
+	require.Equal(t, address.NewRange(ip(testAddr1), size), r, "Wrong space")
 
-	for i := 0; i < 20; i++ {
+	for i := 0; i < size; i++ {
 		ok, _ := space.Allocate(entireRange)
 		require.True(t, ok, "Failed to get address")
 	}
@@ -149,44 +150,44 @@ func TestSpaceFree(t *testing.T) {
 	require.True(t, r.Size() == 0, "Wrong space")
 
 	// Free in the middle
-	require.NoError(t, space.Free(ip("10.0.3.13")))
-	r = space.biggestFreeRange(entireRange)
-	require.True(t, r.Start == ip("10.0.3.13") && r.Size() == 1, "Wrong space")
-
-	// Free one at the end
 	require.NoError(t, space.Free(ip("10.0.3.23")))
 	r = space.biggestFreeRange(entireRange)
 	require.True(t, r.Start == ip("10.0.3.23") && r.Size() == 1, "Wrong space")
 
+	// Free one at the end
+	require.NoError(t, space.Free(ip("10.0.3.31")))
+	r = space.biggestFreeRange(entireRange)
+	require.True(t, r.Start == ip("10.0.3.31") && r.Size() == 1, "Wrong space")
+
 	// Now free a few at the end
-	require.NoError(t, space.Free(ip("10.0.3.22")))
-	require.NoError(t, space.Free(ip("10.0.3.21")))
+	require.NoError(t, space.Free(ip("10.0.3.30")))
+	require.NoError(t, space.Free(ip("10.0.3.29")))
 
 	require.Equal(t, address.Count(4), space.NumFreeAddresses())
 
-	// Now get the biggest free space; should be 3.21
+	// Now get the biggest free space; should be 3.30
 	r = space.biggestFreeRange(entireRange)
-	require.True(t, r.Start == ip("10.0.3.21") && r.Size() == 3, "Wrong space")
+	require.Equal(t, address.NewRange(ip("10.0.3.30"), 2), r, "Wrong space")
 
 	// Now free a few in the middle
-	require.NoError(t, space.Free(ip("10.0.3.12")))
-	require.NoError(t, space.Free(ip("10.0.3.11")))
-	require.NoError(t, space.Free(ip("10.0.3.10")))
+	require.NoError(t, space.Free(ip("10.0.3.24")))
+	require.NoError(t, space.Free(ip("10.0.3.22")))
+	require.NoError(t, space.Free(ip("10.0.3.21")))
 
 	require.Equal(t, address.Count(7), space.NumFreeAddresses())
 
-	// Now get the biggest free space; should be 3.21
+	// Now get the biggest free space; should be 3.30
 	r = space.biggestFreeRange(entireRange)
-	require.True(t, r.Start == ip("10.0.3.10") && r.Size() == 4, "Wrong space")
+	require.Equal(t, address.NewRange(ip("10.0.3.30"), 2), r, "Wrong space")
 
-	require.Equal(t, []address.Range{{Start: ip("10.0.3.4"), End: ip("10.0.3.24")}}, space.OwnedRanges())
+	require.Equal(t, []address.Range{{Start: ip("10.0.3.16"), End: ip("10.0.3.32")}}, space.OwnedRanges())
 }
 
 func TestDonateSimple(t *testing.T) {
 	const (
 		testAddr1 = "10.0.1.0"
 		testAddr2 = "10.0.1.32"
-		size      = 48
+		size      = 64
 	)
 
 	var (
@@ -199,7 +200,7 @@ func TestDonateSimple(t *testing.T) {
 	r, ok := ps1.Donate(address.NewRange(ip(testAddr1), size))
 	numGivenUp := r.Size()
 	require.True(t, ok, "Donate result")
-	require.Equal(t, "10.0.1.24", r.Start.String(), "Invalid start")
+	require.Equal(t, "10.0.1.32", r.Start.String(), "Invalid start")
 	require.Equal(t, address.Count(size/2), numGivenUp)
 	require.Equal(t, address.Count(size/2), ps1.NumFreeAddresses())
 
@@ -241,13 +242,13 @@ func TestDonateHard(t *testing.T) {
 	// Now split
 	newRange, ok := spaceset.Donate(address.NewRange(start, size))
 	require.True(t, ok, "GiveUpSpace result")
-	require.Equal(t, ip("10.0.1.23"), newRange.Start)
-	require.Equal(t, address.Count(24), newRange.Size())
-	require.Equal(t, address.Count(23), spaceset.NumFreeAddresses())
+	require.Equal(t, address.NewRange(ip("10.0.1.16"), 16), newRange, "Wrong space")
+	require.Equal(t, address.Count(31), spaceset.NumFreeAddresses())
 
-	//Space set should now have 2 spaces
-	expected := New()
-	expected.Add(start, 23)
-	expected.ours = add(nil, ip("10.0.1.47"), ip("10.0.1.48"))
+	//Space set should now have 3 spaces
+	expected := &Space{
+		ours: add(nil, ip("10.0.1.47"), ip("10.0.1.48")),
+		free: add(add(nil, ip("10.0.1.0"), ip("10.0.1.16")), ip("10.0.1.32"), ip("10.0.1.47")),
+	}
 	require.Equal(t, expected, spaceset)
 }

ipam/testutils_test.go

@@ -143,7 +143,7 @@ func makeAllocator(name string, cidrStr string, quorum uint) (*Allocator, addres
 		OurName:     peername,
 		OurUID:      mesh.PeerUID(rand.Int63()),
 		OurNickname: "nick-" + name,
-		Universe:    cidr.Range(),
+		Universe:    cidr,
 		IsObserver:  quorum == 0,
 		Quorum:      func() uint { return quorum },
 		Db:          new(mockDB),

ipam/tracker/awsvpc.go

@@ -0,0 +1,286 @@
+package tracker
+
+// The AWSVPC tracker tracks the IPAM ring changes and accordingly updates
+// the AWS VPC route table and, if the announced change is local
+// (denoted by local=true) to the host, the host route table.
+//
+// During the initialization, the tracker detects AWS VPC route table id which
+// is associated with the host's subnet on the AWS network. If such a table does
+// not exist, the default VPC route table is used.
+//
+// When a host A donates a range to a host B, the necessary route table
+// updates (removal) happen on the host A first, and afterwards on
+// the host B (installation).
+//
+// NB: there is a hard limit for 50 routes within any VPC route table
+// (practically, it is 48, because one route is used by the AWS Internet GW and
+// one by the AWS host subnet), therefore it is suggested to avoid
+// an excessive fragmentation within the IPAM ring which might happen due to
+// the claim operations or uneven distribution of containers across the hosts.
+
+import (
+	"fmt"
+	"net"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/ec2metadata"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/vishvananda/netlink"
+
+	"github.com/weaveworks/weave/common"
+	wnet "github.com/weaveworks/weave/net"
+	"github.com/weaveworks/weave/net/address"
+)
+
+type AWSVPCTracker struct {
+	ec2          *ec2.EC2
+	instanceID   string // EC2 Instance ID
+	routeTableID string // VPC Route Table ID
+	linkIndex    int    // The weave bridge link index
+}
+
+// NewAWSVPCTracker creates and initialises AWS VPC based tracker.
+func NewAWSVPCTracker() (*AWSVPCTracker, error) {
+	var (
+		err     error
+		session = session.New()
+		t       = &AWSVPCTracker{}
+	)
+
+	// Detect region and instance id
+	meta := ec2metadata.New(session)
+	t.instanceID, err = meta.GetMetadata("instance-id")
+	if err != nil {
+		return nil, fmt.Errorf("cannot detect instance-id: %s", err)
+	}
+	region, err := meta.Region()
+	if err != nil {
+		return nil, fmt.Errorf("cannot detect region: %s", err)
+	}
+
+	t.ec2 = ec2.New(session, aws.NewConfig().WithRegion(region))
+
+	routeTableID, err := t.detectRouteTableID()
+	if err != nil {
+		return nil, err
+	}
+	t.routeTableID = *routeTableID
+
+	// Detect Weave bridge link index
+	link, err := netlink.LinkByName(wnet.WeaveBridgeName)
+	if err != nil {
+		return nil, fmt.Errorf("cannot find \"%s\" interface: %s", wnet.WeaveBridgeName, err)
+	}
+	t.linkIndex = link.Attrs().Index
+
+	t.infof("AWSVPC has been initialized on %s instance for %s route table at %s region",
+		t.instanceID, t.routeTableID, region)
+
+	return t, nil
+}
+
+// HandleUpdate method updates the AWS VPC and the host route tables.
+func (t *AWSVPCTracker) HandleUpdate(prevRanges, currRanges []address.Range, local bool) error {
+	t.debugf("replacing %q by %q; local(%t)", prevRanges, currRanges, local)
+
+	prev, curr := removeCommon(address.NewCIDRs(merge(prevRanges)), address.NewCIDRs(merge(currRanges)))
+
+	// It might make sense to do the removal first and then add entries
+	// because of the 50 routes limit. However, in such case a container might
+	// not be reachable for a short period of time which is not a desired behavior.
+
+	// Add new entries
+	for _, cidr := range curr {
+		cidrStr := cidr.String()
+		t.debugf("adding route %s to %s", cidrStr, t.instanceID)
+		_, err := t.createVPCRoute(cidrStr)
+		if err != nil {
+			return fmt.Errorf("createVPCRoutes failed: %s", err)
+		}
+		if local {
+			err = t.createHostRoute(cidrStr)
+			if err != nil {
+				return fmt.Errorf("createHostRoute failed: %s", err)
+			}
+		}
+	}
+
+	// Remove obsolete entries
+	for _, cidr := range prev {
+		cidrStr := cidr.String()
+		t.debugf("removing %s route", cidrStr)
+		_, err := t.deleteVPCRoute(cidrStr)
+		if err != nil {
+			return fmt.Errorf("deleteVPCRoute failed: %s", err)
+		}
+		if local {
+			err = t.deleteHostRoute(cidrStr)
+			if err != nil {
+				return fmt.Errorf("deleteHostRoute failed: %s", err)
+			}
+		}
+	}
+
+	return nil
+}
+
+func (t *AWSVPCTracker) createVPCRoute(cidr string) (*ec2.CreateRouteOutput, error) {
+	route := &ec2.CreateRouteInput{
+		RouteTableId:         &t.routeTableID,
+		InstanceId:           &t.instanceID,
+		DestinationCidrBlock: &cidr,
+	}
+	return t.ec2.CreateRoute(route)
+}
+
+func (t *AWSVPCTracker) createHostRoute(cidr string) error {
+	dst, err := parseCIDR(cidr)
+	if err != nil {
+		return err
+	}
+	route := &netlink.Route{
+		LinkIndex: t.linkIndex,
+		Dst:       dst,
+		Scope:     netlink.SCOPE_LINK,
+	}
+	return netlink.RouteAdd(route)
+}
+
+func (t *AWSVPCTracker) deleteVPCRoute(cidr string) (*ec2.DeleteRouteOutput, error) {
+	route := &ec2.DeleteRouteInput{
+		RouteTableId:         &t.routeTableID,
+		DestinationCidrBlock: &cidr,
+	}
+	return t.ec2.DeleteRoute(route)
+}
+
+func (t *AWSVPCTracker) deleteHostRoute(cidr string) error {
+	dst, err := parseCIDR(cidr)
+	if err != nil {
+		return err
+	}
+	route := &netlink.Route{
+		LinkIndex: t.linkIndex,
+		Dst:       dst,
+		Scope:     netlink.SCOPE_LINK,
+	}
+	return netlink.RouteDel(route)
+}
+
+// detectRouteTableID detects AWS VPC Route Table ID of the given tracker instance.
+func (t *AWSVPCTracker) detectRouteTableID() (*string, error) {
+	instancesParams := &ec2.DescribeInstancesInput{
+		InstanceIds: []*string{aws.String(t.instanceID)},
+	}
+	instancesResp, err := t.ec2.DescribeInstances(instancesParams)
+	if err != nil {
+		return nil, fmt.Errorf("DescribeInstances failed: %s", err)
+	}
+	if len(instancesResp.Reservations) == 0 ||
+		len(instancesResp.Reservations[0].Instances) == 0 {
+		return nil, fmt.Errorf("cannot find %s instance within reservations", t.instanceID)
+	}
+	vpcID := instancesResp.Reservations[0].Instances[0].VpcId
+	subnetID := instancesResp.Reservations[0].Instances[0].SubnetId
+
+	// First try to find a routing table associated with the subnet of the instance
+	tablesParams := &ec2.DescribeRouteTablesInput{
+		Filters: []*ec2.Filter{
+			{
+				Name:   aws.String("association.subnet-id"),
+				Values: []*string{subnetID},
+			},
+		},
+	}
+	tablesResp, err := t.ec2.DescribeRouteTables(tablesParams)
+	if err != nil {
+		return nil, fmt.Errorf("DescribeRouteTables failed: %s", err)
+	}
+	if len(tablesResp.RouteTables) != 0 {
+		return tablesResp.RouteTables[0].RouteTableId, nil
+	}
+	// Fallback to the default routing table
+	tablesParams = &ec2.DescribeRouteTablesInput{
+		Filters: []*ec2.Filter{
+			{
+				Name:   aws.String("association.main"),
+				Values: []*string{aws.String("true")},
+			},
+			{
+				Name:   aws.String("vpc-id"),
+				Values: []*string{vpcID},
+			},
+		},
+	}
+	tablesResp, err = t.ec2.DescribeRouteTables(tablesParams)
+	if err != nil {
+		return nil, fmt.Errorf("DescribeRouteTables failed: %s", err)
+	}
+	if len(tablesResp.RouteTables) != 0 {
+		return tablesResp.RouteTables[0].RouteTableId, nil
+	}
+
+	return nil, fmt.Errorf("cannot find routetable for %s instance", t.instanceID)
+}
+
+func (t *AWSVPCTracker) debugf(fmt string, args ...interface{}) {
+	common.Log.Debugf("[tracker] "+fmt, args...)
+}
+
+func (t *AWSVPCTracker) infof(fmt string, args ...interface{}) {
+	common.Log.Infof("[tracker] "+fmt, args...)
+}
+
+// Helpers
+
+// merge merges adjacent range entries.
+// The given slice has to be sorted in increasing order.
+func merge(r []address.Range) []address.Range {
+	var merged []address.Range
+
+	for i := range r {
+		if prev := len(merged) - 1; prev >= 0 && merged[prev].End == r[i].Start {
+			merged[prev].End = r[i].End
+		} else {
+			merged = append(merged, r[i])
+		}
+	}
+
+	return merged
+}
+
+// removeCommon filters out CIDR ranges which are contained in both a and b slices.
+// Both slices have to be sorted in increasing order.
+func removeCommon(a, b []address.CIDR) (newA, newB []address.CIDR) {
+	i, j := 0, 0
+
+	for i < len(a) && j < len(b) {
+		switch {
+		case a[i].Start() < b[j].Start() || a[i].End() < b[j].End():
+			newA = append(newA, a[i])
+			i++
+		case a[i].Start() > b[j].Start() || a[i].End() > b[j].End():
+			newB = append(newB, b[j])
+			j++
+		default:
+			i++
+			j++
+		}
+
+	}
+	newA = append(newA, a[i:]...)
+	newB = append(newB, b[j:]...)
+
+	return
+}
+
+func parseCIDR(cidr string) (*net.IPNet, error) {
+	ip, ipnet, err := net.ParseCIDR(cidr)
+	if err != nil {
+		return nil, err
+	}
+	ipnet.IP = ip
+
+	return ipnet, nil
+}

ipam/tracker/awsvpc_test.go

@@ -0,0 +1,52 @@
+package tracker
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/weaveworks/weave/net/address"
+)
+
+var (
+	r0to127     = cidr("10.0.0.0", "10.0.0.127")
+	r128to255   = cidr("10.0.0.128", "10.0.0.255")
+	r0to255     = cidr("10.0.0.0", "10.0.0.255")
+	r1dot0to255 = cidr("10.0.1.0", "10.0.1.255")
+	r2dot0to255 = cidr("10.0.2.0", "10.0.2.255")
+)
+
+func TestRemoveCommon(t *testing.T) {
+	a := []address.CIDR{r0to127, r1dot0to255}
+	b := []address.CIDR{r1dot0to255, r2dot0to255}
+	newA, newB := removeCommon(a, b)
+	require.Equal(t, []address.CIDR{r0to127}, newA)
+	require.Equal(t, []address.CIDR{r2dot0to255}, newB)
+}
+
+func TestMerge(t *testing.T) {
+	ranges := []address.Range{
+		r0to127.Range(),
+		r128to255.Range(),
+		r2dot0to255.Range(),
+	}
+	require.Equal(t, []address.Range{r0to255.Range(), r2dot0to255.Range()}, merge(ranges))
+}
+
+// Helper
+
+// TODO(mp) DRY with helpers of other tests.
+
+func ip(s string) address.Address {
+	addr, _ := address.ParseIP(s)
+	return addr
+}
+
+// [start; end]
+func cidr(start, end string) address.CIDR {
+	c := address.Range{Start: ip(start), End: ip(end) + 1}.CIDRs()
+	if len(c) != 1 {
+		panic("invalid cidr")
+	}
+	return c[0]
+}

ipam/tracker/tracker.go

@@ -0,0 +1,20 @@
+package tracker
+
+import (
+	"github.com/weaveworks/weave/net/address"
+)
+
+// LocalRangeTracker is an interface for tracking changes in the IPAM ring.
+type LocalRangeTracker interface {
+	// HandleUpdate is called whenever an address ring gets updated.
+	//
+	// prevRanges corresponds to ranges which were owned by a peer before
+	// a change in the ring, while currRanges to the ones which are currently
+	// owned by the peer.
+	// Both slices have to be sorted in increasing order.
+	// Adjacent ranges within each slice might appear as separate ranges.
+	//
+	// The local parameter indicates whether the ranges belong to the peer
+	// by which the method is called.
+	HandleUpdate(prevRanges, currRanges []address.Range, local bool) error
+}

nameserver/dns.go

@@ -5,8 +5,10 @@ import (
 	"fmt"
 	"math/rand"
 	"net"
+	"os"
 	"sort"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/miekg/dns"
@@ -17,7 +19,6 @@ import (
 const (
 	topDomain        = "."
 	reverseDNSdomain = "in-addr.arpa."
-	etcResolvConf    = "/etc/resolv.conf"
 	udpBuffSize      = uint16(4096)
 	minUDPSize       = 512
 
@@ -26,16 +27,48 @@ const (
 	DefaultClientTimeout = 5 * time.Second
 )
 
-type DNSServer struct {
-	ns      *Nameserver
-	domain  string
-	ttl     uint32
-	address string
+type Upstream interface {
+	Config() *dns.ClientConfig
+}
 
-	servers   []*dns.Server
-	upstream  *dns.ClientConfig
-	tcpClient *dns.Client
-	udpClient *dns.Client
+func NewUpstream(resolvConf, filterAddress string) Upstream {
+	return &upstream{
+		resolvConf:    resolvConf,
+		filterAddress: filterAddress,
+		cachedConfig:  &dns.ClientConfig{}}
+}
+
+type upstream struct {
+	sync.Mutex
+
+	resolvConf    string
+	filterAddress string
+	cachedConfig  *dns.ClientConfig
+	lastStat      time.Time
+	lastModified  time.Time
+}
+
+func (u *upstream) Config() *dns.ClientConfig {
+	u.Lock()
+	defer u.Unlock()
+
+	// If we checked less than five seconds ago return what we already have
+	now := time.Now()
+	if now.Sub(u.lastStat) < 5*time.Second {
+		return u.cachedConfig
+	}
+	u.lastStat = now
+
+	// Reread file if it has changed
+	if fi, err := os.Stat(u.resolvConf); err == nil && fi.ModTime() != u.lastModified {
+		if config, err := dns.ClientConfigFromFile(u.resolvConf); err == nil {
+			config.Servers = filter(config.Servers, u.filterAddress)
+			u.lastModified = fi.ModTime()
+			u.cachedConfig = config
+		}
+	}
+
+	return u.cachedConfig
 }
 
 func filter(ss []string, s string) []string {
@@ -49,24 +82,30 @@ func filter(ss []string, s string) []string {
 	return ss
 }
 
-func NewDNSServer(ns *Nameserver, domain, address, effectiveAddress string, ttl uint32, clientTimeout time.Duration) (*DNSServer, error) {
+type DNSServer struct {
+	ns      *Nameserver
+	domain  string
+	ttl     uint32
+	address string
+
+	servers   []*dns.Server
+	upstream  Upstream
+	tcpClient *dns.Client
+	udpClient *dns.Client
+}
+
+func NewDNSServer(ns *Nameserver, domain, address string, upstream Upstream, ttl uint32, clientTimeout time.Duration) (*DNSServer, error) {
 	s := &DNSServer{
 		ns:        ns,
 		domain:    dns.Fqdn(domain),
 		ttl:       ttl,
 		address:   address,
+		upstream:  upstream,
 		tcpClient: &dns.Client{Net: "tcp", ReadTimeout: clientTimeout},
 		udpClient: &dns.Client{Net: "udp", ReadTimeout: clientTimeout, UDPSize: udpBuffSize},
 	}
-	var err error
-	if s.upstream, err = dns.ClientConfigFromFile(etcResolvConf); err != nil {
-		return nil, err
-	}
-	if s.upstream != nil {
-		s.upstream.Servers = filter(s.upstream.Servers, effectiveAddress)
-	}
 
-	err = s.listen(address)
+	err := s.listen(address)
 	return s, err
 }
 
@@ -218,10 +257,11 @@ func (h *handler) handleRecursive(w dns.ResponseWriter, req *dns.Msg) {
 		}
 	}
 
-	for _, server := range h.upstream.Servers {
+	upstreamConfig := h.upstream.Config()
+	for _, server := range upstreamConfig.Servers {
 		reqCopy := req.Copy()
 		reqCopy.Id = dns.Id()
-		response, _, err := h.client.Exchange(reqCopy, fmt.Sprintf("%s:%s", server, h.upstream.Port))
+		response, _, err := h.client.Exchange(reqCopy, fmt.Sprintf("%s:%s", server, upstreamConfig.Port))
 		if (err != nil && err != dns.ErrTruncated) || response == nil {
 			h.ns.debugf("error trying %s: %v", server, err)
 			continue

nameserver/dns_test.go

@@ -15,17 +15,22 @@ import (
 	"github.com/weaveworks/weave/net/address"
 )
 
+type mockUpstream struct {
+	config *dns.ClientConfig
+}
+
+func (mu *mockUpstream) Config() *dns.ClientConfig {
+	return mu.config
+}
+
 func startServer(t *testing.T, upstream *dns.ClientConfig) (*DNSServer, *Nameserver, int, int) {
 	peername, err := mesh.PeerNameFromString("00:00:00:02:00:00")
 	require.Nil(t, err)
 	nameserver := New(peername, "", func(mesh.PeerName) bool { return true })
-	dnsserver, err := NewDNSServer(nameserver, "weave.local.", "0.0.0.0:0", "", 30, 5*time.Second)
+	dnsserver, err := NewDNSServer(nameserver, "weave.local.", "0.0.0.0:0", &mockUpstream{upstream}, 30, 5*time.Second)
 	require.Nil(t, err)
 	udpPort := dnsserver.servers[0].PacketConn.LocalAddr().(*net.UDPAddr).Port
 	tcpPort := dnsserver.servers[1].Listener.Addr().(*net.TCPAddr).Port
-	if upstream != nil {
-		dnsserver.upstream = upstream
-	}
 	go dnsserver.ActivateAndServe()
 	return dnsserver, nameserver, udpPort, tcpPort
 }

nameserver/status.go

@@ -38,7 +38,7 @@ func NewStatus(ns *Nameserver, dnsServer *DNSServer) *Status {
 
 	return &Status{
 		dnsServer.domain,
-		dnsServer.upstream.Servers,
+		dnsServer.upstream.Config().Servers,
 		dnsServer.address,
 		dnsServer.ttl,
 		entryStatusSlice}

net/address/address.go

@@ -35,6 +35,35 @@ func (r Range) AsCIDRString() string {
 	return CIDR{Addr: r.Start, PrefixLen: prefixLen}.String()
 }
 
+// return the highest bit set in v
+// algorithm from http://graphics.stanford.edu/~seander/bithacks.html#RoundUpPowerOf2
+func msb(v Count) Count {
+	v |= v >> 1
+	v |= v >> 2
+	v |= v >> 4
+	v |= v >> 8
+	v |= v >> 16
+	// Cast to 64-bit to avoid overflow when original v >= 0x80000000
+	return Count((uint64(v) + 1) / 2)
+}
+
+func (r Range) BiggestCIDRRange() Range {
+	sizeMsb := Offset(msb(r.Size()))
+	maskedSize := Offset(r.Size()) & (sizeMsb - 1)
+	maskedStart := Offset(r.Start) & (sizeMsb - 1)
+	if maskedStart == 0 {
+		return NewRange(r.Start, sizeMsb)
+	}
+	if sizeMsb-maskedStart <= maskedSize {
+		return NewRange(Add(r.Start, sizeMsb-maskedStart), sizeMsb)
+	}
+	size := sizeMsb / 2
+	if size >= maskedStart {
+		return NewRange(Add(r.Start, size-maskedStart), size)
+	}
+	return NewRange(Add(r.Start, sizeMsb-maskedStart), size)
+}
+
 func MakeCIDR(subnet CIDR, addr Address) CIDR {
 	return CIDR{Addr: addr, PrefixLen: subnet.PrefixLen}
 }
@@ -44,6 +73,40 @@ type CIDR struct {
 	PrefixLen int
 }
 
+// CIDRs returns a list of CIDR-aligned ranges which cover this range.
+func (r Range) CIDRs() []CIDR {
+	const (
+		fullMask         = ^Address(0)
+		cidrMaxPrefixLen = 32
+	)
+	var cidrs []CIDR
+
+	for start, end := r.Start, r.End-1; end >= start; {
+		mask, prefixLen := fullMask, cidrMaxPrefixLen
+		// Find the smallest mask which would cover some part of [start;end].
+		// Once we found such, apply it by OR'ing
+		for mask > 0 {
+			tmpMask := mask << 1
+			// Check whether mask neither too short nor too long
+			if (start&tmpMask) != start || (start|^tmpMask) > end {
+				break
+			}
+			mask = tmpMask
+			prefixLen--
+		}
+		cidrs = append(cidrs, CIDR{start, prefixLen})
+		// Apply mask
+		start |= ^mask
+		// Check for overflow
+		if start+1 < start {
+			break
+		}
+		start++
+	}
+
+	return cidrs
+}
+
 func ParseIP(s string) (Address, error) {
 	if ip := net.ParseIP(s); ip != nil {
 		return FromIP4(ip), nil
@@ -62,6 +125,22 @@ func ParseCIDR(s string) (CIDR, error) {
 	}
 }
 
+func NewCIDRs(ranges []Range) (cidrs []CIDR) {
+	for _, r := range ranges {
+		cidrs = append(cidrs, r.CIDRs()...)
+	}
+	return cidrs
+}
+
+func (cidr CIDR) Start() Address {
+	return cidr.Addr
+}
+
+// cidr = [Start; End)
+func (cidr CIDR) End() Address {
+	return cidr.Range().End
+}
+
 func (cidr CIDR) IsSubnet() bool {
 	mask := cidr.Size() - 1
 	return Offset(cidr.Addr)&mask == 0
@@ -122,7 +201,7 @@ func Length(a, b Address) Count {
 	return Count(a - b)
 }
 
-func Min(a, b Offset) Offset {
+func Min(a, b Count) Count {
 	if a > b {
 		return b
 	}

net/address/address_test.go

@@ -0,0 +1,69 @@
+package address
+
+import (
+	"testing"
+	"testing/quick"
+
+	"github.com/stretchr/testify/require"
+)
+
+func isPower2(x Count) bool {
+	if x == 0 {
+		return false
+	}
+	for ; x > 1; x /= 2 {
+		if x&1 != 0 {
+			return false
+		}
+	}
+	return true
+}
+
+func TestBiggestPow2AlignedRange(t *testing.T) {
+	require.Equal(t, NewRange(0, 1), NewRange(0, 1).BiggestCIDRRange())
+	require.Equal(t, NewRange(1, 1), NewRange(1, 2).BiggestCIDRRange())
+	require.Equal(t, NewRange(2, 2), NewRange(1, 3).BiggestCIDRRange())
+	require.Equal(t, NewRange(0, 0x40000000), NewRange(0, 0x7fffffff).BiggestCIDRRange())
+	require.Equal(t, NewRange(0xfffffffe, 1), NewRange(0xfffffffe, 1).BiggestCIDRRange())
+	prop := func(start Address, size Offset) bool {
+		if size > Offset(0xffffffff)-Offset(start) { // out of range
+			return true
+		}
+		r := NewRange(start, size)
+		result := r.BiggestCIDRRange()
+		return r.Contains(result.Start) &&
+			r.Contains(result.End-1) &&
+			isPower2(result.Size()) &&
+			result.Size() > r.Size()/4 &&
+			Count(result.Start)%result.Size() == 0
+	}
+	require.NoError(t, quick.Check(prop, &quick.Config{MaxCount: 1000000}))
+}
+
+func ip(s string) Address {
+	addr, _ := ParseIP(s)
+	return addr
+}
+
+func cidr(s string) CIDR {
+	c, err := ParseCIDR(s)
+	if err != nil {
+		panic(err)
+	}
+	return c
+}
+
+func TestCIDRs(t *testing.T) {
+	start := ip("10.0.0.1")
+	end := ip("10.0.0.9")
+	r := NewRange(start, Subtract(end, start))
+	require.Equal(t,
+		[]CIDR{cidr("10.0.0.1/32"), cidr("10.0.0.2/31"), cidr("10.0.0.4/30"), cidr("10.0.0.8/32")},
+		r.CIDRs())
+}
+
+func TestCIDRStartAndEnd(t *testing.T) {
+	cidr, _ := ParseCIDR("10.0.0.0/24")
+	require.Equal(t, ip("10.0.0.0"), cidr.Start(), "")
+	require.Equal(t, ip("10.0.1.0"), cidr.End(), "")
+}

net/arp.go

@@ -1,4 +1,4 @@
-package common
+package net
 
 import "fmt"
 import "io"

net/bridge.go

@@ -0,0 +1,53 @@
+package net
+
+import (
+	"github.com/vishvananda/netlink"
+)
+
+type BridgeType int
+
+const (
+	WeaveBridgeName = "weave"
+	DatapathName    = "datapath"
+
+	None BridgeType = iota
+	Bridge
+	Fastdp
+	BridgedFastdp
+	Inconsistent
+)
+
+func DetectBridgeType(weaveBridgeName, datapathName string) BridgeType {
+	bridge, _ := netlink.LinkByName(weaveBridgeName)
+	datapath, _ := netlink.LinkByName(datapathName)
+
+	switch {
+	case bridge == nil && datapath == nil:
+		return None
+	case isBridge(bridge) && datapath == nil:
+		return Bridge
+	case isDatapath(bridge) && datapath == nil:
+		return Fastdp
+	case isDatapath(datapath) && isBridge(bridge):
+		return BridgedFastdp
+	default:
+		return Inconsistent
+	}
+}
+
+func isBridge(link netlink.Link) bool {
+	_, isBridge := link.(*netlink.Bridge)
+	return isBridge
+}
+
+func isDatapath(link netlink.Link) bool {
+	switch link.(type) {
+	case *netlink.GenericLink:
+		return link.Type() == "openvswitch"
+	case *netlink.Device:
+		// Assume it's our openvswitch device, and the kernel has not been updated to report the kind.
+		return true
+	default:
+		return false
+	}
+}

net/ethtool.go

@@ -0,0 +1,52 @@
+package net
+
+import "fmt"
+import "syscall"
+import "unsafe"
+
+const (
+	SIOCETHTOOL     = 0x8946     // linux/sockios.h
+	ETHTOOL_STXCSUM = 0x00000017 // linux/ethtool.h
+	IFNAMSIZ        = 16         // linux/if.h
+)
+
+// linux/if.h 'struct ifreq'
+type IFReqData struct {
+	Name [IFNAMSIZ]byte
+	Data uintptr
+}
+
+// linux/ethtool.h 'struct ethtool_value'
+type EthtoolValue struct {
+	Cmd  uint32
+	Data uint32
+}
+
+// Disable TX checksum offload on specified interface
+func EthtoolTXOff(name string) error {
+	if len(name)+1 > IFNAMSIZ {
+		return fmt.Errorf("name too long")
+	}
+
+	value := EthtoolValue{ETHTOOL_STXCSUM, 0}
+	request := IFReqData{Data: uintptr(unsafe.Pointer(&value))}
+
+	copy(request.Name[:], name)
+
+	socket, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_DGRAM, 0)
+	if err != nil {
+		return err
+	}
+	defer syscall.Close(socket)
+
+	_, _, errno := syscall.RawSyscall(syscall.SYS_IOCTL,
+		uintptr(socket),
+		uintptr(SIOCETHTOOL),
+		uintptr(unsafe.Pointer(&request)))
+
+	if errno != 0 {
+		return errno
+	}
+
+	return nil
+}

net/netns.go

@@ -0,0 +1,37 @@
+package net
+
+import (
+	"runtime"
+
+	"github.com/vishvananda/netlink"
+	"github.com/vishvananda/netns"
+)
+
+func WithNetNS(ns netns.NsHandle, work func() error) error {
+	runtime.LockOSThread()
+	defer runtime.UnlockOSThread()
+
+	oldNs, err := netns.Get()
+	if err == nil {
+		defer oldNs.Close()
+
+		err = netns.Set(ns)
+		if err == nil {
+			defer netns.Set(oldNs)
+
+			err = work()
+		}
+	}
+
+	return err
+}
+
+func WithNetNSLink(ns netns.NsHandle, ifName string, work func(link netlink.Link) error) error {
+	return WithNetNS(ns, func() error {
+		link, err := netlink.LinkByName(ifName)
+		if err != nil {
+			return err
+		}
+		return work(link)
+	})
+}

net/route.go

@@ -3,6 +3,7 @@ package net
 import (
 	"fmt"
 	"net"
+	"os"
 
 	"github.com/vishvananda/netlink"
 )
@@ -55,3 +56,16 @@ func forEachRoute(ignoreIfaceNames map[string]struct{}, check func(r netlink.Rou
 	}
 	return nil
 }
+
+func AddRoute(link netlink.Link, scope netlink.Scope, dst *net.IPNet, gw net.IP) error {
+	err := netlink.RouteAdd(&netlink.Route{
+		LinkIndex: link.Attrs().Index,
+		Scope:     scope,
+		Dst:       dst,
+		Gw:        gw,
+	})
+	if os.IsExist(err) { // squash duplicate route errors
+		err = nil
+	}
+	return err
+}

net/veth.go

@@ -0,0 +1,204 @@
+package net
+
+import (
+	"fmt"
+	"net"
+	"time"
+
+	"github.com/j-keck/arping"
+	"github.com/vishvananda/netlink"
+	"github.com/vishvananda/netns"
+
+	"github.com/weaveworks/weave/common/odp"
+)
+
+// create and attach a veth to the Weave bridge
+func CreateAndAttachVeth(name, peerName, bridgeName string, mtu int, keepTXOn bool, init func(peer netlink.Link) error) (*netlink.Veth, error) {
+	bridge, err := netlink.LinkByName(bridgeName)
+	if err != nil {
+		return nil, fmt.Errorf(`bridge "%s" not present; did you launch weave?`, bridgeName)
+	}
+
+	if mtu == 0 {
+		mtu = bridge.Attrs().MTU
+	}
+	veth := &netlink.Veth{
+		LinkAttrs: netlink.LinkAttrs{
+			Name: name,
+			MTU:  mtu},
+		PeerName: peerName,
+	}
+	if err := netlink.LinkAdd(veth); err != nil {
+		return nil, fmt.Errorf(`could not create veth pair %s-%s: %s`, name, peerName, err)
+	}
+
+	cleanup := func(format string, a ...interface{}) (*netlink.Veth, error) {
+		netlink.LinkDel(veth)
+		return nil, fmt.Errorf(format, a...)
+	}
+
+	switch bridgeType := DetectBridgeType(bridgeName, DatapathName); bridgeType {
+	case Bridge, BridgedFastdp:
+		if err := netlink.LinkSetMasterByIndex(veth, bridge.Attrs().Index); err != nil {
+			return cleanup(`unable to set master of %s: %s`, name, err)
+		}
+		if bridgeType == Bridge && !keepTXOn {
+			if err := EthtoolTXOff(peerName); err != nil {
+				return cleanup(`unable to set tx off on %q: %s`, peerName, err)
+			}
+		}
+	case Fastdp:
+		if err := odp.AddDatapathInterface(bridgeName, name); err != nil {
+			return cleanup(`failed to attach %s to device "%s": %s`, name, bridgeName, err)
+		}
+	default:
+		return cleanup(`invalid bridge configuration`)
+	}
+
+	if init != nil {
+		peer, err := netlink.LinkByName(peerName)
+		if err != nil {
+			return cleanup("unable to find peer veth %s: %s", peerName, err)
+		}
+		if err := init(peer); err != nil {
+			return cleanup("initializing veth: %s", err)
+		}
+	}
+
+	if err := netlink.LinkSetUp(veth); err != nil {
+		return cleanup("unable to bring veth up: %s", err)
+	}
+
+	return veth, nil
+}
+
+func AddAddresses(link netlink.Link, cidrs []*net.IPNet) (newAddrs []*net.IPNet, err error) {
+	existingAddrs, err := netlink.AddrList(link, netlink.FAMILY_V4)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get IP address for %q: %v", link.Attrs().Name, err)
+	}
+	for _, ipnet := range cidrs {
+		if contains(existingAddrs, ipnet) {
+			continue
+		}
+		if err := netlink.AddrAdd(link, &netlink.Addr{IPNet: ipnet}); err != nil {
+			return nil, fmt.Errorf("failed to add IP address to %q: %v", link.Attrs().Name, err)
+		}
+		newAddrs = append(newAddrs, ipnet)
+	}
+	return newAddrs, nil
+}
+
+func contains(addrs []netlink.Addr, addr *net.IPNet) bool {
+	for _, x := range addrs {
+		if addr.IP.Equal(x.IPNet.IP) {
+			return true
+		}
+	}
+	return false
+}
+
+const (
+	VethName   = "ethwe"        // name inside container namespace
+	vethPrefix = "v" + VethName // starts with "veth" to suppress UI notifications
+)
+
+func interfaceExistsInNamespace(ns netns.NsHandle, ifName string) bool {
+	err := WithNetNS(ns, func() error {
+		_, err := netlink.LinkByName(ifName)
+		return err
+	})
+	return err == nil
+}
+
+func AttachContainer(ns netns.NsHandle, id, ifName, bridgeName string, mtu int, withMulticastRoute bool, cidrs []*net.IPNet, keepTXOn bool) error {
+	if !interfaceExistsInNamespace(ns, ifName) {
+		maxIDLen := IFNAMSIZ - 1 - len(vethPrefix+"pl")
+		if len(id) > maxIDLen {
+			id = id[:maxIDLen] // trim passed ID if too long
+		}
+		name, peerName := vethPrefix+"pl"+id, vethPrefix+"pg"+id
+		_, err := CreateAndAttachVeth(name, peerName, bridgeName, mtu, keepTXOn, func(veth netlink.Link) error {
+			if err := netlink.LinkSetNsFd(veth, int(ns)); err != nil {
+				return fmt.Errorf("failed to move veth to container netns: %s", err)
+			}
+			if err := WithNetNS(ns, func() error {
+				if err := netlink.LinkSetName(veth, ifName); err != nil {
+					return err
+				}
+				if err := ConfigureARPCache(ifName); err != nil {
+					return err
+				}
+				return nil
+			}); err != nil {
+				return fmt.Errorf("error setting up interface: %s", err)
+			}
+			return nil
+		})
+		if err != nil {
+			return err
+		}
+	}
+
+	if err := WithNetNSLink(ns, ifName, func(veth netlink.Link) error {
+		newAddresses, err := AddAddresses(veth, cidrs)
+		if err != nil {
+			return err
+		}
+		if err := netlink.LinkSetUp(veth); err != nil {
+			return err
+		}
+		for _, ipnet := range newAddresses {
+			// If we don't wait for a bit here, we see the arp fail to reach the bridge.
+			time.Sleep(1 * time.Millisecond)
+			arping.GratuitousArpOverIfaceByName(ipnet.IP, ifName)
+		}
+		if withMulticastRoute {
+			/* Route multicast packets across the weave network.
+			This must come last in 'attach'. If you change this, change weavewait to match.
+
+			TODO: Add the MTU lock to prevent PMTU discovery for multicast
+			destinations. Without that, the kernel sets the DF flag on
+			multicast packets. Since RFC1122 prohibits sending of ICMP
+			errors for packets with multicast destinations, that causes
+			packets larger than the PMTU to be dropped silently.  */
+
+			_, multicast, _ := net.ParseCIDR("224.0.0.0/4")
+			if err := AddRoute(veth, netlink.SCOPE_LINK, multicast, nil); err != nil {
+				return err
+			}
+		}
+		return nil
+	}); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func DetachContainer(ns netns.NsHandle, id, ifName string, cidrs []*net.IPNet) error {
+	return WithNetNSLink(ns, ifName, func(veth netlink.Link) error {
+		existingAddrs, err := netlink.AddrList(veth, netlink.FAMILY_V4)
+		if err != nil {
+			return fmt.Errorf("failed to get IP address for %q: %v", veth.Attrs().Name, err)
+		}
+		for _, ipnet := range cidrs {
+			if !contains(existingAddrs, ipnet) {
+				continue
+			}
+			if err := netlink.AddrDel(veth, &netlink.Addr{IPNet: ipnet}); err != nil {
+				return fmt.Errorf("failed to remove IP address from %q: %v", veth.Attrs().Name, err)
+			}
+		}
+		addrs, err := netlink.AddrList(veth, netlink.FAMILY_V4)
+		if err != nil {
+			return fmt.Errorf("failed to get IP address for %q: %v", veth.Attrs().Name, err)
+		}
+		if len(addrs) == 0 { // all addresses gone: remove the interface
+			if err := netlink.LinkDel(veth); err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+}

plugin/ipam/driver.go

@@ -54,6 +54,21 @@ func (i *Ipam) ReleasePool(poolID string) error {
 	return nil
 }
 
+func splitPoolID(poolID string) (subnet, iprange *net.IPNet, err error) {
+	parts := strings.Split(poolID, "-")
+	if len(parts) != 3 || parts[0] != "weave" {
+		err = fmt.Errorf("Unrecognized pool ID: %s", poolID)
+		return
+	}
+	if _, subnet, err = net.ParseCIDR(parts[1]); err != nil {
+		return
+	}
+	if _, iprange, err = net.ParseCIDR(parts[2]); err != nil {
+		return
+	}
+	return
+}
+
 func (i *Ipam) RequestAddress(poolID string, address net.IP, options map[string]string) (ip *net.IPNet, _ map[string]string, err error) {
 	i.logReq("RequestAddress", poolID, address, options)
 	defer func() { i.logRes("RequestAddress", err, ip) }()
@@ -62,13 +77,8 @@ func (i *Ipam) RequestAddress(poolID string, address net.IP, options map[string]
 		ip, err = i.weave.AllocateIP("_")
 		return
 	}
-	parts := strings.Split(poolID, "-")
-	if len(parts) != 3 || parts[0] != "weave" {
-		err = fmt.Errorf("Unrecognized pool ID: %s", poolID)
-		return
-	}
-	var subnet, iprange *net.IPNet
-	if _, subnet, err = net.ParseCIDR(parts[1]); err != nil {
+	subnet, iprange, err := splitPoolID(poolID)
+	if err != nil {
 		return
 	}
 	if address != nil { // try to claim specific address requested
@@ -77,9 +87,6 @@ func (i *Ipam) RequestAddress(poolID string, address net.IP, options map[string]
 			return
 		}
 	} else {
-		if _, iprange, err = net.ParseCIDR(parts[2]); err != nil {
-			return
-		}
 		// We are lying slightly to IPAM here: the range is not a subnet
 		if ip, err = i.weave.AllocateIPInSubnet("_", iprange); err != nil {
 			return
@@ -91,6 +98,11 @@ func (i *Ipam) RequestAddress(poolID string, address net.IP, options map[string]
 
 func (i *Ipam) ReleaseAddress(poolID string, address net.IP) error {
 	i.logReq("ReleaseAddress", poolID, address)
+	if subnet, _, err := splitPoolID(poolID); err != nil {
+		return err
+	} else if address.Equal(subnet.IP) { // is it the gateway address we faked earlier?
+		return nil
+	}
 	return i.weave.ReleaseIPsFor(address.String())
 }
 

plugin/net/cni.go

@@ -5,7 +5,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"net"
-	"os"
 
 	"github.com/appc/cni/pkg/ipam"
 	"github.com/appc/cni/pkg/skel"
@@ -14,6 +13,7 @@ import (
 	"github.com/vishvananda/netns"
 	weaveapi "github.com/weaveworks/weave/api"
 	"github.com/weaveworks/weave/common"
+	weavenet "github.com/weaveworks/weave/net"
 	ipamplugin "github.com/weaveworks/weave/plugin/ipam"
 )
 
@@ -32,7 +32,7 @@ func NewCNIPlugin(weave *weaveapi.Client) *CNIPlugin {
 
 func loadNetConf(bytes []byte) (*NetConf, error) {
 	n := &NetConf{
-		BrName: "weave",
+		BrName: weavenet.WeaveBridgeName,
 	}
 	if err := json.Unmarshal(bytes, n); err != nil {
 		return nil, fmt.Errorf("failed to load netconf: %v", err)
@@ -53,6 +53,29 @@ func (c *CNIPlugin) CmdAdd(args *skel.CmdArgs) error {
 		return fmt.Errorf("IP Masquerading functionality not supported")
 	}
 
+	var result *types.Result
+	// Default IPAM is Weave's own
+	if conf.IPAM.Type == "" {
+		result, err = ipamplugin.NewIpam(c.weave).Allocate(args)
+	} else {
+		result, err = ipam.ExecAdd(conf.IPAM.Type, args.StdinData)
+	}
+	if err != nil {
+		return fmt.Errorf("unable to allocate IP address: %s", err)
+	}
+	if result.IP4 == nil {
+		return fmt.Errorf("IPAM plugin failed to allocate IP address")
+	}
+
+	// If config says nothing about routes or gateway, default one will be via the bridge
+	if result.IP4.Routes == nil && result.IP4.Gateway == nil {
+		bridgeIP, err := findBridgeIP(conf.BrName, result.IP4.IP)
+		if err != nil {
+			return err
+		}
+		result.IP4.Gateway = bridgeIP
+	}
+
 	ns, err := netns.GetFromPath(args.Netns)
 	if err != nil {
 		return err
@@ -69,93 +92,36 @@ func (c *CNIPlugin) CmdAdd(args *skel.CmdArgs) error {
 		id = fmt.Sprintf("%x", data)
 	}
 
-	local, err := createAndAttach(id, conf.BrName, conf.MTU)
-	if err != nil {
+	if err := weavenet.AttachContainer(ns, id, args.IfName, conf.BrName, conf.MTU, false, []*net.IPNet{&result.IP4.IP}, false); err != nil {
 		return err
 	}
-
-	cleanup := func(err error) error {
-		netlink.LinkDel(local)
-		return err
-	}
-	guest, err := netlink.LinkByName(local.PeerName)
-	if err != nil {
-		return cleanup(err)
-	}
-	if err = netlink.LinkSetNsFd(guest, int(ns)); err != nil {
-		return cleanup(fmt.Errorf("failed to move veth to container netns: %s", err))
-	}
-
-	if err := netlink.LinkSetUp(local); err != nil {
-		return cleanup(fmt.Errorf("unable to bring veth up: %s", err))
-	}
-
-	var result *types.Result
-	// Default IPAM is Weave's own
-	if conf.IPAM.Type == "" {
-		result, err = ipamplugin.NewIpam(c.weave).Allocate(args)
-	} else {
-		result, err = ipam.ExecAdd(conf.IPAM.Type, args.StdinData)
-	}
-	if err != nil {
-		return cleanup(fmt.Errorf("unable to allocate IP address: %s", err))
-	}
-	if result.IP4 == nil {
-		return cleanup(fmt.Errorf("IPAM plugin failed to allocate IP address"))
-	}
-
-	// If config says nothing about routes or gateway, default one will be via the bridge
-	if result.IP4.Routes == nil && result.IP4.Gateway == nil {
-		bridgeIP, err := findBridgeIP(conf.BrName, result.IP4.IP)
-		if err != nil {
-			return cleanup(err)
-		}
-		result.IP4.Gateway = bridgeIP
-	}
-
-	err = common.WithNetNS(ns, func() error {
-		return setupGuestIP4(local.PeerName, args.IfName, result.IP4.IP, result.IP4.Gateway, result.IP4.Routes)
-	})
-	if err != nil {
-		return cleanup(fmt.Errorf("error setting up interface: %s", err))
+	if err := weavenet.WithNetNSLink(ns, args.IfName, func(link netlink.Link) error {
+		return setupRoutes(link, args.IfName, result.IP4.IP, result.IP4.Gateway, result.IP4.Routes)
+	}); err != nil {
+		return fmt.Errorf("error setting up routes: %s", err)
 	}
 
 	result.DNS = conf.DNS
 	return result.Print()
 }
 
-func setupGuestIP4(origName, name string, ipnet net.IPNet, gw net.IP, routes []types.Route) error {
-	guest, err := netlink.LinkByName(origName)
-	if err != nil {
-		return err
-	}
-	if err = netlink.LinkSetName(guest, name); err != nil {
-		return err
-	}
-	if err = netlink.LinkSetUp(guest); err != nil {
-		return err
-	}
-	if err = common.ConfigureARPCache(name); err != nil {
-		return err
-	}
+func setupRoutes(link netlink.Link, name string, ipnet net.IPNet, gw net.IP, routes []types.Route) error {
+	var err error
 	if routes == nil { // If config says nothing about routes, add a default one
 		if !ipnet.Contains(gw) {
 			// The bridge IP is not on the same subnet; add a specific route to it
 			gw32 := &net.IPNet{IP: gw, Mask: mask32}
-			if err = addRoute(guest, netlink.SCOPE_LINK, gw32, nil); err != nil {
+			if err = weavenet.AddRoute(link, netlink.SCOPE_LINK, gw32, nil); err != nil {
 				return err
 			}
 		}
 		routes = []types.Route{{Dst: zeroNetwork}}
 	}
-	if err = netlink.AddrAdd(guest, &netlink.Addr{IPNet: &ipnet}); err != nil {
-		return fmt.Errorf("failed to add IP address to %q: %v", name, err)
-	}
 	for _, r := range routes {
 		if r.GW != nil {
-			err = addRoute(guest, netlink.SCOPE_UNIVERSE, &r.Dst, r.GW)
+			err = weavenet.AddRoute(link, netlink.SCOPE_UNIVERSE, &r.Dst, r.GW)
 		} else {
-			err = addRoute(guest, netlink.SCOPE_UNIVERSE, &r.Dst, gw)
+			err = weavenet.AddRoute(link, netlink.SCOPE_UNIVERSE, &r.Dst, gw)
 		}
 		if err != nil {
 			return fmt.Errorf("failed to add route '%v via %v dev %v': %v", r.Dst, gw, name, err)
@@ -164,19 +130,6 @@ func setupGuestIP4(origName, name string, ipnet net.IPNet, gw net.IP, routes []t
 	return nil
 }
 
-func addRoute(link netlink.Link, scope netlink.Scope, dst *net.IPNet, gw net.IP) error {
-	err := netlink.RouteAdd(&netlink.Route{
-		LinkIndex: link.Attrs().Index,
-		Scope:     scope,
-		Dst:       dst,
-		Gw:        gw,
-	})
-	if os.IsExist(err) { // squash duplicate route errors
-		err = nil
-	}
-	return err
-}
-
 func findBridgeIP(bridgeName string, subnet net.IPNet) (net.IP, error) {
 	netdevs, err := common.GetBridgeNetDev(bridgeName)
 	if err != nil {
@@ -208,7 +161,7 @@ func (c *CNIPlugin) CmdDel(args *skel.CmdArgs) error {
 		return err
 	}
 	defer ns.Close()
-	err = common.WithNetNS(ns, func() error {
+	err = weavenet.WithNetNS(ns, func() error {
 		link, err := netlink.LinkByName(args.IfName)
 		if err != nil {
 			return err

plugin/net/driver.go

@@ -2,35 +2,44 @@ package plugin
 
 import (
 	"fmt"
+	"strconv"
 	"sync"
 
 	"github.com/docker/libnetwork/drivers/remote/api"
+	"github.com/docker/libnetwork/netlabel"
 	"github.com/docker/libnetwork/types"
 
 	"github.com/vishvananda/netlink"
 	weaveapi "github.com/weaveworks/weave/api"
 	"github.com/weaveworks/weave/common"
 	"github.com/weaveworks/weave/common/docker"
-	"github.com/weaveworks/weave/common/odp"
+	weavenet "github.com/weaveworks/weave/net"
 	"github.com/weaveworks/weave/plugin/skel"
 )
 
 const (
-	WeaveBridge = "weave"
+	MulticastOption = "works.weave.multicast"
 )
 
-type driver struct {
-	scope            string
-	noMulticastRoute bool
-	sync.RWMutex
-	endpoints map[string]struct{}
+type network struct {
+	isOurs            bool
+	hasMulticastRoute bool
 }
 
-func New(client *docker.Client, weave *weaveapi.Client, scope string, noMulticastRoute bool) (skel.Driver, error) {
+type driver struct {
+	name   string
+	scope  string
+	docker *docker.Client
+	sync.RWMutex
+	networks map[string]network
+}
+
+func New(client *docker.Client, weave *weaveapi.Client, name, scope string) (skel.Driver, error) {
 	driver := &driver{
-		noMulticastRoute: noMulticastRoute,
-		scope:            scope,
-		endpoints:        make(map[string]struct{}),
+		name:     name,
+		scope:    scope,
+		docker:   client,
+		networks: make(map[string]network),
 	}
 
 	_, err := NewWatcher(client, weave, driver)
@@ -53,24 +62,42 @@ func (driver *driver) GetCapabilities() (*api.GetCapabilityResponse, error) {
 
 func (driver *driver) CreateNetwork(create *api.CreateNetworkRequest) error {
 	driver.logReq("CreateNetwork", create, create.NetworkID)
+	_, err := driver.setupNetworkInfo(create.NetworkID, true, stringOptions(create))
+	return err
+}
+
+// Deal with excessively-generic way the options get decoded from JSON
+func stringOptions(create *api.CreateNetworkRequest) map[string]string {
+	if create.Options != nil {
+		if data, found := create.Options[netlabel.GenericData]; found {
+			if options, ok := data.(map[string]interface{}); ok {
+				out := make(map[string]string, len(options))
+				for key, value := range options {
+					if str, ok := value.(string); ok {
+						out[key] = str
+					}
+				}
+				return out
+			}
+		}
+	}
 	return nil
 }
 
-func (driver *driver) DeleteNetwork(delete *api.DeleteNetworkRequest) error {
-	driver.logReq("DeleteNetwork", delete, delete.NetworkID)
+func (driver *driver) DeleteNetwork(delreq *api.DeleteNetworkRequest) error {
+	driver.logReq("DeleteNetwork", delreq, delreq.NetworkID)
+	driver.Lock()
+	delete(driver.networks, delreq.NetworkID)
+	driver.Unlock()
 	return nil
 }
 
 func (driver *driver) CreateEndpoint(create *api.CreateEndpointRequest) (*api.CreateEndpointResponse, error) {
 	driver.logReq("CreateEndpoint", create, create.EndpointID)
-	endID := create.EndpointID
 
 	if create.Interface == nil {
 		return nil, driver.error("CreateEndpoint", "Not supported: creating an interface from within CreateEndpoint")
 	}
-	driver.Lock()
-	driver.endpoints[endID] = struct{}{}
-	driver.Unlock()
 	resp := &api.CreateEndpointResponse{}
 
 	driver.logRes("CreateEndpoint", resp)
@@ -79,19 +106,9 @@ func (driver *driver) CreateEndpoint(create *api.CreateEndpointRequest) (*api.Cr
 
 func (driver *driver) DeleteEndpoint(deleteReq *api.DeleteEndpointRequest) error {
 	driver.logReq("DeleteEndpoint", deleteReq, deleteReq.EndpointID)
-	driver.Lock()
-	delete(driver.endpoints, deleteReq.EndpointID)
-	driver.Unlock()
 	return nil
 }
 
-func (driver *driver) HasEndpoint(endpointID string) bool {
-	driver.Lock()
-	_, found := driver.endpoints[endpointID]
-	driver.Unlock()
-	return found
-}
-
 func (driver *driver) EndpointInfo(req *api.EndpointInfoRequest) (*api.EndpointInfoResponse, error) {
 	driver.logReq("EndpointInfo", req, req.EndpointID)
 	return &api.EndpointInfoResponse{Value: map[string]interface{}{}}, nil
@@ -100,24 +117,23 @@ func (driver *driver) EndpointInfo(req *api.EndpointInfoRequest) (*api.EndpointI
 func (driver *driver) JoinEndpoint(j *api.JoinRequest) (*api.JoinResponse, error) {
 	driver.logReq("JoinEndpoint", j, fmt.Sprintf("%s:%s to %s", j.NetworkID, j.EndpointID, j.SandboxKey))
 
-	local, err := createAndAttach(j.EndpointID, WeaveBridge, 0)
+	network, err := driver.findNetworkInfo(j.NetworkID)
 	if err != nil {
+		return nil, driver.error("JoinEndpoint", "unable to find network info: %s", err)
+	}
+
+	name, peerName := vethPair(j.EndpointID)
+	if _, err := weavenet.CreateAndAttachVeth(name, peerName, weavenet.WeaveBridgeName, 0, false, nil); err != nil {
 		return nil, driver.error("JoinEndpoint", "%s", err)
 	}
 
-	if err := netlink.LinkSetUp(local); err != nil {
-		return nil, driver.error("JoinEndpoint", "unable to bring up veth %s-%s: %s", local.Name, local.PeerName, err)
-	}
-
-	ifname := &api.InterfaceName{
-		SrcName:   local.PeerName,
-		DstPrefix: "ethwe",
-	}
-
 	response := &api.JoinResponse{
-		InterfaceName: ifname,
+		InterfaceName: &api.InterfaceName{
+			SrcName:   peerName,
+			DstPrefix: weavenet.VethName,
+		},
 	}
-	if !driver.noMulticastRoute {
+	if network.hasMulticastRoute {
 		multicastRoute := api.StaticRoute{
 			Destination: "224.0.0.0/4",
 			RouteType:   types.CONNECTED,
@@ -128,51 +144,52 @@ func (driver *driver) JoinEndpoint(j *api.JoinRequest) (*api.JoinResponse, error
 	return response, nil
 }
 
-// create and attach local name to the Weave bridge
-func createAndAttach(id, bridgeName string, mtu int) (*netlink.Veth, error) {
-	maybeBridge, err := netlink.LinkByName(bridgeName)
+func (driver *driver) findNetworkInfo(id string) (network, error) {
+	driver.Lock()
+	network, found := driver.networks[id]
+	driver.Unlock()
+	if found {
+		return network, nil
+	}
+	info, err := driver.docker.NetworkInfo(id)
 	if err != nil {
-		return nil, fmt.Errorf(`bridge "%s" not present; did you launch weave?`, bridgeName)
+		return network, err
 	}
+	return driver.setupNetworkInfo(id, info.Driver == driver.name, info.Options)
+}
 
-	local := vethPair(id[:5])
-	if mtu == 0 {
-		local.Attrs().MTU = maybeBridge.Attrs().MTU
-	} else {
-		local.Attrs().MTU = mtu
-	}
-	if err := netlink.LinkAdd(local); err != nil {
-		return nil, fmt.Errorf(`could not create veth pair %s-%s: %s`, local.Name, local.PeerName, err)
-	}
+func (driver *driver) setupNetworkInfo(id string, isOurs bool, options map[string]string) (network, error) {
+	network := network{isOurs: isOurs}
+	if isOurs {
+		for key, value := range options {
+			switch key {
+			case MulticastOption:
+				if value == "" { // interpret "--opt works.weave.multicast" as "turn it on"
+					network.hasMulticastRoute = true
+				} else {
+					var err error
+					if network.hasMulticastRoute, err = strconv.ParseBool(value); err != nil {
+						return network, fmt.Errorf("unrecognized value %q for option %s", value, key)
+					}
 
-	switch maybeBridge.(type) {
-	case *netlink.Bridge:
-		if err := netlink.LinkSetMasterByIndex(local, maybeBridge.Attrs().Index); err != nil {
-			return nil, fmt.Errorf(`unable to set master of %s: %s`, local.Name, err)
+				}
+			default:
+				driver.warn("setupNetworkInfo", "unrecognized option: %s", key)
+			}
 		}
-	case *netlink.GenericLink:
-		if maybeBridge.Type() != "openvswitch" {
-			return nil, fmt.Errorf(`device "%s" is of type "%s"`, bridgeName, maybeBridge.Type())
-		}
-		if err := odp.AddDatapathInterface(bridgeName, local.Name); err != nil {
-			return nil, fmt.Errorf(`failed to attach %s to device "%s": %s`, local.Name, bridgeName, err)
-		}
-	case *netlink.Device:
-		// Assume it's our openvswitch device, and the kernel has not been updated to report the kind.
-		if err := odp.AddDatapathInterface(bridgeName, local.Name); err != nil {
-			return nil, fmt.Errorf(`failed to attach %s to device "%s": %s`, local.Name, bridgeName, err)
-		}
-	default:
-		return nil, fmt.Errorf(`device "%s" is not a bridge`, bridgeName)
 	}
-	return local, nil
+	driver.Lock()
+	driver.networks[id] = network
+	driver.Unlock()
+	return network, nil
 }
 
 func (driver *driver) LeaveEndpoint(leave *api.LeaveRequest) error {
 	driver.logReq("LeaveEndpoint", leave, fmt.Sprintf("%s:%s", leave.NetworkID, leave.EndpointID))
 
-	local := vethPair(leave.EndpointID[:5])
-	if err := netlink.LinkDel(local); err != nil {
+	name, _ := vethPair(leave.EndpointID)
+	veth := &netlink.Veth{LinkAttrs: netlink.LinkAttrs{Name: name}}
+	if err := netlink.LinkDel(veth); err != nil {
 		driver.warn("LeaveEndpoint", "unable to delete veth: %s", err)
 	}
 	return nil
@@ -188,13 +205,8 @@ func (driver *driver) DiscoverDelete(disco *api.DiscoveryNotification) error {
 	return nil
 }
 
-// ===
-
-func vethPair(suffix string) *netlink.Veth {
-	return &netlink.Veth{
-		LinkAttrs: netlink.LinkAttrs{Name: "vethwl" + suffix},
-		PeerName:  "vethwg" + suffix,
-	}
+func vethPair(id string) (string, string) {
+	return "vethwl" + id[:5], "vethwg" + id[:5]
 }
 
 // logging

plugin/net/watcher.go

@@ -4,7 +4,9 @@ import (
 	"fmt"
 
 	weaveapi "github.com/weaveworks/weave/api"
+	"github.com/weaveworks/weave/common"
 	"github.com/weaveworks/weave/common/docker"
+	weavenet "github.com/weaveworks/weave/net"
 )
 
 const (
@@ -32,13 +34,21 @@ func (w *watcher) ContainerStarted(id string) {
 		w.driver.warn("ContainerStarted", "error inspecting container %s: %s", id, err)
 		return
 	}
-	// check that it's on our network, via the endpointID
+	// check that it's on our network
 	for _, net := range info.NetworkSettings.Networks {
-		if w.driver.HasEndpoint(net.EndpointID) {
+		network, err := w.driver.findNetworkInfo(net.NetworkID)
+		if err != nil {
+			w.driver.warn("ContainerStarted", "unable to find network %s info: %s", net.NetworkID, err)
+			continue
+		}
+		if network.isOurs {
 			fqdn := fmt.Sprintf("%s.%s", info.Config.Hostname, info.Config.Domainname)
 			if err := w.weave.RegisterWithDNS(id, fqdn, net.IPAddress); err != nil {
 				w.driver.warn("ContainerStarted", "unable to register %s with weaveDNS: %s", id, err)
 			}
+			if err := common.ConfigureARPforVeths(info.State.Pid, weavenet.VethName); err != nil {
+				w.driver.warn("ContainerStarted", "unable to configure interfaces: %s", err)
+			}
 		}
 	}
 }

prog/plugin/main.go

@@ -7,6 +7,7 @@ import (
 	"net/http"
 	"os"
 	"os/signal"
+	"path"
 	"strings"
 	"syscall"
 
@@ -42,7 +43,7 @@ func main() {
 	flag.StringVar(&logLevel, "log-level", "info", "logging level (debug, info, warning, error)")
 	flag.StringVar(&address, "socket", "/run/docker/plugins/weave.sock", "socket on which to listen")
 	flag.StringVar(&meshAddress, "meshsocket", "/run/docker/plugins/weavemesh.sock", "socket on which to listen in mesh mode")
-	flag.BoolVar(&noMulticastRoute, "no-multicast-route", false, "do not add a multicast route to network endpoints")
+	flag.BoolVar(&noMulticastRoute, "no-multicast-route", false, "deprecated (this is now the default)")
 
 	flag.Parse()
 
@@ -73,18 +74,21 @@ func main() {
 	}
 
 	Log.Println("Weave plugin", version, "Command line options:", os.Args[1:])
+	if noMulticastRoute {
+		Log.Warning("--no-multicast-route option has been removed; multicast is off by default")
+	}
 	Log.Info(dockerClient.Info())
 
-	err = run(dockerClient, weave, address, meshAddress, noMulticastRoute)
+	err = run(dockerClient, weave, address, meshAddress)
 	if err != nil {
 		Log.Fatal(err)
 	}
 }
 
-func run(dockerClient *docker.Client, weave *weaveapi.Client, address, meshAddress string, noMulticastRoute bool) error {
+func run(dockerClient *docker.Client, weave *weaveapi.Client, address, meshAddress string) error {
 	endChan := make(chan error, 1)
 	if address != "" {
-		globalListener, err := listenAndServe(dockerClient, weave, address, noMulticastRoute, endChan, "global", false)
+		globalListener, err := listenAndServe(dockerClient, weave, address, endChan, "global", false)
 		if err != nil {
 			return err
 		}
@@ -92,7 +96,7 @@ func run(dockerClient *docker.Client, weave *weaveapi.Client, address, meshAddre
 		defer globalListener.Close()
 	}
 	if meshAddress != "" {
-		meshListener, err := listenAndServe(dockerClient, weave, meshAddress, noMulticastRoute, endChan, "local", true)
+		meshListener, err := listenAndServe(dockerClient, weave, meshAddress, endChan, "local", true)
 		if err != nil {
 			return err
 		}
@@ -117,8 +121,9 @@ func run(dockerClient *docker.Client, weave *weaveapi.Client, address, meshAddre
 	}
 }
 
-func listenAndServe(dockerClient *docker.Client, weave *weaveapi.Client, address string, noMulticastRoute bool, endChan chan<- error, scope string, withIpam bool) (net.Listener, error) {
-	d, err := netplugin.New(dockerClient, weave, scope, noMulticastRoute)
+func listenAndServe(dockerClient *docker.Client, weave *weaveapi.Client, address string, endChan chan<- error, scope string, withIpam bool) (net.Listener, error) {
+	name := strings.TrimSuffix(path.Base(address), ".sock")
+	d, err := netplugin.New(dockerClient, weave, name, scope)
 	if err != nil {
 		return nil, err
 	}

prog/weaver/checkpoint.go

@@ -0,0 +1,63 @@
+package main
+
+import (
+	"sync/atomic"
+	"syscall"
+	"time"
+
+	"github.com/weaveworks/go-checkpoint"
+)
+
+var checker *checkpoint.Checker
+var newVersion atomic.Value
+
+const (
+	updateCheckPeriod = 6 * time.Hour
+)
+
+func checkForUpdates(dockerVersion string, network string) {
+	newVersion.Store("")
+
+	handleResponse := func(r *checkpoint.CheckResponse, err error) {
+		if err != nil {
+			Log.Printf("Error checking version: %v", err)
+			return
+		}
+		if r.Outdated {
+			newVersion.Store(r.CurrentVersion)
+			Log.Printf("Weave version %s is available; please update at %s",
+				r.CurrentVersion, r.CurrentDownloadURL)
+		}
+	}
+
+	var uts syscall.Utsname
+	syscall.Uname(&uts)
+	flags := map[string]string{
+		"docker-version": dockerVersion,
+		"kernel-version": charsToString(uts.Release[:]),
+	}
+	if network != "" {
+		flags["network"] = network
+	}
+
+	// Start background version checking
+	params := checkpoint.CheckParams{
+		Product:       "weave-net",
+		Version:       version,
+		SignatureFile: "",
+		Flags:         flags,
+	}
+	checker = checkpoint.CheckInterval(&params, updateCheckPeriod, handleResponse)
+}
+
+func charsToString(ca []int8) string {
+	s := make([]byte, len(ca))
+	i := 0
+	for ; i < len(ca); i++ {
+		if ca[i] == 0 {
+			break
+		}
+		s[i] = uint8(ca[i])
+	}
+	return string(s[:i])
+}

prog/weaver/main.go

@@ -9,19 +9,18 @@ import (
 	"runtime"
 	"strconv"
 	"strings"
-	"sync/atomic"
 	"time"
 
-	"github.com/davecheney/profile"
 	"github.com/docker/docker/pkg/mflag"
 	"github.com/gorilla/mux"
+	"github.com/pkg/profile"
 	"github.com/weaveworks/mesh"
 
-	"github.com/weaveworks/go-checkpoint"
 	"github.com/weaveworks/weave/common"
 	"github.com/weaveworks/weave/common/docker"
 	"github.com/weaveworks/weave/db"
 	"github.com/weaveworks/weave/ipam"
+	"github.com/weaveworks/weave/ipam/tracker"
 	"github.com/weaveworks/weave/nameserver"
 	weavenet "github.com/weaveworks/weave/net"
 	"github.com/weaveworks/weave/net/address"
@@ -29,8 +28,6 @@ import (
 )
 
 var version = "(unreleased version)"
-var checker *checkpoint.Checker
-var newVersion atomic.Value
 
 var Log = common.Log
 
@@ -49,34 +46,7 @@ type dnsConfig struct {
 	TTL                    int
 	ClientTimeout          time.Duration
 	EffectiveListenAddress string
-}
-
-const (
-	updateCheckPeriod = 6 * time.Hour
-)
-
-func checkForUpdates() {
-	newVersion.Store("")
-
-	handleResponse := func(r *checkpoint.CheckResponse, err error) {
-		if err != nil {
-			Log.Printf("Error checking version: %v", err)
-			return
-		}
-		if r.Outdated {
-			newVersion.Store(r.CurrentVersion)
-			Log.Printf("Weave version %s is available; please update at %s",
-				r.CurrentVersion, r.CurrentDownloadURL)
-		}
-	}
-
-	// Start background version checking
-	params := checkpoint.CheckParams{
-		Product:       "weave-net",
-		Version:       version,
-		SignatureFile: "",
-	}
-	checker = checkpoint.CheckInterval(&params, updateCheckPeriod, handleResponse)
+	ResolvConf             string
 }
 
 func (c *ipamConfig) Enabled() bool {
@@ -151,6 +121,7 @@ func main() {
 		config             mesh.Config
 		networkConfig      weave.NetworkConfig
 		protocolMinVersion int
+		resume             bool
 		ifaceName          string
 		routerName         string
 		nickName           string
@@ -169,6 +140,7 @@ func main() {
 		datapathName       string
 		trustedSubnetStr   string
 		dbPrefix           string
+		isAWSVPC           bool
 
 		defaultDockerHost = "unix:///var/run/docker.sock"
 	)
@@ -181,6 +153,7 @@ func main() {
 	mflag.StringVar(&config.Host, []string{"-host"}, "", "router host")
 	mflag.IntVar(&config.Port, []string{"#port", "-port"}, mesh.Port, "router port")
 	mflag.IntVar(&protocolMinVersion, []string{"-min-protocol-version"}, mesh.ProtocolMinVersion, "minimum weave protocol version")
+	mflag.BoolVar(&resume, []string{"-resume"}, false, "resume connections to previous peers")
 	mflag.StringVar(&ifaceName, []string{"#iface", "-iface"}, "", "name of interface to capture/inject from (disabled if blank)")
 	mflag.StringVar(&routerName, []string{"#name", "-name"}, "", "name of router (defaults to MAC of interface)")
 	mflag.StringVar(&nickName, []string{"#nickname", "-nickname"}, "", "nickname of peer (defaults to hostname)")
@@ -203,9 +176,11 @@ func main() {
 	mflag.IntVar(&dnsConfig.TTL, []string{"-dns-ttl"}, nameserver.DefaultTTL, "TTL for DNS request from our domain")
 	mflag.DurationVar(&dnsConfig.ClientTimeout, []string{"-dns-fallback-timeout"}, nameserver.DefaultClientTimeout, "timeout for fallback DNS requests")
 	mflag.StringVar(&dnsConfig.EffectiveListenAddress, []string{"-dns-effective-listen-address"}, "", "address DNS will actually be listening, after Docker port mapping")
+	mflag.StringVar(&dnsConfig.ResolvConf, []string{"-resolv-conf"}, "", "path to resolver configuration for fallback DNS lookups")
 	mflag.StringVar(&datapathName, []string{"-datapath"}, "", "ODP datapath name")
 	mflag.StringVar(&trustedSubnetStr, []string{"-trusted-subnets"}, "", "comma-separated list of trusted subnets in CIDR notation")
 	mflag.StringVar(&dbPrefix, []string{"-db-prefix"}, "/weavedb/weave", "pathname/prefix of filename to store data")
+	mflag.BoolVar(&isAWSVPC, []string{"#awsvpc", "-awsvpc"}, false, "use AWS VPC for routing")
 
 	// crude way of detecting that we probably have been started in a
 	// container, with `weave launch` --> suppress misleading paths in
@@ -218,6 +193,9 @@ func main() {
 	mflag.Parse()
 
 	peers = mflag.Args()
+	if resume && len(peers) > 0 {
+		Log.Fatalf("You must not specify an initial peer list in conjuction with --resume")
+	}
 
 	common.SetLogLevel(logLevel)
 
@@ -228,13 +206,8 @@ func main() {
 
 	Log.Println("Command line options:", options())
 
-	checkForUpdates()
-
 	if prof != "" {
-		p := *profile.CPUProfile
-		p.ProfilePath = prof
-		p.NoShutdownHook = true
-		defer profile.Start(&p).Stop()
+		defer profile.Start(profile.CPUProfile, profile.ProfilePath(prof), profile.NoShutdownHook).Stop()
 	}
 
 	if protocolMinVersion < mesh.ProtocolMinVersion || protocolMinVersion > mesh.ProtocolMaxVersion {
@@ -248,7 +221,7 @@ func main() {
 		networkConfig.PacketLogging = nopPacketLogging{}
 	}
 
-	overlay, bridge := createOverlay(datapathName, ifaceName, config.Host, config.Port, bufSzMB)
+	overlay, bridge := createOverlay(datapathName, ifaceName, isAWSVPC, config.Host, config.Port, bufSzMB)
 	networkConfig.Bridge = bridge
 
 	name := peerName(routerName, bridge.Interface())
@@ -263,6 +236,10 @@ func main() {
 	config.TrustedSubnets = parseTrustedSubnets(trustedSubnetStr)
 	config.PeerDiscovery = !noDiscovery
 
+	if isAWSVPC && len(config.Password) > 0 {
+		Log.Fatalf("--awsvpc mode is not compatible with the --password option")
+	}
+
 	db, err := db.NewBoltDB(dbPrefix + "data.db")
 	checkFatal(err)
 	defer db.Close()
@@ -270,7 +247,7 @@ func main() {
 	router := weave.NewNetworkRouter(config, networkConfig, name, nickName, overlay, db)
 	Log.Println("Our name is", router.Ourself)
 
-	if peers, err = router.InitialPeers(peers); err != nil {
+	if peers, err = router.InitialPeers(resume, peers); err != nil {
 		Log.Fatal("Unable to get initial peer set: ", err)
 	}
 
@@ -284,6 +261,13 @@ func main() {
 		}
 		dockerCli = dc
 	}
+
+	network := ""
+	if isAWSVPC {
+		network = "awsvpc"
+	}
+	checkForUpdates(dockerCli.DockerVersion(), network)
+
 	observeContainers := func(o docker.ContainerObserver) {
 		if dockerCli != nil {
 			if err := dockerCli.AddObserver(o); err != nil {
@@ -298,13 +282,23 @@ func main() {
 	var (
 		allocator     *ipam.Allocator
 		defaultSubnet address.CIDR
+		trackerName   string
 	)
 	if ipamConfig.Enabled() {
-		allocator, defaultSubnet = createAllocator(router, ipamConfig, db, isKnownPeer)
+		var t tracker.LocalRangeTracker
+		if isAWSVPC {
+			Log.Infoln("Creating AWSVPC LocalRangeTracker")
+			t, err = tracker.NewAWSVPCTracker()
+			if err != nil {
+				Log.Fatalf("Cannot create AWSVPC LocalRangeTracker: %s", err)
+			}
+			trackerName = "awsvpc"
+		}
+		allocator, defaultSubnet = createAllocator(router, ipamConfig, db, t, isKnownPeer)
 		observeContainers(allocator)
 		ids, err := dockerCli.AllContainerIDs()
 		checkFatal(err)
-		allocator.AllContainerIDs(ids)
+		allocator.PruneOwned(ids)
 	}
 
 	var (
@@ -331,16 +325,16 @@ func main() {
 	if httpAddr != "" {
 		muxRouter := mux.NewRouter()
 		if allocator != nil {
-			allocator.HandleHTTP(muxRouter, defaultSubnet, dockerCli)
+			allocator.HandleHTTP(muxRouter, defaultSubnet, trackerName, dockerCli)
 		}
 		if ns != nil {
 			ns.HandleHTTP(muxRouter, dockerCli)
 		}
 		router.HandleHTTP(muxRouter)
 		HandleHTTP(muxRouter, version, router, allocator, defaultSubnet, ns, dnsserver)
-		http.Handle("/", muxRouter)
+		http.Handle("/", common.LoggingHTTPHandler(muxRouter))
 		Log.Println("Listening for HTTP control messages on", httpAddr)
-		go listenAndServeHTTP(httpAddr, muxRouter)
+		go listenAndServeHTTP(httpAddr)
 	}
 
 	common.SignalHandlerLoop(router)
@@ -386,10 +380,18 @@ func (nopPacketLogging) LogPacket(string, weave.PacketKey) {
 func (nopPacketLogging) LogForwardPacket(string, weave.ForwardPacketKey) {
 }
 
-func createOverlay(datapathName string, ifaceName string, host string, port int, bufSzMB int) (weave.NetworkOverlay, weave.Bridge) {
+func createOverlay(datapathName string, ifaceName string, isAWSVPC bool, host string, port int, bufSzMB int) (weave.NetworkOverlay, weave.Bridge) {
 	overlay := weave.NewOverlaySwitch()
 	var bridge weave.Bridge
+	var ignoreSleeve bool
+
 	switch {
+	case isAWSVPC:
+		vpc := weave.NewAWSVPC()
+		overlay.Add("awsvpc", vpc)
+		bridge = weave.NullBridge{}
+		// Currently, we do not support any overlay with AWSVPC
+		ignoreSleeve = true
 	case datapathName != "" && ifaceName != "":
 		Log.Fatal("At most one of --datapath and --iface must be specified.")
 	case datapathName != "":
@@ -407,30 +409,23 @@ func createOverlay(datapathName string, ifaceName string, host string, port int,
 	default:
 		bridge = weave.NullBridge{}
 	}
-	sleeve := weave.NewSleeveOverlay(host, port)
-	overlay.Add("sleeve", sleeve)
-	overlay.SetCompatOverlay(sleeve)
+
+	if !ignoreSleeve {
+		sleeve := weave.NewSleeveOverlay(host, port)
+		overlay.Add("sleeve", sleeve)
+		overlay.SetCompatOverlay(sleeve)
+	}
+
 	return overlay, bridge
 }
 
-func parseAndCheckCIDR(cidrStr string) address.CIDR {
-	cidr, err := address.ParseCIDR(cidrStr)
+func createAllocator(router *weave.NetworkRouter, config ipamConfig, db db.DB, track tracker.LocalRangeTracker, isKnownPeer func(mesh.PeerName) bool) (*ipam.Allocator, address.CIDR) {
+	ipRange, err := ipam.ParseCIDRSubnet(config.IPRangeCIDR)
 	checkFatal(err)
-
-	if !cidr.IsSubnet() {
-		Log.Fatalf("Invalid allocation range %s - bits after network prefix are not all zero", cidrStr)
-	}
-	if cidr.Size() < ipam.MinSubnetSize {
-		Log.Fatalf("Allocation range smaller than minimum size %d: %s", ipam.MinSubnetSize, cidrStr)
-	}
-	return cidr
-}
-
-func createAllocator(router *weave.NetworkRouter, config ipamConfig, db db.DB, isKnownPeer func(mesh.PeerName) bool) (*ipam.Allocator, address.CIDR) {
-	ipRange := parseAndCheckCIDR(config.IPRangeCIDR)
 	defaultSubnet := ipRange
 	if config.IPSubnetCIDR != "" {
-		defaultSubnet = parseAndCheckCIDR(config.IPSubnetCIDR)
+		defaultSubnet, err = ipam.ParseCIDRSubnet(config.IPSubnetCIDR)
+		checkFatal(err)
 		if !ipRange.Range().Overlaps(defaultSubnet.Range()) {
 			Log.Fatalf("IP address allocation default subnet %s does not overlap with allocation range %s", defaultSubnet, ipRange)
 		}
@@ -441,17 +436,19 @@ func createAllocator(router *weave.NetworkRouter, config ipamConfig, db db.DB, i
 		OurUID:      router.Ourself.Peer.UID,
 		OurNickname: router.Ourself.Peer.NickName,
 		Seed:        config.SeedPeerNames,
-		Universe:    ipRange.Range(),
+		Universe:    ipRange,
 		IsObserver:  config.Observer,
 		Quorum:      func() uint { return determineQuorum(config.PeerCount, router) },
 		Db:          db,
 		IsKnownPeer: isKnownPeer,
+		Tracker:     track,
 	}
 
 	allocator := ipam.NewAllocator(c)
 
 	allocator.SetInterfaces(router.NewGossip("IPallocation", allocator))
 	allocator.Start()
+	router.Peers.OnGC(func(peer *mesh.Peer) { allocator.PeerGone(peer.Name) })
 
 	return allocator, defaultSubnet
 }
@@ -460,8 +457,9 @@ func createDNSServer(config dnsConfig, router *mesh.Router, isKnownPeer func(mes
 	ns := nameserver.New(router.Ourself.Peer.Name, config.Domain, isKnownPeer)
 	router.Peers.OnGC(func(peer *mesh.Peer) { ns.PeerGone(peer.Name) })
 	ns.SetGossip(router.NewGossip("nameserver", ns))
+	upstream := nameserver.NewUpstream(config.ResolvConf, config.EffectiveListenAddress)
 	dnsserver, err := nameserver.NewDNSServer(ns, config.Domain, config.ListenAddress,
-		config.EffectiveListenAddress, uint32(config.TTL), config.ClientTimeout)
+		upstream, uint32(config.TTL), config.ClientTimeout)
 	if err != nil {
 		Log.Fatal("Unable to start dns server: ", err)
 	}
@@ -552,7 +550,7 @@ func parsePeerNames(s string) ([]mesh.PeerName, error) {
 	return peerNames, nil
 }
 
-func listenAndServeHTTP(httpAddr string, muxRouter *mux.Router) {
+func listenAndServeHTTP(httpAddr string) {
 	protocol := "tcp"
 	if strings.HasPrefix(httpAddr, "/") {
 		os.Remove(httpAddr) // in case it's there from last time

prog/weaveutil/attach.go

@@ -0,0 +1,111 @@
+package main
+
+import (
+	"fmt"
+	"net"
+	"strconv"
+	"syscall"
+
+	docker "github.com/fsouza/go-dockerclient"
+	"github.com/vishvananda/netns"
+
+	weavenet "github.com/weaveworks/weave/net"
+)
+
+func attach(args []string) error {
+	if len(args) < 4 {
+		cmdUsage("attach-container", "[--no-multicast-route] [--keep-tx-on] <container-id> <bridge-name> <mtu> <cidr>...")
+	}
+
+	keepTXOn := false
+	withMulticastRoute := true
+	for i := 0; i < len(args); {
+		switch args[i] {
+		case "--no-multicast-route":
+			withMulticastRoute = false
+			args = append(args[:i], args[i+1:]...)
+		case "--keep-tx-on":
+			keepTXOn = true
+			args = append(args[:i], args[i+1:]...)
+		default:
+			i++
+		}
+	}
+
+	pid, nsContainer, err := containerPidAndNs(args[0])
+	if err != nil {
+		return err
+	}
+	if nsHost, err := netns.GetFromPid(1); err != nil {
+		return fmt.Errorf("unable to open host namespace: %s", err)
+	} else if nsHost.Equal(nsContainer) {
+		return fmt.Errorf("Container is running in the host network namespace, and therefore cannot be\nconnected to weave. Perhaps the container was started with --net=host.")
+	}
+	mtu, err := strconv.Atoi(args[2])
+	if err != nil && args[3] != "" {
+		return fmt.Errorf("unable to parse mtu %q: %s", args[2], err)
+	}
+	cidrs, err := parseCIDRs(args[3:])
+	if err != nil {
+		return err
+	}
+
+	err = weavenet.AttachContainer(nsContainer, fmt.Sprint(pid), weavenet.VethName, args[1], mtu, withMulticastRoute, cidrs, keepTXOn)
+	// If we detected an error but the container has died, tell the user that instead.
+	if err != nil && !processExists(pid) {
+		err = fmt.Errorf("Container %s died", args[0])
+	}
+	return err
+}
+
+func containerPidAndNs(containerID string) (int, netns.NsHandle, error) {
+	c, err := docker.NewVersionedClientFromEnv("1.18")
+	if err != nil {
+		return 0, 0, fmt.Errorf("unable to connect to docker: %s", err)
+	}
+	container, err := c.InspectContainer(containerID)
+	if err != nil {
+		return 0, 0, fmt.Errorf("unable to inspect container %s: %s", containerID, err)
+	}
+	if container.State.Pid == 0 {
+		return 0, 0, fmt.Errorf("container %s not running", containerID)
+	}
+	ns, err := netns.GetFromPid(container.State.Pid)
+	if err != nil {
+		return 0, 0, fmt.Errorf("unable to open namespace for container %s: %s", containerID, err)
+	}
+	return container.State.Pid, ns, nil
+}
+
+func processExists(pid int) bool {
+	err := syscall.Kill(pid, syscall.Signal(0))
+	return err == nil || err == syscall.EPERM
+}
+
+func parseCIDRs(args []string) (cidrs []*net.IPNet, err error) {
+	for _, ipstr := range args {
+		ip, ipnet, err := net.ParseCIDR(ipstr)
+		if err != nil {
+			return nil, err
+		}
+		ipnet.IP = ip // we want the specific IP plus the mask
+		cidrs = append(cidrs, ipnet)
+	}
+	return
+}
+
+func detach(args []string) error {
+	if len(args) < 2 {
+		cmdUsage("detach-container", "<container-id> <cidr>...")
+	}
+
+	_, ns, err := containerPidAndNs(args[0])
+	if err != nil {
+		return err
+	}
+	cidrs, err := parseCIDRs(args[1:])
+	if err != nil {
+		return err
+	}
+	return weavenet.DetachContainer(ns, args[0], weavenet.VethName, cidrs)
+}

prog/weaveutil/datapath.go

@@ -31,6 +31,19 @@ func deleteDatapath(args []string) error {
 	return odp.DeleteDatapath(args[0])
 }
 
+// Checks whether a datapath can be created by actually creating and destroying it
+func checkDatapath(args []string) error {
+	if len(args) != 1 {
+		cmdUsage("check-datapath", "<datapath>")
+	}
+
+	if err := createDatapath(args); err != nil {
+		return err
+	}
+
+	return odp.DeleteDatapath(args[0])
+}
+
 func addDatapathInterface(args []string) error {
 	if len(args) != 2 {
 		cmdUsage("add-datapath-interface", "<datapath> <interface>")

prog/weaveutil/docker_tls_args.go

@@ -26,7 +26,12 @@ func dockerTLSArgs(args []string) error {
 			continue
 		}
 
-		if comm, err := ioutil.ReadFile(filepath.Join(procRoot, dirName, "comm")); err != nil || string(comm) != "docker\n" {
+		isDaemon := false
+		if comm, err := ioutil.ReadFile(filepath.Join(procRoot, dirName, "comm")); err != nil {
+			continue
+		} else if string(comm) == "dockerd\n" {
+			isDaemon = true
+		} else if string(comm) != "docker\n" {
 			continue
 		}
 
@@ -35,7 +40,6 @@ func dockerTLSArgs(args []string) error {
 			continue
 		}
 
-		isDaemon := false
 		tlsArgs := []string{}
 		args := bytes.Split(cmdline, []byte{'\000'})
 		for i := 0; i < len(args); i++ {

prog/weaveutil/main.go

@@ -15,10 +15,13 @@ func init() {
 		"docker-tls-args":        dockerTLSArgs,
 		"create-datapath":        createDatapath,
 		"delete-datapath":        deleteDatapath,
+		"check-datapath":         checkDatapath,
 		"add-datapath-interface": addDatapathInterface,
 		"create-plugin-network":  createPluginNetwork,
 		"remove-plugin-network":  removePluginNetwork,
 		"container-addrs":        containerAddrs,
+		"attach-container":       attach,
+		"detach-container":       detach,
 	}
 }
 

prog/weaveutil/plugin_network.go

@@ -4,7 +4,9 @@ package main
 import (
 	"fmt"
 
-	"github.com/fsouza/go-dockerclient"
+	docker "github.com/fsouza/go-dockerclient"
+
+	"github.com/weaveworks/weave/plugin/net"
 )
 
 func createPluginNetwork(args []string) error {
@@ -23,6 +25,7 @@ func createPluginNetwork(args []string) error {
 			CheckDuplicate: true,
 			Driver:         driverName,
 			IPAM:           docker.IPAMOptions{Driver: driverName},
+			Options:        map[string]interface{}{plugin.MulticastOption: "true"},
 		})
 	if err != docker.ErrNetworkAlreadyExists && err != nil {
 		// Despite appearances to the contrary, CreateNetwork does
@@ -47,6 +50,18 @@ func removePluginNetwork(args []string) error {
 	}
 	err = d.RemoveNetwork(networkName)
 	if _, ok := err.(*docker.NoSuchNetwork); !ok && err != nil {
+		if info, err2 := d.NetworkInfo(networkName); err2 == nil {
+			if len(info.Containers) > 0 {
+				containers := ""
+				for container := range info.Containers {
+					containers += fmt.Sprintf("  %.12s ", container)
+				}
+				return fmt.Errorf(`WARNING: the following containers are still attached to network %q:
+%s
+Docker operations involving those containers may pause or fail
+while Weave is not running`, networkName, containers)
+			}
+		}
 		return fmt.Errorf("unable to remove network: %s", err)
 	}
 	return nil

prog/weavewait/check_network_iface.go

@@ -7,6 +7,6 @@ import (
 )
 
 func checkNetwork() error {
-	_, err := weavenet.EnsureInterface("ethwe")
+	_, err := weavenet.EnsureInterface(weavenet.VethName)
 	return err
 }

prog/weavewait/check_network_iface_mcast.go

@@ -7,6 +7,6 @@ import (
 )
 
 func checkNetwork() error {
-	_, err := weavenet.EnsureInterfaceAndMcastRoute("ethwe")
+	_, err := weavenet.EnsureInterfaceAndMcastRoute(weavenet.VethName)
 	return err
 }

proxy/proxy.go

@@ -20,6 +20,7 @@ import (
 	"github.com/weaveworks/weave/common"
 	weavedocker "github.com/weaveworks/weave/common/docker"
 	weavenet "github.com/weaveworks/weave/net"
+	"github.com/weaveworks/weave/net/address"
 )
 
 const (
@@ -103,7 +104,7 @@ func (proxy *Proxy) attachWithRetry(id string) {
 
 	j := &attachJob{id: id, tryInterval: initialInterval}
 	j.timer = time.AfterFunc(time.Duration(0), func() {
-		if err := proxy.attach(id, false, false); err != nil {
+		if err := proxy.attach(id); err != nil {
 			// The delay at the nth retry is a random value in the range
 			// [i-i/2,i+i/2], where i = initialInterval * 1.5^(n-1).
 			j.timer.Reset(j.tryInterval)
@@ -383,9 +384,26 @@ func (proxy *Proxy) listen(protoAndAddr string) (net.Listener, string, error) {
 
 // weavedocker.ContainerObserver interface
 func (proxy *Proxy) ContainerStarted(ident string) {
-	// In case of attach failure, if we have a request waiting on the start, kill the container,
-	// otherwise assume it is a Docker-initated restart and kill the process inside.
-	err := proxy.attach(ident, true, proxy.waitChan(ident) == nil)
+	err := proxy.attach(ident)
+	if err != nil {
+		var e error
+		// attach failed: if we have a request waiting on the start, kill the container,
+		// otherwise assume it is a Docker-initated restart and kill the process inside.
+		if proxy.waitChan(ident) != nil {
+			e = proxy.client.KillContainer(docker.KillContainerOptions{ID: ident})
+		} else {
+			var c *docker.Container
+			if c, e = proxy.client.InspectContainer(ident); e == nil {
+				var process *os.Process
+				if process, e = os.FindProcess(c.State.Pid); e == nil {
+					e = process.Kill()
+				}
+			}
+		}
+		if e != nil {
+			Log.Warningf("Error killing %s: %s", ident, e)
+		}
+	}
 	proxy.notifyWaiters(ident, err)
 }
 
@@ -466,7 +484,7 @@ func (proxy *Proxy) ContainerDestroyed(ident string) {}
 
 // Check if this container needs to be attached, if so then attach it,
 // and return nil on success or not needed.
-func (proxy *Proxy) attach(containerID string, orDie, killProcess bool) error {
+func (proxy *Proxy) attach(containerID string) error {
 	container, err := proxy.client.InspectContainer(containerID)
 	if err != nil {
 		if _, ok := err.(*docker.NoSuchContainer); !ok {
@@ -488,6 +506,10 @@ func (proxy *Proxy) attach(containerID string, orDie, killProcess bool) error {
 		return nil
 	}
 	Log.Infof("Attaching container %s with WEAVE_CIDR \"%s\" to weave network", container.ID, strings.Join(cidrs, " "))
+	if err := validateCIDRs(cidrs); err != nil {
+		return err
+	}
+
 	args := []string{"attach"}
 	args = append(args, cidrs...)
 	if !proxy.NoRewriteHosts {
@@ -502,13 +524,6 @@ func (proxy *Proxy) attach(containerID string, orDie, killProcess bool) error {
 	if proxy.NoMulticastRoute {
 		args = append(args, "--no-multicast-route")
 	}
-	if orDie {
-		if killProcess {
-			args = append(args, "--or-kill")
-		} else {
-			args = append(args, "--or-die")
-		}
-	}
 	args = append(args, container.ID)
 	return callWeaveAttach(container, args)
 }
@@ -523,8 +538,27 @@ func callWeaveAttach(container *docker.Container, args []string) error {
 	return nil
 }
 
+func validateCIDRs(cidrs []string) error {
+	for _, cidr := range cidrs {
+		if cidr == "net:default" {
+			continue
+		}
+		for _, prefix := range []string{"ip:", "net:", ""} {
+			if strings.HasPrefix(cidr, prefix) {
+				if _, err := address.ParseCIDR(strings.TrimPrefix(cidr, prefix)); err == nil {
+					break
+				}
+				return fmt.Errorf("invalid WEAVE_CIDR: %s", cidr)
+			}
+		}
+	}
+	return nil
+}
+
 func (proxy *Proxy) weaveCIDRs(networkMode string, env []string) ([]string, error) {
-	if networkMode == "host" || strings.HasPrefix(networkMode, "container:") {
+	if networkMode == "host" || strings.HasPrefix(networkMode, "container:") ||
+		// Anything else, other than blank/none/default/bridge, is some sort of network plugin
+		(networkMode != "" && networkMode != "none" && networkMode != "default" && networkMode != "bridge") {
 		return nil, fmt.Errorf("the container has '--net=%s'", networkMode)
 	}
 	for _, e := range env {
@@ -611,10 +645,10 @@ func (proxy *Proxy) symlink(unixAddrs []string) (err error) {
 	binds := []string{"/var/run/weave:/var/run/weave"}
 	froms := []string{}
 	for _, addr := range unixAddrs {
-		if addr == weaveSockUnix {
+		from := strings.TrimPrefix(addr, "unix://")
+		if from == weaveSock {
 			continue
 		}
-		from := strings.TrimPrefix(addr, "unix://")
 		dir := filepath.Dir(from)
 		binds = append(binds, dir+":"+filepath.Join("/host", dir))
 		froms = append(froms, filepath.Join("/host", from))

proxy/proxy_intercept.go

@@ -77,19 +77,22 @@ func doRawStream(w http.ResponseWriter, resp *http.Response, client *httputil.Cl
 	}
 	defer down.Close()
 	defer up.Close()
+	defer func() {
+		if err != nil {
+			Log.Warning(err)
+		}
+	}()
 
-	if _, err := down.Write([]byte("HTTP/1.1 200 OK\n")); err != nil {
-		Log.Warning(err)
+	if _, err = downBuf.Write([]byte("HTTP/1.1 " + resp.Status + "\n")); err != nil {
 		return
 	}
-
-	if err := resp.Header.Write(down); err != nil {
-		Log.Warning(err)
+	if err = resp.Header.Write(downBuf); err != nil {
 		return
 	}
-
-	if _, err := down.Write([]byte("\n")); err != nil {
-		Log.Warning(err)
+	if _, err = downBuf.Write([]byte("\n")); err != nil {
+		return
+	}
+	if err = downBuf.Flush(); err != nil {
 		return
 	}
 
@@ -133,6 +136,7 @@ func doChunkedResponse(w http.ResponseWriter, resp *http.Response, client *httpu
 	}
 
 	w.WriteHeader(resp.StatusCode)
+	wf.Flush()
 
 	up, remaining := client.Hijack()
 	defer up.Close()

router/awsvpc.go

@@ -0,0 +1,75 @@
+package router
+
+// A dummy overlay for the AWSVPC underlay to make `weave status` to return
+// a valid information about peer connections.
+
+import (
+	"github.com/weaveworks/mesh"
+)
+
+// mesh.OverlayConnection
+
+type AWSVPCConnection struct {
+	establishedChan chan struct{}
+	errorChan       chan error
+}
+
+func (conn *AWSVPCConnection) Confirm() {
+	// We close the channel to notify mesh that the connection has been established.
+	close(conn.establishedChan)
+}
+
+func (conn *AWSVPCConnection) EstablishedChannel() <-chan struct{} {
+	return conn.establishedChan
+}
+
+func (conn *AWSVPCConnection) ErrorChannel() <-chan error {
+	return conn.errorChan
+}
+
+func (conn *AWSVPCConnection) Stop() {}
+
+func (conn *AWSVPCConnection) ControlMessage(tag byte, msg []byte) {
+}
+
+func (conn *AWSVPCConnection) DisplayName() string {
+	return "awsvpc"
+}
+
+// OverlayForwarder
+
+func (conn *AWSVPCConnection) Forward(key ForwardPacketKey) FlowOp {
+	return DiscardingFlowOp{}
+}
+
+type AWSVPC struct{}
+
+func NewAWSVPC() AWSVPC {
+	return AWSVPC{}
+}
+
+// mesh.Overlay
+
+func (vpc AWSVPC) AddFeaturesTo(features map[string]string) {}
+
+func (vpc AWSVPC) PrepareConnection(params mesh.OverlayConnectionParams) (mesh.OverlayConnection, error) {
+	conn := &AWSVPCConnection{
+		establishedChan: make(chan struct{}),
+		errorChan:       make(chan error, 1),
+	}
+	return conn, nil
+}
+
+func (vpc AWSVPC) Diagnostics() interface{} {
+	return nil
+}
+
+// NetworkOverlay
+
+func (vpc AWSVPC) InvalidateRoutes() {}
+
+func (vpc AWSVPC) InvalidateShortIDs() {}
+
+func (vpc AWSVPC) StartConsumingPackets(localPeer *mesh.Peer, peers *mesh.Peers, consumer OverlayConsumer) error {
+	return nil
+}

router/bridge.go

@@ -36,7 +36,7 @@ func (NullBridge) Interface() *net.Interface {
 }
 
 func (NullBridge) String() string {
-	return "<no bridge networking>"
+	return "no overlay bridge"
 }
 
 func (NullBridge) Stats() map[string]int {

router/network_router.go

@@ -1,8 +1,10 @@
 package router
 
 import (
+	"fmt"
 	"math"
 	"net"
+	"os"
 	"time"
 
 	"github.com/weaveworks/mesh"
@@ -211,11 +213,10 @@ func (router *NetworkRouter) relayBroadcast(srcPeer *mesh.Peer, key PacketKey) F
 const peersIdent = "directPeers"
 
 func (router *NetworkRouter) persistPeers() {
-	// Persistence disabled pending rationalisation of the expected behaviour
-	/*if err := router.db.Save(peersIdent, router.ConnectionMaker.Targets(false)); err != nil {
+	if err := router.db.Save(peersIdent, router.ConnectionMaker.Targets(false)); err != nil {
 		log.Errorf("Error persisting peers: %s", err)
 		return
-	}*/
+	}
 }
 
 func (router *NetworkRouter) InitiateConnections(peers []string, replace bool) []error {
@@ -229,30 +230,22 @@ func (router *NetworkRouter) ForgetConnections(peers []string) {
 	router.persistPeers()
 }
 
-func (router *NetworkRouter) InitialPeers(peers []string) ([]string, error) {
-	var storedPeers []string
-	// Persistence disabled pending rationalisation of the expected behaviour
-	/*if _, err := router.db.Load(peersIdent, &storedPeers); err != nil {
-		return nil, err
-	}*/
-
-	if storedPeers != nil && !equal(peers, storedPeers) {
-		log.Println("Overriding initial peer list with stored list:", storedPeers)
-		peers = storedPeers
-	} else {
-		log.Println("Initial set of peers:", peers)
+func (router *NetworkRouter) InitialPeers(resume bool, peers []string) ([]string, error) {
+	if _, err := os.Stat("restart.sentinel"); err == nil || resume {
+		var storedPeers []string
+		if _, err := router.db.Load(peersIdent, &storedPeers); err != nil {
+			return nil, err
+		}
+		log.Println("Restart/resume detected - using persisted peer list:", storedPeers)
+		return storedPeers, nil
 	}
+
+	sentinel, err := os.Create("restart.sentinel")
+	if err != nil {
+		return nil, fmt.Errorf("error creating sentinel: %v", err)
+	}
+	sentinel.Close()
+
+	log.Println("Launch detected - using supplied peer list:", peers)
 	return peers, nil
 }
-
-func equal(a, b []string) bool {
-	if len(a) != len(b) {
-		return false
-	}
-	for i, v := range a {
-		if v != b[i] {
-			return false
-		}
-	}
-	return true
-}

site/faq.md

@@ -91,6 +91,14 @@ You must permit traffic to flow through TCP 6783 and UDP 6783/6784,
 which are Weave’s control and data ports.
 
 
+**<a name=own-image></a>Q: Why do you use your own Docker image `weaveworks/ubuntu`?**
+
+The official Ubuntu image does not contain the `ping` and `nc`
+commands which are used in many of our examples throughout the
+documentation. The `weaveworks/ubuntu` image is simply the official
+Ubuntu image with those two commands added.
+
+
 **See Also**
 
  * [Troubleshooting Weave](/site/troubleshooting.md)

site/features.md

@@ -81,7 +81,7 @@ See [Using the Weave Docker API Proxy](/site/weave-docker-api/using-proxy.md).
 Weave Net can also be used as a [Docker plugin](https://docs.docker.com/engine/extend/plugins_network/).  A Docker network 
 named `weave` is created by `weave launch`, which is used as follows:
 
-    $ docker run --net=weave -ti ubuntu 
+    $ docker run --net=weave -ti weaveworks/ubuntu
 
 Using the Weave plugin enables you to take advantage of [Docker's network functionality](https://docs.docker.com/engine/extend/plugins_network/).
 
@@ -119,8 +119,8 @@ For a discussion on how Weave Net uses IPAM, see [Automatic IP Address Managemen
 Named containers are automatically registered in [weaveDNS](/site/weavedns.md), 
 and are discoverable by using standard, simple name lookups:
 
-    host1$ docker run -dti --name=service ubuntu
-    host1$ docker run -ti ubuntu
+    host1$ docker run -dti --name=service weaveworks/ubuntu
+    host1$ docker run -ti weaveworks/ubuntu
     root@7b21498fb103:/# ping service
 
 WeaveDNS also supports [load balancing](/site/weavedns/load-balance-fault-weavedns.md), [fault resilience](/site/weavedns/load-balance-fault-weavedns.md) and [hot swapping](/site/weavedns/managing-entries-weavedns.md). 

site/how-it-works.md

@@ -47,4 +47,4 @@ first send it to peer 3:
 **See Also**
 
  * [Weave Router Sleeve Encapsulation](/site/how-it-works/router-encapsulation.md)
- * [How Weave Inteprets Network Topology](/site/how-it-works/network-topology.md)
+ * [How Weave Interprets Network Topology](/site/how-it-works/network-topology.md)

site/how-it-works/ip-addresses.md

@@ -63,4 +63,4 @@ find a match, so uses the default rule.
 
 **See Also**
 
- * [Allocating IPs in a Specific Range](/site/ip-addresses/configuring-weave.md) 
+ * [Allocating IPs in a Specific Range](/site/using-weave/configuring-weave.md)

site/how-it-works/router-encapsulation.md

@@ -61,4 +61,4 @@ MAC discovery.
 
 **See Also**
 
- * [How Weave Net Inteprets Network Topology](/site/how-it-works/network-topology.md)
+ * [How Weave Net Interprets Network Topology](/site/how-it-works/network-topology.md)

site/installing-weave.md

@@ -12,13 +12,11 @@ Install Weave Net by running the following:
     sudo curl -L git.io/weave -o /usr/local/bin/weave
     sudo chmod a+x /usr/local/bin/weave
 
-If you are on OSX and are using Docker Machine you need to make sure
-that a VM is running and configured before getting Weave Net. Setting up a VM is shown in [the Docker Machine
-documentation](https://docs.docker.com/installation/mac/#from-your-shell).
-After the VM is configured with Docker Machine, Weave Net can be launched directly from the OSX host.
+If you are on OSX and you are using Docker Machine ensure that a VM is running and configured 
+before downloading Weave Net. To set up a VM see [the Docker Machine
+documentation](https://docs.docker.com/installation/mac/#from-your-shell) or refer to ["Part 1: Launching Weave Net with Docker Machine"](/guides/part-1-launching-weave-net-with-docker-machine/).
 
-Weave Net respects the environment variable `DOCKER_HOST`, so that you can run
-and control a Weave Network locally on a remote host. See [Using The Weave Docker API Proxy](/site/weave-docker-api/using-proxy.md).
+After your VM is setup with Docker Machine, Weave Net can be launched directly from the OSX host. Weave Net respects the environment variable `DOCKER_HOST`, so that you can run and control a Weave Network locally on a remote host. See [Using The Weave Docker API Proxy](/site/weave-docker-api/using-proxy.md).
 
 With Weave Net downloaded onto your VMs or hosts, you are ready to launch a Weave network and deploy apps onto it. See [Using Weave Net](/site/using-weave.md).
 
@@ -28,13 +26,38 @@ With Weave Net downloaded onto your VMs or hosts, you are ready to launch a Weav
   <img src="hello-screencast.png" alt="Click to watch the screencast" />
 </a>
 
-###Guides for specific platforms
+###Checkpoint
+
+Weave Net [periodically contacts Weaveworks servers for available
+versions](https://github.com/weaveworks/go-checkpoint).  New versions
+are announced in the log and in [the status
+summary](/site/troubleshooting.md#weave-status).
+
+The information sent in this check is:
+
+ * Host UUID hash
+ * Kernel version
+ * Docker version
+ * Weave Net version
+ * Network mode, e.g. 'awsvpc'
+
+To disable this check, run the following before launching Weave Net:
+
+    export CHECKPOINT_DISABLE=1
+
+###Guides for Specific Platforms
 
 CoreOS users see [here](/guides/networking-docker-containers-with-weave-on-coreos/) for an example of installing Weave using cloud-config.
 
 Amazon ECS users see [here](https://github.com/weaveworks/integrations/blob/master/aws/ecs/README.md)
 for the latest Weave AMIs and [here](http://weave.works/guides/service-discovery-with-weave-aws-ecs.html) to get started with Weave Net on ECS.
 
+If you're on Amazon EC2, the standard installation instructions at the
+top of this page, provide the simplest setup and the most flexibility.
+A [special no-overlay mode for EC2](/site/using-weave/awsvpc.md) can
+optionally be enabled, which allows containers to communicate at the
+full speed of the underlying network.
+
 **See Also** 
 
  * [Using Weave Net](/site/using-weave.md)

site/installing-weave/systemd.md

@@ -19,7 +19,7 @@ normally placed in `/etc/systemd/system/weave.service`.
     After=docker.service
     [Service]
     EnvironmentFile=-/etc/sysconfig/weave
-    ExecStartPre=/usr/local/bin/weave launch $PEERS
+    ExecStartPre=/usr/local/bin/weave launch --no-restart $PEERS
     ExecStart=/usr/bin/docker attach weave
     ExecStop=/usr/local/bin/weave stop
     [Install]

site/ipam/allocation-multi-ipam.md

@@ -15,13 +15,13 @@ allocation of an address from a particular subnet, set the
 `WEAVE_CIDR` environment variable to `net:<subnet>` when creating the
 container, for example:
 
-    host1$ docker run -e WEAVE_CIDR=net:10.2.7.0/24 -ti ubuntu
+    host1$ docker run -e WEAVE_CIDR=net:10.2.7.0/24 -ti weaveworks/ubuntu
 
 You can ask for multiple addresses in different subnets and add in
 manually-assigned addresses (outside the automatic allocation range),
 for instance:
 
-    host1$ docker run -e WEAVE_CIDR="net:10.2.7.0/24 net:10.2.8.0/24 ip:10.3.9.1/24" -ti ubuntu
+    host1$ docker run -e WEAVE_CIDR="net:10.2.7.0/24 net:10.2.8.0/24 ip:10.3.9.1/24" -ti weaveworks/ubuntu
 
 >**Note:** The ".0" and ".-1" addresses in a subnet are not used, as required by
 [RFC 1122](https://tools.ietf.org/html/rfc1122#page-29)).

site/operational-guide.md

@@ -0,0 +1,5 @@
+---
+title: Operational Guide
+menu_order: 35
+---
+Operations manual.

site/operational-guide/autoscaling.md

@@ -0,0 +1,57 @@
+---
+title: Autoscaling
+menu_order: 40
+---
+
+### Bootstrapping
+
+An autoscaling configuration begins with a fixed cluster:
+
+* Configured as per the [Uniform Fixed Cluster](/site/operational-guide/uniform-fixed-cluster.md)
+  scenario.
+* Hosted on reserved or protected instances to ensure long-term
+  stability.
+* Ideally sized at a minimum of three or five nodes (you can make your
+  fixed cluster bigger to accommodate base load as required; a minimum
+  is specified here in the interests of resilience only.)
+
+Building on this foundation, arbitrary numbers of dynamic peers can be
+added or removed concurrently as desired, without requiring any
+changes to the configuration of the fixed cluster. As with the fixed
+cluster, dynamically added nodes recover automatically from reboots
+and partitions.
+
+### Scaling Out
+
+On the additional dynamic peer, at boot, via
+[systemd](/site/installing-weave/systemd.md) or equivalent:
+
+    weave launch --no-restart --ipalloc-init=observer $PEERS
+
+Where, 
+
+ * `$PEERS` means all peers in the _fixed cluster_, initial
+and subsequently added, which have not been explicitly removed. It
+should include fixed peers which are temporarily offline or stopped.
+
+You do not have to keep track of and specify the addresses
+of other dynamic peers in `$PEERS` - they will discover and connect to
+each other via the fixed cluster.
+
+>>**Note:** The use of `--ipalloc-init=observer` prevents dynamic peers from
+coming to a consensus on their own - this is important to stop a
+clique forming amongst a group of dynamically added peers if they
+become partitioned from the fixed cluster after having learned about
+each other via discovery.
+
+### Scaling In
+
+On the dynamic peer to be removed:
+
+    weave reset
+
+If for any reason you cannot arrange for `weave reset` to be run on
+the peer before the underlying host is destroyed (for example when
+using spot instances that can be destroyed without notice), you will
+need an asynchronous process to [reclaim lost IP address
+space](/site/operational-guide/tasks.md#detect-reclaim-ipam).

site/operational-guide/concepts.md

@@ -0,0 +1,144 @@
+---
+title: Concepts
+menu_order: 10
+---
+This section describes some of the essential concepts with which you will
+need to be familiar before continuing to the example deployment
+scenarios.
+
+## Host
+
+For the purposes of this documentation a host is an
+installation of the Linux operating system that is running an
+instance of the Docker Engine. The host may be executing directly on bare
+hardware or inside a virtual machine.
+
+## Peer
+
+A peer is a running instance of Weave Net, typically one per host.
+
+## Peer Name
+
+Weave Net peers are identified by a 48-bit value formatted like an
+ethernet MAC address, for example, `01:23:45:67:89:ab`. The 'peer
+name' is used for various purposes:
+
+* Routing of packets between containers on the overlay network
+* Recording the origin peer of DNS entries
+* Recording ownership of IP address ranges
+
+While it is desirable for the peer name to remain stable across
+restarts, it is essential that it is unique. If two or more peers
+share the same name chaos will ensue, which includes but is not limited to
+double allocation of addresses and the inability to route packets on the
+overlay network. 
+
+When the router is launched on a host, it derives its peer name in order of preference:
+
+* From the command line, where the user is responsible for uniqueness and
+  stability
+* From the BIOS product UUID, which is generally stable across
+  restarts and unique across different physical hardware and certain
+  cloned VMs
+* From the hypervisor UUID, which is generally stable across restarts
+  and unique across VMs which do not provide access to a BIOS product
+  UUID
+* From a random value, practically unique across different physical
+  hardware and cloned VMs but not stable across restarts
+
+The appropriate strategy for assigning peer names depends on the type
+and method of your particular deployment and is discussed in more
+detail below.
+
+## Peer Discovery
+
+Peer discovery is a mechanism that allows peers to learn about new
+Weave Net hosts from existing peers without being explicitly told. Peer
+discovery is
+[enabled by default](/site/using-weave/finding-adding-hosts-dynamically.md).
+
+## Network Partition
+
+A network partition is a transient condition whereby some arbitrary
+subsets of peers are unable to communicate with each other for the
+duration - perhaps because a network switch has failed, or a fibre
+optic line severed. Weave Net is designed to allow peers and their
+containers to make maximum safe progress under conditions of
+partition, healing automatically once the partition is over.
+
+## IP Address Manager (IPAM)
+
+[IPAM](/site/ipam.md) is the subsystem responsible for dividing up a
+large contiguous block of IP addresses (known as the IP allocation
+range) amongst peers so that individual addresses may be uniquely
+assigned to containers anywhere on the overlay network.
+
+When a new network is formed an initial division of the IP allocation
+range must be made. Two (mutually exclusive) mechanisms with different
+tradeoffs are provided to perform this task: seeding and consensus.
+
+### Seeding
+
+Seeding requires each peer to be told the list of peer names amongst
+which the address space is to be divided initially. There are some
+constraints and consequences:
+
+* Every peer added to the network _must_ receive the same seed list,
+  for all time, or they will not be able to join together to form a
+  single cohesive whole.
+* Because the 'product UUID' and 'random value' methods of peer name
+  assignment are unpredictable, the end user must by necessity also
+  specify peer names.
+* Even though every peer _must_ receive the same seed, that seed does
+  _not_ have to include every peer in the network, nor does it have to
+  be updated when new peers are added (in fact due to the first
+  constraint above it may not be).
+
+
+Example configurations are given in the section on deployment
+scenarios:
+
+* [Uniform Dynamic Cluster](/site/operational-guide/uniform-dynamic-cluster.md)
+
+### Consensus
+
+Alternatively, when a new network is formed for the first time, peers
+can be configured to co-ordinate amongst themselves to automatically
+divide up the IP allocation range. This process is known as consensus
+and it requires each peer to be told the total number of expected peers
+(the 'initial peer count') in order to prevent the formation of disjoint
+peer groups which would, ultimately, result in duplicate IP
+addresses.
+
+Example configurations are given in the section on deployment
+scenarios:
+
+* [Interactive Deployment](/site/operational-guide/interactive.md)
+* [Uniform Fixed Cluster](/site/operational-guide/uniform-fixed-cluster.md)
+
+### Observers
+
+Finally, an option is provided to start a peer as an _observer_. Such
+peers do not require a seed peer name list or an initial peer
+count; instead they rely on the existence of other peers in the
+network which have been so configured. When an observer needs address
+space, it asks for it from one of the peers which partook of the
+initial division, triggering consensus if necessary.
+
+Example configurations are given in the section on deployment
+scenarios:
+
+* [Autoscaling](/site/operational-guide/autoscaling.md)
+
+## Persistence
+
+Certain information is remembered between launches of Weave Net (for
+example across reboots):
+
+* The division of the IP allocation range amongst peers
+* Allocation of addresses to containers
+
+The persistence of this information is managed transparently in a
+volume container but can be
+[destroyed explicitly](/site/operational-guide/tasks.md#reset)
+if necessary.

site/operational-guide/interactive.md

@@ -0,0 +1,65 @@
+---
+title: Interactive Deployment
+menu_order: 20
+---
+Weave Net can be launched interactively on the command line, and as 
+long as Docker is configured to start on boot, the network will survive 
+host reboots without the use of a systemd. However, since launching 
+Weave Net commands in interactive mode is not amenable to automation 
+and configuration management, it is recommended that deploying Weave Net
+in this mode be reserved for exploration and evaluation only. 
+
+### Bootstrapping
+
+On the initial peer:
+
+    weave launch
+
+### Adding a Peer
+
+On the new peer:
+
+    weave launch <extant peers>
+
+Where, 
+
+* `<extant peers>` indicates all peers in the network, initial and
+subsequently added, which have not been explicitly removed. It should
+include peers that are temporarily offline or stopped.
+
+To ensure that the new peer has joined the existing network, 
+execute the following:
+
+    weave prime
+
+Before adding any new peers, you _must_ wait for this to complete. 
+If this command waits and does not exit, it means that there is some
+issue (such as a network partition or failed peers) that is preventing
+a quorum from being reached - you will need to [address
+that](/site/troubleshooting.md) before moving on.
+
+### Stopping a Peer
+
+A peer can be stopped temporarily with the following command:
+
+    weave stop
+
+A temporarily stopped peer will remember IP address allocation information on the
+next `weave launch` but will forget any discovered peers or
+modifications to the initial peer list that were made with `weave
+connect` or `weave forget`. Note that if the host reboots, Docker
+automatically restarts the peer.
+
+### Removing a Peer
+
+On the peer to be removed:
+
+    weave reset
+
+Then optionally on each remaining peer:
+
+    weave forget <removed peer>
+
+This step is not mandatory, but it will eliminate log noise and
+spurious network traffic by stopping reconnection attempts and
+preventing further connection attempts after a restart.

site/operational-guide/tasks.md

@@ -0,0 +1,119 @@
+---
+title: Administrative Tasks
+menu_order: 60
+---
+
+The following administrative tasks are discussed: 
+
+* [Configuring Weave Net to Start Automatically on Boot](#start-on-boot)
+* [Detecting and Reclaiming Lost IP Address Space](#detect-reclaim-ipam)
+    *[Manually Reclaiming Lost Address Space](#reclaim-address-space)
+* [Upgrading a Cluster](#cluster-upgrade)
+* [Resetting Persisted Data](#reset)
+
+
+##<a name="start-on-boot"></a>Configuring Weave Net to Start Automatically on Boot
+
+`weave launch` runs all of Weave Net's containers with a Docker restart
+policy set to `always`.  If you have launched Weave Net manually at least
+once and your system is configured to start Docker on boot, then Weave Net
+will start automatically on system restarts.
+
+If you are aiming for a non-interactive installation, use
+[systemd]((/site/installing-weave/systemd.md) or a similar init system to
+launch Weave using the `--no-restart` flag after Docker has been started.
+
+##<a name="detect-reclaim-ipam"></a>Detecting and Reclaiming Lost IP Address Space
+
+The recommended method of removing a peer is to run `weave reset` on that
+peer before the underlying host is decommissioned or repurposed. This
+ensures that the portion of the IPAM allocation range assigned to the
+peer is released for reuse. 
+
+Under certain circumstances this operation may not be successful, 
+or possible:
+
+* If the peer is partitioned from the rest of the network
+  when `weave reset` is executed on it
+* If the underlying host is no longer available to execute `weave
+  reset` due to a hardware failure or other unplanned termination (for
+  example when using autoscaling with spot-instances that can be
+  destroyed without notice)
+
+In some cases you may already be aware of the problem, as you were
+unable to execute `weave reset` successfully or because you know
+through other channels that the host has died. In these cases you can
+proceed straight to [Manually Reclaiming Lost Address Space](#reclaim-address-space).
+
+However in some scenarios it may not be obvious that space has been
+lost, in which case you can check for it periodically with the
+following command on any peer:
+
+    weave status ipam
+
+This command displays the peer names and nicknames, absolute quantity/percentage of allocation 
+range managed by peer and also identifies the names of unreachable peers. If you are satisifed
+that the peer is truly gone, rather than temporarily unreachable due to a
+partition, you can reclaim their space manually.
+
+###<a name="manually-reclaim-address-space"></a>Manually Reclaiming Address Space
+
+When a peer dies unexpectedly the remaining peers will consider its
+address space to be unavailable even after it has remained unreachable
+for prolonged periods. There is no universally applicable time limit
+after which one of the remaining peers could decide unilaterally that
+it is safe to appropriate the space for itself, and so an
+administrative action is required to reclaim it.
+
+The [`weave rmpeer`](/site/ipam/stop-remove-peers-ipam.md)
+command is provided to perform this task, and must
+be executed on _one_ of the remaining peers. That peer will then take
+ownership of the freed address space.
+
+##<a name="cluster-upgrade"></a>Upgrading a Cluster
+
+Protocol versioning and feature negotiation are employed in Weave Net
+to enable incremental rolling upgrades. Each major maintains
+the ability to speak to the preceding major release at a minimum, and
+connected peers only utilize features which both support. 
+
+The general upgrade procedure is as follows:
+
+On each peer:
+
+* Stop the old Weave Net with `weave stop` (or `systemctl stop weave` if
+  you're using a systemd unit file)
+* Download the new Weave Net script and replace the existing one
+* Start the new Weave with `weave launch <existing peer list>` (or
+  `systemctl start weave` if you're using a systemd unit file)
+
+To minimize downtime while the new script is pulling the new container images:
+
+* Download the new Weave Net script to a temporary location, for example,
+  `/path/to/new/weave`
+* Pull the new images with `/path/to/new/weave setup`
+* Stop the old Weave Net with `weave stop` (or `systemctl stop weave` if
+  you're using a systemd unit file)
+* Replace the existing script with the new one
+* Start the new Weave Net with `weave launch <existing peer list>` (or
+  `systemctl start weave` if you're using a systemd unit file)
+
+>>**Note:** Always check the Release Notes for specific versions in case
+there are any special caveats or deviations from the standard
+procedure.
+
+##<a name="reset"></a>Resetting Persisted Data
+
+Weave Net persists information in a data volume container named
+`weavedb`. If you wish to start from a completely clean slate (for
+example to withdraw a peer from one network and join it to another)
+you can issue the following command:
+
+    weave reset
+    
+
+**See Also**
+
+ * [Allocating IP Addresses](/site/ipam.md)
+ * [Troubleshooting the IP Allocator](/site/ipam/troubleshooting-ipam.md)
+

site/operational-guide/uniform-dynamic-cluster.md

@@ -0,0 +1,96 @@
+---
+title: Uniform Dynamic Clusters
+menu_order: 50
+---
+
+A uniform dynamic cluster has the following characteristics:
+
+* Recovers automatically after reboots and partitions.
+* Identical configuration for all of its peers.
+* Initial cluster peers can make progress from the outset even if
+  bootstrapping occurs under conditions of partition.
+* Once an initial cluster has been bootstrapped, arbitrary numbers of
+  new peers can be added in parallel without coordination. This makes
+  it ideally suited for use with convergent provisioning tools that
+  operate across multiple hosts in an asynchronous fashion.
+
+## Bootstrapping
+
+On each initial peer N, at boot, via
+[systemd](/site/installing-weave/systemd.md):
+
+    hostN$ weave launch --no-restart --name ::N --ipalloc-init seed=$SEED $PEERS
+
+Where, 
+
+* `$SEED` is a comma separated list of the _names_ of the initial
+peers, for example, `::1,::2,::3` and, 
+* `$PEERS` is obtained from `/etc/sysconfig/weave` as described in the linked systemd
+documentation, and includes the complete set of initial peer
+_addresses_.
+
+For example, if you have three initial peers you would specify the
+following:
+
+    host1$ weave launch --no-restart --name ::1 --ipalloc-init seed=$SEED $PEERS
+    host2$ weave launch --no-restart --name ::2 --ipalloc-init seed=$SEED $PEERS
+    host3$ weave launch --no-restart --name ::3 --ipalloc-init seed=$SEED $PEERS
+
+Where
+
+    SEED="::1,::2,::3"
+    PEERS="host1 host2 host3"
+
+## Adding a Peer
+
+On each new peer, at boot, via
+[systemd](/site/installing-weave/systemd.md):
+
+    hostN$ weave launch --no-restart --name ::N --ipam-seed $SEED $PEERS
+
+Where,
+
+* `--no-restart` disables the Docker restart policy, since this will be
+  handled by systemd.
+* `--name` specifies a unique name for this new peer.
+* `--ipam-seed` specifies the names of only those peers that were
+  involved in the initial cluster bootstrap - even if they have been
+  subsequently removed from the cluster. You can view this as a kind
+  of 'cluster identity',where  peers may only interoperate in the same
+  cluster if they share the same seed.
+* `$PEERS` is obtained from `/etc/sysconfig/weave` as described in the
+  linked systemd documentation. For convenience, this may contain the
+  address of the peer that is being launched, so that you don't have
+  to compute separate lists of 'other' peers tailored to each peer -
+  just supply the same complete list of peer addresses to every peer.
+
+Note that unlike [Interactive](/site/operational-guide/interactive.md)
+and [Uniform Fixed Cluster](/site/operational-guide/uniform-fixed-cluster.md) deployments
+there is no `weave prime` step. You can add as many new peers in
+parallel as you like, even under conditions of partition, and they
+will all (eventually) join safely. This is ideal for use in
+conjunction with asynchronous provisioning systems such as puppet or
+chef. 
+
+For maximum robustness, you should distribute an updated
+`/etc/sysconfig/weave` file including the new peer to all existing
+peers.
+
+### Removing a Peer
+
+On the peer to be removed:
+
+    weave reset
+
+You may remove a seed peer, as long as there is at least one other
+seed peer left in the network.
+
+Then, distribute an updated `/etc/sysconfig/weave` to the remaining
+peers, omitting the removed peer from `$PEERS`.
+
+On each remaining peer:
+
+    weave forget <removed peer>
+
+This step is not mandatory, but it will eliminate log noise and
+spurious network traffic by stopping reconnection attempts.

site/operational-guide/uniform-fixed-cluster.md

@@ -0,0 +1,87 @@
+---
+title: Uniform Fixed Clusters
+menu_order: 30
+---
+
+This scenario describes a production deployment of a fixed number of N
+nodes (N=1 in the simplest case). 
+
+A uniform fixed cluster has the following characteristics:
+
+* Recovers automatically from reboots and partitions.
+* All peers have identical configuration.
+* There is a controlled process for adding or removing nodes, however
+  the end user is responsible for ensuring that only one instance of
+  the process is in-flight at a time. While it is possible to
+  automate, the potentially-blocking `weave prime` operation and the
+  need for global serialization make it non-trivial. It is however
+  relatively straightforward for a human to provide the necessary
+  guarantees and exception handling manually, and so this scenario is best
+  suited to deployments which change size infrequently as a planned
+  maintenance event.
+
+### Bootstrapping
+
+On each initial peer, at boot, via
+[systemd](/site/installing-weave/systemd.md):
+
+    weave launch --no-restart $PEERS
+
+Where,
+
+* `--no-restart` disables the Docker restart policy, since this will be
+  handled by systemd.
+* `$PEERS` is obtained from `/etc/sysconfig/weave` as described in the
+  linked systemd documentation. For convenience, this may contain the
+  address of the peer which is being launched, so that you don't have
+  to compute separate lists of 'other' peers tailored to each peer -
+  just supply the same complete list of peer addresses to every peer.
+
+Then on _any_ peer run the following to force consensus:
+
+    weave prime
+
+>>**Note:** You can run this safely on more than one or
+even all peers, but it's only strictly necessary to run it on one of
+them. 
+
+Once this command completes successfully, IP address
+allocations can proceed under partition and it is safe to add new
+peers. If this command waits without exiting, it means that there is an issue (such
+as a network partition or failed peers) that is preventing a quorum
+from being reached – you will need to [address
+that](/site/troubleshooting.md) before moving on.
+
+### Adding a Peer
+
+On the new peer, at boot, via
+[systemd](/site/installing-weave/systemd.md) run:
+
+    weave launch --no-restart $PEERS
+
+Where, 
+
+* `$PEERS` is the new peer plus all other peers in the network,
+initial and subsequently added, which have not been explicitly
+removed. It should include peers which are temporarily offline or
+stopped.
+
+For maximum robustness, distribute an updated
+`/etc/sysconfig/weave` file including the new peer to all existing
+peers.
+
+### Removing a Peer
+
+On the peer to be removed:
+
+    weave reset
+
+Then distribute an updated `/etc/sysconfig/weave` to the remaining
+peers, omitting the removed peer from `$PEERS`.
+
+On each remaining peer:
+
+    weave forget <removed peer>
+
+This final step is not mandatory, but it will eliminate log noise and
+spurious network traffic by stopping any reconnection attempts.

site/plugin.md

@@ -5,13 +5,13 @@ menu_order: 60
 
  * [Launching Weave Net and Running Containers Using the Plugin](#launching)
  * [Restarting the Plugin](#restarting)
- * [Bypassing the Central Cluster Store When Building Docker Apps](#cluster-store)
- * [Using other plugin command-line arguments](#plugin-args)
+ * [Configuring the Plugin](#configuring)
 
 Docker versions 1.9 and later have a plugin mechanism for adding
-different network providers. Weave Net installs itself as a network plugin
-when you start it with `weave launch`. The Weave Docker Networking plugin is fast and easy to use and 
-best of all doesn't require an external cluster store in order to use it.  
+different network providers. Weave Net installs itself as a network
+plugin when you start it with `weave launch`. The Weave Docker
+Networking Plugin is fast and easy to use, and, unlike other
+networking plugins, does not require an external cluster store.
 
 To create a network which can span multiple Docker hosts, Weave Net peers must be connected to each other, by specifying the other hosts during `weave launch` or via
 [`weave connect`](/site/using-weave/finding-adding-hosts-dynamically.md).
@@ -20,18 +20,19 @@ See [Using Weave Net](/site/using-weave.md#peer-connections) for a discussion on
 
 After you've launched Weave Net and peered your hosts,  you can start containers using the following, for example:
 
-    $ docker run --net=weave -ti ubuntu
+    $ docker run --net=weave -ti weaveworks/ubuntu
 
-on any of the hosts, and they can all communicate with each other.
+on any of the hosts, and they can all communicate with each other
+using any protocol, even multicast.
 
->**Warning!** It is inadvisable to attach containers to the Weave network using the Weave Docker Networking Plugin and Weave Docker API Proxy simultaneously. Such containers will end up with two Weave network interfaces and two IP addresses, which is rarely desirable. To ensure that the proxy is not being used, do not run eval $(weave env), or docker $(weave config).
+>**Warning!** It is inadvisable to attach containers to the Weave network using the Weave Docker Networking Plugin and Weave Docker API Proxy simultaneously. Such containers will end up with two Weave network interfaces and two IP addresses, which is rarely desirable. To ensure that the proxy is not being used, do not run `eval $(weave env)`, or `docker $(weave config) ...`.
 
 In order to use Weave Net's [Service Discovery](/site/weavedns.md) you
 must pass the additional arguments `--dns` and `-dns-search`, for
 which a helper is provided in the Weave script:
 
-    $ docker run --net=weave -h foo.weave.local $(weave dns-args) -tdi ubuntu
-    $ docker run --net=weave -h bar.weave.local $(weave dns-args) -ti ubuntu
+    $ docker run --net=weave -h foo.weave.local $(weave dns-args) -tdi weaveworks/ubuntu
+    $ docker run --net=weave -h bar.weave.local $(weave dns-args) -ti weaveworks/ubuntu
     # ping foo
 
 
@@ -44,11 +45,11 @@ Just launch the Weave Net router onto each host and make a peer connection with
 
 then run your containers using the Docker command-line:
 
-    host1$ docker run --net=weave -ti ubuntu
+    host1$ docker run --net=weave -ti weaveworks/ubuntu
     root@1458e848cd90:/# hostname -i
     10.32.0.2
 
-    host2$ docker run --net=weave -ti ubuntu
+    host2$ docker run --net=weave -ti weaveworks/ubuntu
     root@8cc4b5dc5722:/# ping 10.32.0.2
 
     PING 10.32.0.2 (10.32.0.2) 56(84) bytes of data.
@@ -56,6 +57,30 @@ then run your containers using the Docker command-line:
     64 bytes from 10.32.0.2: icmp_seq=2 ttl=64 time=0.052 ms
 
 
+###<a name="multi"></a>Creating multiple Docker Networks
+
+Docker enables you to create multiple independent networks and attach
+different sets of containers to each network. However, coordinating
+this between hosts, Docker requires that you configure a
+["key-value store"](https://docs.docker.com/engine/userguide/networking/get-started-overlay/#step-1-set-up-a-key-value-store).
+
+If your Docker installation has a key-value store, create a network
+based on Weave Net as follows:
+
+    $ docker network create --driver=weave mynetwork
+
+then use it to connect a container:
+
+    $ docker run --net=mynetwork ...
+
+or
+
+    $ docker network connect mynetwork somecontainer
+
+Containers attached to different Docker Networks are
+[isolated through subnets](https://www.weave.works/docs/net/latest/using-weave/application-isolation/).
+
+
 ###<a name="restarting"></a>Restarting the Plugin
 
 The plugin, like all Weave Net components, is started with a policy of `--restart=always`, so that it is always there after a restart or reboot. If you remove this container (for example, when using `weave reset`) before removing all endpoints created using `--net=weave`, Docker may hang for a long time when it subsequently tries to re-establish communications to the plugin.
@@ -66,46 +91,24 @@ If you are using `systemd` with Docker 1.9, it is advised that you modify the Do
 
     TimeoutStartSec=0
 
-###<a name="cluster-store"></a>Bypassing the Central Cluster Store When Building Docker Apps
 
-To run a Docker cluster without a central database, you need to ensure the following:
+###<a name="configuring"></a>Configuring the Plugin
 
- 1. Run in "local" scope. This tells Docker to ignore any cross-host coordination.
- 2. Allow Weave to handle all the cross-host coordination and to set up all networks. This is done by using the `weave launch` command.
- 3. Provide an IP Address Management (IPAM) driver, which links to Weave Net's own IPAM system
-
-All cross-host coordination is handled by Weave Net's "mesh" communication, using gossipDNS and eventual consistency to avoid the need for constant communication and dependency on a central cluster store.
-
-
-###<a name="plugin-args"></a>Using other plugin command-line arguments
-
-If you need to give additional arguments to the plugin independently, don't
-use `weave launch`, but instead run:
+The plugin accepts a number of configuration parameters. To supply
+these, instead of running `weave launch`, start the router and plugin
+separately with:
 
     $ weave launch-router [other peers]
-    $ weave launch-plugin [plugin arguments]
+    $ weave launch-plugin [plugin parameters]
 
-The plugin command-line arguments are:
+The plugin configuration parameters are:
 
  * `--log-level=debug|info|warning|error` --tells the plugin
    how much information to emit for debugging.
  * `--no-restart` -- remove the default policy of `--restart=always`, if
    you want to control start-up of the plugin yourself
- * `--no-multicast-route` -- stops weave from adding a static IP route for
-   multicast traffic onto its interface
 
-By default, multicast traffic is routed over the weave network.
-To turn this off, e.g. you want to configure your own multicast
-route, add the `--no-multicast-route` flag to `weave launch-plugin`.
-
-
->Note: When using the Docker Plugin, there is no need to run `eval
- $(weave env)` to enable the Proxy. If you do, you may end up with two
- weave network interfaces and two IP addresses for each container.
 
 **See Also**
 
  * [How the Weave Network Plugin Works](/site/plugin/plugin-how-it-works.md)
- * [Plugin Command-line Arguments](/site/plugin/plug-in-command-line.md)
- 
-

site/plugin/plugin-how-it-works.md

@@ -26,6 +26,15 @@ store, but the first part of
 
 >**Note:** In the case of multiple networks using the `weave` driver, all containers are on the same virtual network but Docker allocates their addresses on different subnets so they cannot talk to each other directly.
 
+The plugin accepts the following options via `docker network create ... --opt`:
+
+ * `works.weave.multicast` -- tells weave to add a static IP
+   route for multicast traffic onto its interface.
+
+>**Note:** If you connect a container to multiple Weave networks, at
+   most one of them can have the multicast route enabled.  The `weave`
+   network created when the plugin is first launched has the multicast
+   option turned on, but for any networks you create it defaults to off.
 
 **See Also**
 

site/using-weave.md

@@ -21,13 +21,13 @@ On `$HOST1` run:
 
     host1$ weave launch
     host1$ eval $(weave env)
-    host1$ docker run --name a1 -ti ubuntu
+    host1$ docker run --name a1 -ti weaveworks/ubuntu
 
 Where, 
 
  * The first line runs Weave Net. 
  * The second line configures the Weave Net environment, so that containers launched via the Docker command line are automatically attached to the Weave network, and, 
- * The third line runs the application container using [Docker commands](https://docs.Docker.com/engine/reference/commandline/daemon/). 
+ * The third line runs the [application container](/site/faq.md#own-image) using [a Docker command](https://docs.docker.com/engine/reference/run/).
 
 >**Note:** If the first command results in an error like
  `http:///var/run/Docker.sock/v1.19/containers/create: dial unix
@@ -49,7 +49,7 @@ To launch Weave Net on an additional host and create a peer connection, run the
 
     host2$ weave launch $HOST1
     host2$ eval $(weave env)
-    host2$ docker run --name a2 -ti ubuntu
+    host2$ docker run --name a2 -ti weaveworks/ubuntu
 
 As noted above, the same steps are repeated for `$HOST2`. The only difference, besides the application container’s name, is that `$HOST2` is told to peer with Weave Net on `$HOST1` during launch. 
 

site/using-weave/application-isolation.md

@@ -26,13 +26,13 @@ specified.
 
 Next, launch the two netcat containers onto the default subnet:
 
-    host1$ docker run --name a1 -ti ubuntu
-    host2$ docker run --name a2 -ti ubuntu
+    host1$ docker run --name a1 -ti weaveworks/ubuntu
+    host2$ docker run --name a2 -ti weaveworks/ubuntu
 
 And then to test the isolation, launch a few more containers onto a different subnet:
 
-    host1$ docker run -e WEAVE_CIDR=net:10.2.2.0/24 --name b1 -ti ubuntu
-    host2$ docker run -e WEAVE_CIDR=net:10.2.2.0.24 --name b2 -ti ubuntu
+    host1$ docker run -e WEAVE_CIDR=net:10.2.2.0/24 --name b1 -ti weaveworks/ubuntu
+    host2$ docker run -e WEAVE_CIDR=net:10.2.2.0.24 --name b2 -ti weaveworks/ubuntu
 
 Ping each container to confirm that they can talk to each other, but not to the containers of our first subnet:
 
@@ -54,7 +54,7 @@ Ping each container to confirm that they can talk to each other, but not to the
 
 If required, a container can also be attached to multiple subnets when it is started using:
 
-    host1$ docker run -e WEAVE_CIDR="net:default net:10.2.2.0/24" -ti ubuntu
+    host1$ docker run -e WEAVE_CIDR="net:default net:10.2.2.0/24" -ti weaveworks/ubuntu
 
 `net:default` is used to request the allocation of an address from the default subnet in addition to one from an explicitly specified range.
 

site/using-weave/awsvpc.md

@@ -0,0 +1,106 @@
+---
+title: Using IP Routing on an Amazon Web Services Virtual Private Cloud
+menu_order: 110
+---
+
+If your container infrastructure is running entirely within Amazon Web Services (AWS) 
+Elastic Compute Cloud (EC2), then you can enable AWS-VPC mode with Weave Net. In AWS-VPC mode, 
+containers are networked without using an overlay and allows network speeds 
+close to that of the underlying network.
+
+With AWS-VPC enabled, Weave Net manages IP addresses and connects
+containers to the network as usual, but instead of wrapping each packet and
+sending it to its destination, Weave Net instructs the AWS network
+router with the ranges of container IP addresses and the instances 
+on which they live. 
+
+![Weave Net AWS-VPC Mode](weave-net-awsvpc-1007x438.png)
+
+###Configuring EC2 Instances to use Weave AWS-VPC Mode
+
+First, your AWS instances need to be given write access to the route
+table via its
+[IAM Role](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html).
+If you have an existing IAM Role then extend it, otherwise create a
+new role. The role must have a
+[policy](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-policies-for-amazon-ec2.html)
+attached which allows the following
+[actions](http://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_Operations.html):
+
+
+```
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Effect": "Allow",
+            "Action": [
+                "ec2:CreateRoute",
+                "ec2:DeleteRoute",
+                "ec2:ReplaceRoute",
+                "ec2:DescribeRouteTables",
+                "ec2:DescribeInstances"
+            ],
+            "Resource": [
+                "*"
+            ]
+        }
+    ]
+}
+```
+
+Secondly, your Security Group must allow network traffic between instances.  
+You must open TCP port 6783 which is the port that Weave Net uses to manage the network and also 
+allow any ports which your own containers use. 
+
+>>**Remember:** There is no network overlay in this mode, and so, IP packets with container
+addresses will flow over the AWS network unmodified.
+
+Finally, since Weave will be operating with IP addresses outside of the 
+range allocated by Amazon, you must disable "Source/Destination check" on each machine.
+
+###Using AWS-VPC Mode
+
+Launch Weave Net with the `--awsvpc` flag:
+
+    $ weave launch --awsvpc [other hosts]
+
+ >>**Note:** You will still need to supply the names or IP addresses of other hosts in
+your cluster.
+
+###Present Limitations
+
+- AWS-VPC mode does not inter-operate with other Weave Net modes; it
+  is all or nothing.  In this mode, all hosts in a cluster must be AWS
+  instances. (We hope to ease this limitation in future.)
+- The `weave launch` command waits until the
+  [IP address manager](/site//operational-guide/concepts.md#ip-address-manager-ipam)
+  is ready, i.e. until after this peer has been able to make contact
+  with other peers and confirm that it has joined the cluster the
+  bridge.  Without AWS-VPC, `weave launch` returns without waiting.
+- The AWS network does not support multicast.
+- The number of hosts in a cluster is limited by the maximum size of
+  your AWS route table.  This is limited to 50 entries though you
+  can request an increase to 100 by contacting Amazon.
+- All of your containers must be on the same network, with no subnet
+  isolation. (We hope to ease this limitation in future.)
+
+###Packet size (MTU)
+
+The Maximum Transmission Unit, or MTU, is the technical term for the
+limit on how big a single packet can be on the network. Weave Net
+defaults to 1410 bytes. This default works across almost all networks, but for better 
+performance you can set it to a larger MTU size. 
+
+The AWS network supports packets of up to 9000 bytes.  
+
+In AWS-VPC mode you can run the following:
+
+    $ WEAVE_MTU=9000 weave launch --awsvpc host2 host3
+
+**See Also**
+
+ * [Using Weave Net](/site/using-weave.md)
+ * [Performance measurements](/blog/weave-docker-networking-performance-aws-vpc/)
+
+(that last blog post doesn't exist yet)

site/using-weave/dynamically-attach-containers.md

@@ -8,13 +8,13 @@ When containers may not know the network to which they will be attached, Weave N
 
 To illustrate...
 
-    host1$ C=$(docker run -e WEAVE_CIDR=none -dti ubuntu)
+    host1$ C=$(docker run -e WEAVE_CIDR=none -dti weaveworks/ubuntu)
     host1$ weave attach $C
     10.2.1.3
 
 where,
 
- *  `C=$(docker run -e WEAVE_CIDR=none -dti ubuntu)` starts a container and assigns its ID to a variable
+ *  `C=$(docker run -e WEAVE_CIDR=none -dti weaveworks/ubuntu)` starts a container and assigns its ID to a variable
  *  `weave attach` – the Weave Net command to attach to the specified container
  *  `10.2.1.3` - the allocated IP address output by `weave attach`, in this case in the default subnet
 

site/using-weave/finding-adding-hosts-dynamically.md

@@ -3,58 +3,69 @@ title: Adding and Removing Hosts Dynamically
 menu_order: 90
 ---
 
+To add a host to an existing Weave network, launch Weave Net on the
+host, and supply the address of at least one host. Weave Net
+automatically discovers any other hosts in the network and establishes
+connections with them if it can (in order to avoid unnecessary
+multi-hop routing).
 
-To add a host to an existing Weave network, simply launch 
-Weave Net on the host, and then supply the address of at least 
-one host. Weave Net automatically discovers any other hosts in 
-the network and establishes connections with them if it 
-can (in order to avoid unnecessary multi-hop routing).
+In some situations existing Weave Net hosts may be unreachable from
+the new host due to firewalls, etc.  However, it is still possible to
+add the new host, provided that inverse connections, for example, from
+existing hosts to the new hosts, are available.
 
-In some situations all existing Weave Net hosts may be 
-unreachable from the new host due to firewalls, etc. 
-However, it is still possible to add the new host, 
-provided that inverse connections, for example, 
-from existing hosts to the new hosts, are available. 
-
-To accomplish this, launch Weave Net onto the new host 
-without supplying any additional addresses.  And then, from one 
-of the existing hosts run:
+To accomplish this, launch Weave Net onto the new host without
+supplying any additional addresses and then, from one of the existing
+hosts run:
 
     host# weave connect $NEW_HOST
 
-Other hosts on the Weave network will automatically attempt
-to establish connections to the new host as well. 
+Any other existing hosts on the Weave network will attempt to
+establish connections to the new host as well.
 
-Alternatively, you can also instruct a peer to forget a 
-particular host specified to it via `weave launch` or 
-`weave connect` by running:
+###Instructing Peers to Forget a Host
+
+To instruct a peer to forget a particular host specified to it via
+`weave launch` or `weave connect` run:
 
     host# weave forget $DECOMMISSIONED_HOST
 
-This prevents the peer from trying to reconnect to that host 
-once connectivity to it is lost, and therefore can be used 
-to administratively remove any decommissioned peers 
-from the network.
+This prevents the peer from reconnecting to that host once
+connectivity to it is lost, and can be used to administratively remove
+any decommissioned peers from the network.
 
-Hosts may also be bulk-replaced. All existing hosts 
-will be forgotten, and the new hosts will be added:
+###Bulk Replacing Hosts
+
+Hosts can also be bulk-replaced. All existing hosts will be forgotten,
+and the new hosts added:
 
     host# weave connect --replace $NEW_HOST1 $NEW_HOST2
 
-For complete control over the peer topology, automatic 
-discovery can be disabled using the `--no-discovery` 
-option with `weave launch`. 
+###Restarting Docker and Weave Net
 
-If discovery if disabled, Weave Net only connects to the 
-addresses specified at launch time and with `weave connect`.
+If Weave Net is restarted by Docker it automatically remembers any
+previous connect and forget operations, however if you stop it
+manually and launch it again, it will not remember any prior connects.
+If you want to launch again and retain the results of those operations
+use `--resume`:
 
-A list of all hosts that a peer has been asked to connect 
-to with `weave launch` and `weave connect` 
-can be obtained using:
+    host# weave launch --resume
+
+> **Note:** In this case, you cannot specify a list of addresses,
+> since the previous peer list is used exclusively.
+
+For complete control over the peer topology, disable automatic
+discovery using the `--no-discovery` option with `weave launch`.
+
+If discovery if disabled, Weave Net only connects to the addresses
+specified at launch time and with `weave connect`.
+
+To return a list of all hosts and their peer connections established
+with `weave launch` and `weave connect` run:
 
     host# weave status targets
 
-**See Also** 
+**See Also**
 
  * [Enabling Multi-Cloud, Multi-Hop Networking and Routing](/site/using-weave/multi-cloud-multi-hop.md)
  * [Stopping and Removing Peers](/site/ipam/stop-remove-peers-ipam.md)

site/using-weave/manual-ip-address.md

@@ -21,12 +21,12 @@ For example, we can launch a couple of containers on `$HOST1` and
 
 On `$HOST1`:
 
-    host1$ docker run -e WEAVE_CIDR=10.2.1.1/24 -ti ubuntu
+    host1$ docker run -e WEAVE_CIDR=10.2.1.1/24 -ti weaveworks/ubuntu
     root@7ca0f6ecf59f:/#
 
 And `$HOST2`:
 
-    host2$ docker run -e WEAVE_CIDR=10.2.1.2/24 -ti ubuntu
+    host2$ docker run -e WEAVE_CIDR=10.2.1.2/24 -ti weaveworks/ubuntu
     root@04c4831fafd3:/#
 
 Then test that the container on `$HOST2` can be reached from the container on `$HOST1`:

site/weave-docker-api/automatic-discovery-proxy.md

@@ -37,7 +37,7 @@ For example, you can launch the proxy using all three flags, as follows:
 
 After launching the Weave Net proxy with these flags, running a container named `aws-12798186823-foo` without labels results in weaveDNS registering the hostname `my-app-foo` and not `aws-12798186823-foo`.
 
-    host1$ docker run -ti --name=aws-12798186823-foo ubuntu ping my-app-foo
+    host1$ docker run -ti --name=aws-12798186823-foo weaveworks/ubuntu ping my-app-foo
     PING my-app-foo.weave.local (10.32.0.2) 56(84) bytes of data.
     64 bytes from my-app-foo.weave.local (10.32.0.2): icmp_seq=1 ttl=64 time=0.027 ms
     64 bytes from my-app-foo.weave.local (10.32.0.2): icmp_seq=2 ttl=64 time=0.067 ms
@@ -45,7 +45,7 @@ After launching the Weave Net proxy with these flags, running a container named
 Also, running a container named `foo` with the label
 `hostname-label=aws-12798186823-foo` leads to the same hostname registration.
 
-    host1$ docker run -ti --name=foo --label=hostname-label=aws-12798186823-foo ubuntu ping my-app-foo
+    host1$ docker run -ti --name=foo --label=hostname-label=aws-12798186823-foo weaveworks/ubuntu ping my-app-foo
     PING my-app-foo.weave.local (10.32.0.2) 56(84) bytes of data.
     64 bytes from my-app-foo.weave.local (10.32.0.2): icmp_seq=1 ttl=64 time=0.031 ms
     64 bytes from my-app-foo.weave.local (10.32.0.2): icmp_seq=2 ttl=64 time=0.042 ms

site/weave-docker-api/ipam-proxy.md

@@ -8,16 +8,16 @@ If [automatic IP address allocation](/site/ipam.md) is enabled in Weave Net (by
 then containers started via the proxy are automatically assigned an IP address, *without having to specify any
 special environment variables or any other options*.
 
-    host1$ docker run -ti ubuntu
+    host1$ docker run -ti weaveworks/ubuntu
 
 To use a specific subnet, you can pass a `WEAVE_CIDR` to the container, for example:
 
-    host1$ docker run -ti -e WEAVE_CIDR=net:10.32.2.0/24 ubuntu
+    host1$ docker run -ti -e WEAVE_CIDR=net:10.32.2.0/24 weaveworks/ubuntu
 
 To start a container without connecting it to the Weave network, pass
 `WEAVE_CIDR=none`, for example:
 
-    host1$ docker run -ti -e WEAVE_CIDR=none ubuntu
+    host1$ docker run -ti -e WEAVE_CIDR=none weaveworks/ubuntu
 
 ###Disabling Automatic IP Address Allocation
 
@@ -33,7 +33,7 @@ Containers started with a `WEAVE_CIDR` environment variable are handled as befor
 To automatically assign an address in this mode, start the
 container with a blank `WEAVE_CIDR`, for example:
 
-    host1$ docker run -ti -e WEAVE_CIDR="" ubuntu
+    host1$ docker run -ti -e WEAVE_CIDR="" weaveworks/ubuntu
     
 **See Also**
 

site/weave-docker-api/launching-without-proxy.md

@@ -7,7 +7,7 @@ menu_order: 50
 If you don't want to use the proxy, you can also launch
 containers on to the Weave network using `weave run`:
 
-    $ weave run -ti ubuntu
+    $ weave run -ti weaveworks/ubuntu
 
 The arguments after `run` are passed through to `docker run`. Therefore you
 can freely specify whatever Docker options you need. 
@@ -17,14 +17,14 @@ this example, it obtains an automatically allocated IP.
 
 You can specify IP addresses manually instead:
 
-    $ weave run 10.2.1.1/24 -ti ubuntu
+    $ weave run 10.2.1.1/24 -ti weaveworks/ubuntu
 
 `weave run` rewrites `/etc/hosts` in the same way
 [the proxy does](/site/weave-docker-api/name-resolution-proxy.md). If you need to keep
 the original file, specify `--no-rewrite-hosts` when running
 the container:
 
-    $ weave run --no-rewrite-hosts 10.2.1.1/24 -ti ubuntu
+    $ weave run --no-rewrite-hosts 10.2.1.1/24 -ti weaveworks/ubuntu
 
 There are some limitations to starting containers using `weave run`:
 

site/weave-docker-api/using-proxy.md

@@ -16,18 +16,17 @@ Weave network.
 
 To create and start a container via the Weave Net proxy run:
 
-    host1$ docker run -ti ubuntu
+    host1$ docker run -ti weaveworks/ubuntu
 
 or, equivalently run:
 
-    host1$ docker create -ti ubuntu
-    5ef831df61d50a1a49272357155a976595e7268e590f0a2c75693337b14e1382
-    host1$ docker start 5ef831df61d50a1a49272357155a976595e7268e590f0a2c75693337b14e1382
+    host1$ docker create -ti --name=foo weaveworks/ubuntu
+    host1$ docker start foo
 
 Specific IP addresses and networks can be supplied in the `WEAVE_CIDR`
 environment variable, for example:
 
-    host1$ docker run -e WEAVE_CIDR=10.2.1.1/24 -ti ubuntu
+    host1$ docker run -e WEAVE_CIDR=10.2.1.1/24 -ti weaveworks/ubuntu
 
 Multiple IP addresses and networks can be supplied in the `WEAVE_CIDR`
 variable by space-separating them, as in

site/weavedns.md

@@ -27,8 +27,8 @@ resolution, and will register themselves if they either have a
 hostname in the weaveDNS domain (`weave.local` by default) or are given an explicit container name:
 
 ```
-host1$ docker run -dti --name=pingme ubuntu
-host1$ docker run -ti  --hostname=ubuntu.weave.local ubuntu
+host1$ docker run -dti --name=pingme weaveworks/ubuntu
+host1$ docker run -ti  --hostname=ubuntu.weave.local weaveworks/ubuntu
 root@ubuntu:/# ping pingme
 ...
 ```

site/weavedns/load-balance-fault-weavedns.md

@@ -15,7 +15,7 @@ some ping tests.
 ```
 host2$ weave launch $HOST1
 host2$ eval $(weave env)
-host2$ docker run -dti --name=pingme ubuntu
+host2$ docker run -dti --name=pingme weaveworks/ubuntu
 
 root@ubuntu:/# ping -nq -c 1 pingme
 PING pingme.weave.local (10.32.0.2) 56(84) bytes of data.

site/weavedns/managing-domains-weavedns.md

@@ -10,12 +10,10 @@ The following topics are discussed:
 
 ## <a name="domain-search-path"></a>Configuring the domain search paths
 
-If you don't supply a domain search path (with `--dns-search=`),
-`weave run ...` tells a container to look for "bare" hostnames, like
-`pingme`, in its own domain (or in `weave.local` if it has no domain).
-That's why you can just invoke `ping pingme` above -- since the
-hostname is `ubuntu.weave.local`, it will look for
-`pingme.weave.local`.
+If you don't supply a domain search path (with `--dns-search=`), Weave
+Net (via the [proxy](/site/weave-docker-api.md) or via `weave run`)
+tells a container to look for "bare" hostnames, like `pingme`, in its
+own domain (or in `weave.local` if it has no domain).
 
 If you want to supply other entries for the domain search path,
 e.g. if you want containers in different sub-domains to resolve
@@ -27,7 +25,7 @@ behaviour.
 docker run -ti \
   --dns-search=zone1.weave.local --dns-search=zone2.weave.local \
   --dns-search=corp1.com --dns-search=corp2.com \
-  --dns-search=weave.local ubuntu
+  --dns-search=weave.local weaveworks/ubuntu
 ```
 
 ## <a name="local-domain"></a>Using a different local domain

site/weavedns/managing-entries-weavedns.md

@@ -20,7 +20,7 @@ If you want to give the container a name in DNS *other* than its
 hostname, you can register it using the `dns-add` command. For example:
 
 ```
-$ C=$(docker run -ti ubuntu)
+$ C=$(docker run -ti weaveworks/ubuntu)
 $ weave dns-add $C -h pingme2.weave.local
 ```
 

test/090_docker_restart_policy_2_test.sh

@@ -4,24 +4,6 @@
 
 start_suite "Test docker restart policy"
 
-check_restart() {
-    OLD_PID=$(container_pid $1 $2)
-
-    run_on $1 sudo kill $OLD_PID
-
-    for i in $(seq 1 10); do
-        NEW_PID=$(container_pid $1 $2)
-
-        if [ $NEW_PID != 0 -a $NEW_PID != $OLD_PID ] ; then
-            return 0
-        fi
-
-        sleep 1
-    done
-
-    return 1
-}
-
 weave_on $HOST1 launch
 
 assert "docker_on $HOST1 inspect -f '{{.HostConfig.RestartPolicy.Name}}' weave weaveproxy weaveplugin" "always\nalways\nalways"

test/140_weave_local_test.sh

@@ -16,10 +16,10 @@ weave_local_on $HOST1 launch --ipalloc-range 10.2.5.0/24
 assert_raises "docker_on $HOST1 ps | grep weave"
 
 weave_local_on $HOST1 run 10.2.6.5/24 -ti --name=c1 $SMALL_IMAGE /bin/sh
-assert_raises "exec_on $HOST1 c1 $CHECK_ETHWE_UP"
+wait_for_attached $HOST1 c1
 
 weave_local_on $HOST1 run             -ti --name=c2 $SMALL_IMAGE /bin/sh
-assert_raises "exec_on $HOST1 c2 $CHECK_ETHWE_UP"
+wait_for_attached $HOST1 c2
 
 assert "weave_local_on $HOST1 ps | wc -l" 3
 

test/175_stop_start_test.sh

@@ -1,30 +0,0 @@
-#! /bin/bash
-
-. ./config.sh
-
-start_suite "Check restart re-uses same container when possible"
-
-ID=$(weave_on $HOST1 launch-router)
-weave_on $HOST1 stop-router
-assert "weave_on $HOST1 launch-router" "$ID"
-# Can't start if already running with different args
-assert_raises "weave_on $HOST1 launch-router --log-level=debug" 1
-# Stop then start with different arg
-weave_on $HOST1 stop-router
-ID2=$(weave_on $HOST1 launch-router --log-level=debug)
-assert "[ $ID != $ID2 ]"
-
-# Stop then start with different password
-weave_on $HOST1 stop-router
-ID3=$(weave_on $HOST1 launch-router --password=xyzzy)
-assert "[ $ID2 != $ID3 ]"
-# Stop then start with same password
-weave_on $HOST1 stop-router
-assert "weave_on $HOST1 launch-router --password=xyzzy" "$ID3"
-
-# Fails on bad argument
-assert_raises "weave_on $HOST1 launch-plugin --foo" 1
-# Fails again on bad argument, although there is a dead container
-assert_raises "weave_on $HOST1 launch-plugin --foo" 1
-
-end_suite

test/175_target_peer_persistence_3_test.sh

@@ -0,0 +1,36 @@
+#! /bin/bash
+
+. ./config.sh
+
+
+assert_targets() {
+    HOST=$1
+    shift
+    EXPECTED=$(for TARGET in $@; do echo $TARGET; done | sort)
+    assert "weave_on $HOST report | jq -r '.Router.Targets[] | tostring' | sort" "$EXPECTED"
+}
+
+start_suite "Check Docker restart uses persisted peer list"
+
+# Launch router and modify initial peer list
+weave_on $HOST1 launch $HOST1 $HOST2
+weave_on $HOST1 forget $HOST1
+weave_on $HOST1 connect $HOST3
+
+# Ensure modified peer list is still in effect after restart
+check_restart $HOST1 weave
+assert_targets $HOST1 $HOST2 $HOST3
+
+# Ensure persisted peer changes are still in effect after --resume
+weave_on $HOST1 stop
+weave_on $HOST1 launch --resume
+assert_targets $HOST1 $HOST2 $HOST3
+
+# Ensure persisted peer changes are ignored after stop and subsequent restart
+weave_on $HOST1 stop
+weave_on $HOST1 launch $HOST1 $HOST2
+assert_targets $HOST1 $HOST1 $HOST2
+check_restart $HOST1 weave
+assert_targets $HOST1 $HOST1 $HOST2
+
+end_suite

test/195_no_multicast_route_1_test.sh

@@ -8,10 +8,13 @@ show_multicast_route_on() {
 
 start_suite "--no-multicast-route operation"
 
+# In case one is left around from a previous test
+$SSH $HOST1 docker network rm testmcasttrue testmcastfalse >/dev/null 2>&1 || true
+
 # Ensure containers run either way have no multicast route
 weave_on $HOST1 launch-router
 weave_on $HOST1 launch-proxy --no-multicast-route
-weave_on $HOST1 launch-plugin --no-multicast-route
+weave_on $HOST1 launch-plugin # plugin defaults to multicast off except for 'net=weave'
 
 start_container $HOST1 --no-multicast-route --name c1
 proxy_start_container $HOST1 --name c2
@@ -19,10 +22,19 @@ start_container_local_plugin $HOST1 --name=c3
 
 assert "show_multicast_route_on $HOST1 c1"
 assert "show_multicast_route_on $HOST1 c2"
-assert "show_multicast_route_on $HOST1 c3"
+assert "show_multicast_route_on $HOST1 c3" "224.0.0.0/4 dev ethwe0 "
+
+# Now try via docker network options
+# using ssh rather than docker -H because CircleCI docker client is older
+$SSH $HOST1 docker network create --driver weavemesh --ipam-driver weavemesh --opt works.weave.multicast=false testmcastfalse
+$SSH $HOST1 docker run --net=testmcastfalse --name c4 -di $SMALL_IMAGE /bin/sh
+assert "show_multicast_route_on $HOST1 c4"
+$SSH $HOST1 docker network create --driver weavemesh --ipam-driver weavemesh --opt works.weave.multicast testmcasttrue
+$SSH $HOST1 docker run --net=testmcasttrue --name c5 -di $SMALL_IMAGE /bin/sh
+assert "show_multicast_route_on $HOST1 c5" "224.0.0.0/4 dev ethwe0 "
 
 # Ensure current proxy options are obeyed on container start
-docker_on $HOST1 stop c2
+docker_on $HOST1 stop -t 1 c2
 weave_on $HOST1 stop-proxy
 weave_on $HOST1 launch-proxy
 
@@ -30,6 +42,7 @@ proxy docker_on $HOST1 start c2
 
 assert "show_multicast_route_on $HOST1 c2" "224.0.0.0/4 dev ethwe "
 
-docker_on $HOST1 rm -f c3
+docker_on $HOST1 rm -f c3 c4 c5
+$SSH $HOST1 docker network rm testmcasttrue testmcastfalse
 
 end_suite

test/300_ip_reuse_test.sh

@@ -19,11 +19,11 @@ start_container $HOST1 $C1/24 --name=c1
 start_container $HOST1 $C2/24 --name=c2
 check
 
-docker_on $HOST1 stop c2
+docker_on $HOST1 stop -t 1 c2
 start_container $HOST1 $C2/24 --name=c3
 check
 
-docker_on $HOST1 stop c3
+docker_on $HOST1 stop -t 1 c3
 start_container $HOST1 $C3/24 --name=c4
 weave_on $HOST1 attach $C2/24 c4
 check

test/310_docker_restart_test.sh

@@ -2,24 +2,6 @@
 
 . ./config.sh
 
-check_attached() {
-    for c in $@; do
-        assert_raises "exec_on $HOST1 $c $CHECK_ETHWE_UP"
-    done
-}
-
-wait_for_proxy() {
-    for i in $(seq 1 120); do
-        echo "Waiting for proxy to start"
-        if proxy docker_on $1 info > /dev/null 2>&1 ; then
-            return
-        fi
-        sleep 1
-    done
-    echo "Timed out waiting for proxy to start" >&2
-    exit 1
-}
-
 start_suite "Containers get same IP address on restart"
 
 weave_on $HOST1 launch-router
@@ -31,15 +13,14 @@ start_container $HOST1 --name=c1
 proxy docker_on $HOST1 run -di --name=c2 --restart=always -dt --entrypoint="/home/weave/sigproxy" weaveworks/weaveexec sleep 600
 C2=$(container_ip $HOST1 c2)
 assert_raises "[ -n $C2 ]"
-check_attached c2
+wait_for_attached $HOST1 c2
 
 docker_on $HOST1 rm -f c1
 
 # Restart docker daemon, using different commands for systemd- and upstart-managed.
 run_on $HOST1 sh -c "command -v systemctl >/dev/null && sudo systemctl restart docker || sudo service docker restart"
 wait_for_proxy $HOST1
-sleep 3 # allow for re-tries of attach
-check_attached c2
+wait_for_attached $HOST1 c2
 # Check same IP address was retained
 assert "container_ip $HOST1 c2" "$C2"
 

test/320_claim_3_test.sh

@@ -2,7 +2,9 @@
 
 . ./config.sh
 
-UNIVERSE=10.32.0.0/12
+UNIVERSE=10.2.0.0/16
+C1=10.2.128.1
+C4=10.2.128.4
 
 delete_persistence() {
     for host in "$@" ; do
@@ -13,12 +15,11 @@ delete_persistence() {
 
 start_suite "Claiming addresses"
 
-weave_on $HOST1 launch-router
-weave_on $HOST2 launch-router $HOST1
+weave_on $HOST1 launch-router --ipalloc-range $UNIVERSE $HOST2
+weave_on $HOST2 launch-router --ipalloc-range $UNIVERSE $HOST1
 
-start_container $HOST1 --name=c1
-start_container $HOST2 --name=c2
-C1=$(container_ip $HOST1 c1)
+start_container $HOST1 $C1/12 --name=c1
+start_container $HOST2        --name=c2
 assert_raises "exec_on $HOST2 c2 $PING $C1"
 
 stop_weave_on $HOST1
@@ -28,8 +29,8 @@ stop_weave_on $HOST2
 delete_persistence $HOST1 $HOST2
 
 # Start hosts in reverse order so c1's address has to be claimed from host2
-weave_on $HOST2 launch-router
-weave_on $HOST1 launch-router $HOST2
+weave_on $HOST2 launch-router --ipalloc-range $UNIVERSE
+weave_on $HOST1 launch-router --ipalloc-range $UNIVERSE $HOST2
 
 # Start another container on host2, so if it hasn't relinquished c1's
 # address it would give that out as the first available.
@@ -47,29 +48,31 @@ delete_persistence $HOST1 $HOST2
 
 # Now make host1 attempt to claim from host2, when host2 is stopped
 # the point being to check whether host1 will hang trying to talk to host2
-weave_on $HOST2 launch-router
+weave_on $HOST2 launch-router --ipalloc-range $UNIVERSE
 # Introduce host3 to remember the IPAM CRDT when we stop host2
-weave_on $HOST3 launch-router $HOST2
+weave_on $HOST3 launch-router --ipalloc-range $UNIVERSE $HOST2
 weave_on $HOST3 prime
 stop_weave_on $HOST2
-weave_on $HOST1 launch-router $HOST3
+weave_on $HOST1 launch-router --ipalloc-range $UNIVERSE $HOST3
 
 stop_weave_on $HOST1
-stop_weave_on $HOST2
 stop_weave_on $HOST3
-delete_persistence $HOST1 $HOST2 $HOST3
 
+# Delete persistence data on host1, so that host1 would try to establish a ring.
+delete_persistence $HOST1
 weave_on $HOST1 launch --ipalloc-range $UNIVERSE $HOST2
+
 # Start another container on host1. The starting should block, because host1 is
 # not able to establish the ring due to host2 being offline.
-CMD="proxy_start_container $HOST1 --name c4 -e WEAVE_CIDR=10.32.0.4/12"
+CMD="proxy_start_container $HOST1 --name c4 -e WEAVE_CIDR=$C4/12"
 assert_raises "timeout 5 cat <( $CMD )" 124
+
 # However, allocation for an external subnet should not block.
-assert_raises "proxy_start_container $HOST1 -e WEAVE_CIDR=10.48.0.1/12"
+assert_raises "proxy_start_container $HOST1 -e WEAVE_CIDR=10.3.0.1/12"
 
 # Launch host2, so that host1 can establish the ring.
 weave_on $HOST2 launch --ipalloc-range $UNIVERSE $HOST1
-weave_on $HOST1 prime
-assert_raises "[ $(container_ip $HOST1 c4) == 10.32.0.4 ]"
+wait_for_attached $HOST1 c4
+assert_raises "[ $(container_ip $HOST1 c4) == $C4 ]"
 
 end_suite

test/500_weave_multi_cidr_test.sh

@@ -77,7 +77,7 @@ CID=$(start_container  $HOST1             10.2.1.1/24 ip:10.2.2.1/24 net:10.2.3.
 assert_ips_and_dns     $HOST1 $CID $NAME. 10.2.1.1/24    10.2.2.1/24     10.2.3.1/24
 
 # Stop the container
-docker_on              $HOST1 stop $CID
+docker_on              $HOST1 stop -t 1 $CID
 assert_ips_and_dns     $HOST1 $CID $NAME.
 
 # Restart with three IPs
@@ -122,4 +122,8 @@ assert_equal "$IPS"                                                      10.2.3.
 CID2=$(start_container $HOST1                                        net:10.2.3.0/24)
 assert_container_cidrs $HOST1 $CID2                                      10.2.3.1/24
 
+# Error conditions: host address not network, subnet too small
+assert_raises "start_container $HOST1                                net:10.2.3.2/30" 1
+assert_raises "start_container $HOST1                                net:10.2.3.2/31" 1
+
 end_suite