Add release v1.4.5

The nil FlowOp check in collectFlowOps was not addressed with the
introduction of DiscardingFlowOp.  This meant that DiscardingFlowOps
produced in the router core would get handled by fastdp as if they were
foreign FlowOps, preventing flow creation and so leading to unwarranted
ODP misses.

To fix this, collectFlowOps cannot simply eliminate FlowOps for which
Discards() is true, because it's true for odpFlowKeyFlowOp.  So the
Discards() case has to be handled in FastDatapath send().

.travis.yml

@@ -1,28 +1,13 @@
-language: go
+language: python
 sudo: false
-go:
-  - 1.4
 
 env:
-  - PATH="${PATH}:${HOME}/.local/bin"
-
-addons:
-  apt:
-    packages:
-    - libpcap-dev
+  - PATH="${PATH}:${HOME}/gsutil"
 
 install:
-  - go clean -i net
-  - go install -tags netgo std
-  - go get github.com/golang/lint/golint
-  - pip install --user --upgrade gsutil
+  - pip install pyopenssl
+  - curl --silent --location https://storage.googleapis.com/pub/gsutil.tar.gz | tar xz -C $HOME
   - bin/setup-ci-secrets $encrypted_5ba036e89377_key $encrypted_5ba036e89377_iv
-  - make travis
-  - go get ./...
 
 script:
-  - make tests
-
-after_success:
-  - go get github.com/mattn/goveralls
-  - test "${TRAVIS_PULL_REQUEST}" = "false" && site/build-and-publish.sh
+  - 'test "${TRAVIS_PULL_REQUEST}" = "true" || site/build-and-publish.sh'

CHANGELOG.md

@@ -1,3 +1,61 @@
+## Release 1.4.5
+
+Higher performance for multicast and broadcast traffic when using
+Weave's Fast Datapath
+
+* The flow rule to deliver broadcast and multicast packets in-kernel
+  was not created correctly, hence every such packet caused a
+  context-switch to the software router #2003/#2008
+
+Other fixes:
+
+* Remove DNS entries for containers that are being restarted by Docker
+  but are not live yet #1977/#1989
+* Don't let one failing allocation attempt prevent others from
+  succeeding; they could be in different subnets which are more
+  available #1996/#2001
+* Don't complain on second router launch on kernels that lack support
+  for Fast Datapath #1929/#1983
+* Fix build broken by change in libnetwork IPAM API #1984/#1985
+
+## Release 1.4.4
+
+Fixing a rather serious issue that slipped through our preparations
+for Docker 1.10:
+
+* Restarting Docker or rebooting your machine while the Weave Net
+  plugin is running causes Docker 1.10.0 to fail to start #1959/#1963
+
+Also one other small fix:
+
+* Avoid a hang when trying to use plugin and proxy at the same time
+  #1956/#1961
+
+## Release 1.4.3
+
+Preparing for Docker 1.10, plus some bug-fixes.
+
+* Avoid hang in Docker v1.10 on `docker volume ls` after `weave stop`,
+  `weave stop-plugin` or `weave reset` #1934/#1936
+* Fix "unexpected EOF" from Docker 1.10 on `docker exec` with Weave
+  proxy `--rewrite-inspect` enabled #1911/#1917
+* Avoid losing DNS entries and potentially double-allocating IP
+  Addresses allocated via plugin, on router restart; also extend
+  `weave ps` to show IP addresses allocated via plugin #1745/#1921
+* Stop creating lots of copies of `weavewait` program in Docker
+  volumes #1757/#1935
+* Prevent container starting prematurely when proxy in
+`--no-multicast-route` mode #1942/#1943
+* Log error message from plugin rather than crashing when weave not
+  running #1906/#1918
+* Warn, don't error, if unable to remove plugin network in 'weave
+  stop', to avoid breaking the usual upgrade or config change process
+  #1900/#1919
+* Don't crash if network conditions suggest only very small packets
+  will get through #1905/#1926
+* Cope with unexpected errors during route traversal when starting
+  container via proxy #1909/#1910/#1932
+
 ## Release 1.4.2
 
 Bug-fixes and minor improvements.

Makefile

@@ -119,7 +119,7 @@ $(WEAVEEXEC_UPTODATE): prog/weaveexec/Dockerfile prog/weaveexec/symlink $(DOCKER
 	$(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker build -t $(WEAVEEXEC_IMAGE) prog/weaveexec
 	touch $@
 
-$(DOCKERPLUGIN_UPTODATE): prog/plugin/Dockerfile $(DOCKERPLUGIN_EXE)
+$(DOCKERPLUGIN_UPTODATE): prog/plugin/Dockerfile $(DOCKERPLUGIN_EXE) $(WEAVEEXEC_UPTODATE)
 	$(SUDO) docker build -t $(DOCKERPLUGIN_IMAGE) prog/plugin
 	touch $@
 

common/docker/client.go

@@ -97,7 +97,7 @@ func (c *Client) AddObserver(ob ContainerObserver) error {
 func (c *Client) IsContainerNotRunning(idStr string) bool {
 	container, err := c.InspectContainer(idStr)
 	if err == nil {
-		return !container.State.Running
+		return !container.State.Running || container.State.Restarting
 	}
 	if _, notThere := err.(*docker.NoSuchContainer); notThere {
 		return true

ipam/allocator.go

@@ -140,29 +140,23 @@ func (alloc *Allocator) cancelOpsFor(ops *[]operation, ident string) bool {
 	return found
 }
 
-// Try all pending operations
-func (alloc *Allocator) tryPendingOps() {
-	// The slightly different semantics requires us to operate on 'claims' and
-	// 'allocates' separately:
-	// Claims must be tried before Allocates
-	for i := 0; i < len(alloc.pendingClaims); {
-		op := alloc.pendingClaims[i]
+// Try all operations in a queue
+func (alloc *Allocator) tryOps(ops *[]operation) {
+	for i := 0; i < len(*ops); {
+		op := (*ops)[i]
 		if !op.Try(alloc) {
 			i++
 			continue
 		}
-		alloc.pendingClaims = append(alloc.pendingClaims[:i], alloc.pendingClaims[i+1:]...)
+		*ops = append((*ops)[:i], (*ops)[i+1:]...)
 	}
+}
 
-	// When the first Allocate fails, bail - no need to
-	// send too many begs for space.
-	for i := 0; i < len(alloc.pendingAllocates); {
-		op := alloc.pendingAllocates[i]
-		if !op.Try(alloc) {
-			break
-		}
-		alloc.pendingAllocates = append(alloc.pendingAllocates[:i], alloc.pendingAllocates[i+1:]...)
-	}
+// Try all pending operations
+func (alloc *Allocator) tryPendingOps() {
+	// Process existing claims before servicing new allocations
+	alloc.tryOps(&alloc.pendingClaims)
+	alloc.tryOps(&alloc.pendingAllocates)
 }
 
 func (alloc *Allocator) spaceRequestDenied(sender mesh.PeerName, r address.Range) {

nameserver/nameserver.go

@@ -84,8 +84,8 @@ func (n *Nameserver) broadcastEntries(es ...Entry) error {
 }
 
 func (n *Nameserver) AddEntry(hostname, containerid string, origin mesh.PeerName, addr address.Address) error {
-	n.infof("adding entry %s -> %s", hostname, addr.String())
 	n.Lock()
+	n.infof("adding entry for %s: %s -> %s", containerid, hostname, addr.String())
 	entry := n.entries.add(hostname, containerid, origin, addr)
 	n.Unlock()
 	return n.broadcastEntries(entry)

net/if.go

@@ -52,16 +52,25 @@ func EnsureInterfaceAndMcastRoute(ifaceName string) (*net.Interface, error) {
 		return nil, err
 	}
 	dest := net.IPv4(224, 0, 0, 0)
-	// check for currently-existing route after subscribing, to avoid race
-	if CheckRouteExists(ifaceName, dest) {
-		return iface, err
+	check := func(route netlink.Route) bool {
+		return route.Dst != nil && route.Dst.IP.Equal(dest)
 	}
-	for update := range ch {
-		link, _ := netlink.LinkByIndex(update.Route.LinkIndex)
-		if link.Attrs().Name == ifaceName && update.Route.Dst.IP.Equal(dest) {
-			break
+	// check for currently-existing route after subscribing, to avoid race
+	routes, err := netlink.RouteList(nil, netlink.FAMILY_V4)
+	if err != nil {
+		return nil, err
+	}
+	for _, route := range routes {
+		if check(route) {
+			return iface, nil
 		}
 	}
+	for update := range ch {
+		if check(update.Route) {
+			return iface, nil
+		}
+	}
+	// should never get here
 	return iface, nil
 }
 

net/route.go

@@ -10,7 +10,7 @@ import (
 // A network is considered free if it does not overlap any existing
 // routes on this host. This is the same approach taken by Docker.
 func CheckNetworkFree(subnet *net.IPNet, ignoreIfaceNames map[string]struct{}) error {
-	return forEachRoute(ignoreIfaceNames, func(name string, route netlink.Route) error {
+	return forEachRoute(ignoreIfaceNames, func(route netlink.Route) error {
 		if route.Dst != nil && overlaps(route.Dst, subnet) {
 			return fmt.Errorf("Network %s overlaps with existing route %s on host.", subnet, route.Dst)
 		}
@@ -26,7 +26,7 @@ func overlaps(n1, n2 *net.IPNet) bool {
 // For a specific address, we only care if it is actually *inside* an
 // existing route, because weave-local traffic never hits IP routing.
 func CheckAddressOverlap(addr net.IP, ignoreIfaceNames map[string]struct{}) error {
-	return forEachRoute(ignoreIfaceNames, func(name string, route netlink.Route) error {
+	return forEachRoute(ignoreIfaceNames, func(route netlink.Route) error {
 		if route.Dst != nil && route.Dst.Contains(addr) {
 			return fmt.Errorf("Address %s overlaps with existing route %s on host.", addr, route.Dst)
 		}
@@ -34,32 +34,24 @@ func CheckAddressOverlap(addr net.IP, ignoreIfaceNames map[string]struct{}) erro
 	})
 }
 
-func forEachRoute(ignoreIfaceNames map[string]struct{}, check func(name string, r netlink.Route) error) error {
+func forEachRoute(ignoreIfaceNames map[string]struct{}, check func(r netlink.Route) error) error {
+	ignoreIfaceIndices := make(map[int]struct{})
+	for ifaceName := range ignoreIfaceNames {
+		if iface, err := net.InterfaceByName(ifaceName); err == nil {
+			ignoreIfaceIndices[iface.Index] = struct{}{}
+		}
+	}
 	routes, err := netlink.RouteList(nil, netlink.FAMILY_V4)
 	if err != nil {
 		return err
 	}
 	for _, route := range routes {
-		link, err := netlink.LinkByIndex(route.LinkIndex)
-		if err == nil {
-			if _, found := ignoreIfaceNames[link.Attrs().Name]; found {
-				continue
-			}
+		if _, found := ignoreIfaceIndices[route.LinkIndex]; found {
+			continue
 		}
-		if err := check(link.Attrs().Name, route); err != nil {
+		if err := check(route); err != nil {
 			return err
 		}
 	}
 	return nil
 }
-
-func CheckRouteExists(ifaceName string, dest net.IP) bool {
-	found := false
-	forEachRoute(map[string]struct{}{}, func(name string, route netlink.Route) error {
-		if name == ifaceName && route.Dst.IP.Equal(dest) {
-			found = true
-		}
-		return nil
-	})
-	return found
-}

plugin/ipam/driver.go

@@ -1,10 +1,13 @@
 package ipamplugin
 
 import (
+	"fmt"
 	"net"
 
+	"github.com/docker/libnetwork/discoverapi"
 	"github.com/docker/libnetwork/ipamapi"
 	"github.com/docker/libnetwork/netlabel"
+	godocker "github.com/fsouza/go-dockerclient"
 	"github.com/weaveworks/weave/api"
 	. "github.com/weaveworks/weave/common"
 	"github.com/weaveworks/weave/common/docker"
@@ -20,7 +23,13 @@ type ipam struct {
 }
 
 func NewIpam(client *docker.Client, version string) (ipamapi.Ipam, error) {
-	resolver := func() (string, error) { return client.GetContainerIP(WeaveContainer) }
+	resolver := func() (string, error) {
+		addr, err := client.GetContainerIP(WeaveContainer)
+		if _, ok := err.(*godocker.NoSuchContainer); ok {
+			return "", fmt.Errorf("%s container is not present. Have you launched it?", WeaveContainer)
+		}
+		return addr, err
+	}
 	return &ipam{client: client, weave: api.NewClientWithResolver(resolver)}, nil
 }
 
@@ -33,6 +42,9 @@ func (i *ipam) RequestPool(addressSpace, pool, subPool string, options map[strin
 	Log.Debugln("RequestPool", addressSpace, pool, subPool, options)
 	cidr, err := i.weave.DefaultSubnet()
 	Log.Debugln("RequestPool returning ", cidr, err)
+	if err != nil {
+		return "", nil, nil, err
+	}
 	// Pass back a fake "gateway address"; we don't actually use it,
 	// so just give the network address.
 	data := map[string]string{netlabel.Gateway: cidr.String()}
@@ -56,3 +68,13 @@ func (i *ipam) ReleaseAddress(poolID string, address net.IP) error {
 	Log.Debugln("ReleaseAddress", poolID, address)
 	return i.weave.ReleaseIP(address.String())
 }
+
+// Functions required by ipamapi "contract" but not actually used.
+
+func (i *ipam) DiscoverNew(discoverapi.DiscoveryType, interface{}) error {
+	return nil
+}
+
+func (i *ipam) DiscoverDelete(discoverapi.DiscoveryType, interface{}) error {
+	return nil
+}

plugin/skel/listener.go

@@ -265,6 +265,7 @@ func sendError(w http.ResponseWriter, msg string, code int) {
 }
 
 func errorResponse(w http.ResponseWriter, fmtString string, item ...interface{}) {
+	w.WriteHeader(http.StatusInternalServerError)
 	json.NewEncoder(w).Encode(map[string]string{
 		"Err": fmt.Sprintf(fmtString, item...),
 	})

prog/plugin/main.go

@@ -69,25 +69,35 @@ func main() {
 	Log.Println("Weave plugin", version, "Command line options:", os.Args[1:])
 	Log.Info(dockerClient.Info())
 
-	var globalListener, meshListener net.Listener
+	err = run(dockerClient, address, meshAddress, meshNetworkName, nameserver, noMulticastRoute)
+	if err != nil {
+		Log.Fatal(err)
+	}
+}
+
+func run(dockerClient *docker.Client, address, meshAddress, meshNetworkName, nameserver string, noMulticastRoute bool) error {
 	endChan := make(chan error, 1)
 	if address != "" {
 		globalListener, err := listenAndServe(dockerClient, address, nameserver, noMulticastRoute, endChan, "global", false)
 		if err != nil {
-			Log.Fatalf("unable to create driver: %s", err)
+			return err
 		}
+		defer os.Remove(address)
 		defer globalListener.Close()
 	}
 	if meshAddress != "" {
 		meshListener, err := listenAndServe(dockerClient, meshAddress, nameserver, noMulticastRoute, endChan, "local", true)
 		if err != nil {
-			Log.Fatalf("unable to create driver: %s", err)
+			return err
 		}
+		defer os.Remove(meshAddress)
 		defer meshListener.Close()
 	}
 
 	if meshNetworkName != "" {
-		createNetwork(dockerClient, meshNetworkName, meshAddress)
+		if err := createNetwork(dockerClient, meshNetworkName, meshAddress); err != nil {
+			return err
+		}
 	}
 
 	sigChan := make(chan os.Signal, 1)
@@ -96,16 +106,9 @@ func main() {
 	select {
 	case sig := <-sigChan:
 		Log.Debugf("Caught signal %s; shutting down", sig)
-		if err := dockerClient.Client.RemoveNetwork(meshNetworkName); err != nil {
-			Log.Fatal(err)
-		}
+		return nil
 	case err := <-endChan:
-		if err != nil {
-			Log.Errorf("Error from listener: %s", err)
-			globalListener.Close()
-			meshListener.Close()
-			os.Exit(1)
-		}
+		return err
 	}
 }
 
@@ -141,7 +144,7 @@ func listenAndServe(dockerClient *docker.Client, address, nameserver string, noM
 	return listener, nil
 }
 
-func createNetwork(dockerClient *docker.Client, networkName, address string) {
+func createNetwork(dockerClient *docker.Client, networkName, address string) error {
 	if _, err := dockerClient.Client.NetworkInfo(networkName); err == nil {
 		Log.Printf("Docker network '%s' already exists", networkName)
 	} else if _, ok := err.(*go_docker.NoSuchNetwork); ok {
@@ -157,7 +160,8 @@ func createNetwork(dockerClient *docker.Client, networkName, address string) {
 		}
 		_, err := dockerClient.Client.CreateNetwork(options)
 		if err != nil {
-			Log.Fatalf("Error creating network: %s", err)
+			return fmt.Errorf("Error creating network: %s", err)
 		}
 	}
+	return nil
 }

prog/weaveexec/Dockerfile

@@ -4,9 +4,6 @@ MAINTAINER Weaveworks Inc <help@weave.works>
 LABEL works.weave.role=system
 
 WORKDIR /home/weave
-VOLUME /w
-VOLUME /w-noop
-VOLUME /w-nomcast
 ENTRYPOINT ["/home/weave/sigproxy", "/home/weave/weave"]
 
 RUN apk add --update \

proxy/inspect_exec_interceptor.go

@@ -22,6 +22,10 @@ func (i *inspectExecInterceptor) InterceptResponse(r *http.Response) error {
 		return err
 	}
 
+	if _, ok := exec["Container"]; !ok {
+		return nil
+	}
+
 	container, err := exec.Object("Container")
 	if err != nil {
 		return err

proxy/proxy.go

@@ -132,7 +132,8 @@ func (proxy *Proxy) AttachExistingContainers() {
 			}
 			continue
 		}
-		if containerShouldAttach(container) && (container.State.Running || container.State.Paused) {
+		// The 'Running' check is purely an optimisation.
+		if containerShouldAttach(container) && container.State.Running {
 			proxy.attach(container, false)
 		}
 	}

router/fastdp.go

@@ -775,10 +775,10 @@ func (fwd *fastDatapathForwarder) Forward(key ForwardPacketKey) FlowOp {
 	defer fwd.lock.RUnlock()
 
 	if fwd.remoteAddr == nil {
-		// Returning nil would discard the packet, but also
-		// result in a flow rule, which we would have to
-		// invalidate when we learn the remote IP.  So for
-		// now, just prevent flows.
+		// Returning a DiscardingFlowOp would discard the
+		// packet, but also result in a flow rule, which we
+		// would have to invalidate when we learn the remote
+		// IP.  So for now, just prevent flows.
 		return vetoFlowCreationFlowOp{}
 	}
 
@@ -1086,6 +1086,10 @@ func (fastdp *FastDatapath) send(fops FlowOp, frame []byte, lock *fastDatapathLo
 		case vetoFlowCreationFlowOp:
 			createFlow = false
 		default:
+			if xfop.Discards() {
+				continue
+			}
+
 			// A foreign FlowOp (e.g. a sleeve forwarding
 			// FlowOp), so send the packet through the
 			// FlowOp interface, decoding the packet

router/flow.go

@@ -84,15 +84,12 @@ func (mfop *MultiFlowOp) Discards() bool {
 	return true
 }
 
+// Flatten out a FlowOp to eliminate any MultiFlowOps
 func FlattenFlowOp(fop FlowOp) []FlowOp {
 	return collectFlowOps(nil, fop)
 }
 
 func collectFlowOps(into []FlowOp, fop FlowOp) []FlowOp {
-	if fop == nil {
-		return into
-	}
-
 	if mfop, ok := fop.(*MultiFlowOp); ok {
 		for _, op := range mfop.ops {
 			into = collectFlowOps(into, op)

router/sleeve.go

@@ -917,7 +917,7 @@ func (fwd *sleeveForwarder) processSendError(err error) error {
 			return nil
 		}
 
-		fwd.mtuHighestGood = 8
+		fwd.mtuHighestGood = 552
 		fwd.mtuLowestBad = mtu + 1
 		fwd.mtuCandidate = mtu
 		fwd.mtuTestsSent = 0

site/_includes/header-navbaronly.html

@@ -2,7 +2,7 @@
     <div class="navbar navbar-inverse">
       <div class="container">
         <div class="navbar-header">
-          <a class="navbar-brand" href="http://weave.works/"><img src="http://weave.works/img/weave-logo.png"></a>
+          <a class="navbar-brand" href="http://weave.works/"><img src="http://www.weave.works/wp-content/uploads/2015/11/weave-logo-nav.png" class=" img-responsive" align="left"></a>
         </div>
         <div class="nav-wrapper">
           <ul class="nav navbar-nav navbar-right">

site/features.md

@@ -87,7 +87,12 @@ launched via the Docker
 [command-line interface](https://docs.docker.com/reference/commandline/cli/)
 or
 [remote API](https://docs.docker.com/reference/api/docker_remote_api/)
-are attached to the weave network before they begin execution.
+are attached to the weave network before they begin execution. To use
+the proxy, run
+
+    $ eval $(weave env)
+
+and then start containers as usual.
 
 Containers started in this way that subsequently restart, either by an
 explicit `docker restart` command or by Docker restart policy, are
@@ -100,6 +105,9 @@ named `weave` is created by `weave launch`, which you can use like this:
 
     $ docker run --net=weave -ti ubuntu
 
+> NB: The plugin is an *alternative* to the proxy, hence one should
+> *not* run `eval $(weave env)` beforehand.
+
 For more details see the [plugin documentation](plugin.html).
 
 ### <a name="addressing"></a>Address allocation

site/plugin.md

@@ -5,36 +5,53 @@ layout: default
 
 # Weave Plugin
 
-New in Docker version 1.9 is a plugin mechanism to add different
-network providers.
+Docker versions 1.9 and later have a plugin mechanism for adding
+different network providers. Weave installs itself as a network plugin
+when you start it with `weave launch`. To create a network that spans
+multiple docker hosts, the weave peers must be connected in the usual
+way, i.e. by specifying other hosts in `weave launch` or
+[`weave connect`](features.html#dynamic-topologies).
 
-To install, download the `weave` script as per the [readme][].
-
-The Weave Net plugin runs automatically when you `weave launch`.  You
-must still tell the weave peers to connect to one another, either via
-`weave launch` or `weave connect`.
-
-## Starting a container
+Subsequently you can start containers with, e.g.
 
     $ docker run --net=weave -ti ubuntu
 
-## Using WeaveDNS for Service Discovery
+on any of the hosts, and they can all communicate with each other.
 
-You need to pass the additional arguments `--dns` and `-dns-search`,
-for which we provide a helper in the weave script:
+> WARNING: It is inadvisable to attach containers to the weave network
+> using both the Weave Docker Networking Plugin and
+> [Weave Docker API Proxy](proxy.html) simultaneously. Such containers
+> will end up with two weave network interfaces and two IP addresses,
+> which is rarely desirable. To ensure the proxy is not being used,
+> *do not run `eval $(weave env)`, or `docker $(weave config) ...`*.
+
+In order to use Weave's [Service Discovery](weavedns.html), you
+need to pass the additional arguments `--dns` and `-dns-search`, for
+which we provide a helper in the weave script:
 
     $ docker run --net=weave -h foo.weave.local $(weave dns-args) -tdi ubuntu
-    $ docker run --net=weave -h bar.weave.local $(weave dns-args) -ti ubuntu
-    # ping foo
 
-For more details see the [Weave Service Discovery documentation][service-discovery].
+Here is a complete example of using the plugin for connectivity
+between two containers running on different hosts:
+
+    host1$ weave launch
+    host1$ docker run --net=weave -h foo.weave.local $(weave dns-args) -tdi ubuntu
+    
+    host2$ weave launch $HOST1
+    host2$ docker run --net=weave $(weave dns-args) -ti ubuntu
+    root@cb73d1a8aece:/# ping -c1 -q foo
+    PING foo (10.32.0.1) 56(84) bytes of data.
+    
+    --- foo ping statistics ---
+    1 packets transmitted, 1 received, 0% packet loss, time 0ms
+    rtt min/avg/max/mdev = 1.341/1.341/1.341/0.000 ms
 
 ## Under the hood
 
-The Weave Net plugin actually provides *two* network drivers to Docker
-- one named `weavemesh` that can operate without a [cluster
-store](#cluster-store) (like Docker's overlay driver) and one named
-`weave` that can only work with one.
+The Weave plugin actually provides *two* network drivers to Docker
+- one named `weavemesh` that can operate without a cluster store and
+one named `weave` that can only work with one (like Docker's overlay
+driver).
 
 ### `weavemesh` driver
 
@@ -54,7 +71,8 @@ networks.
 * Used with Docker's cluster store based IPAM
 
 There's no specific documentation from Docker on using a cluster
-store, but the first part of [Getting Started with Docker Multi-host Networking][docker-net]
+store, but the first part of
+[Getting Started with Docker Multi-host Networking](https://github.com/docker/docker/blob/master/docs/userguide/networking/get-started-overlay.md)
 should point the way.
 
 Note that in the case of multiple networks using the `weave` driver, all containers are
@@ -87,9 +105,14 @@ route, add the `--no-multicast-route` flag to `weave launch-plugin`.
 We start the plugin with a policy of `--restart=always`, so that it is
 there after a restart or reboot. If you remove this container
 (e.g. using `weave reset`) before removing all endpoints created using
-`--net=weave`, Docker can
-[hang](https://github.com/docker/libnetwork/issues/813).
+`--net=weave`, Docker may hang for a long time when it subsequently
+tries to talk to the plugin.
 
-[readme]: https://github.com/weaveworks/weave/blob/master/README.md#installation
-[service-discovery]: weavedns.html
-[docker-net]: https://github.com/docker/docker/blob/master/docs/userguide/networking/get-started-overlay.md
+Unfortunately, [Docker 1.9 may also try to talk to the plugin before it has even started it](https://github.com/docker/libnetwork/issues/813).
+If using `systemd`, we advise that you modify the Docker unit to
+remove the timeout on startup, to give Docker enough time to abandon
+its attempts.
+
+E.g. in the file `/lib/systemd/system/docker.service`, add under `[Service]`:
+
+    TimeoutStartSec=0

site/troubleshooting.md

@@ -124,6 +124,13 @@ Proxy](proxy.html#troubleshooting).
 
 ### <a name="weave-status-connections"></a>List connections
 
+Connections between weave peers carry control traffic over TCP and
+data traffic over UDP. For a connection to be fully established, the
+TCP connection and UDP data path must be able to transmit information
+in both directions. Weave routers check this regularly with
+heartbeats. Failed connections are automatically retried, with an
+exponential back-off.
+
 Detailed information on the local weave router's connections can be
 obtained with `weave status connections`:
 

site/weavedns.md

@@ -18,7 +18,6 @@ hosts.
 * [Adding and removing extra DNS entries](#add-remove)
 * [Resolve weaveDNS entries from host](#resolve-weavedns-entries-from-host)
 * [Hot-swapping service containers](#hot-swapping)
-* [Retaining DNS entries when containers stop](#retain-stopped)
 * [Configuring a custom TTL](#ttl)
 * [Configuring the domain search path](#domain-search-path)
 * [Using a different local domain](#local-domain)

test/105_frag_2_test.sh

@@ -8,7 +8,8 @@ C2=10.2.1.71
 # 700 = Linux's minimum PMTU (552) + weave framing overheads
 # (currently about 60, but we leave some margin). That way the PMTUs
 # seen by containers is still >= 552.
-drop_large_udp_packets="FORWARD -p udp --destination-port 6783 -m length '!' --length 0:700 -j DROP"
+drop_large_sleeve_packets="INPUT -p udp --destination-port 6783 -m length '!' --length 0:700 -j DROP"
+drop_large_fastdp_packets="INPUT -p udp --destination-port 6784 -m length '!' --length 0:700 -j DROP"
 
 start_suite "PMTU discovery and packet fragmentation"
 
@@ -18,7 +19,8 @@ weave_on $HOST2 launch
 start_container $HOST1 $C1/24 --name=c1
 start_container $HOST2 $C2/24 --name=c2
 
-run_on $HOST2 "sudo iptables -I $drop_large_udp_packets"
+run_on $HOST1 "sudo iptables -I $drop_large_sleeve_packets"
+run_on $HOST1 "sudo iptables -I $drop_large_fastdp_packets"
 assert_raises "weave_on $HOST2 connect $HOST1"
 sleep 35 # give weave time to discover the PMTU
 
@@ -28,6 +30,7 @@ sleep 35 # give weave time to discover the PMTU
 exec_on $HOST2 c2 $PING -s 10000 $C1 1>/dev/null 2>&1 || true
 assert_raises "exec_on $HOST2 c2 $PING -s 10000 $C1"
 
-run_on $HOST2 "sudo iptables -D $drop_large_udp_packets"
+run_on $HOST1 "sudo iptables -D $drop_large_fastdp_packets"
+run_on $HOST1 "sudo iptables -D $drop_large_sleeve_packets"
 
 end_suite

test/610_proxy_wait_for_weave_test.sh

@@ -6,15 +6,26 @@ entrypoint() {
   docker_on $HOST1 inspect --format="{{.Config.Entrypoint}}" "$@"
 }
 
+check_iface_ready() {
+    assert_raises "proxy docker_on $HOST1 run -e 'WEAVE_CIDR=$1' $BASE_IMAGE $CHECK_ETHWE_UP"
+    assert_raises "proxy docker_on $HOST1 run -e 'WEAVE_CIDR=$1' $BASE_IMAGE ip -4 -o addr show ethwe | grep -q '$1'"
+}
+
 start_suite "Proxy waits for weave to be ready before running container commands"
-weave_on $HOST1 launch-proxy
+
 BASE_IMAGE=busybox
 # Ensure the base image does not exist, so that it will be pulled
 ! docker_on $HOST1 inspect --format=" " $BASE_IMAGE >/dev/null 2>&1 || docker_on $HOST1 rmi $BASE_IMAGE
 
-assert_raises "proxy docker_on $HOST1 run --name c1 -e 'WEAVE_CIDR=10.2.1.1/24' $BASE_IMAGE $CHECK_ETHWE_UP"
+# check that interface is ready, with and without --no-multicast-route
+weave_on $HOST1 launch-proxy --no-multicast-route
+check_iface_ready 10.2.1.1/24
+weave_on $HOST1 stop-proxy
+weave_on $HOST1 launch-proxy
+check_iface_ready 10.2.1.1/24
 
 # Check committed containers only have one weavewait prepended
+proxy_start_container $HOST1 --name c1 -e 'WEAVE_CIDR=10.2.1.1/24'
 COMMITTED_IMAGE=$(proxy docker_on $HOST1 commit c1)
 assert_raises "proxy docker_on $HOST1 run --name c2 $COMMITTED_IMAGE"
 assert "entrypoint c2" "$(entrypoint $COMMITTED_IMAGE)"

test/810_plugin_router_restart.sh

@@ -0,0 +1,24 @@
+#! /bin/bash
+
+. ./config.sh
+
+C1_NAME=c1.weave.local
+
+start_suite "Recovery of container details on router restart (via 'local' plugin)"
+
+weave_on $HOST1 launch
+start_container_local_plugin $HOST1 --name=c1 --hostname=$C1_NAME
+c1ip=$(container_ip $HOST1 c1)
+
+weave_on $HOST1 stop-router
+weave_on $HOST1 launch-router
+
+assert "container_ip $HOST1 c1" "$c1ip"
+assert "weave_on $HOST1 dns-lookup $C1_NAME" "$c1ip"
+
+# check that c1 IP has been reclaimed and doesn't get assigned to a
+# fresh container
+start_container $HOST1 --name=c2
+assert_raises "container_ip $HOST1 c2 | grep -v $c1ip"
+
+end_suite

test/config.sh

@@ -176,7 +176,7 @@ proxy_start_container_with_dns() {
 rm_containers() {
     host=$1
     shift
-    [ $# -eq 0 ] || docker_on $host rm -f "$@" >/dev/null
+    [ $# -eq 0 ] || docker_on $host rm -f -v "$@" >/dev/null
 }
 
 container_ip() {

weave

@@ -344,6 +344,7 @@ IMAGE=$BASE_IMAGE:$IMAGE_VERSION
 BASE_PLUGIN_IMAGE=$DOCKERHUB_USER/plugin
 PLUGIN_IMAGE=$BASE_PLUGIN_IMAGE:$IMAGE_VERSION
 PLUGIN_CONTAINER_NAME=weaveplugin
+VOLUMES_CONTAINER_NAME=weavevolumes-$IMAGE_VERSION
 
 PROCFS=${PROCFS:-/proc}
 DOCKER_BRIDGE=${DOCKER_BRIDGE:-docker0}
@@ -535,7 +536,9 @@ create_bridge() {
         add_iptables_rule nat POSTROUTING -j WEAVE
     else
         if [ -n "$LAUNCHING_ROUTER" ] ; then
-            if [ "$BRIDGE_TYPE" = bridge -a -z "$WEAVE_NO_FASTDP" ] ; then
+            if [ "$BRIDGE_TYPE" = bridge -a -z "$WEAVE_NO_FASTDP" ] &&
+                util_op create-datapath $DATAPATH 2>/dev/null &&
+                util_op delete-datapath $DATAPATH 2>/dev/null ; then
                 cat <<EOF >&1
 WEAVE_NO_FASTDP is not set, but there is already a bridge present of
 the wrong type for fast datapath.  Please do 'weave reset' to remove
@@ -762,11 +765,11 @@ connect_container_to_bridge() {
     # veth pair in the 'up' or 'down' state.  Furthermore, moving one
     # end into a netns normally sets them to 'down'.  But, moving an
     # end into the default netns is a null operation, so they stay up,
-    # which leads to errors later on.  So to get a consistent result,
-    # we have to be explicit in setting them 'up' or 'down' as
-    # necessary.
+    # which leads to errors in 'set name' since that requires the
+    # interface to be down. So to get a consistent result, we set the
+    # interface 'down' to begin with.
     if ! netnsenter ip link set $GUEST_IFNAME down ||
-       ! netnsenter ip link set $GUEST_IFNAME name $1 up ||
+       ! netnsenter ip link set $GUEST_IFNAME name $1 ||
        ! ip link set $LOCAL_IFNAME up ||
        ! add_iface_$BRIDGE_TYPE $LOCAL_IFNAME ||
        ! configure_arp_cache $1 "netnsenter" ; then
@@ -833,8 +836,12 @@ setup_router_iface_bridged_fastdp() {
     setup_router_iface_fastdp "$@"
 }
 
+container_in_host_ns() {
+    [ -h "$PROCFS/$CONTAINER_PID/ns/net" -a -h "/proc/self/ns/net" -a "$(readlink $PROCFS/$CONTAINER_PID/ns/net)" = "$(readlink /proc/self/ns/net)" ]
+}
+
 attach() {
-    if [ -h "$PROCFS/$CONTAINER_PID/ns/net" -a -h "/proc/self/ns/net" -a "$(readlink $PROCFS/$CONTAINER_PID/ns/net)" = "$(readlink /proc/self/ns/net)" ] ; then
+    if container_in_host_ns ; then
         echo "Container is running in the host network namespace, and therefore cannot be" >&2
         echo "connected to weave. Perhaps the container was started with --net=host." >&2
         return 1
@@ -895,8 +902,8 @@ detach() {
     netnsenter ip link del $CONTAINER_IFNAME type veth
 }
 
-container_weave_addrs() {
-    netnsenter ip addr show dev $CONTAINER_IFNAME
+container_ip_addr_show() {
+    ! container_in_host_ns && netnsenter ip addr show
 }
 
 ######################################################################
@@ -1027,6 +1034,17 @@ wait_for_status() {
     done
 }
 
+# Parse the output of `ip addr show`, outputing weave addresses of kind $1
+addrs_from_ip_addr_show() {
+    while read first second rest ; do
+        if [ "$first" != "${first%:}" ] ; then
+            iname="${second%:}"
+        elif [ "$first" = "$1" -a \( "$iname" = "weave" -o "$iname" != "${iname#ethwe}" \) ] ; then
+            echo $second
+        fi
+    done
+}
+
 # Call $1 for all containers, passing container ID, all MACs and all IPs
 with_container_addresses() {
     COMMAND=$1
@@ -1035,12 +1053,14 @@ with_container_addresses() {
         if [ "$CONTAINER_ID" = "weave:expose" ] ; then
             ADDRS_CMD="ip addr show dev $BRIDGE"
         else
-            ADDRS_CMD="with_container_netns $CONTAINER_ID container_weave_addrs"
+            ADDRS_CMD="with_container_netns $CONTAINER_ID container_ip_addr_show"
         fi
         if CONTAINER_ADDRS=$($ADDRS_CMD 2>/dev/null) ; then
-            CONTAINER_MAC=$(echo "$CONTAINER_ADDRS" | grep -o 'link/ether .*' | cut -d ' ' -f 2)
-            CONTAINER_IPS=$(echo "$CONTAINER_ADDRS" | grep -o 'inet .*'       | cut -d ' ' -f 2)
-            $COMMAND "$CONTAINER_ID" "$CONTAINER_MAC" "$CONTAINER_IPS"
+            CONTAINER_MAC=$(echo "$CONTAINER_ADDRS" | addrs_from_ip_addr_show 'link/ether')
+            CONTAINER_IPS=$(echo "$CONTAINER_ADDRS" | addrs_from_ip_addr_show 'inet')
+            if [ -n "$CONTAINER_MAC$CONTAINER_IPS" ] ; then
+                $COMMAND "$CONTAINER_ID" "$CONTAINER_MAC" "$CONTAINER_IPS"
+            fi
         fi
     done
 }
@@ -1278,7 +1298,7 @@ collect_dns_add_remove_args() {
 # with_container_addresses.
 ipam_reclaim() {
     for CIDR in $3 ; do
-        # NB: CONTAINER_IP is the IP of the weave container; it is set by wait_for_status.
+        # NB: CONTAINER_IP is the IP of the weave container.
         http_call $CONTAINER_IP:$HTTP_PORT PUT /ip/$1/${CIDR%/*}?noErrorOnUnknown=true
     done
 }
@@ -1476,7 +1496,7 @@ proxy_args() {
 }
 
 proxy_addrs() {
-    if addr="$(http_call_unix $PROXY_CONTAINER_NAME status.sock GET /status 2>&1)" ; then
+    if addr="$(http_call_unix $PROXY_CONTAINER_NAME status.sock GET /status 2>/dev/null)" ; then
       echo "$addr" | sed "s/0.0.0.0/$PROXY_HOST/g"
 
     else
@@ -1650,7 +1670,7 @@ attach_router() {
     if [ -z "$NO_DNS_OPT" ] ; then
         # Tell the newly-started weaveDNS about existing weave IPs
         for CONTAINER in $(docker ps -q --no-trunc) ; do
-            if CONTAINER_IPS=$(with_container_netns $CONTAINER container_weave_addrs 2>&1 | sed -n -e 's/inet \([^/]*\)\/\(.*\)/\1/p') ; then
+            if CONTAINER_IPS=$(with_container_netns $CONTAINER container_ip_addr_show 2>/dev/null | addrs_from_ip_addr_show 'inet') && [ -n "$CONTAINER_IPS" ] ; then
                 with_container_fqdn $CONTAINER put_dns_fqdn $CONTAINER_IPS
             fi
         done
@@ -1669,8 +1689,14 @@ launch_proxy() {
     docker_client_args $DOCKER_CLIENT_ARGS
     proxy_args "$@"
     mkdir -p /var/run/weave
+    # Create a data-only container to mount the weavewait files from
+    if ! docker inspect -f ' ' $VOLUMES_CONTAINER_NAME > /dev/null 2>&1 ; then
+       docker create -v /w -v /w-noop -v /w-nomcast --name=$VOLUMES_CONTAINER_NAME \
+           --label=weavevolumes --entrypoint=/bin/false $EXEC_IMAGE >/dev/null
+    fi
     PROXY_CONTAINER=$(docker run --privileged -d --name=$PROXY_CONTAINER_NAME --net=host \
         $PROXY_VOLUMES \
+        --volumes-from $VOLUMES_CONTAINER_NAME \
         $(docker_sock_options) \
         -v /var/run/weave:/var/run/weave \
         -v /proc:/hostproc \
@@ -1715,8 +1741,12 @@ remove_plugin_network() {
         $PLUGIN_IMAGE --remove-network
 }
 
+warn_plugin_active() {
+    echo "WARNING: docker may hang in this state; re-launch weave and remove attached containers before shutting down" >&2
+}
+
 stop_plugin() {
-    remove_plugin_network
+    remove_plugin_network || warn_plugin_active
     stop $PLUGIN_CONTAINER_NAME "Plugin"
 }
 
@@ -2152,14 +2182,15 @@ EOF
         ;;
     reset)
         [ $# -eq 0 ] || usage
-        plugin_disabled || remove_plugin_network || \
-            echo "WARNING: docker may hang in this state; re-launch weave and remove attached containers before shutting down" >&2
+        plugin_disabled || remove_plugin_network || warn_plugin_active
         warn_if_stopping_proxy_in_env
         call_weave DELETE /peer >/dev/null 2>&1 || true
         for NAME in $PLUGIN_CONTAINER_NAME $CONTAINER_NAME $PROXY_CONTAINER_NAME ; do
             docker stop  $NAME >/dev/null 2>&1 || true
             docker rm -f $NAME >/dev/null 2>&1 || true
         done
+        VOLUME_CONTAINERS=$(docker ps -qa --filter label=weavevolumes)
+        [ -n "$VOLUME_CONTAINERS" ] && docker rm -v $VOLUME_CONTAINERS  >/dev/null 2>&1 || true
         conntrack -D -p udp --dport $PORT >/dev/null 2>&1 || true
         destroy_bridge
         for LOCAL_IFNAME in $(ip link show | grep v${CONTAINER_IFNAME}pl | cut -d ' ' -f 2 | tr -d ':') ; do