Add release v1.5.2

Check validity of IP subnets

.gitmodules

@@ -70,3 +70,6 @@
 [submodule "vendor/github.com/coreos/go-iptables"]
 	path = vendor/github.com/coreos/go-iptables
 	url = https://github.com/coreos/go-iptables
+[submodule "vendor/github.com/j-keck/arping"]
+	path = vendor/github.com/j-keck/arping
+	url = https://github.com/j-keck/arping

CHANGELOG.md

@@ -1,3 +1,17 @@
+## Release 1.5.2
+
+Bug fixes and minor improvements
+
+* Weave Proxy did not flush the initial http header in the Docker event stream, which could cause Docker Swarm to show all nodes pending. #2306/#2311
+* When using the CNI plugin, if a container was removed and quickly replaced by another using the same IP address, other containers might be unable to contact it. Send an address resolution protocol message to update them. #2313
+* Avoid Docker hanging for 1 minute in `weave launch` if the plugin had not shut down cleanly #2286/#2292
+* Print an error message when Weave bridge mode is changed without `weave reset` #2304
+* Eliminate spurious warning message from IP allocator on plugin shutdown #2300/#2319
+* Display error message when address requested in a subnet that is too small (/31 or /32) #2282/#2321
+* Add short wait after `weave reset` to allow updates to reach peers #2280
+* Weave was occasionally unable to claim existing IP address immediately after launch #2275/#2281
+* Refactor some integration tests to run faster and more reliably #2291
+
 ## Release 1.5.1
 
 Bug fixes and minor improvements

circle.yml

@@ -68,11 +68,11 @@ deployment:
       - docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS
       - cd $SRCDIR; make clean-bin
       - cd $SRCDIR; make publish
-      - go get github.com/weaveworks/wordepress && cd /home/ubuntu/src/github.com/weaveworks/wordepress && git checkout v1.0.0 && cd cmd/wordepress && go get
-      - cd $SRCDIR; bin/publish-site "$WP_DEV_URL" "$WP_DEV_USER" "$WP_DEV_PASSWORD"
-  issues:
-    branch: /.*/
-    owner: weaveworks
-    commands:
-      - go get github.com/weaveworks/wordepress && cd /home/ubuntu/src/github.com/weaveworks/wordepress && git checkout v1.0.0 && cd cmd/wordepress && go get
-      - cd $SRCDIR; bin/publish-site "$WP_DEV_URL" "$WP_DEV_USER" "$WP_DEV_PASSWORD"
+#      - go get github.com/weaveworks/wordepress && cd /home/ubuntu/src/github.com/weaveworks/wordepress && git checkout v1.0.0 && cd cmd/wordepress && go get
+#      - cd $SRCDIR; bin/publish-site "$WP_DEV_URL" "$WP_DEV_USER" "$WP_DEV_PASSWORD"
+#  issues:
+#    branch: /.*/
+#    owner: weaveworks
+#    commands:
+#      - go get github.com/weaveworks/wordepress && cd /home/ubuntu/src/github.com/weaveworks/wordepress && git checkout v1.0.0 && cd cmd/wordepress && go get
+#      - cd $SRCDIR; bin/publish-site "$WP_DEV_URL" "$WP_DEV_USER" "$WP_DEV_PASSWORD"

docs/release-process.md

@@ -6,8 +6,6 @@
 * Create a [github token for
   github-release](https://help.github.com/articles/creating-an-access-token-for-command-line-use/);
 set and export `$GITHUB_TOKEN` with this value
-* Update all dependencies with `make update`
-* Ensure you are still on the right branch, since Go may switch to `master` as part of `make update`
 
 ## Release Types
 
@@ -133,12 +131,13 @@ Finally, for **Mainline** releases only:
 ### Finish up
 
 * If not on master, merge branch into master and push to GitHub.
-* Merge the branch you are on into `latest_release_doc_updates` and push to GitHub.
 * Close the [milestone](https://github.com/weaveworks/weave/milestones) in GitHub and create the next milestone
 * Update the `#weavenetwork` topic heading on freenode (requires 'chanops' permission)
 * For a mainline release vX.Y.0, create a release branch X.Y from the
   tag and push to GitHub - this will result in X.Y.0 site docs being
   published to https://www.weave.works
+* Add the new version of `weave-net` to the checkpoint system at
+  https://checkpoint-api.weave.works/admin
 
 ## Troubleshooting
 

ipam/allocator.go

@@ -111,6 +111,20 @@ func NewAllocator(config Config) *Allocator {
 	}
 }
 
+func ParseCIDRSubnet(cidrStr string) (cidr address.CIDR, err error) {
+	cidr, err = address.ParseCIDR(cidrStr)
+	if err != nil {
+		return
+	}
+	if !cidr.IsSubnet() {
+		err = fmt.Errorf("invalid subnet - bits after network prefix are not all zero: %s", cidrStr)
+	}
+	if cidr.Size() < MinSubnetSize {
+		err = fmt.Errorf("invalid subnet - smaller than minimum size %d: %s", MinSubnetSize, cidrStr)
+	}
+	return
+}
+
 // Start runs the allocator goroutine
 func (alloc *Allocator) Start() {
 	loadedPersistedData := alloc.loadPersistedData()
@@ -419,7 +433,6 @@ func (alloc *Allocator) Shutdown() {
 			alloc.persistRing()
 			alloc.space.Clear()
 			alloc.gossip.GossipBroadcast(alloc.Gossip())
-			time.Sleep(100 * time.Millisecond)
 		}
 		doneChan <- struct{}{}
 	}
@@ -512,9 +525,11 @@ func (alloc *Allocator) OnGossipUnicast(sender mesh.PeerName, msg []byte) error
 	alloc.actionChan <- func() {
 		switch msg[0] {
 		case msgSpaceRequest:
-			// some other peer asked us for space
+			alloc.debugln("Peer", sender, "asked me for space")
 			r, err := decodeRange(msg[1:])
-			if err == nil {
+			// If we don't have a ring, just ignore a request for space.
+			// They'll probably ask again later.
+			if err == nil && !alloc.ring.Empty() {
 				alloc.donateSpace(r, sender)
 			}
 			resultChan <- err
@@ -823,7 +838,6 @@ func (alloc *Allocator) donateSpace(r address.Range, to mesh.PeerName) {
 	// more.
 	defer alloc.sendRingUpdate(to)
 
-	alloc.debugln("Peer", to, "asked me for space")
 	chunk, ok := alloc.space.Donate(r)
 	if !ok {
 		free := alloc.space.NumFreeAddressesInRange(r)

ipam/http.go

@@ -17,15 +17,17 @@ func badRequest(w http.ResponseWriter, err error) {
 }
 
 func parseCIDR(w http.ResponseWriter, cidrStr string, net bool) (address.CIDR, bool) {
-	cidr, err := address.ParseCIDR(cidrStr)
+	var cidr address.CIDR
+	var err error
+	if net {
+		cidr, err = ParseCIDRSubnet(cidrStr)
+	} else {
+		cidr, err = address.ParseCIDR(cidrStr)
+	}
 	if err != nil {
 		badRequest(w, err)
 		return address.CIDR{}, false
 	}
-	if net && !cidr.IsSubnet() {
-		badRequest(w, fmt.Errorf("Invalid subnet %s - bits after network prefix are not all zero", cidrStr))
-		return address.CIDR{}, false
-	}
 	return cidr, true
 }
 
@@ -38,7 +40,7 @@ func writeAddresses(w http.ResponseWriter, cidrs []address.CIDR) {
 	}
 }
 
-func hasBeenCanceled(dockerCli *docker.Client, closedChan <-chan bool, ident string, checkAlive bool) func() bool {
+func hasBeenCancelled(dockerCli *docker.Client, closedChan <-chan bool, ident string, checkAlive bool) func() bool {
 	return func() bool {
 		select {
 		case <-closedChan:
@@ -62,7 +64,7 @@ func cancellationErr(w http.ResponseWriter, err error) bool {
 
 func (alloc *Allocator) handleHTTPAllocate(dockerCli *docker.Client, w http.ResponseWriter, ident string, checkAlive bool, subnet address.CIDR) {
 	addr, err := alloc.Allocate(ident, subnet, checkAlive,
-		hasBeenCanceled(dockerCli, w.(http.CloseNotifier).CloseNotify(), ident, checkAlive))
+		hasBeenCancelled(dockerCli, w.(http.CloseNotifier).CloseNotify(), ident, checkAlive))
 	if err != nil {
 		if !cancellationErr(w, err) {
 			badRequest(w, err)
@@ -74,7 +76,7 @@ func (alloc *Allocator) handleHTTPAllocate(dockerCli *docker.Client, w http.Resp
 
 func (alloc *Allocator) handleHTTPClaim(dockerCli *docker.Client, w http.ResponseWriter, ident string, cidr address.CIDR, checkAlive, noErrorOnUnknown bool) {
 	err := alloc.Claim(ident, cidr, checkAlive, noErrorOnUnknown,
-		hasBeenCanceled(dockerCli, w.(http.CloseNotifier).CloseNotify(), ident, checkAlive))
+		hasBeenCancelled(dockerCli, w.(http.CloseNotifier).CloseNotify(), ident, checkAlive))
 	if err != nil {
 		if !cancellationErr(w, err) {
 			badRequest(w, fmt.Errorf("Unable to claim: %s", err))

plugin/ipam/driver.go

@@ -54,6 +54,21 @@ func (i *Ipam) ReleasePool(poolID string) error {
 	return nil
 }
 
+func splitPoolID(poolID string) (subnet, iprange *net.IPNet, err error) {
+	parts := strings.Split(poolID, "-")
+	if len(parts) != 3 || parts[0] != "weave" {
+		err = fmt.Errorf("Unrecognized pool ID: %s", poolID)
+		return
+	}
+	if _, subnet, err = net.ParseCIDR(parts[1]); err != nil {
+		return
+	}
+	if _, iprange, err = net.ParseCIDR(parts[2]); err != nil {
+		return
+	}
+	return
+}
+
 func (i *Ipam) RequestAddress(poolID string, address net.IP, options map[string]string) (ip *net.IPNet, _ map[string]string, err error) {
 	i.logReq("RequestAddress", poolID, address, options)
 	defer func() { i.logRes("RequestAddress", err, ip) }()
@@ -62,13 +77,8 @@ func (i *Ipam) RequestAddress(poolID string, address net.IP, options map[string]
 		ip, err = i.weave.AllocateIP("_")
 		return
 	}
-	parts := strings.Split(poolID, "-")
-	if len(parts) != 3 || parts[0] != "weave" {
-		err = fmt.Errorf("Unrecognized pool ID: %s", poolID)
-		return
-	}
-	var subnet, iprange *net.IPNet
-	if _, subnet, err = net.ParseCIDR(parts[1]); err != nil {
+	subnet, iprange, err := splitPoolID(poolID)
+	if err != nil {
 		return
 	}
 	if address != nil { // try to claim specific address requested
@@ -77,9 +87,6 @@ func (i *Ipam) RequestAddress(poolID string, address net.IP, options map[string]
 			return
 		}
 	} else {
-		if _, iprange, err = net.ParseCIDR(parts[2]); err != nil {
-			return
-		}
 		// We are lying slightly to IPAM here: the range is not a subnet
 		if ip, err = i.weave.AllocateIPInSubnet("_", iprange); err != nil {
 			return
@@ -91,6 +98,11 @@ func (i *Ipam) RequestAddress(poolID string, address net.IP, options map[string]
 
 func (i *Ipam) ReleaseAddress(poolID string, address net.IP) error {
 	i.logReq("ReleaseAddress", poolID, address)
+	if subnet, _, err := splitPoolID(poolID); err != nil {
+		return err
+	} else if address.Equal(subnet.IP) { // is it the gateway address we faked earlier?
+		return nil
+	}
 	return i.weave.ReleaseIPsFor(address.String())
 }
 

plugin/net/cni.go

@@ -10,6 +10,7 @@ import (
 	"github.com/appc/cni/pkg/ipam"
 	"github.com/appc/cni/pkg/skel"
 	"github.com/appc/cni/pkg/types"
+	"github.com/j-keck/arping"
 	"github.com/vishvananda/netlink"
 	"github.com/vishvananda/netns"
 	weaveapi "github.com/weaveworks/weave/api"
@@ -151,6 +152,7 @@ func setupGuestIP4(origName, name string, ipnet net.IPNet, gw net.IP, routes []t
 	if err = netlink.AddrAdd(guest, &netlink.Addr{IPNet: &ipnet}); err != nil {
 		return fmt.Errorf("failed to add IP address to %q: %v", name, err)
 	}
+	arping.GratuitousArpOverIfaceByName(ipnet.IP, name)
 	for _, r := range routes {
 		if r.GW != nil {
 			err = addRoute(guest, netlink.SCOPE_UNIVERSE, &r.Dst, r.GW)

prog/weaver/main.go

@@ -413,24 +413,13 @@ func createOverlay(datapathName string, ifaceName string, host string, port int,
 	return overlay, bridge
 }
 
-func parseAndCheckCIDR(cidrStr string) address.CIDR {
-	cidr, err := address.ParseCIDR(cidrStr)
-	checkFatal(err)
-
-	if !cidr.IsSubnet() {
-		Log.Fatalf("Invalid allocation range %s - bits after network prefix are not all zero", cidrStr)
-	}
-	if cidr.Size() < ipam.MinSubnetSize {
-		Log.Fatalf("Allocation range smaller than minimum size %d: %s", ipam.MinSubnetSize, cidrStr)
-	}
-	return cidr
-}
-
 func createAllocator(router *weave.NetworkRouter, config ipamConfig, db db.DB, isKnownPeer func(mesh.PeerName) bool) (*ipam.Allocator, address.CIDR) {
-	ipRange := parseAndCheckCIDR(config.IPRangeCIDR)
+	ipRange, err := ipam.ParseCIDRSubnet(config.IPRangeCIDR)
+	checkFatal(err)
 	defaultSubnet := ipRange
 	if config.IPSubnetCIDR != "" {
-		defaultSubnet = parseAndCheckCIDR(config.IPSubnetCIDR)
+		defaultSubnet, err = ipam.ParseCIDRSubnet(config.IPSubnetCIDR)
+		checkFatal(err)
 		if !ipRange.Range().Overlaps(defaultSubnet.Range()) {
 			Log.Fatalf("IP address allocation default subnet %s does not overlap with allocation range %s", defaultSubnet, ipRange)
 		}

proxy/proxy_intercept.go

@@ -133,6 +133,7 @@ func doChunkedResponse(w http.ResponseWriter, resp *http.Response, client *httpu
 	}
 
 	w.WriteHeader(resp.StatusCode)
+	wf.Flush()
 
 	up, remaining := client.Hijack()
 	defer up.Close()

site/using-weave.md

@@ -21,7 +21,7 @@ On `$HOST1` run:
 
     host1$ weave launch
     host1$ eval $(weave env)
-    host1$ docker run --name a1 -ti ubuntu
+    host1$ docker run --name a1 -ti ubuntu:trusty
 
 Where, 
 
@@ -49,7 +49,7 @@ To launch Weave Net on an additional host and create a peer connection, run the
 
     host2$ weave launch $HOST1
     host2$ eval $(weave env)
-    host2$ docker run --name a2 -ti ubuntu
+    host2$ docker run --name a2 -ti ubuntu:trusty
 
 As noted above, the same steps are repeated for `$HOST2`. The only difference, besides the application container’s name, is that `$HOST2` is told to peer with Weave Net on `$HOST1` during launch. 
 

test/140_weave_local_test.sh

@@ -16,10 +16,10 @@ weave_local_on $HOST1 launch --ipalloc-range 10.2.5.0/24
 assert_raises "docker_on $HOST1 ps | grep weave"
 
 weave_local_on $HOST1 run 10.2.6.5/24 -ti --name=c1 $SMALL_IMAGE /bin/sh
-assert_raises "exec_on $HOST1 c1 $CHECK_ETHWE_UP"
+wait_for_attached $HOST1 c1
 
 weave_local_on $HOST1 run             -ti --name=c2 $SMALL_IMAGE /bin/sh
-assert_raises "exec_on $HOST1 c2 $CHECK_ETHWE_UP"
+wait_for_attached $HOST1 c2
 
 assert "weave_local_on $HOST1 ps | wc -l" 3
 

test/310_docker_restart_test.sh

@@ -2,24 +2,6 @@
 
 . ./config.sh
 
-check_attached() {
-    for c in $@; do
-        assert_raises "exec_on $HOST1 $c $CHECK_ETHWE_UP"
-    done
-}
-
-wait_for_proxy() {
-    for i in $(seq 1 120); do
-        echo "Waiting for proxy to start"
-        if proxy docker_on $1 info > /dev/null 2>&1 ; then
-            return
-        fi
-        sleep 1
-    done
-    echo "Timed out waiting for proxy to start" >&2
-    exit 1
-}
-
 start_suite "Containers get same IP address on restart"
 
 weave_on $HOST1 launch-router
@@ -31,15 +13,14 @@ start_container $HOST1 --name=c1
 proxy docker_on $HOST1 run -di --name=c2 --restart=always -dt --entrypoint="/home/weave/sigproxy" weaveworks/weaveexec sleep 600
 C2=$(container_ip $HOST1 c2)
 assert_raises "[ -n $C2 ]"
-check_attached c2
+wait_for_attached $HOST1 c2
 
 docker_on $HOST1 rm -f c1
 
 # Restart docker daemon, using different commands for systemd- and upstart-managed.
 run_on $HOST1 sh -c "command -v systemctl >/dev/null && sudo systemctl restart docker || sudo service docker restart"
 wait_for_proxy $HOST1
-sleep 3 # allow for re-tries of attach
-check_attached c2
+wait_for_attached $HOST1 c2
 # Check same IP address was retained
 assert "container_ip $HOST1 c2" "$C2"
 

test/320_claim_3_test.sh

@@ -2,7 +2,9 @@
 
 . ./config.sh
 
-UNIVERSE=10.32.0.0/12
+UNIVERSE=10.2.0.0/16
+C1=10.2.128.1
+C4=10.2.128.4
 
 delete_persistence() {
     for host in "$@" ; do
@@ -13,12 +15,11 @@ delete_persistence() {
 
 start_suite "Claiming addresses"
 
-weave_on $HOST1 launch-router
-weave_on $HOST2 launch-router $HOST1
+weave_on $HOST1 launch-router --ipalloc-range $UNIVERSE $HOST2
+weave_on $HOST2 launch-router --ipalloc-range $UNIVERSE $HOST1
 
-start_container $HOST1 --name=c1
-start_container $HOST2 --name=c2
-C1=$(container_ip $HOST1 c1)
+start_container $HOST1 $C1/12 --name=c1
+start_container $HOST2        --name=c2
 assert_raises "exec_on $HOST2 c2 $PING $C1"
 
 stop_weave_on $HOST1
@@ -28,8 +29,8 @@ stop_weave_on $HOST2
 delete_persistence $HOST1 $HOST2
 
 # Start hosts in reverse order so c1's address has to be claimed from host2
-weave_on $HOST2 launch-router
-weave_on $HOST1 launch-router $HOST2
+weave_on $HOST2 launch-router --ipalloc-range $UNIVERSE
+weave_on $HOST1 launch-router --ipalloc-range $UNIVERSE $HOST2
 
 # Start another container on host2, so if it hasn't relinquished c1's
 # address it would give that out as the first available.
@@ -47,29 +48,31 @@ delete_persistence $HOST1 $HOST2
 
 # Now make host1 attempt to claim from host2, when host2 is stopped
 # the point being to check whether host1 will hang trying to talk to host2
-weave_on $HOST2 launch-router
+weave_on $HOST2 launch-router --ipalloc-range $UNIVERSE
 # Introduce host3 to remember the IPAM CRDT when we stop host2
-weave_on $HOST3 launch-router $HOST2
+weave_on $HOST3 launch-router --ipalloc-range $UNIVERSE $HOST2
 weave_on $HOST3 prime
 stop_weave_on $HOST2
-weave_on $HOST1 launch-router $HOST3
+weave_on $HOST1 launch-router --ipalloc-range $UNIVERSE $HOST3
 
 stop_weave_on $HOST1
-stop_weave_on $HOST2
 stop_weave_on $HOST3
-delete_persistence $HOST1 $HOST2 $HOST3
 
+# Delete persistence data on host1, so that host1 would try to establish a ring.
+delete_persistence $HOST1
 weave_on $HOST1 launch --ipalloc-range $UNIVERSE $HOST2
+
 # Start another container on host1. The starting should block, because host1 is
 # not able to establish the ring due to host2 being offline.
-CMD="proxy_start_container $HOST1 --name c4 -e WEAVE_CIDR=10.32.0.4/12"
+CMD="proxy_start_container $HOST1 --name c4 -e WEAVE_CIDR=$C4/12"
 assert_raises "timeout 5 cat <( $CMD )" 124
+
 # However, allocation for an external subnet should not block.
-assert_raises "proxy_start_container $HOST1 -e WEAVE_CIDR=10.48.0.1/12"
+assert_raises "proxy_start_container $HOST1 -e WEAVE_CIDR=10.3.0.1/12"
 
 # Launch host2, so that host1 can establish the ring.
 weave_on $HOST2 launch --ipalloc-range $UNIVERSE $HOST1
-weave_on $HOST1 prime
-assert_raises "[ $(container_ip $HOST1 c4) == 10.32.0.4 ]"
+wait_for_attached $HOST1 c4
+assert_raises "[ $(container_ip $HOST1 c4) == $C4 ]"
 
 end_suite

test/500_weave_multi_cidr_test.sh

@@ -122,4 +122,8 @@ assert_equal "$IPS"                                                      10.2.3.
 CID2=$(start_container $HOST1                                        net:10.2.3.0/24)
 assert_container_cidrs $HOST1 $CID2                                      10.2.3.1/24
 
+# Error conditions: host address not network, subnet too small
+assert_raises "start_container $HOST1                                net:10.2.3.2/30" 1
+assert_raises "start_container $HOST1                                net:10.2.3.2/31" 1
+
 end_suite

test/640_proxy_restart_reattaches_test.sh

@@ -4,23 +4,6 @@
 
 NAME=seetwo.weave.local
 
-check_attached() {
-    assert_raises "exec_on $HOST1 c2 $CHECK_ETHWE_UP"
-    assert_dns_record $HOST1 c1 $NAME $C2
-}
-
-wait_for_proxy() {
-    for i in $(seq 1 120); do
-        echo "Waiting for proxy to start"
-        if proxy docker_on $1 info > /dev/null 2>&1 ; then
-            return
-        fi
-        sleep 1
-    done
-    echo "Timed out waiting for proxy to start" >&2
-    exit 1
-}
-
 N=50
 # Create and remove a lot of containers in a small subnet; the failure
 # mode is that this takes a long time as it has to wait for the old
@@ -39,39 +22,41 @@ proxy_start_container_with_dns $HOST1 -di --name=c1 --restart=always
 C2=$(container_ip $HOST1 c2)
 
 proxy docker_on $HOST1 restart -t=1 c2
-check_attached
+wait_for_attached $HOST1 c2
+assert_dns_record $HOST1 c1 $NAME $C2
 
 # Restart weave router
 docker_on $HOST1 restart weave
+wait_for_attached $HOST1 c2
 sleep 1
-check_attached
+assert_dns_record $HOST1 c1 $NAME $C2
 
 # Kill outside of Docker so Docker will restart it
 run_on $HOST1 sudo kill -KILL $(docker_on $HOST1 inspect --format='{{.State.Pid}}' c2)
+wait_for_attached $HOST1 c2
 sleep 1
-check_attached
+assert_dns_record $HOST1 c1 $NAME $C2
 
 run_on $HOST1 "sudo service docker restart"
 wait_for_proxy $HOST1
-sleep 5 # allow for re-tries of attach
+wait_for_attached $HOST1 c2
 # Re-fetch the IP since it is not retained on docker restart
 C2=$(container_ip $HOST1 c2) || (echo "container c2 has no IP address" 2>&1; exit 1)
-check_attached
+assert_dns_record $HOST1 c1 $NAME $C2
 
 assert_raises "timeout 90 cat <( run_many )"
 
 # Start a container that needs IPAM, then restart it when the router is stopped
 proxy_start_container $HOST1 -di --name=c3 --restart=always
-assert_raises "exec_on $HOST1 c3 $CHECK_ETHWE_UP"
+wait_for_attached $HOST1 c3
 weave_on $HOST1 stop-router
 docker_on $HOST1 restart c3
 weave_on $HOST1 launch-router
-sleep 5
-assert_raises "exec_on $HOST1 c3 $CHECK_ETHWE_UP"
+wait_for_attached $HOST1 c3
 
 # Restarting proxy shouldn't kill unattachable containers
 weave_on $HOST1 stop
 weave_on $HOST1 launch-proxy
-assert_raises "exec_on $HOST1 c3 $CHECK_ETHWE_UP"
+wait_for_attached $HOST1 c3
 
 end_suite

test/830_cni_plugin_test.sh

@@ -6,7 +6,8 @@ start_suite "Test CNI plugin"
 
 cni_connect() {
     pid=$(container_pid $1 $2)
-    run_on $1 CNI_VERSION=1 CNI_COMMAND=ADD CNI_CONTAINERID=c1 CNI_IFNAME=eth0 \
+    id=$(docker_on $1 inspect -f '{{.Id}}' $2)
+    run_on $1 CNI_VERSION=1 CNI_COMMAND=ADD CNI_CONTAINERID=$id CNI_IFNAME=eth0 \
     CNI_NETNS=/proc/$pid/ns/net CNI_PATH=/opt/cni/bin /opt/cni/bin/weave-net 
 }
 
@@ -42,4 +43,16 @@ C2IP=$(container_ip $HOST1 c2)
 assert_raises "exec_on $HOST1 c1 $PING $C2IP"
 assert_raises "exec_on $HOST1 c2 $PING $C1IP"
 
+# Now remove and start a new container to see if IP address re-use breaks things
+docker_on $HOST1 rm -f c2
+sleep 1
+
+docker_on $HOST1 run --net=none --name=c3 -dt $SMALL_IMAGE /bin/sh
+
+cni_connect $HOST1 c3 <<EOF
+{ "name": "weave", "type": "weave-net" }
+EOF
+
+assert_raises "exec_on $HOST1 c1 $PING $C2IP"
+
 end_suite

test/config.sh

@@ -171,6 +171,18 @@ proxy_start_container_with_dns() {
     proxy docker_on $host run "$@" -dt $DNS_IMAGE /bin/sh
 }
 
+wait_for_proxy() {
+    for i in $(seq 1 120); do
+        echo "Waiting for proxy to start"
+        if proxy docker_on $1 info > /dev/null 2>&1 ; then
+            return
+        fi
+        sleep 1
+    done
+    echo "Timed out waiting for proxy to start" >&2
+    exit 1
+}
+
 rm_containers() {
     host=$1
     shift
@@ -185,6 +197,20 @@ container_pid() {
     docker_on $1 inspect -f '{{.State.Pid}}' $2
 }
 
+wait_for_attached() {
+    host=$1
+    container=$2
+    for i in $(seq 1 10); do
+        echo "Waiting for $container on $host to be attached"
+        if exec_on $host $container $CHECK_ETHWE_UP > /dev/null 2>&1 ; then
+            return
+        fi
+        sleep 1
+    done
+    echo "Timed out waiting for $container on $host to be attached" >&2
+    exit 1
+}
+
 # assert_dns_record <host> <container> <name> [<ip> ...]
 assert_dns_record() {
     local host=$1

weave

@@ -621,7 +621,7 @@ create_bridge() {
             if [ "$BRIDGE_TYPE" = bridge -a -z "$WEAVE_NO_FASTDP" ] &&
                 util_op create-datapath $DATAPATH 2>/dev/null &&
                 util_op delete-datapath $DATAPATH 2>/dev/null ; then
-                cat <<EOF >&1
+                cat <<EOF >&2
 WEAVE_NO_FASTDP is not set, but there is already a bridge present of
 the wrong type for fast datapath.  Please do 'weave reset' to remove
 the bridge first.
@@ -629,7 +629,7 @@ EOF
                 return 1
             fi
             if [ "$BRIDGE_TYPE" != bridge -a -n "$WEAVE_NO_FASTDP" ] ; then
-                cat <<EOF >&1
+                cat <<EOF >&2
 WEAVE_NO_FASTDP is set, but there is already a weave fast datapath
 bridge present.  Please do 'weave reset' to remove the bridge first.
 EOF
@@ -1740,6 +1740,7 @@ launch_router() {
 
     # Create a data-only container for persistence data
     if ! docker inspect -f ' ' $DB_CONTAINER_NAME > /dev/null 2>&1 ; then
+       protect_against_docker_hang
        docker create -v /weavedb --name=$DB_CONTAINER_NAME \
            --label=weavevolumes $WEAVEDB_IMAGE >/dev/null
     fi
@@ -1803,6 +1804,7 @@ launch_proxy() {
     mkdir -p /var/run/weave
     # Create a data-only container to mount the weavewait files from
     if ! docker inspect -f ' ' $VOLUMES_CONTAINER_NAME > /dev/null 2>&1 ; then
+       protect_against_docker_hang
        docker create -v /w -v /w-noop -v /w-nomcast --name=$VOLUMES_CONTAINER_NAME \
            --label=weavevolumes --entrypoint=/bin/false $EXEC_IMAGE >/dev/null
     fi
@@ -1864,6 +1866,13 @@ stop_plugin() {
     stop $PLUGIN_CONTAINER_NAME "Plugin"
 }
 
+protect_against_docker_hang() {
+    # If the plugin is not running, remove its socket so Docker doesn't try to talk to it
+    if !check_running $PLUGIN_CONTAINER_NAME 2>/dev/null ; then
+        rm -f /run/docker/plugins/weave.sock /run/docker/plugins/weavemesh.sock
+    fi
+}
+
 ##########################################################################################
 
 [ $(id -u) = 0 ] || {
@@ -2308,6 +2317,7 @@ EOF
         plugin_disabled || util_op remove-plugin-network weave || warn_plugin_active
         warn_if_stopping_proxy_in_env
         call_weave DELETE /peer >/dev/null 2>&1 || true
+        fractional_sleep 0.5 # Allow some time for broadcast updates to go out
         for NAME in $PLUGIN_CONTAINER_NAME $CONTAINER_NAME $PROXY_CONTAINER_NAME ; do
             docker stop  $NAME >/dev/null 2>&1 || true
             docker rm -f $NAME >/dev/null 2>&1 || true