Merge pull request #1518 from weaveworks/401-reattach-weave2

Get proxy to re-attach weave router if it restarts

proxy/common.go

@@ -20,6 +20,7 @@ import (
 var (
 	containerIDRegexp   = regexp.MustCompile("^(/v[0-9\\.]*)?/containers/([^/]*)/.*")
 	weaveWaitEntrypoint = []string{"/w/w"}
+	weaveEntrypoint     = "/home/weave/weaver"
 )
 
 func callWeave(args ...string) ([]byte, []byte, error) {

proxy/proxy.go

@@ -305,6 +305,8 @@ func (proxy *Proxy) ContainerStarted(ident string) {
 	// If this was a container we modified the entrypoint for, attach it to the network
 	if containerShouldAttach(container) {
 		proxy.attach(container, true)
+	} else if containerIsWeaveRouter(container) {
+		proxy.attachRouter(container)
 	}
 	proxy.notifyWaiters(container.ID)
 }
@@ -313,6 +315,10 @@ func containerShouldAttach(container *docker.Container) bool {
 	return len(container.Config.Entrypoint) > 0 && container.Config.Entrypoint[0] == weaveWaitEntrypoint[0]
 }
 
+func containerIsWeaveRouter(container *docker.Container) bool {
+	return len(container.Config.Entrypoint) > 0 && container.Config.Entrypoint[0] == weaveEntrypoint
+}
+
 func (proxy *Proxy) createWait(r *http.Request, ident string) {
 	proxy.Lock()
 	ch := make(chan struct{})
@@ -386,6 +392,18 @@ func (proxy *Proxy) attach(container *docker.Container, orDie bool) error {
 	return nil
 }
 
+func (proxy *Proxy) attachRouter(container *docker.Container) error {
+	Log.Infof("Attaching weave router container: %s", container.ID)
+	args := []string{"attach-router"}
+	if _, stderr, err := callWeave(args...); err != nil {
+		Log.Warningf("Attaching container %s to weave network failed: %s", container.ID, string(stderr))
+		return errors.New(string(stderr))
+	} else if len(stderr) > 0 {
+		Log.Warningf("Attaching container %s to weave network: %s", container.ID, string(stderr))
+	}
+	return nil
+}
+
 func (proxy *Proxy) weaveCIDRs(networkMode string, env []string) ([]string, error) {
 	if networkMode == "host" || strings.HasPrefix(networkMode, "container:") {
 		return nil, fmt.Errorf("the container was created with the '--net=%s'", networkMode)

test/640_proxy_restart_reattaches_test.sh

@@ -13,28 +13,32 @@ check_attached() {
 
 start_suite "Proxy restart reattaches networking to containers"
 
-weave_on $HOST1 launch
+WEAVE_DOCKER_ARGS=--restart=always WEAVEPROXY_DOCKER_ARGS=--restart=always weave_on $HOST1 launch
 proxy_start_container          $HOST1 -e WEAVE_CIDR=$C2/24 -di --name=c2 --restart=always -h $NAME
 proxy_start_container_with_dns $HOST1 -e WEAVE_CIDR=$C1/24 -di --name=c1 --restart=always
 
 proxy docker_on $HOST1 restart -t=1 c2
 check_attached
 
+# Restart weave router
+docker_on $HOST1 restart weave
+sleep 1
+check_attached
+
 # Kill outside of Docker so Docker will restart it
 run_on $HOST1 sudo kill -KILL $(docker_on $HOST1 inspect --format='{{.State.Pid}}' c2)
 sleep 1
 check_attached
 
+# Restart docker itself, using different commands for systemd- and upstart-managed.
+run_on $HOST1 sh -c "command -v systemctl >/dev/null && sudo systemctl restart docker || sudo service docker restart"
+sleep 5
+check_attached
+
 # Restarting proxy shouldn't kill unattachable containers
 proxy_start_container $HOST1 -di --name=c3 --restart=always # Use ipam, so it won't be attachable w/o weave
 weave_on $HOST1 stop
 weave_on $HOST1 launch-proxy
 assert_raises "proxy exec_on $HOST1 c3 $CHECK_ETHWE_UP"
 
-# Restart docker itself, using different commands for systemd- and upstart-managed.
-run_on $HOST1 sh -c "command -v systemctl >/dev/null && sudo systemctl restart docker || sudo service docker restart"
-sleep 1
-weave_on $HOST1 launch
-check_attached
-
 end_suite

weave

@@ -673,8 +673,10 @@ router_opts_bridge() {
 setup_router_iface_fastdp() {
     if [ -n "$WEAVE_PASSWORD" ] ; then
         # See router_opts_fastdp
-        # In case there is a lingering weave2 netdev
-        ip link del $ROUTER_HOSTNETNS_IFNAME >/dev/null 2>&1 || true
+        # No-op if already attached
+        if ip link show $LOCAL_IFNAME >/dev/null 2>&1 ; then
+            return 0
+        fi
         connect_container_to_bridge $ROUTER_HOSTNETNS_IFNAME &&
             ip link set $ROUTER_HOSTNETNS_IFNAME up
     fi
@@ -685,6 +687,10 @@ setup_router_iface_bridge() {
         echo "Perhaps you are running the docker daemon with container networking disabled (-b=none)." >&2
         return 1
     fi
+    # No-op if already attached
+    if netnsenter ip link show $CONTAINER_IFNAME >/dev/null 2>&1 ; then
+        return 0
+    fi
     connect_container_to_bridge $CONTAINER_IFNAME &&
         netnsenter ethtool -K eth0 tx off >/dev/null &&
         netnsenter ip link set $CONTAINER_IFNAME up
@@ -1451,6 +1457,8 @@ launch_router() {
 
     if [ "$BRIDGE_TYPE" = fastdp ] ; then
         NETHOST_OPT="--net=host"
+        # In case there is a lingering weave2 netdev
+        ip link del $ROUTER_HOSTNETNS_IFNAME >/dev/null 2>&1 || true
     fi
 
     # Set WEAVE_DOCKER_ARGS in the environment in order to supply
@@ -1469,7 +1477,19 @@ launch_router() {
         --dns-effective-listen-address $DOCKER_BRIDGE_IP \
         ${NETHOST_OPT:+$DNS_ROUTER_OPTS} $NO_DNS_OPT \
         --docker-api "unix:///var/run/docker.sock" "$@")
-    with_container_netns_or_die $ROUTER_CONTAINER setup_router_iface_$BRIDGE_TYPE
+    with_container_netns_or_die $CONTAINER_NAME setup_router_iface_$BRIDGE_TYPE
+    attach_router
+}
+
+# Recreate the parameter values that are set when the router is first launched
+fetch_router_args() {
+    CONTAINER_ARGS=$(docker inspect -f '{{.Args}}' $CONTAINER_NAME)
+    IPRANGE=$(echo $CONTAINER_ARGS | grep -o -e '-ipalloc-range [^ ]*') || true
+    NO_DNS_OPT=$(echo $CONTAINER_ARGS | grep -o -e '--no-dns') || true
+    WEAVE_PASSWORD=$(docker inspect -f '{{.Config.Env}}' $CONTAINER_NAME | sed -n -e 's/.*WEAVE_PASSWORD=\([^] ]*\).*/\1/p')
+}
+
+attach_router() {
     err_msg=$(death_msg $CONTAINER_NAME)
     container_ip $CONTAINER_NAME "$err_msg" "$err_msg" || return 1
     wait_for_status $CONTAINER_NAME http_call $CONTAINER_IP:$HTTP_PORT
@@ -1653,6 +1673,14 @@ case "$COMMAND" in
         launch_router "$@"
         echo $ROUTER_CONTAINER
         ;;
+    attach-router)
+        check_running $CONTAINER_NAME
+        enforce_docker_bridge_addr_assign_type
+        create_bridge
+        fetch_router_args
+        with_container_netns $CONTAINER_NAME setup_router_iface_$BRIDGE_TYPE
+        attach_router
+        ;;
     launch-proxy)
         deprecation_warnings "$@"
         check_not_running $PROXY_CONTAINER_NAME $BASE_EXEC_IMAGE