From a87cc5c3b15be2bd47c51c74fce9ec88fe0bfd6f Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Tue, 25 Aug 2015 11:09:16 +0100 Subject: [PATCH 01/11] Get the proxy to re-attach the weave router if it sees it restart --- proxy/proxy.go | 18 ++++++++++++++++ test/640_proxy_restart_reattaches_test.sh | 8 ++++++-- weave | 25 +++++++++++++++++++++++ 3 files changed, 49 insertions(+), 2 deletions(-) diff --git a/proxy/proxy.go b/proxy/proxy.go index 1ab70579..77cc7ca9 100644 --- a/proxy/proxy.go +++ b/proxy/proxy.go @@ -305,6 +305,8 @@ func (proxy *Proxy) ContainerStarted(ident string) { // If this was a container we modified the entrypoint for, attach it to the network if containerShouldAttach(container) { proxy.attach(container, true) + } else if containerIsWeaveRouter(container) { + proxy.attachRouter(container) } proxy.notifyWaiters(container.ID) } @@ -313,6 +315,10 @@ func containerShouldAttach(container *docker.Container) bool { return len(container.Config.Entrypoint) > 0 && container.Config.Entrypoint[0] == weaveWaitEntrypoint[0] } +func containerIsWeaveRouter(container *docker.Container) bool { + return strings.HasPrefix(container.Config.Image, "weaveworks/weave") +} + func (proxy *Proxy) createWait(r *http.Request, ident string) { proxy.Lock() ch := make(chan struct{}) @@ -386,6 +392,18 @@ func (proxy *Proxy) attach(container *docker.Container, orDie bool) error { return nil } +func (proxy *Proxy) attachRouter(container *docker.Container) error { + Log.Infof("Attaching weave router container") + args := []string{"attach-router"} + if _, stderr, err := callWeave(args...); err != nil { + Log.Warningf("Attaching container %s to weave network failed: %s", container.ID, string(stderr)) + return errors.New(string(stderr)) + } else if len(stderr) > 0 { + Log.Warningf("Attaching container %s to weave network: %s", container.ID, string(stderr)) + } + return nil +} + func (proxy *Proxy) weaveCIDRs(networkMode string, env []string) ([]string, error) { if networkMode == "host" || strings.HasPrefix(networkMode, "container:") { return nil, fmt.Errorf("the container was created with the '--net=%s'", networkMode) diff --git a/test/640_proxy_restart_reattaches_test.sh b/test/640_proxy_restart_reattaches_test.sh index ff02e678..6bdd65ec 100755 --- a/test/640_proxy_restart_reattaches_test.sh +++ b/test/640_proxy_restart_reattaches_test.sh @@ -13,13 +13,18 @@ check_attached() { start_suite "Proxy restart reattaches networking to containers" -weave_on $HOST1 launch +WEAVE_DOCKER_ARGS=--restart=always WEAVEPROXY_DOCKER_ARGS=--restart=always weave_on $HOST1 launch proxy_start_container $HOST1 -e WEAVE_CIDR=$C2/24 -di --name=c2 --restart=always -h $NAME proxy_start_container_with_dns $HOST1 -e WEAVE_CIDR=$C1/24 -di --name=c1 --restart=always proxy docker_on $HOST1 restart -t=1 c2 check_attached +# Restart weave router +docker_on $HOST1 restart weave +sleep 1 +check_attached + # Kill outside of Docker so Docker will restart it run_on $HOST1 sudo kill -KILL $(docker_on $HOST1 inspect --format='{{.State.Pid}}' c2) sleep 1 @@ -34,7 +39,6 @@ assert_raises "proxy exec_on $HOST1 c3 $CHECK_ETHWE_UP" # Restart docker itself, using different commands for systemd- and upstart-managed. run_on $HOST1 sh -c "command -v systemctl >/dev/null && sudo systemctl restart docker || sudo service docker restart" sleep 1 -weave_on $HOST1 launch check_attached end_suite diff --git a/weave b/weave index d560c278..b89f0173 100755 --- a/weave +++ b/weave @@ -685,6 +685,10 @@ setup_router_iface_bridge() { echo "Perhaps you are running the docker daemon with container networking disabled (-b=none)." >&2 return 1 fi + # No-op if already attached + if netnsenter ip link show $CONTAINER_IFNAME >/dev/null 2>&1 ; then + return 0 + fi connect_container_to_bridge $CONTAINER_IFNAME && netnsenter ethtool -K eth0 tx off >/dev/null && netnsenter ip link set $CONTAINER_IFNAME up @@ -1467,6 +1471,17 @@ launch_router() { --dns-effective-listen-address $DOCKER_BRIDGE_IP \ ${NETHOST_OPT:+$DNS_ROUTER_OPTS} $NO_DNS_OPT \ --docker-api "unix:///var/run/docker.sock" "$@") +} + +# Recreate the parameter values that are set when the router is first launched +fetch_router_args() { + IPRANGE=$(docker inspect -f '{{.Args}}' $1 | grep -o -e '-ipalloc-range [^ ]*') + # This doesn't get back the right syntax, but it's enough for our purposes + DNS_PORT_MAPPING=$(docker inspect -f '{{with index .HostConfig.PortBindings "53/udp"}}{{.}}{{end}}' $1) +} + +attach_router() { + ROUTER_CONTAINER=$1 with_container_netns_or_die $ROUTER_CONTAINER setup_router_iface_$BRIDGE_TYPE err_msg=$(death_msg $CONTAINER_NAME) container_ip $CONTAINER_NAME "$err_msg" "$err_msg" || return 1 @@ -1643,14 +1658,24 @@ case "$COMMAND" in check_not_running $PROXY_CONTAINER_NAME $BASE_EXEC_IMAGE COMMON_ARGS=$(common_launch_args "$@") launch_router "$@" + attach_router $CONTAINER_NAME launch_proxy $COMMON_ARGS ;; launch-router) deprecation_warnings "$@" check_not_running $CONTAINER_NAME $BASE_IMAGE launch_router "$@" + attach_router $CONTAINER_NAME echo $ROUTER_CONTAINER ;; + attach-router) + deprecation_warnings "$@" + check_running $CONTAINER_NAME + enforce_docker_bridge_addr_assign_type + create_bridge + fetch_router_args $CONTAINER_NAME + attach_router $CONTAINER_NAME + ;; launch-proxy) deprecation_warnings "$@" check_not_running $PROXY_CONTAINER_NAME $BASE_EXEC_IMAGE From a1bbb06e2f13085a214db9510b53d8e80940ddf0 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Fri, 9 Oct 2015 15:57:18 +0100 Subject: [PATCH 02/11] Don't need depreciation warnings on internal-only command --- weave | 1 - 1 file changed, 1 deletion(-) diff --git a/weave b/weave index b89f0173..48b88883 100755 --- a/weave +++ b/weave @@ -1669,7 +1669,6 @@ case "$COMMAND" in echo $ROUTER_CONTAINER ;; attach-router) - deprecation_warnings "$@" check_running $CONTAINER_NAME enforce_docker_bridge_addr_assign_type create_bridge From 20ac0edda8f03684c35d5b72b8533d241fc39925 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Mon, 12 Oct 2015 09:56:14 +0100 Subject: [PATCH 03/11] Try a bigger sleep to get Docker restarted --- test/640_proxy_restart_reattaches_test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/640_proxy_restart_reattaches_test.sh b/test/640_proxy_restart_reattaches_test.sh index 6bdd65ec..703c3c56 100755 --- a/test/640_proxy_restart_reattaches_test.sh +++ b/test/640_proxy_restart_reattaches_test.sh @@ -38,7 +38,7 @@ assert_raises "proxy exec_on $HOST1 c3 $CHECK_ETHWE_UP" # Restart docker itself, using different commands for systemd- and upstart-managed. run_on $HOST1 sh -c "command -v systemctl >/dev/null && sudo systemctl restart docker || sudo service docker restart" -sleep 1 +sleep 5 check_attached end_suite From 697ee62d9a2ae8773c69064bcd2253f12c4da33c Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Fri, 16 Oct 2015 15:01:38 +0100 Subject: [PATCH 04/11] Use entrypoint to detect weave router --- proxy/common.go | 1 + proxy/proxy.go | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/proxy/common.go b/proxy/common.go index 3f41eb17..1dfe787e 100644 --- a/proxy/common.go +++ b/proxy/common.go @@ -20,6 +20,7 @@ import ( var ( containerIDRegexp = regexp.MustCompile("^(/v[0-9\\.]*)?/containers/([^/]*)/.*") weaveWaitEntrypoint = []string{"/w/w"} + weaveEntrypoint = "/home/weave/weaver" ) func callWeave(args ...string) ([]byte, []byte, error) { diff --git a/proxy/proxy.go b/proxy/proxy.go index 77cc7ca9..171a91a0 100644 --- a/proxy/proxy.go +++ b/proxy/proxy.go @@ -316,7 +316,7 @@ func containerShouldAttach(container *docker.Container) bool { } func containerIsWeaveRouter(container *docker.Container) bool { - return strings.HasPrefix(container.Config.Image, "weaveworks/weave") + return len(container.Config.Entrypoint) > 0 && container.Config.Entrypoint[0] == weaveEntrypoint } func (proxy *Proxy) createWait(r *http.Request, ident string) { @@ -393,7 +393,7 @@ func (proxy *Proxy) attach(container *docker.Container, orDie bool) error { } func (proxy *Proxy) attachRouter(container *docker.Container) error { - Log.Infof("Attaching weave router container") + Log.Infof("Attaching weave router container:", container.ID) args := []string{"attach-router"} if _, stderr, err := callWeave(args...); err != nil { Log.Warningf("Attaching container %s to weave network failed: %s", container.ID, string(stderr)) From b8b5fb9eae7318c28689c426bf065738cc55dca7 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Fri, 16 Oct 2015 16:56:46 +0100 Subject: [PATCH 05/11] Review feedback --- weave | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/weave b/weave index 48b88883..609573bd 100755 --- a/weave +++ b/weave @@ -1471,18 +1471,18 @@ launch_router() { --dns-effective-listen-address $DOCKER_BRIDGE_IP \ ${NETHOST_OPT:+$DNS_ROUTER_OPTS} $NO_DNS_OPT \ --docker-api "unix:///var/run/docker.sock" "$@") + attach_router } # Recreate the parameter values that are set when the router is first launched fetch_router_args() { - IPRANGE=$(docker inspect -f '{{.Args}}' $1 | grep -o -e '-ipalloc-range [^ ]*') + IPRANGE=$(docker inspect -f '{{.Args}}' $CONTAINER_NAME | grep -o -e '-ipalloc-range [^ ]*') # This doesn't get back the right syntax, but it's enough for our purposes - DNS_PORT_MAPPING=$(docker inspect -f '{{with index .HostConfig.PortBindings "53/udp"}}{{.}}{{end}}' $1) + DNS_PORT_MAPPING=$(docker inspect -f '{{with index .HostConfig.PortBindings "53/udp"}}{{.}}{{end}}' $CONTAINER_NAME) } attach_router() { - ROUTER_CONTAINER=$1 - with_container_netns_or_die $ROUTER_CONTAINER setup_router_iface_$BRIDGE_TYPE + with_container_netns_or_die $CONTAINER_NAME setup_router_iface_$BRIDGE_TYPE err_msg=$(death_msg $CONTAINER_NAME) container_ip $CONTAINER_NAME "$err_msg" "$err_msg" || return 1 wait_for_status $CONTAINER_NAME http_call $CONTAINER_IP:$HTTP_PORT @@ -1658,22 +1658,20 @@ case "$COMMAND" in check_not_running $PROXY_CONTAINER_NAME $BASE_EXEC_IMAGE COMMON_ARGS=$(common_launch_args "$@") launch_router "$@" - attach_router $CONTAINER_NAME launch_proxy $COMMON_ARGS ;; launch-router) deprecation_warnings "$@" check_not_running $CONTAINER_NAME $BASE_IMAGE launch_router "$@" - attach_router $CONTAINER_NAME echo $ROUTER_CONTAINER ;; attach-router) check_running $CONTAINER_NAME enforce_docker_bridge_addr_assign_type create_bridge - fetch_router_args $CONTAINER_NAME - attach_router $CONTAINER_NAME + fetch_router_args + attach_router ;; launch-proxy) deprecation_warnings "$@" From 284d78c6b2b04360c3566c7f4e94231961c900d3 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Mon, 19 Oct 2015 12:35:05 +0000 Subject: [PATCH 06/11] Update fetch_router_args in line with changes elsewhere --- weave | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/weave b/weave index 609573bd..231a4d33 100755 --- a/weave +++ b/weave @@ -1476,9 +1476,9 @@ launch_router() { # Recreate the parameter values that are set when the router is first launched fetch_router_args() { - IPRANGE=$(docker inspect -f '{{.Args}}' $CONTAINER_NAME | grep -o -e '-ipalloc-range [^ ]*') - # This doesn't get back the right syntax, but it's enough for our purposes - DNS_PORT_MAPPING=$(docker inspect -f '{{with index .HostConfig.PortBindings "53/udp"}}{{.}}{{end}}' $CONTAINER_NAME) + CONTAINER_ARGS=$(docker inspect -f '{{.Args}}' $CONTAINER_NAME) + IPRANGE=$(echo $CONTAINER_ARGS | grep -o -e '-ipalloc-range [^ ]*') + NO_DNS_OPT=$(echo $CONTAINER_ARGS | grep -o -e '--no-dns') } attach_router() { From 08f6dcf08dbda49ae68a687cee8a595f7eaf23d3 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Mon, 19 Oct 2015 13:28:16 +0000 Subject: [PATCH 07/11] Missing format --- proxy/proxy.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proxy/proxy.go b/proxy/proxy.go index 171a91a0..81d8e79c 100644 --- a/proxy/proxy.go +++ b/proxy/proxy.go @@ -393,7 +393,7 @@ func (proxy *Proxy) attach(container *docker.Container, orDie bool) error { } func (proxy *Proxy) attachRouter(container *docker.Container) error { - Log.Infof("Attaching weave router container:", container.ID) + Log.Infof("Attaching weave router container: %s", container.ID) args := []string{"attach-router"} if _, stderr, err := callWeave(args...); err != nil { Log.Warningf("Attaching container %s to weave network failed: %s", container.ID, string(stderr)) From 2a99a2a597106cd7347ccb8e5984881419a20efb Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Mon, 19 Oct 2015 13:49:42 +0000 Subject: [PATCH 08/11] Don't fail if the arg we're looking for isn't there --- weave | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/weave b/weave index 231a4d33..110bc01d 100755 --- a/weave +++ b/weave @@ -1477,8 +1477,8 @@ launch_router() { # Recreate the parameter values that are set when the router is first launched fetch_router_args() { CONTAINER_ARGS=$(docker inspect -f '{{.Args}}' $CONTAINER_NAME) - IPRANGE=$(echo $CONTAINER_ARGS | grep -o -e '-ipalloc-range [^ ]*') - NO_DNS_OPT=$(echo $CONTAINER_ARGS | grep -o -e '--no-dns') + IPRANGE=$(echo $CONTAINER_ARGS | grep -o -e '-ipalloc-range [^ ]*') || true + NO_DNS_OPT=$(echo $CONTAINER_ARGS | grep -o -e '--no-dns') || true } attach_router() { From c09d84efc65926f129031a735699bd5be946b556 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Mon, 19 Oct 2015 15:40:45 +0000 Subject: [PATCH 09/11] Move setup_router_iface call so we don't kill an already-running router on attach-router --- weave | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/weave b/weave index 110bc01d..08bc5eac 100755 --- a/weave +++ b/weave @@ -1471,6 +1471,7 @@ launch_router() { --dns-effective-listen-address $DOCKER_BRIDGE_IP \ ${NETHOST_OPT:+$DNS_ROUTER_OPTS} $NO_DNS_OPT \ --docker-api "unix:///var/run/docker.sock" "$@") + with_container_netns_or_die $CONTAINER_NAME setup_router_iface_$BRIDGE_TYPE attach_router } @@ -1482,7 +1483,6 @@ fetch_router_args() { } attach_router() { - with_container_netns_or_die $CONTAINER_NAME setup_router_iface_$BRIDGE_TYPE err_msg=$(death_msg $CONTAINER_NAME) container_ip $CONTAINER_NAME "$err_msg" "$err_msg" || return 1 wait_for_status $CONTAINER_NAME http_call $CONTAINER_IP:$HTTP_PORT @@ -1671,6 +1671,7 @@ case "$COMMAND" in enforce_docker_bridge_addr_assign_type create_bridge fetch_router_args + with_container_netns $CONTAINER_NAME setup_router_iface_$BRIDGE_TYPE attach_router ;; launch-proxy) From bb5515113f42252263928f8cee93a6ca8df5556a Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Mon, 19 Oct 2015 15:41:35 +0000 Subject: [PATCH 10/11] Make attach-router work in fastdp+encryption case --- weave | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/weave b/weave index 08bc5eac..983be841 100755 --- a/weave +++ b/weave @@ -673,8 +673,10 @@ router_opts_bridge() { setup_router_iface_fastdp() { if [ -n "$WEAVE_PASSWORD" ] ; then # See router_opts_fastdp - # In case there is a lingering weave2 netdev - ip link del $ROUTER_HOSTNETNS_IFNAME >/dev/null 2>&1 || true + # No-op if already attached + if ip link show $LOCAL_IFNAME >/dev/null 2>&1 ; then + return 0 + fi connect_container_to_bridge $ROUTER_HOSTNETNS_IFNAME && ip link set $ROUTER_HOSTNETNS_IFNAME up fi @@ -1453,6 +1455,8 @@ launch_router() { if [ "$BRIDGE_TYPE" = fastdp ] ; then NETHOST_OPT="--net=host" + # In case there is a lingering weave2 netdev + ip link del $ROUTER_HOSTNETNS_IFNAME >/dev/null 2>&1 || true fi # Set WEAVE_DOCKER_ARGS in the environment in order to supply @@ -1480,6 +1484,7 @@ fetch_router_args() { CONTAINER_ARGS=$(docker inspect -f '{{.Args}}' $CONTAINER_NAME) IPRANGE=$(echo $CONTAINER_ARGS | grep -o -e '-ipalloc-range [^ ]*') || true NO_DNS_OPT=$(echo $CONTAINER_ARGS | grep -o -e '--no-dns') || true + WEAVE_PASSWORD=$(docker inspect -f '{{.Config.Env}}' $CONTAINER_NAME | sed -n -e 's/.*WEAVE_PASSWORD=\([^] ]*\).*/\1/p') } attach_router() { From 26d43f193e0ddbb9a76d743511a249d2ccb742b0 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Tue, 20 Oct 2015 07:07:04 +0000 Subject: [PATCH 11/11] Move docker restart test above the place where we stop part of weave --- test/640_proxy_restart_reattaches_test.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/640_proxy_restart_reattaches_test.sh b/test/640_proxy_restart_reattaches_test.sh index 703c3c56..521eae30 100755 --- a/test/640_proxy_restart_reattaches_test.sh +++ b/test/640_proxy_restart_reattaches_test.sh @@ -30,15 +30,15 @@ run_on $HOST1 sudo kill -KILL $(docker_on $HOST1 inspect --format='{{.State.Pid} sleep 1 check_attached +# Restart docker itself, using different commands for systemd- and upstart-managed. +run_on $HOST1 sh -c "command -v systemctl >/dev/null && sudo systemctl restart docker || sudo service docker restart" +sleep 5 +check_attached + # Restarting proxy shouldn't kill unattachable containers proxy_start_container $HOST1 -di --name=c3 --restart=always # Use ipam, so it won't be attachable w/o weave weave_on $HOST1 stop weave_on $HOST1 launch-proxy assert_raises "proxy exec_on $HOST1 c3 $CHECK_ETHWE_UP" -# Restart docker itself, using different commands for systemd- and upstart-managed. -run_on $HOST1 sh -c "command -v systemctl >/dev/null && sudo systemctl restart docker || sudo service docker restart" -sleep 5 -check_attached - end_suite