From a91357978d7abb836dc7b4f4307dd9b9da4e4379 Mon Sep 17 00:00:00 2001 From: adieu Date: Fri, 12 Sep 2014 00:36:21 +0800 Subject: [PATCH 01/26] Add attach command to weave. --- weaver/weave | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/weaver/weave b/weaver/weave index 5dab5d9b..559680e6 100755 --- a/weaver/weave +++ b/weaver/weave @@ -5,6 +5,7 @@ usage() { echo "Usage:" echo "weave launch / [-passwd ] ..." echo "weave run / ..." + echo "weave attach / " echo "weave expose /" echo "weave hide /" echo "weave status" @@ -131,6 +132,13 @@ case "$COMMAND" in echo $CONTAINER IFACE=$CONTAINER_IFNAME ;; + attach) + [ $# -gt 1 ] || usage + IPADDR=$1 + create_bridge + CONTAINER=$2 + IFACE=$CONTAINER_IFNAME + ;; expose) [ $# -gt 0 ] || usage IPADDR=$1 From 5490826666921dc2d538739251c68ea8b1a989f1 Mon Sep 17 00:00:00 2001 From: Kai Davenport Date: Mon, 15 Sep 2014 10:25:59 -0700 Subject: [PATCH 02/26] attach command that accepts and existing container id and runs configure_container_networking using it --- weaver/weave | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/weaver/weave b/weaver/weave index 77a28173..3faf7c19 100755 --- a/weaver/weave +++ b/weaver/weave @@ -5,6 +5,7 @@ usage() { echo "Usage:" echo "weave launch / [-passwd ] ..." echo "weave run / ..." + echo "weave attach / ..." echo "weave expose /" echo "weave hide /" echo "weave status" @@ -205,6 +206,14 @@ case "$COMMAND" in configure_container_networking $CONTAINER $IPADDR $CONTAINER_IFNAME echo $CONTAINER ;; + attach) + [ $# -gt 1 ] || usage + IPADDR=$1 + shift 1 + CONTAINER=$1 + shift 1 + configure_container_networking $CONTAINER $IPADDR $CONTAINER_IFNAME + ;; expose) [ $# -gt 0 ] || usage IPADDR=$1 From e51f89268c0b03ffc8ab3f5d176d15b09a7f0b60 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Thu, 18 Sep 2014 05:50:13 +0100 Subject: [PATCH 03/26] Stop waiting for interface after 5 seconds --- weaver/main.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/weaver/main.go b/weaver/main.go index 63bb60d1..a16aabc9 100644 --- a/weaver/main.go +++ b/weaver/main.go @@ -24,11 +24,15 @@ func ensureInterface(ifaceName string, wait bool) (iface *net.Interface, err err return } log.Println("Waiting for interface", ifaceName, "to come up") - for err != nil { + count := 0 + for err != nil && count < 5 { + count += 1 time.Sleep(1 * time.Second) iface, err = findInterface(ifaceName) } - log.Println("Interface", ifaceName, "is up") + if err == nil { + log.Println("Interface", ifaceName, "is up") + } return } @@ -47,6 +51,7 @@ func main() { log.SetPrefix(weave.Protocol + " ") log.SetFlags(log.Ldate | log.Ltime | log.Lmicroseconds) + log.Println(os.Args) procs := runtime.NumCPU() // packet sniffing can block an OS thread, so we need one thread @@ -87,6 +92,9 @@ func main() { if err != nil { log.Fatal(err) } + if wait && iface == nil { + os.Exit(1) + } if connLimit < 0 { connLimit = 0 From add13ff8578365158b25a7c82b0c539227003da5 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Thu, 18 Sep 2014 06:18:43 +0100 Subject: [PATCH 04/26] Add section to README on reboots --- README.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/README.md b/README.md index 475ec6b2..4f86dfe6 100644 --- a/README.md +++ b/README.md @@ -437,6 +437,28 @@ Any running application containers will permanently lose connectivity with the weave network and have to be restarted in order to re-connect. +### Reboots + +If the host you are running your containers on gets rebooted, the +default behaviour of Docker is to restart any containers that were +running, but weave relies on a bridge and virtual ethernet devices +being set up outside of the containers, so the weave network will not +function in this state. + +For the weave router, the best thing is to stop any container that +Docker may have restarted and re-launch, i.e. something like: + + host1# weave stop + host1# weave launch 10.0.0.1/16 + +After that, other containers may be re-attached if you can identify +them, like this: + + host1# weave attach 10.0.1.1/24 + +Alternatively, you may prefer to disable the auto-restart feature in +Docker as described [here](https://docs.docker.com/articles/host_integration/). + ## Installation with Boot2Docker If you are running Docker inside the Boot2Docker VM, e.g. because you From 4868585d6e30c5a9d9d6d15ba22eb4edbe0d1393 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Thu, 18 Sep 2014 17:18:54 +0100 Subject: [PATCH 05/26] Add detach command --- weaver/weave | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/weaver/weave b/weaver/weave index 371f3cec..b9b2d69c 100755 --- a/weaver/weave +++ b/weaver/weave @@ -6,6 +6,7 @@ usage() { echo "weave launch / [-passwd ] ..." echo "weave run / ..." echo "weave attach / " + echo "weave detach " echo "weave expose /" echo "weave hide /" echo "weave status" @@ -102,7 +103,7 @@ stop_masquerading() { } # the following borrows from https://github.com/jpetazzo/pipework -configure_container_networking() { +find_container_pid() { DOCKERPID=$(docker inspect --format='{{ .State.Pid }}' $1) [ "$DOCKERPID" = 0 ] && { echo "Container $1 not running." >&2 @@ -112,6 +113,11 @@ configure_container_networking() { echo "Container $1 unknown to Docker." >&2 exit 1 } + return 0 +} + +configure_container_networking() { + find_container_pid $1 NSPID=$DOCKERPID [ ! -d /var/run/netns ] && mkdir -p /var/run/netns @@ -154,6 +160,15 @@ configure_container_networking() { return $EXITCODE } +takedown_container_networking() { + find_container_pid $1 + NSPID=$DOCKERPID + + LOCAL_IFNAME="v${CONTAINER_IFNAME}pl${NSPID}" + + ip link del $LOCAL_IFNAME type veth || true +} + case "$COMMAND" in launch) [ $# -gt 0 ] || usage @@ -236,6 +251,12 @@ case "$COMMAND" in shift 1 configure_container_networking $CONTAINER $IPADDR $CONTAINER_IFNAME ;; + detach) + [ $# -gt 0 ] || usage + CONTAINER=$1 + shift 1 + takedown_container_networking $CONTAINER + ;; expose) [ $# -gt 0 ] || usage IPADDR=$1 From 0947fc204f9eb73e3950ffa9777a8b9f2c0e1d0f Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Thu, 18 Sep 2014 17:20:08 +0100 Subject: [PATCH 06/26] Untabify --- weaver/weave | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/weaver/weave b/weaver/weave index b9b2d69c..7340dc3f 100755 --- a/weaver/weave +++ b/weaver/weave @@ -113,11 +113,11 @@ find_container_pid() { echo "Container $1 unknown to Docker." >&2 exit 1 } - return 0 + return 0 } configure_container_networking() { - find_container_pid $1 + find_container_pid $1 NSPID=$DOCKERPID [ ! -d /var/run/netns ] && mkdir -p /var/run/netns @@ -161,7 +161,7 @@ configure_container_networking() { } takedown_container_networking() { - find_container_pid $1 + find_container_pid $1 NSPID=$DOCKERPID LOCAL_IFNAME="v${CONTAINER_IFNAME}pl${NSPID}" From 47310fbfc57876c705dd9eaa1719da6f08928591 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Thu, 18 Sep 2014 18:00:58 +0100 Subject: [PATCH 07/26] Described attach and detach commands --- README.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/README.md b/README.md index 475ec6b2..ff0e2028 100644 --- a/README.md +++ b/README.md @@ -221,6 +221,30 @@ will work. And, more interestingly, will work too, which is talking to a container that resides on $HOST1. +### Attach to already-running containers + +If you don't want to start containers via the `weave` script, e.g. because +you are controlling them as part of some larger orchestration scheme, you can +attach the weave network to an already-running container. To illustrate, we +can achieve the same effect as the first example like this: + + host1# C=$(docker run -d -t -i ubuntu) + host1# weave attach 10.0.1.1/24 $C + +There is a matching `weave detach` command: + + host1# weave detach $C + +You can detach a container from one subnet and re-attach it to another: + + host1# weave detach $C + host1# weave attach 10.0.2.1/24 $C + +or attach a container to multiple subnets: + + host1# weave attach 10.0.1.1/24 $C + host1# weave attach 10.0.2.1/24 $C + ### Service export Services running in containers on a weave network can be made From 5b935770541927dc737fd8535b8d0530c49e1d49 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Mon, 22 Sep 2014 16:42:18 +0100 Subject: [PATCH 08/26] log.Fatal() does os.Exit(1) so we don't need to do it again --- weaver/main.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/weaver/main.go b/weaver/main.go index a16aabc9..4ff2349d 100644 --- a/weaver/main.go +++ b/weaver/main.go @@ -92,9 +92,6 @@ func main() { if err != nil { log.Fatal(err) } - if wait && iface == nil { - os.Exit(1) - } if connLimit < 0 { connLimit = 0 From 0d3eebb208854cb0c6fccc84e043d0482a27fbe4 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Mon, 22 Sep 2014 16:48:55 +0100 Subject: [PATCH 09/26] Change -wait from bool to int number of seconds to wait --- weaver/Dockerfile | 2 +- weaver/main.go | 12 +++++------- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/weaver/Dockerfile b/weaver/Dockerfile index adf86d35..9baf50bb 100644 --- a/weaver/Dockerfile +++ b/weaver/Dockerfile @@ -3,4 +3,4 @@ MAINTAINER Zettio weave@zett.io WORKDIR /home/weave ADD ./weaver /home/weave/ CMD [] -ENTRYPOINT ["/home/weave/weaver", "-iface", "ethwe", "-wait"] +ENTRYPOINT ["/home/weave/weaver", "-iface", "ethwe", "-wait", "5"] diff --git a/weaver/main.go b/weaver/main.go index 4ff2349d..bd9014c0 100644 --- a/weaver/main.go +++ b/weaver/main.go @@ -18,15 +18,13 @@ import ( "time" ) -func ensureInterface(ifaceName string, wait bool) (iface *net.Interface, err error) { +func ensureInterface(ifaceName string, wait int) (iface *net.Interface, err error) { iface, err = findInterface(ifaceName) - if err == nil || !wait { + if err == nil || wait == 0 { return } log.Println("Waiting for interface", ifaceName, "to come up") - count := 0 - for err != nil && count < 5 { - count += 1 + for ; err != nil && wait > 0; wait -= 1 { time.Sleep(1 * time.Second) iface, err = findInterface(ifaceName) } @@ -65,7 +63,7 @@ func main() { ifaceName string routerName string password string - wait bool + wait int debug bool prof string peers []string @@ -76,7 +74,7 @@ func main() { flag.StringVar(&ifaceName, "iface", "", "name of interface to read from") flag.StringVar(&routerName, "name", "", "name of router (defaults to MAC)") flag.StringVar(&password, "password", "", "network password") - flag.BoolVar(&wait, "wait", false, "wait for interface to be created and come up") + flag.IntVar(&wait, "wait", 0, "number of seconds to wait for interface to be created and come up") flag.BoolVar(&debug, "debug", false, "enable debug logging") flag.StringVar(&prof, "profile", "", "enable profiling and write profiles to given path") flag.IntVar(&connLimit, "connlimit", 10, "connection limit (defaults to 10, set to 0 for unlimited)") From d1896e86d03e9da712aa816c091cc30e6038fd46 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Tue, 23 Sep 2014 13:01:21 +0100 Subject: [PATCH 10/26] Redesigned attach/detach --- weaver/weave | 73 ++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 51 insertions(+), 22 deletions(-) diff --git a/weaver/weave b/weaver/weave index 7340dc3f..589a2710 100755 --- a/weaver/weave +++ b/weaver/weave @@ -103,7 +103,7 @@ stop_masquerading() { } # the following borrows from https://github.com/jpetazzo/pipework -find_container_pid() { +container_netns_preamble() { DOCKERPID=$(docker inspect --format='{{ .State.Pid }}' $1) [ "$DOCKERPID" = 0 ] && { echo "Container $1 not running." >&2 @@ -113,20 +113,40 @@ find_container_pid() { echo "Container $1 unknown to Docker." >&2 exit 1 } - return 0 -} -configure_container_networking() { - find_container_pid $1 NSPID=$DOCKERPID [ ! -d /var/run/netns ] && mkdir -p /var/run/netns rm -f /var/run/netns/$NSPID ln -s /proc/$NSPID/ns/net /var/run/netns/$NSPID - - EXITCODE=1 LOCAL_IFNAME="v${CONTAINER_IFNAME}pl${NSPID}" GUEST_IFNAME="v${CONTAINER_IFNAME}pg${NSPID}" + + return 0 +} + +container_netns_cleanup() { + rm -f /var/run/netns/$NSPID +} + +# Parameters are container ID, CIDR address +configure_container_networking() { + container_netns_preamble $1 + + EXITCODE=1 + + # If veth name already exists then assume we set it up already; + # just add the IP address + if ip link show $LOCAL_IFNAME >/dev/null 2>&1 ; then + if ip netns exec $NSPID ip addr add $2 dev $CONTAINER_IFNAME ; then + EXITCODE=0 + else + echo "Failed to attach address $2 to container $1" >&2 + fi + container_netns_cleanup + return $EXITCODE + fi + IP_TMPFILE=/tmp/weave_ip_output_$$ rm -f $IP_TMPFILE if ip link add name $LOCAL_IFNAME mtu $MTU type veth peer name $GUEST_IFNAME mtu $MTU >$IP_TMPFILE 2>&1 ; then @@ -155,18 +175,33 @@ configure_container_networking() { fi rm -f $IP_TMPFILE - rm -f /var/run/netns/$NSPID + container_netns_cleanup return $EXITCODE } -takedown_container_networking() { - find_container_pid $1 - NSPID=$DOCKERPID +# Parameters are container ID, CIDR address, interface-name +detach() { + container_netns_preamble $1 - LOCAL_IFNAME="v${CONTAINER_IFNAME}pl${NSPID}" + EXITCODE=1 + if ip netns exec $NSPID ip addr del $2 dev $CONTAINER_IFNAME ; then + # if no inet addresses remain then delete the veth + if [ $(ip netns exec $NSPID ip -f inet addr show dev ethwe | wc -c) -eq 0 ] ; then + if ip link del $LOCAL_IFNAME type veth ; then + EXITCODE=0 + else + echo "Failed to delete virtual ethernet device $LOCAL_IFNAME from container $1" >&2 + fi + else + EXITCODE=0 + fi + else + echo "Failed to detach address $2 from container $1" >&2 + fi - ip link del $LOCAL_IFNAME type veth || true + container_netns_cleanup + return $EXITCODE } case "$COMMAND" in @@ -244,18 +279,12 @@ case "$COMMAND" in ;; attach) [ $# -gt 1 ] || usage - IPADDR=$1 - shift 1 create_bridge - CONTAINER=$1 - shift 1 - configure_container_networking $CONTAINER $IPADDR $CONTAINER_IFNAME + configure_container_networking $2 $1 $CONTAINER_IFNAME ;; detach) - [ $# -gt 0 ] || usage - CONTAINER=$1 - shift 1 - takedown_container_networking $CONTAINER + [ $# -gt 1 ] || usage + detach $2 $1 ;; expose) [ $# -gt 0 ] || usage From a6b83a1234689c8412a74e6c26d514be97e74c41 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Tue, 23 Sep 2014 13:52:41 +0100 Subject: [PATCH 11/26] Updated instructions for attach and detach --- README.md | 4 ++-- weaver/weave | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index ff0e2028..53ac9e33 100644 --- a/README.md +++ b/README.md @@ -233,11 +233,11 @@ can achieve the same effect as the first example like this: There is a matching `weave detach` command: - host1# weave detach $C + host1# weave detach 10.0.1.1/24 $C You can detach a container from one subnet and re-attach it to another: - host1# weave detach $C + host1# weave detach 10.0.1.1/24 $C host1# weave attach 10.0.2.1/24 $C or attach a container to multiple subnets: diff --git a/weaver/weave b/weaver/weave index 589a2710..80d220c1 100755 --- a/weaver/weave +++ b/weaver/weave @@ -6,7 +6,7 @@ usage() { echo "weave launch / [-passwd ] ..." echo "weave run / ..." echo "weave attach / " - echo "weave detach " + echo "weave detach / " echo "weave expose /" echo "weave hide /" echo "weave status" From 2d8318b1584b84a7148367f46c59ee6dc55d79a5 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Tue, 23 Sep 2014 13:53:20 +0100 Subject: [PATCH 12/26] Improved test for blank string --- weaver/weave | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weaver/weave b/weaver/weave index 80d220c1..d7a617ed 100755 --- a/weaver/weave +++ b/weaver/weave @@ -187,7 +187,7 @@ detach() { EXITCODE=1 if ip netns exec $NSPID ip addr del $2 dev $CONTAINER_IFNAME ; then # if no inet addresses remain then delete the veth - if [ $(ip netns exec $NSPID ip -f inet addr show dev ethwe | wc -c) -eq 0 ] ; then + if [ -z "$(ip netns exec $NSPID ip -f inet addr list dev ethwe)" ] ; then if ip link del $LOCAL_IFNAME type veth ; then EXITCODE=0 else From 8e50e0759df13a341f40a2221953506e1e0ceacb Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Tue, 23 Sep 2014 14:49:00 +0100 Subject: [PATCH 13/26] Make attach and detach idempotent --- weaver/weave | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/weaver/weave b/weaver/weave index d7a617ed..de0e72e7 100755 --- a/weaver/weave +++ b/weaver/weave @@ -113,7 +113,6 @@ container_netns_preamble() { echo "Container $1 unknown to Docker." >&2 exit 1 } - NSPID=$DOCKERPID [ ! -d /var/run/netns ] && mkdir -p /var/run/netns @@ -133,20 +132,21 @@ container_netns_cleanup() { configure_container_networking() { container_netns_preamble $1 - EXITCODE=1 - # If veth name already exists then assume we set it up already; - # just add the IP address + # just add the IP address, if it's not already there if ip link show $LOCAL_IFNAME >/dev/null 2>&1 ; then - if ip netns exec $NSPID ip addr add $2 dev $CONTAINER_IFNAME ; then - EXITCODE=0 - else - echo "Failed to attach address $2 to container $1" >&2 + if ! ip netns exec $NSPID ip addr list dev $CONTAINER_IFNAME | grep -qF $2 ; then + if ! ip netns exec $NSPID ip addr add $2 dev $CONTAINER_IFNAME ; then + echo "Failed to attach address $2 to container $1" >&2 + container_netns_cleanup + return 1 + fi fi container_netns_cleanup - return $EXITCODE + return 0 fi + EXITCODE=1 IP_TMPFILE=/tmp/weave_ip_output_$$ rm -f $IP_TMPFILE if ip link add name $LOCAL_IFNAME mtu $MTU type veth peer name $GUEST_IFNAME mtu $MTU >$IP_TMPFILE 2>&1 ; then @@ -185,19 +185,21 @@ detach() { container_netns_preamble $1 EXITCODE=1 - if ip netns exec $NSPID ip addr del $2 dev $CONTAINER_IFNAME ; then - # if no inet addresses remain then delete the veth - if [ -z "$(ip netns exec $NSPID ip -f inet addr list dev ethwe)" ] ; then - if ip link del $LOCAL_IFNAME type veth ; then - EXITCODE=0 + if ip netns exec $NSPID ip addr list dev $CONTAINER_IFNAME | grep -qF $2 ; then + if ip netns exec $NSPID ip addr del $2 dev $CONTAINER_IFNAME ; then + # if no inet addresses remain then delete the veth + if [ -z "$(ip netns exec $NSPID ip -f inet addr list dev ethwe)" ] ; then + if ip link del $LOCAL_IFNAME type veth ; then + EXITCODE=0 + else + echo "Failed to delete virtual ethernet device $LOCAL_IFNAME from container $1" >&2 + fi else - echo "Failed to delete virtual ethernet device $LOCAL_IFNAME from container $1" >&2 + EXITCODE=0 fi else - EXITCODE=0 + echo "Failed to detach address $2 from container $1" >&2 fi - else - echo "Failed to detach address $2 from container $1" >&2 fi container_netns_cleanup From ff3fbaede915de404527f74080d7313bac6cabc8 Mon Sep 17 00:00:00 2001 From: Matthias Radestock Date: Tue, 23 Sep 2014 15:32:53 +0100 Subject: [PATCH 14/26] tweak & move docs for 'attach' and 'detach' --- README.md | 53 +++++++++++++++++++++++++++++------------------------ 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 49fe82dd..22092da7 100644 --- a/README.md +++ b/README.md @@ -185,6 +185,35 @@ each other but not the containers of our first application... This isolation-through-subnets scheme is an example of carrying over a well-known technique from the 'on metal' days to containers. +### Dynamic network attachment + +In some scenarios containers are started independently, e.g. via some +existing tool chain, or require more complex startup sequences than +provided by `weave run`. And sometimes the decision which application +network a container should be part of is made post startup. For these +situation, weave allows an existing, running container to be attached +to the weave network. To illustrate, we can achieve the same effect as +the first example with + + host1# C=$(docker run -d -t -i ubuntu) + host1# weave attach 10.0.1.1/24 $C + +There is a matching `weave detach` command: + + host1# weave detach 10.0.1.1/24 $C + +You can detach a container from one application network and attach it +to another: + + host1# weave detach 10.0.1.1/24 $C + host1# weave attach 10.0.2.1/24 $C + +or attach a container to multiple application networks, effectively +sharing it between applications: + + host1# weave attach 10.0.1.1/24 $C + host1# weave attach 10.0.2.1/24 $C + ### Security In order to connect containers across untrusted networks, weave peers @@ -221,30 +250,6 @@ will work. And, more interestingly, will work too, which is talking to a container that resides on $HOST1. -### Attach to already-running containers - -If you don't want to start containers via the `weave` script, e.g. because -you are controlling them as part of some larger orchestration scheme, you can -attach the weave network to an already-running container. To illustrate, we -can achieve the same effect as the first example like this: - - host1# C=$(docker run -d -t -i ubuntu) - host1# weave attach 10.0.1.1/24 $C - -There is a matching `weave detach` command: - - host1# weave detach 10.0.1.1/24 $C - -You can detach a container from one subnet and re-attach it to another: - - host1# weave detach 10.0.1.1/24 $C - host1# weave attach 10.0.2.1/24 $C - -or attach a container to multiple subnets: - - host1# weave attach 10.0.1.1/24 $C - host1# weave attach 10.0.2.1/24 $C - ### Service export Services running in containers on a weave network can be made From 9e3cbba275a6a955df32877470d19c87a144ef2e Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Tue, 23 Sep 2014 16:00:03 +0100 Subject: [PATCH 15/26] Capture error output from ip command and check if error was caused by container dying --- weaver/weave | 46 ++++++++++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/weaver/weave b/weaver/weave index de0e72e7..2870314e 100755 --- a/weaver/weave +++ b/weaver/weave @@ -131,25 +131,21 @@ container_netns_cleanup() { # Parameters are container ID, CIDR address configure_container_networking() { container_netns_preamble $1 + EXITCODE=1 + IP_TMPFILE=/tmp/weave_ip_output_$$ + rm -f $IP_TMPFILE # If veth name already exists then assume we set it up already; # just add the IP address, if it's not already there if ip link show $LOCAL_IFNAME >/dev/null 2>&1 ; then - if ! ip netns exec $NSPID ip addr list dev $CONTAINER_IFNAME | grep -qF $2 ; then - if ! ip netns exec $NSPID ip addr add $2 dev $CONTAINER_IFNAME ; then - echo "Failed to attach address $2 to container $1" >&2 - container_netns_cleanup - return 1 + if ! (ip netns exec $NSPID ip addr list dev $CONTAINER_IFNAME | grep -qF $2) 2>$IP_TMPFILE ; then + if ip netns exec $NSPID ip addr add $2 dev $CONTAINER_IFNAME 2>$IP_TMPFILE ; then + EXITCODE=0 # succeeded in adding the address fi + else + EXITCODE=0 # address was there already fi - container_netns_cleanup - return 0 - fi - - EXITCODE=1 - IP_TMPFILE=/tmp/weave_ip_output_$$ - rm -f $IP_TMPFILE - if ip link add name $LOCAL_IFNAME mtu $MTU type veth peer name $GUEST_IFNAME mtu $MTU >$IP_TMPFILE 2>&1 ; then + elif ip link add name $LOCAL_IFNAME mtu $MTU type veth peer name $GUEST_IFNAME mtu $MTU >$IP_TMPFILE 2>&1 ; then if (ip link set $LOCAL_IFNAME master $BRIDGE && ip link set $LOCAL_IFNAME up && ip link set $GUEST_IFNAME netns $NSPID) >$IP_TMPFILE 2>&1 @@ -185,23 +181,33 @@ detach() { container_netns_preamble $1 EXITCODE=1 - if ip netns exec $NSPID ip addr list dev $CONTAINER_IFNAME | grep -qF $2 ; then - if ip netns exec $NSPID ip addr del $2 dev $CONTAINER_IFNAME ; then + IP_TMPFILE=/tmp/weave_ip_output_$$ + rm -f $IP_TMPFILE + if (ip netns exec $NSPID ip addr list dev $CONTAINER_IFNAME | grep -qF $2) 2>$IP_TMPFILE ; then + if ip netns exec $NSPID ip addr del $2 dev $CONTAINER_IFNAME 2>$IP_TMPFILE ; then # if no inet addresses remain then delete the veth - if [ -z "$(ip netns exec $NSPID ip -f inet addr list dev ethwe)" ] ; then - if ip link del $LOCAL_IFNAME type veth ; then + if [ -z "$(ip netns exec $NSPID ip -f inet addr list dev ethwe 2>$IP_TMPFILE)" ] ; then + if ip link del $LOCAL_IFNAME type veth 2>$IP_TMPFILE ; then EXITCODE=0 - else - echo "Failed to delete virtual ethernet device $LOCAL_IFNAME from container $1" >&2 fi else EXITCODE=0 fi + fi + else + EXITCODE=0 + fi + + if [ "$EXITCODE" -ne 0 ] ; then + if ! ps -p $DOCKERPID >/dev/null 2>&1 ; then + echo "Container $1 died" >&2 else - echo "Failed to detach address $2 from container $1" >&2 + echo "Failure while detaching address $2 from container $1" >&2 + cat $IP_TMPFILE >&2 fi fi + rm -f $IP_TMPFILE container_netns_cleanup return $EXITCODE } From 02efccc36ae41493a1948e7b9c8c28523ff6d247 Mon Sep 17 00:00:00 2001 From: Matthias Radestock Date: Tue, 23 Sep 2014 16:18:00 +0100 Subject: [PATCH 16/26] cosmetic --- weaver/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weaver/main.go b/weaver/main.go index bd9014c0..3ca078db 100644 --- a/weaver/main.go +++ b/weaver/main.go @@ -74,7 +74,7 @@ func main() { flag.StringVar(&ifaceName, "iface", "", "name of interface to read from") flag.StringVar(&routerName, "name", "", "name of router (defaults to MAC)") flag.StringVar(&password, "password", "", "network password") - flag.IntVar(&wait, "wait", 0, "number of seconds to wait for interface to be created and come up") + flag.IntVar(&wait, "wait", 0, "number of seconds to wait for interface to be created and come up (defaults to 0, i.e. don't wait)") flag.BoolVar(&debug, "debug", false, "enable debug logging") flag.StringVar(&prof, "profile", "", "enable profiling and write profiles to given path") flag.IntVar(&connLimit, "connlimit", 10, "connection limit (defaults to 10, set to 0 for unlimited)") From d10ce1e801f19ac44f8d8f1cfc763365d70a34d6 Mon Sep 17 00:00:00 2001 From: Matthias Radestock Date: Tue, 23 Sep 2014 17:24:52 +0100 Subject: [PATCH 17/26] reword --- README.md | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index ff45947c..a6a109e8 100644 --- a/README.md +++ b/README.md @@ -434,25 +434,18 @@ re-connect. ### Reboots -If the host you are running your containers on gets rebooted, the -default behaviour of Docker is to restart any containers that were -running, but weave relies on a bridge and virtual ethernet devices -being set up outside of the containers, so the weave network will not +When a host reboots, docker's default behaviour is to restart any +containers that were running. Since weave relies on special network +configuration outside of the containers, the weave network will not function in this state. -For the weave router, the best thing is to stop any container that -Docker may have restarted and re-launch, i.e. something like: +To remedy this, stop and re-launch the weave container, and re-attach +the application containers with `weave attach`. - host1# weave stop - host1# weave launch 10.0.0.1/16 - -After that, other containers may be re-attached if you can identify -them, like this: - - host1# weave attach 10.0.1.1/24 - -Alternatively, you may prefer to disable the auto-restart feature in -Docker as described [here](https://docs.docker.com/articles/host_integration/). +For a mor permanent solution, +[disable Docker's auto-restart feature](https://docs.docker.com/articles/host_integration/), +and create appropriate startup scripts to launch weave and run +application containers from your favourite process manager. ## Installation with Boot2Docker From 025315363d5ea8b8265f1b3ea800f24df2be51a4 Mon Sep 17 00:00:00 2001 From: Matthias Radestock Date: Tue, 23 Sep 2014 17:54:40 +0100 Subject: [PATCH 18/26] prevent CPU churn by setting long pcap timeout pcap timeouts are a mystery. Setting the timeout to -1 used to reduce latency, but that was before we started setting immediate mode. And it turns out that with that timeout, pcap chews through a lot of CPU polling its file descriptor, checking timers, etc. However, the behaviour of setting *no* timeout is defined as, err, undefined. So instead we set a very long timeout indeed. And loop if it expires. Fixes #80 --- connection_maker.go | 5 ++--- consts.go | 2 ++ pcap.go | 9 +++++++-- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/connection_maker.go b/connection_maker.go index 94064646..718b2fdc 100644 --- a/connection_maker.go +++ b/connection_maker.go @@ -4,7 +4,6 @@ import ( "bytes" "fmt" "log" - "math" "math/rand" "net" "time" @@ -59,7 +58,7 @@ func (cm *ConnectionMaker) String() string { } func (cm *ConnectionMaker) queryLoop(queryChan <-chan *ConnectionMakerInteraction) { - timer := time.NewTimer(math.MaxInt64) + timer := time.NewTimer(MaxDuration) run := func() { timer.Reset(cm.checkStateAndAttemptConnections()) } for { select { @@ -131,7 +130,7 @@ func (cm *ConnectionMaker) checkStateAndAttemptConnections() time.Duration { }) now := time.Now() // make sure we catch items just added - after := time.Duration(math.MaxInt64) + after := MaxDuration for address, target := range cm.targets { if our_connected_targets[address] { delete(cm.targets, address) diff --git a/consts.go b/consts.go index e90a27ad..06add6f9 100644 --- a/consts.go +++ b/consts.go @@ -2,6 +2,7 @@ package weave import ( "time" + "math" ) const ( @@ -23,6 +24,7 @@ const ( FragTestInterval = 5 * time.Minute PMTUVerifyAttempts = 8 PMTUVerifyTimeout = 10 * time.Millisecond // gets doubled with every attempt + MaxDuration = time.Duration(math.MaxInt64) ) const ( diff --git a/pcap.go b/pcap.go index 354ec818..bcca3edd 100644 --- a/pcap.go +++ b/pcap.go @@ -40,7 +40,7 @@ func newPcapIO(ifName string, promisc bool, snaplen int, bufSz int) (handle *Pca if err = inactive.SetSnapLen(snaplen); err != nil { return } - if err = inactive.SetTimeout(-1); err != nil { + if err = inactive.SetTimeout(MaxDuration); err != nil { return } if err = inactive.SetImmediateMode(true); err != nil { @@ -60,7 +60,12 @@ func newPcapIO(ifName string, promisc bool, snaplen int, bufSz int) (handle *Pca } func (pi *PcapIO) ReadPacket() (data []byte, err error) { - data, _, err = pi.handle.ZeroCopyReadPacketData() + for { + data, _, err = pi.handle.ZeroCopyReadPacketData() + if err == nil || err != pcap.NextErrorTimeoutExpired { + break + } + } return } From dc611fa0af793002e3f7057f18238754689d5d9e Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Wed, 24 Sep 2014 09:39:44 +0100 Subject: [PATCH 19/26] Refactor common code in attach/detach --- weaver/weave | 45 ++++++++++++++++----------------------------- 1 file changed, 16 insertions(+), 29 deletions(-) diff --git a/weaver/weave b/weaver/weave index 2870314e..f9853fc0 100755 --- a/weaver/weave +++ b/weaver/weave @@ -120,20 +120,30 @@ container_netns_preamble() { ln -s /proc/$NSPID/ns/net /var/run/netns/$NSPID LOCAL_IFNAME="v${CONTAINER_IFNAME}pl${NSPID}" GUEST_IFNAME="v${CONTAINER_IFNAME}pg${NSPID}" + EXITCODE=1 + IP_TMPFILE=/tmp/weave_ip_output_$$ + rm -f $IP_TMPFILE return 0 } -container_netns_cleanup() { +container_netns_postamble() { + if [ "$EXITCODE" -ne 0 ] ; then + if ! ps -p $DOCKERPID >/dev/null 2>&1 ; then + echo "Container $1 died" >&2 + else + echo "Failure during network configuration for container $1:" >&2 + cat $IP_TMPFILE >&2 + fi + fi + + rm -f $IP_TMPFILE rm -f /var/run/netns/$NSPID } # Parameters are container ID, CIDR address configure_container_networking() { container_netns_preamble $1 - EXITCODE=1 - IP_TMPFILE=/tmp/weave_ip_output_$$ - rm -f $IP_TMPFILE # If veth name already exists then assume we set it up already; # just add the IP address, if it's not already there @@ -161,17 +171,7 @@ configure_container_networking() { fi fi - if [ "$EXITCODE" -ne 0 ] ; then - if ! ps -p $DOCKERPID >/dev/null 2>&1 ; then - echo "Container $1 died" >&2 - else - echo "Failure during network configuration for container $1:" >&2 - cat $IP_TMPFILE >&2 - fi - fi - - rm -f $IP_TMPFILE - container_netns_cleanup + container_netns_postamble $1 return $EXITCODE } @@ -180,9 +180,6 @@ configure_container_networking() { detach() { container_netns_preamble $1 - EXITCODE=1 - IP_TMPFILE=/tmp/weave_ip_output_$$ - rm -f $IP_TMPFILE if (ip netns exec $NSPID ip addr list dev $CONTAINER_IFNAME | grep -qF $2) 2>$IP_TMPFILE ; then if ip netns exec $NSPID ip addr del $2 dev $CONTAINER_IFNAME 2>$IP_TMPFILE ; then # if no inet addresses remain then delete the veth @@ -198,17 +195,7 @@ detach() { EXITCODE=0 fi - if [ "$EXITCODE" -ne 0 ] ; then - if ! ps -p $DOCKERPID >/dev/null 2>&1 ; then - echo "Container $1 died" >&2 - else - echo "Failure while detaching address $2 from container $1" >&2 - cat $IP_TMPFILE >&2 - fi - fi - - rm -f $IP_TMPFILE - container_netns_cleanup + container_netns_postamble $1 return $EXITCODE } From d5acb312b432376aabf5632c49964960defa6433 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Wed, 24 Sep 2014 10:01:52 +0100 Subject: [PATCH 20/26] Rename 'configure_container_networking' to 'attach' --- weaver/weave | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/weaver/weave b/weaver/weave index f9853fc0..dc6dbc24 100755 --- a/weaver/weave +++ b/weaver/weave @@ -141,8 +141,9 @@ container_netns_postamble() { rm -f /var/run/netns/$NSPID } -# Parameters are container ID, CIDR address -configure_container_networking() { +# Attach Container-ID $1 to the weave network at CIDR address $2, +# creating a virtual ethernet device if necessary. +attach() { container_netns_preamble $1 # If veth name already exists then assume we set it up already; @@ -176,7 +177,8 @@ configure_container_networking() { return $EXITCODE } -# Parameters are container ID, CIDR address, interface-name +# Detach Container-ID $1 at CIDR address $2 from the weave network +# removing the virtual ethernet device if there are no more addresses detach() { container_netns_preamble $1 @@ -232,7 +234,7 @@ case "$COMMAND" in # restarts. MACADDR=`cat /sys/class/net/$BRIDGE/address` CONTAINER=$(docker run --privileged -d --name=weave -p 6783:6783/tcp -p 6783:6783/udp $WEAVE_DOCKER_ARGS zettio/weave -name $MACADDR "$@") - configure_container_networking $CONTAINER $IPADDR eth0 + attach $CONTAINER $IPADDR eth0 echo $CONTAINER ;; status) @@ -269,13 +271,13 @@ case "$COMMAND" in shift 1 create_bridge CONTAINER=$(docker run -d "$@") - configure_container_networking $CONTAINER $IPADDR $CONTAINER_IFNAME + attach $CONTAINER $IPADDR $CONTAINER_IFNAME echo $CONTAINER ;; attach) [ $# -gt 1 ] || usage create_bridge - configure_container_networking $2 $1 $CONTAINER_IFNAME + attach $2 $1 $CONTAINER_IFNAME ;; detach) [ $# -gt 1 ] || usage From a4ffb4e45ac9c599abd453314e92340d5e844788 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Wed, 24 Sep 2014 10:07:49 +0100 Subject: [PATCH 21/26] Revert "don't set pcap timeout, to prevent cpu churn" This reverts commit b5df59660fe83fd5a5d4d3ddd23e6b084ef56465. --- pcap.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pcap.go b/pcap.go index 912d0fba..354ec818 100644 --- a/pcap.go +++ b/pcap.go @@ -40,6 +40,9 @@ func newPcapIO(ifName string, promisc bool, snaplen int, bufSz int) (handle *Pca if err = inactive.SetSnapLen(snaplen); err != nil { return } + if err = inactive.SetTimeout(-1); err != nil { + return + } if err = inactive.SetImmediateMode(true); err != nil { return } From ad3a36686f69bf2a90b43c5f6c9bd77f55fb2698 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Wed, 24 Sep 2014 10:11:36 +0100 Subject: [PATCH 22/26] Cosmetic --- weaver/weave | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/weaver/weave b/weaver/weave index dc6dbc24..5d45c55e 100755 --- a/weaver/weave +++ b/weaver/weave @@ -118,9 +118,10 @@ container_netns_preamble() { [ ! -d /var/run/netns ] && mkdir -p /var/run/netns rm -f /var/run/netns/$NSPID ln -s /proc/$NSPID/ns/net /var/run/netns/$NSPID + + EXITCODE=1 LOCAL_IFNAME="v${CONTAINER_IFNAME}pl${NSPID}" GUEST_IFNAME="v${CONTAINER_IFNAME}pg${NSPID}" - EXITCODE=1 IP_TMPFILE=/tmp/weave_ip_output_$$ rm -f $IP_TMPFILE From 704c3dad67b6c6f9cbb4faee3e4992c07a32ca00 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Wed, 24 Sep 2014 10:17:45 +0100 Subject: [PATCH 23/26] Improved comment --- weaver/weave | 1 + 1 file changed, 1 insertion(+) diff --git a/weaver/weave b/weaver/weave index 5d45c55e..db6756f2 100755 --- a/weaver/weave +++ b/weaver/weave @@ -144,6 +144,7 @@ container_netns_postamble() { # Attach Container-ID $1 to the weave network at CIDR address $2, # creating a virtual ethernet device if necessary. +# Also turn off tx offload on device $3 inside the namespace. attach() { container_netns_preamble $1 From ebe8207f8adaf7fb7f73efa98ad36f47ac30ebe3 Mon Sep 17 00:00:00 2001 From: Matthias Radestock Date: Wed, 24 Sep 2014 12:23:51 +0100 Subject: [PATCH 24/26] typo --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a6a109e8..4b2cb115 100644 --- a/README.md +++ b/README.md @@ -442,8 +442,8 @@ function in this state. To remedy this, stop and re-launch the weave container, and re-attach the application containers with `weave attach`. -For a mor permanent solution, -[disable Docker's auto-restart feature](https://docs.docker.com/articles/host_integration/), +For a more permanent solution, +[disable Docker's auto-restart feature](https://docs.docker.com/articles/host_integration/) and create appropriate startup scripts to launch weave and run application containers from your favourite process manager. From 573ec220872a9b7020b5d6466e3fd1e396cdc746 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Wed, 24 Sep 2014 12:43:56 +0100 Subject: [PATCH 25/26] Fixed typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 23748005..e764013c 100644 --- a/README.md +++ b/README.md @@ -191,7 +191,7 @@ In some scenarios containers are started independently, e.g. via some existing tool chain, or require more complex startup sequences than provided by `weave run`. And sometimes the decision which application network a container should be part of is made post startup. For these -situation, weave allows an existing, running container to be attached +situations, weave allows an existing, running container to be attached to the weave network. To illustrate, we can achieve the same effect as the first example with From f6b8e9a6757d5211b560be76512500f3173dbdde Mon Sep 17 00:00:00 2001 From: Kai Davenport Date: Wed, 24 Sep 2014 14:19:40 -0700 Subject: [PATCH 26/26] change -passwd to -password in usage function --- weaver/weave | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weaver/weave b/weaver/weave index db6756f2..7c04fdd0 100755 --- a/weaver/weave +++ b/weaver/weave @@ -3,7 +3,7 @@ set -e usage() { echo "Usage:" - echo "weave launch / [-passwd ] ..." + echo "weave launch / [-password ] ..." echo "weave run / ..." echo "weave attach / " echo "weave detach / "