From 7c979d6725071c53ec0aaa9ac8ca3d190e9f37ed Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Wed, 17 Sep 2014 15:44:34 +0100 Subject: [PATCH 01/15] Remove veth and/or netns symlink if interface configuration fails part-way through --- weaver/weave | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/weaver/weave b/weaver/weave index 8ba2b3ac..4e76309a 100755 --- a/weaver/weave +++ b/weaver/weave @@ -117,19 +117,28 @@ configure_container_networking() { [ -f /var/run/netns/$NSPID ] && rm -f /var/run/netns/$NSPID ln -s /proc/$NSPID/ns/net /var/run/netns/$NSPID + EXITCODE=1 LOCAL_IFNAME="v${CONTAINER_IFNAME}pl${NSPID}" GUEST_IFNAME="v${CONTAINER_IFNAME}pg${NSPID}" - ip link add name $LOCAL_IFNAME mtu $MTU type veth peer name $GUEST_IFNAME mtu $MTU - ip link set $LOCAL_IFNAME master $BRIDGE - ip link set $LOCAL_IFNAME up - ip link set $GUEST_IFNAME netns $NSPID - ip netns exec $NSPID ip link set $GUEST_IFNAME name $CONTAINER_IFNAME - ip netns exec $NSPID ip addr add $2 dev $CONTAINER_IFNAME - ip netns exec $NSPID ip link set $CONTAINER_IFNAME up - - check_command_and_run ethtool ip netns exec $NSPID ethtool -K $3 tx off >/dev/null + if ip link add name $LOCAL_IFNAME mtu $MTU type veth peer name $GUEST_IFNAME mtu $MTU ; then + if ip link set $LOCAL_IFNAME master $BRIDGE && \ + ip link set $LOCAL_IFNAME up && \ + ip link set $GUEST_IFNAME netns $NSPID + then + ip netns exec $NSPID ip link set $GUEST_IFNAME name $CONTAINER_IFNAME && \ + ip netns exec $NSPID ip addr add $2 dev $CONTAINER_IFNAME && \ + ip netns exec $NSPID ip link set $CONTAINER_IFNAME up && \ + $(check_command_and_run ethtool ip netns exec $NSPID ethtool -K $3 tx off >/dev/null) && \ + EXITCODE=0 + else + # failed before we assigned the veth to the container's namespace - delete it + ip link delete $LOCAL_IFNAME type veth || true + fi + fi [ -f /var/run/netns/$NSPID ] && rm -f /var/run/netns/$NSPID + + exit $EXITCODE } case "$COMMAND" in From fd7b0347131af66064271b1118aaf67767436e6f Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Wed, 17 Sep 2014 16:11:30 +0100 Subject: [PATCH 02/15] untabify --- weaver/weave | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/weaver/weave b/weaver/weave index 4e76309a..d0598469 100755 --- a/weaver/weave +++ b/weaver/weave @@ -121,24 +121,24 @@ configure_container_networking() { LOCAL_IFNAME="v${CONTAINER_IFNAME}pl${NSPID}" GUEST_IFNAME="v${CONTAINER_IFNAME}pg${NSPID}" if ip link add name $LOCAL_IFNAME mtu $MTU type veth peer name $GUEST_IFNAME mtu $MTU ; then - if ip link set $LOCAL_IFNAME master $BRIDGE && \ - ip link set $LOCAL_IFNAME up && \ - ip link set $GUEST_IFNAME netns $NSPID - then - ip netns exec $NSPID ip link set $GUEST_IFNAME name $CONTAINER_IFNAME && \ - ip netns exec $NSPID ip addr add $2 dev $CONTAINER_IFNAME && \ - ip netns exec $NSPID ip link set $CONTAINER_IFNAME up && \ - $(check_command_and_run ethtool ip netns exec $NSPID ethtool -K $3 tx off >/dev/null) && \ - EXITCODE=0 - else - # failed before we assigned the veth to the container's namespace - delete it - ip link delete $LOCAL_IFNAME type veth || true - fi + if ip link set $LOCAL_IFNAME master $BRIDGE && \ + ip link set $LOCAL_IFNAME up && \ + ip link set $GUEST_IFNAME netns $NSPID + then + ip netns exec $NSPID ip link set $GUEST_IFNAME name $CONTAINER_IFNAME && \ + ip netns exec $NSPID ip addr add $2 dev $CONTAINER_IFNAME && \ + ip netns exec $NSPID ip link set $CONTAINER_IFNAME up && \ + $(check_command_and_run ethtool ip netns exec $NSPID ethtool -K $3 tx off >/dev/null) && \ + EXITCODE=0 + else + # failed before we assigned the veth to the container's namespace - delete it + ip link delete $LOCAL_IFNAME type veth || true + fi fi [ -f /var/run/netns/$NSPID ] && rm -f /var/run/netns/$NSPID - exit $EXITCODE + exit $EXITCODE } case "$COMMAND" in From eabcc55f7368797c272e3f6795e7c08613a61c5c Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Wed, 17 Sep 2014 16:25:46 +0100 Subject: [PATCH 03/15] fix subshell syntax and make del/delete consistent --- weaver/weave | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/weaver/weave b/weaver/weave index d0598469..fcb5a492 100755 --- a/weaver/weave +++ b/weaver/weave @@ -128,11 +128,11 @@ configure_container_networking() { ip netns exec $NSPID ip link set $GUEST_IFNAME name $CONTAINER_IFNAME && \ ip netns exec $NSPID ip addr add $2 dev $CONTAINER_IFNAME && \ ip netns exec $NSPID ip link set $CONTAINER_IFNAME up && \ - $(check_command_and_run ethtool ip netns exec $NSPID ethtool -K $3 tx off >/dev/null) && \ + (check_command_and_run ethtool ip netns exec $NSPID ethtool -K $3 tx off >/dev/null) && \ EXITCODE=0 else # failed before we assigned the veth to the container's namespace - delete it - ip link delete $LOCAL_IFNAME type veth || true + ip link del $LOCAL_IFNAME type veth || true fi fi From 09aed84638fb0561c7e2e7b43bff55097ca1bcfe Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Wed, 17 Sep 2014 16:30:50 +0100 Subject: [PATCH 04/15] return not exit from function --- weaver/weave | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/weaver/weave b/weaver/weave index fcb5a492..b622b07c 100755 --- a/weaver/weave +++ b/weaver/weave @@ -1,5 +1,6 @@ #!/bin/sh set -e +set -x usage() { echo "Usage:" @@ -138,7 +139,7 @@ configure_container_networking() { [ -f /var/run/netns/$NSPID ] && rm -f /var/run/netns/$NSPID - exit $EXITCODE + return $EXITCODE } case "$COMMAND" in From 0a403dd0818d3481772a8858864e2087199a5f01 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Wed, 17 Sep 2014 16:31:52 +0100 Subject: [PATCH 05/15] untabify --- weaver/weave | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weaver/weave b/weaver/weave index b622b07c..c54a6d5d 100755 --- a/weaver/weave +++ b/weaver/weave @@ -118,7 +118,7 @@ configure_container_networking() { [ -f /var/run/netns/$NSPID ] && rm -f /var/run/netns/$NSPID ln -s /proc/$NSPID/ns/net /var/run/netns/$NSPID - EXITCODE=1 + EXITCODE=1 LOCAL_IFNAME="v${CONTAINER_IFNAME}pl${NSPID}" GUEST_IFNAME="v${CONTAINER_IFNAME}pg${NSPID}" if ip link add name $LOCAL_IFNAME mtu $MTU type veth peer name $GUEST_IFNAME mtu $MTU ; then From 2f5a9733b8d3b302b7c552592529df30ea0e4beb Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Wed, 17 Sep 2014 16:32:21 +0100 Subject: [PATCH 06/15] Take out debugging -x mode --- weaver/weave | 1 - 1 file changed, 1 deletion(-) diff --git a/weaver/weave b/weaver/weave index c54a6d5d..b7fa02d7 100755 --- a/weaver/weave +++ b/weaver/weave @@ -1,6 +1,5 @@ #!/bin/sh set -e -set -x usage() { echo "Usage:" From f5d9feac95b7f16dca8aa2090f7638c5eca7d6f6 Mon Sep 17 00:00:00 2001 From: Matthias Radestock Date: Wed, 17 Sep 2014 19:24:32 +0100 Subject: [PATCH 07/15] note that local traffic isn't captured --- README.md | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index b5cb0d8d..8769081c 100644 --- a/README.md +++ b/README.md @@ -483,10 +483,14 @@ connected to the bridge is the weave router container, which is given the IP address & netmask supplied in 'weave launch'. A weave router captures Ethernet packets from its bridge-connected -interface in promiscuous mode, using 'pcap'. It forwards these packets -over UDP to weave router peers running on other hosts. On receipt of -such a packet, a router injects the packet on its bridge interface -using 'pcap' and/or forwards the packet to peers. +interface in promiscuous mode, using 'pcap'. This typically excludes +traffic between local containers, or between the host and local +containers, which is routed straight over the bridge by the +kernel. Captured packets are forwarded over UDP to weave router peers +running on other hosts. On receipt of such a packet, a router injects +the packet on its bridge interface using 'pcap' and/or forwards the +packet to peers. + Weave routers learn which peer host a particular MAC address resides on. They combine this knowledge with topology information in order to From ce1d40afb89baafd272247097f74c6508016f7e4 Mon Sep 17 00:00:00 2001 From: Matthias Radestock Date: Wed, 17 Sep 2014 19:25:24 +0100 Subject: [PATCH 08/15] cosmetic --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8769081c..475ec6b2 100644 --- a/README.md +++ b/README.md @@ -484,8 +484,8 @@ the IP address & netmask supplied in 'weave launch'. A weave router captures Ethernet packets from its bridge-connected interface in promiscuous mode, using 'pcap'. This typically excludes -traffic between local containers, or between the host and local -containers, which is routed straight over the bridge by the +traffic between local containers, and between the host and local +containers, all of which is routed straight over the bridge by the kernel. Captured packets are forwarded over UDP to weave router peers running on other hosts. On receipt of such a packet, a router injects the packet on its bridge interface using 'pcap' and/or forwards the From 12b969004ec74584a8aae6e4c2d01eddfff6be16 Mon Sep 17 00:00:00 2001 From: David Wragg Date: Wed, 17 Sep 2014 23:42:37 +0100 Subject: [PATCH 09/15] Filter outbound packets in the kernel It turns out that under Linux, libpcap implements the SetDirection filtering in userspace, causing some spurious wake ups of the packet capture thread. So set a BPF filter to discard outbound packets inside the kernel. --- pcap.go | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/pcap.go b/pcap.go index 2a326d16..354ec818 100644 --- a/pcap.go +++ b/pcap.go @@ -8,9 +8,19 @@ type PcapIO struct { handle *pcap.Handle } -func NewPcapIO(ifName string, bufSz int) (pio PacketSourceSink, err error) { - pio, err = newPcapIO(ifName, true, 65535, bufSz) - return +func NewPcapIO(ifName string, bufSz int) (PacketSourceSink, error) { + pio, err := newPcapIO(ifName, true, 65535, bufSz) + if err != nil { + return pio, err + } + + // Under Linux, libpcap implements the SetDirection filtering + // in userspace. So set a BPF filter to discard outbound + // packets inside the kernel. We do this here rather than in + // newPcapIO because libpcap doesn't like this in combination + // with a 0 snaplen. + err = pio.handle.SetBPFFilter("inbound") + return pio, err } func NewPcapO(ifName string) (po PacketSink, err error) { From bfd500733912bbaa55594d1951f52a369ca617d7 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Thu, 18 Sep 2014 11:03:06 +0100 Subject: [PATCH 10/15] Don't test if symlink exists before 'rm -f' When the symlink's target doesn't exist, '[ -f' is false, so the symlink doesn't get removed. Since we do always want it to be removed, just call 'rm -f' anyway, which won't complain if the file doesn't exist. --- weaver/weave | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/weaver/weave b/weaver/weave index b7fa02d7..7aa4d14d 100755 --- a/weaver/weave +++ b/weaver/weave @@ -114,7 +114,7 @@ configure_container_networking() { NSPID=$DOCKERPID [ ! -d /var/run/netns ] && mkdir -p /var/run/netns - [ -f /var/run/netns/$NSPID ] && rm -f /var/run/netns/$NSPID + rm -f /var/run/netns/$NSPID ln -s /proc/$NSPID/ns/net /var/run/netns/$NSPID EXITCODE=1 @@ -136,7 +136,7 @@ configure_container_networking() { fi fi - [ -f /var/run/netns/$NSPID ] && rm -f /var/run/netns/$NSPID + rm -f /var/run/netns/$NSPID return $EXITCODE } From a75a0d870947f3b7d207e01edf6e78d920f492e3 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Thu, 18 Sep 2014 11:24:08 +0100 Subject: [PATCH 11/15] If and only if network setup because the container died, say so. This involves capturing stderr output from all the 'ip' commands so we can hide that output if we see the process died, or show it to the user if the process didn't die, in which case we don't know why we failed. --- weaver/weave | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/weaver/weave b/weaver/weave index 7aa4d14d..8da45d72 100755 --- a/weaver/weave +++ b/weaver/weave @@ -120,15 +120,16 @@ configure_container_networking() { EXITCODE=1 LOCAL_IFNAME="v${CONTAINER_IFNAME}pl${NSPID}" GUEST_IFNAME="v${CONTAINER_IFNAME}pg${NSPID}" - if ip link add name $LOCAL_IFNAME mtu $MTU type veth peer name $GUEST_IFNAME mtu $MTU ; then - if ip link set $LOCAL_IFNAME master $BRIDGE && \ - ip link set $LOCAL_IFNAME up && \ - ip link set $GUEST_IFNAME netns $NSPID + IP_TMPFILE=/tmp/weave_ip_output_$$ + if ip link add name $LOCAL_IFNAME mtu $MTU type veth peer name $GUEST_IFNAME mtu $MTU > $IP_TMPFILE 2>&1 ; then + if (ip link set $LOCAL_IFNAME master $BRIDGE && + ip link set $LOCAL_IFNAME up && + ip link set $GUEST_IFNAME netns $NSPID) > $IP_TMPFILE 2>&1 then - ip netns exec $NSPID ip link set $GUEST_IFNAME name $CONTAINER_IFNAME && \ - ip netns exec $NSPID ip addr add $2 dev $CONTAINER_IFNAME && \ - ip netns exec $NSPID ip link set $CONTAINER_IFNAME up && \ - (check_command_and_run ethtool ip netns exec $NSPID ethtool -K $3 tx off >/dev/null) && \ + (ip netns exec $NSPID ip link set $GUEST_IFNAME name $CONTAINER_IFNAME && + ip netns exec $NSPID ip addr add $2 dev $CONTAINER_IFNAME && + ip netns exec $NSPID ip link set $CONTAINER_IFNAME up && + check_command_and_run ethtool ip netns exec $NSPID ethtool -K $3 tx off) > $IP_TMPFILE 2>&1 && EXITCODE=0 else # failed before we assigned the veth to the container's namespace - delete it @@ -136,6 +137,16 @@ configure_container_networking() { fi fi + if [ "$EXITCODE" -ne 0 ] ; then + if ! ps -p $DOCKERPID > /dev/null 2>&1 ; then + echo "Container died during network set-up. ID $1" >&2 + else + echo "Failure during network set-up:" >&2 + cat $IP_TMPFILE + fi + fi + + rm -f $IP_TMPFILE rm -f /var/run/netns/$NSPID return $EXITCODE From c48ae19c5523129bd787d2b6186b91a05b7bdc4c Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Thu, 18 Sep 2014 11:47:47 +0100 Subject: [PATCH 12/15] Tidy up messages; ensure errors to to stderr --- weaver/weave | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/weaver/weave b/weaver/weave index 8da45d72..9b1d8b83 100755 --- a/weaver/weave +++ b/weaver/weave @@ -129,7 +129,7 @@ configure_container_networking() { (ip netns exec $NSPID ip link set $GUEST_IFNAME name $CONTAINER_IFNAME && ip netns exec $NSPID ip addr add $2 dev $CONTAINER_IFNAME && ip netns exec $NSPID ip link set $CONTAINER_IFNAME up && - check_command_and_run ethtool ip netns exec $NSPID ethtool -K $3 tx off) > $IP_TMPFILE 2>&1 && + check_command_and_run ethtool ip netns exec $NSPID ethtool -K $3 tx off > /dev/null) > $IP_TMPFILE 2>&1 && EXITCODE=0 else # failed before we assigned the veth to the container's namespace - delete it @@ -139,10 +139,10 @@ configure_container_networking() { if [ "$EXITCODE" -ne 0 ] ; then if ! ps -p $DOCKERPID > /dev/null 2>&1 ; then - echo "Container died during network set-up. ID $1" >&2 + echo "Container $1 died" >&2 else echo "Failure during network set-up:" >&2 - cat $IP_TMPFILE + cat $IP_TMPFILE >&2 fi fi From 86644f7615ad22c3853b0b0b75c83b8af02d7829 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Thu, 18 Sep 2014 11:55:47 +0100 Subject: [PATCH 13/15] cosmetic --- weaver/weave | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/weaver/weave b/weaver/weave index 9b1d8b83..0d4e1fb3 100755 --- a/weaver/weave +++ b/weaver/weave @@ -121,15 +121,15 @@ configure_container_networking() { LOCAL_IFNAME="v${CONTAINER_IFNAME}pl${NSPID}" GUEST_IFNAME="v${CONTAINER_IFNAME}pg${NSPID}" IP_TMPFILE=/tmp/weave_ip_output_$$ - if ip link add name $LOCAL_IFNAME mtu $MTU type veth peer name $GUEST_IFNAME mtu $MTU > $IP_TMPFILE 2>&1 ; then + if ip link add name $LOCAL_IFNAME mtu $MTU type veth peer name $GUEST_IFNAME mtu $MTU >$IP_TMPFILE 2>&1 ; then if (ip link set $LOCAL_IFNAME master $BRIDGE && ip link set $LOCAL_IFNAME up && - ip link set $GUEST_IFNAME netns $NSPID) > $IP_TMPFILE 2>&1 + ip link set $GUEST_IFNAME netns $NSPID) >$IP_TMPFILE 2>&1 then (ip netns exec $NSPID ip link set $GUEST_IFNAME name $CONTAINER_IFNAME && ip netns exec $NSPID ip addr add $2 dev $CONTAINER_IFNAME && ip netns exec $NSPID ip link set $CONTAINER_IFNAME up && - check_command_and_run ethtool ip netns exec $NSPID ethtool -K $3 tx off > /dev/null) > $IP_TMPFILE 2>&1 && + check_command_and_run ethtool ip netns exec $NSPID ethtool -K $3 tx off >/dev/null) >$IP_TMPFILE 2>&1 && EXITCODE=0 else # failed before we assigned the veth to the container's namespace - delete it @@ -138,7 +138,7 @@ configure_container_networking() { fi if [ "$EXITCODE" -ne 0 ] ; then - if ! ps -p $DOCKERPID > /dev/null 2>&1 ; then + if ! ps -p $DOCKERPID >/dev/null 2>&1 ; then echo "Container $1 died" >&2 else echo "Failure during network set-up:" >&2 From 0ed57bad674b47392099c8f3fa965a3c7af66225 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Thu, 18 Sep 2014 11:56:15 +0100 Subject: [PATCH 14/15] rm tempfile before we use it, to avoid attack via symlink --- weaver/weave | 1 + 1 file changed, 1 insertion(+) diff --git a/weaver/weave b/weaver/weave index 0d4e1fb3..1c52da6c 100755 --- a/weaver/weave +++ b/weaver/weave @@ -121,6 +121,7 @@ configure_container_networking() { LOCAL_IFNAME="v${CONTAINER_IFNAME}pl${NSPID}" GUEST_IFNAME="v${CONTAINER_IFNAME}pg${NSPID}" IP_TMPFILE=/tmp/weave_ip_output_$$ + rm -f $IP_TMPFILE if ip link add name $LOCAL_IFNAME mtu $MTU type veth peer name $GUEST_IFNAME mtu $MTU >$IP_TMPFILE 2>&1 ; then if (ip link set $LOCAL_IFNAME master $BRIDGE && ip link set $LOCAL_IFNAME up && From 331c8553dbc6d75b7e0521597c9028068c1b9301 Mon Sep 17 00:00:00 2001 From: Matthias Radestock Date: Thu, 18 Sep 2014 12:30:43 +0100 Subject: [PATCH 15/15] cosmetic --- weaver/weave | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/weaver/weave b/weaver/weave index 1c52da6c..e673f9f9 100755 --- a/weaver/weave +++ b/weaver/weave @@ -124,13 +124,13 @@ configure_container_networking() { rm -f $IP_TMPFILE if ip link add name $LOCAL_IFNAME mtu $MTU type veth peer name $GUEST_IFNAME mtu $MTU >$IP_TMPFILE 2>&1 ; then if (ip link set $LOCAL_IFNAME master $BRIDGE && - ip link set $LOCAL_IFNAME up && - ip link set $GUEST_IFNAME netns $NSPID) >$IP_TMPFILE 2>&1 + ip link set $LOCAL_IFNAME up && + ip link set $GUEST_IFNAME netns $NSPID) >$IP_TMPFILE 2>&1 then (ip netns exec $NSPID ip link set $GUEST_IFNAME name $CONTAINER_IFNAME && - ip netns exec $NSPID ip addr add $2 dev $CONTAINER_IFNAME && - ip netns exec $NSPID ip link set $CONTAINER_IFNAME up && - check_command_and_run ethtool ip netns exec $NSPID ethtool -K $3 tx off >/dev/null) >$IP_TMPFILE 2>&1 && + ip netns exec $NSPID ip addr add $2 dev $CONTAINER_IFNAME && + ip netns exec $NSPID ip link set $CONTAINER_IFNAME up && + check_command_and_run ethtool ip netns exec $NSPID ethtool -K $3 tx off >/dev/null) >$IP_TMPFILE 2>&1 && EXITCODE=0 else # failed before we assigned the veth to the container's namespace - delete it @@ -142,7 +142,7 @@ configure_container_networking() { if ! ps -p $DOCKERPID >/dev/null 2>&1 ; then echo "Container $1 died" >&2 else - echo "Failure during network set-up:" >&2 + echo "Failure during network configuration for container $1:" >&2 cat $IP_TMPFILE >&2 fi fi