From e8ed9c707d9f83bb322f79ab50824deeb81eaaf7 Mon Sep 17 00:00:00 2001 From: David Wragg Date: Fri, 13 Mar 2015 11:13:16 +0000 Subject: [PATCH 1/9] Refactor --- weave | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/weave b/weave index af0763c9..1e90266e 100755 --- a/weave +++ b/weave @@ -225,17 +225,22 @@ connect_container_to_bridge() { return 1 fi - if run_tool host ethtool -K $GUEST_IFNAME tx off >/dev/null && - ip link set $LOCAL_IFNAME master $BRIDGE && - ip link set $LOCAL_IFNAME up && - ip link set $GUEST_IFNAME netns $NETNS ; then - ip netns exec $NETNS ip link set $GUEST_IFNAME name $CONTAINER_IFNAME - else + if ! run_tool host ethtool -K $GUEST_IFNAME tx off >/dev/null || + ! ip link set $LOCAL_IFNAME master $BRIDGE || + ! ip link set $LOCAL_IFNAME up || + ! ip link set $GUEST_IFNAME netns $NETNS ; then # failed before we assigned the veth to the container's - # namespace - delete it + # namespace ip link del $LOCAL_IFNAME type veth || true return 1 fi + + if ! ip netns exec $NETNS ip link set $GUEST_IFNAME name $CONTAINER_IFNAME ; then + # failed after we assigned the veth to the container's + # namespace + ip netns exec $NETNS ip link del $GUEST_IFNAME type veth || true + return 1 + fi } launch() { From 11af5440710a563e5641547bbd6145d0bf705a64 Mon Sep 17 00:00:00 2001 From: David Wragg Date: Fri, 13 Mar 2015 17:45:26 +0000 Subject: [PATCH 2/9] Configure ARP cache to react more quickly to MAC address changes When an IP address was reused by a new container, other containers could experience delays of many seconds the first time they tried to talk to that new container before successful communication would occur. This delay came from three sources related to ARP cache behaviour: - The kernel would take a while to decide that an ARP mapping was stale due to lack of positive evidence (due to the base_reachable_time default of 30s) - Even once an ARP mapping was considered stale, the kernel would wait a few seconds before sending an ARP request, in the hope that a higher layer (e.g. TCP) would confirm the ARP entry (due to the delay_first_probe default of 5s) - When the kernel sent out an ARP request to update a stale entry, initial attempts would unicast the request to the old MAC address before resorting to broadcast (due to the ucast_solicit default of 3) This change reduces all these values, so that any delays are a few seconds. For most workloads, this should not greatly increase the amount of ARP traffic, because communication via TCP is often sufficient to confirm ARP entries. And one unicast ARP request is still sent just in case. Unfortunately, there's a fixed-but-only-recently bug in the kernel that means that changing base_reachable_time can take a while to take effect: . This means that a container will continue to use a long reachable_time delay for the first few minutes of its life. --- weave | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/weave b/weave index 1e90266e..4b93f736 100755 --- a/weave +++ b/weave @@ -148,6 +148,12 @@ create_bridge() { fi ip link set dev $BRIDGE up + + # Configure the ARP cache parameters on the bridge interface for + # the sake of 'weave expose' + echo 5 >/proc/sys/net/ipv4/neigh/$BRIDGE/base_reachable_time + echo 2 >/proc/sys/net/ipv4/neigh/$BRIDGE/delay_first_probe_time + echo 1 >/proc/sys/net/ipv4/neigh/$BRIDGE/ucast_solicit } destroy_bridge() { @@ -235,10 +241,15 @@ connect_container_to_bridge() { return 1 fi - if ! ip netns exec $NETNS ip link set $GUEST_IFNAME name $CONTAINER_IFNAME ; then + if ! ip netns exec $NETNS sh -c "\ + ip link set $GUEST_IFNAME name $CONTAINER_IFNAME && + echo 5 >/proc/sys/net/ipv4/neigh/$CONTAINER_IFNAME/base_reachable_time && + echo 2 >/proc/sys/net/ipv4/neigh/$CONTAINER_IFNAME/delay_first_probe_time && + echo 1 >/proc/sys/net/ipv4/neigh/$CONTAINER_IFNAME/ucast_solicit" ; then # failed after we assigned the veth to the container's - # namespace - ip netns exec $NETNS ip link del $GUEST_IFNAME type veth || true + # namespace, and we don't know if it was renamed or not. + ip netns exec $NETNS ip link del $GUEST_IFNAME type veth 2>/dev/null || true + ip netns exec $NETNS ip link del $CONTAINER_IFNAME type veth 2>/dev/null || true return 1 fi } From 7a8a7010e2e4dccba75c07ed2008189bd3afd28a Mon Sep 17 00:00:00 2001 From: David Wragg Date: Mon, 16 Mar 2015 15:10:36 +0000 Subject: [PATCH 3/9] Refactor for uniform handling of CIDRs Not quite a pure refactoring because it adds a missing 'validate_cidr'. --- weave | 46 ++++++++++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/weave b/weave index 4b93f736..8282f3e6 100755 --- a/weave +++ b/weave @@ -522,6 +522,7 @@ case "$COMMAND" in launch-dns) [ $# -gt 0 ] || usage CIDR=$1 + validate_cidr $CIDR shift 1 check_not_running $DNS_CONTAINER_NAME $BASE_DNS_IMAGE create_bridge @@ -589,8 +590,8 @@ case "$COMMAND" in esac; done fi - validate_cidr $1 CIDR=$1 + validate_cidr $CIDR shift 1 create_bridge CONTAINER=$(docker run $DNS_ARG $DNS_SEARCH_ARG -d "$@") @@ -600,46 +601,51 @@ case "$COMMAND" in ;; start) [ $# -eq 2 ] || usage - validate_cidr $1 + CIDR=$1 + validate_cidr $CIDR create_bridge CONTAINER=$(docker start $2) - with_container_netns $CONTAINER attach $1 >/dev/null - tell_dns PUT $CONTAINER $1 + with_container_netns $CONTAINER attach $CIDR >/dev/null + tell_dns PUT $CONTAINER $CIDR echo $CONTAINER ;; attach) [ $# -eq 2 ] || usage - validate_cidr $1 + CIDR=$1 + validate_cidr $CIDR create_bridge - with_container_netns $2 attach $1 >/dev/null - tell_dns PUT $2 $1 + with_container_netns $2 attach $CIDR >/dev/null + tell_dns PUT $2 $CIDR ;; detach) [ $# -eq 2 ] || usage - validate_cidr $1 - with_container_netns $2 detach $1 >/dev/null - tell_dns DELETE $2 $1 + CIDR=$1 + validate_cidr $CIDR + with_container_netns $2 detach $CIDR >/dev/null + tell_dns DELETE $2 $CIDR ;; expose) [ $# -eq 1 ] || usage - validate_cidr $1 + CIDR=$1 + validate_cidr $CIDR create_bridge --without-ethtool - if ! ip addr show dev $BRIDGE | grep -qF $1 + if ! ip addr show dev $BRIDGE | grep -qF $CIDR then - ip addr add dev $BRIDGE $1 - add_iptables_rule nat WEAVE -o $BRIDGE ! -s $1 -j MASQUERADE - add_iptables_rule nat WEAVE -s $1 ! -o $BRIDGE -j MASQUERADE + ip addr add dev $BRIDGE $CIDR + add_iptables_rule nat WEAVE -o $BRIDGE ! -s $CIDR -j MASQUERADE + add_iptables_rule nat WEAVE -s $CIDR ! -o $BRIDGE -j MASQUERADE fi ;; hide) [ $# -eq 1 ] || usage - validate_cidr $1 + CIDR=$1 + validate_cidr $CIDR create_bridge --without-ethtool - if ip addr show dev $BRIDGE | grep -qF $1 + if ip addr show dev $BRIDGE | grep -qF $CIDR then - ip addr del dev $BRIDGE $1 - delete_iptables_rule nat WEAVE -o $BRIDGE ! -s $1 -j MASQUERADE - delete_iptables_rule nat WEAVE -s $1 ! -o $BRIDGE -j MASQUERADE + ip addr del dev $BRIDGE $CIDR + delete_iptables_rule nat WEAVE -o $BRIDGE ! -s $CIDR -j MASQUERADE + delete_iptables_rule nat WEAVE -s $CIDR ! -o $BRIDGE -j MASQUERADE fi ;; stop) From 9b62bcb75dd3af78b8f680f8decd5beefddb46d2 Mon Sep 17 00:00:00 2001 From: David Wragg Date: Fri, 13 Mar 2015 18:17:26 +0000 Subject: [PATCH 4/9] Don't need to use sed to get the IP address part of a CIDR --- weave | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/weave b/weave index 8282f3e6..7613b3c7 100755 --- a/weave +++ b/weave @@ -353,9 +353,8 @@ tell_dns() { # get the long form of the container ID CONTAINER=$(docker inspect --format='{{.Id}}' $2 2>/dev/null) # extract IP address and routing prefix from CIDR - WEAVE_ADDR_IP=$(echo $3 | sed -e 's/\([^/]*\)\/\(.*\)/\1/') MORE_ARGS=$(docker inspect --format='--data-urlencode fqdn={{.Config.Hostname}}.{{.Config.Domainname}}.' $CONTAINER 2>/dev/null) && true - http_call $DNS_CONTAINER_NAME $DNS_HTTP_PORT $1 /name/$CONTAINER/$WEAVE_ADDR_IP $MORE_ARGS || true + http_call $DNS_CONTAINER_NAME $DNS_HTTP_PORT $1 /name/$CONTAINER/${3%/*} $MORE_ARGS || true } # Tell the newly-started weaveDNS about existing weave IPs From 78b0a57007d7e52beb39041c708665e0b506b722 Mon Sep 17 00:00:00 2001 From: David Wragg Date: Fri, 13 Mar 2015 19:09:35 +0000 Subject: [PATCH 5/9] Send an unsolicited ARP response when a container attaches Using arping, which is part of the iputils package and so seems to be ubiquitous. --- weave | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/weave b/weave index 7613b3c7..14d197b4 100755 --- a/weave +++ b/weave @@ -290,6 +290,17 @@ attach() { if ! ip netns exec $NETNS ip route show | grep '^224\.0\.0\.0/4' >/dev/null ; then ip netns exec $NETNS ip route add 224.0.0.0/4 dev $CONTAINER_IFNAME fi + + arp_update $CONTAINER_IFNAME $1 "ip netns exec $NETNS" +} + +# Send out an unsolicated ARP response to flush ARP cache entries +arp_update() { + # It's not the end of the world if this doesn't run - we configure + # ARP caches so that stale entries will be noticed quickly. + if command_exists arping ; then + $3 arping -U -q -I $1 -c 1 ${2%/*} + fi } detach() { @@ -631,6 +642,7 @@ case "$COMMAND" in if ! ip addr show dev $BRIDGE | grep -qF $CIDR then ip addr add dev $BRIDGE $CIDR + arp_update $BRIDGE $CIDR add_iptables_rule nat WEAVE -o $BRIDGE ! -s $CIDR -j MASQUERADE add_iptables_rule nat WEAVE -s $CIDR ! -o $BRIDGE -j MASQUERADE fi From e844753a21de99942fdc05d11ea2ebc3c909000c Mon Sep 17 00:00:00 2001 From: David Wragg Date: Mon, 16 Mar 2015 14:41:50 +0000 Subject: [PATCH 6/9] Don't duplicate ARP cache configuration --- weave | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/weave b/weave index 14d197b4..976cc4b9 100755 --- a/weave +++ b/weave @@ -151,9 +151,7 @@ create_bridge() { # Configure the ARP cache parameters on the bridge interface for # the sake of 'weave expose' - echo 5 >/proc/sys/net/ipv4/neigh/$BRIDGE/base_reachable_time - echo 2 >/proc/sys/net/ipv4/neigh/$BRIDGE/delay_first_probe_time - echo 1 >/proc/sys/net/ipv4/neigh/$BRIDGE/ucast_solicit + configure_arp_cache $BRIDGE } destroy_bridge() { @@ -241,11 +239,8 @@ connect_container_to_bridge() { return 1 fi - if ! ip netns exec $NETNS sh -c "\ - ip link set $GUEST_IFNAME name $CONTAINER_IFNAME && - echo 5 >/proc/sys/net/ipv4/neigh/$CONTAINER_IFNAME/base_reachable_time && - echo 2 >/proc/sys/net/ipv4/neigh/$CONTAINER_IFNAME/delay_first_probe_time && - echo 1 >/proc/sys/net/ipv4/neigh/$CONTAINER_IFNAME/ucast_solicit" ; then + if ! ip netns exec $NETNS ip link set $GUEST_IFNAME name $CONTAINER_IFNAME || + ! configure_arp_cache $CONTAINER_IFNAME "ip netns exec $NETNS" ; then # failed after we assigned the veth to the container's # namespace, and we don't know if it was renamed or not. ip netns exec $NETNS ip link del $GUEST_IFNAME type veth 2>/dev/null || true @@ -294,6 +289,13 @@ attach() { arp_update $CONTAINER_IFNAME $1 "ip netns exec $NETNS" } +# Configure the ARP cache parameters for the given interface +configure_arp_cache() { + $2 sh -c "echo 5 >/proc/sys/net/ipv4/neigh/$1/base_reachable_time && + echo 2 >/proc/sys/net/ipv4/neigh/$1/delay_first_probe_time && + echo 1 >/proc/sys/net/ipv4/neigh/$1/ucast_solicit" +} + # Send out an unsolicated ARP response to flush ARP cache entries arp_update() { # It's not the end of the world if this doesn't run - we configure From 39011a5218500ab5628d572a2b303ffec8bfa7c1 Mon Sep 17 00:00:00 2001 From: David Wragg Date: Mon, 16 Mar 2015 14:58:37 +0000 Subject: [PATCH 7/9] Don't clean up the container interface after moving it into the netns The most likely cause of failure is the container going away, in which. case we don't need to clean up. And there are other points of possible failure in the same category in attach(), so making sure we clean up the container interface under all conditions is a broader issue. --- weave | 4 ---- 1 file changed, 4 deletions(-) diff --git a/weave b/weave index 976cc4b9..dab90e71 100755 --- a/weave +++ b/weave @@ -241,10 +241,6 @@ connect_container_to_bridge() { if ! ip netns exec $NETNS ip link set $GUEST_IFNAME name $CONTAINER_IFNAME || ! configure_arp_cache $CONTAINER_IFNAME "ip netns exec $NETNS" ; then - # failed after we assigned the veth to the container's - # namespace, and we don't know if it was renamed or not. - ip netns exec $NETNS ip link del $GUEST_IFNAME type veth 2>/dev/null || true - ip netns exec $NETNS ip link del $CONTAINER_IFNAME type veth 2>/dev/null || true return 1 fi } From 4dcf77c210a2474bedef8e93a80029ba87827551 Mon Sep 17 00:00:00 2001 From: David Wragg Date: Mon, 16 Mar 2015 15:03:49 +0000 Subject: [PATCH 8/9] Commentary --- weave | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/weave b/weave index dab90e71..f3ad2e8f 100755 --- a/weave +++ b/weave @@ -285,14 +285,22 @@ attach() { arp_update $CONTAINER_IFNAME $1 "ip netns exec $NETNS" } -# Configure the ARP cache parameters for the given interface +# Configure the ARP cache parameters for the given interface. This +# makes containers react more quickly to a change in the MAC address +# associated with an IP address. configure_arp_cache() { $2 sh -c "echo 5 >/proc/sys/net/ipv4/neigh/$1/base_reachable_time && echo 2 >/proc/sys/net/ipv4/neigh/$1/delay_first_probe_time && echo 1 >/proc/sys/net/ipv4/neigh/$1/ucast_solicit" } -# Send out an unsolicated ARP response to flush ARP cache entries +# Send out an unsolicated ARP response to update ARP cache entries +# accross the weave network. We do this in addition to +# configure_arp_cache because a) with those ARP cache settings it +# still takes a few seconds to correct a stale ARP mapping, and b) +# there is a kernel bug that means that the base_reachable_time +# setting is not promptly obeyed +# (>). arp_update() { # It's not the end of the world if this doesn't run - we configure # ARP caches so that stale entries will be noticed quickly. From 2369b43c8dc332f173409be7854e33b156c7fd6a Mon Sep 17 00:00:00 2001 From: Matthias Radestock Date: Mon, 16 Mar 2015 15:33:05 +0000 Subject: [PATCH 9/9] cosmetic: move new funs to 'right' place --- weave | 48 ++++++++++++++++++++++++------------------------ 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/weave b/weave index f3ad2e8f..89763f0f 100755 --- a/weave +++ b/weave @@ -121,6 +121,30 @@ delete_iptables_rule() { fi } +# Configure the ARP cache parameters for the given interface. This +# makes containers react more quickly to a change in the MAC address +# associated with an IP address. +configure_arp_cache() { + $2 sh -c "echo 5 >/proc/sys/net/ipv4/neigh/$1/base_reachable_time && + echo 2 >/proc/sys/net/ipv4/neigh/$1/delay_first_probe_time && + echo 1 >/proc/sys/net/ipv4/neigh/$1/ucast_solicit" +} + +# Send out an unsolicated ARP response to update ARP cache entries +# accross the weave network. We do this in addition to +# configure_arp_cache because a) with those ARP cache settings it +# still takes a few seconds to correct a stale ARP mapping, and b) +# there is a kernel bug that means that the base_reachable_time +# setting is not promptly obeyed +# (>). +arp_update() { + # It's not the end of the world if this doesn't run - we configure + # ARP caches so that stale entries will be noticed quickly. + if command_exists arping ; then + $3 arping -U -q -I $1 -c 1 ${2%/*} + fi +} + create_bridge() { [ ! -d /sys/class/net/$BRIDGE ] && { ip link add name $BRIDGE type bridge @@ -285,30 +309,6 @@ attach() { arp_update $CONTAINER_IFNAME $1 "ip netns exec $NETNS" } -# Configure the ARP cache parameters for the given interface. This -# makes containers react more quickly to a change in the MAC address -# associated with an IP address. -configure_arp_cache() { - $2 sh -c "echo 5 >/proc/sys/net/ipv4/neigh/$1/base_reachable_time && - echo 2 >/proc/sys/net/ipv4/neigh/$1/delay_first_probe_time && - echo 1 >/proc/sys/net/ipv4/neigh/$1/ucast_solicit" -} - -# Send out an unsolicated ARP response to update ARP cache entries -# accross the weave network. We do this in addition to -# configure_arp_cache because a) with those ARP cache settings it -# still takes a few seconds to correct a stale ARP mapping, and b) -# there is a kernel bug that means that the base_reachable_time -# setting is not promptly obeyed -# (>). -arp_update() { - # It's not the end of the world if this doesn't run - we configure - # ARP caches so that stale entries will be noticed quickly. - if command_exists arping ; then - $3 arping -U -q -I $1 -c 1 ${2%/*} - fi -} - detach() { if ! ip netns exec $NETNS ip addr show dev $CONTAINER_IFNAME | grep -F $1 >/dev/null ; then # address is not there, leave the device alone