From 431197042d14570cb2e1e8e1fb70418e2142b4e4 Mon Sep 17 00:00:00 2001 From: David Wragg Date: Thu, 28 May 2015 10:37:46 +0100 Subject: [PATCH] Weave script changes to enable fast datapath This means: * The 'weave' netdev is an OVS datapath * The weave router container runs with --net=host in order to access the datapath. * The MTU is set to a smaller value, to allow vxlan encapsulation. Setting the WEAVE_NO_FASTDP environment variable disables these things, effectively running the router without fast datapath. It is an environment variable rather than an option to 'weave launch' because it takes effect in create_bridge, which is called from many subcommands, and it doesn't seem practical to add option processing to all of them. --- weave | 221 ++++++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 185 insertions(+), 36 deletions(-) diff --git a/weave b/weave index 6750df5f..554185b4 100755 --- a/weave +++ b/weave @@ -107,6 +107,8 @@ exec_remote() { -e WEAVE_PASSWORD \ -e WEAVE_PORT \ -e WEAVE_CONTAINER_NAME \ + -e WEAVE_MTU \ + -e WEAVE_NO_FASTDP \ -e DOCKER_BRIDGE \ -e DOCKER_CLIENT_HOST="$DOCKER_CLIENT_HOST" \ -e DOCKER_CLIENT_TLS_VERIFY="$DOCKER_CLIENT_TLS_VERIFY" \ @@ -263,7 +265,8 @@ DOCKER_BRIDGE=${DOCKER_BRIDGE:-docker0} CONTAINER_NAME=${WEAVE_CONTAINER_NAME:-weave} BRIDGE=weave CONTAINER_IFNAME=ethwe -MTU=65535 +# ROUTER_HOSTNETNS_IFNAME is only used for fastdp with encryption +ROUTER_HOSTNETNS_IFNAME=weave2 PORT=${WEAVE_PORT:-6783} HTTP_PORT=6784 PROXY_PORT=12375 @@ -366,33 +369,57 @@ random_mac() { ###################################################################### create_bridge() { + BRIDGE_TYPE= + if [ ! -d /sys/class/net/$BRIDGE ] ; then - ip link add name $BRIDGE type bridge - # Set a random MAC address on the bridge. Current Linux - # kernels already do this when creating a bridge, but there - # are rumours it was not always so. - ip link set dev $BRIDGE address $(random_mac) - # Attempting to set the bridge MTU to a high value directly - # fails. Bridges take the lowest MTU of their interfaces. So - # instead we create a temporary interface with the desired - # MTU, attach that to the bridge, and then remove it again. - ip link add name v${CONTAINER_IFNAME}du mtu $MTU type dummy - ip link set dev v${CONTAINER_IFNAME}du master $BRIDGE - ip link del dev v${CONTAINER_IFNAME}du - # Drop traffic from Docker bridge to Weave; it can break subnet isolation + if [ -n "$WEAVE_NO_FASTDP" ] ; then + BRIDGE_TYPE=bridge + elif docker run --rm --privileged --net=host $IMAGE $COVERAGE_ARGS --create-datapath --datapath=$BRIDGE ; then + BRIDGE_TYPE=fastdp + elif [ $? = 17 ] ; then + # Exit status of 17 means the kernel doesn't have ODP + BRIDGE_TYPE=bridge + else + return 1 + fi + + init_bridge_$BRIDGE_TYPE + + # Drop traffic from Docker bridge to Weave; it can break + # subnet isolation if [ "$DOCKER_BRIDGE" != "$BRIDGE" ] ; then # Note using -I to insert ahead of Docker's bridge rules run_iptables -t filter -I FORWARD -i $DOCKER_BRIDGE -o $BRIDGE -j DROP fi + # Work around the situation where there are no rules allowing traffic # across our bridge. E.g. ufw add_iptables_rule filter FORWARD -i $BRIDGE -o $BRIDGE -j ACCEPT + # create a chain for masquerading run_iptables -t nat -N WEAVE >/dev/null 2>&1 || true add_iptables_rule nat POSTROUTING -j WEAVE + else + # Detect whether fast datapath is in use on + # $BRIDGE. Unfortunately there's no simple way to positively + # check whether $BRIDGE is a ODP netdev, so we have to check + # whether it is a bridge instead. + if [ -d /sys/class/net/$BRIDGE/bridge ] ; then + BRIDGE_TYPE=bridge + else + BRIDGE_TYPE=fastdp + fi + + # WEAVE_MTU may have been specified when the bridge was + # created (perhaps implicitly with WEAVE_NO_FASTDP). So take + # the MTU from the bridge unless it is explicitly specified + # for this invocation. + MTU=${WEAVE_MTU:-$(cat /sys/class/net/$BRIDGE/mtu)} fi - [ "$1" != "--without-ethtool" ] && ethtool -K $BRIDGE tx off >/dev/null + if [ "$1" != "--without-ethtool" ] ; then + ethtool_tx_off_$BRIDGE_TYPE $BRIDGE + fi ip link set dev $BRIDGE up @@ -401,11 +428,58 @@ create_bridge() { configure_arp_cache $BRIDGE } +init_bridge_fastdp () { + # GCE has the lowest underlay network MTU we're likely to encounter on + # a local network, at 1460 bytes. To get the overlay MTU from that we + # subtract 20 bytes for the outer IPv4 header, 8 bytes for the outer + # UDP header, 8 bytes for the vxlan header, and 14 bytes for the inner + # ethernet header. + MTU=${WEAVE_MTU:-1410} + + # create_bridge already created the datapath netdev + ip link set dev $BRIDGE mtu $MTU +} + +init_bridge_bridge () { + MTU=${WEAVE_MTU:-65535} + + ip link add name $BRIDGE type bridge + + # Set a random MAC address on the bridge. Current Linux + # kernels already do this when creating a bridge, but there + # are rumours it was not always so. + ip link set dev $BRIDGE address $(random_mac) + + # Attempting to set the bridge MTU to a high value directly + # fails. Bridges take the lowest MTU of their interfaces. So + # instead we create a temporary interface with the desired + # MTU, attach that to the bridge, and then remove it again. + ip link add name v${CONTAINER_IFNAME}du mtu $MTU type dummy + ip link set dev v${CONTAINER_IFNAME}du master $BRIDGE + ip link del dev v${CONTAINER_IFNAME}du +} + +ethtool_tx_off_fastdp () { + true +} + +ethtool_tx_off_bridge () { + ethtool -K $1 tx off >/dev/null +} + destroy_bridge() { - [ -d /sys/class/net/$BRIDGE ] && ip link del dev $BRIDGE + if [ -d /sys/class/net/$BRIDGE ] ; then + if [ -d /sys/class/net/$BRIDGE/bridge ] ; then + ip link del dev $BRIDGE + else + docker run --rm --privileged --net=host $IMAGE $COVERAGE_ARGS --delete-datapath --datapath=$BRIDGE + fi + fi + if [ "$DOCKER_BRIDGE" != "$BRIDGE" ] ; then run_iptables -t filter -D FORWARD -i $DOCKER_BRIDGE -o $BRIDGE -j DROP 2>/dev/null || true fi + run_iptables -t filter -D FORWARD -i $BRIDGE -o $BRIDGE -j ACCEPT 2>/dev/null || true run_iptables -t nat -F WEAVE >/dev/null 2>&1 || true run_iptables -t nat -D POSTROUTING -j WEAVE >/dev/null 2>&1 || true @@ -474,17 +548,11 @@ netnsenter() { nsenter --net=$PROCFS/$CONTAINER_PID/ns/net "$@" } +# connect_container_to_bridge connect_container_to_bridge() { - if [ -h "$PROCFS/$CONTAINER_PID/ns/net" -a -h "/proc/$$/ns/net" -a "$(readlink $PROCFS/$CONTAINER_PID/ns/net)" = "$(readlink /proc/$$/ns/net)" ] ; then - echo "Container is running in the host network namespace, and therefore cannot be" >&2 - echo "connected to weave. Perhaps the container was started with --net=host." >&2 - return 1 - fi ip link add name $LOCAL_IFNAME mtu $MTU type veth peer name $GUEST_IFNAME mtu $MTU || return 1 - if ! ethtool -K $GUEST_IFNAME tx off >/dev/null || - ! ip link set $LOCAL_IFNAME master $BRIDGE || - ! ip link set $LOCAL_IFNAME up || + if ! ethtool_tx_off_$BRIDGE_TYPE $GUEST_IFNAME || ! ip link set $GUEST_IFNAME netns $PROCFS/$CONTAINER_PID/ns/net ; then # failed before we assigned the veth to the container's # namespace @@ -492,12 +560,30 @@ connect_container_to_bridge() { return 1 fi - if ! netnsenter ip link set $GUEST_IFNAME name $CONTAINER_IFNAME || - ! configure_arp_cache $CONTAINER_IFNAME "netnsenter" ; then + # Versions of the 'ip' command differ in whether they create a + # veth pair in the 'up' or 'down' state. Furthermore, moving one + # end into a netns normally sets them to 'down'. But, moving an + # end into the default netns is a null operation, so they stay up, + # which leads to errors later on. So to get a consistent result, + # we have to be explicit in setting them 'up' or 'down' as + # necessary. + if ! netnsenter ip link set $GUEST_IFNAME down || + ! netnsenter ip link set $GUEST_IFNAME name $1 up || + ! ip link set $LOCAL_IFNAME up || + ! add_iface_$BRIDGE_TYPE $LOCAL_IFNAME || + ! configure_arp_cache $1 "netnsenter" ; then return 1 fi } +add_iface_fastdp () { + docker run --rm --privileged --net=host $IMAGE $COVERAGE_ARGS --datapath=$BRIDGE --add-datapath-iface=$1 +} + +add_iface_bridge () { + ip link set $1 master $BRIDGE +} + ask_version() { if ! DOCKERIMAGE=$(docker inspect --format='{{.Image}}' $1 2>/dev/null) ; then if ! DOCKERIMAGE=$(docker inspect --format='{{.Id}}' $2 2>/dev/null) ; then @@ -507,23 +593,65 @@ ask_version() { [ -n "$DOCKERIMAGE" ] && docker run --rm -e WEAVE_CIDR=none $3 $DOCKERIMAGE $COVERAGE_ARGS --version } +router_opts_fastdp () { + if [ -z "$WEAVE_PASSWORD" ] ; then + echo "--datapath $BRIDGE" + else + # When using encryption, we still do bridging on the ODP + # datapath, because you can 'weave launch' without encryption + # and then later restart the router with encryption, or vice + # versa. Encryption disables the use of the fastdp Overlay, + # but the router could still use the fastdp Bridge to receive + # packets. However, pcap has better performance when sniffing + # every packet. So we pass --iface to use the pcap Bridge. + # + # Why don't we simply pass "--iface $BRIDGE". We could, + # except for the fact that NetworkManager likes to down the + # odp $BRIDGE netdev (at least under ubuntu), and you can only + # use pcap on an interface that is up. We avoid that by use + # pcap via a veth pair (NetworkManager leaves them alone). + # Having a netdev in the host netns called "ethwe" might + # surprise people, so it is called $ROUTER_HOSTNETNS_IFNAME + # instead. + echo "--datapath $BRIDGE --iface $ROUTER_HOSTNETNS_IFNAME" + fi +} + +router_opts_bridge () { + echo "--iface $CONTAINER_IFNAME" +} + ###################################################################### # functions invoked through with_container_netns ###################################################################### -launch() { +setup_router_iface_fastdp() { + if [ -n "$WEAVE_PASSWORD" ] ; then + # See router_opts_fastdp + connect_container_to_bridge $ROUTER_HOSTNETNS_IFNAME && + ip link set $ROUTER_HOSTNETNS_IFNAME up + fi +} + +setup_router_iface_bridge() { if ! netnsenter ip link show eth0 >/dev/null ; then echo "Perhaps you are running the docker daemon with container networking disabled (-b=none)." >&2 return 1 fi - connect_container_to_bridge && + connect_container_to_bridge $CONTAINER_IFNAME && netnsenter ethtool -K eth0 tx off >/dev/null && netnsenter ip link set $CONTAINER_IFNAME up } attach() { + if [ -h "$PROCFS/$CONTAINER_PID/ns/net" -a -h "/proc/$$/ns/net" -a "$(readlink $PROCFS/$CONTAINER_PID/ns/net)" = "$(readlink /proc/$$/ns/net)" ] ; then + echo "Container is running in the host network namespace, and therefore cannot be" >&2 + echo "connected to weave. Perhaps the container was started with --net=host." >&2 + return 1 + fi + if ! netnsenter ip link show $CONTAINER_IFNAME >/dev/null 2>&1 ; then - connect_container_to_bridge || return 1 + connect_container_to_bridge $CONTAINER_IFNAME || return 1 fi NEW_ADDRS= @@ -645,8 +773,8 @@ container_ip() { fi case "$status" in "true ") - echo "$1 container has no IP address; is Docker networking enabled?" >&2 - return 1 + #echo "$1 container has no IP address; is Docker networking enabled?" >&2 + CONTAINER_IP=127.0.0.1 ;; true*) CONTAINER_IP="${status#true }" @@ -1127,10 +1255,17 @@ launch_router() { shift 1 fi CONTAINER_PORT=$PORT - ARGS="" + ARGS= IPRANGE= IPRANGE_SPECIFIED= + + # DNS_PORT_MAPPING is only used if the router runs in a container + # netns. DNS_ROUTER_OPTS is only used if the router runs in the + # host netns. DNS_PORT_MAPPING="-p $DOCKER_BRIDGE_IP:53:53/udp -p $DOCKER_BRIDGE_IP:53:53/tcp" + DNS_ROUTER_OPTS="--dns-listen-address $DOCKER_BRIDGE_IP:53" + NO_DNS_OPT= + while [ $# -gt 0 ] ; do case "$1" in -password|--password) @@ -1163,6 +1298,8 @@ launch_router() { ;; --no-dns) DNS_PORT_MAPPING= + DNS_ROUTER_OPTS= + NO_DNS_OPT="--no-dns" ARGS="$ARGS $1" ;; *) @@ -1185,16 +1322,28 @@ launch_router() { echo "Unless this is deliberate, you must pick another range and set it on all hosts." >&2 fi fi + + if [ "$BRIDGE_TYPE" = fastdp ] ; then + NETHOST_OPT="--net=host" + fi + # Set WEAVE_DOCKER_ARGS in the environment in order to supply # additional parameters, such as resource limits, to docker # when launching the weave container. ROUTER_CONTAINER=$(docker run --privileged -d --name=$CONTAINER_NAME \ - -p $PORT:$CONTAINER_PORT/tcp -p $PORT:$CONTAINER_PORT/udp $DNS_PORT_MAPPING \ -v /var/run/docker.sock:/var/run/docker.sock \ + -p $PORT:$CONTAINER_PORT/tcp -p $PORT:$CONTAINER_PORT/udp \ + ${NETHOST_OPT:-$DNS_PORT_MAPPING} \ -e WEAVE_PASSWORD \ -e WEAVE_CIDR=none \ - $WEAVE_DOCKER_ARGS $IMAGE $COVERAGE_ARGS --iface $CONTAINER_IFNAME --port $CONTAINER_PORT --name "$PEERNAME" --nickname "$(hostname)" --ipalloc-range "$IPRANGE" --dns-effective-listen-address "$DOCKER_BRIDGE_IP" --docker-api "unix:///var/run/docker.sock" "$@") - with_container_netns_or_die $ROUTER_CONTAINER launch + $WEAVE_DOCKER_ARGS $IMAGE $COVERAGE_ARGS \ + --port $CONTAINER_PORT --name "$PEERNAME" --nickname "$(hostname)" \ + $(router_opts_$BRIDGE_TYPE) \ + --ipalloc-range "$IPRANGE" \ + --dns-effective-listen-address $DOCKER_BRIDGE_IP \ + ${NETHOST_OPT:+$DNS_ROUTER_OPTS} $NO_DNS_OPT \ + --docker-api "unix:///var/run/docker.sock" "$@") + with_container_netns_or_die $ROUTER_CONTAINER setup_router_iface_$BRIDGE_TYPE err_msg=$(death_msg $CONTAINER_NAME) container_ip $CONTAINER_NAME "$err_msg" "$err_msg" || return 1 wait_for_status $CONTAINER_NAME http_call $CONTAINER_IP:$HTTP_PORT @@ -1202,7 +1351,7 @@ launch_router() { # Tell the newly-started weave IP allocator about existing weave IPs with_container_addresses ipam_reclaim weave:expose $(docker ps -q --no-trunc) fi - if [ -n "$DNS_PORT_MAPPING" ] ; then + if [ -z "$NO_DNS_OPT" ] ; then # Tell the newly-started weaveDNS about existing weave IPs for CONTAINER in $(docker ps -q --no-trunc) ; do if CONTAINER_IPS=$(with_container_netns $CONTAINER container_weave_addrs 2>&1 | sed -n -e 's/inet \([^/]*\)\/\(.*\)/\1/p') ; then