From 2b4d13c5b1cf95f40a4c522d875bc8715773ac4a Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Thu, 10 Nov 2016 16:58:52 +0000 Subject: [PATCH 1/6] Don't error on bridge address already there --- plugin/net/cni.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/plugin/net/cni.go b/plugin/net/cni.go index 1d9e4407..92e96c59 100644 --- a/plugin/net/cni.go +++ b/plugin/net/cni.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "net" + "syscall" "github.com/appc/cni/pkg/ipam" "github.com/appc/cni/pkg/skel" @@ -150,6 +151,11 @@ func assignBridgeIP(bridgeName string, ipnet net.IPNet) error { return err } if err := netlink.AddrAdd(link, &netlink.Addr{IPNet: &ipnet}); err != nil { + // Treat as non-error if this address is already there + // - maybe another copy of this program just added it + if err == syscall.Errno(syscall.EEXIST) { + return nil + } return fmt.Errorf("failed to add IP address to %q: %v", bridgeName, err) } return nil From a07e3301b1a8330f697acfdd591cd2b50d5d4144 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Thu, 10 Nov 2016 17:02:57 +0000 Subject: [PATCH 2/6] Add 'weave expose' NAT iptables rules inside CNI plugin This avoids a race between the plugin and the script --- net/veth.go | 18 ++++++++++++++++++ plugin/net/cni.go | 3 +++ prog/weave-kube/launch.sh | 3 --- 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/net/veth.go b/net/veth.go index f33d7b58..d0c81d0b 100644 --- a/net/veth.go +++ b/net/veth.go @@ -271,3 +271,21 @@ func subnets(addrs []netlink.Addr) map[string]struct{} { } return subnets } + +func ExposeNAT(ipnet net.IPNet) error { + ipt, err := iptables.New() + if err != nil { + return err + } + cidr := ipnet.String() + if err := ipt.AppendUnique("nat", "WEAVE", "-s", cidr, "-d", "224.0.0.0/4", "-j", "RETURN"); err != nil { + return err + } + if err := ipt.AppendUnique("nat", "WEAVE", "-d", cidr, "!", "-s", cidr, "-j", "MASQUERADE"); err != nil { + return err + } + if err := ipt.AppendUnique("nat", "WEAVE", "-s", cidr, "!", "-d", cidr, "-j", "MASQUERADE"); err != nil { + return err + } + return nil +} diff --git a/plugin/net/cni.go b/plugin/net/cni.go index 92e96c59..3145e49d 100644 --- a/plugin/net/cni.go +++ b/plugin/net/cni.go @@ -84,6 +84,9 @@ func (c *CNIPlugin) CmdAdd(args *skel.CmdArgs) error { if err := assignBridgeIP(conf.BrName, bridgeIPResult.IP4.IP); err != nil { return fmt.Errorf("unable to assign IP address to bridge: %s", err) } + if err := weavenet.ExposeNAT(bridgeIPResult.IP4.IP); err != nil { + return fmt.Errorf("unable to create NAT rules: %s", err) + } bridgeIP = bridgeIPResult.IP4.IP.IP } else if err != nil { return err diff --git a/prog/weave-kube/launch.sh b/prog/weave-kube/launch.sh index 7c82e8ef..0bb9dcd9 100755 --- a/prog/weave-kube/launch.sh +++ b/prog/weave-kube/launch.sh @@ -152,7 +152,4 @@ done reclaim_ips "_" $IPS done -# Expose the weave network so host processes can communicate with pods -/home/weave/weave --local expose $WEAVE_EXPOSE_IP - wait $WEAVE_PID From e458c11e669ab2f7da7f584831d666a5ef061d44 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Thu, 10 Nov 2016 19:42:43 +0000 Subject: [PATCH 3/6] Weave script calls Go implementation of expose_nat --- prog/weaveutil/expose.go | 23 +++++++++++++++++++++++ prog/weaveutil/main.go | 1 + weave | 10 +--------- 3 files changed, 25 insertions(+), 9 deletions(-) create mode 100644 prog/weaveutil/expose.go diff --git a/prog/weaveutil/expose.go b/prog/weaveutil/expose.go new file mode 100644 index 00000000..758285a7 --- /dev/null +++ b/prog/weaveutil/expose.go @@ -0,0 +1,23 @@ +package main + +import ( + weavenet "github.com/weaveworks/weave/net" +) + +func exposeNAT(args []string) error { + if len(args) < 1 { + cmdUsage("expose-nat", "...") + } + + cidrs, err := parseCIDRs(args) + if err != nil { + return err + } + + for _, cidr := range cidrs { + if err := weavenet.ExposeNAT(*cidr); err != nil { + return err + } + } + return nil +} diff --git a/prog/weaveutil/main.go b/prog/weaveutil/main.go index f8bcc500..b4dabc65 100644 --- a/prog/weaveutil/main.go +++ b/prog/weaveutil/main.go @@ -34,6 +34,7 @@ func init() { "list-netdevs": listNetDevs, "cni-net": cniNet, "cni-ipam": cniIPAM, + "expose-nat": exposeNAT, } } diff --git a/weave b/weave index 8e9247d3..3d24a630 100755 --- a/weave +++ b/weave @@ -661,14 +661,6 @@ expose_ip() { done } -expose_nat() { - for CIDR in $ALL_CIDRS ; do - add_iptables_rule nat WEAVE -s $CIDR -d 224.0.0.0/4 -j RETURN - add_iptables_rule nat WEAVE -d $CIDR ! -s $CIDR -j MASQUERADE - add_iptables_rule nat WEAVE -s $CIDR ! -d $CIDR -j MASQUERADE - done -} - # create veth with ends $1-$2, and then invoke $3..., removing the # veth on failure. No-op of veth already exists. create_veth() { @@ -2222,7 +2214,7 @@ EOF fi create_bridge --without-ethtool expose_ip - expose_nat + util_op expose-nat $ALL_CIDRS show_addrs $ALL_CIDRS ;; hide) From 10baab38d3101c0784031831ae1603b743faa0a5 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Fri, 11 Nov 2016 11:07:26 +0000 Subject: [PATCH 4/6] Extend smoke-test to include pinging --- test/840_weave_kube_3_test.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/840_weave_kube_3_test.sh b/test/840_weave_kube_3_test.sh index 14c31d95..9bfe5bec 100755 --- a/test/840_weave_kube_3_test.sh +++ b/test/840_weave_kube_3_test.sh @@ -44,6 +44,14 @@ wait_for_connections() { assert_raises wait_for_connections +# Check we can ping between the Weave bridg IPs on each host +HOST1EXPIP=$($SSH $HOST1 "weave expose") +HOST2EXPIP=$($SSH $HOST2 "weave expose") +HOST3EXPIP=$($SSH $HOST3 "weave expose") +assert_raises "run_on $HOST1 $PING $HOST2EXPIP" +assert_raises "run_on $HOST2 $PING $HOST1EXPIP" +assert_raises "run_on $HOST3 $PING $HOST2EXPIP" + # See if we can get some pods running that connect to the network run_on $HOST1 "kubectl run hello --image=weaveworks/hello-world --replicas=3" From 1629f4f3f00d494cae6d3aec11240bd9fbc4305c Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Fri, 11 Nov 2016 18:29:58 +0000 Subject: [PATCH 5/6] Loop round iptables commands on EAGAIN --- weave | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/weave b/weave index 3d24a630..dadd9b02 100755 --- a/weave +++ b/weave @@ -398,7 +398,13 @@ add_iptables_rule() { IPTABLES_TABLE="$1" shift 1 if ! run_iptables -t $IPTABLES_TABLE -C "$@" >/dev/null 2>&1 ; then - run_iptables -t $IPTABLES_TABLE -A "$@" >/dev/null + ## Loop until we get an exit code other than "temporarily unavailable" + while true ; do + run_iptables -t $IPTABLES_TABLE -A "$@" >/dev/null && return 0 + if [ $? != 4 ] ; then + return 1 + fi + done fi } @@ -407,7 +413,13 @@ insert_iptables_rule() { IPTABLES_TABLE="$1" shift 1 if ! run_iptables -t $IPTABLES_TABLE -C "$@" >/dev/null 2>&1 ; then - run_iptables -t $IPTABLES_TABLE -I "$@" >/dev/null + ## Loop until we get an exit code other than "temporarily unavailable" + while true ; do + run_iptables -t $IPTABLES_TABLE -I "$@" >/dev/null && return 0 + if [ $? != 4 ] ; then + return 1 + fi + done fi } From f8693074a36308deddc4b92bbfbbbe09ad537698 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Mon, 14 Nov 2016 10:00:37 +0000 Subject: [PATCH 6/6] Move CNI config file to the end of launch script So something waiting for the config (e.g. Kubernetes) doesn't try to talk to Weave before then (this only helps on first install, but that tends to be when things are busiest). --- prog/weave-kube/launch.sh | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/prog/weave-kube/launch.sh b/prog/weave-kube/launch.sh index 0bb9dcd9..ad7ddef9 100755 --- a/prog/weave-kube/launch.sh +++ b/prog/weave-kube/launch.sh @@ -18,17 +18,6 @@ fi echo 1 > /proc/sys/net/bridge/bridge-nf-call-iptables -# Create CNI config, if not already there -if [ ! -f /etc/cni/net.d/10-weave.conf ] ; then - mkdir -p /etc/cni/net.d - cat > /etc/cni/net.d/10-weave.conf < /etc/cni/net.d/10-weave.conf <