Add release v1.8.2

Detect looping flows

CHANGELOG.md

@@ -1,3 +1,22 @@
+## Release 1.8.2
+
+Bug fixes and minor improvements
+
+* Fixed a bug where looping flows were installed which caused high CPU
+  usage #2650, #2674
+* Fixed a bug where Kubernetes master could not contact pods #2673, #2683
+* Fixed a bug where weave-kube was crashing in a loop due to invalid
+  Weave bridge state #2657
+* Fixed a bug where iptables NAT rules were not appended due to
+  "temporary unavailable" iptables error #2679
+* Added a detection of enabling the hairpin mode on the Weave bridge port
+  which caused installation of looping flows #2674
+* Added a detection of overlaps between Weave and the host IP address
+  ranges when launching weave-kube #2669, #2672
+* Added logging of connections blocked by weave-npc #2546, #2573
+
+More details in the [change log](https://github.com/weaveworks/weave/issues?q=milestone%3A1.8.2).
+
 ## Release 1.8.1
 
 Bug fixes and minor improvements

net/bridge.go

@@ -1,6 +1,9 @@
 package net
 
 import (
+	"fmt"
+
+	"github.com/Sirupsen/logrus"
 	"github.com/vishvananda/netlink"
 )
 
@@ -51,3 +54,35 @@ func isDatapath(link netlink.Link) bool {
 		return false
 	}
 }
+
+func DetectHairpin(portIfName string, log *logrus.Logger) error {
+	link, err := netlink.LinkByName(portIfName)
+	if err != nil {
+		return fmt.Errorf("Unable to find link %q: %s", portIfName, err)
+	}
+
+	ch := make(chan netlink.LinkUpdate)
+	// See EnsureInterface for why done channel is not passed
+	if err := netlink.LinkSubscribe(ch, nil); err != nil {
+		return fmt.Errorf("Unable to subscribe to netlink updates: %s", err)
+	}
+
+	pi, err := netlink.LinkGetProtinfo(link)
+	if err != nil {
+		return fmt.Errorf("Unable to get link protinfo %q: %s", portIfName, err)
+	}
+	if pi.Hairpin {
+		return fmt.Errorf("Hairpin mode enabled on %q", portIfName)
+	}
+
+	go func() {
+		for up := range ch {
+			if up.Attrs().Name == portIfName && up.Attrs().Protinfo != nil &&
+				up.Attrs().Protinfo.Hairpin {
+				log.Errorf("Hairpin mode enabled on %q", portIfName)
+			}
+		}
+	}()
+
+	return nil
+}

net/veth.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"net"
 	"strings"
+	"syscall"
 	"time"
 
 	"github.com/coreos/go-iptables/iptables"
@@ -272,19 +273,37 @@ func subnets(addrs []netlink.Addr) map[string]struct{} {
 	return subnets
 }
 
+func addNatRule(ipt *iptables.IPTables, rulespec ...string) error {
+	// Loop until we get an exit code other than "temporarily unavailable"
+	for {
+		if err := ipt.AppendUnique("nat", "WEAVE", rulespec...); err != nil {
+			if ierr, ok := err.(*iptables.Error); ok {
+				if status, ok := ierr.ExitError.Sys().(syscall.WaitStatus); ok {
+					// (magic exit code 4 found in iptables source code; undocumented)
+					if status.ExitStatus() == 4 {
+						continue
+					}
+				}
+			}
+			return err
+		}
+		return nil
+	}
+}
+
 func ExposeNAT(ipnet net.IPNet) error {
 	ipt, err := iptables.New()
 	if err != nil {
 		return err
 	}
 	cidr := ipnet.String()
-	if err := ipt.AppendUnique("nat", "WEAVE", "-s", cidr, "-d", "224.0.0.0/4", "-j", "RETURN"); err != nil {
+	if err := addNatRule(ipt, "-s", cidr, "-d", "224.0.0.0/4", "-j", "RETURN"); err != nil {
 		return err
 	}
-	if err := ipt.AppendUnique("nat", "WEAVE", "-d", cidr, "!", "-s", cidr, "-j", "MASQUERADE"); err != nil {
+	if err := addNatRule(ipt, "-d", cidr, "!", "-s", cidr, "-j", "MASQUERADE"); err != nil {
 		return err
 	}
-	if err := ipt.AppendUnique("nat", "WEAVE", "-s", cidr, "!", "-d", cidr, "-j", "MASQUERADE"); err != nil {
+	if err := addNatRule(ipt, "-s", cidr, "!", "-d", cidr, "-j", "MASQUERADE"); err != nil {
 		return err
 	}
 	return nil

npc/metrics/metrics.go

@@ -46,6 +46,7 @@ func gatherMetrics() {
 			tcp, _ := tcpLayer.(*layers.TCP)
 			if tcp.SYN && !tcp.ACK { // Only plain SYN constitutes a NEW TCP connection
 				blockedConnections.With(prometheus.Labels{"protocol": "tcp", "dport": strconv.Itoa(int(tcp.DstPort))}).Inc()
+				log.Warnf("TCP connection from %v:%d to %v:%d blocked by Weave NPC.", srcIP(packet), tcp.SrcPort, dstIP(packet), tcp.DstPort)
 				continue
 			}
 		}
@@ -53,11 +54,42 @@ func gatherMetrics() {
 		if udpLayer := packet.Layer(layers.LayerTypeUDP); udpLayer != nil {
 			udp, _ := udpLayer.(*layers.UDP)
 			blockedConnections.With(prometheus.Labels{"protocol": "udp", "dport": strconv.Itoa(int(udp.DstPort))}).Inc()
+			log.Warnf("UDP connection from %v:%d to %v:%d blocked by Weave NPC.", srcIP(packet), udp.SrcPort, dstIP(packet), udp.DstPort)
 			continue
 		}
 	}
 }
 
+const unknownIP string = "<unknown IP>"
+
+func srcIP(packet gopacket.Packet) string {
+	if layer := packet.Layer(layers.LayerTypeIPv4); layer != nil {
+		if ip, ok := layer.(*layers.IPv4); ok {
+			return ip.SrcIP.String()
+		}
+	}
+	if layer := packet.Layer(layers.LayerTypeIPv6); layer != nil {
+		if ip, ok := layer.(*layers.IPv6); ok {
+			return ip.SrcIP.String()
+		}
+	}
+	return unknownIP
+}
+
+func dstIP(packet gopacket.Packet) string {
+	if layer := packet.Layer(layers.LayerTypeIPv4); layer != nil {
+		if ip, ok := layer.(*layers.IPv4); ok {
+			return ip.DstIP.String()
+		}
+	}
+	if layer := packet.Layer(layers.LayerTypeIPv6); layer != nil {
+		if ip, ok := layer.(*layers.IPv6); ok {
+			return ip.DstIP.String()
+		}
+	}
+	return unknownIP
+}
+
 func Start(addr string) error {
 	if err := prometheus.Register(blockedConnections); err != nil {
 		return err

prog/weave-kube/launch.sh

@@ -7,6 +7,9 @@ IPALLOC_RANGE=${IPALLOC_RANGE:-10.32.0.0/12}
 HTTP_ADDR=${WEAVE_HTTP_ADDR:-127.0.0.1:6784}
 STATUS_ADDR=${WEAVE_STATUS_ADDR:-0.0.0.0:6782}
 
+# Check if the IP range overlaps anything existing on the host
+/usr/bin/weaveutil netcheck $IPALLOC_RANGE weave
+
 # Default for network policy
 EXPECT_NPC=${EXPECT_NPC:-1}
 
@@ -152,4 +155,7 @@ if [ ! -f /etc/cni/net.d/10-weave.conf ] ; then
 EOF
 fi
 
+# Expose the weave network so host processes can communicate with pods
+/home/weave/weave --local expose $WEAVE_EXPOSE_IP
+
 wait $WEAVE_PID

prog/weaver/main.go

@@ -226,6 +226,12 @@ func main() {
 	overlay, bridge := createOverlay(datapathName, ifaceName, isAWSVPC, config.Host, config.Port, bufSzMB)
 	networkConfig.Bridge = bridge
 
+	if bridge != nil {
+		if err := weavenet.DetectHairpin("vethwe-bridge", Log); err != nil {
+			Log.Errorf("DetectHairpin failed: %s", err)
+		}
+	}
+
 	name := peerName(routerName, bridge.Interface())
 
 	if nickName == "" {

router/fastdp.go

@@ -50,7 +50,8 @@ type FastDatapath struct {
 	seenMACs map[MAC]struct{}
 
 	// vxlan vports associated with the given UDP ports
-	vxlanVportIDs    map[int]odp.VportID
+	vxlanUDPPorts    map[int]odp.VportID
+	vxlanVportIDs    map[odp.VportID]struct{}
 	mainVxlanVportID odp.VportID
 
 	// A singleton pool for the occasions when we need to decode
@@ -87,7 +88,8 @@ func NewFastDatapath(iface *net.Interface, port int) (*FastDatapath, error) {
 		sendToPort:    nil,
 		sendToMAC:     make(map[MAC]bridgeSender),
 		seenMACs:      make(map[MAC]struct{}),
-		vxlanVportIDs: make(map[int]odp.VportID),
+		vxlanUDPPorts: make(map[int]odp.VportID),
+		vxlanVportIDs: make(map[odp.VportID]struct{}),
 		forwarders:    make(map[mesh.PeerName]*fastDatapathForwarder),
 	}
 
@@ -433,7 +435,7 @@ func (fastdp *FastDatapath) getVxlanVportID(udpPort int) (odp.VportID, error) {
 	fastdp.lock.Lock()
 	defer fastdp.lock.Unlock()
 
-	if vxlanVportID, present := fastdp.vxlanVportIDs[udpPort]; present {
+	if vxlanVportID, present := fastdp.vxlanUDPPorts[udpPort]; present {
 		return vxlanVportID, nil
 	}
 
@@ -459,7 +461,8 @@ func (fastdp *FastDatapath) getVxlanVportID(udpPort int) (odp.VportID, error) {
 		}
 	}
 
-	fastdp.vxlanVportIDs[udpPort] = vxlanVportID
+	fastdp.vxlanUDPPorts[udpPort] = vxlanVportID
+	fastdp.vxlanVportIDs[vxlanVportID] = struct{}{}
 	fastdp.missHandlers[vxlanVportID] = func(fks odp.FlowKeys, lock *fastDatapathLock) FlowOp {
 		log.Debug("ODP miss: ", fks, " on port ", vxlanVportID)
 		tunnel := fks[odp.OVS_KEY_ATTR_TUNNEL].(odp.TunnelFlowKey)
@@ -1134,6 +1137,11 @@ func (fastdp *FastDatapath) send(fops FlowOp, frame []byte, lock *fastDatapathLo
 		}
 	}
 
+	if fastdp.isHairpinFlow(&flow) {
+		log.Error("Vetoed installation of hairpin flow ", flow)
+		return
+	}
+
 	if dec != nil {
 		// put the decoder back
 		lock.relock()
@@ -1171,6 +1179,50 @@ func (fastdp *FastDatapath) takeDecoder(lock *fastDatapathLock) *EthernetDecoder
 	return dec
 }
 
+// A isHairpinFlow checks whether the flow is created due to enabled hairpin
+// mode on the weave bridge port which attaches the datapath. Such flow is
+// identified by either:
+//
+// * in_port == out_port, where in_port is non-vxlan vport;
+// * a packet is sent back to a vxlan tunnel it has been received from and
+//   the tunnel id is either the same or dstPeer and srcPeer are reversed.
+func (fastdp *FastDatapath) isHairpinFlow(flow *odp.FlowSpec) bool {
+	var (
+		vxlanKey odp.TunnelAttrs
+		inVport  odp.VportID
+		inVxlan  bool
+	)
+
+	for _, key := range flow.FlowKeys {
+		switch k := key.(type) {
+		case odp.InPortFlowKey:
+			inVport = k.VportID()
+		case odp.TunnelFlowKey:
+			inVxlan = true
+			vxlanKey = k.Key()
+		}
+	}
+
+	for _, action := range flow.Actions {
+		switch a := action.(type) {
+		case odp.SetTunnelAction:
+			if inVxlan && a.TunnelAttrs.TunnelId == vxlanKey.TunnelId &&
+				a.TunnelAttrs.Ipv4Src == vxlanKey.Ipv4Dst &&
+				a.TunnelAttrs.Ipv4Dst == vxlanKey.Ipv4Src {
+				return true
+			}
+		case odp.OutputAction:
+			if a.VportID() == inVport {
+				if _, ok := fastdp.vxlanVportIDs[a.VportID()]; !ok {
+					return true
+				}
+			}
+		}
+	}
+
+	return false
+}
+
 type odpActionsFlowOp struct {
 	NonDiscardingFlowOp
 	fastdp  *FastDatapath

router/network_router.go

@@ -124,6 +124,13 @@ func (router *NetworkRouter) handleCapturedPacket(key PacketKey) FlowOp {
 }
 
 func (router *NetworkRouter) handleForwardedPacket(key ForwardPacketKey) FlowOp {
+	srcMac := net.HardwareAddr(key.SrcMAC[:])
+	dstMac := net.HardwareAddr(key.DstMAC[:])
+
+	if key.SrcPeer == router.Ourself.Peer {
+		log.Warn("Received own packet to peer ", key.DstPeer, " from MAC (", srcMac, ") to (", dstMac, ")")
+	}
+
 	if key.DstPeer != router.Ourself.Peer {
 		// it's not for us, we're just relaying it
 		router.PacketLogging.LogForwardPacket("Relaying", key)
@@ -134,9 +141,6 @@ func (router *NetworkRouter) handleForwardedPacket(key ForwardPacketKey) FlowOp
 	// (because the DstPeer on a forwarded broadcast packet is
 	// always set to the peer being forwarded to)
 
-	srcMac := net.HardwareAddr(key.SrcMAC[:])
-	dstMac := net.HardwareAddr(key.DstMAC[:])
-
 	switch newSrcMac, conflictPeer := router.Macs.AddForced(srcMac, key.SrcPeer); {
 	case newSrcMac:
 		log.Print("Discovered remote MAC ", srcMac, " at ", key.SrcPeer)

site/kube-addon.md

@@ -3,60 +3,70 @@ title: Integrating Kubernetes via the Addon
 menu_order: 63
 ---
 
-## Installation
+The following topics are discussed:
 
-If you're using Kubernetes 1.4 or later, and it is configured to use
-CNI, it is possible to install Weave Net on your cluster with a single
-command:
+* [Installation](#install)
+* [Network Policy Controller](#npc)
+ * [Upgrading the Daemon Sets](#daemon-sets)
+* [Network Policy Controller](#npc)
+ * [Troubleshooting Blocked Connections](#blocked-connections)
+ * [Changing Configuration Options](#configuration-options)
+
+
+##<a name=”install”></a> Installation
+
+Weave Net can be installed onto your CNI-enabled Kubernetes cluster
+with a single command:
 
 ```
 kubectl apply -f https://git.io/weave-kube
 ```
 
-After a few seconds, you should have a Weave Net pod running on each
-node, and any further pods you create will be attached to the Weave
+After a few seconds, a Weave Net pod should be running on each
+Node and any further pods you create will be automatically attached to the Weave
 network.
 
+**Note:** This command requires Kubernetes 1.4 or later.
+
+> CNI, the [_Container Network Interface_](https://github.com/containernetworking/cni),
+> is a proposed standard for configuring network interfaces for Linux
+> containers.
+>
 > If you do not already have a CNI-enabled cluster, you can bootstrap
 > one easily with
-> [kubeadm](http://kubernetes.io/docs/getting-started-guides/kubeadm/);
-> alternatively if you are using the older cluster set-up scripts from
-> the Kubernetes repo, you can use
+> [kubeadm](http://kubernetes.io/docs/getting-started-guides/kubeadm/).
 >
-> ```
-> NETWORK_PROVIDER=cni cluster/kube-up.sh
-> ```
+> Alternatively, you can [configure CNI yourself](http://kubernetes.io/docs/admin/network-plugins/#cni)
 
-NB If you were previously using the [Weave CNI
-driver](/site/cni-plugin.md) from a full install of Weave Net with your
-cluster, you will need to uninstall it first. Shut down Kubernetes,
-then perform the following _on all nodes_:
+**Note:** If using the [Weave CNI
+Plugin](/site/cni-plugin.md) from a prior full install of Weave Net with your
+cluster, you must first uninstall it before applying the Weave-kube addon.
+Shut down Kubernetes, and _on all nodes_ perform the following:
 
  * `weave reset`
- * Remove any separate provision you have made to run weave at
+ * Remove any separate provisions you may have made to run Weave at
    boot-time, e.g. `systemd` units
  * `rm /opt/cni/bin/weave-*`
 
-You can then relaunch Kubernetes and install the addon as described
+Then relaunch Kubernetes and install the addon as described
 above.
 
 The URL [https://git.io/weave-kube](https://git.io/weave-kube) points
-to the addon YAML for the latest release of the Weave Net addon;
-historic versions will be archived on our [GitHub release
+to the YAML file for the latest release of the Weave Net addon.
+Historic versions are archived on our [GitHub release
 page](https://github.com/weaveworks/weave/releases).
 
-## Upgrading
+###<a name=”daemon-sets”></a> Upgrading the Daemon Sets
 
 Kubernetes does not currently support rolling upgrades of daemon sets,
-so you will need to perform the procedure manually:
+and so you will need to perform the procedure manually:
 
 * Apply the updated addon manifest `kubectl apply -f https://git.io/weave-kube`
-* Kill each weave net pod in turn with kubectl delete; wait for the
-  replacement to begin running before moving on to the next.
+* Kill each Weave Net pod with `kubectl delete` and then wait for it to reboot before moving on to the next pod.
 
 ##<a name="npc"></a>Network Policy Controller
 
-The addon supports the [Kubernetes policy
+The addon also supports the [Kubernetes policy
 API](http://kubernetes.io/docs/user-guide/networkpolicies/) so that
 you can securely isolate pods from each other based on namespaces and
 labels. For more information on configuring network policies in
@@ -65,9 +75,57 @@ Kubernetes see the
 and the [NetworkPolicy API object
 definition](http://kubernetes.io/docs/api-reference/extensions/v1beta1/definitions/#_v1beta1_networkpolicy).
 
-## Changing Configuration Options
+###<a name=”blocked-connections”></a> Troubleshooting Blocked Connections
 
-You can change the default configuration by saving and editing the
-addon YAML before you `kubectl apply`. Additional arguments can be
+If you suspect that legitimate traffic is being blocked by the Weave Network Policy Controller, check the `weave-npc` container's logs:
+
+```
+$ kubectl logs $(kubectl get pods --all-namespaces | grep weave-net | awk '{print $2}') -n kube-system weave-npc
+```
+
+When the Weave Network Policy Controller blocks a connection, it logs the following details about it:
+
+* protocol used, 
+* source IP and port, 
+* destination IP and port, 
+
+as per the below example:
+
+```
+TCP connection from 10.32.0.7:56648 to 10.32.0.11:80 blocked by Weave NPC.
+UDP connection from 10.32.0.7:56648 to 10.32.0.11:80 blocked by Weave NPC.
+```
+
+###<a name=”configuration-options”></a> Changing Configuration Options
+
+The default configuration settings can be changed by saving and editing the
+addon YAML before running `kubectl apply`. Additional arguments may be
 supplied to the Weave router process by adding them to the `command:`
-array in the yaml file.
+array in the YAML file.
+
+Some parameters are changed by environment variables; these can be
+inserted into the YAML file like this:
+
+```
+      containers:
+        - name: weave
+          env:
+            - name: IPALLOC_RANGE
+              value: 10.0.0.0/16
+```
+
+The list of variables you can set is:
+
+* CHECKPOINT\_DISABLE - if set to 1, disable checking for new Weave Net
+  versions (default is blank, i.e. check is enabled)
+* IPALLOC\_RANGE - the range of IP addresses used by Weave Net
+  and the subnet they are placed in (CIDR format; default 10.32.0.0/12)
+* EXPECT\_NPC - set to 0 to disable Network Policy Controller (default is on)
+* KUBE\_PEERS - list of addresses of peers in the Kubernetes cluster
+  (default is to fetch the list from the api-server)
+* IPALLOC\_INIT - set the initialization mode of the [IP Address
+  Manager](/site/operational-guide/concepts.md#ip-address-manager)
+  (defaults to consensus amongst the KUBE\_PEERS)
+* WEAVE\_EXPOSE\_IP - set the IP address used as a gateway from the
+  Weave network to the host network - this is useful if you are
+  configuring the addon as a static pod.

test/700_status_and_report_test.sh

@@ -41,12 +41,17 @@ $SSH $HOST1 tee /tmp/hosts.checkpoint-api > /dev/null <<EOF
 EOF
 CHECKPOINT_DISABLE="" WEAVE_DOCKER_ARGS="-v /tmp/hosts.checkpoint-api:/etc/hosts" weave_on $HOST1 launch
 
+wait_for_version_check_error() {
+    while true; do
+        docker_on $HOST1 logs weave 2>&1 | grep 'Error checking version' && return
+        sleep 1
+    done
+}
+assert_raises "timeout 30 cat <( wait_for_version_check_error )"
+
 assert "weave_on $HOST1 report -f '{{.VersionCheck.Enabled}}'" "true"
 assert "weave_on $HOST1 report -f '{{.VersionCheck.Success}}'" "false"
 assert "weave_on $HOST1 report -f '{{.VersionCheck.NewVersion}}'" ""
-assert_raises "weave_on $HOST1 status | grep 'failed to check latest version - see logs; next check at'"
-
-
 
 weave_on $HOST1 reset
 

test/705_detect_hairpin_test.sh

@@ -0,0 +1,12 @@
+#! /bin/bash
+
+. ./config.sh
+
+start_suite "Detect hairpin mode"
+
+weave_on $HOST1 launch
+
+assert_raises "run_on $HOST1 sudo ip link set vethwe-bridge type bridge_slave hairpin on"
+assert_raises "docker_on $HOST1 logs weave 2>&1 | grep -q 'Hairpin mode enabled on \"vethwe-bridge\"'"
+
+end_suite

test/gce.sh

@@ -34,6 +34,9 @@ function vm_names {
 
 # Delete all vms in this account
 function destroy {
+	if [ $(gcloud compute firewall-rules list test-allow-docker$SUFFIX 2>/dev/null | wc -l) -gt 0 ] ; then
+		gcloud compute firewall-rules delete test-allow-docker$SUFFIX
+	fi
 	names="$(vm_names)"
 	if [ $(gcloud compute instances list --zones $ZONE -q $names | wc -l) -le 1 ] ; then
 		return 0
@@ -124,7 +127,9 @@ function setup {
 	destroy
 
 	names="$(vm_names)"
-	gcloud compute instances create $names --image $TEMPLATE_NAME --zone $ZONE
+	gcloud compute instances create $names --image $TEMPLATE_NAME --zone $ZONE --tags test$SUFFIX --network=test
+	my_ip="$(curl -s http://ipinfo.io/ip)"
+	gcloud compute firewall-rules create test-allow-docker$SUFFIX --network=test --allow tcp:2375,tcp:12375 --target-tags test$SUFFIX --source-ranges $my_ip
 	gcloud compute config-ssh --ssh-key-file $SSH_KEY_FILE
 	sed -i '/UserKnownHostsFile=\/dev\/null/d' ~/.ssh/config
 

weave

@@ -571,7 +571,7 @@ detect_bridge_type() {
     MTU=${WEAVE_MTU:-$(cat /sys/class/net/$BRIDGE/mtu)}
 }
 
-create_bridge() {
+try_create_bridge() {
     if ! detect_bridge_type ; then
         BRIDGE_TYPE=bridge
         if [ -z "$WEAVE_NO_FASTDP" ] ; then
@@ -656,6 +656,16 @@ EOF
     configure_arp_cache $BRIDGE
 }
 
+create_bridge() {
+    if ! try_create_bridge "$@" ; then
+        echo "Creating bridge '$BRIDGE' failed" >&2
+        # reset to original value so we destroy both kinds
+        DATAPATH=datapath
+        destroy_bridge
+        exit 1
+    fi
+}
+
 expose_ip() {
     ipam_cidrs allocate_no_check_alive weave:expose $CIDR_ARGS
     for CIDR in $ALL_CIDRS ; do