Add release v1.8.2

Detect looping flows

CHANGELOG.md

@@ -1,3 +1,43 @@
+## Release 1.8.2
+
+Bug fixes and minor improvements
+
+* Fixed a bug where looping flows were installed which caused high CPU
+  usage #2650, #2674
+* Fixed a bug where Kubernetes master could not contact pods #2673, #2683
+* Fixed a bug where weave-kube was crashing in a loop due to invalid
+  Weave bridge state #2657
+* Fixed a bug where iptables NAT rules were not appended due to
+  "temporary unavailable" iptables error #2679
+* Added a detection of enabling the hairpin mode on the Weave bridge port
+  which caused installation of looping flows #2674
+* Added a detection of overlaps between Weave and the host IP address
+  ranges when launching weave-kube #2669, #2672
+* Added logging of connections blocked by weave-npc #2546, #2573
+
+More details in the [change log](https://github.com/weaveworks/weave/issues?q=milestone%3A1.8.2).
+
+## Release 1.8.1
+
+Bug fixes and minor improvements
+
+* Fixed weave-npc crash from Succeeded/Failed pods #2632,#2658
+* Fixed occasional failure to create Weave bridge on node reboot
+  #2617,#2637
+* Fixed a bug where weave-kube would fail to install when run with
+  unrelease snapshot builds #2642
+* Improved conformance to CNI spec by not releasing IP addresses when
+  a container dies #2643
+* Improved troubleshooting of install failure by creating CNI config
+  after Weave Net is up #2570
+* "up to date" shown even when the version check was blocked by
+  firewall #2537,#2565,#2645
+* "Unable to claim" message on re-launching Weave after using CNI
+  #2548,#2577
+* Eliminated spurious IP reclaim operations when IPAM was disabled
+  #2567,#2644
+* Include `jq` tool in our build VM configuration #2656
+
 ## Release 1.8.0
 
 Highlights:

Vagrantfile

@@ -14,6 +14,7 @@ pkgs = %w(
   git
   mercurial
   bc
+  jq
 )
 
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|

api/ipam.go

@@ -5,6 +5,15 @@ import (
 	"net"
 )
 
+// Special token used in place of a container identifier when:
+// - the caller does not know the container ID, or
+// - the caller is unsure whether IPAM was previously told the container ID.
+//
+// While addresses are typically stored under their container IDs, when this
+// token is used, the address will be stored under its own string, something
+// which should be kept in mind if the entry needs to be removed at any point.
+const NoContainerID string = "_"
+
 func (client *Client) ipamOp(ID string, op string) (*net.IPNet, error) {
 	ip, err := client.httpVerb(op, fmt.Sprintf("/ip/%s", ID), nil)
 	if err != nil {

ipam/allocate.go

@@ -3,6 +3,7 @@ package ipam
 import (
 	"fmt"
 
+	"github.com/weaveworks/weave/api"
 	"github.com/weaveworks/weave/net/address"
 )
 
@@ -43,7 +44,7 @@ func (g *allocate) Try(alloc *Allocator) bool {
 	if ok, addr := alloc.space.Allocate(g.r.HostRange()); ok {
 		// If caller hasn't supplied a unique ID, file it under the IP address
 		// which lets the caller then release the address using DELETE /ip/address
-		if g.ident == "_" {
+		if g.ident == api.NoContainerID {
 			g.ident = addr.String()
 		}
 		alloc.debugln("Allocated", addr, "for", g.ident, "in", g.r)

ipam/allocator.go

@@ -322,7 +322,7 @@ func (alloc *Allocator) Claim(ident string, cidr address.CIDR, isContainer, noEr
 // ContainerDied called from the updater interface.  Async.
 func (alloc *Allocator) ContainerDied(ident string) {
 	alloc.actionChan <- func() {
-		if alloc.hasOwned(ident) {
+		if alloc.hasOwnedByContainer(ident) {
 			alloc.debugln("Container", ident, "died; noting to remove later")
 			alloc.dead[ident] = alloc.now()
 		}
@@ -335,7 +335,7 @@ func (alloc *Allocator) ContainerDied(ident string) {
 // ContainerDestroyed called from the updater interface.  Async.
 func (alloc *Allocator) ContainerDestroyed(ident string) {
 	alloc.actionChan <- func() {
-		if alloc.hasOwned(ident) {
+		if alloc.hasOwnedByContainer(ident) {
 			alloc.debugln("Container", ident, "destroyed; removing addresses")
 			alloc.delete(ident)
 			delete(alloc.dead, ident)
@@ -996,9 +996,9 @@ func (alloc *Allocator) persistOwned() {
 
 // Owned addresses
 
-func (alloc *Allocator) hasOwned(ident string) bool {
-	_, b := alloc.owned[ident]
-	return b
+func (alloc *Allocator) hasOwnedByContainer(ident string) bool {
+	d, b := alloc.owned[ident]
+	return b && d.IsContainer
 }
 
 // NB: addr must not be owned by ident already

ipam/claim.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/weaveworks/mesh"
 
+	"github.com/weaveworks/weave/api"
 	"github.com/weaveworks/weave/common"
 	"github.com/weaveworks/weave/net/address"
 )
@@ -77,26 +78,34 @@ func (c *claim) Try(alloc *Allocator) bool {
 		return false
 	}
 
-	if c.ident == "_" { // Special "I don't have a unique ID" identifier
-		c.ident = c.cidr.Addr.String()
-	}
 	// We are the owner, check we haven't given it to another container
-	switch existingIdent := alloc.findOwner(c.cidr.Addr); existingIdent {
-	case "":
+	existingIdent := alloc.findOwner(c.cidr.Addr)
+	switch {
+	case existingIdent == "":
+		// Unused address, we try to claim it:
 		if err := alloc.space.Claim(c.cidr.Addr); err == nil {
 			alloc.debugln("Claimed", c.cidr, "for", c.ident)
-			alloc.addOwned(c.ident, c.cidr, c.isContainer)
+			if c.ident == api.NoContainerID {
+				alloc.addOwned(c.cidr.Addr.String(), c.cidr, c.isContainer)
+			} else {
+				alloc.addOwned(c.ident, c.cidr, c.isContainer)
+			}
 			c.sendResult(nil)
 		} else {
 			c.sendResult(err)
 		}
-	case c.ident:
+	case (existingIdent == c.ident) || (c.ident == api.NoContainerID && existingIdent == c.cidr.Addr.String()):
 		// same identifier is claiming same address; that's OK
 		alloc.debugln("Re-Claimed", c.cidr, "for", c.ident)
 		c.sendResult(nil)
-	case c.cidr.Addr.String():
-		// Address already allocated via "_" name
+	case existingIdent == c.cidr.Addr.String():
+		// Address already allocated via api.NoContainerID name and current ID is a real container ID:
 		c.sendResult(fmt.Errorf("address %s already in use", c.cidr))
+	case c.ident == api.NoContainerID:
+		// We do not know whether this is the same container or another one,
+		// but we also cannot prove otherwise, so we let it reclaim the address:
+		alloc.debugln("Re-Claimed", c.cidr, "for ID", c.ident, "having existing ID as", existingIdent)
+		c.sendResult(nil)
 	default:
 		// Addr already owned by container on this machine
 		c.sendResult(fmt.Errorf("address %s is already owned by %s", c.cidr.String(), existingIdent))

net/bridge.go

@@ -1,6 +1,9 @@
 package net
 
 import (
+	"fmt"
+
+	"github.com/Sirupsen/logrus"
 	"github.com/vishvananda/netlink"
 )
 
@@ -51,3 +54,35 @@ func isDatapath(link netlink.Link) bool {
 		return false
 	}
 }
+
+func DetectHairpin(portIfName string, log *logrus.Logger) error {
+	link, err := netlink.LinkByName(portIfName)
+	if err != nil {
+		return fmt.Errorf("Unable to find link %q: %s", portIfName, err)
+	}
+
+	ch := make(chan netlink.LinkUpdate)
+	// See EnsureInterface for why done channel is not passed
+	if err := netlink.LinkSubscribe(ch, nil); err != nil {
+		return fmt.Errorf("Unable to subscribe to netlink updates: %s", err)
+	}
+
+	pi, err := netlink.LinkGetProtinfo(link)
+	if err != nil {
+		return fmt.Errorf("Unable to get link protinfo %q: %s", portIfName, err)
+	}
+	if pi.Hairpin {
+		return fmt.Errorf("Hairpin mode enabled on %q", portIfName)
+	}
+
+	go func() {
+		for up := range ch {
+			if up.Attrs().Name == portIfName && up.Attrs().Protinfo != nil &&
+				up.Attrs().Protinfo.Hairpin {
+				log.Errorf("Hairpin mode enabled on %q", portIfName)
+			}
+		}
+	}()
+
+	return nil
+}

net/veth.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"net"
 	"strings"
+	"syscall"
 	"time"
 
 	"github.com/coreos/go-iptables/iptables"
@@ -271,3 +272,39 @@ func subnets(addrs []netlink.Addr) map[string]struct{} {
 	}
 	return subnets
 }
+
+func addNatRule(ipt *iptables.IPTables, rulespec ...string) error {
+	// Loop until we get an exit code other than "temporarily unavailable"
+	for {
+		if err := ipt.AppendUnique("nat", "WEAVE", rulespec...); err != nil {
+			if ierr, ok := err.(*iptables.Error); ok {
+				if status, ok := ierr.ExitError.Sys().(syscall.WaitStatus); ok {
+					// (magic exit code 4 found in iptables source code; undocumented)
+					if status.ExitStatus() == 4 {
+						continue
+					}
+				}
+			}
+			return err
+		}
+		return nil
+	}
+}
+
+func ExposeNAT(ipnet net.IPNet) error {
+	ipt, err := iptables.New()
+	if err != nil {
+		return err
+	}
+	cidr := ipnet.String()
+	if err := addNatRule(ipt, "-s", cidr, "-d", "224.0.0.0/4", "-j", "RETURN"); err != nil {
+		return err
+	}
+	if err := addNatRule(ipt, "-d", cidr, "!", "-s", cidr, "-j", "MASQUERADE"); err != nil {
+		return err
+	}
+	if err := addNatRule(ipt, "-s", cidr, "!", "-d", cidr, "-j", "MASQUERADE"); err != nil {
+		return err
+	}
+	return nil
+}

npc/metrics/metrics.go

@@ -46,6 +46,7 @@ func gatherMetrics() {
 			tcp, _ := tcpLayer.(*layers.TCP)
 			if tcp.SYN && !tcp.ACK { // Only plain SYN constitutes a NEW TCP connection
 				blockedConnections.With(prometheus.Labels{"protocol": "tcp", "dport": strconv.Itoa(int(tcp.DstPort))}).Inc()
+				log.Warnf("TCP connection from %v:%d to %v:%d blocked by Weave NPC.", srcIP(packet), tcp.SrcPort, dstIP(packet), tcp.DstPort)
 				continue
 			}
 		}
@@ -53,11 +54,42 @@ func gatherMetrics() {
 		if udpLayer := packet.Layer(layers.LayerTypeUDP); udpLayer != nil {
 			udp, _ := udpLayer.(*layers.UDP)
 			blockedConnections.With(prometheus.Labels{"protocol": "udp", "dport": strconv.Itoa(int(udp.DstPort))}).Inc()
+			log.Warnf("UDP connection from %v:%d to %v:%d blocked by Weave NPC.", srcIP(packet), udp.SrcPort, dstIP(packet), udp.DstPort)
 			continue
 		}
 	}
 }
 
+const unknownIP string = "<unknown IP>"
+
+func srcIP(packet gopacket.Packet) string {
+	if layer := packet.Layer(layers.LayerTypeIPv4); layer != nil {
+		if ip, ok := layer.(*layers.IPv4); ok {
+			return ip.SrcIP.String()
+		}
+	}
+	if layer := packet.Layer(layers.LayerTypeIPv6); layer != nil {
+		if ip, ok := layer.(*layers.IPv6); ok {
+			return ip.SrcIP.String()
+		}
+	}
+	return unknownIP
+}
+
+func dstIP(packet gopacket.Packet) string {
+	if layer := packet.Layer(layers.LayerTypeIPv4); layer != nil {
+		if ip, ok := layer.(*layers.IPv4); ok {
+			return ip.DstIP.String()
+		}
+	}
+	if layer := packet.Layer(layers.LayerTypeIPv6); layer != nil {
+		if ip, ok := layer.(*layers.IPv6); ok {
+			return ip.DstIP.String()
+		}
+	}
+	return unknownIP
+}
+
 func Start(addr string) error {
 	if err := prometheus.Register(blockedConnections); err != nil {
 		return err

npc/namespace.go

@@ -302,8 +302,9 @@ func (ns *ns) deleteNamespace(obj *coreapi.Namespace) error {
 }
 
 func hasIP(pod *coreapi.Pod) bool {
-	// Ensure pod has an IP address and isn't sharing the host network namespace
-	return len(pod.Status.PodIP) > 0 && !pod.Spec.HostNetwork
+	// Ensure pod isn't dead, has an IP address and isn't sharing the host network namespace
+	return pod.Status.Phase != "Succeeded" && pod.Status.Phase != "Failed" &&
+		len(pod.Status.PodIP) > 0 && !pod.Spec.HostNetwork
 }
 
 func equals(a, b map[string]string) bool {

plugin/ipam/driver.go

@@ -72,9 +72,8 @@ func splitPoolID(poolID string) (subnet, iprange *net.IPNet, err error) {
 func (i *Ipam) RequestAddress(poolID string, address net.IP, options map[string]string) (ip *net.IPNet, _ map[string]string, err error) {
 	i.logReq("RequestAddress", poolID, address, options)
 	defer func() { i.logRes("RequestAddress", err, ip) }()
-	// If we pass magic string "_" to weave IPAM it stores the address under its own string
 	if poolID == "weavepool" { // old-style
-		ip, err = i.weave.AllocateIP("_")
+		ip, err = i.weave.AllocateIP(api.NoContainerID)
 		return
 	}
 	subnet, iprange, err := splitPoolID(poolID)
@@ -83,12 +82,12 @@ func (i *Ipam) RequestAddress(poolID string, address net.IP, options map[string]
 	}
 	if address != nil { // try to claim specific address requested
 		ip = &net.IPNet{IP: address, Mask: subnet.Mask}
-		if err = i.weave.ClaimIP("_", ip); err != nil {
+		if err = i.weave.ClaimIP(api.NoContainerID, ip); err != nil {
 			return
 		}
 	} else {
 		// We are lying slightly to IPAM here: the range is not a subnet
-		if ip, err = i.weave.AllocateIPInSubnet("_", iprange); err != nil {
+		if ip, err = i.weave.AllocateIPInSubnet(api.NoContainerID, iprange); err != nil {
 			return
 		}
 		ip.Mask = subnet.Mask // fix up the subnet we lied about

plugin/net/cni.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"net"
+	"syscall"
 
 	"github.com/appc/cni/pkg/ipam"
 	"github.com/appc/cni/pkg/skel"
@@ -83,6 +84,9 @@ func (c *CNIPlugin) CmdAdd(args *skel.CmdArgs) error {
 			if err := assignBridgeIP(conf.BrName, bridgeIPResult.IP4.IP); err != nil {
 				return fmt.Errorf("unable to assign IP address to bridge: %s", err)
 			}
+			if err := weavenet.ExposeNAT(bridgeIPResult.IP4.IP); err != nil {
+				return fmt.Errorf("unable to create NAT rules: %s", err)
+			}
 			bridgeIP = bridgeIPResult.IP4.IP.IP
 		} else if err != nil {
 			return err
@@ -150,6 +154,11 @@ func assignBridgeIP(bridgeName string, ipnet net.IPNet) error {
 		return err
 	}
 	if err := netlink.AddrAdd(link, &netlink.Addr{IPNet: &ipnet}); err != nil {
+		// Treat as non-error if this address is already there
+		// - maybe another copy of this program just added it
+		if err == syscall.Errno(syscall.EEXIST) {
+			return nil
+		}
 		return fmt.Errorf("failed to add IP address to %q: %v", bridgeName, err)
 	}
 	return nil

prog/plugin/main.go

@@ -22,7 +22,7 @@ import (
 	"github.com/weaveworks/weave/plugin/skel"
 )
 
-var version = "(unreleased version)"
+var version = "unreleased"
 
 var Log = common.Log
 

prog/weave-kube/launch.sh

@@ -7,6 +7,9 @@ IPALLOC_RANGE=${IPALLOC_RANGE:-10.32.0.0/12}
 HTTP_ADDR=${WEAVE_HTTP_ADDR:-127.0.0.1:6784}
 STATUS_ADDR=${WEAVE_STATUS_ADDR:-0.0.0.0:6782}
 
+# Check if the IP range overlaps anything existing on the host
+/usr/bin/weaveutil netcheck $IPALLOC_RANGE weave
+
 # Default for network policy
 EXPECT_NPC=${EXPECT_NPC:-1}
 
@@ -18,25 +21,14 @@ fi
 
 echo 1 > /proc/sys/net/bridge/bridge-nf-call-iptables
 
-# Create CNI config, if not already there
-if [ ! -f /etc/cni/net.d/10-weave.conf ] ; then
-    mkdir -p /etc/cni/net.d
-    cat > /etc/cni/net.d/10-weave.conf <<EOF
-{
-    "name": "weave",
-    "type": "weave-net"
-}
-EOF
-fi
-
 SOURCE_BINARY=/usr/bin/weaveutil
-VERSION=$(/home/weave/weaver --version | sed -E 's/weave router (.*?)/\1/')
+VERSION=$(/home/weave/weaver $EXTRA_ARGS --version | sed -e 's/weave router //')
 PLUGIN="weave-plugin-$VERSION"
 
 install_cni_plugin() {
     mkdir -p $1 || return 1
-    if [ ! -f $1/$PLUGIN ]; then
-        cp "$SOURCE_BINARY" $1/$PLUGIN
+    if [ ! -f "$1/$PLUGIN" ]; then
+        cp "$SOURCE_BINARY" "$1/$PLUGIN"
     fi
 }
 
@@ -44,7 +36,7 @@ upgrade_cni_plugin_symlink() {
     # Remove potential temporary symlink from previous failed upgrade:
     rm -f $1/$2.tmp
     # Atomically create a symlink to the plugin:
-    ln -s $1/$PLUGIN $1/$2.tmp && mv -f $1/$2.tmp $1/$2
+    ln -s "$1/$PLUGIN" $1/$2.tmp && mv -f $1/$2.tmp $1/$2
 }
 
 upgrade_cni_plugin() {
@@ -152,6 +144,17 @@ done
     reclaim_ips "_" $IPS
 done
 
+# Create CNI config, if not already there
+if [ ! -f /etc/cni/net.d/10-weave.conf ] ; then
+    mkdir -p /etc/cni/net.d
+    cat > /etc/cni/net.d/10-weave.conf <<EOF
+{
+    "name": "weave",
+    "type": "weave-net"
+}
+EOF
+fi
+
 # Expose the weave network so host processes can communicate with pods
 /home/weave/weave --local expose $WEAVE_EXPOSE_IP
 

prog/weave-npc/main.go

@@ -24,7 +24,7 @@ import (
 )
 
 var (
-	version     = "(unreleased)"
+	version     = "unreleased"
 	metricsAddr string
 )
 

prog/weaveproxy/main.go

@@ -11,7 +11,7 @@ import (
 	"github.com/weaveworks/weave/proxy"
 )
 
-var version = "(unreleased version)"
+var version = "unreleased"
 
 var Log = common.Log
 

prog/weaver/checkpoint.go

@@ -10,6 +10,7 @@ import (
 
 var checker *checkpoint.Checker
 var newVersion atomic.Value
+var success atomic.Value
 
 const (
 	updateCheckPeriod = 6 * time.Hour
@@ -17,9 +18,11 @@ const (
 
 func checkForUpdates(dockerVersion string, network string) {
 	newVersion.Store("")
+	success.Store(true)
 
 	handleResponse := func(r *checkpoint.CheckResponse, err error) {
 		if err != nil {
+			success.Store(false)
 			Log.Printf("Error checking version: %v", err)
 			return
 		}

prog/weaver/http.go

@@ -240,6 +240,7 @@ var ipamTemplate = defTemplate("ipamTemplate", `{{printIPAMRanges .Router .IPAM}
 
 type VersionCheck struct {
 	Enabled     bool
+	Success     bool
 	NewVersion  string
 	NextCheckAt time.Time
 }
@@ -251,6 +252,7 @@ func versionCheck() *VersionCheck {
 	}
 
 	v.Enabled = true
+	v.Success = success.Load().(bool)
 	v.NewVersion = newVersion.Load().(string)
 	v.NextCheckAt = checker.NextCheckAt()
 
@@ -261,12 +263,12 @@ func (v *VersionCheck) String() string {
 	switch {
 	case !v.Enabled:
 		return "version check update disabled"
+	case !v.Success:
+		return fmt.Sprintf("failed to check latest version - see logs; next check at %s", v.NextCheckAt.Format("2006/01/02 15:04:05"))
 	case v.NewVersion != "":
-		return fmt.Sprintf("version %s available - please upgrade!",
-			v.NewVersion)
+		return fmt.Sprintf("version %s available - please upgrade!", v.NewVersion)
 	default:
-		return fmt.Sprintf("up to date; next check at %s",
-			v.NextCheckAt.Format("2006/01/02 15:04:05"))
+		return fmt.Sprintf("up to date; next check at %s", v.NextCheckAt.Format("2006/01/02 15:04:05"))
 	}
 }
 

prog/weaver/main.go

@@ -27,7 +27,7 @@ import (
 	weave "github.com/weaveworks/weave/router"
 )
 
-var version = "(unreleased version)"
+var version = "unreleased"
 
 var Log = common.Log
 
@@ -226,6 +226,12 @@ func main() {
 	overlay, bridge := createOverlay(datapathName, ifaceName, isAWSVPC, config.Host, config.Port, bufSzMB)
 	networkConfig.Bridge = bridge
 
+	if bridge != nil {
+		if err := weavenet.DetectHairpin("vethwe-bridge", Log); err != nil {
+			Log.Errorf("DetectHairpin failed: %s", err)
+		}
+	}
+
 	name := peerName(routerName, bridge.Interface())
 
 	if nickName == "" {

prog/weaveutil/expose.go

@@ -0,0 +1,23 @@
+package main
+
+import (
+	weavenet "github.com/weaveworks/weave/net"
+)
+
+func exposeNAT(args []string) error {
+	if len(args) < 1 {
+		cmdUsage("expose-nat", "<cidr>...")
+	}
+
+	cidrs, err := parseCIDRs(args)
+	if err != nil {
+		return err
+	}
+
+	for _, cidr := range cidrs {
+		if err := weavenet.ExposeNAT(*cidr); err != nil {
+			return err
+		}
+	}
+	return nil
+}

prog/weaveutil/main.go

@@ -34,6 +34,7 @@ func init() {
 		"list-netdevs":           listNetDevs,
 		"cni-net":                cniNet,
 		"cni-ipam":               cniIPAM,
+		"expose-nat":             exposeNAT,
 	}
 }
 

router/fastdp.go

@@ -50,7 +50,8 @@ type FastDatapath struct {
 	seenMACs map[MAC]struct{}
 
 	// vxlan vports associated with the given UDP ports
-	vxlanVportIDs    map[int]odp.VportID
+	vxlanUDPPorts    map[int]odp.VportID
+	vxlanVportIDs    map[odp.VportID]struct{}
 	mainVxlanVportID odp.VportID
 
 	// A singleton pool for the occasions when we need to decode
@@ -87,7 +88,8 @@ func NewFastDatapath(iface *net.Interface, port int) (*FastDatapath, error) {
 		sendToPort:    nil,
 		sendToMAC:     make(map[MAC]bridgeSender),
 		seenMACs:      make(map[MAC]struct{}),
-		vxlanVportIDs: make(map[int]odp.VportID),
+		vxlanUDPPorts: make(map[int]odp.VportID),
+		vxlanVportIDs: make(map[odp.VportID]struct{}),
 		forwarders:    make(map[mesh.PeerName]*fastDatapathForwarder),
 	}
 
@@ -433,7 +435,7 @@ func (fastdp *FastDatapath) getVxlanVportID(udpPort int) (odp.VportID, error) {
 	fastdp.lock.Lock()
 	defer fastdp.lock.Unlock()
 
-	if vxlanVportID, present := fastdp.vxlanVportIDs[udpPort]; present {
+	if vxlanVportID, present := fastdp.vxlanUDPPorts[udpPort]; present {
 		return vxlanVportID, nil
 	}
 
@@ -459,7 +461,8 @@ func (fastdp *FastDatapath) getVxlanVportID(udpPort int) (odp.VportID, error) {
 		}
 	}
 
-	fastdp.vxlanVportIDs[udpPort] = vxlanVportID
+	fastdp.vxlanUDPPorts[udpPort] = vxlanVportID
+	fastdp.vxlanVportIDs[vxlanVportID] = struct{}{}
 	fastdp.missHandlers[vxlanVportID] = func(fks odp.FlowKeys, lock *fastDatapathLock) FlowOp {
 		log.Debug("ODP miss: ", fks, " on port ", vxlanVportID)
 		tunnel := fks[odp.OVS_KEY_ATTR_TUNNEL].(odp.TunnelFlowKey)
@@ -1134,6 +1137,11 @@ func (fastdp *FastDatapath) send(fops FlowOp, frame []byte, lock *fastDatapathLo
 		}
 	}
 
+	if fastdp.isHairpinFlow(&flow) {
+		log.Error("Vetoed installation of hairpin flow ", flow)
+		return
+	}
+
 	if dec != nil {
 		// put the decoder back
 		lock.relock()
@@ -1171,6 +1179,50 @@ func (fastdp *FastDatapath) takeDecoder(lock *fastDatapathLock) *EthernetDecoder
 	return dec
 }
 
+// A isHairpinFlow checks whether the flow is created due to enabled hairpin
+// mode on the weave bridge port which attaches the datapath. Such flow is
+// identified by either:
+//
+// * in_port == out_port, where in_port is non-vxlan vport;
+// * a packet is sent back to a vxlan tunnel it has been received from and
+//   the tunnel id is either the same or dstPeer and srcPeer are reversed.
+func (fastdp *FastDatapath) isHairpinFlow(flow *odp.FlowSpec) bool {
+	var (
+		vxlanKey odp.TunnelAttrs
+		inVport  odp.VportID
+		inVxlan  bool
+	)
+
+	for _, key := range flow.FlowKeys {
+		switch k := key.(type) {
+		case odp.InPortFlowKey:
+			inVport = k.VportID()
+		case odp.TunnelFlowKey:
+			inVxlan = true
+			vxlanKey = k.Key()
+		}
+	}
+
+	for _, action := range flow.Actions {
+		switch a := action.(type) {
+		case odp.SetTunnelAction:
+			if inVxlan && a.TunnelAttrs.TunnelId == vxlanKey.TunnelId &&
+				a.TunnelAttrs.Ipv4Src == vxlanKey.Ipv4Dst &&
+				a.TunnelAttrs.Ipv4Dst == vxlanKey.Ipv4Src {
+				return true
+			}
+		case odp.OutputAction:
+			if a.VportID() == inVport {
+				if _, ok := fastdp.vxlanVportIDs[a.VportID()]; !ok {
+					return true
+				}
+			}
+		}
+	}
+
+	return false
+}
+
 type odpActionsFlowOp struct {
 	NonDiscardingFlowOp
 	fastdp  *FastDatapath

router/network_router.go

@@ -124,6 +124,13 @@ func (router *NetworkRouter) handleCapturedPacket(key PacketKey) FlowOp {
 }
 
 func (router *NetworkRouter) handleForwardedPacket(key ForwardPacketKey) FlowOp {
+	srcMac := net.HardwareAddr(key.SrcMAC[:])
+	dstMac := net.HardwareAddr(key.DstMAC[:])
+
+	if key.SrcPeer == router.Ourself.Peer {
+		log.Warn("Received own packet to peer ", key.DstPeer, " from MAC (", srcMac, ") to (", dstMac, ")")
+	}
+
 	if key.DstPeer != router.Ourself.Peer {
 		// it's not for us, we're just relaying it
 		router.PacketLogging.LogForwardPacket("Relaying", key)
@@ -134,9 +141,6 @@ func (router *NetworkRouter) handleForwardedPacket(key ForwardPacketKey) FlowOp
 	// (because the DstPeer on a forwarded broadcast packet is
 	// always set to the peer being forwarded to)
 
-	srcMac := net.HardwareAddr(key.SrcMAC[:])
-	dstMac := net.HardwareAddr(key.DstMAC[:])
-
 	switch newSrcMac, conflictPeer := router.Macs.AddForced(srcMac, key.SrcPeer); {
 	case newSrcMac:
 		log.Print("Discovered remote MAC ", srcMac, " at ", key.SrcPeer)

site/kube-addon.md

@@ -3,60 +3,70 @@ title: Integrating Kubernetes via the Addon
 menu_order: 63
 ---
 
-## Installation
+The following topics are discussed:
 
-If you're using Kubernetes 1.4 or later, and it is configured to use
-CNI, it is possible to install Weave Net on your cluster with a single
-command:
+* [Installation](#install)
+* [Network Policy Controller](#npc)
+ * [Upgrading the Daemon Sets](#daemon-sets)
+* [Network Policy Controller](#npc)
+ * [Troubleshooting Blocked Connections](#blocked-connections)
+ * [Changing Configuration Options](#configuration-options)
+
+
+##<a name=”install”></a> Installation
+
+Weave Net can be installed onto your CNI-enabled Kubernetes cluster
+with a single command:
 
 ```
-kubectl create -f https://git.io/weave-kube
+kubectl apply -f https://git.io/weave-kube
 ```
 
-After a few seconds, you should have a Weave Net pod running on each
-node, and any further pods you create will be attached to the Weave
+After a few seconds, a Weave Net pod should be running on each
+Node and any further pods you create will be automatically attached to the Weave
 network.
 
+**Note:** This command requires Kubernetes 1.4 or later.
+
+> CNI, the [_Container Network Interface_](https://github.com/containernetworking/cni),
+> is a proposed standard for configuring network interfaces for Linux
+> containers.
+>
 > If you do not already have a CNI-enabled cluster, you can bootstrap
 > one easily with
-> [kubeadm](http://kubernetes.io/docs/getting-started-guides/kubeadm/);
-> alternatively if you are using the older cluster set-up scripts from
-> the Kubernetes repo, you can use
+> [kubeadm](http://kubernetes.io/docs/getting-started-guides/kubeadm/).
 >
-> ```
-> NETWORK_PROVIDER=cni cluster/kube-up.sh
-> ```
+> Alternatively, you can [configure CNI yourself](http://kubernetes.io/docs/admin/network-plugins/#cni)
 
-NB If you were previously using the [Weave CNI
-driver](/site/cni-plugin.md) from a full install of Weave Net with your
-cluster, you will need to uninstall it first. Shut down Kubernetes,
-then perform the following _on all nodes_:
+**Note:** If using the [Weave CNI
+Plugin](/site/cni-plugin.md) from a prior full install of Weave Net with your
+cluster, you must first uninstall it before applying the Weave-kube addon.
+Shut down Kubernetes, and _on all nodes_ perform the following:
 
  * `weave reset`
- * Remove any separate provision you have made to run weave at
+ * Remove any separate provisions you may have made to run Weave at
    boot-time, e.g. `systemd` units
  * `rm /opt/cni/bin/weave-*`
 
-You can then relaunch Kubernetes and install the addon as described
+Then relaunch Kubernetes and install the addon as described
 above.
 
 The URL [https://git.io/weave-kube](https://git.io/weave-kube) points
-to the addon YAML for the latest release of the Weave Net addon;
-historic versions will be archived on our [GitHub release
+to the YAML file for the latest release of the Weave Net addon.
+Historic versions are archived on our [GitHub release
 page](https://github.com/weaveworks/weave/releases).
 
-## Upgrading
+###<a name=”daemon-sets”></a> Upgrading the Daemon Sets
 
 Kubernetes does not currently support rolling upgrades of daemon sets,
-so you will need to perform the procedure manually:
+and so you will need to perform the procedure manually:
 
 * Apply the updated addon manifest `kubectl apply -f https://git.io/weave-kube`
-* Kill each weave net pod in turn with kubectl delete; wait for the
-  replacement to begin running before moving on to the next.
+* Kill each Weave Net pod with `kubectl delete` and then wait for it to reboot before moving on to the next pod.
 
 ##<a name="npc"></a>Network Policy Controller
 
-The addon supports the [Kubernetes policy
+The addon also supports the [Kubernetes policy
 API](http://kubernetes.io/docs/user-guide/networkpolicies/) so that
 you can securely isolate pods from each other based on namespaces and
 labels. For more information on configuring network policies in
@@ -65,9 +75,57 @@ Kubernetes see the
 and the [NetworkPolicy API object
 definition](http://kubernetes.io/docs/api-reference/extensions/v1beta1/definitions/#_v1beta1_networkpolicy).
 
-## Changing Configuration Options
+###<a name=”blocked-connections”></a> Troubleshooting Blocked Connections
 
-You can change the default configuration by saving and editing the
-addon YAML before you `kubectl apply`. Additional arguments can be
+If you suspect that legitimate traffic is being blocked by the Weave Network Policy Controller, check the `weave-npc` container's logs:
+
+```
+$ kubectl logs $(kubectl get pods --all-namespaces | grep weave-net | awk '{print $2}') -n kube-system weave-npc
+```
+
+When the Weave Network Policy Controller blocks a connection, it logs the following details about it:
+
+* protocol used, 
+* source IP and port, 
+* destination IP and port, 
+
+as per the below example:
+
+```
+TCP connection from 10.32.0.7:56648 to 10.32.0.11:80 blocked by Weave NPC.
+UDP connection from 10.32.0.7:56648 to 10.32.0.11:80 blocked by Weave NPC.
+```
+
+###<a name=”configuration-options”></a> Changing Configuration Options
+
+The default configuration settings can be changed by saving and editing the
+addon YAML before running `kubectl apply`. Additional arguments may be
 supplied to the Weave router process by adding them to the `command:`
-array in the yaml file.
+array in the YAML file.
+
+Some parameters are changed by environment variables; these can be
+inserted into the YAML file like this:
+
+```
+      containers:
+        - name: weave
+          env:
+            - name: IPALLOC_RANGE
+              value: 10.0.0.0/16
+```
+
+The list of variables you can set is:
+
+* CHECKPOINT\_DISABLE - if set to 1, disable checking for new Weave Net
+  versions (default is blank, i.e. check is enabled)
+* IPALLOC\_RANGE - the range of IP addresses used by Weave Net
+  and the subnet they are placed in (CIDR format; default 10.32.0.0/12)
+* EXPECT\_NPC - set to 0 to disable Network Policy Controller (default is on)
+* KUBE\_PEERS - list of addresses of peers in the Kubernetes cluster
+  (default is to fetch the list from the api-server)
+* IPALLOC\_INIT - set the initialization mode of the [IP Address
+  Manager](/site/operational-guide/concepts.md#ip-address-manager)
+  (defaults to consensus amongst the KUBE\_PEERS)
+* WEAVE\_EXPOSE\_IP - set the IP address used as a gateway from the
+  Weave network to the host network - this is useful if you are
+  configuring the addon as a static pod.

site/troubleshooting.md

@@ -302,7 +302,7 @@ latest snapshot release using:
     sudo chmod a+x /usr/local/bin/weave
     weave setup
 
-Snapshot releases report the script version as "(unreleased version)",
+Snapshot releases report the script version as "unreleased",
 and the container image versions as git hashes.
 
 

test/700_status_and_report_test.sh

@@ -6,6 +6,8 @@ UNIVERSE=10.2.0.0/16
 
 start_suite "weave status/report"
 
+
+
 weave_on $HOST1 launch --ipalloc-range $UNIVERSE --name 8a:3e:3e:3e:3e:3e --nickname nicknamington
 
 check() {
@@ -29,10 +31,33 @@ assert "weave_on $HOST1 status connections | tr -s ' ' | cut -d ' ' -f 2" "10.2.
 assert "weave_on $HOST1 report -f '{{.VersionCheck.Enabled}}'" "false"
 assert_raises "weave_on $HOST1 status | grep 'version check update disabled'"
 
+
+
+weave_on $HOST1 reset
+# Guarantee the version check fails by feeding an unresponsive IP address into the resolver
+$SSH $HOST1 tee /tmp/hosts.checkpoint-api > /dev/null <<EOF
+127.0.0.1 localhost
+127.1.1.1 checkpoint-api.weave.works
+EOF
+CHECKPOINT_DISABLE="" WEAVE_DOCKER_ARGS="-v /tmp/hosts.checkpoint-api:/etc/hosts" weave_on $HOST1 launch
+
+wait_for_version_check_error() {
+    while true; do
+        docker_on $HOST1 logs weave 2>&1 | grep 'Error checking version' && return
+        sleep 1
+    done
+}
+assert_raises "timeout 30 cat <( wait_for_version_check_error )"
+
+assert "weave_on $HOST1 report -f '{{.VersionCheck.Enabled}}'" "true"
+assert "weave_on $HOST1 report -f '{{.VersionCheck.Success}}'" "false"
+assert "weave_on $HOST1 report -f '{{.VersionCheck.NewVersion}}'" ""
+
 weave_on $HOST1 reset
 
 CHECKPOINT_DISABLE="" weave_on $HOST1 launch
 assert "weave_on $HOST1 report -f '{{.VersionCheck.Enabled}}'" "true"
+assert "weave_on $HOST1 report -f '{{.VersionCheck.Success}}'" "true"
 
 NEW_VSN=$(weave_on $HOST1 report  -f "{{.VersionCheck.NewVersion}}")
 if [ -z "$NEW_VSN" ]; then
@@ -41,4 +66,6 @@ else
     assert_raises "weave_on $HOST1 status | grep \"version $NEW_VSN available - please upgrade!\""
 fi
 
+
+
 end_suite

test/705_detect_hairpin_test.sh

@@ -0,0 +1,12 @@
+#! /bin/bash
+
+. ./config.sh
+
+start_suite "Detect hairpin mode"
+
+weave_on $HOST1 launch
+
+assert_raises "run_on $HOST1 sudo ip link set vethwe-bridge type bridge_slave hairpin on"
+assert_raises "docker_on $HOST1 logs weave 2>&1 | grep -q 'Hairpin mode enabled on \"vethwe-bridge\"'"
+
+end_suite

test/830_cni_plugin_test.sh

@@ -45,15 +45,33 @@ C2IP=$(container_ip $HOST1 c2)
 assert_raises "exec_on $HOST1 c1 $PING $C2IP"
 assert_raises "exec_on $HOST1 c2 $PING $C1IP"
 
-# Now remove and start a new container to see if IP address re-use breaks things
+# Now remove and start a new container to see if anything breaks
 docker_on $HOST1 rm -f c2
 
-docker_on $HOST1 run --net=none --name=c3 -dt $SMALL_IMAGE /bin/sh
+C3=$(docker_on $HOST1 run --net=none --name=c3 -dt $SMALL_IMAGE /bin/sh)
 
 cni_connect $HOST1 c3 <<EOF
 { "name": "weave", "type": "weave-net" }
 EOF
 
-assert_raises "exec_on $HOST1 c1 $PING $C2IP"
+C3IP=$(container_ip $HOST1 c3)
+
+# CNI shouldn't re-use the address until we call DEL
+assert_raises "[ $C2IP != $C3IP ]"
+assert_raises "exec_on $HOST1 c1 $PING $C3IP"
+
+
+# Ensure existing containers can reclaim their IP addresses after CNI has been used -- see #2548
+stop_weave_on $HOST1
+
+# Ensure no warning is printed to the standard error:
+ACTUAL_OUTPUT=$(weave_on $HOST1 launch-router 2>&1)
+EXPECTED_OUTPUT=$(docker_on $HOST1 inspect --format="{{.Id}}" weave)
+
+assert_raises "[ $EXPECTED_OUTPUT == $ACTUAL_OUTPUT ]"
+
+assert "$SSH $HOST1 \"curl -s -X GET 127.0.0.1:6784/ip/$C1 | grep -o -E '[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}'\"" "$C1IP"
+assert "$SSH $HOST1 \"curl -s -X GET 127.0.0.1:6784/ip/$C3 | grep -o -E '[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}'\"" "$C3IP"
+
 
 end_suite

test/840_weave_kube_3_test.sh

@@ -44,6 +44,14 @@ wait_for_connections() {
 
 assert_raises wait_for_connections
 
+# Check we can ping between the Weave bridg IPs on each host
+HOST1EXPIP=$($SSH $HOST1 "weave expose")
+HOST2EXPIP=$($SSH $HOST2 "weave expose")
+HOST3EXPIP=$($SSH $HOST3 "weave expose")
+assert_raises "run_on $HOST1 $PING $HOST2EXPIP"
+assert_raises "run_on $HOST2 $PING $HOST1EXPIP"
+assert_raises "run_on $HOST3 $PING $HOST2EXPIP"
+
 # See if we can get some pods running that connect to the network
 run_on $HOST1 "kubectl run hello --image=weaveworks/hello-world --replicas=3"
 

test/gce.sh

@@ -34,6 +34,9 @@ function vm_names {
 
 # Delete all vms in this account
 function destroy {
+	if [ $(gcloud compute firewall-rules list test-allow-docker$SUFFIX 2>/dev/null | wc -l) -gt 0 ] ; then
+		gcloud compute firewall-rules delete test-allow-docker$SUFFIX
+	fi
 	names="$(vm_names)"
 	if [ $(gcloud compute instances list --zones $ZONE -q $names | wc -l) -le 1 ] ; then
 		return 0
@@ -124,7 +127,9 @@ function setup {
 	destroy
 
 	names="$(vm_names)"
-	gcloud compute instances create $names --image $TEMPLATE_NAME --zone $ZONE
+	gcloud compute instances create $names --image $TEMPLATE_NAME --zone $ZONE --tags test$SUFFIX --network=test
+	my_ip="$(curl -s http://ipinfo.io/ip)"
+	gcloud compute firewall-rules create test-allow-docker$SUFFIX --network=test --allow tcp:2375,tcp:12375 --target-tags test$SUFFIX --source-ranges $my_ip
 	gcloud compute config-ssh --ssh-key-file $SSH_KEY_FILE
 	sed -i '/UserKnownHostsFile=\/dev\/null/d' ~/.ssh/config
 

weave

@@ -3,9 +3,9 @@ set -e
 
 [ -n "$WEAVE_DEBUG" ] && set -x
 
-SCRIPT_VERSION="(unreleased version)"
+SCRIPT_VERSION="unreleased"
 IMAGE_VERSION=latest
-[ "$SCRIPT_VERSION" = "(unreleased version)" ] || IMAGE_VERSION=$SCRIPT_VERSION
+[ "$SCRIPT_VERSION" = "unreleased" ] || IMAGE_VERSION=$SCRIPT_VERSION
 IMAGE_VERSION=${WEAVE_VERSION:-$IMAGE_VERSION}
 
 # - Docker versions prior to 1.6 cannot pull images from DockerHub
@@ -398,7 +398,13 @@ add_iptables_rule() {
     IPTABLES_TABLE="$1"
     shift 1
     if ! run_iptables -t $IPTABLES_TABLE -C "$@" >/dev/null 2>&1 ; then
-        run_iptables -t $IPTABLES_TABLE -A "$@" >/dev/null
+        ## Loop until we get an exit code other than "temporarily unavailable"
+        while true ; do
+            run_iptables -t $IPTABLES_TABLE -A "$@" >/dev/null && return 0
+            if [ $? != 4 ] ; then
+                return 1
+            fi
+        done
     fi
 }
 
@@ -407,7 +413,13 @@ insert_iptables_rule() {
     IPTABLES_TABLE="$1"
     shift 1
     if ! run_iptables -t $IPTABLES_TABLE -C "$@" >/dev/null 2>&1 ; then
-        run_iptables -t $IPTABLES_TABLE -I "$@" >/dev/null
+        ## Loop until we get an exit code other than "temporarily unavailable"
+        while true ; do
+            run_iptables -t $IPTABLES_TABLE -I "$@" >/dev/null && return 0
+            if [ $? != 4 ] ; then
+                return 1
+            fi
+        done
     fi
 }
 
@@ -559,7 +571,7 @@ detect_bridge_type() {
     MTU=${WEAVE_MTU:-$(cat /sys/class/net/$BRIDGE/mtu)}
 }
 
-create_bridge() {
+try_create_bridge() {
     if ! detect_bridge_type ; then
         BRIDGE_TYPE=bridge
         if [ -z "$WEAVE_NO_FASTDP" ] ; then
@@ -644,6 +656,16 @@ EOF
     configure_arp_cache $BRIDGE
 }
 
+create_bridge() {
+    if ! try_create_bridge "$@" ; then
+        echo "Creating bridge '$BRIDGE' failed" >&2
+        # reset to original value so we destroy both kinds
+        DATAPATH=datapath
+        destroy_bridge
+        exit 1
+    fi
+}
+
 expose_ip() {
     ipam_cidrs allocate_no_check_alive weave:expose $CIDR_ARGS
     for CIDR in $ALL_CIDRS ; do
@@ -661,14 +683,6 @@ expose_ip() {
     done
 }
 
-expose_nat() {
-    for CIDR in $ALL_CIDRS ; do
-        add_iptables_rule nat WEAVE -s $CIDR -d 224.0.0.0/4 -j RETURN
-        add_iptables_rule nat WEAVE -d $CIDR ! -s $CIDR -j MASQUERADE
-        add_iptables_rule nat WEAVE -s $CIDR ! -d $CIDR -j MASQUERADE
-    done
-}
-
 # create veth with ends $1-$2, and then invoke $3..., removing the
 # veth on failure. No-op of veth already exists.
 create_veth() {
@@ -2222,7 +2236,7 @@ EOF
         fi
         create_bridge --without-ethtool
         expose_ip
-        expose_nat
+        util_op expose-nat $ALL_CIDRS
         show_addrs $ALL_CIDRS
         ;;
     hide)