Merge pull request #1499 from weaveworks/1463-ipam-seed-mismatch

detect when IPAM has been seeded by different peers

ipam/allocator.go

@@ -342,6 +342,18 @@ func (alloc *Allocator) pruneNicknames() {
 	}
 }
 
+func (alloc *Allocator) annotatePeernames(names []router.PeerName) []string {
+	var res []string
+	for _, name := range names {
+		if nickname, found := alloc.nicknames[name]; found {
+			res = append(res, fmt.Sprint(name, "(", nickname, ")"))
+		} else {
+			res = append(res, name.String())
+		}
+	}
+	return res
+}
+
 func decodeRange(msg []byte) (r address.Range, err error) {
 	decoder := gob.NewDecoder(bytes.NewReader(msg))
 	return r, decoder.Decode(&r)
@@ -606,16 +618,20 @@ func (alloc *Allocator) update(sender router.PeerName, msg []byte) error {
 	// shouldn't get updates for a empty Ring. But tolerate
 	// them just in case.
 	if data.Ring != nil {
-		if data.Ring.Range() != alloc.universe {
-			return fmt.Errorf("Incompatible IP allocation range %s; ours is %s",
-				data.Ring.Range().AsCIDRString(), alloc.universe.AsCIDRString())
+		switch err = alloc.ring.Merge(*data.Ring); err {
+		case ring.ErrDifferentSeeds:
+			return fmt.Errorf("IP allocation was seeded by different peers (received: %v, ours: %v)",
+				alloc.annotatePeernames(data.Ring.Seeds), alloc.annotatePeernames(alloc.ring.Seeds))
+		case ring.ErrDifferentRange:
+			return fmt.Errorf("Incompatible IP allocation ranges (received: %s, ours: %s)",
+				data.Ring.Range().AsCIDRString(), alloc.ring.Range().AsCIDRString())
+		default:
+			if err == nil && !alloc.ring.Empty() {
+				alloc.pruneNicknames()
+				alloc.ringUpdated()
+			}
+			return err
 		}
-		err = alloc.ring.Merge(*data.Ring)
-		if !alloc.ring.Empty() {
-			alloc.pruneNicknames()
-			alloc.ringUpdated()
-		}
-		return err
 	}
 
 	if data.Paxos != nil {

ipam/ring/ring.go

@@ -19,9 +19,10 @@ import (
 
 // Ring represents the ring itself
 type Ring struct {
-	Start, End address.Address // [min, max) tokens in this ring.  Due to wrapping, min == max (effectively)
-	Peer       router.PeerName // name of peer owning this ring instance
-	Entries    entries         // list of entries sorted by token
+	Start, End address.Address   // [min, max) tokens in this ring.  Due to wrapping, min == max (effectively)
+	Peer       router.PeerName   // name of peer owning this ring instance
+	Entries    entries           // list of entries sorted by token
+	Seeds      []router.PeerName // peers with which the ring was seeded
 }
 
 func (r *Ring) assertInvariants() {
@@ -36,6 +37,7 @@ var (
 	ErrNotSorted       = errors.New("Ring not sorted")
 	ErrTokenRepeated   = errors.New("Token appears twice in ring")
 	ErrTokenOutOfRange = errors.New("Token is out of range")
+	ErrDifferentSeeds  = errors.New("Received ring was seeded differently from ours!")
 	ErrDifferentRange  = errors.New("Received range differs from ours!")
 	ErrNewerVersion    = errors.New("Received new version for entry I own!")
 	ErrInvalidEntry    = errors.New("Received invalid state update!")
@@ -188,6 +190,17 @@ func (r *Ring) Merge(gossip Ring) error {
 		return err
 	}
 
+	if len(gossip.Seeds) > 0 && len(r.Seeds) > 0 {
+		if len(gossip.Seeds) != len(r.Seeds) {
+			return ErrDifferentSeeds
+		}
+		for i, seed := range gossip.Seeds {
+			if seed != r.Seeds[i] {
+				return ErrDifferentSeeds
+			}
+		}
+	}
+
 	if r.Start != gossip.Start || r.End != gossip.End {
 		return ErrDifferentRange
 	}
@@ -254,6 +267,9 @@ func (r *Ring) Merge(gossip Ring) error {
 		previousOwner = nil
 	}
 
+	if len(r.Seeds) == 0 {
+		r.Seeds = gossip.Seeds
+	}
 	r.Entries = result
 	return nil
 }
@@ -330,6 +346,8 @@ func (r *Ring) ClaimForPeers(peers []router.PeerName) {
 	}
 
 	common.Assert(pos == r.End)
+
+	r.Seeds = peers
 }
 
 func (r *Ring) FprintWithNicknames(w io.Writer, m map[router.PeerName]string) {

test/180_ipalloc_seed_mismatch_3.sh

@@ -0,0 +1,28 @@
+#! /bin/bash
+
+. ./config.sh
+
+start_suite "IPAM ring seed mismatch"
+
+weave_on $HOST1 launch-router --no-discovery
+weave_on $HOST2 launch-router --no-discovery
+weave_on $HOST3 launch-router --no-discovery
+
+# Do some allocations to get the ring initialized on two hosts
+start_container $HOST1
+start_container $HOST2
+
+# Connect 2 to 3 and allocate so they will each have half the ring
+weave_on $HOST2 connect $HOST3
+start_container $HOST3
+
+# More allocations on 1 to increase the version number
+start_container $HOST1
+start_container $HOST1
+
+weave_on $HOST1 connect $HOST3
+
+# Check that weave on 3 is still running
+assert_raises "weave_on $HOST3 status"
+
+end_suite