Ring CRDT for IP Allocation Management

The basic data structure used to share information about
IP allocations between peers

ipam/ring/entry.go

@@ -0,0 +1,102 @@
+package ring
+
+import (
+	"sort"
+
+	"github.com/weaveworks/weave/common"
+	"github.com/weaveworks/weave/ipam/address"
+	"github.com/weaveworks/weave/router"
+)
+
+// Entry represents entries around the ring
+type entry struct {
+	Token   address.Address // The start of this range
+	Peer    router.PeerName // Who owns this range
+	Version uint32          // Version of this range
+	Free    address.Offset  // Number of free IPs in this range
+}
+
+func (e *entry) Equal(e2 *entry) bool {
+	return e.Token == e2.Token && e.Peer == e2.Peer &&
+		e.Version == e2.Version
+}
+
+func (e *entry) update(peername router.PeerName, free address.Offset) {
+	e.Peer = peername
+	e.Version++
+	e.Free = free
+}
+
+// For compatibility with sort.Interface
+type entries []*entry
+
+func (es entries) Len() int           { return len(es) }
+func (es entries) Less(i, j int) bool { return es[i].Token < es[j].Token }
+func (es entries) Swap(i, j int)      { panic("Should never be swapping entries!") }
+
+func (es entries) entry(i int) *entry {
+	i = i % len(es)
+	if i < 0 {
+		i += len(es)
+	}
+	return es[i]
+}
+
+func (es *entries) insert(e entry) {
+	i := sort.Search(len(*es), func(j int) bool {
+		return (*es)[j].Token >= e.Token
+	})
+
+	if i < len(*es) && (*es)[i].Token == e.Token {
+		panic("Trying to insert an existing token!")
+	}
+
+	*es = append(*es, &entry{})
+	copy((*es)[i+1:], (*es)[i:])
+	(*es)[i] = &e
+}
+
+func (es entries) get(token address.Address) (*entry, bool) {
+	i := sort.Search(len(es), func(j int) bool {
+		return es[j].Token >= token
+	})
+
+	if i < len(es) && es[i].Token == token {
+		return es[i], true
+	}
+
+	return nil, false
+}
+
+func (es *entries) remove(i int) {
+	*es = (*es)[:i+copy((*es)[i:], (*es)[i+1:])]
+}
+
+// Is token between entries at i and j?
+// NB i and j can overflow and will wrap
+// NBB this function does not work very well if there is only one
+//     token on the ring; luckily an accurate answer is not needed
+//     by the call sites in this case.
+func (es entries) between(token address.Address, i, j int) bool {
+	common.Assert(i < j)
+
+	first := es.entry(i)
+	second := es.entry(j)
+
+	switch {
+	case first.Token == second.Token:
+		// This implies there is only one token
+		// on the ring (i < j and i.token == j.token)
+		// In which case everything is between, expect
+		// this one token
+		return token != first.Token
+
+	case first.Token < second.Token:
+		return first.Token <= token && token < second.Token
+
+	case second.Token < first.Token:
+		return first.Token <= token || token < second.Token
+	}
+
+	panic("Should never get here - switch covers all possibilities.")
+}

ipam/ring/expvar.go

@@ -0,0 +1,31 @@
+package ring
+
+/* Exported variables for monitoring and management */
+
+import (
+	"expvar"
+	"strconv"
+)
+
+var (
+	expRingSize    = expvar.NewMap("ipam.ringSize")
+	expRingEntries = expvar.NewMap("ipam.ringEntries")
+)
+
+type _int int
+
+func (i _int) String() string {
+	return strconv.Itoa(int(i))
+}
+
+type _uint32 uint32
+
+func (i _uint32) String() string {
+	return strconv.FormatUint(uint64(i), 10)
+}
+
+func (r *Ring) updateExportedVariables() {
+	ringName := r.Start.String()
+	expRingSize.Set(ringName, _uint32(r.End-r.Start))
+	expRingEntries.Set(ringName, _int(len(r.Entries)))
+}

ipam/ring/ring.go

@@ -0,0 +1,521 @@
+/*
+Package ring implements a simple ring CRDT.
+*/
+package ring
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"math/rand"
+	"sort"
+
+	"github.com/weaveworks/weave/common"
+	"github.com/weaveworks/weave/ipam/address"
+	"github.com/weaveworks/weave/router"
+)
+
+// Ring represents the ring itself
+type Ring struct {
+	Start, End address.Address // [min, max) tokens in this ring.  Due to wrapping, min == max (effectively)
+	Peer       router.PeerName // name of peer owning this ring instance
+	Entries    entries         // list of entries sorted by token
+}
+
+func (r *Ring) assertInvariants() {
+	err := r.checkInvariants()
+	if err != nil {
+		panic(err.Error())
+	}
+}
+
+// Errors returned by Merge
+var (
+	ErrNotSorted        = errors.New("Ring not sorted")
+	ErrTokenRepeated    = errors.New("Token appears twice in ring")
+	ErrTokenOutOfRange  = errors.New("Token is out of range")
+	ErrDifferentSubnets = errors.New("IP Allocator with different subnet detected")
+	ErrNewerVersion     = errors.New("Received new version for entry I own!")
+	ErrInvalidEntry     = errors.New("Received invalid state update!")
+	ErrEntryInMyRange   = errors.New("Received new entry in my range!")
+	ErrNoFreeSpace      = errors.New("No free space found!")
+	ErrTooMuchFreeSpace = errors.New("Entry reporting too much free space!")
+	ErrInvalidTimeout   = errors.New("dt must be greater than 0")
+	ErrNotFound         = errors.New("No entries for peer found")
+)
+
+func (r *Ring) checkInvariants() error {
+	if !sort.IsSorted(r.Entries) {
+		return ErrNotSorted
+	}
+
+	// Check no token appears twice
+	// We know it's sorted...
+	for i := 1; i < len(r.Entries); i++ {
+		if r.Entries[i-1].Token == r.Entries[i].Token {
+			return ErrTokenRepeated
+		}
+	}
+
+	if len(r.Entries) == 0 {
+		return nil
+	}
+
+	// Check tokens are in range
+	if r.Entries.entry(0).Token < r.Start {
+		return ErrTokenOutOfRange
+	}
+	if r.Entries.entry(-1).Token >= r.End {
+		return ErrTokenOutOfRange
+	}
+
+	// Check all the freespaces are in range
+	for i, entry := range r.Entries {
+		next := r.Entries.entry(i + 1)
+		distance := r.distance(entry.Token, next.Token)
+
+		if entry.Free > distance {
+			return ErrTooMuchFreeSpace
+		}
+	}
+
+	return nil
+}
+
+// New creates an empty ring belonging to peer.
+func New(start, end address.Address, peer router.PeerName) *Ring {
+	common.Assert(start < end)
+
+	ring := &Ring{Start: start, End: end, Peer: peer, Entries: make([]*entry, 0)}
+	ring.updateExportedVariables()
+	return ring
+}
+
+// TotalRemoteFree returns the approximate number of free IPs
+// on other hosts.
+func (r *Ring) TotalRemoteFree() address.Offset {
+	result := address.Offset(0)
+	for _, entry := range r.Entries {
+		if entry.Peer != r.Peer {
+			result += entry.Free
+		}
+	}
+	return result
+}
+
+// Returns the distance between two tokens on this ring, dealing
+// with ranges which cross the origin
+func (r *Ring) distance(start, end address.Address) address.Offset {
+	if end > start {
+		return address.Offset(end - start)
+	}
+
+	return address.Offset((r.End - start) + (end - r.Start))
+}
+
+// GrantRangeToHost modifies the ring such that range [start, end)
+// is assigned to peer.  This may insert up to two new tokens.
+// Preconditions:
+// - start < end
+// - [start, end) must be owned by the calling peer
+func (r *Ring) GrantRangeToHost(start, end address.Address, peer router.PeerName) {
+	//fmt.Printf("%s GrantRangeToHost [%v,%v) -> %s\n", r.Peer, start, end, peer)
+
+	r.assertInvariants()
+	defer r.assertInvariants()
+	defer r.updateExportedVariables()
+
+	// ----------------- Start of Checks -----------------
+
+	common.Assert(start < end)
+	common.Assert(r.Start <= start && start < r.End)
+	common.Assert(r.Start < end && end <= r.End)
+	common.Assert(len(r.Entries) > 0)
+
+	// Look for the left-most entry greater than start, then go one previous
+	// to get the right-most entry less than or equal to start
+	preceedingPos := sort.Search(len(r.Entries), func(j int) bool {
+		return r.Entries[j].Token > start
+	})
+	preceedingPos--
+
+	// Check all tokens up to end are owned by us
+	for pos := preceedingPos; pos < len(r.Entries) && r.Entries.entry(pos).Token < end; pos++ {
+		common.Assert(r.Entries.entry(pos).Peer == r.Peer)
+	}
+
+	// ----------------- End of Checks -----------------
+
+	// Free space at start is max(length of range, distance to next token)
+	startFree := r.distance(start, r.Entries.entry(preceedingPos+1).Token)
+	if length := r.distance(start, end); startFree > length {
+		startFree = length
+	}
+	// Is there already a token at start, update it
+	if previousEntry := r.Entries.entry(preceedingPos); previousEntry.Token == start {
+		previousEntry.update(peer, startFree)
+	} else {
+		// Otherwise, these isn't a token here, insert a new one.
+		r.Entries.insert(entry{Token: start, Peer: peer, Free: startFree})
+		preceedingPos++
+		// Reset free space on previous entry, which we own.
+		previousEntry.update(r.Peer, r.distance(previousEntry.Token, start))
+	}
+
+	// Give all intervening tokens to the other peer
+	pos := preceedingPos + 1
+	for ; pos < len(r.Entries) && r.Entries.entry(pos).Token < end; pos++ {
+		entry := r.Entries.entry(pos)
+		entry.update(peer, entry.Free)
+	}
+
+	// There is never an entry with a token of r.End, as the end of
+	// the ring is exclusive.
+	if end == r.End {
+		end = r.Start
+	}
+
+	//  If there is a token equal to the end of the range, we don't need to do anything further
+	if _, found := r.Entries.get(end); found {
+		return
+	}
+
+	// If not, we need to insert a token such that we claim this bit on the end.
+	endFree := r.distance(end, r.Entries.entry(pos).Token)
+	r.Entries.insert(entry{Token: end, Peer: r.Peer, Free: endFree})
+}
+
+// Merge the given ring into this ring and return any new ranges added
+func (r *Ring) Merge(gossip Ring) error {
+	r.assertInvariants()
+	defer r.assertInvariants()
+	defer r.updateExportedVariables()
+
+	// Don't panic when checking the gossiped in ring.
+	// In this case just return any error found.
+	if err := gossip.checkInvariants(); err != nil {
+		return err
+	}
+
+	if r.Start != gossip.Start || r.End != gossip.End {
+		return ErrDifferentSubnets
+	}
+
+	// Now merge their ring with yours, in a temporary ring.
+	var result entries
+	addToResult := func(e entry) { result = append(result, &e) }
+
+	var mine, theirs *entry
+	var previousOwner *router.PeerName
+	// i is index into r.Entries; j is index into gossip.Entries
+	var i, j int
+	for i < len(r.Entries) && j < len(gossip.Entries) {
+		mine, theirs = r.Entries[i], gossip.Entries[j]
+		switch {
+		case mine.Token < theirs.Token:
+			addToResult(*mine)
+			previousOwner = &mine.Peer
+			i++
+		case mine.Token > theirs.Token:
+			// insert, checking that a range owned by us hasn't been split
+			if previousOwner != nil && *previousOwner == r.Peer && theirs.Peer != r.Peer {
+				return ErrEntryInMyRange
+			}
+			addToResult(*theirs)
+			previousOwner = nil
+			j++
+		case mine.Token == theirs.Token:
+			// merge
+			switch {
+			case mine.Version >= theirs.Version:
+				if mine.Version == theirs.Version && !mine.Equal(theirs) {
+					common.Debug.Printf("Error merging entries at %s - %v != %v\n", mine.Token, mine, theirs)
+					return ErrInvalidEntry
+				}
+				addToResult(*mine)
+				previousOwner = &mine.Peer
+			case mine.Version < theirs.Version:
+				if mine.Peer == r.Peer { // We shouldn't receive updates to our own tokens
+					return ErrNewerVersion
+				}
+				addToResult(*theirs)
+				previousOwner = nil
+			}
+			i++
+			j++
+		}
+	}
+
+	// At this point, either i is at the end of r or j is at the end
+	// of gossip, so copy over the remaining entries.
+
+	for ; i < len(r.Entries); i++ {
+		mine = r.Entries[i]
+		addToResult(*mine)
+	}
+
+	for ; j < len(gossip.Entries); j++ {
+		theirs = gossip.Entries[j]
+		if previousOwner != nil && *previousOwner == r.Peer && theirs.Peer != r.Peer {
+			return ErrEntryInMyRange
+		}
+		addToResult(*theirs)
+		previousOwner = nil
+	}
+
+	r.Entries = result
+	return nil
+}
+
+// Empty returns true if the ring has no entries
+func (r *Ring) Empty() bool {
+	return len(r.Entries) == 0
+}
+
+// Given a slice of ranges which are all in the right order except
+// possibly the last one spans zero, fix that up and return the slice
+func (r *Ring) splitRangesOverZero(ranges []address.Range) []address.Range {
+	if len(ranges) == 0 {
+		return ranges
+	}
+	lastRange := ranges[len(ranges)-1]
+	// if end token == start (ie last) entry on ring, we want to actually use r.End
+	if lastRange.End == r.Start {
+		ranges[len(ranges)-1].End = r.End
+	} else if lastRange.End <= lastRange.Start {
+		// We wrapped; want to split around 0
+		// First shuffle everything up as we want results to be sorted
+		ranges = append(ranges, address.Range{})
+		copy(ranges[1:], ranges[:len(ranges)-1])
+		ranges[0] = address.Range{Start: r.Start, End: lastRange.End}
+		ranges[len(ranges)-1].End = r.End
+	}
+	return ranges
+}
+
+// OwnedRanges returns slice of Ranges, ordered by IP, indicating which
+// ranges are owned by this peer.  Will split ranges which
+// span 0 in the ring.
+func (r *Ring) OwnedRanges() (result []address.Range) {
+	r.assertInvariants()
+
+	for i, entry := range r.Entries {
+		if entry.Peer == r.Peer {
+			nextEntry := r.Entries.entry(i + 1)
+			result = append(result, address.Range{Start: entry.Token, End: nextEntry.Token})
+		}
+	}
+
+	return r.splitRangesOverZero(result)
+}
+
+// ClaimForPeers claims the entire ring for the array of peers passed
+// in.  Only works for empty rings.
+func (r *Ring) ClaimForPeers(peers []router.PeerName) {
+	common.Assert(r.Empty())
+	defer r.assertInvariants()
+	defer r.updateExportedVariables()
+
+	totalSize := r.distance(r.Start, r.End)
+	share := totalSize/address.Offset(len(peers)) + 1
+	remainder := totalSize % address.Offset(len(peers))
+	pos := r.Start
+
+	for i, peer := range peers {
+		if address.Offset(i) == remainder {
+			share--
+			if share == 0 {
+				break
+			}
+		}
+
+		if e, found := r.Entries.get(pos); found {
+			e.update(peer, share)
+		} else {
+			r.Entries.insert(entry{Token: pos, Peer: peer, Free: share})
+		}
+
+		pos += address.Address(share)
+	}
+
+	common.Assert(pos == r.End)
+}
+
+func (r *Ring) ClaimItAll() {
+	r.ClaimForPeers([]router.PeerName{r.Peer})
+}
+
+func (r *Ring) FprintWithNicknames(w io.Writer, m map[router.PeerName]string) {
+	fmt.Fprintf(w, "Ring [%s, %s)\n", r.Start, r.End)
+	for _, entry := range r.Entries {
+		nickname, found := m[entry.Peer]
+		if found {
+			nickname = fmt.Sprintf(" (%s)", nickname)
+		}
+
+		fmt.Fprintf(w, "  %s -> %s%s (version: %d, free: %d)\n", entry.Token,
+			entry.Peer, nickname, entry.Version, entry.Free)
+	}
+}
+
+func (r *Ring) String() string {
+	var buffer bytes.Buffer
+	r.FprintWithNicknames(&buffer, make(map[router.PeerName]string))
+	return buffer.String()
+}
+
+// ReportFree is used by the allocator to tell the ring
+// how many free ips are in a given range, so that ChoosePeerToAskForSpace
+// can make more intelligent decisions.
+func (r *Ring) ReportFree(freespace map[address.Address]address.Offset) {
+	r.assertInvariants()
+	defer r.assertInvariants()
+	defer r.updateExportedVariables()
+
+	common.Assert(!r.Empty())
+	entries := r.Entries
+
+	// As OwnedRanges splits around the origin, we need to
+	// detect that here and fix up freespace
+	if free, found := freespace[r.Start]; found && entries.entry(0).Token != r.Start {
+		lastToken := entries.entry(-1).Token
+		prevFree, found := freespace[lastToken]
+		common.Assert(found)
+		freespace[lastToken] = prevFree + free
+		delete(freespace, r.Start)
+	}
+
+	for start, free := range freespace {
+		// Look for entry
+		i := sort.Search(len(entries), func(j int) bool {
+			return entries[j].Token >= start
+		})
+
+		// Are you trying to report free on space I don't own?
+		common.Assert(i < len(entries) && entries[i].Token == start && entries[i].Peer == r.Peer)
+
+		// Check we're not reporting more space than the range
+		entry, next := entries.entry(i), entries.entry(i+1)
+		maxSize := r.distance(entry.Token, next.Token)
+		common.Assert(free <= maxSize)
+
+		if entries[i].Free == free {
+			return
+		}
+
+		entries[i].Free = free
+		entries[i].Version++
+	}
+}
+
+// ChoosePeerToAskForSpace chooses a weighted-random peer to ask
+// for space.
+func (r *Ring) ChoosePeerToAskForSpace() (result router.PeerName, err error) {
+	var (
+		sum               address.Offset
+		totalSpacePerPeer = make(map[router.PeerName]address.Offset) // Compute total free space per peer
+	)
+
+	// iterate through tokens
+	for _, entry := range r.Entries {
+		// Ignore ranges with no free space
+		if entry.Free <= 0 {
+			continue
+		}
+
+		// Don't talk to yourself
+		if entry.Peer == r.Peer {
+			continue
+		}
+
+		totalSpacePerPeer[entry.Peer] += entry.Free
+		sum += entry.Free
+	}
+
+	if sum == 0 {
+		err = ErrNoFreeSpace
+		return
+	}
+
+	// Pick random peer, weighted by total free space
+	rn := rand.Int63n(int64(sum))
+	for peername, space := range totalSpacePerPeer {
+		rn -= int64(space)
+		if rn < 0 {
+			return peername, nil
+		}
+	}
+
+	panic("Should never reach this point")
+}
+
+func (r *Ring) PickPeerForTransfer() router.PeerName {
+	for _, entry := range r.Entries {
+		if entry.Peer != r.Peer {
+			return entry.Peer
+		}
+	}
+	return router.UnknownPeerName
+}
+
+// Transfer will mark all entries associated with 'from' peer as owned by 'to' peer
+// and return ranges indicating the new space we picked up
+func (r *Ring) Transfer(from, to router.PeerName) (error, []address.Range) {
+	r.assertInvariants()
+	defer r.assertInvariants()
+	defer r.updateExportedVariables()
+
+	var newRanges []address.Range
+	found := false
+
+	for i, entry := range r.Entries {
+		if entry.Peer == from {
+			found = true
+			entry.Peer = to
+			entry.Version++
+			newRanges = append(newRanges, address.Range{Start: entry.Token, End: r.Entries.entry(i + 1).Token})
+		}
+	}
+
+	if !found {
+		return ErrNotFound, nil
+	}
+
+	return nil, r.splitRangesOverZero(newRanges)
+}
+
+// Contains returns true if addr is in this ring
+func (r *Ring) Contains(addr address.Address) bool {
+	return addr >= r.Start && addr < r.End
+}
+
+// Owner returns the peername which owns the range containing addr
+func (r *Ring) Owner(token address.Address) router.PeerName {
+	common.Assert(r.Start <= token && token < r.End)
+
+	r.assertInvariants()
+	// There can be no owners on an empty ring
+	if r.Empty() {
+		return router.UnknownPeerName
+	}
+
+	// Look for the right-most entry, less than or equal to token
+	preceedingEntry := sort.Search(len(r.Entries), func(j int) bool {
+		return r.Entries[j].Token > token
+	})
+	preceedingEntry--
+	entry := r.Entries.entry(preceedingEntry)
+	return entry.Peer
+}
+
+// Get the set of PeerNames mentioned in the ring
+func (r *Ring) PeerNames() map[router.PeerName]struct{} {
+	res := make(map[router.PeerName]struct{})
+
+	for _, entry := range r.Entries {
+		res[entry.Peer] = struct{}{}
+	}
+
+	return res
+}

ipam/ring/ring_test.go

@@ -0,0 +1,713 @@
+package ring
+
+import (
+	"bytes"
+	"fmt"
+	"math/rand"
+	"sort"
+	"testing"
+
+	"github.com/weaveworks/weave/common"
+	"github.com/weaveworks/weave/ipam/address"
+	"github.com/weaveworks/weave/router"
+	wt "github.com/weaveworks/weave/testing"
+)
+
+var (
+	peer1name, _ = router.PeerNameFromString("01:00:00:00:02:00")
+	peer2name, _ = router.PeerNameFromString("02:00:00:00:02:00")
+	peer3name, _ = router.PeerNameFromString("03:00:00:00:02:00")
+
+	start, end    = ParseIP("10.0.0.0"), ParseIP("10.0.0.255")
+	dot10, dot245 = ParseIP("10.0.0.10"), ParseIP("10.0.0.245")
+	dot250        = ParseIP("10.0.0.250")
+	middle        = ParseIP("10.0.0.128")
+)
+
+func ParseIP(s string) address.Address {
+	addr, _ := address.ParseIP(s)
+	return addr
+}
+
+func TestInvariants(t *testing.T) {
+	ring := New(start, end, peer1name)
+
+	// Check ring is sorted
+	ring.Entries = []*entry{{Token: dot245, Peer: peer1name}, {Token: dot10, Peer: peer2name}}
+	wt.AssertTrue(t, ring.checkInvariants() == ErrNotSorted, "Expected error")
+
+	// Check tokens don't appear twice
+	ring.Entries = []*entry{{Token: dot245, Peer: peer1name}, {Token: dot245, Peer: peer2name}}
+	wt.AssertTrue(t, ring.checkInvariants() == ErrTokenRepeated, "Expected error")
+
+	// Check tokens are in bounds
+	ring = New(dot10, dot245, peer1name)
+	ring.Entries = []*entry{{Token: start, Peer: peer1name}}
+	wt.AssertTrue(t, ring.checkInvariants() == ErrTokenOutOfRange, "Expected error")
+
+	ring.Entries = []*entry{{Token: end, Peer: peer1name}}
+	wt.AssertTrue(t, ring.checkInvariants() == ErrTokenOutOfRange, "Expected error")
+}
+
+func TestInsert(t *testing.T) {
+	ring := New(start, end, peer1name)
+	ring.Entries = []*entry{{Token: start, Peer: peer1name, Free: 255}}
+
+	wt.AssertPanic(t, func() {
+		ring.Entries.insert(entry{Token: start, Peer: peer1name})
+	})
+
+	ring.Entries.entry(0).Free = 0
+	ring.Entries.insert(entry{Token: dot245, Peer: peer1name})
+	ring2 := New(start, end, peer1name)
+	ring2.Entries = []*entry{{Token: start, Peer: peer1name, Free: 0}, {Token: dot245, Peer: peer1name}}
+	wt.AssertEquals(t, ring, ring2)
+
+	ring.Entries.insert(entry{Token: dot10, Peer: peer1name})
+	ring2.Entries = []*entry{{Token: start, Peer: peer1name, Free: 0}, {Token: dot10, Peer: peer1name}, {Token: dot245, Peer: peer1name}}
+	wt.AssertEquals(t, ring, ring2)
+}
+
+func TestBetween(t *testing.T) {
+	ring1 := New(start, end, peer1name)
+	ring1.Entries = []*entry{{Token: start, Peer: peer1name, Free: 255}}
+
+	// First off, in a ring where everything is owned by the peer
+	// between should return true for everything
+	for i := 1; i <= 255; i++ {
+		ip := ParseIP(fmt.Sprintf("10.0.0.%d", i))
+		wt.AssertTrue(t, ring1.Entries.between(ip, 0, 1), "between should be true!")
+	}
+
+	// Now, construct a ring with entries at +10 and -10
+	// And check the correct behaviour
+
+	ring1.Entries = []*entry{{Token: dot10, Peer: peer1name}, {Token: dot245, Peer: peer2name}}
+	ring1.assertInvariants()
+	for i := 10; i <= 244; i++ {
+		ipStr := fmt.Sprintf("10.0.0.%d", i)
+		ip := ParseIP(ipStr)
+		wt.AssertTrue(t, ring1.Entries.between(ip, 0, 1),
+			fmt.Sprintf("Between should be true for %s!", ipStr))
+		wt.AssertFalse(t, ring1.Entries.between(ip, 1, 2),
+			fmt.Sprintf("Between should be false for %s!", ipStr))
+	}
+	for i := 0; i <= 9; i++ {
+		ipStr := fmt.Sprintf("10.0.0.%d", i)
+		ip := ParseIP(ipStr)
+		wt.AssertFalse(t, ring1.Entries.between(ip, 0, 1),
+			fmt.Sprintf("Between should be false for %s!", ipStr))
+		wt.AssertTrue(t, ring1.Entries.between(ip, 1, 2),
+			fmt.Sprintf("Between should be true for %s!", ipStr))
+	}
+	for i := 245; i <= 255; i++ {
+		ipStr := fmt.Sprintf("10.0.0.%d", i)
+		ip := ParseIP(ipStr)
+		wt.AssertFalse(t, ring1.Entries.between(ip, 0, 1),
+			fmt.Sprintf("Between should be false for %s!", ipStr))
+		wt.AssertTrue(t, ring1.Entries.between(ip, 1, 2),
+			fmt.Sprintf("Between should be true for %s!", ipStr))
+	}
+}
+
+func TestGrantSimple(t *testing.T) {
+	ring1 := New(start, end, peer1name)
+	ring2 := New(start, end, peer2name)
+
+	// Claim everything for peer1
+	ring1.ClaimItAll()
+	wt.AssertEquals(t, ring1.Entries, entries{{Token: start, Peer: peer1name, Free: 255}})
+
+	// Now grant everything to peer2
+	ring1.GrantRangeToHost(start, end, peer2name)
+	ring2.Entries = []*entry{{Token: start, Peer: peer2name, Free: 255, Version: 1}}
+	wt.AssertEquals(t, ring1.Entries, ring2.Entries)
+
+	// Now spint back to peer 1
+	ring2.GrantRangeToHost(dot10, end, peer1name)
+	ring1.Entries = []*entry{{Token: start, Peer: peer2name, Free: 10, Version: 2},
+		{Token: dot10, Peer: peer1name, Free: 245}}
+	wt.AssertEquals(t, ring1.Entries, ring2.Entries)
+
+	// And spint back to peer 2 again
+	ring1.GrantRangeToHost(dot245, end, peer2name)
+	wt.AssertEquals(t, ring1.Entries, entries{{Token: start, Peer: peer2name, Free: 10, Version: 2},
+		{Token: dot10, Peer: peer1name, Free: 235, Version: 1},
+		{Token: dot245, Peer: peer2name, Free: 10}})
+
+	// Grant range spanning a live token
+	ring1.Entries = []*entry{{Token: start, Peer: peer1name, Free: 10, Version: 2},
+		{Token: dot10, Peer: peer1name, Free: 235}, {Token: dot245, Peer: peer1name, Free: 10}}
+	ring1.GrantRangeToHost(dot10, end, peer2name)
+	wt.AssertEquals(t, ring1.Entries, entries{{Token: start, Peer: peer1name, Free: 10, Version: 2},
+		{Token: dot10, Peer: peer2name, Free: 235, Version: 1},
+		{Token: dot245, Peer: peer2name, Free: 10, Version: 1}})
+}
+
+func TestGrantSplit(t *testing.T) {
+	ring1 := New(start, end, peer1name)
+	ring2 := New(start, end, peer2name)
+
+	// Claim everything for peer1
+	ring1.Entries = []*entry{{Token: start, Peer: peer1name, Free: 255}}
+	ring2.Merge(*ring1)
+	wt.AssertEquals(t, ring1.Entries, ring2.Entries)
+
+	// Now grant a split range to peer2
+	ring1.GrantRangeToHost(dot10, dot245, peer2name)
+	wt.AssertEquals(t, ring1.Entries, entries{{Token: start, Peer: peer1name, Free: 10, Version: 1},
+		{Token: dot10, Peer: peer2name, Free: 235},
+		{Token: dot245, Peer: peer1name, Free: 10}})
+}
+
+func TestMergeSimple(t *testing.T) {
+	ring1 := New(start, end, peer1name)
+	ring2 := New(start, end, peer2name)
+
+	// Claim everything for peer1
+	ring1.ClaimItAll()
+	ring1.GrantRangeToHost(middle, end, peer2name)
+	wt.AssertSuccess(t, ring2.Merge(*ring1))
+
+	wt.AssertEquals(t, ring1.Entries, entries{{Token: start, Peer: peer1name, Free: 128, Version: 1},
+		{Token: middle, Peer: peer2name, Free: 127}})
+	wt.AssertEquals(t, ring1.Entries, ring2.Entries)
+
+	// Now to two different operations on either side,
+	// check we can Merge again
+	ring1.GrantRangeToHost(start, middle, peer2name)
+	ring2.GrantRangeToHost(middle, end, peer1name)
+
+	wt.AssertSuccess(t, ring2.Merge(*ring1))
+	wt.AssertSuccess(t, ring1.Merge(*ring2))
+
+	wt.AssertEquals(t, ring1.Entries, entries{{Token: start, Peer: peer2name, Free: 128, Version: 2},
+		{Token: middle, Peer: peer1name, Free: 127, Version: 1}})
+	wt.AssertEquals(t, ring1.Entries, ring2.Entries)
+}
+
+func TestMergeErrors(t *testing.T) {
+	// Cannot Merge in an invalid ring
+	ring1 := New(start, end, peer1name)
+	ring2 := New(start, end, peer2name)
+	ring2.Entries = []*entry{{Token: middle, Peer: peer2name}, {Token: start, Peer: peer2name}}
+	wt.AssertTrue(t, ring1.Merge(*ring2) == ErrNotSorted, "Expected ErrNotSorted")
+
+	// Should Merge two rings for different ranges
+	ring2 = New(start, middle, peer2name)
+	ring2.Entries = []*entry{}
+	wt.AssertTrue(t, ring1.Merge(*ring2) == ErrDifferentSubnets, "Expected ErrDifferentSubnets")
+
+	// Cannot Merge newer version of entry I own
+	ring2 = New(start, end, peer2name)
+	ring1.Entries = []*entry{{Token: start, Peer: peer1name}}
+	ring2.Entries = []*entry{{Token: start, Peer: peer1name, Version: 1}}
+	wt.AssertTrue(t, ring1.Merge(*ring2) == ErrNewerVersion, "Expected ErrNewerVersion")
+
+	// Cannot Merge two entries with same version but different hosts
+	ring1.Entries = []*entry{{Token: start, Peer: peer1name}}
+	ring2.Entries = []*entry{{Token: start, Peer: peer2name}}
+	wt.AssertTrue(t, ring1.Merge(*ring2) == ErrInvalidEntry, "Expected ErrInvalidEntry")
+
+	// Cannot Merge an entry into a range I own
+	ring1.Entries = []*entry{{Token: start, Peer: peer1name}}
+	ring2.Entries = []*entry{{Token: middle, Peer: peer2name}}
+	wt.AssertTrue(t, ring1.Merge(*ring2) == ErrEntryInMyRange, "Expected ErrEntryInMyRange")
+}
+
+func TestMergeMore(t *testing.T) {
+	ring1 := New(start, end, peer1name)
+	ring2 := New(start, end, peer2name)
+
+	assertRing := func(ring *Ring, entries entries) {
+		wt.AssertEquals(t, ring.Entries, entries)
+	}
+
+	assertRing(ring1, []*entry{})
+	assertRing(ring2, []*entry{})
+
+	// Claim everything for peer1
+	ring1.ClaimItAll()
+	assertRing(ring1, []*entry{{Token: start, Peer: peer1name, Free: 255}})
+	assertRing(ring2, []*entry{})
+
+	// Check the Merge sends it to the other ring
+	wt.AssertSuccess(t, ring2.Merge(*ring1))
+	assertRing(ring1, []*entry{{Token: start, Peer: peer1name, Free: 255}})
+	assertRing(ring2, []*entry{{Token: start, Peer: peer1name, Free: 255}})
+
+	// Give everything to peer2
+	ring1.GrantRangeToHost(start, end, peer2name)
+	assertRing(ring1, []*entry{{Token: start, Peer: peer2name, Free: 255, Version: 1}})
+	assertRing(ring2, []*entry{{Token: start, Peer: peer1name, Free: 255}})
+
+	wt.AssertSuccess(t, ring2.Merge(*ring1))
+	assertRing(ring1, []*entry{{Token: start, Peer: peer2name, Free: 255, Version: 1}})
+	assertRing(ring2, []*entry{{Token: start, Peer: peer2name, Free: 255, Version: 1}})
+
+	// And carve off some space
+	ring2.GrantRangeToHost(middle, end, peer1name)
+	assertRing(ring2, []*entry{{Token: start, Peer: peer2name, Free: 128, Version: 2},
+		{Token: middle, Peer: peer1name, Free: 127}})
+	assertRing(ring1, []*entry{{Token: start, Peer: peer2name, Free: 255, Version: 1}})
+
+	// And Merge back
+	wt.AssertSuccess(t, ring1.Merge(*ring2))
+	assertRing(ring1, []*entry{{Token: start, Peer: peer2name, Free: 128, Version: 2},
+		{Token: middle, Peer: peer1name, Free: 127}})
+	assertRing(ring2, []*entry{{Token: start, Peer: peer2name, Free: 128, Version: 2},
+		{Token: middle, Peer: peer1name, Free: 127}})
+
+	// This should be a no-op
+	wt.AssertSuccess(t, ring2.Merge(*ring1))
+	assertRing(ring1, []*entry{{Token: start, Peer: peer2name, Free: 128, Version: 2},
+		{Token: middle, Peer: peer1name, Free: 127}})
+	assertRing(ring2, []*entry{{Token: start, Peer: peer2name, Free: 128, Version: 2},
+		{Token: middle, Peer: peer1name, Free: 127}})
+}
+
+func TestMergeSplit(t *testing.T) {
+	ring1 := New(start, end, peer1name)
+	ring2 := New(start, end, peer2name)
+
+	// Claim everything for peer2
+	ring1.Entries = []*entry{{Token: start, Peer: peer2name, Free: 255}}
+	wt.AssertSuccess(t, ring2.Merge(*ring1))
+	wt.AssertEquals(t, ring1.Entries, ring2.Entries)
+
+	// Now grant a split range to peer1
+	ring2.GrantRangeToHost(dot10, dot245, peer1name)
+	wt.AssertEquals(t, ring2.Entries, entries{{Token: start, Peer: peer2name, Free: 10, Version: 1},
+		{Token: dot10, Peer: peer1name, Free: 235},
+		{Token: dot245, Peer: peer2name, Free: 10}})
+	wt.AssertSuccess(t, ring1.Merge(*ring2))
+	wt.AssertEquals(t, ring1.Entries, entries{{Token: start, Peer: peer2name, Free: 10, Version: 1},
+		{Token: dot10, Peer: peer1name, Free: 235},
+		{Token: dot245, Peer: peer2name, Free: 10}})
+	wt.AssertEquals(t, ring1.Entries, ring2.Entries)
+}
+
+func TestMergeSplit2(t *testing.T) {
+	ring1 := New(start, end, peer1name)
+	ring2 := New(start, end, peer2name)
+
+	// Claim everything for peer2
+	ring1.Entries = []*entry{{Token: start, Peer: peer2name, Free: 250}, {Token: dot250, Peer: peer2name, Free: 5}}
+	wt.AssertSuccess(t, ring2.Merge(*ring1))
+	wt.AssertEquals(t, ring1.Entries, ring2.Entries)
+
+	// Now grant a split range to peer1
+	ring2.GrantRangeToHost(dot10, dot245, peer1name)
+	wt.AssertEquals(t, ring2.Entries, entries{{Token: start, Peer: peer2name, Free: 10, Version: 1},
+		{Token: dot10, Peer: peer1name, Free: 235},
+		{Token: dot245, Peer: peer2name, Free: 5}, {Token: dot250, Peer: peer2name, Free: 5}})
+	wt.AssertSuccess(t, ring1.Merge(*ring2))
+	wt.AssertEquals(t, ring1.Entries, entries{{Token: start, Peer: peer2name, Free: 10, Version: 1},
+		{Token: dot10, Peer: peer1name, Free: 235},
+		{Token: dot245, Peer: peer2name, Free: 5}, {Token: dot250, Peer: peer2name, Free: 5}})
+	wt.AssertEquals(t, ring1.Entries, ring2.Entries)
+}
+
+// A simple test, very similar to above, but using the marshalling to byte[]s
+func TestGossip(t *testing.T) {
+	ring1 := New(start, end, peer1name)
+	ring2 := New(start, end, peer2name)
+
+	assertRing := func(ring *Ring, entries entries) {
+		wt.AssertEquals(t, ring.Entries, entries)
+	}
+
+	assertRing(ring1, []*entry{})
+	assertRing(ring2, []*entry{})
+
+	// Claim everything for peer1
+	ring1.ClaimItAll()
+	assertRing(ring1, []*entry{{Token: start, Peer: peer1name, Free: 255}})
+	assertRing(ring2, []*entry{})
+
+	// Check the Merge sends it to the other ring
+	wt.AssertSuccess(t, ring2.Merge(*ring1))
+	assertRing(ring1, []*entry{{Token: start, Peer: peer1name, Free: 255}})
+	assertRing(ring2, []*entry{{Token: start, Peer: peer1name, Free: 255}})
+}
+
+func TestFindFree(t *testing.T) {
+	ring1 := New(start, end, peer1name)
+
+	_, err := ring1.ChoosePeerToAskForSpace()
+	wt.AssertTrue(t, err == ErrNoFreeSpace, "Expected ErrNoFreeSpace")
+
+	ring1.Entries = []*entry{{Token: start, Peer: peer1name}}
+	_, err = ring1.ChoosePeerToAskForSpace()
+	wt.AssertTrue(t, err == ErrNoFreeSpace, "Expected ErrNoFreeSpace")
+
+	// We shouldn't return outselves
+	ring1.ReportFree(map[address.Address]address.Offset{start: 10})
+	_, err = ring1.ChoosePeerToAskForSpace()
+	wt.AssertTrue(t, err == ErrNoFreeSpace, "Expected ErrNoFreeSpace")
+
+	ring1.Entries = []*entry{{Token: start, Peer: peer1name, Free: 1},
+		{Token: start, Peer: peer1name, Free: 1}}
+	_, err = ring1.ChoosePeerToAskForSpace()
+	wt.AssertTrue(t, err == ErrNoFreeSpace, "Expected ErrNoFreeSpace")
+
+	// We should return others
+	ring1.Entries = []*entry{{Token: start, Peer: peer2name, Free: 1}}
+	peer, err := ring1.ChoosePeerToAskForSpace()
+	wt.AssertSuccess(t, err)
+	wt.AssertEquals(t, peer, peer2name)
+
+	ring1.Entries = []*entry{{Token: start, Peer: peer2name, Free: 1},
+		{Token: start, Peer: peer2name, Free: 1}}
+	peer, err = ring1.ChoosePeerToAskForSpace()
+	wt.AssertSuccess(t, err)
+	wt.AssertEquals(t, peer, peer2name)
+}
+
+func TestReportFree(t *testing.T) {
+	ring1 := New(start, end, peer1name)
+	ring2 := New(start, end, peer2name)
+
+	ring1.ClaimItAll()
+	ring1.GrantRangeToHost(middle, end, peer2name)
+	wt.AssertSuccess(t, ring2.Merge(*ring1))
+
+	freespace := make(map[address.Address]address.Offset)
+	for _, r := range ring2.OwnedRanges() {
+		freespace[r.Start] = 0
+	}
+	ring2.ReportFree(freespace)
+}
+
+func TestMisc(t *testing.T) {
+	ring := New(start, end, peer1name)
+
+	wt.AssertTrue(t, ring.Empty(), "empty")
+
+	ring.ClaimItAll()
+	println(ring.String())
+}
+
+func TestEmptyGossip(t *testing.T) {
+	ring1 := New(start, end, peer1name)
+	ring2 := New(start, end, peer2name)
+
+	ring1.ClaimItAll()
+	// This used to panic, and it shouldn't
+	wt.AssertSuccess(t, ring1.Merge(*ring2))
+}
+
+func TestMergeOldMessage(t *testing.T) {
+	ring1 := New(start, end, peer1name)
+	ring2 := New(start, end, peer2name)
+
+	ring1.ClaimItAll()
+	wt.AssertSuccess(t, ring2.Merge(*ring1))
+
+	ring1.GrantRangeToHost(middle, end, peer1name)
+	wt.AssertSuccess(t, ring1.Merge(*ring2))
+}
+
+func TestSplitRangeAtBeginning(t *testing.T) {
+	ring1 := New(start, end, peer1name)
+	ring2 := New(start, end, peer2name)
+
+	ring1.ClaimItAll()
+	wt.AssertSuccess(t, ring2.Merge(*ring1))
+
+	ring1.GrantRangeToHost(start, middle, peer2name)
+	wt.AssertSuccess(t, ring2.Merge(*ring1))
+}
+
+func TestOwnedRange(t *testing.T) {
+	ring1 := New(start, end, peer1name)
+	ring1.ClaimItAll()
+
+	wt.AssertEquals(t, ring1.OwnedRanges(), []address.Range{{Start: start, End: end}})
+
+	ring1.GrantRangeToHost(middle, end, peer2name)
+	wt.AssertEquals(t, ring1.OwnedRanges(), []address.Range{{Start: start, End: middle}})
+
+	ring2 := New(start, end, peer2name)
+	ring2.Merge(*ring1)
+	wt.AssertEquals(t, ring2.OwnedRanges(), []address.Range{{Start: middle, End: end}})
+
+	ring2.Entries = []*entry{{Token: middle, Peer: peer2name}}
+	wt.AssertEquals(t, ring2.OwnedRanges(),
+		[]address.Range{{Start: start, End: middle}, {Start: middle, End: end}})
+
+	ring2.Entries = []*entry{{Token: dot10, Peer: peer2name}, {Token: middle, Peer: peer2name}}
+	wt.AssertEquals(t, ring2.OwnedRanges(),
+		[]address.Range{{Start: start, End: dot10}, {Start: dot10, End: middle},
+			{Start: middle, End: end}})
+}
+
+func TestTransfer(t *testing.T) {
+	// First test just checks if we can grant some range to a host, when we transfer it, we get it back
+	ring1 := New(start, end, peer1name)
+	ring1.ClaimItAll()
+	ring1.GrantRangeToHost(middle, end, peer2name)
+	ring1.Transfer(peer2name, peer1name)
+	wt.AssertEquals(t, ring1.OwnedRanges(), []address.Range{{start, middle}, {middle, end}})
+
+	// Second test is what happens when a token exists at the end of a range but is transferred
+	// - does it get resurrected correctly?
+	ring1 = New(start, end, peer1name)
+	ring1.ClaimItAll()
+	ring1.GrantRangeToHost(middle, end, peer2name)
+	ring1.Transfer(peer2name, peer1name)
+	ring1.GrantRangeToHost(dot10, middle, peer2name)
+	wt.AssertEquals(t, ring1.OwnedRanges(), []address.Range{{start, dot10}, {middle, end}})
+}
+
+func TestOwner(t *testing.T) {
+	ring1 := New(start, end, peer1name)
+	wt.AssertTrue(t, ring1.Contains(start), "start should be in ring")
+	wt.AssertFalse(t, ring1.Contains(end), "end should not be in ring")
+
+	wt.AssertEquals(t, ring1.Owner(start), router.UnknownPeerName)
+
+	ring1.ClaimItAll()
+	ring1.GrantRangeToHost(middle, end, peer2name)
+	wt.AssertEquals(t, ring1.Owner(start), peer1name)
+	wt.AssertEquals(t, ring1.Owner(middle), peer2name)
+	wt.AssertPanic(t, func() {
+		ring1.Owner(end)
+	})
+}
+
+type addressSlice []address.Address
+
+func (s addressSlice) Len() int           { return len(s) }
+func (s addressSlice) Less(i, j int) bool { return s[i] < s[j] }
+func (s addressSlice) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
+
+func TestFuzzRing(t *testing.T) {
+	var (
+		numPeers   = 25
+		iterations = 1000
+	)
+
+	peers := make([]router.PeerName, numPeers)
+	for i := 0; i < numPeers; i++ {
+		peer, _ := router.PeerNameFromString(fmt.Sprintf("%02d:00:00:00:02:00", i))
+		peers[i] = peer
+	}
+
+	// Make a valid, random ring
+	makeGoodRandomRing := func() *Ring {
+		addressSpace := end - start
+		numTokens := rand.Intn(int(addressSpace))
+
+		tokenMap := make(map[address.Address]bool)
+		for i := 0; i < numTokens; i++ {
+			tokenMap[address.Address(rand.Intn(int(addressSpace)))] = true
+		}
+		var tokens []address.Address
+		for token := range tokenMap {
+			tokens = append(tokens, token)
+		}
+		sort.Sort(addressSlice(tokens))
+
+		peer := peers[rand.Intn(len(peers))]
+		ring := New(start, end, peer)
+		for _, token := range tokens {
+			peer = peers[rand.Intn(len(peers))]
+			ring.Entries = append(ring.Entries, &entry{Token: start + token, Peer: peer})
+		}
+
+		ring.assertInvariants()
+		return ring
+	}
+
+	for i := 0; i < iterations; i++ {
+		// make 2 random rings
+		ring1 := makeGoodRandomRing()
+		ring2 := makeGoodRandomRing()
+
+		// Merge them - this might fail, we don't care
+		// We just want to make sure it doesn't panic
+		ring1.Merge(*ring2)
+
+		// Check whats left still passes assertions
+		ring1.assertInvariants()
+		ring2.assertInvariants()
+	}
+
+	// Make an invalid, random ring
+	makeBadRandomRing := func() *Ring {
+		addressSpace := end - start
+		numTokens := rand.Intn(int(addressSpace))
+		tokens := make([]address.Address, numTokens)
+		for i := 0; i < numTokens; i++ {
+			tokens[i] = address.Address(rand.Intn(int(addressSpace)))
+		}
+
+		peer := peers[rand.Intn(len(peers))]
+		ring := New(start, end, peer)
+		for _, token := range tokens {
+			peer = peers[rand.Intn(len(peers))]
+			ring.Entries = append(ring.Entries, &entry{Token: start + token, Peer: peer})
+		}
+
+		return ring
+	}
+
+	for i := 0; i < iterations; i++ {
+		// make 2 random rings
+		ring1 := makeGoodRandomRing()
+		ring2 := makeBadRandomRing()
+
+		// Merge them - this might fail, we don't care
+		// We just want to make sure it doesn't panic
+		ring1.Merge(*ring2)
+
+		// Check whats left still passes assertions
+		ring1.assertInvariants()
+	}
+}
+
+func TestFuzzRingHard(t *testing.T) {
+	//common.InitDefaultLogging(true)
+	var (
+		numPeers   = 100
+		iterations = 3000
+		peers      []router.PeerName
+		rings      []*Ring
+		nextPeerID = 0
+	)
+
+	addPeer := func() {
+		peer, _ := router.PeerNameFromString(fmt.Sprintf("%02d:%02d:00:00:00:00", nextPeerID/10, nextPeerID%10))
+		common.Debug.Printf("%s: Adding peer", peer)
+		nextPeerID++
+		peers = append(peers, peer)
+		rings = append(rings, New(start, end, peer))
+	}
+
+	for i := 0; i < numPeers; i++ {
+		addPeer()
+	}
+
+	rings[0].ClaimItAll()
+
+	randomPeer := func(exclude int) (int, router.PeerName, *Ring) {
+		var peerIndex int
+		if exclude >= 0 {
+			peerIndex = rand.Intn(len(peers) - 1)
+			if peerIndex == exclude {
+				peerIndex++
+			}
+		} else {
+			peerIndex = rand.Intn(len(peers))
+		}
+		return peerIndex, peers[peerIndex], rings[peerIndex]
+	}
+
+	// Keep a map of index -> ranges, as these are a little expensive to
+	// calculate for every ring on every iteration.
+	var theRanges = make(map[int][]address.Range)
+	theRanges[0] = rings[0].OwnedRanges()
+
+	addOrRmPeer := func() {
+		if len(peers) < numPeers {
+			addPeer()
+			return
+		}
+
+		peerIndex, peername, _ := randomPeer(-1)
+		// Remove peer from our state
+		peers = append(peers[:peerIndex], peers[peerIndex+1:]...)
+		rings = append(rings[:peerIndex], rings[peerIndex+1:]...)
+		theRanges = make(map[int][]address.Range)
+
+		// Transfer the space for this peer on another peer, but not this one
+		_, otherPeername, otherRing := randomPeer(peerIndex)
+
+		// We need to be in a ~converged ring to rmpeer
+		for _, ring := range rings {
+			wt.AssertSuccess(t, otherRing.Merge(*ring))
+		}
+
+		common.Debug.Printf("%s: transferring from peer %s", otherPeername, peername)
+		otherRing.Transfer(peername, peername)
+
+		// And now tell everyone about the transfer - rmpeer is
+		// not partition safe
+		for i, ring := range rings {
+			wt.AssertSuccess(t, ring.Merge(*otherRing))
+			theRanges[i] = ring.OwnedRanges()
+		}
+	}
+
+	doGrantOrGossip := func() {
+		var ringsWithRanges = make([]int, 0, len(rings))
+		for index, ranges := range theRanges {
+			if len(ranges) > 0 {
+				ringsWithRanges = append(ringsWithRanges, index)
+			}
+		}
+
+		if len(ringsWithRanges) > 0 {
+			// Produce a random split in a random owned range, given to a random peer
+			indexWithRanges := ringsWithRanges[rand.Intn(len(ringsWithRanges))]
+			ownedRanges := theRanges[indexWithRanges]
+			ring := rings[indexWithRanges]
+
+			rangeToSplit := ownedRanges[rand.Intn(len(ownedRanges))]
+			size := address.Subtract(rangeToSplit.End, rangeToSplit.Start)
+			ipInRange := address.Add(rangeToSplit.Start, address.Offset(rand.Intn(int(size))))
+			_, peerToGiveTo, _ := randomPeer(-1)
+			common.Debug.Printf("%s: Granting [%v, %v) to %s", ring.Peer, ipInRange, rangeToSplit.End, peerToGiveTo)
+			ring.GrantRangeToHost(ipInRange, rangeToSplit.End, peerToGiveTo)
+
+			// Now 'gossip' this to a random host (note, note could be same host as above)
+			otherIndex, _, otherRing := randomPeer(-1)
+			common.Debug.Printf("%s: 'Gossiping' to %s", ring.Peer, otherRing.Peer)
+			wt.AssertSuccess(t, otherRing.Merge(*ring))
+
+			theRanges[indexWithRanges] = ring.OwnedRanges()
+			theRanges[otherIndex] = otherRing.OwnedRanges()
+			return
+		}
+
+		// No rings think they own anything (as gossip might be behind)
+		// We're going to pick a random host (which has entries) and gossip
+		// it to a random host (which may or may not have entries).
+		var ringsWithEntries = make([]*Ring, 0, len(rings))
+		for _, ring := range rings {
+			if len(ring.Entries) > 0 {
+				ringsWithEntries = append(ringsWithEntries, ring)
+			}
+		}
+		ring1 := ringsWithEntries[rand.Intn(len(ringsWithEntries))]
+		ring2index, _, ring2 := randomPeer(-1)
+		common.Debug.Printf("%s: 'Gossiping' to %s", ring1.Peer, ring2.Peer)
+		wt.AssertSuccess(t, ring2.Merge(*ring1))
+		theRanges[ring2index] = ring2.OwnedRanges()
+	}
+
+	for i := 0; i < iterations; i++ {
+		// about 1 in 10 times, rmpeer or add host
+		n := rand.Intn(10)
+		switch {
+		case n < 1:
+			addOrRmPeer()
+		default:
+			doGrantOrGossip()
+		}
+	}
+}
+
+func (es entries) String() string {
+	var buffer bytes.Buffer
+	fmt.Fprintf(&buffer, "[")
+	for i, entry := range es {
+		fmt.Fprintf(&buffer, "%+v", *entry)
+		if i+1 < len(es) {
+			fmt.Fprintf(&buffer, " ")
+		}
+	}
+	fmt.Fprintf(&buffer, "]")
+	return buffer.String()
+}

ipam/utils/utils.go

@@ -0,0 +1,49 @@
+package utils
+
+import (
+	"net"
+)
+
+// We shouldn't ever get any errors on *encoding*, but if we do, this will make sure we get to hear about them.
+func panicOnError(err error) {
+	if err != nil {
+		panic(err)
+	}
+}
+
+// IP4int converts an ipv4 address to a uint32
+func IP4int(ip4 net.IP) (r uint32) {
+	for _, b := range ip4.To4() {
+		r <<= 8
+		r |= uint32(b)
+	}
+	return
+}
+
+// IntIP4 converts a uint32 to an ipv4 address
+func IntIP4(key uint32) (r net.IP) {
+	r = make([]byte, net.IPv4len)
+	for i := 3; i >= 0; i-- {
+		r[i] = byte(key)
+		key >>= 8
+	}
+	return
+}
+
+// Add - convert to 32-bit unsigned integer, add, and convert back
+func Add(addr net.IP, i uint32) net.IP {
+	sum := IP4int(addr) + i
+	return IntIP4(sum)
+}
+
+// Subtract - convert to 32-bit unsigned integer, subtract, and convert back
+func Subtract(a, b net.IP) int64 {
+	return int64(IP4int(a)) - int64(IP4int(b))
+}
+
+// Assert test is true, panic otherwise
+func Assert(test bool, message string) {
+	if !test {
+		panic(message)
+	}
+}

testing/util.go

@@ -40,12 +40,29 @@ func AssertNoErr(t *testing.T, err error) {
 	}
 }
 
+func AssertBool(t *testing.T, got, wanted bool, desc string) {
+	if got != wanted {
+		Fatalf(t, "Expected %s %t but got %t", desc, wanted, got)
+	}
+}
+
+func AssertEqualUint32(t *testing.T, got, wanted uint32, desc string) {
+	if got != wanted {
+		Fatalf(t, "Expected %s %d but got %d", desc, wanted, got)
+	}
+}
+
 func AssertEqualuint64(t *testing.T, got, wanted uint64, desc string) {
 	if got != wanted {
 		Fatalf(t, "Expected %s %d but got %d", desc, wanted, got)
 	}
 }
 
+func AssertEqualInt64(t *testing.T, got, wanted int64, desc string) {
+	if got != wanted {
+		Fatalf(t, "Expected %s %d but got %d", desc, wanted, got)
+	}
+}
 func AssertEqualInt(t *testing.T, got, wanted int, desc string) {
 	if got != wanted {
 		Fatalf(t, "Expected %s %d but got %d", desc, wanted, got)
@@ -64,6 +81,18 @@ func AssertEqualString(t *testing.T, got, wanted string, desc string) {
 	}
 }
 
+func AssertNotEqualString(t *testing.T, got, wanted string, desc string) {
+	if got == wanted {
+		Fatalf(t, "Expected %s unlike '%s'", desc, wanted)
+	}
+}
+
+func AssertEquals(t *testing.T, a1, a2 interface{}) {
+	if !reflect.DeepEqual(a1, a2) {
+		Fatalf(t, "Expected %+v == %+v", a1, a2)
+	}
+}
+
 func AssertStatus(t *testing.T, got int, wanted int, desc string) {
 	if got != wanted {
 		Fatalf(t, "Expected %s %d but got %d", desc, wanted, got)
@@ -72,6 +101,9 @@ func AssertStatus(t *testing.T, got int, wanted int, desc string) {
 
 func AssertErrorInterface(t *testing.T, got interface{}, wanted interface{}, desc string) {
 	gotT, wantedT := reflect.TypeOf(got), reflect.TypeOf(wanted).Elem()
+	if got == nil {
+		Fatalf(t, "Expected %s but got nil (%s)", wantedT.String(), desc)
+	}
 	if !gotT.Implements(wantedT) {
 		Fatalf(t, "Expected %s but got %s (%s)", wantedT.String(), gotT.String(), desc)
 	}
@@ -79,6 +111,9 @@ func AssertErrorInterface(t *testing.T, got interface{}, wanted interface{}, des
 
 func AssertErrorType(t *testing.T, got interface{}, wanted interface{}, desc string) {
 	gotT, wantedT := reflect.TypeOf(got), reflect.TypeOf(wanted).Elem()
+	if got == nil {
+		Fatalf(t, "Expected %s but got nil (%s)", wantedT.String(), desc)
+	}
 	if gotT != wantedT {
 		Fatalf(t, "Expected %s but got %s (%s)", wantedT.String(), gotT.String(), desc)
 	}
@@ -97,6 +132,26 @@ func AssertEmpty(t *testing.T, array interface{}, desc string) {
 	}
 }
 
+func AssertSuccess(t *testing.T, err error) {
+	if err != nil {
+		Fatalf(t, "Expected success, got '%s'", err.Error())
+	}
+}
+
+func AssertPanic(t *testing.T, f func()) {
+	wrapper := func() (paniced bool) {
+		defer func() {
+			if err := recover(); err != nil {
+				paniced = true
+			}
+		}()
+
+		f()
+		return
+	}
+	AssertTrue(t, wrapper(), "Expected panic")
+}
+
 // Like testing.Fatalf, but adds the stack trace of the current call
 func Fatalf(t *testing.T, format string, args ...interface{}) {
 	t.Fatalf(format+"\n%s", append(args, StackTrace())...)