package ring import ( "bytes" "fmt" "math/rand" "sort" "testing" "github.com/stretchr/testify/require" "github.com/weaveworks/mesh" "github.com/rajch/weave/common" "github.com/rajch/weave/net/address" ) var ( peer1name, _ = mesh.PeerNameFromString("01:00:00:00:02:00") peer2name, _ = mesh.PeerNameFromString("02:00:00:00:02:00") peer3name, _ = mesh.PeerNameFromString("03:00:00:00:02:00") start, end = ParseIP("10.0.0.0"), ParseIP("10.0.1.0") dot8 = ParseIP("10.0.0.8") dot10, dot245 = ParseIP("10.0.0.10"), ParseIP("10.0.0.245") dot250 = ParseIP("10.0.0.250") middle = ParseIP("10.0.0.128") ) func ParseIP(s string) address.Address { addr, _ := address.ParseIP(s) return addr } func merge(r1, r2 *Ring) error { distance := func(start, end address.Address) address.Count { if end > start { return address.Count(end - start) } return address.Count((r1.End - start) + (end - r1.Start)) } check := func(rs []address.Range) bool { for _, r := range rs { size := distance(r.Start, r.End) entry, found := r1.Entries.get(r.Start) if !found { continue } if entry.Free < size { return true } } return false } _, err := r1.Merge(*r2, check) return err } func NewRing(start, end address.Address, peer mesh.PeerName) *Ring { return New(start, end, peer, nil) } func TestInvariants(t *testing.T) { ring := NewRing(start, end, peer1name) // Check ring is sorted ring.Entries = []*entry{{Token: dot245, Peer: peer1name}, {Token: dot10, Peer: peer2name}} require.True(t, ring.checkInvariants() == ErrNotSorted, "Expected error") // Check tokens don't appear twice ring.Entries = []*entry{{Token: dot245, Peer: peer1name}, {Token: dot245, Peer: peer2name}} require.True(t, ring.checkInvariants() == ErrTokenRepeated, "Expected error") // Check tokens are in bounds ring = NewRing(dot10, dot245, peer1name) ring.Entries = []*entry{{Token: start, Peer: peer1name}} require.True(t, ring.checkInvariants() == ErrTokenOutOfRange, "Expected error") ring.Entries = []*entry{{Token: end, Peer: peer1name}} require.True(t, ring.checkInvariants() == ErrTokenOutOfRange, "Expected error") } func TestInsert(t *testing.T) { ring := NewRing(start, end, peer1name) ring.Entries = []*entry{{Token: start, Peer: peer1name, Free: 255}} require.Panics(t, func() { ring.Entries.insert(entry{Token: start, Peer: peer1name}) }) ring.Entries.entry(0).Free = 0 ring.Entries.insert(entry{Token: dot245, Peer: peer1name}) check := []RangeInfo{ {Peer: peer1name, Range: address.Range{Start: start, End: dot245}}, {Peer: peer1name, Range: address.Range{Start: dot245, End: end}}, } require.Equal(t, check, ring.AllRangeInfo()) ring.Entries.insert(entry{Token: dot10, Peer: peer1name}) check2 := []RangeInfo{ {Peer: peer1name, Range: address.Range{Start: start, End: dot10}}, {Peer: peer1name, Range: address.Range{Start: dot10, End: dot245}}, {Peer: peer1name, Range: address.Range{Start: dot245, End: end}}, } require.Equal(t, check2, ring.AllRangeInfo()) } func TestBetween(t *testing.T) { ring1 := NewRing(start, end, peer1name) ring1.Entries = []*entry{{Token: start, Peer: peer1name, Free: 255}} // First off, in a ring where everything is owned by the peer // between should return true for everything for i := 1; i <= 255; i++ { ip := ParseIP(fmt.Sprintf("10.0.0.%d", i)) require.True(t, ring1.Entries.between(ip, 0, 1), "between should be true!") } // Now, construct a ring with entries at +10 and -10 // And check the correct behaviour ring1.Entries = []*entry{{Token: dot10, Peer: peer1name}, {Token: dot245, Peer: peer2name}} ring1.assertInvariants() for i := 10; i <= 244; i++ { ipStr := fmt.Sprintf("10.0.0.%d", i) ip := ParseIP(ipStr) require.True(t, ring1.Entries.between(ip, 0, 1), fmt.Sprintf("Between should be true for %s!", ipStr)) require.False(t, ring1.Entries.between(ip, 1, 2), fmt.Sprintf("Between should be false for %s!", ipStr)) } for i := 0; i <= 9; i++ { ipStr := fmt.Sprintf("10.0.0.%d", i) ip := ParseIP(ipStr) require.False(t, ring1.Entries.between(ip, 0, 1), fmt.Sprintf("Between should be false for %s!", ipStr)) require.True(t, ring1.Entries.between(ip, 1, 2), fmt.Sprintf("Between should be true for %s!", ipStr)) } for i := 245; i <= 255; i++ { ipStr := fmt.Sprintf("10.0.0.%d", i) ip := ParseIP(ipStr) require.False(t, ring1.Entries.between(ip, 0, 1), fmt.Sprintf("Between should be false for %s!", ipStr)) require.True(t, ring1.Entries.between(ip, 1, 2), fmt.Sprintf("Between should be true for %s!", ipStr)) } } func TestGrantSimple(t *testing.T) { ring1 := NewRing(start, end, peer1name) ring2 := NewRing(start, end, peer2name) // Claim everything for peer1 ring1.ClaimItAll() require.Equal(t, entries{{Token: start, Peer: peer1name, Free: 256}}, ring1.Entries) // Now grant everything to peer2 ring1.GrantRangeToHost(start, end, peer2name) ring2.Entries = []*entry{{Token: start, Peer: peer2name, Free: 256, Version: 1}} require.Equal(t, ring2.Entries, ring1.Entries) // Now spint back to peer 1 ring2.GrantRangeToHost(dot10, end, peer1name) ring1.Entries = []*entry{{Token: start, Peer: peer2name, Free: 10, Version: 2}, {Token: dot10, Peer: peer1name, Free: 246}} require.Equal(t, ring2.Entries, ring1.Entries) // And spint back to peer 2 again ring1.GrantRangeToHost(dot245, end, peer2name) require.Equal(t, entries{{Token: start, Peer: peer2name, Free: 10, Version: 2}, {Token: dot10, Peer: peer1name, Free: 235, Version: 1}, {Token: dot245, Peer: peer2name, Free: 11}}, ring1.Entries) // Grant range spanning a live token ring1.Entries = []*entry{{Token: start, Peer: peer1name, Free: 10, Version: 2}, {Token: dot10, Peer: peer1name, Free: 235}, {Token: dot245, Peer: peer1name, Free: 10}} ring1.GrantRangeToHost(dot10, end, peer2name) require.Equal(t, entries{{Token: start, Peer: peer1name, Free: 10, Version: 2}, {Token: dot10, Peer: peer2name, Free: 235, Version: 1}, {Token: dot245, Peer: peer2name, Free: 10, Version: 1}}, ring1.Entries) } func TestGrantSplit(t *testing.T) { ring1 := NewRing(start, end, peer1name) ring2 := NewRing(start, end, peer2name) // Claim everything for peer1 ring1.Entries = []*entry{{Token: start, Peer: peer1name, Free: 256}} merge(ring2, ring1) require.Equal(t, ring2.Entries, ring1.Entries) // Now grant a split range to peer2 ring1.GrantRangeToHost(dot10, dot245, peer2name) require.Equal(t, entries{{Token: start, Peer: peer1name, Free: 10, Version: 1}, {Token: dot10, Peer: peer2name, Free: 235}, {Token: dot245, Peer: peer1name, Free: 11}}, ring1.Entries) ring1.assertInvariants() // Grant range spanning a live token, and inserting a new token ring1.Entries = []*entry{{Token: start, Peer: peer1name, Free: 10, Version: 2}, {Token: dot10, Peer: peer1name, Free: 118}, {Token: middle, Peer: peer1name, Free: 127}} ring1.GrantRangeToHost(dot10, dot245, peer2name) require.Equal(t, entries{{Token: start, Peer: peer1name, Free: 10, Version: 2}, {Token: dot10, Peer: peer2name, Free: 118, Version: 1}, {Token: middle, Peer: peer2name, Free: 117, Version: 1}, {Token: dot245, Peer: peer1name, Free: 11, Version: 0}}, ring1.Entries) ring1.assertInvariants() } func TestMergeSimple(t *testing.T) { ring1 := NewRing(start, end, peer1name) ring2 := NewRing(start, end, peer2name) // Claim everything for peer1 ring1.ClaimItAll() ring1.GrantRangeToHost(middle, end, peer2name) require.NoError(t, merge(ring2, ring1)) require.Equal(t, entries{{Token: start, Peer: peer1name, Free: 128, Version: 1}, {Token: middle, Peer: peer2name, Free: 128}}, ring1.Entries) require.Equal(t, ring2.Entries, ring1.Entries) // Now to two different operations on either side, // check we can Merge again ring1.GrantRangeToHost(start, middle, peer2name) ring2.GrantRangeToHost(middle, end, peer1name) require.NoError(t, merge(ring2, ring1)) require.NoError(t, merge(ring1, ring2)) require.Equal(t, entries{{Token: start, Peer: peer2name, Free: 128, Version: 2}, {Token: middle, Peer: peer1name, Free: 128, Version: 1}}, ring1.Entries) require.Equal(t, ring2.Entries, ring1.Entries) } func TestMergeWithConflicts(t *testing.T) { // received update to entry with a token and version identical to an entry we have // but but holding different content, if there are no allocations, resolve the // conflict by picking entry one with high free count ring1 := NewRing(start, end, peer1name) ring2 := NewRing(start, end, peer2name) ring1.Entries = []*entry{{Token: start, Peer: peer1name, Free: 256, Version: 1}} ring2.Entries = []*entry{{Token: start, Peer: peer2name, Free: 256, Version: 1}} require.NoError(t, merge(ring1, ring2)) // received update to entry with a token and version identical to an entry we have // but but holding different content, if there are allocations, then reject the received // entry ring1 = NewRing(start, end, peer1name) ring2 = NewRing(start, end, peer2name) ring1.Entries = []*entry{{Token: start, Peer: peer1name, Free: 128, Version: 1}} ring2.Entries = []*entry{{Token: start, Peer: peer2name, Free: 256, Version: 1}} require.Error(t, merge(ring1, ring2), "Expected error") // received an entry with update to one of our own tokens and with new version, // accept the received entry if its still going to belong to us ring1 = NewRing(start, end, peer1name) ring2 = NewRing(start, end, peer2name) ring1.Entries = []*entry{{Token: start, Peer: peer1name}} ring2.Entries = []*entry{{Token: start, Peer: peer1name, Version: 1}} require.NoError(t, merge(ring1, ring2)) // received an entry with update to one of our own tokens and with new version, // but belongs to a different peer, accept the received entry if we do not // have allocations in the range ring1 = NewRing(start, end, peer1name) ring2 = NewRing(start, end, peer2name) ring1.Entries = []*entry{{Token: start, Peer: peer1name, Free: 256}} ring2.Entries = []*entry{{Token: start, Peer: peer2name, Free: 256, Version: 1}} require.NoError(t, merge(ring1, ring2)) // received an entry with update to one of our own tokens and with new version, // reject received entry if we have allocations in the range ring1 = NewRing(start, end, peer1name) ring2 = NewRing(start, end, peer2name) ring1.Entries = []*entry{{Token: start, Peer: peer1name, Free: 128}} ring2.Entries = []*entry{{Token: start, Peer: peer2name, Version: 1}} require.Error(t, merge(ring1, ring2), "Expected error") // we receive an entry that splits one of our ranges, giving some of it // away to another peer. accept the entry provided that we have no allocations // in the range that got given away. ring1 = NewRing(start, end, peer1name) ring2 = NewRing(start, end, peer2name) ring1.Entries = []*entry{{Token: start, Peer: peer1name, Free: 256, Version: 1}} ring2.Entries = []*entry{{Token: start, Peer: peer1name, Free: 128, Version: 2}, {Token: middle, Peer: peer2name, Free: 128}} require.NoError(t, merge(ring1, ring2)) } func TestMergeErrors(t *testing.T) { // Cannot Merge in an invalid ring ring1 := NewRing(start, end, peer1name) ring2 := NewRing(start, end, peer2name) ring2.Entries = []*entry{{Token: middle, Peer: peer2name}, {Token: start, Peer: peer2name}} require.True(t, merge(ring1, ring2) == ErrNotSorted, "Expected ErrNotSorted") // Should Merge two rings for different ranges ring2 = NewRing(start, middle, peer2name) ring2.Entries = []*entry{} require.True(t, merge(ring1, ring2) == ErrDifferentRange, "Expected ErrDifferentRange") // Cannot Merge two entries with same version but different hosts ring1.Entries = []*entry{{Token: start, Peer: peer1name}} ring2.Entries = []*entry{{Token: start, Peer: peer2name}} require.Error(t, merge(ring1, ring2), "Expected error") // Cannot Merge an entry into a range I own ring1.Entries = []*entry{{Token: start, Peer: peer1name}} ring2.Entries = []*entry{{Token: middle, Peer: peer2name}} require.Error(t, merge(ring1, ring2), "Expected error") } func TestMergeMore(t *testing.T) { ring1 := NewRing(start, end, peer1name) ring2 := NewRing(start, end, peer2name) assertRing := func(ring *Ring, entries entries) { require.Equal(t, entries, ring.Entries) } assertRing(ring1, []*entry{}) assertRing(ring2, []*entry{}) // Claim everything for peer1 ring1.ClaimItAll() assertRing(ring1, []*entry{{Token: start, Peer: peer1name, Free: 256}}) assertRing(ring2, []*entry{}) // Check the Merge sends it to the other ring require.NoError(t, merge(ring2, ring1)) assertRing(ring1, []*entry{{Token: start, Peer: peer1name, Free: 256}}) assertRing(ring2, []*entry{{Token: start, Peer: peer1name, Free: 256}}) // Give everything to peer2 ring1.GrantRangeToHost(start, end, peer2name) assertRing(ring1, []*entry{{Token: start, Peer: peer2name, Free: 256, Version: 1}}) assertRing(ring2, []*entry{{Token: start, Peer: peer1name, Free: 256}}) require.NoError(t, merge(ring2, ring1)) assertRing(ring1, []*entry{{Token: start, Peer: peer2name, Free: 256, Version: 1}}) assertRing(ring2, []*entry{{Token: start, Peer: peer2name, Free: 256, Version: 1}}) // And carve off some space ring2.GrantRangeToHost(middle, end, peer1name) assertRing(ring2, []*entry{{Token: start, Peer: peer2name, Free: 128, Version: 2}, {Token: middle, Peer: peer1name, Free: 128}}) assertRing(ring1, []*entry{{Token: start, Peer: peer2name, Free: 256, Version: 1}}) // And Merge back require.NoError(t, merge(ring1, ring2)) assertRing(ring1, []*entry{{Token: start, Peer: peer2name, Free: 128, Version: 2}, {Token: middle, Peer: peer1name, Free: 128}}) assertRing(ring2, []*entry{{Token: start, Peer: peer2name, Free: 128, Version: 2}, {Token: middle, Peer: peer1name, Free: 128}}) // This should be a no-op require.NoError(t, merge(ring2, ring1)) assertRing(ring1, []*entry{{Token: start, Peer: peer2name, Free: 128, Version: 2}, {Token: middle, Peer: peer1name, Free: 128}}) assertRing(ring2, []*entry{{Token: start, Peer: peer2name, Free: 128, Version: 2}, {Token: middle, Peer: peer1name, Free: 128}}) } func TestMergeSplit(t *testing.T) { ring1 := NewRing(start, end, peer1name) ring2 := NewRing(start, end, peer2name) // Claim everything for peer2 ring1.Entries = []*entry{{Token: start, Peer: peer2name, Free: 256}} require.NoError(t, merge(ring2, ring1)) require.Equal(t, ring2.Entries, ring1.Entries) // Now grant a split range to peer1 ring2.GrantRangeToHost(dot10, dot245, peer1name) require.Equal(t, entries{{Token: start, Peer: peer2name, Free: 10, Version: 1}, {Token: dot10, Peer: peer1name, Free: 235}, {Token: dot245, Peer: peer2name, Free: 11}}, ring2.Entries) require.NoError(t, merge(ring1, ring2)) require.Equal(t, entries{{Token: start, Peer: peer2name, Free: 10, Version: 1}, {Token: dot10, Peer: peer1name, Free: 235}, {Token: dot245, Peer: peer2name, Free: 11}}, ring1.Entries) require.Equal(t, ring2.Entries, ring1.Entries) } func TestMergeSplit2(t *testing.T) { ring1 := NewRing(start, end, peer1name) ring2 := NewRing(start, end, peer2name) // Claim everything for peer2 ring1.Entries = []*entry{{Token: start, Peer: peer2name, Free: 250}, {Token: dot250, Peer: peer2name, Free: 5}} require.NoError(t, merge(ring2, ring1)) require.Equal(t, ring2.Entries, ring1.Entries) // Now grant a split range to peer1 ring2.GrantRangeToHost(dot10, dot245, peer1name) require.Equal(t, entries{{Token: start, Peer: peer2name, Free: 10, Version: 1}, {Token: dot10, Peer: peer1name, Free: 235}, {Token: dot245, Peer: peer2name, Free: 5}, {Token: dot250, Peer: peer2name, Free: 5}}, ring2.Entries) require.NoError(t, merge(ring1, ring2)) require.Equal(t, entries{{Token: start, Peer: peer2name, Free: 10, Version: 1}, {Token: dot10, Peer: peer1name, Free: 235}, {Token: dot245, Peer: peer2name, Free: 5}, {Token: dot250, Peer: peer2name, Free: 5}}, ring1.Entries) require.Equal(t, ring2.Entries, ring1.Entries) } // A simple test, very similar to above, but using the marshalling to byte[]s func TestGossip(t *testing.T) { ring1 := NewRing(start, end, peer1name) ring2 := NewRing(start, end, peer2name) assertRing := func(ring *Ring, entries entries) { require.Equal(t, entries, ring.Entries) } assertRing(ring1, []*entry{}) assertRing(ring2, []*entry{}) // Claim everything for peer1 ring1.ClaimItAll() assertRing(ring1, []*entry{{Token: start, Peer: peer1name, Free: 256}}) assertRing(ring2, []*entry{}) // Check the Merge sends it to the other ring require.NoError(t, merge(ring2, ring1)) assertRing(ring1, []*entry{{Token: start, Peer: peer1name, Free: 256}}) assertRing(ring2, []*entry{{Token: start, Peer: peer1name, Free: 256}}) } func assertPeersWithSpace(t *testing.T, ring *Ring, start, end address.Address, expected int) []mesh.PeerName { peers := ring.ChoosePeersToAskForSpace(start, end) require.Equal(t, expected, len(peers)) return peers } func TestFindFree(t *testing.T) { ring1 := NewRing(start, end, peer1name) assertPeersWithSpace(t, ring1, start, end, 0) ring1.Entries = []*entry{{Token: start, Peer: peer1name}} assertPeersWithSpace(t, ring1, start, end, 0) // We shouldn't return outselves ring1.ReportFree(map[address.Address]address.Count{start: 10}) assertPeersWithSpace(t, ring1, start, end, 0) ring1.Entries = []*entry{{Token: start, Peer: peer1name, Free: 1}, {Token: middle, Peer: peer1name, Free: 1}} assertPeersWithSpace(t, ring1, start, end, 0) ring1.assertInvariants() // We should return others var peers []mesh.PeerName ring1.Entries = []*entry{{Token: start, Peer: peer2name, Free: 1}} peers = assertPeersWithSpace(t, ring1, start, end, 1) require.Equal(t, peer2name, peers[0]) ring1.Entries = []*entry{{Token: start, Peer: peer2name, Free: 1}, {Token: middle, Peer: peer3name, Free: 1}} peers = assertPeersWithSpace(t, ring1, start, middle, 1) require.Equal(t, peer2name, peers[0]) peers = assertPeersWithSpace(t, ring1, middle, end, 1) require.Equal(t, peer3name, peers[0]) assertPeersWithSpace(t, ring1, start, end, 2) ring1.assertInvariants() } func TestReportFree(t *testing.T) { ring1 := NewRing(start, end, peer1name) ring2 := NewRing(start, end, peer2name) ring1.ClaimItAll() ring1.GrantRangeToHost(middle, end, peer2name) require.NoError(t, merge(ring2, ring1)) freespace := make(map[address.Address]address.Count) for _, r := range ring2.OwnedRanges() { freespace[r.Start] = 0 } ring2.ReportFree(freespace) } func TestMisc(t *testing.T) { ring := NewRing(start, end, peer1name) require.True(t, ring.Empty(), "empty") ring.ClaimItAll() println(ring.String()) } func TestEmptyGossip(t *testing.T) { ring1 := NewRing(start, end, peer1name) ring2 := NewRing(start, end, peer2name) ring1.ClaimItAll() // This used to panic, and it shouldn't require.NoError(t, merge(ring1, ring2)) } func TestMergeOldMessage(t *testing.T) { ring1 := NewRing(start, end, peer1name) ring2 := NewRing(start, end, peer2name) ring1.ClaimItAll() require.NoError(t, merge(ring2, ring1)) ring1.GrantRangeToHost(middle, end, peer1name) require.NoError(t, merge(ring1, ring2)) } func TestSplitRangeAtBeginning(t *testing.T) { ring1 := NewRing(start, end, peer1name) ring2 := NewRing(start, end, peer2name) ring1.ClaimItAll() require.NoError(t, merge(ring2, ring1)) ring1.GrantRangeToHost(start, middle, peer2name) require.NoError(t, merge(ring2, ring1)) } func TestOwnedRange(t *testing.T) { ring1 := NewRing(start, end, peer1name) ring1.ClaimItAll() require.Equal(t, []address.Range{{Start: start, End: end}}, ring1.OwnedRanges()) ring1.GrantRangeToHost(middle, end, peer2name) require.Equal(t, []address.Range{{Start: start, End: middle}}, ring1.OwnedRanges()) ring2 := NewRing(start, end, peer2name) merge(ring2, ring1) require.Equal(t, []address.Range{{Start: middle, End: end}}, ring2.OwnedRanges()) ring2.Entries = []*entry{{Token: middle, Peer: peer2name}} require.Equal(t, []address.Range{{Start: start, End: middle}, {Start: middle, End: end}}, ring2.OwnedRanges()) ring2.Entries = []*entry{{Token: dot10, Peer: peer2name}, {Token: middle, Peer: peer2name}} require.Equal(t, []address.Range{{Start: start, End: dot10}, {Start: dot10, End: middle}, {Start: middle, End: end}}, ring2.OwnedRanges()) } func TestTransfer(t *testing.T) { // First test just checks if we can grant some range to a host, when we transfer it, we get it back ring1 := NewRing(start, end, peer1name) ring1.ClaimItAll() ring1.GrantRangeToHost(middle, end, peer2name) ring1.Transfer(peer2name, peer1name) require.Equal(t, []address.Range{{Start: start, End: middle}, {Start: middle, End: end}}, ring1.OwnedRanges()) // Second test is what happens when a token exists at the end of a range but is transferred // - does it get resurrected correctly? ring1 = NewRing(start, end, peer1name) ring1.ClaimItAll() ring1.GrantRangeToHost(middle, end, peer2name) ring1.Transfer(peer2name, peer1name) ring1.GrantRangeToHost(dot10, middle, peer2name) require.Equal(t, []address.Range{{Start: start, End: dot10}, {Start: middle, End: end}}, ring1.OwnedRanges()) } func TestOwner(t *testing.T) { ring1 := NewRing(start, end, peer1name) require.True(t, ring1.Contains(start), "start should be in ring") require.False(t, ring1.Contains(end), "end should not be in ring") require.Equal(t, mesh.UnknownPeerName, ring1.Owner(start)) ring1.ClaimItAll() ring1.GrantRangeToHost(middle, end, peer2name) require.Equal(t, peer1name, ring1.Owner(start)) require.Equal(t, peer2name, ring1.Owner(middle)) require.Panics(t, func() { ring1.Owner(end) }) } func makePeerName(i int) mesh.PeerName { if i >= 10000 { panic("makePeerName: invalid value") } peer, _ := mesh.PeerNameFromString(fmt.Sprintf("%02d:%02d:00:00:00:ff", i/100, i%100)) return peer } func makePeers(numPeers int) []mesh.PeerName { peers := make([]mesh.PeerName, numPeers) for i := 0; i < numPeers; i++ { peers[i] = makePeerName(i) } return peers } func TestClaimForPeers(t *testing.T) { const numPeers = 12 // Different end to usual so we get a number of addresses that a) // is smaller than the max number of peers, and b) is divisible by // some number of peers. This maximises coverage of edge cases. end := dot8 peers := makePeers(numPeers) // Test for a range of peer counts for i := 0; i < numPeers; i++ { ring := NewRing(start, end, peers[0]) ring.ClaimForPeers(peers[:i+1]) } } type addressSlice []address.Address func (s addressSlice) Len() int { return len(s) } func (s addressSlice) Less(i, j int) bool { return s[i] < s[j] } func (s addressSlice) Swap(i, j int) { s[i], s[j] = s[j], s[i] } func TestFuzzRing(t *testing.T) { var ( numPeers = 25 iterations = 1000 ) peers := make([]mesh.PeerName, numPeers) for i := 0; i < numPeers; i++ { peers[i] = makePeerName(i) } // Make a valid, random ring makeGoodRandomRing := func() *Ring { addressSpace := end - start numTokens := rand.Intn(int(addressSpace)) tokenMap := make(map[address.Address]bool) for i := 0; i < numTokens; i++ { tokenMap[address.Address(rand.Intn(int(addressSpace)))] = true } var tokens []address.Address for token := range tokenMap { tokens = append(tokens, token) } sort.Sort(addressSlice(tokens)) peer := peers[rand.Intn(len(peers))] ring := NewRing(start, end, peer) for _, token := range tokens { peer = peers[rand.Intn(len(peers))] ring.Entries = append(ring.Entries, &entry{Token: start + token, Peer: peer}) } ring.assertInvariants() return ring } for i := 0; i < iterations; i++ { // make 2 random rings ring1 := makeGoodRandomRing() ring2 := makeGoodRandomRing() // Merge them - this might fail, we don't care // We just want to make sure it doesn't panic merge(ring1, ring2) // Check whats left still passes assertions ring1.assertInvariants() ring2.assertInvariants() } // Make an invalid, random ring makeBadRandomRing := func() *Ring { addressSpace := end - start numTokens := rand.Intn(int(addressSpace)) tokens := make([]address.Address, numTokens) for i := 0; i < numTokens; i++ { tokens[i] = address.Address(rand.Intn(int(addressSpace))) } peer := peers[rand.Intn(len(peers))] ring := NewRing(start, end, peer) for _, token := range tokens { peer = peers[rand.Intn(len(peers))] ring.Entries = append(ring.Entries, &entry{Token: start + token, Peer: peer}) } return ring } for i := 0; i < iterations; i++ { // make 2 random rings ring1 := makeGoodRandomRing() ring2 := makeBadRandomRing() // Merge them - this might fail, we don't care // We just want to make sure it doesn't panic merge(ring1, ring2) // Check whats left still passes assertions ring1.assertInvariants() } } func TestFuzzRingHard(t *testing.T) { //common.SetLogLevel("debug") var ( numPeers = 100 iterations = 3000 peers []mesh.PeerName rings []*Ring nextPeerID = 0 ) addPeer := func() { peer := makePeerName(nextPeerID) common.Log.Debugf("%s: Adding peer", peer) nextPeerID++ peers = append(peers, peer) rings = append(rings, NewRing(start, end, peer)) } for i := 0; i < numPeers; i++ { addPeer() } rings[0].ClaimItAll() randomPeer := func(exclude int) (int, mesh.PeerName, *Ring) { var peerIndex int if exclude >= 0 { peerIndex = rand.Intn(len(peers) - 1) if peerIndex == exclude { peerIndex++ } } else { peerIndex = rand.Intn(len(peers)) } return peerIndex, peers[peerIndex], rings[peerIndex] } // Keep a map of index -> ranges, as these are a little expensive to // calculate for every ring on every iteration. var theRanges = make(map[int][]address.Range) theRanges[0] = rings[0].OwnedRanges() addOrRmPeer := func() { if len(peers) < numPeers { addPeer() return } // Pick one peer to remove, and a different one to transfer to peerIndex, peername, _ := randomPeer(-1) _, otherPeername, otherRing := randomPeer(peerIndex) // We need to be in a ~converged ring to rmpeer for _, ring := range rings { require.NoError(t, merge(otherRing, ring)) } common.Log.Debugf("%s: transferring from peer %s", otherPeername, peername) otherRing.Transfer(peername, otherPeername) // Remove peer from our state peers = append(peers[:peerIndex], peers[peerIndex+1:]...) rings = append(rings[:peerIndex], rings[peerIndex+1:]...) theRanges = make(map[int][]address.Range) // And now tell everyone about the transfer - rmpeer is // not partition safe for i, ring := range rings { require.NoError(t, merge(ring, otherRing)) theRanges[i] = ring.OwnedRanges() } } doGrantOrGossip := func() { var ringsWithRanges = make([]int, 0, len(rings)) for index, ranges := range theRanges { if len(ranges) > 0 { ringsWithRanges = append(ringsWithRanges, index) } } if len(ringsWithRanges) > 0 { // Produce a random split in a random owned range, given to a random peer indexWithRanges := ringsWithRanges[rand.Intn(len(ringsWithRanges))] ownedRanges := theRanges[indexWithRanges] ring := rings[indexWithRanges] rangeToSplit := ownedRanges[rand.Intn(len(ownedRanges))] size := address.Subtract(rangeToSplit.End, rangeToSplit.Start) ipInRange := address.Add(rangeToSplit.Start, address.Offset(rand.Intn(int(size)))) _, peerToGiveTo, _ := randomPeer(-1) common.Log.Debugf("%s: Granting [%v, %v) to %s", ring.Peer, ipInRange, rangeToSplit.End, peerToGiveTo) ring.GrantRangeToHost(ipInRange, rangeToSplit.End, peerToGiveTo) // Now 'gossip' this to a random host (note, note could be same host as above) otherIndex, _, otherRing := randomPeer(-1) common.Log.Debugf("%s: 'Gossiping' to %s", ring.Peer, otherRing.Peer) require.NoError(t, merge(otherRing, ring)) theRanges[indexWithRanges] = ring.OwnedRanges() theRanges[otherIndex] = otherRing.OwnedRanges() return } // No rings think they own anything (as gossip might be behind) // We're going to pick a random host (which has entries) and gossip // it to a random host (which may or may not have entries). var ringsWithEntries = make([]*Ring, 0, len(rings)) for _, ring := range rings { if len(ring.Entries) > 0 { ringsWithEntries = append(ringsWithEntries, ring) } } ring1 := ringsWithEntries[rand.Intn(len(ringsWithEntries))] ring2index, _, ring2 := randomPeer(-1) common.Log.Debugf("%s: 'Gossiping' to %s", ring1.Peer, ring2.Peer) require.NoError(t, merge(ring2, ring1)) theRanges[ring2index] = ring2.OwnedRanges() } for i := 0; i < iterations; i++ { // about 1 in 10 times, rmpeer or add host n := rand.Intn(10) switch { case n < 1: addOrRmPeer() default: doGrantOrGossip() } } } func (r *Ring) ClaimItAll() { r.ClaimForPeers([]mesh.PeerName{r.Peer}) } func (es entries) String() string { var buffer bytes.Buffer fmt.Fprintf(&buffer, "[") for i, entry := range es { fmt.Fprintf(&buffer, "%+v", *entry) if i+1 < len(es) { fmt.Fprintf(&buffer, " ") } } fmt.Fprintf(&buffer, "]") return buffer.String() }