From b3cda39b0bc4dcca88eafe172ed517f7c174b0aa Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Mon, 2 Nov 2015 18:01:40 +0000 Subject: [PATCH 1/5] Let IPAM pick any peer we know about to transfer ownership to, when shutting down --- ipam/allocator.go | 16 ++++++++++++++-- ipam/testutils_test.go | 2 +- prog/weaver/main.go | 3 ++- test/165_reset_2_test.sh | 22 ++++++++++++++++++++++ 4 files changed, 39 insertions(+), 4 deletions(-) create mode 100755 test/165_reset_2_test.sh diff --git a/ipam/allocator.go b/ipam/allocator.go index 04ad0cd9..958788cf 100644 --- a/ipam/allocator.go +++ b/ipam/allocator.go @@ -56,10 +56,11 @@ type Allocator struct { paxosTicker *time.Ticker shuttingDown bool // to avoid doing any requests while trying to shut down now func() time.Time + namesFunc func() map[router.PeerName]struct{} } // NewAllocator creates and initialises a new Allocator -func NewAllocator(ourName router.PeerName, ourUID router.PeerUID, ourNickname string, universe address.Range, quorum uint) *Allocator { +func NewAllocator(ourName router.PeerName, ourUID router.PeerUID, ourNickname string, universe address.Range, quorum uint, names func() map[router.PeerName]struct{}) *Allocator { return &Allocator{ ourName: ourName, universe: universe, @@ -68,6 +69,7 @@ func NewAllocator(ourName router.PeerName, ourUID router.PeerUID, ourNickname st paxos: paxos.NewNode(ourName, ourUID, quorum), nicknames: map[router.PeerName]string{ourName: ourNickname}, now: time.Now, + namesFunc: names, } } @@ -282,7 +284,17 @@ func (alloc *Allocator) Shutdown() { alloc.shuttingDown = true alloc.cancelOps(&alloc.pendingClaims) alloc.cancelOps(&alloc.pendingAllocates) - if heir := alloc.ring.PickPeerForTransfer(); heir != router.UnknownPeerName { + heir := alloc.ring.PickPeerForTransfer() + if heir == router.UnknownPeerName && alloc.namesFunc != nil { + // No peers in the ring; pick one via the function we were given at init + for peerName := range alloc.namesFunc() { + if peerName != alloc.ourName { + heir = peerName + break + } + } + } + if heir != router.UnknownPeerName { alloc.ring.Transfer(alloc.ourName, heir) alloc.space.Clear() alloc.gossip.GossipBroadcast(alloc.Gossip()) diff --git a/ipam/testutils_test.go b/ipam/testutils_test.go index 95f6407a..ea954aa0 100644 --- a/ipam/testutils_test.go +++ b/ipam/testutils_test.go @@ -135,7 +135,7 @@ func makeAllocator(name string, cidrStr string, quorum uint) (*Allocator, addres } alloc := NewAllocator(peername, router.PeerUID(rand.Int63()), - "nick-"+name, cidr.Range(), quorum) + "nick-"+name, cidr.Range(), quorum, nil) return alloc, cidr.HostRange() } diff --git a/prog/weaver/main.go b/prog/weaver/main.go index bf85b916..5444f2f4 100644 --- a/prog/weaver/main.go +++ b/prog/weaver/main.go @@ -368,7 +368,8 @@ func createAllocator(router *weave.Router, ipRangeStr string, defaultSubnetStr s Log.Fatalf("IP address allocation default subnet %s does not overlap with allocation range %s", defaultSubnet, ipRange) } } - allocator := ipam.NewAllocator(router.Ourself.Peer.Name, router.Ourself.Peer.UID, router.Ourself.Peer.NickName, ipRange.Range(), quorum) + namesFunc := func() map[weave.PeerName]struct{} { return router.Peers.Names() } + allocator := ipam.NewAllocator(router.Ourself.Peer.Name, router.Ourself.Peer.UID, router.Ourself.Peer.NickName, ipRange.Range(), quorum, namesFunc) allocator.SetInterfaces(router.NewGossip("IPallocation", allocator)) allocator.Start() diff --git a/test/165_reset_2_test.sh b/test/165_reset_2_test.sh new file mode 100755 index 00000000..cc4cdaed --- /dev/null +++ b/test/165_reset_2_test.sh @@ -0,0 +1,22 @@ +#! /bin/bash + +. ./config.sh + +start_suite "Claiming addresses" + +weave_on $HOST1 launch-router +weave_on $HOST2 launch-router $HOST1 + +start_container $HOST1 --name=c1 +start_container $HOST2 --name=c2 + +# Now reset the routers so they should give their space away, and restart +weave_on $HOST1 reset +weave_on $HOST1 launch-router $HOST2 +weave_on $HOST2 reset +weave_on $HOST2 launch-router $HOST1 + +# And try to start a container +assert_raises "timeout 10 cat <( start_container $HOST1 )" + +end_suite From 27c5f5f49f976a74f8a5c241b5e759e359b96196 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Tue, 3 Nov 2015 12:03:00 +0000 Subject: [PATCH 2/5] Fix title of test --- test/165_reset_2_test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/165_reset_2_test.sh b/test/165_reset_2_test.sh index cc4cdaed..05d68677 100755 --- a/test/165_reset_2_test.sh +++ b/test/165_reset_2_test.sh @@ -2,7 +2,7 @@ . ./config.sh -start_suite "Claiming addresses" +start_suite "Rolling restart with reset" weave_on $HOST1 launch-router weave_on $HOST2 launch-router $HOST1 From ace372cfc65bb3c2022d22a74fc751319eb7a882 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Tue, 3 Nov 2015 12:03:28 +0000 Subject: [PATCH 3/5] More extensive choice of transferee --- ipam/allocator.go | 63 +++++++++++++++++++++++++++--------------- ipam/ring/ring.go | 4 +-- ipam/testutils_test.go | 2 +- prog/weaver/main.go | 4 +-- 4 files changed, 45 insertions(+), 28 deletions(-) diff --git a/ipam/allocator.go b/ipam/allocator.go index 958788cf..8e6cd6a8 100644 --- a/ipam/allocator.go +++ b/ipam/allocator.go @@ -56,20 +56,20 @@ type Allocator struct { paxosTicker *time.Ticker shuttingDown bool // to avoid doing any requests while trying to shut down now func() time.Time - namesFunc func() map[router.PeerName]struct{} + peerValidFunc func(router.PeerName) bool } // NewAllocator creates and initialises a new Allocator -func NewAllocator(ourName router.PeerName, ourUID router.PeerUID, ourNickname string, universe address.Range, quorum uint, names func() map[router.PeerName]struct{}) *Allocator { +func NewAllocator(ourName router.PeerName, ourUID router.PeerUID, ourNickname string, universe address.Range, quorum uint, validPeer func(name router.PeerName) bool) *Allocator { return &Allocator{ - ourName: ourName, - universe: universe, - ring: ring.New(universe.Start, universe.End, ourName), - owned: make(map[string][]address.Address), - paxos: paxos.NewNode(ourName, ourUID, quorum), - nicknames: map[router.PeerName]string{ourName: ourNickname}, - now: time.Now, - namesFunc: names, + ourName: ourName, + universe: universe, + ring: ring.New(universe.Start, universe.End, ourName), + owned: make(map[string][]address.Address), + paxos: paxos.NewNode(ourName, ourUID, quorum), + nicknames: map[router.PeerName]string{ourName: ourNickname}, + now: time.Now, + peerValidFunc: validPeer, } } @@ -276,6 +276,33 @@ func (alloc *Allocator) Free(ident string, addrToFree address.Address) error { return <-errChan } +func (alloc *Allocator) pickPeerFromNicknames(isValid func(router.PeerName) bool) router.PeerName { + for name := range alloc.nicknames { + if name != alloc.ourName && isValid(name) { + return name + } + } + return router.UnknownPeerName +} + +func (alloc *Allocator) pickPeerForTransfer() router.PeerName { + // first try alive peers that actively participate in IPAM (i.e. have entries) + if heir := alloc.ring.PickPeerForTransfer(alloc.peerValidFunc); heir != router.UnknownPeerName { + return heir + } + // next try alive peers that have IPAM enabled but have no entries + if heir := alloc.pickPeerFromNicknames(alloc.peerValidFunc); heir != router.UnknownPeerName { + return heir + } + // next try disappeared peers that still have entries + t := func(router.PeerName) bool { return true } + if heir := alloc.ring.PickPeerForTransfer(t); heir != router.UnknownPeerName { + return heir + } + // finally, disappeared peers that that passively participated in IPAM + return alloc.pickPeerFromNicknames(t) +} + // Shutdown (Sync) func (alloc *Allocator) Shutdown() { alloc.infof("Shutdown") @@ -284,16 +311,7 @@ func (alloc *Allocator) Shutdown() { alloc.shuttingDown = true alloc.cancelOps(&alloc.pendingClaims) alloc.cancelOps(&alloc.pendingAllocates) - heir := alloc.ring.PickPeerForTransfer() - if heir == router.UnknownPeerName && alloc.namesFunc != nil { - // No peers in the ring; pick one via the function we were given at init - for peerName := range alloc.namesFunc() { - if peerName != alloc.ourName { - heir = peerName - break - } - } - } + heir := alloc.pickPeerForTransfer() if heir != router.UnknownPeerName { alloc.ring.Transfer(alloc.ourName, heir) alloc.space.Clear() @@ -322,7 +340,6 @@ func (alloc *Allocator) AdminTakeoverRanges(peerNameOrNickname string) error { return } - delete(alloc.nicknames, peername) newRanges, err := alloc.ring.Transfer(peername, alloc.ourName) alloc.space.AddRanges(newRanges) resultChan <- err @@ -344,11 +361,11 @@ func (alloc *Allocator) lookupPeername(name string) (router.PeerName, error) { return router.PeerNameFromString(name) } -// Restrict the peers in "nicknames" to those in the ring and our own +// Restrict the peers in "nicknames" to those in the ring plus peers known to the router func (alloc *Allocator) pruneNicknames() { ringPeers := alloc.ring.PeerNames() for name := range alloc.nicknames { - if _, ok := ringPeers[name]; !ok && name != alloc.ourName { + if _, ok := ringPeers[name]; !ok && !alloc.peerValidFunc(name) { delete(alloc.nicknames, name) } } diff --git a/ipam/ring/ring.go b/ipam/ring/ring.go index b82bad1d..fee7991e 100644 --- a/ipam/ring/ring.go +++ b/ipam/ring/ring.go @@ -470,9 +470,9 @@ func (r *Ring) ChoosePeersToAskForSpace(start, end address.Address) []router.Pee return result } -func (r *Ring) PickPeerForTransfer() router.PeerName { +func (r *Ring) PickPeerForTransfer(isValid func(router.PeerName) bool) router.PeerName { for _, entry := range r.Entries { - if entry.Peer != r.Peer { + if entry.Peer != r.Peer && isValid(entry.Peer) { return entry.Peer } } diff --git a/ipam/testutils_test.go b/ipam/testutils_test.go index ea954aa0..144e313f 100644 --- a/ipam/testutils_test.go +++ b/ipam/testutils_test.go @@ -135,7 +135,7 @@ func makeAllocator(name string, cidrStr string, quorum uint) (*Allocator, addres } alloc := NewAllocator(peername, router.PeerUID(rand.Int63()), - "nick-"+name, cidr.Range(), quorum, nil) + "nick-"+name, cidr.Range(), quorum, func(router.PeerName) bool { return true }) return alloc, cidr.HostRange() } diff --git a/prog/weaver/main.go b/prog/weaver/main.go index 5444f2f4..dae87086 100644 --- a/prog/weaver/main.go +++ b/prog/weaver/main.go @@ -368,8 +368,8 @@ func createAllocator(router *weave.Router, ipRangeStr string, defaultSubnetStr s Log.Fatalf("IP address allocation default subnet %s does not overlap with allocation range %s", defaultSubnet, ipRange) } } - namesFunc := func() map[weave.PeerName]struct{} { return router.Peers.Names() } - allocator := ipam.NewAllocator(router.Ourself.Peer.Name, router.Ourself.Peer.UID, router.Ourself.Peer.NickName, ipRange.Range(), quorum, namesFunc) + isValidFunc := func(name weave.PeerName) bool { return router.Peers.Fetch(name) != nil } + allocator := ipam.NewAllocator(router.Ourself.Peer.Name, router.Ourself.Peer.UID, router.Ourself.Peer.NickName, ipRange.Range(), quorum, isValidFunc) allocator.SetInterfaces(router.NewGossip("IPallocation", allocator)) allocator.Start() From 8742957f75904f6de1114c4c06d67d7a06517ef9 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Tue, 3 Nov 2015 14:15:49 +0000 Subject: [PATCH 4/5] Tweak naming to be closer to code changes in nameserver --- ipam/allocator.go | 26 +++++++++++++------------- prog/weaver/main.go | 4 ++-- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/ipam/allocator.go b/ipam/allocator.go index 8e6cd6a8..d52ba6d3 100644 --- a/ipam/allocator.go +++ b/ipam/allocator.go @@ -56,20 +56,20 @@ type Allocator struct { paxosTicker *time.Ticker shuttingDown bool // to avoid doing any requests while trying to shut down now func() time.Time - peerValidFunc func(router.PeerName) bool + isKnownPeer func(router.PeerName) bool } // NewAllocator creates and initialises a new Allocator -func NewAllocator(ourName router.PeerName, ourUID router.PeerUID, ourNickname string, universe address.Range, quorum uint, validPeer func(name router.PeerName) bool) *Allocator { +func NewAllocator(ourName router.PeerName, ourUID router.PeerUID, ourNickname string, universe address.Range, quorum uint, isKnownPeer func(name router.PeerName) bool) *Allocator { return &Allocator{ - ourName: ourName, - universe: universe, - ring: ring.New(universe.Start, universe.End, ourName), - owned: make(map[string][]address.Address), - paxos: paxos.NewNode(ourName, ourUID, quorum), - nicknames: map[router.PeerName]string{ourName: ourNickname}, - now: time.Now, - peerValidFunc: validPeer, + ourName: ourName, + universe: universe, + ring: ring.New(universe.Start, universe.End, ourName), + owned: make(map[string][]address.Address), + paxos: paxos.NewNode(ourName, ourUID, quorum), + nicknames: map[router.PeerName]string{ourName: ourNickname}, + now: time.Now, + isKnownPeer: isKnownPeer, } } @@ -287,11 +287,11 @@ func (alloc *Allocator) pickPeerFromNicknames(isValid func(router.PeerName) bool func (alloc *Allocator) pickPeerForTransfer() router.PeerName { // first try alive peers that actively participate in IPAM (i.e. have entries) - if heir := alloc.ring.PickPeerForTransfer(alloc.peerValidFunc); heir != router.UnknownPeerName { + if heir := alloc.ring.PickPeerForTransfer(alloc.isKnownPeer); heir != router.UnknownPeerName { return heir } // next try alive peers that have IPAM enabled but have no entries - if heir := alloc.pickPeerFromNicknames(alloc.peerValidFunc); heir != router.UnknownPeerName { + if heir := alloc.pickPeerFromNicknames(alloc.isKnownPeer); heir != router.UnknownPeerName { return heir } // next try disappeared peers that still have entries @@ -365,7 +365,7 @@ func (alloc *Allocator) lookupPeername(name string) (router.PeerName, error) { func (alloc *Allocator) pruneNicknames() { ringPeers := alloc.ring.PeerNames() for name := range alloc.nicknames { - if _, ok := ringPeers[name]; !ok && !alloc.peerValidFunc(name) { + if _, ok := ringPeers[name]; !ok && !alloc.isKnownPeer(name) { delete(alloc.nicknames, name) } } diff --git a/prog/weaver/main.go b/prog/weaver/main.go index dae87086..57a54f72 100644 --- a/prog/weaver/main.go +++ b/prog/weaver/main.go @@ -368,8 +368,8 @@ func createAllocator(router *weave.Router, ipRangeStr string, defaultSubnetStr s Log.Fatalf("IP address allocation default subnet %s does not overlap with allocation range %s", defaultSubnet, ipRange) } } - isValidFunc := func(name weave.PeerName) bool { return router.Peers.Fetch(name) != nil } - allocator := ipam.NewAllocator(router.Ourself.Peer.Name, router.Ourself.Peer.UID, router.Ourself.Peer.NickName, ipRange.Range(), quorum, isValidFunc) + isKnownPeer := func(name weave.PeerName) bool { return router.Peers.Fetch(name) != nil } + allocator := ipam.NewAllocator(router.Ourself.Peer.Name, router.Ourself.Peer.UID, router.Ourself.Peer.NickName, ipRange.Range(), quorum, isKnownPeer) allocator.SetInterfaces(router.NewGossip("IPallocation", allocator)) allocator.Start() From 67e9310cc18f9edfc03d20a1fd2731e23f814178 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Tue, 3 Nov 2015 17:00:24 +0000 Subject: [PATCH 5/5] Review feedback --- ipam/allocator.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/ipam/allocator.go b/ipam/allocator.go index d52ba6d3..05fdcda1 100644 --- a/ipam/allocator.go +++ b/ipam/allocator.go @@ -55,8 +55,8 @@ type Allocator struct { paxos *paxos.Node paxosTicker *time.Ticker shuttingDown bool // to avoid doing any requests while trying to shut down - now func() time.Time isKnownPeer func(router.PeerName) bool + now func() time.Time } // NewAllocator creates and initialises a new Allocator @@ -68,8 +68,8 @@ func NewAllocator(ourName router.PeerName, ourUID router.PeerUID, ourNickname st owned: make(map[string][]address.Address), paxos: paxos.NewNode(ourName, ourUID, quorum), nicknames: map[router.PeerName]string{ourName: ourNickname}, - now: time.Now, isKnownPeer: isKnownPeer, + now: time.Now, } } @@ -311,8 +311,7 @@ func (alloc *Allocator) Shutdown() { alloc.shuttingDown = true alloc.cancelOps(&alloc.pendingClaims) alloc.cancelOps(&alloc.pendingAllocates) - heir := alloc.pickPeerForTransfer() - if heir != router.UnknownPeerName { + if heir := alloc.pickPeerForTransfer(); heir != router.UnknownPeerName { alloc.ring.Transfer(alloc.ourName, heir) alloc.space.Clear() alloc.gossip.GossipBroadcast(alloc.Gossip())