From 8405e0759aa3e2b8778c3e2fe0dc1134ae4abc7f Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Fri, 29 Apr 2016 13:59:25 +0100 Subject: [PATCH 1/3] Clarify the function of the periodic ticker. - it either retries for consensus or retries operations we would have tried previously. There is no point retrying pending operations if we are waiting for consensus. --- ipam/allocator.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ipam/allocator.go b/ipam/allocator.go index 741390fa..6d1ed7c4 100644 --- a/ipam/allocator.go +++ b/ipam/allocator.go @@ -614,11 +614,13 @@ func (alloc *Allocator) actorLoop(actionChan <-chan func(), stopChan <-chan stru case <-stopChan: return case <-alloc.ticker.C: + // Retry things in case messages got lost between here and recipients if alloc.awaitingConsensus { alloc.propose() + } else { + alloc.tryPendingOps() } alloc.removeDeadContainers() - alloc.tryPendingOps() } alloc.assertInvariants() From 6847a3551df322cfda90db867863d0bfac90df06 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Thu, 28 Apr 2016 16:22:47 +0000 Subject: [PATCH 2/3] Persist the ring in Shutdown() and AdminTakeoverRanges(). Also make AdminTakeoverRanges cause pending ops to be retried. --- ipam/allocator.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ipam/allocator.go b/ipam/allocator.go index 6d1ed7c4..404716ec 100644 --- a/ipam/allocator.go +++ b/ipam/allocator.go @@ -402,6 +402,7 @@ func (alloc *Allocator) Shutdown() { alloc.cancelOps(&alloc.pendingPrimes) if heir := alloc.pickPeerForTransfer(); heir != mesh.UnknownPeerName { alloc.ring.Transfer(alloc.ourName, heir) + alloc.persistRing() alloc.space.Clear() alloc.gossip.GossipBroadcast(alloc.Gossip()) time.Sleep(100 * time.Millisecond) @@ -439,7 +440,7 @@ func (alloc *Allocator) AdminTakeoverRanges(peerNameOrNickname string) address.C } before := alloc.space.NumFreeAddresses() - alloc.space.AddRanges(newRanges) + alloc.ringUpdated() after := alloc.space.NumFreeAddresses() alloc.gossip.GossipBroadcast(alloc.Gossip()) From 3252a25e00ba153d387a457ebdba6d9cc5b81395 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Fri, 29 Apr 2016 13:53:33 +0100 Subject: [PATCH 3/3] Broadcast initial ring after persisting it, in case we die between those operations --- ipam/allocator.go | 2 +- ipam/allocator_test.go | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ipam/allocator.go b/ipam/allocator.go index 404716ec..8479d443 100644 --- a/ipam/allocator.go +++ b/ipam/allocator.go @@ -650,8 +650,8 @@ func (alloc *Allocator) establishRing() { func (alloc *Allocator) createRing(peers []mesh.PeerName) { alloc.debugln("Paxos consensus:", peers) alloc.ring.ClaimForPeers(normalizeConsensus(peers)) - alloc.gossip.GossipBroadcast(alloc.Gossip()) alloc.ringUpdated() + alloc.gossip.GossipBroadcast(alloc.Gossip()) } func (alloc *Allocator) ringUpdated() { diff --git a/ipam/allocator_test.go b/ipam/allocator_test.go index c851a344..4234858c 100644 --- a/ipam/allocator_test.go +++ b/ipam/allocator_test.go @@ -176,6 +176,7 @@ func TestAllocatorClaim(t *testing.T) { alloc.Prime() // Do an allocate on the other peer, which we will try to claim later addrx, err := allocs[0].Allocate(container1, subnet, true, returnFalse) + router.Flush() // Now try the claim again err = alloc.Claim(container3, addr1, true, true)