From 551394a831ad0ab5edfa4eb69395c7174aea66b0 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Thu, 4 Jun 2015 10:29:37 +0100 Subject: [PATCH] Add subnet info to space requests from one peer to another --- ipam/allocate.go | 4 ++-- ipam/allocator.go | 30 +++++++++++++++++------ ipam/space/space.go | 15 ++++++++---- ipam/space/space_test.go | 51 +++++++++++++++++++++++----------------- 4 files changed, 65 insertions(+), 35 deletions(-) diff --git a/ipam/allocate.go b/ipam/allocate.go index 8fe28ad0..71eb8e82 100644 --- a/ipam/allocate.go +++ b/ipam/allocate.go @@ -50,8 +50,8 @@ func (g *allocate) Try(alloc *Allocator) bool { // out of space if donor, err := alloc.ring.ChoosePeerToAskForSpace(start, end); err == nil { - alloc.debugln("Decided to ask peer", donor, "for space") - alloc.sendRequest(donor, msgSpaceRequest) + alloc.debugln("Decided to ask peer", donor, "for space in subnet", g.subnet) + alloc.sendSpaceRequest(donor, g.subnet) } return false diff --git a/ipam/allocator.go b/ipam/allocator.go index cd714f3e..f30d3949 100644 --- a/ipam/allocator.go +++ b/ipam/allocator.go @@ -341,7 +341,13 @@ func (alloc *Allocator) OnGossipUnicast(sender router.PeerName, msg []byte) erro switch msg[0] { case msgSpaceRequest: // some other peer asked us for space - alloc.donateSpace(sender) + decoder := gob.NewDecoder(bytes.NewReader(msg[1:])) + var cidr address.CIDR + if err := decoder.Decode(&cidr); err != nil { + resultChan <- err + return + } + alloc.donateSpace(cidr, sender) resultChan <- nil case msgRingUpdate: resultChan <- alloc.update(msg[1:]) @@ -562,8 +568,18 @@ func (alloc *Allocator) propose() { alloc.gossip.GossipBroadcast(alloc.Gossip()) } -func (alloc *Allocator) sendRequest(dest router.PeerName, kind byte) { - msg := router.Concat([]byte{kind}, alloc.encode()) +func (alloc *Allocator) sendSpaceRequest(dest router.PeerName, subnet address.CIDR) { + buf := new(bytes.Buffer) + enc := gob.NewEncoder(buf) + if err := enc.Encode(subnet); err != nil { + panic(err) + } + msg := router.Concat([]byte{msgSpaceRequest}, buf.Bytes()) + alloc.gossip.GossipUnicast(dest, msg) +} + +func (alloc *Allocator) sendRingUpdate(dest router.PeerName) { + msg := router.Concat([]byte{msgRingUpdate}, alloc.encode()) alloc.gossip.GossipUnicast(dest, msg) } @@ -615,18 +631,18 @@ func (alloc *Allocator) update(msg []byte) error { return nil } -func (alloc *Allocator) donateSpace(to router.PeerName) { +func (alloc *Allocator) donateSpace(subnet address.CIDR, to router.PeerName) { // No matter what we do, we'll send a unicast gossip // of our ring back to tha chap who asked for space. // This serves to both tell him of any space we might // have given him, or tell him where he might find some // more. - defer alloc.sendRequest(to, msgRingUpdate) + defer alloc.sendRingUpdate(to) alloc.debugln("Peer", to, "asked me for space") - start, size, ok := alloc.space.Donate() + start, size, ok := alloc.space.Donate(subnet.Start, subnet.End()) if !ok { - free := alloc.space.NumFreeAddresses() + free := alloc.space.NumFreeAddressesInRange(subnet.Start, subnet.End()) common.Assert(free == 0) alloc.debugln("No space to give to peer", to) return diff --git a/ipam/space/space.go b/ipam/space/space.go index c4d8ea5f..2e2bd8d5 100644 --- a/ipam/space/space.go +++ b/ipam/space/space.go @@ -108,12 +108,19 @@ func (s *Space) Free(addr address.Address) error { return nil } -func (s *Space) biggestFreeRange() (int, address.Offset) { +func (s *Space) biggestFreeRange(rangeStart, rangeEnd address.Address) (int, address.Offset) { pos := -1 biggest := address.Offset(0) for i := 0; i < len(s.free); i += 2 { - size := address.Subtract(s.free[i+1], s.free[i]) + start, end := s.free[i], s.free[i+1] + if start < rangeStart { + start = rangeStart + } + if end > rangeEnd { + end = rangeEnd + } + size := address.Subtract(end, start) if size >= biggest { pos = i biggest = size @@ -122,12 +129,12 @@ func (s *Space) biggestFreeRange() (int, address.Offset) { return pos, biggest } -func (s *Space) Donate() (address.Address, address.Offset, bool) { +func (s *Space) Donate(rangeStart, rangeEnd address.Address) (address.Address, address.Offset, bool) { if len(s.free) == 0 { return 0, 0, false } - pos, biggest := s.biggestFreeRange() + pos, biggest := s.biggestFreeRange(rangeStart, rangeEnd) // Donate half of that biggest free range, rounding up so // that the donation can't be empty diff --git a/ipam/space/space_test.go b/ipam/space/space_test.go index 11df7b58..1585a2c5 100644 --- a/ipam/space/space_test.go +++ b/ipam/space/space_test.go @@ -67,19 +67,19 @@ func TestLowlevel(t *testing.T) { wt.AssertErrorInterface(t, s.Free(0), (*error)(nil), "free not allocated") wt.AssertErrorInterface(t, s.Free(100), (*error)(nil), "double free") - start, size, ok := s.Donate() + start, size, ok := s.Donate(0, 1000) wt.AssertTrue(t, ok && start == 125 && size == 25, "donate") // test Donate when addresses are scarce s = New() - start, size, ok = s.Donate() + start, size, ok = s.Donate(0, 1000) wt.AssertTrue(t, !ok, "donate on empty space should fail") s.Add(0, 3) wt.AssertNoErr(t, s.Claim(0)) wt.AssertNoErr(t, s.Claim(2)) - start, size, ok = s.Donate() + start, size, ok = s.Donate(0, 1000) wt.AssertTrue(t, ok && start == 1 && size == 1, "donate") - start, size, ok = s.Donate() + start, size, ok = s.Donate(0, 1000) wt.AssertTrue(t, !ok, "donate should fail") } @@ -90,19 +90,24 @@ func TestSpaceAllocate(t *testing.T) { testAddrx = "10.0.3.19" testAddry = "10.0.9.19" containerID = "deadbeef" + size = 20 + ) + var ( + start = ip(testAddr1) + end = address.Add(ip(testAddr1), 20) ) - space1 := makeSpace(ip(testAddr1), 20) + space1 := makeSpace(start, size) wt.AssertEquals(t, space1.NumFreeAddresses(), address.Offset(20)) space1.assertInvariants() - _, addr1 := space1.Allocate(ip(testAddr2), ip(testAddrx)) - wt.AssertEqualString(t, addr1.String(), testAddr2, "address") + _, addr1 := space1.Allocate(start, end) + wt.AssertEqualString(t, addr1.String(), testAddr1, "address") wt.AssertEquals(t, space1.NumFreeAddresses(), address.Offset(19)) space1.assertInvariants() - _, addr2 := space1.Allocate(ip(testAddr2), ip(testAddrx)) - wt.AssertFalse(t, addr2.String() == testAddr2, "address") + _, addr2 := space1.Allocate(start, end) + wt.AssertFalse(t, addr2.String() == testAddr1, "address") wt.AssertEquals(t, space1.NumFreeAddresses(), address.Offset(18)) wt.AssertEquals(t, space1.NumFreeAddressesInRange(ip(testAddr1), ip(testAddrx)), address.Offset(13)) wt.AssertEquals(t, space1.NumFreeAddressesInRange(ip(testAddr1), ip(testAddry)), address.Offset(18)) @@ -129,7 +134,7 @@ func TestSpaceFree(t *testing.T) { space := makeSpace(ip(testAddr1), 20) // Check we are prepared to give up the entire space - pos, size := space.biggestFreeRange() + pos, size := space.biggestFreeRange(ip(testAddr1), ip(testAddry)) wt.AssertTrue(t, space.free[pos] == ip(testAddr1) && size == 20, "Wrong space") for i := 0; i < 20; i++ { @@ -140,17 +145,17 @@ func TestSpaceFree(t *testing.T) { // Check we are full ok, _ := space.Allocate(ip(testAddr1), ip(testAddry)) wt.AssertTrue(t, !ok, "Should have failed to get address") - _, size, ok = space.Donate() + _, size, ok = space.Donate(ip(testAddr1), ip(testAddry)) wt.AssertTrue(t, size == 0, "Wrong space") // Free in the middle wt.AssertSuccess(t, space.Free(ip("10.0.3.13"))) - pos, size = space.biggestFreeRange() + pos, size = space.biggestFreeRange(ip(testAddr1), ip(testAddry)) wt.AssertTrue(t, space.free[pos] == ip("10.0.3.13") && size == 1, "Wrong space") // Free one at the end wt.AssertSuccess(t, space.Free(ip("10.0.3.23"))) - pos, size = space.biggestFreeRange() + pos, size = space.biggestFreeRange(ip(testAddr1), ip(testAddry)) wt.AssertTrue(t, space.free[pos] == ip("10.0.3.23") && size == 1, "Wrong space") // Now free a few at the end @@ -160,7 +165,7 @@ func TestSpaceFree(t *testing.T) { wt.AssertEquals(t, space.NumFreeAddresses(), address.Offset(4)) // Now get the biggest free space; should be 3.21 - pos, size = space.biggestFreeRange() + pos, size = space.biggestFreeRange(ip(testAddr1), ip(testAddry)) wt.AssertTrue(t, space.free[pos] == ip("10.0.3.21") && size == 3, "Wrong space") // Now free a few in the middle @@ -171,7 +176,7 @@ func TestSpaceFree(t *testing.T) { wt.AssertEquals(t, space.NumFreeAddresses(), address.Offset(7)) // Now get the biggest free space; should be 3.21 - pos, size = space.biggestFreeRange() + pos, size = space.biggestFreeRange(ip(testAddr1), ip(testAddry)) wt.AssertTrue(t, space.free[pos] == ip("10.0.3.10") && size == 4, "Wrong space") wt.AssertEquals(t, space.OwnedRanges(), []address.Range{address.Range{Start: ip("10.0.3.4"), End: ip("10.0.3.24")}}) @@ -185,32 +190,33 @@ func TestDonateSimple(t *testing.T) { const ( testAddr1 = "10.0.1.0" testAddr2 = "10.0.1.32" + size = 48 ) var ( ipAddr1 = ip(testAddr1) ) - ps1 := makeSpace(ipAddr1, 48) + ps1 := makeSpace(ipAddr1, size) // Empty space set should split in two and give me the second half - start, numGivenUp, ok := ps1.Donate() + start, numGivenUp, ok := ps1.Donate(ip(testAddr1), ip(testAddr1)+size) wt.AssertTrue(t, ok, "Donate result") wt.AssertEqualString(t, start.String(), "10.0.1.24", "Invalid start") - wt.AssertEquals(t, numGivenUp, address.Offset(24)) - wt.AssertEquals(t, ps1.NumFreeAddresses(), address.Offset(24)) + wt.AssertEquals(t, numGivenUp, address.Offset(size/2)) + wt.AssertEquals(t, ps1.NumFreeAddresses(), address.Offset(size/2)) // Now check we can give the rest up. count := 0 // count to avoid infinite loop for ; count < 1000; count++ { - _, size, ok := ps1.Donate() + _, size, ok := ps1.Donate(ip(testAddr1), ip(testAddr1)+size) if !ok { break } numGivenUp += size } wt.AssertEquals(t, ps1.NumFreeAddresses(), address.Offset(0)) - wt.AssertEquals(t, numGivenUp, address.Offset(48)) + wt.AssertEquals(t, numGivenUp, address.Offset(size)) } func TestDonateHard(t *testing.T) { @@ -218,6 +224,7 @@ func TestDonateHard(t *testing.T) { var ( start = ip("10.0.1.0") size address.Offset = 48 + end = address.Add(start, size) ) // Fill a fresh space @@ -236,7 +243,7 @@ func TestDonateHard(t *testing.T) { } // Now split - newRange, numGivenUp, ok := spaceset.Donate() + newRange, numGivenUp, ok := spaceset.Donate(start, end) wt.AssertTrue(t, ok, "GiveUpSpace result") wt.AssertEquals(t, newRange, ip("10.0.1.23")) wt.AssertEquals(t, numGivenUp, address.Offset(24))