From c7a9dfbb3f3e6181c07fb1bce3e1124de944ad9a Mon Sep 17 00:00:00 2001
From: Bryan Boreham <bjboreham@gmail.com>
Date: Tue, 2 Jun 2015 15:27:31 +0100
Subject: [PATCH] Remember multiple addresses owned by each container, for
 multi-subnet use

---
 ipam/address/address.go |  9 +++++++++
 ipam/allocate.go        | 15 ++++++++++++---
 ipam/allocator.go       | 34 ++++++++++++++++++----------------
 3 files changed, 39 insertions(+), 19 deletions(-)

diff --git a/ipam/address/address.go b/ipam/address/address.go
index 0eae12b5..b5dbc2cb 100644
--- a/ipam/address/address.go
+++ b/ipam/address/address.go
@@ -46,6 +46,15 @@ func (cidr CIDR) String() string {
 	return fmt.Sprintf("%s/%d", cidr.Start.String(), cidr.PrefixLen)
 }
 
+func (cidr1 CIDR) Overlaps(cidr2 CIDR) bool {
+	return cidr1.Start >= cidr2.Start && cidr1.Start < cidr2.End() ||
+		cidr2.Start >= cidr1.Start && cidr2.Start < cidr1.End()
+}
+
+func (cidr CIDR) Contains(addr Address) bool {
+	return addr >= cidr.Start && addr < cidr.End()
+}
+
 // FromIP4 converts an ipv4 address to our integer address type
 func FromIP4(ip4 net.IP) (r Address) {
 	for _, b := range ip4.To4() {
diff --git a/ipam/allocate.go b/ipam/allocate.go
index 9fe2522c..8fe28ad0 100644
--- a/ipam/allocate.go
+++ b/ipam/allocate.go
@@ -24,9 +24,18 @@ func (g *allocate) Try(alloc *Allocator) bool {
 		return true
 	}
 
-	// If we have previously stored an address for this container, return it.
-	if addr, found := alloc.owned[g.ident]; found {
-		g.resultChan <- allocateResult{addr, nil}
+	// If we have previously stored an address for this container in this subnet, return it.
+	if addrs, found := alloc.owned[g.ident]; found {
+		for _, addr := range addrs {
+			if g.subnet.Contains(addr) {
+				g.resultChan <- allocateResult{addr, nil}
+				return true
+			}
+		}
+	}
+
+	if !alloc.universe.Overlaps(g.subnet) {
+		g.resultChan <- allocateResult{0, fmt.Errorf("Subnet %s out of bounds: %s", g.subnet, alloc.universe)}
 		return true
 	}
 
diff --git a/ipam/allocator.go b/ipam/allocator.go
index ceea6caf..cd714f3e 100644
--- a/ipam/allocator.go
+++ b/ipam/allocator.go
@@ -45,14 +45,14 @@ type Allocator struct {
 	actionChan       chan<- func()
 	ourName          router.PeerName
 	defaultSubnet    address.CIDR
-	universe         address.CIDR               // superset of all ranges
-	ring             *ring.Ring                 // information on ranges owned by all peers
-	space            space.Space                // more detail on ranges owned by us
-	owned            map[string]address.Address // who owns what address, indexed by container-ID -- fixme - needs to be multi-subnet
-	nicknames        map[router.PeerName]string // so we can map nicknames for rmpeer
-	pendingAllocates []operation                // held until we get some free space
-	pendingClaims    []operation                // held until we know who owns the space
-	gossip           router.Gossip              // our link to the outside world for sending messages
+	universe         address.CIDR                 // superset of all ranges
+	ring             *ring.Ring                   // information on ranges owned by all peers
+	space            space.Space                  // more detail on ranges owned by us
+	owned            map[string][]address.Address // who owns what address, indexed by container-ID
+	nicknames        map[router.PeerName]string   // so we can map nicknames for rmpeer
+	pendingAllocates []operation                  // held until we get some free space
+	pendingClaims    []operation                  // held until we know who owns the space
+	gossip           router.Gossip                // our link to the outside world for sending messages
 	paxos            *paxos.Node
 	paxosTicker      *time.Ticker
 	shuttingDown     bool // to avoid doing any requests while trying to shut down
@@ -65,7 +65,7 @@ func NewAllocator(ourName router.PeerName, ourUID router.PeerUID, ourNickname st
 		ourName:   ourName,
 		universe:  universe,
 		ring:      ring.New(universe.Start, address.Add(universe.Start, universe.Size()), ourName),
-		owned:     make(map[string]address.Address),
+		owned:     make(map[string][]address.Address),
 		paxos:     paxos.NewNode(ourName, ourUID, quorum),
 		nicknames: map[router.PeerName]string{ourName: ourNickname},
 		now:       time.Now,
@@ -205,7 +205,7 @@ func (alloc *Allocator) Lookup(ident string) (address.Address, error) {
 	alloc.actionChan <- func() {
 		addr, found := alloc.owned[ident]
 		if found {
-			resultChan <- allocateResult{addr: addr}
+			resultChan <- allocateResult{addr: addr[0]}
 			return
 		}
 		resultChan <- allocateResult{err: fmt.Errorf("lookup: no address found")}
@@ -237,8 +237,8 @@ func (alloc *Allocator) ContainerDied(ident string) error {
 func (alloc *Allocator) free(ident string) error {
 	errChan := make(chan error)
 	alloc.actionChan <- func() {
-		addr, found := alloc.owned[ident]
-		if found {
+		addrs, found := alloc.owned[ident]
+		for _, addr := range addrs {
 			alloc.space.Free(addr)
 		}
 		delete(alloc.owned, ident)
@@ -669,13 +669,15 @@ func (alloc *Allocator) reportFreeSpace() {
 // Owned addresses
 
 func (alloc *Allocator) addOwned(ident string, addr address.Address) {
-	alloc.owned[ident] = addr
+	alloc.owned[ident] = append(alloc.owned[ident], addr)
 }
 
 func (alloc *Allocator) findOwner(addr address.Address) string {
-	for ident, candidate := range alloc.owned {
-		if candidate == addr {
-			return ident
+	for ident, addrs := range alloc.owned {
+		for _, candidate := range addrs {
+			if candidate == addr {
+				return ident
+			}
 		}
 	}
 	return ""