Merge pull request #827 from bboreham/675-ipam-subnets

multi-subnet ipam

Closes #675. Fixes #861. Closes #862.

docs/ipam.md

@@ -20,11 +20,16 @@ plugin) uses to request IP addresses.
 
 The commands supported by the allocator via the http interface are:
 
-- Allocate: request one IP address for a container
+- Allocate: request one IP address for a container in a subnet
+- Lookup: fetch the previously-allocated IP address for a container in a subnet
 - Free: return an IP address that is currently allocated
 - Claim: request a specific IP address for a container (e.g. because
   it is already using that address)
 
+Each http request either specifies a subnet, or if no subnet is
+specified this is taken as a request to allocate in a pre-defined
+default subnet.
+
 The allocator also watches via the Docker event mechanism: if a
 container dies then all IP addresses allocated to that container are
 freed.
@@ -48,16 +53,17 @@ freed.
 
 When a peer owns some range(s), it can allocate freely from within
 those ranges to containers on the same machine. If it runs out of
-space (all owned ranges are full), it will ask another peer for space:
+space in a subnet (all owned ranges are full), it will ask another
+peer for space:
   - it picks a peer to ask at random, weighted by the amount of space
-    owned by each peer
+    owned by each peer in the subnet
     - if the target peer decides to give up space, it unicasts a message
       back to the asker with the newly-updated ring.
     - if the target peer has no space, it unicasts a message back to the
       asker with its current copy of the ring, on the basis that the
       requestor must have acted on out-of-date information.
   - it will continue to ask for space until it receives some, or its
-    copy of the ring tells it all peers are full.
+    copy of the ring tells it all peers are full in that subnet.
 
 ### Claiming an address
 

ipam/address/address.go

@@ -1,8 +1,10 @@
 package address
 
 import (
-	"github.com/weaveworks/weave/common"
+	"fmt"
 	"net"
+
+	"github.com/weaveworks/weave/common"
 )
 
 // Using 32-bit integer to represent IPv4 address
@@ -13,6 +15,19 @@ type Range struct {
 	Start, End Address // [Start, End); Start <= End
 }
 
+func NewRange(start Address, size Offset) Range {
+	return Range{Start: start, End: Add(start, size)}
+}
+func (r Range) Size() Offset               { return Subtract(r.End, r.Start) }
+func (r Range) String() string             { return fmt.Sprintf("[%s-%s)", r.Start, r.End) }
+func (r Range) Overlaps(or Range) bool     { return !(r.Start >= or.End || r.End <= or.Start) }
+func (r Range) Contains(addr Address) bool { return addr >= r.Start && addr < r.End }
+
+type CIDR struct {
+	Start     Address
+	PrefixLen int
+}
+
 func ParseIP(s string) (Address, error) {
 	if ip := net.ParseIP(s); ip != nil {
 		return FromIP4(ip), nil
@@ -20,6 +35,31 @@ func ParseIP(s string) (Address, error) {
 	return 0, &net.ParseError{Type: "IP Address", Text: s}
 }
 
+func ParseCIDR(s string) (Address, CIDR, error) {
+	if ip, ipnet, err := net.ParseCIDR(s); err != nil {
+		return 0, CIDR{}, err
+	} else if ipnet.IP.To4() == nil {
+		return 0, CIDR{}, &net.ParseError{Type: "Non-IPv4 address not supported", Text: s}
+	} else {
+		prefixLen, _ := ipnet.Mask.Size()
+		return FromIP4(ip), CIDR{Start: FromIP4(ipnet.IP), PrefixLen: prefixLen}, nil
+	}
+}
+
+func (cidr CIDR) Size() Offset { return 1 << uint(32-cidr.PrefixLen) }
+
+func (cidr CIDR) Range() Range {
+	return NewRange(cidr.Start, cidr.Size())
+}
+func (cidr CIDR) HostRange() Range {
+	// Respect RFC1122 exclusions of first and last addresses
+	return NewRange(cidr.Start+1, cidr.Size()-2)
+}
+
+func (cidr CIDR) String() string {
+	return fmt.Sprintf("%s/%d", cidr.Start.String(), cidr.PrefixLen)
+}
+
 // FromIP4 converts an ipv4 address to our integer address type
 func FromIP4(ip4 net.IP) (r Address) {
 	for _, b := range ip4.To4() {

ipam/allocate.go

@@ -12,8 +12,9 @@ type allocateResult struct {
 
 type allocate struct {
 	resultChan       chan<- allocateResult
-	hasBeenCancelled func() bool
 	ident            string
+	r                address.Range // Range we are trying to allocate within
+	hasBeenCancelled func() bool
 }
 
 // Try returns true if the request is completed, false if pending
@@ -23,23 +24,27 @@ func (g *allocate) Try(alloc *Allocator) bool {
 		return true
 	}
 
-	// If we have previously stored an address for this container, return it.
-	if addr, found := alloc.owned[g.ident]; found {
+	if addr, found := alloc.lookupOwned(g.ident, g.r); found {
 		g.resultChan <- allocateResult{addr, nil}
 		return true
 	}
 
-	if ok, addr := alloc.space.Allocate(); ok {
-		alloc.debugln("Allocated", addr, "for", g.ident)
+	if !alloc.universe.Overlaps(g.r) {
+		g.resultChan <- allocateResult{0, fmt.Errorf("range %s out of bounds: %s", g.r, alloc.universe)}
+		return true
+	}
+
+	if ok, addr := alloc.space.Allocate(g.r); ok {
+		alloc.debugln("Allocated", addr, "for", g.ident, "in", g.r)
 		alloc.addOwned(g.ident, addr)
 		g.resultChan <- allocateResult{addr, nil}
 		return true
 	}
 
 	// out of space
-	if donor, err := alloc.ring.ChoosePeerToAskForSpace(); err == nil {
-		alloc.debugln("Decided to ask peer", donor, "for space")
-		alloc.sendRequest(donor, msgSpaceRequest)
+	if donor, err := alloc.ring.ChoosePeerToAskForSpace(g.r.Start, g.r.End); err == nil {
+		alloc.debugln("Decided to ask peer", donor, "for space in range", g.r)
+		alloc.sendSpaceRequest(donor, g.r)
 	}
 
 	return false

ipam/allocator.go

@@ -3,9 +3,7 @@ package ipam
 import (
 	"bytes"
 	"encoding/gob"
-	"errors"
 	"fmt"
-	"net"
 	"sort"
 	"time"
 
@@ -23,6 +21,7 @@ const (
 	msgRingUpdate
 
 	paxosInterval = time.Second * 5
+	MinSubnetSize = 4 // first and last addresses are excluded, so 2 would be too small
 )
 
 // operation represents something which Allocator wants to do, but
@@ -46,16 +45,14 @@ type operation interface {
 type Allocator struct {
 	actionChan       chan<- func()
 	ourName          router.PeerName
-	subnetStart      address.Address            // start address of space all peers are allocating from
-	subnetSize       address.Offset             // length of space all peers are allocating from
-	prefixLen        int                        // network prefix length, e.g. 24 for a /24 network
-	ring             *ring.Ring                 // information on ranges owned by all peers
-	space            space.Space                // more detail on ranges owned by us
-	owned            map[string]address.Address // who owns what address, indexed by container-ID
-	nicknames        map[router.PeerName]string // so we can map nicknames for rmpeer
-	pendingAllocates []operation                // held until we get some free space
-	pendingClaims    []operation                // held until we know who owns the space
-	gossip           router.Gossip              // our link to the outside world for sending messages
+	universe         address.Range                // superset of all ranges
+	ring             *ring.Ring                   // information on ranges owned by all peers
+	space            space.Space                  // more detail on ranges owned by us
+	owned            map[string][]address.Address // who owns what addresses, indexed by container-ID
+	nicknames        map[router.PeerName]string   // so we can map nicknames for rmpeer
+	pendingAllocates []operation                  // held until we get some free space
+	pendingClaims    []operation                  // held until we know who owns the space
+	gossip           router.Gossip                // our link to the outside world for sending messages
 	paxos            *paxos.Node
 	paxosTicker      *time.Ticker
 	shuttingDown     bool // to avoid doing any requests while trying to shut down
@@ -63,34 +60,16 @@ type Allocator struct {
 }
 
 // NewAllocator creates and initialises a new Allocator
-func NewAllocator(ourName router.PeerName, ourUID router.PeerUID, ourNickname string, subnetCIDR string, quorum uint) (*Allocator, error) {
-	_, subnet, err := net.ParseCIDR(subnetCIDR)
-	if err != nil {
-		return nil, err
-	}
-	if subnet.IP.To4() == nil {
-		return nil, errors.New("Non-IPv4 address not supported")
-	}
-	// Get the size of the network from the mask
-	ones, bits := subnet.Mask.Size()
-	var subnetSize address.Offset = 1 << uint(bits-ones)
-	if subnetSize < 4 {
-		return nil, errors.New("Allocation subnet too small")
-	}
-	subnetStart := address.FromIP4(subnet.IP)
-	alloc := &Allocator{
-		ourName:     ourName,
-		subnetStart: subnetStart,
-		subnetSize:  subnetSize,
-		prefixLen:   ones,
-		// per RFC 1122, don't allocate the first and last address in the subnet
-		ring:      ring.New(address.Add(subnetStart, 1), address.Add(subnetStart, subnetSize-1), ourName),
-		owned:     make(map[string]address.Address),
+func NewAllocator(ourName router.PeerName, ourUID router.PeerUID, ourNickname string, universe address.Range, quorum uint) *Allocator {
+	return &Allocator{
+		ourName:   ourName,
+		universe:  universe,
+		ring:      ring.New(universe.Start, address.Add(universe.Start, universe.Size()), ourName),
+		owned:     make(map[string][]address.Address),
 		paxos:     paxos.NewNode(ourName, ourUID, quorum),
 		nicknames: map[router.PeerName]string{ourName: ourNickname},
 		now:       time.Now,
 	}
-	return alloc, nil
 }
 
 // Start runs the allocator goroutine
@@ -197,26 +176,26 @@ func hasBeenCancelled(cancelChan <-chan bool) func() bool {
 
 // Actor client API
 
-// Allocate (Sync) - get IP address for container with given name
-// if there isn't any space we block indefinitely
-func (alloc *Allocator) Allocate(ident string, cancelChan <-chan bool) (address.Address, error) {
+// Allocate (Sync) - get new IP address for container with given name in range
+// if there isn't any space in that range we block indefinitely
+func (alloc *Allocator) Allocate(ident string, r address.Range, cancelChan <-chan bool) (address.Address, error) {
 	resultChan := make(chan allocateResult)
-	op := &allocate{resultChan: resultChan, ident: ident,
+	op := &allocate{resultChan: resultChan, ident: ident, r: r,
 		hasBeenCancelled: hasBeenCancelled(cancelChan)}
 	alloc.doOperation(op, &alloc.pendingAllocates)
 	result := <-resultChan
 	return result.addr, result.err
 }
 
-func (alloc *Allocator) Lookup(ident string) (address.Address, error) {
+// Lookup (Sync) - get existing IP address for container with given name in range
+func (alloc *Allocator) Lookup(ident string, r address.Range) (address.Address, error) {
 	resultChan := make(chan allocateResult)
 	alloc.actionChan <- func() {
-		addr, found := alloc.owned[ident]
-		if found {
+		if addr, found := alloc.lookupOwned(ident, r); found {
 			resultChan <- allocateResult{addr: addr}
 			return
 		}
-		resultChan <- allocateResult{err: fmt.Errorf("lookup: no address found")}
+		resultChan <- allocateResult{err: fmt.Errorf("lookup: no address found for %s in range %s", ident, r)}
 	}
 	result := <-resultChan
 	return result.addr, result.err
@@ -231,22 +210,21 @@ func (alloc *Allocator) Claim(ident string, addr address.Address, cancelChan <-c
 	return <-resultChan
 }
 
-// Free (Sync) - release IP address for container with given name
-func (alloc *Allocator) Free(ident string) error {
-	return alloc.free(ident)
-}
-
-// ContainerDied is provided to satisfy the updater interface; does a free underneath.  Async.
+// ContainerDied is provided to satisfy the updater interface; does a 'Delete' underneath.  Sync.
 func (alloc *Allocator) ContainerDied(ident string) error {
-	alloc.debugln("Container", ident, "died; releasing addresses")
-	return alloc.free(ident)
+	err := alloc.Delete(ident)
+	if err == nil {
+		alloc.debugln("Container", ident, "died; released addresses")
+	}
+	return err
 }
 
-func (alloc *Allocator) free(ident string) error {
+// Delete (Sync) - release all IP addresses for container with given name
+func (alloc *Allocator) Delete(ident string) error {
 	errChan := make(chan error)
 	alloc.actionChan <- func() {
-		addr, found := alloc.owned[ident]
-		if found {
+		addrs, found := alloc.owned[ident]
+		for _, addr := range addrs {
 			alloc.space.Free(addr)
 		}
 		delete(alloc.owned, ident)
@@ -256,7 +234,7 @@ func (alloc *Allocator) free(ident string) error {
 		found = alloc.cancelOpsFor(&alloc.pendingClaims, ident) || found
 
 		if !found {
-			errChan <- fmt.Errorf("Free: no addresses for %s", ident)
+			errChan <- fmt.Errorf("Delete: no addresses for %s", ident)
 			return
 		}
 		errChan <- nil
@@ -264,6 +242,30 @@ func (alloc *Allocator) free(ident string) error {
 	return <-errChan
 }
 
+// Free (Sync) - release single IP address for container
+func (alloc *Allocator) Free(ident string, addrToFree address.Address) error {
+	errChan := make(chan error)
+	alloc.actionChan <- func() {
+		addrs := alloc.owned[ident]
+		for i, ownedAddr := range addrs {
+			if ownedAddr == addrToFree {
+				alloc.debugln("Freed", addrToFree, "for", ident)
+				if len(addrs) == 1 {
+					delete(alloc.owned, ident)
+				} else {
+					alloc.owned[ident] = append(addrs[:i], addrs[i+1:]...)
+				}
+				alloc.space.Free(addrToFree)
+				errChan <- nil
+				return
+			}
+		}
+
+		errChan <- fmt.Errorf("Free: address %s not found for %s", addrToFree, ident)
+	}
+	return <-errChan
+}
+
 // Sync.
 func (alloc *Allocator) String() string {
 	resultChan := make(chan string)
@@ -349,7 +351,13 @@ func (alloc *Allocator) OnGossipUnicast(sender router.PeerName, msg []byte) erro
 		switch msg[0] {
 		case msgSpaceRequest:
 			// some other peer asked us for space
-			alloc.donateSpace(sender)
+			decoder := gob.NewDecoder(bytes.NewReader(msg[1:]))
+			var r address.Range
+			if err := decoder.Decode(&r); err != nil {
+				resultChan <- err
+				return
+			}
+			alloc.donateSpace(r, sender)
 			resultChan <- nil
 		case msgRingUpdate:
 			resultChan <- alloc.update(msg[1:])
@@ -470,20 +478,14 @@ func (alloc *Allocator) actorLoop(actionChan <-chan func()) {
 
 func (alloc *Allocator) string() string {
 	var buf bytes.Buffer
-	fmt.Fprintf(&buf, "Allocator subnet %s/%d", alloc.subnetStart.String(), alloc.prefixLen)
+	fmt.Fprintf(&buf, "Allocator range %s", alloc.universe)
 
 	if alloc.ring.Empty() {
 		if alloc.paxosTicker != nil {
 			fmt.Fprintf(&buf, " awaiting consensus: %s", alloc.paxos.String())
 		}
 	} else {
-		localFreeSpace := alloc.space.NumFreeAddresses()
-		remoteFreeSpace := alloc.ring.TotalRemoteFree()
-		percentFree := 100 * float64(localFreeSpace+remoteFreeSpace) / float64(alloc.subnetSize)
-		fmt.Fprintf(&buf, "  Free IPs: ~%.1f%%, %d local, ~%d remote\n",
-			percentFree, localFreeSpace, remoteFreeSpace)
-
-		fmt.Fprint(&buf, "Owned Ranges:")
+		fmt.Fprint(&buf, "\nOwned Ranges:")
 		alloc.ring.FprintWithNicknames(&buf, alloc.nicknames)
 	}
 	if len(alloc.pendingAllocates)+len(alloc.pendingClaims) > 0 {
@@ -573,8 +575,18 @@ func (alloc *Allocator) propose() {
 	alloc.gossip.GossipBroadcast(alloc.Gossip())
 }
 
-func (alloc *Allocator) sendRequest(dest router.PeerName, kind byte) {
-	msg := router.Concat([]byte{kind}, alloc.encode())
+func (alloc *Allocator) sendSpaceRequest(dest router.PeerName, r address.Range) {
+	buf := new(bytes.Buffer)
+	enc := gob.NewEncoder(buf)
+	if err := enc.Encode(r); err != nil {
+		panic(err)
+	}
+	msg := router.Concat([]byte{msgSpaceRequest}, buf.Bytes())
+	alloc.gossip.GossipUnicast(dest, msg)
+}
+
+func (alloc *Allocator) sendRingUpdate(dest router.PeerName) {
+	msg := router.Concat([]byte{msgRingUpdate}, alloc.encode())
 	alloc.gossip.GossipUnicast(dest, msg)
 }
 
@@ -626,25 +638,24 @@ func (alloc *Allocator) update(msg []byte) error {
 	return nil
 }
 
-func (alloc *Allocator) donateSpace(to router.PeerName) {
+func (alloc *Allocator) donateSpace(r address.Range, to router.PeerName) {
 	// No matter what we do, we'll send a unicast gossip
 	// of our ring back to tha chap who asked for space.
 	// This serves to both tell him of any space we might
 	// have given him, or tell him where he might find some
 	// more.
-	defer alloc.sendRequest(to, msgRingUpdate)
+	defer alloc.sendRingUpdate(to)
 
 	alloc.debugln("Peer", to, "asked me for space")
-	start, size, ok := alloc.space.Donate()
+	chunk, ok := alloc.space.Donate(r)
 	if !ok {
-		free := alloc.space.NumFreeAddresses()
+		free := alloc.space.NumFreeAddressesInRange(r)
 		common.Assert(free == 0)
 		alloc.debugln("No space to give to peer", to)
 		return
 	}
-	end := address.Add(start, size)
-	alloc.debugln("Giving range", start, end, size, "to", to)
-	alloc.ring.GrantRangeToHost(start, end, to)
+	alloc.debugln("Giving range", chunk, "to", to)
+	alloc.ring.GrantRangeToHost(chunk.Start, chunk.End, to)
 }
 
 func (alloc *Allocator) assertInvariants() {
@@ -672,21 +683,33 @@ func (alloc *Allocator) reportFreeSpace() {
 
 	freespace := make(map[address.Address]address.Offset)
 	for _, r := range ranges {
-		freespace[r.Start] = alloc.space.NumFreeAddressesInRange(r.Start, r.End)
+		freespace[r.Start] = alloc.space.NumFreeAddressesInRange(r)
 	}
 	alloc.ring.ReportFree(freespace)
 }
 
 // Owned addresses
 
+// NB: addr must not be owned by ident already
 func (alloc *Allocator) addOwned(ident string, addr address.Address) {
-	alloc.owned[ident] = addr
+	alloc.owned[ident] = append(alloc.owned[ident], addr)
+}
+
+func (alloc *Allocator) lookupOwned(ident string, r address.Range) (address.Address, bool) {
+	for _, addr := range alloc.owned[ident] {
+		if r.Contains(addr) {
+			return addr, true
+		}
+	}
+	return 0, false
 }
 
 func (alloc *Allocator) findOwner(addr address.Address) string {
-	for ident, candidate := range alloc.owned {
-		if candidate == addr {
-			return ident
+	for ident, addrs := range alloc.owned {
+		for _, candidate := range addrs {
+			if candidate == addr {
+				return ident
+			}
 		}
 	}
 	return ""

ipam/allocator_test.go

@@ -24,37 +24,50 @@ func TestAllocFree(t *testing.T) {
 		container1 = "abcdef"
 		container2 = "baddf00d"
 		container3 = "b01df00d"
-		universe   = "10.0.3.0/28"
-		testAddr1  = "10.0.3.1" // first address allocated should be .1 because .0 is network addr
-		spaceSize  = 14         // 16 IP addresses in /28, minus .0 and .15
+		universe   = "10.0.3.0/26"
+		subnet1    = "10.0.3.0/28"
+		subnet2    = "10.0.3.32/28"
+		testAddr1  = "10.0.3.1"
+		testAddr2  = "10.0.3.33"
+		spaceSize  = 62 // 64 IP addresses in /26, minus .0 and .63
 	)
 
-	alloc := makeAllocatorWithMockGossip(t, "01:00:00:01:00:00", universe, 1)
+	alloc, subnet := makeAllocatorWithMockGossip(t, "01:00:00:01:00:00", universe, 1)
 	defer alloc.Stop()
+	_, cidr1, _ := address.ParseCIDR(subnet1)
+	_, cidr2, _ := address.ParseCIDR(subnet2)
 
 	alloc.claimRingForTesting()
-	addr1, _ := alloc.Allocate(container1, nil)
+	addr1, err := alloc.Allocate(container1, cidr1.HostRange(), nil)
+	wt.AssertNoErr(t, err)
 	wt.AssertEqualString(t, addr1.String(), testAddr1, "address")
 
+	addr2, err := alloc.Allocate(container1, cidr2.HostRange(), nil)
+	wt.AssertNoErr(t, err)
+	wt.AssertEqualString(t, addr2.String(), testAddr2, "address")
+
 	// Ask for another address for a different container and check it's different
-	addr2, _ := alloc.Allocate(container2, nil)
-	if addr2.String() == testAddr1 {
-		t.Fatalf("Expected different address but got %s", addr2.String())
+	addr1b, _ := alloc.Allocate(container2, cidr1.HostRange(), nil)
+	if addr1b.String() == testAddr1 {
+		t.Fatalf("Expected different address but got %s", addr1b.String())
 	}
 
-	// Ask for the first container again and we should get the same address again
-	addr1a, _ := alloc.Allocate(container1, nil)
+	// Ask for the first container again and we should get the same addresses again
+	addr1a, _ := alloc.Allocate(container1, cidr1.HostRange(), nil)
 	wt.AssertEqualString(t, addr1a.String(), testAddr1, "address")
+	addr2a, _ := alloc.Allocate(container1, cidr2.HostRange(), nil)
+	wt.AssertEqualString(t, addr2a.String(), testAddr2, "address")
 
-	// Now free the first one, and we should get it back when we ask
-	wt.AssertSuccess(t, alloc.Free(container1))
-	addr3, _ := alloc.Allocate(container3, nil)
+	// Now delete the first container, and we should get its addresses back
+	wt.AssertSuccess(t, alloc.Delete(container1))
+	addr3, _ := alloc.Allocate(container3, cidr1.HostRange(), nil)
 	wt.AssertEqualString(t, addr3.String(), testAddr1, "address")
+	addr4, _ := alloc.Allocate(container3, cidr2.HostRange(), nil)
+	wt.AssertEqualString(t, addr4.String(), testAddr2, "address")
 
 	alloc.ContainerDied(container2)
 	alloc.ContainerDied(container3)
-	alloc.String() // force sync-up after async call
-	wt.AssertEquals(t, alloc.NumFreeAddresses(), address.Offset(spaceSize))
+	wt.AssertEquals(t, alloc.NumFreeAddresses(subnet), address.Offset(spaceSize))
 }
 
 func TestBootstrap(t *testing.T) {
@@ -66,11 +79,11 @@ func TestBootstrap(t *testing.T) {
 		peerNameString = "02:00:00:02:00:00"
 	)
 
-	alloc1 := makeAllocatorWithMockGossip(t, ourNameString, testStart1+"/22", 2)
+	alloc1, subnet := makeAllocatorWithMockGossip(t, ourNameString, testStart1+"/22", 2)
 	defer alloc1.Stop()
 
 	// Simulate another peer on the gossip network
-	alloc2 := makeAllocatorWithMockGossip(t, peerNameString, testStart1+"/22", 2)
+	alloc2, _ := makeAllocatorWithMockGossip(t, peerNameString, testStart1+"/22", 2)
 	defer alloc2.Stop()
 
 	alloc1.OnGossipBroadcast(alloc2.Encode())
@@ -80,7 +93,7 @@ func TestBootstrap(t *testing.T) {
 	ExpectBroadcastMessage(alloc1, nil) // alloc1 will try to form consensus
 	done := make(chan bool)
 	go func() {
-		alloc1.Allocate("somecontainer", nil)
+		alloc1.Allocate("somecontainer", subnet, nil)
 		done <- true
 	}()
 	time.Sleep(100 * time.Millisecond)
@@ -124,19 +137,16 @@ func TestAllocatorClaim(t *testing.T) {
 		testAddr1  = "10.0.3.1" // first address allocated should be .1 because .0 is network addr
 	)
 
-	alloc := makeAllocatorWithMockGossip(t, "01:00:00:01:00:00", universe, 1)
+	alloc, subnet := makeAllocatorWithMockGossip(t, "01:00:00:01:00:00", universe, 1)
 	defer alloc.Stop()
 
 	alloc.claimRingForTesting()
-	addr1, _ := alloc.Allocate(container1, nil)
-	alloc.Allocate(container2, nil)
+	addr1, _ := address.ParseIP(testAddr1)
 
-	// Now free the first one, and try to claim it
-	wt.AssertSuccess(t, alloc.Free(container1))
-	t.Log(alloc)
 	err := alloc.Claim(container3, addr1, nil)
 	wt.AssertNoErr(t, err)
-	addr3, _ := alloc.Allocate(container3, nil)
+	// Check we get this address back if we try an allocate
+	addr3, _ := alloc.Allocate(container3, subnet, nil)
 	wt.AssertEqualString(t, addr3.String(), testAddr1, "address")
 }
 
@@ -160,10 +170,10 @@ func TestCancel(t *testing.T) {
 
 	router := TestGossipRouter{make(map[router.PeerName]chan gossipMessage), 0.0}
 
-	alloc1 := makeAllocator("01:00:00:02:00:00", CIDR, 2)
+	alloc1, subnet := makeAllocator("01:00:00:02:00:00", CIDR, 2)
 	alloc1.SetInterfaces(router.connect(alloc1.ourName, alloc1))
 
-	alloc2 := makeAllocator("02:00:00:02:00:00", CIDR, 2)
+	alloc2, _ := makeAllocator("02:00:00:02:00:00", CIDR, 2)
 	alloc2.SetInterfaces(router.connect(alloc2.ourName, alloc2))
 	alloc1.claimRingForTesting(alloc1, alloc2)
 	alloc2.claimRingForTesting(alloc1, alloc2)
@@ -175,9 +185,9 @@ func TestCancel(t *testing.T) {
 	alloc1.OnGossipBroadcast(alloc2.Encode())
 
 	// Get some IPs, so each allocator has some space
-	res1, _ := alloc1.Allocate("foo", nil)
+	res1, _ := alloc1.Allocate("foo", subnet, nil)
 	common.Debug.Printf("res1 = %s", res1.String())
-	res2, _ := alloc2.Allocate("bar", nil)
+	res2, _ := alloc2.Allocate("bar", subnet, nil)
 	common.Debug.Printf("res2 = %s", res2.String())
 	if res1 == res2 {
 		wt.Fatalf(t, "Error: got same ips!")
@@ -188,13 +198,13 @@ func TestCancel(t *testing.T) {
 	unpause := alloc2.pause()
 
 	// Use up all the IPs that alloc1 owns, so the allocation after this will prompt a request to alloc2
-	for i := 0; alloc1.NumFreeAddresses() > 0; i++ {
-		alloc1.Allocate(fmt.Sprintf("tmp%d", i), nil)
+	for i := 0; alloc1.NumFreeAddresses(subnet) > 0; i++ {
+		alloc1.Allocate(fmt.Sprintf("tmp%d", i), subnet, nil)
 	}
 	cancelChan := make(chan bool, 1)
 	doneChan := make(chan bool)
 	go func() {
-		_, ok := alloc1.Allocate("baz", cancelChan)
+		_, ok := alloc1.Allocate("baz", subnet, cancelChan)
 		doneChan <- ok == nil
 	}()
 
@@ -214,19 +224,17 @@ func TestGossipShutdown(t *testing.T) {
 		container1 = "abcdef"
 		container2 = "baddf00d"
 		universe   = "10.0.3.0/30"
-		testAddr1  = "10.0.3.1" // first address allocated should be .1 because .0 is network addr
 	)
 
-	alloc := makeAllocatorWithMockGossip(t, "01:00:00:01:00:00", universe, 1)
+	alloc, subnet := makeAllocatorWithMockGossip(t, "01:00:00:01:00:00", universe, 1)
 	defer alloc.Stop()
 
 	alloc.claimRingForTesting()
-	addr1, _ := alloc.Allocate(container1, nil)
-	wt.AssertEqualString(t, addr1.String(), testAddr1, "address")
+	alloc.Allocate(container1, subnet, nil)
 
 	alloc.Shutdown()
 
-	_, err := alloc.Allocate(container2, nil) // trying to allocate after shutdown should fail
+	_, err := alloc.Allocate(container2, subnet, nil) // trying to allocate after shutdown should fail
 	wt.AssertFalse(t, err == nil, "no address")
 
 	CheckAllExpectedMessagesSent(alloc)
@@ -236,15 +244,15 @@ func TestTransfer(t *testing.T) {
 	const (
 		cidr = "10.0.1.7/22"
 	)
-	allocs, router := makeNetworkOfAllocators(3, cidr)
+	allocs, router, subnet := makeNetworkOfAllocators(3, cidr)
 	alloc1 := allocs[0]
 	alloc2 := allocs[1]
 	alloc3 := allocs[2] // This will be 'master' and get the first range
 
-	_, err := alloc2.Allocate("foo", nil)
+	_, err := alloc2.Allocate("foo", subnet, nil)
 	wt.AssertTrue(t, err == nil, "Failed to get address")
 
-	_, err = alloc3.Allocate("bar", nil)
+	_, err = alloc3.Allocate("bar", subnet, nil)
 	wt.AssertTrue(t, err == nil, "Failed to get address")
 
 	router.GossipBroadcast(alloc2.Gossip())
@@ -256,9 +264,9 @@ func TestTransfer(t *testing.T) {
 	wt.AssertSuccess(t, alloc1.AdminTakeoverRanges(alloc2.ourName.String()))
 	wt.AssertSuccess(t, alloc1.AdminTakeoverRanges(alloc3.ourName.String()))
 
-	wt.AssertEquals(t, alloc1.NumFreeAddresses(), address.Offset(1022))
+	wt.AssertEquals(t, alloc1.NumFreeAddresses(subnet), address.Offset(1022))
 
-	_, err = alloc1.Allocate("foo", nil)
+	_, err = alloc1.Allocate("foo", subnet, nil)
 	wt.AssertTrue(t, err == nil, "Failed to get address")
 	alloc1.Stop()
 }
@@ -268,14 +276,13 @@ func TestFakeRouterSimple(t *testing.T) {
 	const (
 		cidr = "10.0.1.7/22"
 	)
-	allocs, _ := makeNetworkOfAllocators(2, cidr)
+	allocs, _, subnet := makeNetworkOfAllocators(2, cidr)
 	defer stopNetworkOfAllocators(allocs)
 
 	alloc1 := allocs[0]
 	//alloc2 := allocs[1]
 
-	addr, _ := alloc1.Allocate("foo", nil)
-	println("Got addr", addr)
+	alloc1.Allocate("foo", subnet, nil)
 }
 
 func TestAllocatorFuzz(t *testing.T) {
@@ -288,7 +295,7 @@ func TestAllocatorFuzz(t *testing.T) {
 		concurrency  = 10
 		cidr         = "10.0.1.7/22"
 	)
-	allocs, _ := makeNetworkOfAllocators(nodes, cidr)
+	allocs, _, subnet := makeNetworkOfAllocators(nodes, cidr)
 	defer stopNetworkOfAllocators(allocs)
 
 	// Test state
@@ -332,7 +339,7 @@ func TestAllocatorFuzz(t *testing.T) {
 		allocIndex := rand.Int31n(nodes)
 		alloc := allocs[allocIndex]
 		//common.Info.Printf("Allocate: asking allocator %d", allocIndex)
-		addr, err := alloc.Allocate(name, nil)
+		addr, err := alloc.Allocate(name, subnet, nil)
 
 		if err != nil {
 			panic(fmt.Sprintf("Could not allocate addr"))
@@ -377,7 +384,11 @@ func TestAllocatorFuzz(t *testing.T) {
 		alloc := allocs[res.alloc]
 		//common.Info.Printf("Freeing %s (%s) on allocator %d", res.name, addr, res.alloc)
 
-		wt.AssertSuccess(t, alloc.Free(res.name))
+		oldAddr, err := address.ParseIP(addr)
+		if err != nil {
+			panic(err)
+		}
+		wt.AssertSuccess(t, alloc.Free(res.name, oldAddr))
 	}
 
 	// Do a Allocate on an existing container & allocator
@@ -394,7 +405,7 @@ func TestAllocatorFuzz(t *testing.T) {
 
 		//common.Info.Printf("Asking for %s on allocator %d again", addr, res.alloc)
 
-		newAddr, _ := alloc.Allocate(res.name, nil)
+		newAddr, _ := alloc.Allocate(res.name, subnet, nil)
 		oldAddr, _ := address.ParseIP(addr)
 		if newAddr != oldAddr {
 			panic(fmt.Sprintf("Got different address for repeat request for %s: %s != %s", res.name, newAddr, oldAddr))
@@ -453,9 +464,9 @@ func TestAllocatorFuzz(t *testing.T) {
 }
 
 func TestGossipSkew(t *testing.T) {
-	alloc1 := makeAllocatorWithMockGossip(t, "01:00:00:01:00:00", "10.0.1.0/22", 2)
+	alloc1, _ := makeAllocatorWithMockGossip(t, "01:00:00:01:00:00", "10.0.1.0/22", 2)
 	defer alloc1.Stop()
-	alloc2 := makeAllocatorWithMockGossip(t, "02:00:00:02:00:00", "10.0.1.0/22", 2)
+	alloc2, _ := makeAllocatorWithMockGossip(t, "02:00:00:02:00:00", "10.0.1.0/22", 2)
 	alloc2.now = func() time.Time { return time.Now().Add(time.Hour * 2) }
 	defer alloc2.Stop()
 

ipam/claim.go

@@ -9,9 +9,9 @@ import (
 
 type claim struct {
 	resultChan       chan<- error
-	hasBeenCancelled func() bool
 	ident            string
 	addr             address.Address
+	hasBeenCancelled func() bool
 }
 
 // Try returns true for success (or failure), false if we need to try again later
@@ -57,6 +57,7 @@ func (c *claim) Try(alloc *Allocator) bool {
 			c.resultChan <- err
 			return true
 		}
+		alloc.debugln("Claimed", c.addr, "for", c.ident)
 		alloc.addOwned(c.ident, c.addr)
 		c.resultChan <- nil
 		return true

ipam/http.go

@@ -16,7 +16,7 @@ func badRequest(w http.ResponseWriter, err error) {
 }
 
 // HandleHTTP wires up ipams HTTP endpoints to the provided mux.
-func (alloc *Allocator) HandleHTTP(router *mux.Router) {
+func (alloc *Allocator) HandleHTTP(router *mux.Router, defaultSubnet address.CIDR) {
 	router.Methods("PUT").Path("/ip/{id}/{ip}").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		closedChan := w.(http.CloseNotifier).CloseNotify()
 		vars := mux.Vars(r)
@@ -33,30 +33,84 @@ func (alloc *Allocator) HandleHTTP(router *mux.Router) {
 		w.WriteHeader(204)
 	})
 
-	router.Methods("GET").Path("/ip/{id}").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		addr, err := alloc.Lookup(mux.Vars(r)["id"])
+	router.Methods("GET").Path("/ip/{id}/{ip}/{prefixlen}").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		vars := mux.Vars(r)
+		cidr := vars["ip"] + "/" + vars["prefixlen"]
+		_, subnet, err := address.ParseCIDR(cidr)
+		if err != nil {
+			badRequest(w, err)
+			return
+		}
+		addr, err := alloc.Lookup(vars["id"], subnet.HostRange())
 		if err != nil {
 			http.NotFound(w, r)
 			return
 		}
-		fmt.Fprintf(w, "%s/%d", addr.String(), alloc.prefixLen)
+		fmt.Fprintf(w, "%s/%d", addr, subnet.PrefixLen)
+	})
+
+	router.Methods("GET").Path("/ip/{id}").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		addr, err := alloc.Lookup(mux.Vars(r)["id"], defaultSubnet.HostRange())
+		if err != nil {
+			http.NotFound(w, r)
+			return
+		}
+		fmt.Fprintf(w, "%s/%d", addr, defaultSubnet.PrefixLen)
+	})
+
+	router.Methods("POST").Path("/ip/{id}/{ip}/{prefixlen}").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		closedChan := w.(http.CloseNotifier).CloseNotify()
+		vars := mux.Vars(r)
+		ident := vars["id"]
+		cidrStr := vars["ip"] + "/" + vars["prefixlen"]
+		subnetAddr, cidr, err := address.ParseCIDR(cidrStr)
+		if err != nil {
+			badRequest(w, err)
+			return
+		}
+		if cidr.Start != subnetAddr {
+			badRequest(w, fmt.Errorf("Invalid subnet %s - bits after network prefix are not all zero", cidrStr))
+			return
+		}
+		addr, err := alloc.Allocate(ident, cidr.HostRange(), closedChan)
+		if err != nil {
+			badRequest(w, fmt.Errorf("Unable to allocate: %s", err))
+			return
+		}
+
+		fmt.Fprintf(w, "%s/%d", addr, cidr.PrefixLen)
 	})
 
 	router.Methods("POST").Path("/ip/{id}").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		closedChan := w.(http.CloseNotifier).CloseNotify()
 		ident := mux.Vars(r)["id"]
-		newAddr, err := alloc.Allocate(ident, closedChan)
+		newAddr, err := alloc.Allocate(ident, defaultSubnet.HostRange(), closedChan)
 		if err != nil {
 			badRequest(w, err)
 			return
 		}
 
-		fmt.Fprintf(w, "%s/%d", newAddr.String(), alloc.prefixLen)
+		fmt.Fprintf(w, "%s/%d", newAddr, defaultSubnet.PrefixLen)
+	})
+
+	router.Methods("DELETE").Path("/ip/{id}/{ip}").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		vars := mux.Vars(r)
+		ident := vars["id"]
+		ipStr := vars["ip"]
+		if ip, err := address.ParseIP(ipStr); err != nil {
+			badRequest(w, err)
+			return
+		} else if err := alloc.Free(ident, ip); err != nil {
+			badRequest(w, fmt.Errorf("Unable to free: %s", err))
+			return
+		}
+
+		w.WriteHeader(204)
 	})
 
 	router.Methods("DELETE").Path("/ip/{id}").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		ident := mux.Vars(r)["id"]
-		if err := alloc.Free(ident); err != nil {
+		if err := alloc.Delete(ident); err != nil {
 			badRequest(w, err)
 			return
 		}

ipam/http_test.go

@@ -13,6 +13,7 @@ import (
 	"github.com/gorilla/mux"
 
 	"github.com/weaveworks/weave/common"
+	"github.com/weaveworks/weave/ipam/address"
 	wt "github.com/weaveworks/weave/testing"
 )
 
@@ -38,12 +39,12 @@ func doHTTP(method string, url string) (resp *http.Response, err error) {
 	return http.DefaultClient.Do(req)
 }
 
-func listenHTTP(alloc *Allocator) int {
+func listenHTTP(alloc *Allocator, subnet address.CIDR) int {
 	router := mux.NewRouter()
 	router.HandleFunc("/status", func(w http.ResponseWriter, r *http.Request) {
 		io.WriteString(w, fmt.Sprintln(alloc))
 	})
-	alloc.HandleHTTP(router)
+	alloc.HandleHTTP(router, subnet)
 
 	httpListener, err := net.Listen("tcp", ":0")
 	if err != nil {
@@ -59,45 +60,55 @@ func listenHTTP(alloc *Allocator) int {
 	return httpListener.Addr().(*net.TCPAddr).Port
 }
 
-func allocURL(port int, containerID string) string {
+func identURL(port int, containerID string) string {
 	return fmt.Sprintf("http://localhost:%d/ip/%s", port, containerID)
 }
 
+func allocURL(port int, cidr string, containerID string) string {
+	return fmt.Sprintf("http://localhost:%d/ip/%s/%s", port, containerID, cidr)
+}
+
 func TestHttp(t *testing.T) {
 	var (
 		containerID = "deadbeef"
 		container2  = "baddf00d"
 		container3  = "b01df00d"
-		testAddr1   = "10.0.3.9"
-		netSuffix   = "/29"
-		testCIDR1   = "10.0.3.8" + netSuffix
+		universe    = "10.0.0.0/8"
+		testCIDR1   = "10.0.3.8/29"
+		testCIDR2   = "10.2.0.0/16"
+		testAddr1   = "10.0.3.9/29"
+		testAddr2   = "10.2.0.1/16"
 	)
 
-	alloc := makeAllocatorWithMockGossip(t, "08:00:27:01:c3:9a", testCIDR1, 1)
-	port := listenHTTP(alloc)
-
+	alloc, _ := makeAllocatorWithMockGossip(t, "08:00:27:01:c3:9a", universe, 1)
+	_, cidr, _ := address.ParseCIDR(universe)
+	port := listenHTTP(alloc, cidr)
 	alloc.claimRingForTesting()
-	// Ask the http server for a new address
-	cidr1 := HTTPPost(t, allocURL(port, containerID))
-	wt.AssertEqualString(t, cidr1, testAddr1+netSuffix, "address")
-	check := HTTPGet(t, allocURL(port, containerID))
-	wt.AssertEqualString(t, check, cidr1, "address")
 
-	// Ask the http server for another address and check it's different
-	cidr2 := HTTPPost(t, allocURL(port, container2))
-	wt.AssertFalse(t, cidr2 == testAddr1+netSuffix, "address")
+	// Allocate an address in each subnet, and check we got what we expected
+	cidr1a := HTTPPost(t, allocURL(port, testCIDR1, containerID))
+	wt.AssertEqualString(t, cidr1a, testAddr1, "address")
+	cidr2a := HTTPPost(t, allocURL(port, testCIDR2, containerID))
+	wt.AssertEqualString(t, cidr2a, testAddr2, "address")
+	// Now, make the same requests again to check the operation is idempotent
+	check := HTTPGet(t, allocURL(port, testCIDR1, containerID))
+	wt.AssertEqualString(t, check, cidr1a, "address")
+	check = HTTPGet(t, allocURL(port, testCIDR2, containerID))
+	wt.AssertEqualString(t, check, cidr2a, "address")
 
-	// Ask for the first container again and we should get the same address again
-	cidr1a := HTTPPost(t, allocURL(port, containerID))
-	wt.AssertEqualString(t, cidr1a, testAddr1+netSuffix, "address")
-	// Now free the first one, and we should get it back when we ask
-	doHTTP("DELETE", allocURL(port, containerID))
-	resp, err := doHTTP("GET", allocURL(port, containerID))
-	wt.AssertNoErr(t, err)
-	wt.AssertStatus(t, resp.StatusCode, http.StatusNotFound, "http response")
+	// Ask the http server for a pair of addresses for another container and check they're different
+	cidr1b := HTTPPost(t, allocURL(port, testCIDR1, container2))
+	wt.AssertFalse(t, cidr1b == testAddr1, "address")
+	cidr2b := HTTPPost(t, allocURL(port, testCIDR2, container2))
+	wt.AssertFalse(t, cidr2b == testAddr2, "address")
 
-	cidr3 := HTTPPost(t, allocURL(port, container3))
-	wt.AssertEqualString(t, cidr3, testAddr1+netSuffix, "address")
+	// Now free the first container, and we should get its addresses back when we ask
+	doHTTP("DELETE", identURL(port, containerID))
+
+	cidr1c := HTTPPost(t, allocURL(port, testCIDR1, container3))
+	wt.AssertEqualString(t, cidr1c, testAddr1, "address")
+	cidr2c := HTTPPost(t, allocURL(port, testCIDR2, container3))
+	wt.AssertEqualString(t, cidr2c, testAddr2, "address")
 
 	// Would like to shut down the http server at the end of this test
 	// but it's complicated.
@@ -110,12 +121,13 @@ func TestBadHttp(t *testing.T) {
 		testCIDR1   = "10.0.0.0/8"
 	)
 
-	alloc := makeAllocatorWithMockGossip(t, "08:00:27:01:c3:9a", testCIDR1, 1)
+	alloc, _ := makeAllocatorWithMockGossip(t, "08:00:27:01:c3:9a", testCIDR1, 1)
 	defer alloc.Stop()
-	port := listenHTTP(alloc)
+	_, cidr, _ := address.ParseCIDR(testCIDR1)
+	port := listenHTTP(alloc, cidr)
 
 	alloc.claimRingForTesting()
-	cidr1 := HTTPPost(t, allocURL(port, containerID))
+	cidr1 := HTTPPost(t, allocURL(port, testCIDR1, containerID))
 	parts := strings.Split(cidr1, "/")
 	testAddr1 := parts[0]
 	// Verb that's not handled
@@ -142,20 +154,21 @@ func impTestHTTPCancel(t *testing.T) {
 	common.InitDefaultLogging(true)
 	var (
 		containerID = "deadbeef"
-		testCIDR1   = "10.0.3.5/29"
+		testCIDR1   = "10.0.3.0/29"
 	)
 
-	alloc := makeAllocatorWithMockGossip(t, "08:00:27:01:c3:9a", testCIDR1, 1)
+	alloc, _ := makeAllocatorWithMockGossip(t, "08:00:27:01:c3:9a", testCIDR1, 1)
 	defer alloc.Stop()
 	alloc.claimRingForTesting()
-	port := listenHTTP(alloc)
+	_, cidr, _ := address.ParseCIDR(testCIDR1)
+	port := listenHTTP(alloc, cidr)
 
 	// Stop the alloc so nothing actually works
 	unpause := alloc.pause()
 
 	// Ask the http server for a new address
 	done := make(chan *http.Response)
-	req, _ := http.NewRequest("POST", allocURL(port, containerID), nil)
+	req, _ := http.NewRequest("POST", allocURL(port, testCIDR1, containerID), nil)
 	go func() {
 		res, _ := http.DefaultClient.Do(req)
 		done <- res

ipam/ring/ring.go

@@ -91,18 +91,6 @@ func New(start, end address.Address, peer router.PeerName) *Ring {
 	return ring
 }
 
-// TotalRemoteFree returns the approximate number of free IPs
-// on other hosts.
-func (r *Ring) TotalRemoteFree() address.Offset {
-	result := address.Offset(0)
-	for _, entry := range r.Entries {
-		if entry.Peer != r.Peer {
-			result += entry.Free
-		}
-	}
-	return result
-}
-
 // Returns the distance between two tokens on this ring, dealing
 // with ranges which cross the origin
 func (r *Ring) distance(start, end address.Address) address.Offset {
@@ -341,10 +329,6 @@ func (r *Ring) ClaimForPeers(peers []router.PeerName) {
 	common.Assert(pos == r.End)
 }
 
-func (r *Ring) ClaimItAll() {
-	r.ClaimForPeers([]router.PeerName{r.Peer})
-}
-
 func (r *Ring) FprintWithNicknames(w io.Writer, m map[router.PeerName]string) {
 	for _, entry := range r.Entries {
 		nickname, found := m[entry.Peer]
@@ -409,15 +393,22 @@ func (r *Ring) ReportFree(freespace map[address.Address]address.Offset) {
 }
 
 // ChoosePeerToAskForSpace chooses a weighted-random peer to ask
-// for space.
-func (r *Ring) ChoosePeerToAskForSpace() (result router.PeerName, err error) {
+// for space in the range [start, end).  Assumes start<end.
+func (r *Ring) ChoosePeerToAskForSpace(start, end address.Address) (result router.PeerName, err error) {
 	var (
 		sum               address.Offset
 		totalSpacePerPeer = make(map[router.PeerName]address.Offset) // Compute total free space per peer
 	)
 
 	// iterate through tokens
-	for _, entry := range r.Entries {
+	for i, entry := range r.Entries {
+		// Ignore entries that don't span the range we want
+		if i+1 < len(r.Entries) && r.Entries.entry(i+1).Token-1 < start {
+			continue
+		}
+		if entry.Token >= end {
+			break
+		}
 		// Ignore ranges with no free space
 		if entry.Free <= 0 {
 			continue

ipam/ring/ring_test.go

@@ -345,34 +345,40 @@ func TestGossip(t *testing.T) {
 func TestFindFree(t *testing.T) {
 	ring1 := New(start, end, peer1name)
 
-	_, err := ring1.ChoosePeerToAskForSpace()
+	_, err := ring1.ChoosePeerToAskForSpace(start, end)
 	wt.AssertTrue(t, err == ErrNoFreeSpace, "Expected ErrNoFreeSpace")
 
 	ring1.Entries = []*entry{{Token: start, Peer: peer1name}}
-	_, err = ring1.ChoosePeerToAskForSpace()
+	_, err = ring1.ChoosePeerToAskForSpace(start, end)
 	wt.AssertTrue(t, err == ErrNoFreeSpace, "Expected ErrNoFreeSpace")
 
 	// We shouldn't return outselves
 	ring1.ReportFree(map[address.Address]address.Offset{start: 10})
-	_, err = ring1.ChoosePeerToAskForSpace()
+	_, err = ring1.ChoosePeerToAskForSpace(start, end)
 	wt.AssertTrue(t, err == ErrNoFreeSpace, "Expected ErrNoFreeSpace")
 
 	ring1.Entries = []*entry{{Token: start, Peer: peer1name, Free: 1},
-		{Token: start, Peer: peer1name, Free: 1}}
-	_, err = ring1.ChoosePeerToAskForSpace()
+		{Token: middle, Peer: peer1name, Free: 1}}
+	_, err = ring1.ChoosePeerToAskForSpace(start, end)
 	wt.AssertTrue(t, err == ErrNoFreeSpace, "Expected ErrNoFreeSpace")
+	ring1.assertInvariants()
 
 	// We should return others
 	ring1.Entries = []*entry{{Token: start, Peer: peer2name, Free: 1}}
-	peer, err := ring1.ChoosePeerToAskForSpace()
+	peer, err := ring1.ChoosePeerToAskForSpace(start, end)
 	wt.AssertSuccess(t, err)
 	wt.AssertEquals(t, peer, peer2name)
 
 	ring1.Entries = []*entry{{Token: start, Peer: peer2name, Free: 1},
-		{Token: start, Peer: peer2name, Free: 1}}
-	peer, err = ring1.ChoosePeerToAskForSpace()
+		{Token: middle, Peer: peer3name, Free: 1}}
+	peer, err = ring1.ChoosePeerToAskForSpace(start, middle)
 	wt.AssertSuccess(t, err)
 	wt.AssertEquals(t, peer, peer2name)
+
+	peer, err = ring1.ChoosePeerToAskForSpace(middle, end)
+	wt.AssertSuccess(t, err)
+	wt.AssertEquals(t, peer, peer3name)
+	ring1.assertInvariants()
 }
 
 func TestReportFree(t *testing.T) {
@@ -710,6 +716,10 @@ func TestFuzzRingHard(t *testing.T) {
 	}
 }
 
+func (r *Ring) ClaimItAll() {
+	r.ClaimForPeers([]router.PeerName{r.Peer})
+}
+
 func (es entries) String() string {
 	var buffer bytes.Buffer
 	fmt.Fprintf(&buffer, "[")

ipam/space/space.go

@@ -9,8 +9,6 @@ import (
 	"github.com/weaveworks/weave/ipam/address"
 )
 
-type Addr address.Address
-
 type Space struct {
 	// ours and free represent a set of addresses as a sorted
 	// sequences of ranges.  Even elements give the inclusive
@@ -37,14 +35,47 @@ func (s *Space) Clear() {
 	s.ours = s.ours[:0]
 }
 
-func (s *Space) Allocate() (bool, address.Address) {
-	if len(s.free) == 0 {
-		return false, 0
+// Walk down the free list calling f() on the in-range portions, until
+// f() returns true or we run out of free space.  Return true iff f() returned true
+func (s *Space) walkFree(r address.Range, f func(address.Range) bool) bool {
+	if r.Start >= r.End { // degenerate case
+		return false
 	}
-	res := s.free[0]
-	s.ours = add(s.ours, res, res+1)
-	s.free = subtract(s.free, res, res+1)
-	return true, res
+	for i := 0; i < len(s.free); i += 2 {
+		chunk := address.Range{Start: s.free[i], End: s.free[i+1]}
+		if chunk.End <= r.Start { // this chunk comes before the range
+			continue
+		}
+		if chunk.Start >= r.End {
+			// all remaining free space is completely after range
+			break
+		}
+		// at this point we know chunk.End>chunk.Start &&
+		// chunk.End>r.Start && r.End>chunk.Start && r.End>r.Start
+		// therefore max(start, r.Start) < min(end, r.End)
+		// Restrict this block of free space to be in range
+		if chunk.Start < r.Start {
+			chunk.Start = r.Start
+		}
+		if chunk.End > r.End {
+			chunk.End = r.End
+		}
+		// at this point we know start<end
+		if f(chunk) {
+			return true
+		}
+	}
+	return false
+}
+
+func (s *Space) Allocate(r address.Range) (bool, address.Address) {
+	var result address.Address
+	return s.walkFree(r, func(chunk address.Range) bool {
+		result = chunk.Start
+		s.ours = add(s.ours, result, result+1)
+		s.free = subtract(s.free, result, result+1)
+		return true
+	}), result
 }
 
 func (s *Space) Claim(addr address.Address) error {
@@ -57,29 +88,12 @@ func (s *Space) Claim(addr address.Address) error {
 	return nil
 }
 
-func (s *Space) NumFreeAddresses() address.Offset {
+func (s *Space) NumFreeAddressesInRange(r address.Range) address.Offset {
 	res := address.Offset(0)
-	for i := 0; i < len(s.free); i += 2 {
-		res += address.Subtract(s.free[i+1], s.free[i])
-	}
-	return res
-}
-
-func (s *Space) NumFreeAddressesInRange(start, end address.Address) address.Offset {
-	res := address.Offset(0)
-	for i := 0; i < len(s.free); i += 2 {
-		s, e := s.free[i], s.free[i+1]
-		if s < start {
-			s = start
-		}
-		if e > end {
-			e = end
-		}
-		if s >= e {
-			continue
-		}
-		res += address.Subtract(e, s)
-	}
+	s.walkFree(r, func(chunk address.Range) bool {
+		res += chunk.Size()
+		return false
+	})
 	return res
 }
 
@@ -96,35 +110,32 @@ func (s *Space) Free(addr address.Address) error {
 	return nil
 }
 
-func (s *Space) biggestFreeRange() (int, address.Offset) {
-	pos := -1
-	biggest := address.Offset(0)
-
-	for i := 0; i < len(s.free); i += 2 {
-		size := address.Subtract(s.free[i+1], s.free[i])
-		if size >= biggest {
-			pos = i
-			biggest = size
+func (s *Space) biggestFreeRange(r address.Range) (biggest address.Range) {
+	biggestSize := address.Offset(0)
+	s.walkFree(r, func(chunk address.Range) bool {
+		if size := chunk.Size(); size >= biggestSize {
+			biggest = chunk
+			biggestSize = size
 		}
-	}
-	return pos, biggest
+		return false
+	})
+	return
 }
 
-func (s *Space) Donate() (address.Address, address.Offset, bool) {
-	if len(s.free) == 0 {
-		return 0, 0, false
+func (s *Space) Donate(r address.Range) (address.Range, bool) {
+	biggest := s.biggestFreeRange(r)
+
+	if biggest.Size() == 0 {
+		return address.Range{}, false
 	}
 
-	pos, biggest := s.biggestFreeRange()
+	// Donate half of that biggest free range. Note size/2 rounds down, so
+	// the resulting donation size rounds up, and in particular can't be empty.
+	biggest.Start = address.Add(biggest.Start, biggest.Size()/2)
 
-	// Donate half of that biggest free range, rounding up so
-	// that the donation can't be empty
-	end := s.free[pos+1]
-	start := end - address.Address((biggest+1)/2)
-
-	s.ours = subtract(s.ours, start, end)
-	s.free = subtract(s.free, start, end)
-	return start, address.Subtract(end, start), true
+	s.ours = subtract(s.ours, biggest.Start, biggest.End)
+	s.free = subtract(s.free, biggest.Start, biggest.End)
+	return biggest, true
 }
 
 func firstGreater(a []address.Address, x address.Address) int {

ipam/space/space_test.go

@@ -1,7 +1,6 @@
 package space
 
 import (
-	"reflect"
 	"testing"
 
 	"github.com/weaveworks/weave/ipam/address"
@@ -19,6 +18,16 @@ func ip(s string) address.Address {
 	return addr
 }
 
+// Helper function to avoid 'NumFreeAddressesInRange(start, end)'
+// dozens of times in tests
+func (s *Space) NumFreeAddresses() address.Offset {
+	res := address.Offset(0)
+	for i := 0; i < len(s.free); i += 2 {
+		res += address.Subtract(s.free[i+1], s.free[i])
+	}
+	return res
+}
+
 func TestLowlevel(t *testing.T) {
 	a := []address.Address{}
 	a = add(a, 100, 200)
@@ -52,12 +61,12 @@ func TestLowlevel(t *testing.T) {
 
 	s := New()
 	wt.AssertEquals(t, s.NumFreeAddresses(), address.Offset(0))
-	ok, got := s.Allocate()
+	ok, got := s.Allocate(address.NewRange(0, 1000))
 	wt.AssertFalse(t, ok, "allocate in empty space should fail")
 
 	s.Add(100, 100)
 	wt.AssertEquals(t, s.NumFreeAddresses(), address.Offset(100))
-	ok, got = s.Allocate()
+	ok, got = s.Allocate(address.NewRange(0, 1000))
 	wt.AssertTrue(t, ok && got == 100, "allocate")
 	wt.AssertEquals(t, s.NumFreeAddresses(), address.Offset(99))
 	wt.AssertNoErr(t, s.Claim(150))
@@ -67,44 +76,49 @@ func TestLowlevel(t *testing.T) {
 	wt.AssertErrorInterface(t, s.Free(0), (*error)(nil), "free not allocated")
 	wt.AssertErrorInterface(t, s.Free(100), (*error)(nil), "double free")
 
-	start, size, ok := s.Donate()
-	wt.AssertTrue(t, ok && start == 125 && size == 25, "donate")
+	r, ok := s.Donate(address.NewRange(0, 1000))
+	wt.AssertTrue(t, ok && r.Start == 125 && r.Size() == 25, "donate")
 
 	// test Donate when addresses are scarce
 	s = New()
-	start, size, ok = s.Donate()
+	r, ok = s.Donate(address.NewRange(0, 1000))
 	wt.AssertTrue(t, !ok, "donate on empty space should fail")
 	s.Add(0, 3)
 	wt.AssertNoErr(t, s.Claim(0))
 	wt.AssertNoErr(t, s.Claim(2))
-	start, size, ok = s.Donate()
-	wt.AssertTrue(t, ok && start == 1 && size == 1, "donate")
-	start, size, ok = s.Donate()
+	r, ok = s.Donate(address.NewRange(0, 1000))
+	wt.AssertTrue(t, ok && r.Start == 1 && r.End == 2, "donate")
+	r, ok = s.Donate(address.NewRange(0, 1000))
 	wt.AssertTrue(t, !ok, "donate should fail")
 }
 
 func TestSpaceAllocate(t *testing.T) {
 	const (
 		testAddr1   = "10.0.3.4"
+		testAddr2   = "10.0.3.5"
 		testAddrx   = "10.0.3.19"
 		testAddry   = "10.0.9.19"
 		containerID = "deadbeef"
+		size        = 20
+	)
+	var (
+		start = ip(testAddr1)
 	)
 
-	space1 := makeSpace(ip(testAddr1), 20)
+	space1 := makeSpace(start, size)
 	wt.AssertEquals(t, space1.NumFreeAddresses(), address.Offset(20))
 	space1.assertInvariants()
 
-	_, addr1 := space1.Allocate()
+	_, addr1 := space1.Allocate(address.NewRange(start, size))
 	wt.AssertEqualString(t, addr1.String(), testAddr1, "address")
 	wt.AssertEquals(t, space1.NumFreeAddresses(), address.Offset(19))
 	space1.assertInvariants()
 
-	_, addr2 := space1.Allocate()
+	_, addr2 := space1.Allocate(address.NewRange(start, size))
 	wt.AssertFalse(t, addr2.String() == testAddr1, "address")
 	wt.AssertEquals(t, space1.NumFreeAddresses(), address.Offset(18))
-	wt.AssertEquals(t, space1.NumFreeAddressesInRange(ip(testAddr1), ip(testAddrx)), address.Offset(13))
-	wt.AssertEquals(t, space1.NumFreeAddressesInRange(ip(testAddr1), ip(testAddry)), address.Offset(18))
+	wt.AssertEquals(t, space1.NumFreeAddressesInRange(address.Range{Start: ip(testAddr1), End: ip(testAddrx)}), address.Offset(13))
+	wt.AssertEquals(t, space1.NumFreeAddressesInRange(address.Range{Start: ip(testAddr1), End: ip(testAddry)}), address.Offset(18))
 	space1.assertInvariants()
 
 	space1.Free(addr2)
@@ -125,32 +139,33 @@ func TestSpaceFree(t *testing.T) {
 		containerID = "deadbeef"
 	)
 
+	entireRange := address.NewRange(ip(testAddr1), 20)
 	space := makeSpace(ip(testAddr1), 20)
 
 	// Check we are prepared to give up the entire space
-	pos, size := space.biggestFreeRange()
-	wt.AssertTrue(t, space.free[pos] == ip(testAddr1) && size == 20, "Wrong space")
+	r := space.biggestFreeRange(entireRange)
+	wt.AssertTrue(t, r.Start == ip(testAddr1) && r.Size() == 20, "Wrong space")
 
 	for i := 0; i < 20; i++ {
-		ok, _ := space.Allocate()
+		ok, _ := space.Allocate(entireRange)
 		wt.AssertTrue(t, ok, "Failed to get address")
 	}
 
 	// Check we are full
-	ok, _ := space.Allocate()
+	ok, _ := space.Allocate(entireRange)
 	wt.AssertTrue(t, !ok, "Should have failed to get address")
-	_, size, ok = space.Donate()
-	wt.AssertTrue(t, size == 0, "Wrong space")
+	r, ok = space.Donate(entireRange)
+	wt.AssertTrue(t, r.Size() == 0, "Wrong space")
 
 	// Free in the middle
 	wt.AssertSuccess(t, space.Free(ip("10.0.3.13")))
-	pos, size = space.biggestFreeRange()
-	wt.AssertTrue(t, space.free[pos] == ip("10.0.3.13") && size == 1, "Wrong space")
+	r = space.biggestFreeRange(entireRange)
+	wt.AssertTrue(t, r.Start == ip("10.0.3.13") && r.Size() == 1, "Wrong space")
 
 	// Free one at the end
 	wt.AssertSuccess(t, space.Free(ip("10.0.3.23")))
-	pos, size = space.biggestFreeRange()
-	wt.AssertTrue(t, space.free[pos] == ip("10.0.3.23") && size == 1, "Wrong space")
+	r = space.biggestFreeRange(entireRange)
+	wt.AssertTrue(t, r.Start == ip("10.0.3.23") && r.Size() == 1, "Wrong space")
 
 	// Now free a few at the end
 	wt.AssertSuccess(t, space.Free(ip("10.0.3.22")))
@@ -159,8 +174,8 @@ func TestSpaceFree(t *testing.T) {
 	wt.AssertEquals(t, space.NumFreeAddresses(), address.Offset(4))
 
 	// Now get the biggest free space; should be 3.21
-	pos, size = space.biggestFreeRange()
-	wt.AssertTrue(t, space.free[pos] == ip("10.0.3.21") && size == 3, "Wrong space")
+	r = space.biggestFreeRange(entireRange)
+	wt.AssertTrue(t, r.Start == ip("10.0.3.21") && r.Size() == 3, "Wrong space")
 
 	// Now free a few in the middle
 	wt.AssertSuccess(t, space.Free(ip("10.0.3.12")))
@@ -170,46 +185,44 @@ func TestSpaceFree(t *testing.T) {
 	wt.AssertEquals(t, space.NumFreeAddresses(), address.Offset(7))
 
 	// Now get the biggest free space; should be 3.21
-	pos, size = space.biggestFreeRange()
-	wt.AssertTrue(t, space.free[pos] == ip("10.0.3.10") && size == 4, "Wrong space")
+	r = space.biggestFreeRange(entireRange)
+	wt.AssertTrue(t, r.Start == ip("10.0.3.10") && r.Size() == 4, "Wrong space")
 
 	wt.AssertEquals(t, space.OwnedRanges(), []address.Range{address.Range{Start: ip("10.0.3.4"), End: ip("10.0.3.24")}})
 }
 
-func (s1 *Space) Equal(s2 *Space) bool {
-	return reflect.DeepEqual(s1.ours, s2.ours) && reflect.DeepEqual(s1.free, s2.free)
-}
-
 func TestDonateSimple(t *testing.T) {
 	const (
 		testAddr1 = "10.0.1.0"
 		testAddr2 = "10.0.1.32"
+		size      = 48
 	)
 
 	var (
 		ipAddr1 = ip(testAddr1)
 	)
 
-	ps1 := makeSpace(ipAddr1, 48)
+	ps1 := makeSpace(ipAddr1, size)
 
 	// Empty space set should split in two and give me the second half
-	start, numGivenUp, ok := ps1.Donate()
+	r, ok := ps1.Donate(address.NewRange(ip(testAddr1), size))
+	numGivenUp := r.Size()
 	wt.AssertTrue(t, ok, "Donate result")
-	wt.AssertEqualString(t, start.String(), "10.0.1.24", "Invalid start")
-	wt.AssertEquals(t, numGivenUp, address.Offset(24))
-	wt.AssertEquals(t, ps1.NumFreeAddresses(), address.Offset(24))
+	wt.AssertEqualString(t, r.Start.String(), "10.0.1.24", "Invalid start")
+	wt.AssertEquals(t, numGivenUp, address.Offset(size/2))
+	wt.AssertEquals(t, ps1.NumFreeAddresses(), address.Offset(size/2))
 
 	// Now check we can give the rest up.
 	count := 0 // count to avoid infinite loop
 	for ; count < 1000; count++ {
-		_, size, ok := ps1.Donate()
+		r, ok := ps1.Donate(address.NewRange(ip(testAddr1), size))
 		if !ok {
 			break
 		}
-		numGivenUp += size
+		numGivenUp += r.Size()
 	}
 	wt.AssertEquals(t, ps1.NumFreeAddresses(), address.Offset(0))
-	wt.AssertEquals(t, numGivenUp, address.Offset(48))
+	wt.AssertEquals(t, numGivenUp, address.Offset(size))
 }
 
 func TestDonateHard(t *testing.T) {
@@ -222,7 +235,7 @@ func TestDonateHard(t *testing.T) {
 	// Fill a fresh space
 	spaceset := makeSpace(start, size)
 	for i := address.Offset(0); i < size; i++ {
-		ok, _ := spaceset.Allocate()
+		ok, _ := spaceset.Allocate(address.NewRange(start, size))
 		wt.AssertTrue(t, ok, "Failed to get IP!")
 	}
 
@@ -235,10 +248,10 @@ func TestDonateHard(t *testing.T) {
 	}
 
 	// Now split
-	newRange, numGivenUp, ok := spaceset.Donate()
+	newRange, ok := spaceset.Donate(address.NewRange(start, size))
 	wt.AssertTrue(t, ok, "GiveUpSpace result")
-	wt.AssertEquals(t, newRange, ip("10.0.1.23"))
-	wt.AssertEquals(t, numGivenUp, address.Offset(24))
+	wt.AssertEquals(t, newRange.Start, ip("10.0.1.23"))
+	wt.AssertEquals(t, newRange.Size(), address.Offset(24))
 	wt.AssertEquals(t, spaceset.NumFreeAddresses(), address.Offset(23))
 
 	//Space set should now have 2 spaces

ipam/testutils_test.go

@@ -128,27 +128,29 @@ func CheckAllExpectedMessagesSent(allocs ...*Allocator) {
 	}
 }
 
-func makeAllocator(name string, cidr string, quorum uint) *Allocator {
+func makeAllocator(name string, cidrStr string, quorum uint) (*Allocator, address.Range) {
 	peername, err := router.PeerNameFromString(name)
 	if err != nil {
 		panic(err)
 	}
 
-	alloc, err := NewAllocator(peername, router.PeerUID(rand.Int63()),
-		"nick-"+name, cidr, quorum)
+	_, cidr, err := address.ParseCIDR(cidrStr)
 	if err != nil {
 		panic(err)
 	}
 
-	return alloc
+	alloc := NewAllocator(peername, router.PeerUID(rand.Int63()),
+		"nick-"+name, cidr.Range(), quorum)
+
+	return alloc, cidr.HostRange()
 }
 
-func makeAllocatorWithMockGossip(t *testing.T, name string, universeCIDR string, quorum uint) *Allocator {
-	alloc := makeAllocator(name, universeCIDR, quorum)
+func makeAllocatorWithMockGossip(t *testing.T, name string, universeCIDR string, quorum uint) (*Allocator, address.Range) {
+	alloc, subnet := makeAllocator(name, universeCIDR, quorum)
 	gossip := &mockGossipComms{t: t, name: name}
 	alloc.SetInterfaces(gossip)
 	alloc.Start()
-	return alloc
+	return alloc, subnet
 }
 
 func (alloc *Allocator) claimRingForTesting(allocs ...*Allocator) {
@@ -160,10 +162,10 @@ func (alloc *Allocator) claimRingForTesting(allocs ...*Allocator) {
 	alloc.space.AddRanges(alloc.ring.OwnedRanges())
 }
 
-func (alloc *Allocator) NumFreeAddresses() address.Offset {
+func (alloc *Allocator) NumFreeAddresses(r address.Range) address.Offset {
 	resultChan := make(chan address.Offset)
 	alloc.actionChan <- func() {
-		resultChan <- alloc.space.NumFreeAddresses()
+		resultChan <- alloc.space.NumFreeAddressesInRange(r)
 	}
 	return <-resultChan
 }
@@ -285,13 +287,15 @@ func (client TestGossipRouterClient) GossipBroadcast(update router.GossipData) e
 	return client.router.GossipBroadcast(update)
 }
 
-func makeNetworkOfAllocators(size int, cidr string) ([]*Allocator, TestGossipRouter) {
+func makeNetworkOfAllocators(size int, cidr string) ([]*Allocator, TestGossipRouter, address.Range) {
 
 	gossipRouter := TestGossipRouter{make(map[router.PeerName]chan gossipMessage), 0.0}
 	allocs := make([]*Allocator, size)
+	var subnet address.Range
 
 	for i := 0; i < size; i++ {
-		alloc := makeAllocator(fmt.Sprintf("%02d:00:00:02:00:00", i),
+		var alloc *Allocator
+		alloc, subnet = makeAllocator(fmt.Sprintf("%02d:00:00:02:00:00", i),
 			cidr, uint(size/2+1))
 		alloc.SetInterfaces(gossipRouter.connect(alloc.ourName, alloc))
 		alloc.Start()
@@ -300,7 +304,7 @@ func makeNetworkOfAllocators(size int, cidr string) ([]*Allocator, TestGossipRou
 
 	gossipRouter.GossipBroadcast(allocs[size-1].Gossip())
 	time.Sleep(1000 * time.Millisecond)
-	return allocs, gossipRouter
+	return allocs, gossipRouter, subnet
 }
 
 func stopNetworkOfAllocators(allocs []*Allocator) {

prog/weaver/main.go

@@ -17,6 +17,7 @@ import (
 	. "github.com/weaveworks/weave/common"
 	"github.com/weaveworks/weave/common/updater"
 	"github.com/weaveworks/weave/ipam"
+	"github.com/weaveworks/weave/ipam/address"
 	weavenet "github.com/weaveworks/weave/net"
 	weave "github.com/weaveworks/weave/router"
 )
@@ -37,23 +38,24 @@ func main() {
 	runtime.GOMAXPROCS(procs)
 
 	var (
-		config      weave.Config
-		justVersion bool
-		ifaceName   string
-		routerName  string
-		nickName    string
-		password    string
-		wait        int
-		debug       bool
-		pktdebug    bool
-		prof        string
-		bufSzMB     int
-		noDiscovery bool
-		httpAddr    string
-		iprangeCIDR string
-		peerCount   int
-		apiPath     string
-		peers       []string
+		config       weave.Config
+		justVersion  bool
+		ifaceName    string
+		routerName   string
+		nickName     string
+		password     string
+		wait         int
+		debug        bool
+		pktdebug     bool
+		prof         string
+		bufSzMB      int
+		noDiscovery  bool
+		httpAddr     string
+		iprangeCIDR  string
+		ipsubnetCIDR string
+		peerCount    int
+		apiPath      string
+		peers        []string
 	)
 
 	flag.BoolVar(&justVersion, "version", false, "print version and exit")
@@ -70,7 +72,8 @@ func main() {
 	flag.BoolVar(&noDiscovery, "nodiscovery", false, "disable peer discovery")
 	flag.IntVar(&bufSzMB, "bufsz", 8, "capture buffer size in MB")
 	flag.StringVar(&httpAddr, "httpaddr", fmt.Sprintf(":%d", weave.HTTPPort), "address to bind HTTP interface to (disabled if blank, absolute path indicates unix domain socket)")
-	flag.StringVar(&iprangeCIDR, "iprange", "", "IP address range to allocate within, in CIDR notation")
+	flag.StringVar(&iprangeCIDR, "iprange", "", "IP address range reserved for automatic allocation, in CIDR notation")
+	flag.StringVar(&ipsubnetCIDR, "ipsubnet", "", "subnet to allocate within by default, in CIDR notation")
 	flag.IntVar(&peerCount, "initpeercount", 0, "number of peers in network (for IP address allocation)")
 	flag.StringVar(&apiPath, "api", "unix:///var/run/docker.sock", "Path to Docker API socket")
 	flag.Parse()
@@ -137,8 +140,9 @@ func main() {
 	log.Println("Our name is", router.Ourself)
 
 	var allocator *ipam.Allocator
+	var defaultSubnet address.CIDR
 	if iprangeCIDR != "" {
-		allocator = createAllocator(router, apiPath, iprangeCIDR, determineQuorum(peerCount, peers))
+		allocator, defaultSubnet = createAllocator(router, apiPath, iprangeCIDR, ipsubnetCIDR, determineQuorum(peerCount, peers))
 	} else if peerCount > 0 {
 		log.Fatal("-initpeercount flag specified without -iprange")
 	} else {
@@ -154,7 +158,7 @@ func main() {
 	// so there is no point in doing "weave launch -httpaddr ''".
 	// This is here to support stand-alone use of weaver.
 	if httpAddr != "" {
-		go handleHTTP(router, httpAddr, allocator)
+		go handleHTTP(router, httpAddr, allocator, defaultSubnet)
 	}
 
 	SignalHandlerLoop(router)
@@ -194,18 +198,35 @@ func logFrameFunc(debug bool) weave.LogFrameFunc {
 	}
 }
 
-func createAllocator(router *weave.Router, apiPath string, iprangeCIDR string, quorum uint) *ipam.Allocator {
-	allocator, err := ipam.NewAllocator(router.Ourself.Peer.Name, router.Ourself.Peer.UID, router.Ourself.Peer.NickName, iprangeCIDR, quorum)
+func parseAndCheckCIDR(cidrStr string) address.CIDR {
+	_, cidr, err := address.ParseCIDR(cidrStr)
 	if err != nil {
 		log.Fatal(err)
 	}
+	if cidr.Size() < ipam.MinSubnetSize {
+		log.Fatalf("Allocation range smaller than minimum size %d: %s", ipam.MinSubnetSize, cidrStr)
+	}
+	return cidr
+}
+
+func createAllocator(router *weave.Router, apiPath string, ipRangeStr string, defaultSubnetStr string, quorum uint) (*ipam.Allocator, address.CIDR) {
+	ipRange := parseAndCheckCIDR(ipRangeStr)
+	defaultSubnet := ipRange
+	if defaultSubnetStr != "" {
+		defaultSubnet = parseAndCheckCIDR(defaultSubnetStr)
+		if !ipRange.Range().Overlaps(defaultSubnet.Range()) {
+			log.Fatalf("Default subnet %s out of bounds: %s", defaultSubnet, ipRange)
+		}
+	}
+	allocator := ipam.NewAllocator(router.Ourself.Peer.Name, router.Ourself.Peer.UID, router.Ourself.Peer.NickName, ipRange.Range(), quorum)
+
 	allocator.SetInterfaces(router.NewGossip("IPallocation", allocator))
 	allocator.Start()
-	err = updater.Start(apiPath, allocator)
+	err := updater.Start(apiPath, allocator)
 	if err != nil {
 		log.Fatal("Unable to start watcher", err)
 	}
-	return allocator
+	return allocator, defaultSubnet
 }
 
 // Pick a quorum size heuristically based on the number of peer
@@ -228,11 +249,11 @@ func determineQuorum(initPeerCountFlag int, peers []string) uint {
 	return quorum
 }
 
-func handleHTTP(router *weave.Router, httpAddr string, allocator *ipam.Allocator) {
+func handleHTTP(router *weave.Router, httpAddr string, allocator *ipam.Allocator, defaultSubnet address.CIDR) {
 	muxRouter := mux.NewRouter()
 
 	if allocator != nil {
-		allocator.HandleHTTP(muxRouter)
+		allocator.HandleHTTP(muxRouter, defaultSubnet)
 	}
 
 	muxRouter.Methods("GET").Path("/status").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -240,6 +261,7 @@ func handleHTTP(router *weave.Router, httpAddr string, allocator *ipam.Allocator
 		fmt.Fprintln(w, router.Status())
 		if allocator != nil {
 			fmt.Fprintln(w, allocator.String())
+			fmt.Fprintln(w, "Allocator default subnet:", defaultSubnet)
 		}
 	})
 

router/protocol.go

@@ -2,7 +2,7 @@ package router
 
 const (
 	Protocol        = "weave"
-	ProtocolVersion = 18
+	ProtocolVersion = 20
 )
 
 type ProtocolTag byte

site/ipam.md

@@ -9,10 +9,10 @@ Weave can automatically assign unique IP addresses to each container
 across the network. To make this work, weave must be told on startup
 what range of addresses to allocate from, for example:
 
-    host1# weave launch -iprange 10.2.3.0/24
+    host1# weave launch -iprange 10.2.0.0/16
 
-The `run`, `start`, `attach`, `expose` and `hide` commands will then
-fetch an address automatically if none is specified, i.e.:
+The `run`, `start`, `attach`, `detach`, `expose` and `hide` commands
+will then fetch an address automatically if none is specified, i.e.:
 
     host1# C=$(weave run -ti ubuntu)
 
@@ -22,7 +22,40 @@ You can see which address was allocated with
     host1# weave ps $C
     a7aff7249393 7a:51:d1:09:21:78 10.2.3.1/24
 
-The `-iprange` parameter is given in [CIDR
+You may wish to [run containers on different
+subnets](features.html#application-isolation). To set a default
+subnet, launch with `-ipsubnet`:
+
+    host1# weave launch -iprange 10.2.0.0/16 -ipsubnet 10.2.3.0/24
+
+`-iprange` should cover the entire range that you will ever use for
+allocation, and `-ipsubnet` is the subnet that will be used when
+you don't explicitly specify one.
+
+Then, to request the allocation of an address from a subnet other than
+the default, use the `net:` prefix:
+
+    host1# C=$(weave run net:10.2.7.0/24 -ti ubuntu)
+
+To explicitly request the default subnet, use `net:default`.
+
+And, you can ask for multiple addresses in different subnets and add
+in manually-assigned addresses on the same line, for instance:
+
+    host1# C=$(weave run net:10.2.7.0/24 net:10.2.8.0/24 ip:10.3.9.1/24 -ti ubuntu)
+
+Note that `-iprange` can be smaller than `-ipsubnet`, if you want to
+use a mixture of automatically-allocated addresses and manually-chosen
+addresses, and have the containers communicate with each other. For
+example, if you say:
+
+    host1# weave launch -iprange 10.9.0.0/17 -ipsubnet 10.9.0.0/16
+
+then you can run all containers in the 10.9.0.0/16 subnet, with IPAM
+using the lower half and leaving the upper half free for manual
+allocation.
+
+The subnet parameters are written in [CIDR
 notation](http://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing)
 format - in this example "/24" means the first 24 bits of the address
 form the network address and the allocator is to allocate container
@@ -117,12 +150,12 @@ reports on the current status of the weave router and IP allocator:
 ````
 weave router git-8f675f15c0b5
 ...
-Allocator subnet 10.2.1.0/24
-  Free IPs: ~98.0%, 62 local, ~189 remote
+Allocator universe 10.2.0.0/16
 Owned Ranges:
   10.2.1.1 -> 96:e9:e2:2e:2d:bc (host1) (v3)
   10.2.1.128 -> ea:84:25:9b:31:2e (host2) (v3)
   10.2.1.192 -> ea:6c:21:09:cf:f0 (host3) (v9)
+Allocator default subnet: 10.2.1.0/24
 ````
 
 The first section covers the router; see the [troubleshooting

test/100_cross_hosts_test.sh

@@ -4,20 +4,27 @@
 
 C1=10.2.1.4
 C2=10.2.1.7
-UNIVERSE=10.2.2.0/24
+UNIVERSE=10.2.0.0/16
+SUBNET_1=10.2.2.0/24
+SUBNET_2=10.2.3.0/24
 
 start_suite "Ping over cross-host weave network (with and without IPAM)"
 
-weave_on $HOST1 launch -iprange $UNIVERSE
-weave_on $HOST2 launch -iprange $UNIVERSE $HOST1
+weave_on $HOST1 launch -iprange $UNIVERSE -ipsubnet $SUBNET_1
+weave_on $HOST2 launch -iprange $UNIVERSE -ipsubnet $SUBNET_1 $HOST1
 
 start_container $HOST1 $C1/24 --name=c1
-start_container $HOST2 $C2/24 --name=c2
+start_container $HOST2 ip:$C2/24 --name=c2
 assert_raises "exec_on $HOST1 c1 $PING $C2"
 
 start_container $HOST1 --name=c3
-start_container $HOST2 --name=c4
+start_container $HOST2 net:default --name=c4
 C4=$(container_ip $HOST2 c4)
 assert_raises "exec_on $HOST1 c3 $PING $C4"
 
+start_container $HOST1 net:$SUBNET_2 --name=c5
+start_container $HOST2 net:$SUBNET_2 --name=c6
+C6=$(container_ip $HOST2 c6)
+assert_raises "exec_on $HOST1 c5 $PING $C6"
+
 end_suite

test/500_weave_multi_cidr_test.sh

@@ -11,7 +11,7 @@ assert_container_cidrs() {
     CIDRS="$@"
 
     # Assert container has attached CIDRs
-    assert_raises "weave_on $HOST ps | grep '^$CID [^ ]* $CIDRS$'"
+    assert_raises "weave_on $HOST ps $CID | grep -E '^$CID [0-9a-f:]{17} $CIDRS$'"
 }
 
 # assert_zone_records <host> <cid> <fqdn> <ip> [<ip> ...]
@@ -42,36 +42,45 @@ assert_bridge_cidrs() {
 
 start_suite "Weave run/start/attach/detach with multiple cidr arguments"
 
-weave_on $HOST1 launch
+# NOTE: in these tests, net: arguments are checked against a
+# specific address, i.e. we are assuming that IPAM always returns the
+# lowest available address in the subnet
+
+weave_on $HOST1 launch -debug -iprange 10.2.3.0/24
 weave_on $HOST1 launch-dns 10.254.254.254/24
 
 # Run container with three cidrs
-CID=$(start_container $HOST1 10.2.1.1/24 10.2.2.1/24 10.2.3.1/24 --name=multicidr -h $NAME | cut -b 1-12)
+CID=$(start_container $HOST1 ip:10.2.1.1/24 10.2.2.1/24 net:10.2.3.0/24 --name=multicidr -h $NAME | cut -b 1-12)
 assert_container_cidrs $HOST1 $CID 10.2.1.1/24 10.2.2.1/24 10.2.3.1/24
 assert_zone_records $HOST1 $CID $NAME. 10.2.1.1 10.2.2.1 10.2.3.1
 
 # Remove two of them
-weave_on $HOST1 detach 10.2.1.1/24 10.2.3.1/24 $CID
+weave_on $HOST1 detach 10.2.1.1/24 net:10.2.3.0/24 $CID
 assert_container_cidrs $HOST1 $CID 10.2.2.1/24
 assert_zone_records $HOST1 $CID $NAME. 10.2.2.1
 
 # Put them both back
-weave_on $HOST1 attach 10.2.1.1/24 10.2.3.1/24 $CID
+weave_on $HOST1 attach ip:10.2.1.1/24 net:10.2.3.0/24 $CID
 assert_container_cidrs $HOST1 $CID 10.2.2.1/24 10.2.1.1/24 10.2.3.1/24
 assert_zone_records $HOST1 $CID $NAME. 10.2.2.1 10.2.1.1 10.2.3.1
 
 # Stop the container, restart with three IPs
 docker_on $HOST1 stop $CID
-weave_on $HOST1 start 10.2.1.1/24 10.2.2.1/24 10.2.3.1/24 $CID
+weave_on $HOST1 start 10.2.1.1/24 ip:10.2.2.1/24  net:10.2.3.0/24 $CID
 assert_container_cidrs $HOST1 $CID 10.2.1.1/24 10.2.2.1/24 10.2.3.1/24
 assert_zone_records $HOST1 $CID $NAME. 10.2.1.1 10.2.2.1 10.2.3.1
 
 # Expose some cidrs
-weave_on $HOST1 expose 10.2.1.2/24 10.2.2.2/24 10.2.3.2/24
+weave_on $HOST1 expose 10.2.1.2/24 10.2.2.2/24 net:10.2.3.0/24
 assert_bridge_cidrs $HOST1 weave 10.2.1.2/24 10.2.2.2/24 10.2.3.2/24
 
 # Hide some cidrs
-weave_on $HOST1 hide 10.2.1.2/24 10.2.3.2/24
+weave_on $HOST1 hide 10.2.1.2/24 net:10.2.3.0/24
 assert_bridge_cidrs $HOST1 weave 10.2.2.2/24
 
+# Now detach and run another container to check we have released IPs in IPAM
+weave_on $HOST1 detach $CID
+CID2=$(start_container $HOST1 net:10.2.3.0/24)
+assert_container_cidrs $HOST1 $CID2 10.2.3.1/24
+
 end_suite

weave

@@ -28,19 +28,19 @@ usage() {
     echo "Usage:"
     echo "weave setup"
     echo "weave launch       [-password <password>] [-nickname <nickname>] [-nodiscovery]"
-    echo "                       [-iprange <cidr>] <peer> ..."
+    echo "                       [-iprange <cidr> [-ipsubnet <cidr>]] <peer> ..."
     echo "weave launch-dns   <cidr>"
     echo "weave launch-proxy [-D <docker_endpoint>] [-H <address>] [--with-dns] [--with-ipam]"
     echo "weave connect      [--replace] [<peer> ...]"
     echo "weave forget       <peer> ..."
-    echo "weave run          [--with-dns] [<cidr> ...] <docker run args> ..."
-    echo "weave start        [<cidr> ...] <container_id>"
-    echo "weave attach       [<cidr> ...] <container_id>"
-    echo "weave detach       <cidr> [<cidr> ...] <container_id>"
+    echo "weave run          [--with-dns] [<address_spec> ...] <docker run args> ..."
+    echo "weave start        [<address_spec> ...] <container_id>"
+    echo "weave attach       [<address_spec> ...] <container_id>"
+    echo "weave detach       [<address_spec> ...] <container_id>"
     echo "weave dns-add      <ip_address> [<ip_address> ...] <container_id> [-h <fqdn>]"
     echo "weave dns-remove   <ip_address> [<ip_address> ...] <container_id> [-h <fqdn>]"
-    echo "weave expose       [<cidr> ...] [-h <fqdn>]"
-    echo "weave hide         [<cidr> ...]"
+    echo "weave expose       [<address_spec> ...] [-h <fqdn>]"
+    echo "weave hide         [<address_spec> ...]"
     echo "weave ps           [<container_id> ...]"
     echo "weave status"
     echo "weave proxy-env"
@@ -52,9 +52,10 @@ usage() {
     echo "weave reset"
     echo "weave rmpeer       <peer_id>"
     echo
-    echo "where <peer>    = <ip_address_or_fqdn>[:<port>]"
-    echo "      <cidr>    = <ip_address>/<routing_prefix_length>"
-    echo "      <peer_id> = <nickname> or weave internal peer ID"
+    echo "where <peer>         = <ip_address_or_fqdn>[:<port>]"
+    echo "      <cidr>         = <ip_address>/<routing_prefix_length>"
+    echo "      <peer_id>      = <nickname> or weave internal peer ID"
+    echo "      <address_spec> = [ip:]<cidr> | net:<cidr> | net:default"
     echo "      <address> = [tcp://][<ip_address>]:<port>"
     exit 1
 }
@@ -104,10 +105,10 @@ validate_cidr() {
 
 collect_cidr_args() {
     CIDR_ARGS=""
-    CIDR_COUNT=0
-    while is_cidr "$1" ; do
-        CIDR_ARGS="$CIDR_ARGS $1"
-        CIDR_COUNT=$((CIDR_COUNT + 1))
+    CIDR_ARG_COUNT=0
+    while [ "$1" = "net:default" ] || is_cidr "$1" || is_cidr "${1#ip:}" || is_cidr "${1#net:}" ; do
+        CIDR_ARGS="$CIDR_ARGS ${1#ip:}"
+        CIDR_ARG_COUNT=$((CIDR_ARG_COUNT + 1))
         shift 1
     done
 }
@@ -149,7 +150,7 @@ if [ "$1" = "run" ] ; then
         DNS_ARGS=$(exec_remote dns-args "$@")
     fi
     collect_cidr_args "$@"
-    shift $CIDR_COUNT
+    shift $CIDR_ARG_COUNT
     CONTAINER=$(docker $DOCKER_CLIENT_ARGS run $DNS_ARGS -d "$@")
     exec_remote attach $CIDR_ARGS $CONTAINER
     echo $CONTAINER
@@ -751,17 +752,45 @@ ipam_claim() {
     done
 }
 
-# If we don't already have any CIDRs passed from the command-line, get
-# one from IPAM. $1 is the full container id.
-get_ipam_cidr() {
-    if [ -n "$CIDR_ARGS" ] ; then
-        return 0
-    fi
-    CIDR_ARGS="$(http_call $CONTAINER_NAME $HTTP_PORT POST /ip/$1)"
-    if [ "$CIDR_ARGS" = "404 page not found" ] ; then
-        echo "No IP address supplied (use the -iprange option on 'weave launch' to enable IP address allocation)" >&2
-        return 1
+# Call IPAM as necessary to allocate addresses
+# $1 is the full container id and following args are previously parsed CIDR_ARGS.
+# Returns ALL_CIDRS and IPAM_CIDRS
+ipam_cidrs() {
+    CONTAINER_ID="$1"
+    shift 1
+    ALL_CIDRS=""
+    IPAM_CIDRs=""
+    if [ $# -eq 0 ] ; then
+        # If no addresses passed in, get one in the default subnet
+        IPAM_CIDRS="$(http_call $CONTAINER_NAME $HTTP_PORT POST /ip/$CONTAINER_ID)"
+        if [ "$IPAM_CIDRS" = "404 page not found" ] ; then
+            echo "No IP address supplied (use the -iprange option on 'weave launch' to enable IP address allocation)" >&2
+            return 1
+        fi
+        ALL_CIDRS="$IPAM_CIDRS"
     fi
+    while [ $# -gt 0 ] ; do
+        if [ "${1%:*}" = "net" ] ; then
+            if [ "$1" = "net:default" ] ; then
+                CIDR=$(http_call $CONTAINER_NAME $HTTP_PORT POST /ip/$CONTAINER_ID)
+            else
+                CIDR=$(http_call $CONTAINER_NAME $HTTP_PORT POST /ip/$CONTAINER_ID/"${1#net:}")
+            fi
+            if [ "$CIDR" = "404 page not found" ] ; then
+                echo "IP address allocation must be enabled to use --subnet (use the -iprange option on 'weave launch')" >&2
+                return 1
+            fi
+            if ! is_cidr "$CIDR" ; then
+                echo "$CIDR" >&2
+                return 1
+            fi
+            IPAM_CIDRS="$IPAM_CIDRS $CIDR"
+            ALL_CIDRS="$ALL_CIDRS $CIDR"
+        else
+            ALL_CIDRS="$ALL_CIDRS $1"
+        fi
+        shift 1
+    done
 }
 
 ######################################################################
@@ -1028,12 +1057,12 @@ case "$COMMAND" in
             dns_args "$@"
         fi
         collect_cidr_args "$@"
-        shift $CIDR_COUNT
+        shift $CIDR_ARG_COUNT
         CONTAINER=$(docker run $DNS_ARGS -d "$@")
         create_bridge
-        get_ipam_cidr $CONTAINER
-        with_container_netns_or_die $CONTAINER attach $CIDR_ARGS
-        when_dns_running with_container_fqdn $CONTAINER put_dns_fqdn $CIDR_ARGS
+        ipam_cidrs $CONTAINER $CIDR_ARGS
+        with_container_netns_or_die $CONTAINER attach $ALL_CIDRS
+        when_dns_running with_container_fqdn $CONTAINER put_dns_fqdn $ALL_CIDRS
         echo $CONTAINER
         ;;
     dns-args)
@@ -1046,33 +1075,37 @@ case "$COMMAND" in
         ;;
     start)
         collect_cidr_args "$@"
-        shift $CIDR_COUNT
+        shift $CIDR_ARG_COUNT
         [ $# -eq 1 ] || usage
         RES=$(docker start $1)
         CONTAINER=$(container_id $1)
         create_bridge
-        get_ipam_cidr $CONTAINER
-        with_container_netns_or_die $CONTAINER attach $CIDR_ARGS
-        when_dns_running with_container_fqdn $CONTAINER put_dns_fqdn $CIDR_ARGS
+        ipam_cidrs $CONTAINER $CIDR_ARGS
+        with_container_netns_or_die $CONTAINER attach $ALL_CIDRS
+        when_dns_running with_container_fqdn $CONTAINER put_dns_fqdn $ALL_CIDRS
         echo $RES
         ;;
     attach)
         collect_cidr_args "$@"
-        shift $CIDR_COUNT
+        shift $CIDR_ARG_COUNT
         [ $# -eq 1 ] || usage
         CONTAINER=$(container_id $1)
         create_bridge
-        get_ipam_cidr $CONTAINER
-        with_container_netns $CONTAINER attach $CIDR_ARGS >/dev/null
-        when_dns_running with_container_fqdn $CONTAINER put_dns_fqdn $CIDR_ARGS
+        ipam_cidrs $CONTAINER $CIDR_ARGS
+        with_container_netns $CONTAINER attach $ALL_CIDRS >/dev/null
+        when_dns_running with_container_fqdn $CONTAINER put_dns_fqdn $ALL_CIDRS
         ;;
     detach)
         collect_cidr_args "$@"
-        shift $CIDR_COUNT
+        shift $CIDR_ARG_COUNT
         [ $# -eq 1 ] || usage
         CONTAINER=$(container_id $1)
-        with_container_netns $CONTAINER detach $CIDR_ARGS >/dev/null
-        when_dns_running with_container_fqdn $CONTAINER delete_dns_fqdn $CIDR_ARGS
+        ipam_cidrs $CONTAINER $CIDR_ARGS
+        with_container_netns $CONTAINER detach $ALL_CIDRS >/dev/null
+        when_dns_running with_container_fqdn $CONTAINER delete_dns_fqdn $ALL_CIDRS
+        for CIDR in $IPAM_CIDRS ; do
+            http_call $CONTAINER_NAME $HTTP_PORT DELETE /ip/$CONTAINER/${CIDR%/*}
+        done
         ;;
     dns-add)
         collect_ip_args "$@"
@@ -1100,8 +1133,8 @@ case "$COMMAND" in
         ;;
     expose)
         collect_cidr_args "$@"
-        shift $CIDR_COUNT
-        get_ipam_cidr weave:expose
+        shift $CIDR_ARG_COUNT
+        ipam_cidrs weave:expose $CIDR_ARGS
         if [ $# -eq 0 ] ; then
             FQDN=""
         else
@@ -1109,7 +1142,7 @@ case "$COMMAND" in
             FQDN="$2"
         fi
         create_bridge --without-ethtool
-        for CIDR in $CIDR_ARGS ; do
+        for CIDR in $ALL_CIDRS ; do
             if ! ip addr show dev $BRIDGE | grep -qF $CIDR ; then
                 ip addr add dev $BRIDGE $CIDR
                 arp_update $BRIDGE $CIDR
@@ -1123,10 +1156,10 @@ case "$COMMAND" in
         ;;
     hide)
         collect_cidr_args "$@"
-        shift $CIDR_COUNT
-        get_ipam_cidr weave:expose
+        shift $CIDR_ARG_COUNT
+        ipam_cidrs weave:expose $CIDR_ARGS
         create_bridge --without-ethtool
-        for CIDR in $CIDR_ARGS ; do
+        for CIDR in $ALL_CIDRS ; do
             if ip addr show dev $BRIDGE | grep -qF $CIDR ; then
                 ip addr del dev $BRIDGE $CIDR
                 delete_iptables_rule nat WEAVE -d $CIDR ! -s $CIDR -j MASQUERADE
@@ -1134,7 +1167,9 @@ case "$COMMAND" in
                 when_dns_running delete_dns 'weave:expose' $CIDR
             fi
         done
-        [ $CIDR_COUNT -eq 0 ] && http_call $CONTAINER_NAME $HTTP_PORT DELETE /ip/weave:expose 2>/dev/null || true
+        for CIDR in $IPAM_CIDRS ; do
+            http_call $CONTAINER_NAME $HTTP_PORT DELETE /ip/weave:expose/${CIDR%/*}
+        done
         ;;
     stop)
         [ $# -eq 0 ] || usage