diff --git a/docs/ipam.md b/docs/ipam.md index 706ad842..d8fcb1e3 100644 --- a/docs/ipam.md +++ b/docs/ipam.md @@ -20,11 +20,16 @@ plugin) uses to request IP addresses. The commands supported by the allocator via the http interface are: -- Allocate: request one IP address for a container +- Allocate: request one IP address for a container in a subnet +- Lookup: fetch the previously-allocated IP address for a container in a subnet - Free: return an IP address that is currently allocated - Claim: request a specific IP address for a container (e.g. because it is already using that address) +Each http request either specifies a subnet, or if no subnet is +specified this is taken as a request to allocate in a pre-defined +default subnet. + The allocator also watches via the Docker event mechanism: if a container dies then all IP addresses allocated to that container are freed. @@ -48,16 +53,17 @@ freed. When a peer owns some range(s), it can allocate freely from within those ranges to containers on the same machine. If it runs out of -space (all owned ranges are full), it will ask another peer for space: +space in a subnet (all owned ranges are full), it will ask another +peer for space: - it picks a peer to ask at random, weighted by the amount of space - owned by each peer + owned by each peer in the subnet - if the target peer decides to give up space, it unicasts a message back to the asker with the newly-updated ring. - if the target peer has no space, it unicasts a message back to the asker with its current copy of the ring, on the basis that the requestor must have acted on out-of-date information. - it will continue to ask for space until it receives some, or its - copy of the ring tells it all peers are full. + copy of the ring tells it all peers are full in that subnet. ### Claiming an address diff --git a/ipam/address/address.go b/ipam/address/address.go index 692ffaab..f97d44c8 100644 --- a/ipam/address/address.go +++ b/ipam/address/address.go @@ -1,8 +1,10 @@ package address import ( - "github.com/weaveworks/weave/common" + "fmt" "net" + + "github.com/weaveworks/weave/common" ) // Using 32-bit integer to represent IPv4 address @@ -13,6 +15,19 @@ type Range struct { Start, End Address // [Start, End); Start <= End } +func NewRange(start Address, size Offset) Range { + return Range{Start: start, End: Add(start, size)} +} +func (r Range) Size() Offset { return Subtract(r.End, r.Start) } +func (r Range) String() string { return fmt.Sprintf("[%s-%s)", r.Start, r.End) } +func (r Range) Overlaps(or Range) bool { return !(r.Start >= or.End || r.End <= or.Start) } +func (r Range) Contains(addr Address) bool { return addr >= r.Start && addr < r.End } + +type CIDR struct { + Start Address + PrefixLen int +} + func ParseIP(s string) (Address, error) { if ip := net.ParseIP(s); ip != nil { return FromIP4(ip), nil @@ -20,6 +35,31 @@ func ParseIP(s string) (Address, error) { return 0, &net.ParseError{Type: "IP Address", Text: s} } +func ParseCIDR(s string) (Address, CIDR, error) { + if ip, ipnet, err := net.ParseCIDR(s); err != nil { + return 0, CIDR{}, err + } else if ipnet.IP.To4() == nil { + return 0, CIDR{}, &net.ParseError{Type: "Non-IPv4 address not supported", Text: s} + } else { + prefixLen, _ := ipnet.Mask.Size() + return FromIP4(ip), CIDR{Start: FromIP4(ipnet.IP), PrefixLen: prefixLen}, nil + } +} + +func (cidr CIDR) Size() Offset { return 1 << uint(32-cidr.PrefixLen) } + +func (cidr CIDR) Range() Range { + return NewRange(cidr.Start, cidr.Size()) +} +func (cidr CIDR) HostRange() Range { + // Respect RFC1122 exclusions of first and last addresses + return NewRange(cidr.Start+1, cidr.Size()-2) +} + +func (cidr CIDR) String() string { + return fmt.Sprintf("%s/%d", cidr.Start.String(), cidr.PrefixLen) +} + // FromIP4 converts an ipv4 address to our integer address type func FromIP4(ip4 net.IP) (r Address) { for _, b := range ip4.To4() { diff --git a/ipam/allocate.go b/ipam/allocate.go index d709784d..130380e9 100644 --- a/ipam/allocate.go +++ b/ipam/allocate.go @@ -12,8 +12,9 @@ type allocateResult struct { type allocate struct { resultChan chan<- allocateResult - hasBeenCancelled func() bool ident string + r address.Range // Range we are trying to allocate within + hasBeenCancelled func() bool } // Try returns true if the request is completed, false if pending @@ -23,23 +24,27 @@ func (g *allocate) Try(alloc *Allocator) bool { return true } - // If we have previously stored an address for this container, return it. - if addr, found := alloc.owned[g.ident]; found { + if addr, found := alloc.lookupOwned(g.ident, g.r); found { g.resultChan <- allocateResult{addr, nil} return true } - if ok, addr := alloc.space.Allocate(); ok { - alloc.debugln("Allocated", addr, "for", g.ident) + if !alloc.universe.Overlaps(g.r) { + g.resultChan <- allocateResult{0, fmt.Errorf("range %s out of bounds: %s", g.r, alloc.universe)} + return true + } + + if ok, addr := alloc.space.Allocate(g.r); ok { + alloc.debugln("Allocated", addr, "for", g.ident, "in", g.r) alloc.addOwned(g.ident, addr) g.resultChan <- allocateResult{addr, nil} return true } // out of space - if donor, err := alloc.ring.ChoosePeerToAskForSpace(); err == nil { - alloc.debugln("Decided to ask peer", donor, "for space") - alloc.sendRequest(donor, msgSpaceRequest) + if donor, err := alloc.ring.ChoosePeerToAskForSpace(g.r.Start, g.r.End); err == nil { + alloc.debugln("Decided to ask peer", donor, "for space in range", g.r) + alloc.sendSpaceRequest(donor, g.r) } return false diff --git a/ipam/allocator.go b/ipam/allocator.go index 210eb796..72ef3072 100644 --- a/ipam/allocator.go +++ b/ipam/allocator.go @@ -3,9 +3,7 @@ package ipam import ( "bytes" "encoding/gob" - "errors" "fmt" - "net" "sort" "time" @@ -23,6 +21,7 @@ const ( msgRingUpdate paxosInterval = time.Second * 5 + MinSubnetSize = 4 // first and last addresses are excluded, so 2 would be too small ) // operation represents something which Allocator wants to do, but @@ -46,16 +45,14 @@ type operation interface { type Allocator struct { actionChan chan<- func() ourName router.PeerName - subnetStart address.Address // start address of space all peers are allocating from - subnetSize address.Offset // length of space all peers are allocating from - prefixLen int // network prefix length, e.g. 24 for a /24 network - ring *ring.Ring // information on ranges owned by all peers - space space.Space // more detail on ranges owned by us - owned map[string]address.Address // who owns what address, indexed by container-ID - nicknames map[router.PeerName]string // so we can map nicknames for rmpeer - pendingAllocates []operation // held until we get some free space - pendingClaims []operation // held until we know who owns the space - gossip router.Gossip // our link to the outside world for sending messages + universe address.Range // superset of all ranges + ring *ring.Ring // information on ranges owned by all peers + space space.Space // more detail on ranges owned by us + owned map[string][]address.Address // who owns what addresses, indexed by container-ID + nicknames map[router.PeerName]string // so we can map nicknames for rmpeer + pendingAllocates []operation // held until we get some free space + pendingClaims []operation // held until we know who owns the space + gossip router.Gossip // our link to the outside world for sending messages paxos *paxos.Node paxosTicker *time.Ticker shuttingDown bool // to avoid doing any requests while trying to shut down @@ -63,34 +60,16 @@ type Allocator struct { } // NewAllocator creates and initialises a new Allocator -func NewAllocator(ourName router.PeerName, ourUID router.PeerUID, ourNickname string, subnetCIDR string, quorum uint) (*Allocator, error) { - _, subnet, err := net.ParseCIDR(subnetCIDR) - if err != nil { - return nil, err - } - if subnet.IP.To4() == nil { - return nil, errors.New("Non-IPv4 address not supported") - } - // Get the size of the network from the mask - ones, bits := subnet.Mask.Size() - var subnetSize address.Offset = 1 << uint(bits-ones) - if subnetSize < 4 { - return nil, errors.New("Allocation subnet too small") - } - subnetStart := address.FromIP4(subnet.IP) - alloc := &Allocator{ - ourName: ourName, - subnetStart: subnetStart, - subnetSize: subnetSize, - prefixLen: ones, - // per RFC 1122, don't allocate the first and last address in the subnet - ring: ring.New(address.Add(subnetStart, 1), address.Add(subnetStart, subnetSize-1), ourName), - owned: make(map[string]address.Address), +func NewAllocator(ourName router.PeerName, ourUID router.PeerUID, ourNickname string, universe address.Range, quorum uint) *Allocator { + return &Allocator{ + ourName: ourName, + universe: universe, + ring: ring.New(universe.Start, address.Add(universe.Start, universe.Size()), ourName), + owned: make(map[string][]address.Address), paxos: paxos.NewNode(ourName, ourUID, quorum), nicknames: map[router.PeerName]string{ourName: ourNickname}, now: time.Now, } - return alloc, nil } // Start runs the allocator goroutine @@ -197,26 +176,26 @@ func hasBeenCancelled(cancelChan <-chan bool) func() bool { // Actor client API -// Allocate (Sync) - get IP address for container with given name -// if there isn't any space we block indefinitely -func (alloc *Allocator) Allocate(ident string, cancelChan <-chan bool) (address.Address, error) { +// Allocate (Sync) - get new IP address for container with given name in range +// if there isn't any space in that range we block indefinitely +func (alloc *Allocator) Allocate(ident string, r address.Range, cancelChan <-chan bool) (address.Address, error) { resultChan := make(chan allocateResult) - op := &allocate{resultChan: resultChan, ident: ident, + op := &allocate{resultChan: resultChan, ident: ident, r: r, hasBeenCancelled: hasBeenCancelled(cancelChan)} alloc.doOperation(op, &alloc.pendingAllocates) result := <-resultChan return result.addr, result.err } -func (alloc *Allocator) Lookup(ident string) (address.Address, error) { +// Lookup (Sync) - get existing IP address for container with given name in range +func (alloc *Allocator) Lookup(ident string, r address.Range) (address.Address, error) { resultChan := make(chan allocateResult) alloc.actionChan <- func() { - addr, found := alloc.owned[ident] - if found { + if addr, found := alloc.lookupOwned(ident, r); found { resultChan <- allocateResult{addr: addr} return } - resultChan <- allocateResult{err: fmt.Errorf("lookup: no address found")} + resultChan <- allocateResult{err: fmt.Errorf("lookup: no address found for %s in range %s", ident, r)} } result := <-resultChan return result.addr, result.err @@ -231,22 +210,21 @@ func (alloc *Allocator) Claim(ident string, addr address.Address, cancelChan <-c return <-resultChan } -// Free (Sync) - release IP address for container with given name -func (alloc *Allocator) Free(ident string) error { - return alloc.free(ident) -} - -// ContainerDied is provided to satisfy the updater interface; does a free underneath. Async. +// ContainerDied is provided to satisfy the updater interface; does a 'Delete' underneath. Sync. func (alloc *Allocator) ContainerDied(ident string) error { - alloc.debugln("Container", ident, "died; releasing addresses") - return alloc.free(ident) + err := alloc.Delete(ident) + if err == nil { + alloc.debugln("Container", ident, "died; released addresses") + } + return err } -func (alloc *Allocator) free(ident string) error { +// Delete (Sync) - release all IP addresses for container with given name +func (alloc *Allocator) Delete(ident string) error { errChan := make(chan error) alloc.actionChan <- func() { - addr, found := alloc.owned[ident] - if found { + addrs, found := alloc.owned[ident] + for _, addr := range addrs { alloc.space.Free(addr) } delete(alloc.owned, ident) @@ -256,7 +234,7 @@ func (alloc *Allocator) free(ident string) error { found = alloc.cancelOpsFor(&alloc.pendingClaims, ident) || found if !found { - errChan <- fmt.Errorf("Free: no addresses for %s", ident) + errChan <- fmt.Errorf("Delete: no addresses for %s", ident) return } errChan <- nil @@ -264,6 +242,30 @@ func (alloc *Allocator) free(ident string) error { return <-errChan } +// Free (Sync) - release single IP address for container +func (alloc *Allocator) Free(ident string, addrToFree address.Address) error { + errChan := make(chan error) + alloc.actionChan <- func() { + addrs := alloc.owned[ident] + for i, ownedAddr := range addrs { + if ownedAddr == addrToFree { + alloc.debugln("Freed", addrToFree, "for", ident) + if len(addrs) == 1 { + delete(alloc.owned, ident) + } else { + alloc.owned[ident] = append(addrs[:i], addrs[i+1:]...) + } + alloc.space.Free(addrToFree) + errChan <- nil + return + } + } + + errChan <- fmt.Errorf("Free: address %s not found for %s", addrToFree, ident) + } + return <-errChan +} + // Sync. func (alloc *Allocator) String() string { resultChan := make(chan string) @@ -349,7 +351,13 @@ func (alloc *Allocator) OnGossipUnicast(sender router.PeerName, msg []byte) erro switch msg[0] { case msgSpaceRequest: // some other peer asked us for space - alloc.donateSpace(sender) + decoder := gob.NewDecoder(bytes.NewReader(msg[1:])) + var r address.Range + if err := decoder.Decode(&r); err != nil { + resultChan <- err + return + } + alloc.donateSpace(r, sender) resultChan <- nil case msgRingUpdate: resultChan <- alloc.update(msg[1:]) @@ -470,20 +478,14 @@ func (alloc *Allocator) actorLoop(actionChan <-chan func()) { func (alloc *Allocator) string() string { var buf bytes.Buffer - fmt.Fprintf(&buf, "Allocator subnet %s/%d", alloc.subnetStart.String(), alloc.prefixLen) + fmt.Fprintf(&buf, "Allocator range %s", alloc.universe) if alloc.ring.Empty() { if alloc.paxosTicker != nil { fmt.Fprintf(&buf, " awaiting consensus: %s", alloc.paxos.String()) } } else { - localFreeSpace := alloc.space.NumFreeAddresses() - remoteFreeSpace := alloc.ring.TotalRemoteFree() - percentFree := 100 * float64(localFreeSpace+remoteFreeSpace) / float64(alloc.subnetSize) - fmt.Fprintf(&buf, " Free IPs: ~%.1f%%, %d local, ~%d remote\n", - percentFree, localFreeSpace, remoteFreeSpace) - - fmt.Fprint(&buf, "Owned Ranges:") + fmt.Fprint(&buf, "\nOwned Ranges:") alloc.ring.FprintWithNicknames(&buf, alloc.nicknames) } if len(alloc.pendingAllocates)+len(alloc.pendingClaims) > 0 { @@ -573,8 +575,18 @@ func (alloc *Allocator) propose() { alloc.gossip.GossipBroadcast(alloc.Gossip()) } -func (alloc *Allocator) sendRequest(dest router.PeerName, kind byte) { - msg := router.Concat([]byte{kind}, alloc.encode()) +func (alloc *Allocator) sendSpaceRequest(dest router.PeerName, r address.Range) { + buf := new(bytes.Buffer) + enc := gob.NewEncoder(buf) + if err := enc.Encode(r); err != nil { + panic(err) + } + msg := router.Concat([]byte{msgSpaceRequest}, buf.Bytes()) + alloc.gossip.GossipUnicast(dest, msg) +} + +func (alloc *Allocator) sendRingUpdate(dest router.PeerName) { + msg := router.Concat([]byte{msgRingUpdate}, alloc.encode()) alloc.gossip.GossipUnicast(dest, msg) } @@ -626,25 +638,24 @@ func (alloc *Allocator) update(msg []byte) error { return nil } -func (alloc *Allocator) donateSpace(to router.PeerName) { +func (alloc *Allocator) donateSpace(r address.Range, to router.PeerName) { // No matter what we do, we'll send a unicast gossip // of our ring back to tha chap who asked for space. // This serves to both tell him of any space we might // have given him, or tell him where he might find some // more. - defer alloc.sendRequest(to, msgRingUpdate) + defer alloc.sendRingUpdate(to) alloc.debugln("Peer", to, "asked me for space") - start, size, ok := alloc.space.Donate() + chunk, ok := alloc.space.Donate(r) if !ok { - free := alloc.space.NumFreeAddresses() + free := alloc.space.NumFreeAddressesInRange(r) common.Assert(free == 0) alloc.debugln("No space to give to peer", to) return } - end := address.Add(start, size) - alloc.debugln("Giving range", start, end, size, "to", to) - alloc.ring.GrantRangeToHost(start, end, to) + alloc.debugln("Giving range", chunk, "to", to) + alloc.ring.GrantRangeToHost(chunk.Start, chunk.End, to) } func (alloc *Allocator) assertInvariants() { @@ -672,21 +683,33 @@ func (alloc *Allocator) reportFreeSpace() { freespace := make(map[address.Address]address.Offset) for _, r := range ranges { - freespace[r.Start] = alloc.space.NumFreeAddressesInRange(r.Start, r.End) + freespace[r.Start] = alloc.space.NumFreeAddressesInRange(r) } alloc.ring.ReportFree(freespace) } // Owned addresses +// NB: addr must not be owned by ident already func (alloc *Allocator) addOwned(ident string, addr address.Address) { - alloc.owned[ident] = addr + alloc.owned[ident] = append(alloc.owned[ident], addr) +} + +func (alloc *Allocator) lookupOwned(ident string, r address.Range) (address.Address, bool) { + for _, addr := range alloc.owned[ident] { + if r.Contains(addr) { + return addr, true + } + } + return 0, false } func (alloc *Allocator) findOwner(addr address.Address) string { - for ident, candidate := range alloc.owned { - if candidate == addr { - return ident + for ident, addrs := range alloc.owned { + for _, candidate := range addrs { + if candidate == addr { + return ident + } } } return "" diff --git a/ipam/allocator_test.go b/ipam/allocator_test.go index cfca9867..57a66b2f 100644 --- a/ipam/allocator_test.go +++ b/ipam/allocator_test.go @@ -24,37 +24,50 @@ func TestAllocFree(t *testing.T) { container1 = "abcdef" container2 = "baddf00d" container3 = "b01df00d" - universe = "10.0.3.0/28" - testAddr1 = "10.0.3.1" // first address allocated should be .1 because .0 is network addr - spaceSize = 14 // 16 IP addresses in /28, minus .0 and .15 + universe = "10.0.3.0/26" + subnet1 = "10.0.3.0/28" + subnet2 = "10.0.3.32/28" + testAddr1 = "10.0.3.1" + testAddr2 = "10.0.3.33" + spaceSize = 62 // 64 IP addresses in /26, minus .0 and .63 ) - alloc := makeAllocatorWithMockGossip(t, "01:00:00:01:00:00", universe, 1) + alloc, subnet := makeAllocatorWithMockGossip(t, "01:00:00:01:00:00", universe, 1) defer alloc.Stop() + _, cidr1, _ := address.ParseCIDR(subnet1) + _, cidr2, _ := address.ParseCIDR(subnet2) alloc.claimRingForTesting() - addr1, _ := alloc.Allocate(container1, nil) + addr1, err := alloc.Allocate(container1, cidr1.HostRange(), nil) + wt.AssertNoErr(t, err) wt.AssertEqualString(t, addr1.String(), testAddr1, "address") + addr2, err := alloc.Allocate(container1, cidr2.HostRange(), nil) + wt.AssertNoErr(t, err) + wt.AssertEqualString(t, addr2.String(), testAddr2, "address") + // Ask for another address for a different container and check it's different - addr2, _ := alloc.Allocate(container2, nil) - if addr2.String() == testAddr1 { - t.Fatalf("Expected different address but got %s", addr2.String()) + addr1b, _ := alloc.Allocate(container2, cidr1.HostRange(), nil) + if addr1b.String() == testAddr1 { + t.Fatalf("Expected different address but got %s", addr1b.String()) } - // Ask for the first container again and we should get the same address again - addr1a, _ := alloc.Allocate(container1, nil) + // Ask for the first container again and we should get the same addresses again + addr1a, _ := alloc.Allocate(container1, cidr1.HostRange(), nil) wt.AssertEqualString(t, addr1a.String(), testAddr1, "address") + addr2a, _ := alloc.Allocate(container1, cidr2.HostRange(), nil) + wt.AssertEqualString(t, addr2a.String(), testAddr2, "address") - // Now free the first one, and we should get it back when we ask - wt.AssertSuccess(t, alloc.Free(container1)) - addr3, _ := alloc.Allocate(container3, nil) + // Now delete the first container, and we should get its addresses back + wt.AssertSuccess(t, alloc.Delete(container1)) + addr3, _ := alloc.Allocate(container3, cidr1.HostRange(), nil) wt.AssertEqualString(t, addr3.String(), testAddr1, "address") + addr4, _ := alloc.Allocate(container3, cidr2.HostRange(), nil) + wt.AssertEqualString(t, addr4.String(), testAddr2, "address") alloc.ContainerDied(container2) alloc.ContainerDied(container3) - alloc.String() // force sync-up after async call - wt.AssertEquals(t, alloc.NumFreeAddresses(), address.Offset(spaceSize)) + wt.AssertEquals(t, alloc.NumFreeAddresses(subnet), address.Offset(spaceSize)) } func TestBootstrap(t *testing.T) { @@ -66,11 +79,11 @@ func TestBootstrap(t *testing.T) { peerNameString = "02:00:00:02:00:00" ) - alloc1 := makeAllocatorWithMockGossip(t, ourNameString, testStart1+"/22", 2) + alloc1, subnet := makeAllocatorWithMockGossip(t, ourNameString, testStart1+"/22", 2) defer alloc1.Stop() // Simulate another peer on the gossip network - alloc2 := makeAllocatorWithMockGossip(t, peerNameString, testStart1+"/22", 2) + alloc2, _ := makeAllocatorWithMockGossip(t, peerNameString, testStart1+"/22", 2) defer alloc2.Stop() alloc1.OnGossipBroadcast(alloc2.Encode()) @@ -80,7 +93,7 @@ func TestBootstrap(t *testing.T) { ExpectBroadcastMessage(alloc1, nil) // alloc1 will try to form consensus done := make(chan bool) go func() { - alloc1.Allocate("somecontainer", nil) + alloc1.Allocate("somecontainer", subnet, nil) done <- true }() time.Sleep(100 * time.Millisecond) @@ -124,19 +137,16 @@ func TestAllocatorClaim(t *testing.T) { testAddr1 = "10.0.3.1" // first address allocated should be .1 because .0 is network addr ) - alloc := makeAllocatorWithMockGossip(t, "01:00:00:01:00:00", universe, 1) + alloc, subnet := makeAllocatorWithMockGossip(t, "01:00:00:01:00:00", universe, 1) defer alloc.Stop() alloc.claimRingForTesting() - addr1, _ := alloc.Allocate(container1, nil) - alloc.Allocate(container2, nil) + addr1, _ := address.ParseIP(testAddr1) - // Now free the first one, and try to claim it - wt.AssertSuccess(t, alloc.Free(container1)) - t.Log(alloc) err := alloc.Claim(container3, addr1, nil) wt.AssertNoErr(t, err) - addr3, _ := alloc.Allocate(container3, nil) + // Check we get this address back if we try an allocate + addr3, _ := alloc.Allocate(container3, subnet, nil) wt.AssertEqualString(t, addr3.String(), testAddr1, "address") } @@ -160,10 +170,10 @@ func TestCancel(t *testing.T) { router := TestGossipRouter{make(map[router.PeerName]chan gossipMessage), 0.0} - alloc1 := makeAllocator("01:00:00:02:00:00", CIDR, 2) + alloc1, subnet := makeAllocator("01:00:00:02:00:00", CIDR, 2) alloc1.SetInterfaces(router.connect(alloc1.ourName, alloc1)) - alloc2 := makeAllocator("02:00:00:02:00:00", CIDR, 2) + alloc2, _ := makeAllocator("02:00:00:02:00:00", CIDR, 2) alloc2.SetInterfaces(router.connect(alloc2.ourName, alloc2)) alloc1.claimRingForTesting(alloc1, alloc2) alloc2.claimRingForTesting(alloc1, alloc2) @@ -175,9 +185,9 @@ func TestCancel(t *testing.T) { alloc1.OnGossipBroadcast(alloc2.Encode()) // Get some IPs, so each allocator has some space - res1, _ := alloc1.Allocate("foo", nil) + res1, _ := alloc1.Allocate("foo", subnet, nil) common.Debug.Printf("res1 = %s", res1.String()) - res2, _ := alloc2.Allocate("bar", nil) + res2, _ := alloc2.Allocate("bar", subnet, nil) common.Debug.Printf("res2 = %s", res2.String()) if res1 == res2 { wt.Fatalf(t, "Error: got same ips!") @@ -188,13 +198,13 @@ func TestCancel(t *testing.T) { unpause := alloc2.pause() // Use up all the IPs that alloc1 owns, so the allocation after this will prompt a request to alloc2 - for i := 0; alloc1.NumFreeAddresses() > 0; i++ { - alloc1.Allocate(fmt.Sprintf("tmp%d", i), nil) + for i := 0; alloc1.NumFreeAddresses(subnet) > 0; i++ { + alloc1.Allocate(fmt.Sprintf("tmp%d", i), subnet, nil) } cancelChan := make(chan bool, 1) doneChan := make(chan bool) go func() { - _, ok := alloc1.Allocate("baz", cancelChan) + _, ok := alloc1.Allocate("baz", subnet, cancelChan) doneChan <- ok == nil }() @@ -214,19 +224,17 @@ func TestGossipShutdown(t *testing.T) { container1 = "abcdef" container2 = "baddf00d" universe = "10.0.3.0/30" - testAddr1 = "10.0.3.1" // first address allocated should be .1 because .0 is network addr ) - alloc := makeAllocatorWithMockGossip(t, "01:00:00:01:00:00", universe, 1) + alloc, subnet := makeAllocatorWithMockGossip(t, "01:00:00:01:00:00", universe, 1) defer alloc.Stop() alloc.claimRingForTesting() - addr1, _ := alloc.Allocate(container1, nil) - wt.AssertEqualString(t, addr1.String(), testAddr1, "address") + alloc.Allocate(container1, subnet, nil) alloc.Shutdown() - _, err := alloc.Allocate(container2, nil) // trying to allocate after shutdown should fail + _, err := alloc.Allocate(container2, subnet, nil) // trying to allocate after shutdown should fail wt.AssertFalse(t, err == nil, "no address") CheckAllExpectedMessagesSent(alloc) @@ -236,15 +244,15 @@ func TestTransfer(t *testing.T) { const ( cidr = "10.0.1.7/22" ) - allocs, router := makeNetworkOfAllocators(3, cidr) + allocs, router, subnet := makeNetworkOfAllocators(3, cidr) alloc1 := allocs[0] alloc2 := allocs[1] alloc3 := allocs[2] // This will be 'master' and get the first range - _, err := alloc2.Allocate("foo", nil) + _, err := alloc2.Allocate("foo", subnet, nil) wt.AssertTrue(t, err == nil, "Failed to get address") - _, err = alloc3.Allocate("bar", nil) + _, err = alloc3.Allocate("bar", subnet, nil) wt.AssertTrue(t, err == nil, "Failed to get address") router.GossipBroadcast(alloc2.Gossip()) @@ -256,9 +264,9 @@ func TestTransfer(t *testing.T) { wt.AssertSuccess(t, alloc1.AdminTakeoverRanges(alloc2.ourName.String())) wt.AssertSuccess(t, alloc1.AdminTakeoverRanges(alloc3.ourName.String())) - wt.AssertEquals(t, alloc1.NumFreeAddresses(), address.Offset(1022)) + wt.AssertEquals(t, alloc1.NumFreeAddresses(subnet), address.Offset(1022)) - _, err = alloc1.Allocate("foo", nil) + _, err = alloc1.Allocate("foo", subnet, nil) wt.AssertTrue(t, err == nil, "Failed to get address") alloc1.Stop() } @@ -268,14 +276,13 @@ func TestFakeRouterSimple(t *testing.T) { const ( cidr = "10.0.1.7/22" ) - allocs, _ := makeNetworkOfAllocators(2, cidr) + allocs, _, subnet := makeNetworkOfAllocators(2, cidr) defer stopNetworkOfAllocators(allocs) alloc1 := allocs[0] //alloc2 := allocs[1] - addr, _ := alloc1.Allocate("foo", nil) - println("Got addr", addr) + alloc1.Allocate("foo", subnet, nil) } func TestAllocatorFuzz(t *testing.T) { @@ -288,7 +295,7 @@ func TestAllocatorFuzz(t *testing.T) { concurrency = 10 cidr = "10.0.1.7/22" ) - allocs, _ := makeNetworkOfAllocators(nodes, cidr) + allocs, _, subnet := makeNetworkOfAllocators(nodes, cidr) defer stopNetworkOfAllocators(allocs) // Test state @@ -332,7 +339,7 @@ func TestAllocatorFuzz(t *testing.T) { allocIndex := rand.Int31n(nodes) alloc := allocs[allocIndex] //common.Info.Printf("Allocate: asking allocator %d", allocIndex) - addr, err := alloc.Allocate(name, nil) + addr, err := alloc.Allocate(name, subnet, nil) if err != nil { panic(fmt.Sprintf("Could not allocate addr")) @@ -377,7 +384,11 @@ func TestAllocatorFuzz(t *testing.T) { alloc := allocs[res.alloc] //common.Info.Printf("Freeing %s (%s) on allocator %d", res.name, addr, res.alloc) - wt.AssertSuccess(t, alloc.Free(res.name)) + oldAddr, err := address.ParseIP(addr) + if err != nil { + panic(err) + } + wt.AssertSuccess(t, alloc.Free(res.name, oldAddr)) } // Do a Allocate on an existing container & allocator @@ -394,7 +405,7 @@ func TestAllocatorFuzz(t *testing.T) { //common.Info.Printf("Asking for %s on allocator %d again", addr, res.alloc) - newAddr, _ := alloc.Allocate(res.name, nil) + newAddr, _ := alloc.Allocate(res.name, subnet, nil) oldAddr, _ := address.ParseIP(addr) if newAddr != oldAddr { panic(fmt.Sprintf("Got different address for repeat request for %s: %s != %s", res.name, newAddr, oldAddr)) @@ -453,9 +464,9 @@ func TestAllocatorFuzz(t *testing.T) { } func TestGossipSkew(t *testing.T) { - alloc1 := makeAllocatorWithMockGossip(t, "01:00:00:01:00:00", "10.0.1.0/22", 2) + alloc1, _ := makeAllocatorWithMockGossip(t, "01:00:00:01:00:00", "10.0.1.0/22", 2) defer alloc1.Stop() - alloc2 := makeAllocatorWithMockGossip(t, "02:00:00:02:00:00", "10.0.1.0/22", 2) + alloc2, _ := makeAllocatorWithMockGossip(t, "02:00:00:02:00:00", "10.0.1.0/22", 2) alloc2.now = func() time.Time { return time.Now().Add(time.Hour * 2) } defer alloc2.Stop() diff --git a/ipam/claim.go b/ipam/claim.go index 18e84016..8d96e6e2 100644 --- a/ipam/claim.go +++ b/ipam/claim.go @@ -9,9 +9,9 @@ import ( type claim struct { resultChan chan<- error - hasBeenCancelled func() bool ident string addr address.Address + hasBeenCancelled func() bool } // Try returns true for success (or failure), false if we need to try again later @@ -57,6 +57,7 @@ func (c *claim) Try(alloc *Allocator) bool { c.resultChan <- err return true } + alloc.debugln("Claimed", c.addr, "for", c.ident) alloc.addOwned(c.ident, c.addr) c.resultChan <- nil return true diff --git a/ipam/http.go b/ipam/http.go index 4e7cd907..a9de0988 100644 --- a/ipam/http.go +++ b/ipam/http.go @@ -16,7 +16,7 @@ func badRequest(w http.ResponseWriter, err error) { } // HandleHTTP wires up ipams HTTP endpoints to the provided mux. -func (alloc *Allocator) HandleHTTP(router *mux.Router) { +func (alloc *Allocator) HandleHTTP(router *mux.Router, defaultSubnet address.CIDR) { router.Methods("PUT").Path("/ip/{id}/{ip}").HandlerFunc(func(w http.ResponseWriter, r *http.Request) { closedChan := w.(http.CloseNotifier).CloseNotify() vars := mux.Vars(r) @@ -33,30 +33,84 @@ func (alloc *Allocator) HandleHTTP(router *mux.Router) { w.WriteHeader(204) }) - router.Methods("GET").Path("/ip/{id}").HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - addr, err := alloc.Lookup(mux.Vars(r)["id"]) + router.Methods("GET").Path("/ip/{id}/{ip}/{prefixlen}").HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + cidr := vars["ip"] + "/" + vars["prefixlen"] + _, subnet, err := address.ParseCIDR(cidr) + if err != nil { + badRequest(w, err) + return + } + addr, err := alloc.Lookup(vars["id"], subnet.HostRange()) if err != nil { http.NotFound(w, r) return } - fmt.Fprintf(w, "%s/%d", addr.String(), alloc.prefixLen) + fmt.Fprintf(w, "%s/%d", addr, subnet.PrefixLen) + }) + + router.Methods("GET").Path("/ip/{id}").HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + addr, err := alloc.Lookup(mux.Vars(r)["id"], defaultSubnet.HostRange()) + if err != nil { + http.NotFound(w, r) + return + } + fmt.Fprintf(w, "%s/%d", addr, defaultSubnet.PrefixLen) + }) + + router.Methods("POST").Path("/ip/{id}/{ip}/{prefixlen}").HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + closedChan := w.(http.CloseNotifier).CloseNotify() + vars := mux.Vars(r) + ident := vars["id"] + cidrStr := vars["ip"] + "/" + vars["prefixlen"] + subnetAddr, cidr, err := address.ParseCIDR(cidrStr) + if err != nil { + badRequest(w, err) + return + } + if cidr.Start != subnetAddr { + badRequest(w, fmt.Errorf("Invalid subnet %s - bits after network prefix are not all zero", cidrStr)) + return + } + addr, err := alloc.Allocate(ident, cidr.HostRange(), closedChan) + if err != nil { + badRequest(w, fmt.Errorf("Unable to allocate: %s", err)) + return + } + + fmt.Fprintf(w, "%s/%d", addr, cidr.PrefixLen) }) router.Methods("POST").Path("/ip/{id}").HandlerFunc(func(w http.ResponseWriter, r *http.Request) { closedChan := w.(http.CloseNotifier).CloseNotify() ident := mux.Vars(r)["id"] - newAddr, err := alloc.Allocate(ident, closedChan) + newAddr, err := alloc.Allocate(ident, defaultSubnet.HostRange(), closedChan) if err != nil { badRequest(w, err) return } - fmt.Fprintf(w, "%s/%d", newAddr.String(), alloc.prefixLen) + fmt.Fprintf(w, "%s/%d", newAddr, defaultSubnet.PrefixLen) + }) + + router.Methods("DELETE").Path("/ip/{id}/{ip}").HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + ident := vars["id"] + ipStr := vars["ip"] + if ip, err := address.ParseIP(ipStr); err != nil { + badRequest(w, err) + return + } else if err := alloc.Free(ident, ip); err != nil { + badRequest(w, fmt.Errorf("Unable to free: %s", err)) + return + } + + w.WriteHeader(204) }) router.Methods("DELETE").Path("/ip/{id}").HandlerFunc(func(w http.ResponseWriter, r *http.Request) { ident := mux.Vars(r)["id"] - if err := alloc.Free(ident); err != nil { + if err := alloc.Delete(ident); err != nil { badRequest(w, err) return } diff --git a/ipam/http_test.go b/ipam/http_test.go index f1a828d1..b93360c6 100644 --- a/ipam/http_test.go +++ b/ipam/http_test.go @@ -13,6 +13,7 @@ import ( "github.com/gorilla/mux" "github.com/weaveworks/weave/common" + "github.com/weaveworks/weave/ipam/address" wt "github.com/weaveworks/weave/testing" ) @@ -38,12 +39,12 @@ func doHTTP(method string, url string) (resp *http.Response, err error) { return http.DefaultClient.Do(req) } -func listenHTTP(alloc *Allocator) int { +func listenHTTP(alloc *Allocator, subnet address.CIDR) int { router := mux.NewRouter() router.HandleFunc("/status", func(w http.ResponseWriter, r *http.Request) { io.WriteString(w, fmt.Sprintln(alloc)) }) - alloc.HandleHTTP(router) + alloc.HandleHTTP(router, subnet) httpListener, err := net.Listen("tcp", ":0") if err != nil { @@ -59,45 +60,55 @@ func listenHTTP(alloc *Allocator) int { return httpListener.Addr().(*net.TCPAddr).Port } -func allocURL(port int, containerID string) string { +func identURL(port int, containerID string) string { return fmt.Sprintf("http://localhost:%d/ip/%s", port, containerID) } +func allocURL(port int, cidr string, containerID string) string { + return fmt.Sprintf("http://localhost:%d/ip/%s/%s", port, containerID, cidr) +} + func TestHttp(t *testing.T) { var ( containerID = "deadbeef" container2 = "baddf00d" container3 = "b01df00d" - testAddr1 = "10.0.3.9" - netSuffix = "/29" - testCIDR1 = "10.0.3.8" + netSuffix + universe = "10.0.0.0/8" + testCIDR1 = "10.0.3.8/29" + testCIDR2 = "10.2.0.0/16" + testAddr1 = "10.0.3.9/29" + testAddr2 = "10.2.0.1/16" ) - alloc := makeAllocatorWithMockGossip(t, "08:00:27:01:c3:9a", testCIDR1, 1) - port := listenHTTP(alloc) - + alloc, _ := makeAllocatorWithMockGossip(t, "08:00:27:01:c3:9a", universe, 1) + _, cidr, _ := address.ParseCIDR(universe) + port := listenHTTP(alloc, cidr) alloc.claimRingForTesting() - // Ask the http server for a new address - cidr1 := HTTPPost(t, allocURL(port, containerID)) - wt.AssertEqualString(t, cidr1, testAddr1+netSuffix, "address") - check := HTTPGet(t, allocURL(port, containerID)) - wt.AssertEqualString(t, check, cidr1, "address") - // Ask the http server for another address and check it's different - cidr2 := HTTPPost(t, allocURL(port, container2)) - wt.AssertFalse(t, cidr2 == testAddr1+netSuffix, "address") + // Allocate an address in each subnet, and check we got what we expected + cidr1a := HTTPPost(t, allocURL(port, testCIDR1, containerID)) + wt.AssertEqualString(t, cidr1a, testAddr1, "address") + cidr2a := HTTPPost(t, allocURL(port, testCIDR2, containerID)) + wt.AssertEqualString(t, cidr2a, testAddr2, "address") + // Now, make the same requests again to check the operation is idempotent + check := HTTPGet(t, allocURL(port, testCIDR1, containerID)) + wt.AssertEqualString(t, check, cidr1a, "address") + check = HTTPGet(t, allocURL(port, testCIDR2, containerID)) + wt.AssertEqualString(t, check, cidr2a, "address") - // Ask for the first container again and we should get the same address again - cidr1a := HTTPPost(t, allocURL(port, containerID)) - wt.AssertEqualString(t, cidr1a, testAddr1+netSuffix, "address") - // Now free the first one, and we should get it back when we ask - doHTTP("DELETE", allocURL(port, containerID)) - resp, err := doHTTP("GET", allocURL(port, containerID)) - wt.AssertNoErr(t, err) - wt.AssertStatus(t, resp.StatusCode, http.StatusNotFound, "http response") + // Ask the http server for a pair of addresses for another container and check they're different + cidr1b := HTTPPost(t, allocURL(port, testCIDR1, container2)) + wt.AssertFalse(t, cidr1b == testAddr1, "address") + cidr2b := HTTPPost(t, allocURL(port, testCIDR2, container2)) + wt.AssertFalse(t, cidr2b == testAddr2, "address") - cidr3 := HTTPPost(t, allocURL(port, container3)) - wt.AssertEqualString(t, cidr3, testAddr1+netSuffix, "address") + // Now free the first container, and we should get its addresses back when we ask + doHTTP("DELETE", identURL(port, containerID)) + + cidr1c := HTTPPost(t, allocURL(port, testCIDR1, container3)) + wt.AssertEqualString(t, cidr1c, testAddr1, "address") + cidr2c := HTTPPost(t, allocURL(port, testCIDR2, container3)) + wt.AssertEqualString(t, cidr2c, testAddr2, "address") // Would like to shut down the http server at the end of this test // but it's complicated. @@ -110,12 +121,13 @@ func TestBadHttp(t *testing.T) { testCIDR1 = "10.0.0.0/8" ) - alloc := makeAllocatorWithMockGossip(t, "08:00:27:01:c3:9a", testCIDR1, 1) + alloc, _ := makeAllocatorWithMockGossip(t, "08:00:27:01:c3:9a", testCIDR1, 1) defer alloc.Stop() - port := listenHTTP(alloc) + _, cidr, _ := address.ParseCIDR(testCIDR1) + port := listenHTTP(alloc, cidr) alloc.claimRingForTesting() - cidr1 := HTTPPost(t, allocURL(port, containerID)) + cidr1 := HTTPPost(t, allocURL(port, testCIDR1, containerID)) parts := strings.Split(cidr1, "/") testAddr1 := parts[0] // Verb that's not handled @@ -142,20 +154,21 @@ func impTestHTTPCancel(t *testing.T) { common.InitDefaultLogging(true) var ( containerID = "deadbeef" - testCIDR1 = "10.0.3.5/29" + testCIDR1 = "10.0.3.0/29" ) - alloc := makeAllocatorWithMockGossip(t, "08:00:27:01:c3:9a", testCIDR1, 1) + alloc, _ := makeAllocatorWithMockGossip(t, "08:00:27:01:c3:9a", testCIDR1, 1) defer alloc.Stop() alloc.claimRingForTesting() - port := listenHTTP(alloc) + _, cidr, _ := address.ParseCIDR(testCIDR1) + port := listenHTTP(alloc, cidr) // Stop the alloc so nothing actually works unpause := alloc.pause() // Ask the http server for a new address done := make(chan *http.Response) - req, _ := http.NewRequest("POST", allocURL(port, containerID), nil) + req, _ := http.NewRequest("POST", allocURL(port, testCIDR1, containerID), nil) go func() { res, _ := http.DefaultClient.Do(req) done <- res diff --git a/ipam/ring/ring.go b/ipam/ring/ring.go index ddb2f631..fcdb362e 100644 --- a/ipam/ring/ring.go +++ b/ipam/ring/ring.go @@ -91,18 +91,6 @@ func New(start, end address.Address, peer router.PeerName) *Ring { return ring } -// TotalRemoteFree returns the approximate number of free IPs -// on other hosts. -func (r *Ring) TotalRemoteFree() address.Offset { - result := address.Offset(0) - for _, entry := range r.Entries { - if entry.Peer != r.Peer { - result += entry.Free - } - } - return result -} - // Returns the distance between two tokens on this ring, dealing // with ranges which cross the origin func (r *Ring) distance(start, end address.Address) address.Offset { @@ -341,10 +329,6 @@ func (r *Ring) ClaimForPeers(peers []router.PeerName) { common.Assert(pos == r.End) } -func (r *Ring) ClaimItAll() { - r.ClaimForPeers([]router.PeerName{r.Peer}) -} - func (r *Ring) FprintWithNicknames(w io.Writer, m map[router.PeerName]string) { for _, entry := range r.Entries { nickname, found := m[entry.Peer] @@ -409,15 +393,22 @@ func (r *Ring) ReportFree(freespace map[address.Address]address.Offset) { } // ChoosePeerToAskForSpace chooses a weighted-random peer to ask -// for space. -func (r *Ring) ChoosePeerToAskForSpace() (result router.PeerName, err error) { +// for space in the range [start, end). Assumes start= end { + break + } // Ignore ranges with no free space if entry.Free <= 0 { continue diff --git a/ipam/ring/ring_test.go b/ipam/ring/ring_test.go index f3ec1e0f..ce63a38e 100644 --- a/ipam/ring/ring_test.go +++ b/ipam/ring/ring_test.go @@ -345,34 +345,40 @@ func TestGossip(t *testing.T) { func TestFindFree(t *testing.T) { ring1 := New(start, end, peer1name) - _, err := ring1.ChoosePeerToAskForSpace() + _, err := ring1.ChoosePeerToAskForSpace(start, end) wt.AssertTrue(t, err == ErrNoFreeSpace, "Expected ErrNoFreeSpace") ring1.Entries = []*entry{{Token: start, Peer: peer1name}} - _, err = ring1.ChoosePeerToAskForSpace() + _, err = ring1.ChoosePeerToAskForSpace(start, end) wt.AssertTrue(t, err == ErrNoFreeSpace, "Expected ErrNoFreeSpace") // We shouldn't return outselves ring1.ReportFree(map[address.Address]address.Offset{start: 10}) - _, err = ring1.ChoosePeerToAskForSpace() + _, err = ring1.ChoosePeerToAskForSpace(start, end) wt.AssertTrue(t, err == ErrNoFreeSpace, "Expected ErrNoFreeSpace") ring1.Entries = []*entry{{Token: start, Peer: peer1name, Free: 1}, - {Token: start, Peer: peer1name, Free: 1}} - _, err = ring1.ChoosePeerToAskForSpace() + {Token: middle, Peer: peer1name, Free: 1}} + _, err = ring1.ChoosePeerToAskForSpace(start, end) wt.AssertTrue(t, err == ErrNoFreeSpace, "Expected ErrNoFreeSpace") + ring1.assertInvariants() // We should return others ring1.Entries = []*entry{{Token: start, Peer: peer2name, Free: 1}} - peer, err := ring1.ChoosePeerToAskForSpace() + peer, err := ring1.ChoosePeerToAskForSpace(start, end) wt.AssertSuccess(t, err) wt.AssertEquals(t, peer, peer2name) ring1.Entries = []*entry{{Token: start, Peer: peer2name, Free: 1}, - {Token: start, Peer: peer2name, Free: 1}} - peer, err = ring1.ChoosePeerToAskForSpace() + {Token: middle, Peer: peer3name, Free: 1}} + peer, err = ring1.ChoosePeerToAskForSpace(start, middle) wt.AssertSuccess(t, err) wt.AssertEquals(t, peer, peer2name) + + peer, err = ring1.ChoosePeerToAskForSpace(middle, end) + wt.AssertSuccess(t, err) + wt.AssertEquals(t, peer, peer3name) + ring1.assertInvariants() } func TestReportFree(t *testing.T) { @@ -710,6 +716,10 @@ func TestFuzzRingHard(t *testing.T) { } } +func (r *Ring) ClaimItAll() { + r.ClaimForPeers([]router.PeerName{r.Peer}) +} + func (es entries) String() string { var buffer bytes.Buffer fmt.Fprintf(&buffer, "[") diff --git a/ipam/space/space.go b/ipam/space/space.go index 548d9e55..1d912370 100644 --- a/ipam/space/space.go +++ b/ipam/space/space.go @@ -9,8 +9,6 @@ import ( "github.com/weaveworks/weave/ipam/address" ) -type Addr address.Address - type Space struct { // ours and free represent a set of addresses as a sorted // sequences of ranges. Even elements give the inclusive @@ -37,14 +35,47 @@ func (s *Space) Clear() { s.ours = s.ours[:0] } -func (s *Space) Allocate() (bool, address.Address) { - if len(s.free) == 0 { - return false, 0 +// Walk down the free list calling f() on the in-range portions, until +// f() returns true or we run out of free space. Return true iff f() returned true +func (s *Space) walkFree(r address.Range, f func(address.Range) bool) bool { + if r.Start >= r.End { // degenerate case + return false } - res := s.free[0] - s.ours = add(s.ours, res, res+1) - s.free = subtract(s.free, res, res+1) - return true, res + for i := 0; i < len(s.free); i += 2 { + chunk := address.Range{Start: s.free[i], End: s.free[i+1]} + if chunk.End <= r.Start { // this chunk comes before the range + continue + } + if chunk.Start >= r.End { + // all remaining free space is completely after range + break + } + // at this point we know chunk.End>chunk.Start && + // chunk.End>r.Start && r.End>chunk.Start && r.End>r.Start + // therefore max(start, r.Start) < min(end, r.End) + // Restrict this block of free space to be in range + if chunk.Start < r.Start { + chunk.Start = r.Start + } + if chunk.End > r.End { + chunk.End = r.End + } + // at this point we know start end { - e = end - } - if s >= e { - continue - } - res += address.Subtract(e, s) - } + s.walkFree(r, func(chunk address.Range) bool { + res += chunk.Size() + return false + }) return res } @@ -96,35 +110,32 @@ func (s *Space) Free(addr address.Address) error { return nil } -func (s *Space) biggestFreeRange() (int, address.Offset) { - pos := -1 - biggest := address.Offset(0) - - for i := 0; i < len(s.free); i += 2 { - size := address.Subtract(s.free[i+1], s.free[i]) - if size >= biggest { - pos = i - biggest = size +func (s *Space) biggestFreeRange(r address.Range) (biggest address.Range) { + biggestSize := address.Offset(0) + s.walkFree(r, func(chunk address.Range) bool { + if size := chunk.Size(); size >= biggestSize { + biggest = chunk + biggestSize = size } - } - return pos, biggest + return false + }) + return } -func (s *Space) Donate() (address.Address, address.Offset, bool) { - if len(s.free) == 0 { - return 0, 0, false +func (s *Space) Donate(r address.Range) (address.Range, bool) { + biggest := s.biggestFreeRange(r) + + if biggest.Size() == 0 { + return address.Range{}, false } - pos, biggest := s.biggestFreeRange() + // Donate half of that biggest free range. Note size/2 rounds down, so + // the resulting donation size rounds up, and in particular can't be empty. + biggest.Start = address.Add(biggest.Start, biggest.Size()/2) - // Donate half of that biggest free range, rounding up so - // that the donation can't be empty - end := s.free[pos+1] - start := end - address.Address((biggest+1)/2) - - s.ours = subtract(s.ours, start, end) - s.free = subtract(s.free, start, end) - return start, address.Subtract(end, start), true + s.ours = subtract(s.ours, biggest.Start, biggest.End) + s.free = subtract(s.free, biggest.Start, biggest.End) + return biggest, true } func firstGreater(a []address.Address, x address.Address) int { diff --git a/ipam/space/space_test.go b/ipam/space/space_test.go index d49324b7..46a135c5 100644 --- a/ipam/space/space_test.go +++ b/ipam/space/space_test.go @@ -1,7 +1,6 @@ package space import ( - "reflect" "testing" "github.com/weaveworks/weave/ipam/address" @@ -19,6 +18,16 @@ func ip(s string) address.Address { return addr } +// Helper function to avoid 'NumFreeAddressesInRange(start, end)' +// dozens of times in tests +func (s *Space) NumFreeAddresses() address.Offset { + res := address.Offset(0) + for i := 0; i < len(s.free); i += 2 { + res += address.Subtract(s.free[i+1], s.free[i]) + } + return res +} + func TestLowlevel(t *testing.T) { a := []address.Address{} a = add(a, 100, 200) @@ -52,12 +61,12 @@ func TestLowlevel(t *testing.T) { s := New() wt.AssertEquals(t, s.NumFreeAddresses(), address.Offset(0)) - ok, got := s.Allocate() + ok, got := s.Allocate(address.NewRange(0, 1000)) wt.AssertFalse(t, ok, "allocate in empty space should fail") s.Add(100, 100) wt.AssertEquals(t, s.NumFreeAddresses(), address.Offset(100)) - ok, got = s.Allocate() + ok, got = s.Allocate(address.NewRange(0, 1000)) wt.AssertTrue(t, ok && got == 100, "allocate") wt.AssertEquals(t, s.NumFreeAddresses(), address.Offset(99)) wt.AssertNoErr(t, s.Claim(150)) @@ -67,44 +76,49 @@ func TestLowlevel(t *testing.T) { wt.AssertErrorInterface(t, s.Free(0), (*error)(nil), "free not allocated") wt.AssertErrorInterface(t, s.Free(100), (*error)(nil), "double free") - start, size, ok := s.Donate() - wt.AssertTrue(t, ok && start == 125 && size == 25, "donate") + r, ok := s.Donate(address.NewRange(0, 1000)) + wt.AssertTrue(t, ok && r.Start == 125 && r.Size() == 25, "donate") // test Donate when addresses are scarce s = New() - start, size, ok = s.Donate() + r, ok = s.Donate(address.NewRange(0, 1000)) wt.AssertTrue(t, !ok, "donate on empty space should fail") s.Add(0, 3) wt.AssertNoErr(t, s.Claim(0)) wt.AssertNoErr(t, s.Claim(2)) - start, size, ok = s.Donate() - wt.AssertTrue(t, ok && start == 1 && size == 1, "donate") - start, size, ok = s.Donate() + r, ok = s.Donate(address.NewRange(0, 1000)) + wt.AssertTrue(t, ok && r.Start == 1 && r.End == 2, "donate") + r, ok = s.Donate(address.NewRange(0, 1000)) wt.AssertTrue(t, !ok, "donate should fail") } func TestSpaceAllocate(t *testing.T) { const ( testAddr1 = "10.0.3.4" + testAddr2 = "10.0.3.5" testAddrx = "10.0.3.19" testAddry = "10.0.9.19" containerID = "deadbeef" + size = 20 + ) + var ( + start = ip(testAddr1) ) - space1 := makeSpace(ip(testAddr1), 20) + space1 := makeSpace(start, size) wt.AssertEquals(t, space1.NumFreeAddresses(), address.Offset(20)) space1.assertInvariants() - _, addr1 := space1.Allocate() + _, addr1 := space1.Allocate(address.NewRange(start, size)) wt.AssertEqualString(t, addr1.String(), testAddr1, "address") wt.AssertEquals(t, space1.NumFreeAddresses(), address.Offset(19)) space1.assertInvariants() - _, addr2 := space1.Allocate() + _, addr2 := space1.Allocate(address.NewRange(start, size)) wt.AssertFalse(t, addr2.String() == testAddr1, "address") wt.AssertEquals(t, space1.NumFreeAddresses(), address.Offset(18)) - wt.AssertEquals(t, space1.NumFreeAddressesInRange(ip(testAddr1), ip(testAddrx)), address.Offset(13)) - wt.AssertEquals(t, space1.NumFreeAddressesInRange(ip(testAddr1), ip(testAddry)), address.Offset(18)) + wt.AssertEquals(t, space1.NumFreeAddressesInRange(address.Range{Start: ip(testAddr1), End: ip(testAddrx)}), address.Offset(13)) + wt.AssertEquals(t, space1.NumFreeAddressesInRange(address.Range{Start: ip(testAddr1), End: ip(testAddry)}), address.Offset(18)) space1.assertInvariants() space1.Free(addr2) @@ -125,32 +139,33 @@ func TestSpaceFree(t *testing.T) { containerID = "deadbeef" ) + entireRange := address.NewRange(ip(testAddr1), 20) space := makeSpace(ip(testAddr1), 20) // Check we are prepared to give up the entire space - pos, size := space.biggestFreeRange() - wt.AssertTrue(t, space.free[pos] == ip(testAddr1) && size == 20, "Wrong space") + r := space.biggestFreeRange(entireRange) + wt.AssertTrue(t, r.Start == ip(testAddr1) && r.Size() == 20, "Wrong space") for i := 0; i < 20; i++ { - ok, _ := space.Allocate() + ok, _ := space.Allocate(entireRange) wt.AssertTrue(t, ok, "Failed to get address") } // Check we are full - ok, _ := space.Allocate() + ok, _ := space.Allocate(entireRange) wt.AssertTrue(t, !ok, "Should have failed to get address") - _, size, ok = space.Donate() - wt.AssertTrue(t, size == 0, "Wrong space") + r, ok = space.Donate(entireRange) + wt.AssertTrue(t, r.Size() == 0, "Wrong space") // Free in the middle wt.AssertSuccess(t, space.Free(ip("10.0.3.13"))) - pos, size = space.biggestFreeRange() - wt.AssertTrue(t, space.free[pos] == ip("10.0.3.13") && size == 1, "Wrong space") + r = space.biggestFreeRange(entireRange) + wt.AssertTrue(t, r.Start == ip("10.0.3.13") && r.Size() == 1, "Wrong space") // Free one at the end wt.AssertSuccess(t, space.Free(ip("10.0.3.23"))) - pos, size = space.biggestFreeRange() - wt.AssertTrue(t, space.free[pos] == ip("10.0.3.23") && size == 1, "Wrong space") + r = space.biggestFreeRange(entireRange) + wt.AssertTrue(t, r.Start == ip("10.0.3.23") && r.Size() == 1, "Wrong space") // Now free a few at the end wt.AssertSuccess(t, space.Free(ip("10.0.3.22"))) @@ -159,8 +174,8 @@ func TestSpaceFree(t *testing.T) { wt.AssertEquals(t, space.NumFreeAddresses(), address.Offset(4)) // Now get the biggest free space; should be 3.21 - pos, size = space.biggestFreeRange() - wt.AssertTrue(t, space.free[pos] == ip("10.0.3.21") && size == 3, "Wrong space") + r = space.biggestFreeRange(entireRange) + wt.AssertTrue(t, r.Start == ip("10.0.3.21") && r.Size() == 3, "Wrong space") // Now free a few in the middle wt.AssertSuccess(t, space.Free(ip("10.0.3.12"))) @@ -170,46 +185,44 @@ func TestSpaceFree(t *testing.T) { wt.AssertEquals(t, space.NumFreeAddresses(), address.Offset(7)) // Now get the biggest free space; should be 3.21 - pos, size = space.biggestFreeRange() - wt.AssertTrue(t, space.free[pos] == ip("10.0.3.10") && size == 4, "Wrong space") + r = space.biggestFreeRange(entireRange) + wt.AssertTrue(t, r.Start == ip("10.0.3.10") && r.Size() == 4, "Wrong space") wt.AssertEquals(t, space.OwnedRanges(), []address.Range{address.Range{Start: ip("10.0.3.4"), End: ip("10.0.3.24")}}) } -func (s1 *Space) Equal(s2 *Space) bool { - return reflect.DeepEqual(s1.ours, s2.ours) && reflect.DeepEqual(s1.free, s2.free) -} - func TestDonateSimple(t *testing.T) { const ( testAddr1 = "10.0.1.0" testAddr2 = "10.0.1.32" + size = 48 ) var ( ipAddr1 = ip(testAddr1) ) - ps1 := makeSpace(ipAddr1, 48) + ps1 := makeSpace(ipAddr1, size) // Empty space set should split in two and give me the second half - start, numGivenUp, ok := ps1.Donate() + r, ok := ps1.Donate(address.NewRange(ip(testAddr1), size)) + numGivenUp := r.Size() wt.AssertTrue(t, ok, "Donate result") - wt.AssertEqualString(t, start.String(), "10.0.1.24", "Invalid start") - wt.AssertEquals(t, numGivenUp, address.Offset(24)) - wt.AssertEquals(t, ps1.NumFreeAddresses(), address.Offset(24)) + wt.AssertEqualString(t, r.Start.String(), "10.0.1.24", "Invalid start") + wt.AssertEquals(t, numGivenUp, address.Offset(size/2)) + wt.AssertEquals(t, ps1.NumFreeAddresses(), address.Offset(size/2)) // Now check we can give the rest up. count := 0 // count to avoid infinite loop for ; count < 1000; count++ { - _, size, ok := ps1.Donate() + r, ok := ps1.Donate(address.NewRange(ip(testAddr1), size)) if !ok { break } - numGivenUp += size + numGivenUp += r.Size() } wt.AssertEquals(t, ps1.NumFreeAddresses(), address.Offset(0)) - wt.AssertEquals(t, numGivenUp, address.Offset(48)) + wt.AssertEquals(t, numGivenUp, address.Offset(size)) } func TestDonateHard(t *testing.T) { @@ -222,7 +235,7 @@ func TestDonateHard(t *testing.T) { // Fill a fresh space spaceset := makeSpace(start, size) for i := address.Offset(0); i < size; i++ { - ok, _ := spaceset.Allocate() + ok, _ := spaceset.Allocate(address.NewRange(start, size)) wt.AssertTrue(t, ok, "Failed to get IP!") } @@ -235,10 +248,10 @@ func TestDonateHard(t *testing.T) { } // Now split - newRange, numGivenUp, ok := spaceset.Donate() + newRange, ok := spaceset.Donate(address.NewRange(start, size)) wt.AssertTrue(t, ok, "GiveUpSpace result") - wt.AssertEquals(t, newRange, ip("10.0.1.23")) - wt.AssertEquals(t, numGivenUp, address.Offset(24)) + wt.AssertEquals(t, newRange.Start, ip("10.0.1.23")) + wt.AssertEquals(t, newRange.Size(), address.Offset(24)) wt.AssertEquals(t, spaceset.NumFreeAddresses(), address.Offset(23)) //Space set should now have 2 spaces diff --git a/ipam/testutils_test.go b/ipam/testutils_test.go index c47273cb..6fd369c7 100644 --- a/ipam/testutils_test.go +++ b/ipam/testutils_test.go @@ -128,27 +128,29 @@ func CheckAllExpectedMessagesSent(allocs ...*Allocator) { } } -func makeAllocator(name string, cidr string, quorum uint) *Allocator { +func makeAllocator(name string, cidrStr string, quorum uint) (*Allocator, address.Range) { peername, err := router.PeerNameFromString(name) if err != nil { panic(err) } - alloc, err := NewAllocator(peername, router.PeerUID(rand.Int63()), - "nick-"+name, cidr, quorum) + _, cidr, err := address.ParseCIDR(cidrStr) if err != nil { panic(err) } - return alloc + alloc := NewAllocator(peername, router.PeerUID(rand.Int63()), + "nick-"+name, cidr.Range(), quorum) + + return alloc, cidr.HostRange() } -func makeAllocatorWithMockGossip(t *testing.T, name string, universeCIDR string, quorum uint) *Allocator { - alloc := makeAllocator(name, universeCIDR, quorum) +func makeAllocatorWithMockGossip(t *testing.T, name string, universeCIDR string, quorum uint) (*Allocator, address.Range) { + alloc, subnet := makeAllocator(name, universeCIDR, quorum) gossip := &mockGossipComms{t: t, name: name} alloc.SetInterfaces(gossip) alloc.Start() - return alloc + return alloc, subnet } func (alloc *Allocator) claimRingForTesting(allocs ...*Allocator) { @@ -160,10 +162,10 @@ func (alloc *Allocator) claimRingForTesting(allocs ...*Allocator) { alloc.space.AddRanges(alloc.ring.OwnedRanges()) } -func (alloc *Allocator) NumFreeAddresses() address.Offset { +func (alloc *Allocator) NumFreeAddresses(r address.Range) address.Offset { resultChan := make(chan address.Offset) alloc.actionChan <- func() { - resultChan <- alloc.space.NumFreeAddresses() + resultChan <- alloc.space.NumFreeAddressesInRange(r) } return <-resultChan } @@ -285,13 +287,15 @@ func (client TestGossipRouterClient) GossipBroadcast(update router.GossipData) e return client.router.GossipBroadcast(update) } -func makeNetworkOfAllocators(size int, cidr string) ([]*Allocator, TestGossipRouter) { +func makeNetworkOfAllocators(size int, cidr string) ([]*Allocator, TestGossipRouter, address.Range) { gossipRouter := TestGossipRouter{make(map[router.PeerName]chan gossipMessage), 0.0} allocs := make([]*Allocator, size) + var subnet address.Range for i := 0; i < size; i++ { - alloc := makeAllocator(fmt.Sprintf("%02d:00:00:02:00:00", i), + var alloc *Allocator + alloc, subnet = makeAllocator(fmt.Sprintf("%02d:00:00:02:00:00", i), cidr, uint(size/2+1)) alloc.SetInterfaces(gossipRouter.connect(alloc.ourName, alloc)) alloc.Start() @@ -300,7 +304,7 @@ func makeNetworkOfAllocators(size int, cidr string) ([]*Allocator, TestGossipRou gossipRouter.GossipBroadcast(allocs[size-1].Gossip()) time.Sleep(1000 * time.Millisecond) - return allocs, gossipRouter + return allocs, gossipRouter, subnet } func stopNetworkOfAllocators(allocs []*Allocator) { diff --git a/prog/weaver/main.go b/prog/weaver/main.go index e59b4202..0ce26cd3 100644 --- a/prog/weaver/main.go +++ b/prog/weaver/main.go @@ -17,6 +17,7 @@ import ( . "github.com/weaveworks/weave/common" "github.com/weaveworks/weave/common/updater" "github.com/weaveworks/weave/ipam" + "github.com/weaveworks/weave/ipam/address" weavenet "github.com/weaveworks/weave/net" weave "github.com/weaveworks/weave/router" ) @@ -37,23 +38,24 @@ func main() { runtime.GOMAXPROCS(procs) var ( - config weave.Config - justVersion bool - ifaceName string - routerName string - nickName string - password string - wait int - debug bool - pktdebug bool - prof string - bufSzMB int - noDiscovery bool - httpAddr string - iprangeCIDR string - peerCount int - apiPath string - peers []string + config weave.Config + justVersion bool + ifaceName string + routerName string + nickName string + password string + wait int + debug bool + pktdebug bool + prof string + bufSzMB int + noDiscovery bool + httpAddr string + iprangeCIDR string + ipsubnetCIDR string + peerCount int + apiPath string + peers []string ) flag.BoolVar(&justVersion, "version", false, "print version and exit") @@ -70,7 +72,8 @@ func main() { flag.BoolVar(&noDiscovery, "nodiscovery", false, "disable peer discovery") flag.IntVar(&bufSzMB, "bufsz", 8, "capture buffer size in MB") flag.StringVar(&httpAddr, "httpaddr", fmt.Sprintf(":%d", weave.HTTPPort), "address to bind HTTP interface to (disabled if blank, absolute path indicates unix domain socket)") - flag.StringVar(&iprangeCIDR, "iprange", "", "IP address range to allocate within, in CIDR notation") + flag.StringVar(&iprangeCIDR, "iprange", "", "IP address range reserved for automatic allocation, in CIDR notation") + flag.StringVar(&ipsubnetCIDR, "ipsubnet", "", "subnet to allocate within by default, in CIDR notation") flag.IntVar(&peerCount, "initpeercount", 0, "number of peers in network (for IP address allocation)") flag.StringVar(&apiPath, "api", "unix:///var/run/docker.sock", "Path to Docker API socket") flag.Parse() @@ -137,8 +140,9 @@ func main() { log.Println("Our name is", router.Ourself) var allocator *ipam.Allocator + var defaultSubnet address.CIDR if iprangeCIDR != "" { - allocator = createAllocator(router, apiPath, iprangeCIDR, determineQuorum(peerCount, peers)) + allocator, defaultSubnet = createAllocator(router, apiPath, iprangeCIDR, ipsubnetCIDR, determineQuorum(peerCount, peers)) } else if peerCount > 0 { log.Fatal("-initpeercount flag specified without -iprange") } else { @@ -154,7 +158,7 @@ func main() { // so there is no point in doing "weave launch -httpaddr ''". // This is here to support stand-alone use of weaver. if httpAddr != "" { - go handleHTTP(router, httpAddr, allocator) + go handleHTTP(router, httpAddr, allocator, defaultSubnet) } SignalHandlerLoop(router) @@ -194,18 +198,35 @@ func logFrameFunc(debug bool) weave.LogFrameFunc { } } -func createAllocator(router *weave.Router, apiPath string, iprangeCIDR string, quorum uint) *ipam.Allocator { - allocator, err := ipam.NewAllocator(router.Ourself.Peer.Name, router.Ourself.Peer.UID, router.Ourself.Peer.NickName, iprangeCIDR, quorum) +func parseAndCheckCIDR(cidrStr string) address.CIDR { + _, cidr, err := address.ParseCIDR(cidrStr) if err != nil { log.Fatal(err) } + if cidr.Size() < ipam.MinSubnetSize { + log.Fatalf("Allocation range smaller than minimum size %d: %s", ipam.MinSubnetSize, cidrStr) + } + return cidr +} + +func createAllocator(router *weave.Router, apiPath string, ipRangeStr string, defaultSubnetStr string, quorum uint) (*ipam.Allocator, address.CIDR) { + ipRange := parseAndCheckCIDR(ipRangeStr) + defaultSubnet := ipRange + if defaultSubnetStr != "" { + defaultSubnet = parseAndCheckCIDR(defaultSubnetStr) + if !ipRange.Range().Overlaps(defaultSubnet.Range()) { + log.Fatalf("Default subnet %s out of bounds: %s", defaultSubnet, ipRange) + } + } + allocator := ipam.NewAllocator(router.Ourself.Peer.Name, router.Ourself.Peer.UID, router.Ourself.Peer.NickName, ipRange.Range(), quorum) + allocator.SetInterfaces(router.NewGossip("IPallocation", allocator)) allocator.Start() - err = updater.Start(apiPath, allocator) + err := updater.Start(apiPath, allocator) if err != nil { log.Fatal("Unable to start watcher", err) } - return allocator + return allocator, defaultSubnet } // Pick a quorum size heuristically based on the number of peer @@ -228,11 +249,11 @@ func determineQuorum(initPeerCountFlag int, peers []string) uint { return quorum } -func handleHTTP(router *weave.Router, httpAddr string, allocator *ipam.Allocator) { +func handleHTTP(router *weave.Router, httpAddr string, allocator *ipam.Allocator, defaultSubnet address.CIDR) { muxRouter := mux.NewRouter() if allocator != nil { - allocator.HandleHTTP(muxRouter) + allocator.HandleHTTP(muxRouter, defaultSubnet) } muxRouter.Methods("GET").Path("/status").HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -240,6 +261,7 @@ func handleHTTP(router *weave.Router, httpAddr string, allocator *ipam.Allocator fmt.Fprintln(w, router.Status()) if allocator != nil { fmt.Fprintln(w, allocator.String()) + fmt.Fprintln(w, "Allocator default subnet:", defaultSubnet) } }) diff --git a/router/protocol.go b/router/protocol.go index 5e9c7036..1ffe5f90 100644 --- a/router/protocol.go +++ b/router/protocol.go @@ -2,7 +2,7 @@ package router const ( Protocol = "weave" - ProtocolVersion = 18 + ProtocolVersion = 20 ) type ProtocolTag byte diff --git a/site/ipam.md b/site/ipam.md index 836e980f..c2b4856f 100644 --- a/site/ipam.md +++ b/site/ipam.md @@ -9,10 +9,10 @@ Weave can automatically assign unique IP addresses to each container across the network. To make this work, weave must be told on startup what range of addresses to allocate from, for example: - host1# weave launch -iprange 10.2.3.0/24 + host1# weave launch -iprange 10.2.0.0/16 -The `run`, `start`, `attach`, `expose` and `hide` commands will then -fetch an address automatically if none is specified, i.e.: +The `run`, `start`, `attach`, `detach`, `expose` and `hide` commands +will then fetch an address automatically if none is specified, i.e.: host1# C=$(weave run -ti ubuntu) @@ -22,7 +22,40 @@ You can see which address was allocated with host1# weave ps $C a7aff7249393 7a:51:d1:09:21:78 10.2.3.1/24 -The `-iprange` parameter is given in [CIDR +You may wish to [run containers on different +subnets](features.html#application-isolation). To set a default +subnet, launch with `-ipsubnet`: + + host1# weave launch -iprange 10.2.0.0/16 -ipsubnet 10.2.3.0/24 + +`-iprange` should cover the entire range that you will ever use for +allocation, and `-ipsubnet` is the subnet that will be used when +you don't explicitly specify one. + +Then, to request the allocation of an address from a subnet other than +the default, use the `net:` prefix: + + host1# C=$(weave run net:10.2.7.0/24 -ti ubuntu) + +To explicitly request the default subnet, use `net:default`. + +And, you can ask for multiple addresses in different subnets and add +in manually-assigned addresses on the same line, for instance: + + host1# C=$(weave run net:10.2.7.0/24 net:10.2.8.0/24 ip:10.3.9.1/24 -ti ubuntu) + +Note that `-iprange` can be smaller than `-ipsubnet`, if you want to +use a mixture of automatically-allocated addresses and manually-chosen +addresses, and have the containers communicate with each other. For +example, if you say: + + host1# weave launch -iprange 10.9.0.0/17 -ipsubnet 10.9.0.0/16 + +then you can run all containers in the 10.9.0.0/16 subnet, with IPAM +using the lower half and leaving the upper half free for manual +allocation. + +The subnet parameters are written in [CIDR notation](http://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing) format - in this example "/24" means the first 24 bits of the address form the network address and the allocator is to allocate container @@ -117,12 +150,12 @@ reports on the current status of the weave router and IP allocator: ```` weave router git-8f675f15c0b5 ... -Allocator subnet 10.2.1.0/24 - Free IPs: ~98.0%, 62 local, ~189 remote +Allocator universe 10.2.0.0/16 Owned Ranges: 10.2.1.1 -> 96:e9:e2:2e:2d:bc (host1) (v3) 10.2.1.128 -> ea:84:25:9b:31:2e (host2) (v3) 10.2.1.192 -> ea:6c:21:09:cf:f0 (host3) (v9) +Allocator default subnet: 10.2.1.0/24 ```` The first section covers the router; see the [troubleshooting diff --git a/test/100_cross_hosts_test.sh b/test/100_cross_hosts_test.sh index 869be0ec..2266b9dd 100755 --- a/test/100_cross_hosts_test.sh +++ b/test/100_cross_hosts_test.sh @@ -4,20 +4,27 @@ C1=10.2.1.4 C2=10.2.1.7 -UNIVERSE=10.2.2.0/24 +UNIVERSE=10.2.0.0/16 +SUBNET_1=10.2.2.0/24 +SUBNET_2=10.2.3.0/24 start_suite "Ping over cross-host weave network (with and without IPAM)" -weave_on $HOST1 launch -iprange $UNIVERSE -weave_on $HOST2 launch -iprange $UNIVERSE $HOST1 +weave_on $HOST1 launch -iprange $UNIVERSE -ipsubnet $SUBNET_1 +weave_on $HOST2 launch -iprange $UNIVERSE -ipsubnet $SUBNET_1 $HOST1 start_container $HOST1 $C1/24 --name=c1 -start_container $HOST2 $C2/24 --name=c2 +start_container $HOST2 ip:$C2/24 --name=c2 assert_raises "exec_on $HOST1 c1 $PING $C2" start_container $HOST1 --name=c3 -start_container $HOST2 --name=c4 +start_container $HOST2 net:default --name=c4 C4=$(container_ip $HOST2 c4) assert_raises "exec_on $HOST1 c3 $PING $C4" +start_container $HOST1 net:$SUBNET_2 --name=c5 +start_container $HOST2 net:$SUBNET_2 --name=c6 +C6=$(container_ip $HOST2 c6) +assert_raises "exec_on $HOST1 c5 $PING $C6" + end_suite diff --git a/test/500_weave_multi_cidr_test.sh b/test/500_weave_multi_cidr_test.sh index 1e8a63a5..2d392715 100755 --- a/test/500_weave_multi_cidr_test.sh +++ b/test/500_weave_multi_cidr_test.sh @@ -11,7 +11,7 @@ assert_container_cidrs() { CIDRS="$@" # Assert container has attached CIDRs - assert_raises "weave_on $HOST ps | grep '^$CID [^ ]* $CIDRS$'" + assert_raises "weave_on $HOST ps $CID | grep -E '^$CID [0-9a-f:]{17} $CIDRS$'" } # assert_zone_records [ ...] @@ -42,36 +42,45 @@ assert_bridge_cidrs() { start_suite "Weave run/start/attach/detach with multiple cidr arguments" -weave_on $HOST1 launch +# NOTE: in these tests, net: arguments are checked against a +# specific address, i.e. we are assuming that IPAM always returns the +# lowest available address in the subnet + +weave_on $HOST1 launch -debug -iprange 10.2.3.0/24 weave_on $HOST1 launch-dns 10.254.254.254/24 # Run container with three cidrs -CID=$(start_container $HOST1 10.2.1.1/24 10.2.2.1/24 10.2.3.1/24 --name=multicidr -h $NAME | cut -b 1-12) +CID=$(start_container $HOST1 ip:10.2.1.1/24 10.2.2.1/24 net:10.2.3.0/24 --name=multicidr -h $NAME | cut -b 1-12) assert_container_cidrs $HOST1 $CID 10.2.1.1/24 10.2.2.1/24 10.2.3.1/24 assert_zone_records $HOST1 $CID $NAME. 10.2.1.1 10.2.2.1 10.2.3.1 # Remove two of them -weave_on $HOST1 detach 10.2.1.1/24 10.2.3.1/24 $CID +weave_on $HOST1 detach 10.2.1.1/24 net:10.2.3.0/24 $CID assert_container_cidrs $HOST1 $CID 10.2.2.1/24 assert_zone_records $HOST1 $CID $NAME. 10.2.2.1 # Put them both back -weave_on $HOST1 attach 10.2.1.1/24 10.2.3.1/24 $CID +weave_on $HOST1 attach ip:10.2.1.1/24 net:10.2.3.0/24 $CID assert_container_cidrs $HOST1 $CID 10.2.2.1/24 10.2.1.1/24 10.2.3.1/24 assert_zone_records $HOST1 $CID $NAME. 10.2.2.1 10.2.1.1 10.2.3.1 # Stop the container, restart with three IPs docker_on $HOST1 stop $CID -weave_on $HOST1 start 10.2.1.1/24 10.2.2.1/24 10.2.3.1/24 $CID +weave_on $HOST1 start 10.2.1.1/24 ip:10.2.2.1/24 net:10.2.3.0/24 $CID assert_container_cidrs $HOST1 $CID 10.2.1.1/24 10.2.2.1/24 10.2.3.1/24 assert_zone_records $HOST1 $CID $NAME. 10.2.1.1 10.2.2.1 10.2.3.1 # Expose some cidrs -weave_on $HOST1 expose 10.2.1.2/24 10.2.2.2/24 10.2.3.2/24 +weave_on $HOST1 expose 10.2.1.2/24 10.2.2.2/24 net:10.2.3.0/24 assert_bridge_cidrs $HOST1 weave 10.2.1.2/24 10.2.2.2/24 10.2.3.2/24 # Hide some cidrs -weave_on $HOST1 hide 10.2.1.2/24 10.2.3.2/24 +weave_on $HOST1 hide 10.2.1.2/24 net:10.2.3.0/24 assert_bridge_cidrs $HOST1 weave 10.2.2.2/24 +# Now detach and run another container to check we have released IPs in IPAM +weave_on $HOST1 detach $CID +CID2=$(start_container $HOST1 net:10.2.3.0/24) +assert_container_cidrs $HOST1 $CID2 10.2.3.1/24 + end_suite diff --git a/weave b/weave index 40f8e2a9..8f3153fb 100755 --- a/weave +++ b/weave @@ -28,19 +28,19 @@ usage() { echo "Usage:" echo "weave setup" echo "weave launch [-password ] [-nickname ] [-nodiscovery]" - echo " [-iprange ] ..." + echo " [-iprange [-ipsubnet ]] ..." echo "weave launch-dns " echo "weave launch-proxy [-D ] [-H
] [--with-dns] [--with-ipam]" echo "weave connect [--replace] [ ...]" echo "weave forget ..." - echo "weave run [--with-dns] [ ...] ..." - echo "weave start [ ...] " - echo "weave attach [ ...] " - echo "weave detach [ ...] " + echo "weave run [--with-dns] [ ...] ..." + echo "weave start [ ...] " + echo "weave attach [ ...] " + echo "weave detach [ ...] " echo "weave dns-add [ ...] [-h ]" echo "weave dns-remove [ ...] [-h ]" - echo "weave expose [ ...] [-h ]" - echo "weave hide [ ...]" + echo "weave expose [ ...] [-h ]" + echo "weave hide [ ...]" echo "weave ps [ ...]" echo "weave status" echo "weave proxy-env" @@ -52,9 +52,10 @@ usage() { echo "weave reset" echo "weave rmpeer " echo - echo "where = [:]" - echo " = /" - echo " = or weave internal peer ID" + echo "where = [:]" + echo " = /" + echo " = or weave internal peer ID" + echo " = [ip:] | net: | net:default" echo "
= [tcp://][]:" exit 1 } @@ -104,10 +105,10 @@ validate_cidr() { collect_cidr_args() { CIDR_ARGS="" - CIDR_COUNT=0 - while is_cidr "$1" ; do - CIDR_ARGS="$CIDR_ARGS $1" - CIDR_COUNT=$((CIDR_COUNT + 1)) + CIDR_ARG_COUNT=0 + while [ "$1" = "net:default" ] || is_cidr "$1" || is_cidr "${1#ip:}" || is_cidr "${1#net:}" ; do + CIDR_ARGS="$CIDR_ARGS ${1#ip:}" + CIDR_ARG_COUNT=$((CIDR_ARG_COUNT + 1)) shift 1 done } @@ -149,7 +150,7 @@ if [ "$1" = "run" ] ; then DNS_ARGS=$(exec_remote dns-args "$@") fi collect_cidr_args "$@" - shift $CIDR_COUNT + shift $CIDR_ARG_COUNT CONTAINER=$(docker $DOCKER_CLIENT_ARGS run $DNS_ARGS -d "$@") exec_remote attach $CIDR_ARGS $CONTAINER echo $CONTAINER @@ -751,17 +752,45 @@ ipam_claim() { done } -# If we don't already have any CIDRs passed from the command-line, get -# one from IPAM. $1 is the full container id. -get_ipam_cidr() { - if [ -n "$CIDR_ARGS" ] ; then - return 0 - fi - CIDR_ARGS="$(http_call $CONTAINER_NAME $HTTP_PORT POST /ip/$1)" - if [ "$CIDR_ARGS" = "404 page not found" ] ; then - echo "No IP address supplied (use the -iprange option on 'weave launch' to enable IP address allocation)" >&2 - return 1 +# Call IPAM as necessary to allocate addresses +# $1 is the full container id and following args are previously parsed CIDR_ARGS. +# Returns ALL_CIDRS and IPAM_CIDRS +ipam_cidrs() { + CONTAINER_ID="$1" + shift 1 + ALL_CIDRS="" + IPAM_CIDRs="" + if [ $# -eq 0 ] ; then + # If no addresses passed in, get one in the default subnet + IPAM_CIDRS="$(http_call $CONTAINER_NAME $HTTP_PORT POST /ip/$CONTAINER_ID)" + if [ "$IPAM_CIDRS" = "404 page not found" ] ; then + echo "No IP address supplied (use the -iprange option on 'weave launch' to enable IP address allocation)" >&2 + return 1 + fi + ALL_CIDRS="$IPAM_CIDRS" fi + while [ $# -gt 0 ] ; do + if [ "${1%:*}" = "net" ] ; then + if [ "$1" = "net:default" ] ; then + CIDR=$(http_call $CONTAINER_NAME $HTTP_PORT POST /ip/$CONTAINER_ID) + else + CIDR=$(http_call $CONTAINER_NAME $HTTP_PORT POST /ip/$CONTAINER_ID/"${1#net:}") + fi + if [ "$CIDR" = "404 page not found" ] ; then + echo "IP address allocation must be enabled to use --subnet (use the -iprange option on 'weave launch')" >&2 + return 1 + fi + if ! is_cidr "$CIDR" ; then + echo "$CIDR" >&2 + return 1 + fi + IPAM_CIDRS="$IPAM_CIDRS $CIDR" + ALL_CIDRS="$ALL_CIDRS $CIDR" + else + ALL_CIDRS="$ALL_CIDRS $1" + fi + shift 1 + done } ###################################################################### @@ -1028,12 +1057,12 @@ case "$COMMAND" in dns_args "$@" fi collect_cidr_args "$@" - shift $CIDR_COUNT + shift $CIDR_ARG_COUNT CONTAINER=$(docker run $DNS_ARGS -d "$@") create_bridge - get_ipam_cidr $CONTAINER - with_container_netns_or_die $CONTAINER attach $CIDR_ARGS - when_dns_running with_container_fqdn $CONTAINER put_dns_fqdn $CIDR_ARGS + ipam_cidrs $CONTAINER $CIDR_ARGS + with_container_netns_or_die $CONTAINER attach $ALL_CIDRS + when_dns_running with_container_fqdn $CONTAINER put_dns_fqdn $ALL_CIDRS echo $CONTAINER ;; dns-args) @@ -1046,33 +1075,37 @@ case "$COMMAND" in ;; start) collect_cidr_args "$@" - shift $CIDR_COUNT + shift $CIDR_ARG_COUNT [ $# -eq 1 ] || usage RES=$(docker start $1) CONTAINER=$(container_id $1) create_bridge - get_ipam_cidr $CONTAINER - with_container_netns_or_die $CONTAINER attach $CIDR_ARGS - when_dns_running with_container_fqdn $CONTAINER put_dns_fqdn $CIDR_ARGS + ipam_cidrs $CONTAINER $CIDR_ARGS + with_container_netns_or_die $CONTAINER attach $ALL_CIDRS + when_dns_running with_container_fqdn $CONTAINER put_dns_fqdn $ALL_CIDRS echo $RES ;; attach) collect_cidr_args "$@" - shift $CIDR_COUNT + shift $CIDR_ARG_COUNT [ $# -eq 1 ] || usage CONTAINER=$(container_id $1) create_bridge - get_ipam_cidr $CONTAINER - with_container_netns $CONTAINER attach $CIDR_ARGS >/dev/null - when_dns_running with_container_fqdn $CONTAINER put_dns_fqdn $CIDR_ARGS + ipam_cidrs $CONTAINER $CIDR_ARGS + with_container_netns $CONTAINER attach $ALL_CIDRS >/dev/null + when_dns_running with_container_fqdn $CONTAINER put_dns_fqdn $ALL_CIDRS ;; detach) collect_cidr_args "$@" - shift $CIDR_COUNT + shift $CIDR_ARG_COUNT [ $# -eq 1 ] || usage CONTAINER=$(container_id $1) - with_container_netns $CONTAINER detach $CIDR_ARGS >/dev/null - when_dns_running with_container_fqdn $CONTAINER delete_dns_fqdn $CIDR_ARGS + ipam_cidrs $CONTAINER $CIDR_ARGS + with_container_netns $CONTAINER detach $ALL_CIDRS >/dev/null + when_dns_running with_container_fqdn $CONTAINER delete_dns_fqdn $ALL_CIDRS + for CIDR in $IPAM_CIDRS ; do + http_call $CONTAINER_NAME $HTTP_PORT DELETE /ip/$CONTAINER/${CIDR%/*} + done ;; dns-add) collect_ip_args "$@" @@ -1100,8 +1133,8 @@ case "$COMMAND" in ;; expose) collect_cidr_args "$@" - shift $CIDR_COUNT - get_ipam_cidr weave:expose + shift $CIDR_ARG_COUNT + ipam_cidrs weave:expose $CIDR_ARGS if [ $# -eq 0 ] ; then FQDN="" else @@ -1109,7 +1142,7 @@ case "$COMMAND" in FQDN="$2" fi create_bridge --without-ethtool - for CIDR in $CIDR_ARGS ; do + for CIDR in $ALL_CIDRS ; do if ! ip addr show dev $BRIDGE | grep -qF $CIDR ; then ip addr add dev $BRIDGE $CIDR arp_update $BRIDGE $CIDR @@ -1123,10 +1156,10 @@ case "$COMMAND" in ;; hide) collect_cidr_args "$@" - shift $CIDR_COUNT - get_ipam_cidr weave:expose + shift $CIDR_ARG_COUNT + ipam_cidrs weave:expose $CIDR_ARGS create_bridge --without-ethtool - for CIDR in $CIDR_ARGS ; do + for CIDR in $ALL_CIDRS ; do if ip addr show dev $BRIDGE | grep -qF $CIDR ; then ip addr del dev $BRIDGE $CIDR delete_iptables_rule nat WEAVE -d $CIDR ! -s $CIDR -j MASQUERADE @@ -1134,7 +1167,9 @@ case "$COMMAND" in when_dns_running delete_dns 'weave:expose' $CIDR fi done - [ $CIDR_COUNT -eq 0 ] && http_call $CONTAINER_NAME $HTTP_PORT DELETE /ip/weave:expose 2>/dev/null || true + for CIDR in $IPAM_CIDRS ; do + http_call $CONTAINER_NAME $HTTP_PORT DELETE /ip/weave:expose/${CIDR%/*} + done ;; stop) [ $# -eq 0 ] || usage