Merge pull request #2155 from weaveworks/issues/2065-rationalise-ipam-concepts

Rationalise IPAM concepts

ipam/allocator.go

@@ -56,7 +56,7 @@ type Allocator struct {
 	nicknames         map[mesh.PeerName]string  // so we can map nicknames for rmpeer
 	pendingAllocates  []operation               // held until we get some free space
 	pendingClaims     []operation               // held until we know who owns the space
-	pendingConsenses  []operation               // held until consensus achieved
+	pendingPrimes     []operation               // held while our ring is empty
 	dead              map[string]time.Time      // containers we heard were dead, and when
 	db                db.DB                     // persistence
 	gossip            mesh.Gossip               // our link to the outside world for sending messages
@@ -193,8 +193,8 @@ func (alloc *Allocator) tryOps(ops *[]operation) {
 
 // Try all pending operations
 func (alloc *Allocator) tryPendingOps() {
-	// Unblock pending consenses first
-	alloc.tryOps(&alloc.pendingConsenses)
+	// Unblock pending primes first
+	alloc.tryOps(&alloc.pendingPrimes)
 	// Process existing claims before servicing new allocations
 	alloc.tryOps(&alloc.pendingClaims)
 	alloc.tryOps(&alloc.pendingAllocates)
@@ -223,11 +223,11 @@ func (e *errorCancelled) Error() string {
 
 // Actor client API
 
-// Consense (Sync) - wait for consensus
-func (alloc *Allocator) Consense() {
+// Prime (Sync) - wait for consensus
+func (alloc *Allocator) Prime() {
 	resultChan := make(chan struct{})
-	op := &consense{resultChan: resultChan}
-	alloc.doOperation(op, &alloc.pendingConsenses)
+	op := &prime{resultChan: resultChan}
+	alloc.doOperation(op, &alloc.pendingPrimes)
 	<-resultChan
 }
 
@@ -381,7 +381,7 @@ func (alloc *Allocator) Shutdown() {
 		alloc.shuttingDown = true
 		alloc.cancelOps(&alloc.pendingClaims)
 		alloc.cancelOps(&alloc.pendingAllocates)
-		alloc.cancelOps(&alloc.pendingConsenses)
+		alloc.cancelOps(&alloc.pendingPrimes)
 		if heir := alloc.pickPeerForTransfer(); heir != mesh.UnknownPeerName {
 			alloc.ring.Transfer(alloc.ourName, heir)
 			alloc.space.Clear()

ipam/http.go

@@ -82,8 +82,8 @@ func (alloc *Allocator) HandleHTTP(router *mux.Router, defaultSubnet address.CID
 		}
 	})
 
-	router.Methods("GET").Path("/consensus").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		alloc.Consense()
+	router.Methods("GET").Path("/ring").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		alloc.Prime()
 	})
 
 	router.Methods("GET").Path("/ip/{id}/{ip}/{prefixlen}").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

ipam/prime.go

@@ -1,10 +1,10 @@
 package ipam
 
-type consense struct {
+type prime struct {
 	resultChan chan<- struct{}
 }
 
-func (c *consense) Try(alloc *Allocator) bool {
+func (c *prime) Try(alloc *Allocator) bool {
 	if !alloc.ring.Empty() {
 		close(c.resultChan)
 		return true
@@ -15,10 +15,10 @@ func (c *consense) Try(alloc *Allocator) bool {
 	return false
 }
 
-func (c *consense) Cancel() {
+func (c *prime) Cancel() {
 	close(c.resultChan)
 }
 
-func (c *consense) ForContainer(ident string) bool {
+func (c *prime) ForContainer(ident string) bool {
 	return false
 }

prog/weaver/main.go

@@ -7,6 +7,7 @@ import (
 	_ "net/http/pprof"
 	"os"
 	"runtime"
+	"strconv"
 	"strings"
 	"sync/atomic"
 	"time"
@@ -34,11 +35,12 @@ var newVersion atomic.Value
 var Log = common.Log
 
 type ipamConfig struct {
-	IPRangeCIDR  string
-	IPSubnetCIDR string
-	PeerCount    int
-	Observer     bool
-	Seed         string
+	IPRangeCIDR   string
+	IPSubnetCIDR  string
+	PeerCount     int
+	Mode          string
+	Observer      bool
+	SeedPeerNames []mesh.PeerName
 }
 
 type dnsConfig struct {
@@ -77,26 +79,64 @@ func checkForUpdates() {
 	checker = checkpoint.CheckInterval(&params, updateCheckPeriod, handleResponse)
 }
 
-func (c ipamConfig) Enabled() bool {
+func (c *ipamConfig) Enabled() bool {
 	var (
 		hasPeerCount = c.PeerCount > 0
-		hasSeed      = len(c.Seed) > 0
+		hasMode      = len(c.Mode) > 0
 		hasRange     = c.IPRangeCIDR != ""
 		hasSubnet    = c.IPSubnetCIDR != ""
 	)
 	switch {
-	case !(c.Observer || hasPeerCount || hasSeed || hasRange || hasSubnet):
+	case !(hasPeerCount || hasMode || hasRange || hasSubnet):
 		return false
 	case !hasRange && hasSubnet:
 		Log.Fatal("--ipalloc-default-subnet specified without --ipalloc-range.")
 	case !hasRange:
-		Log.Fatal("--observer, --ipam-seed or --init-peer-count specified without --ipalloc-range.")
-	case c.Observer && hasPeerCount || hasPeerCount && hasSeed || hasSeed && c.Observer:
-		Log.Fatal("At most one of --observer, --ipam-seed or --init-peer-count may be specified.")
+		Log.Fatal("--ipalloc-init or --init-peer-count specified without --ipalloc-range.")
+	case hasMode && hasPeerCount:
+		Log.Fatal("At most one of --ipalloc-init or --init-peer-count may be specified.")
+	}
+	if hasMode {
+		if err := c.parseMode(); err != nil {
+			Log.Fatalf("Unable to parse --ipalloc-init: %s", err)
+		}
 	}
 	return true
 }
 
+func (c *ipamConfig) parseMode() error {
+	modeAndParam := strings.SplitN(c.Mode, "=", 2)
+
+	switch modeAndParam[0] {
+	case "consensus":
+		if len(modeAndParam) == 2 {
+			peerCount, err := strconv.Atoi(modeAndParam[1])
+			if err != nil {
+				return fmt.Errorf("bad consensus parameter: %s", err)
+			}
+			c.PeerCount = peerCount
+		}
+	case "seed":
+		if len(modeAndParam) != 2 {
+			return fmt.Errorf("seed mode requires peer name list")
+		}
+		seedPeerNames, err := parsePeerNames(modeAndParam[1])
+		if err != nil {
+			return fmt.Errorf("bad seed parameter: %s", err)
+		}
+		c.SeedPeerNames = seedPeerNames
+	case "observer":
+		if len(modeAndParam) != 1 {
+			return fmt.Errorf("observer mode takes no parameter")
+		}
+		c.Observer = true
+	default:
+		return fmt.Errorf("unknown mode: %s", modeAndParam[0])
+	}
+
+	return nil
+}
+
 func main() {
 	procs := runtime.NumCPU()
 	// packet sniffing can block an OS thread, so we need one thread
@@ -152,11 +192,10 @@ func main() {
 	mflag.BoolVar(&noDiscovery, []string{"#nodiscovery", "#-nodiscovery", "-no-discovery"}, false, "disable peer discovery")
 	mflag.IntVar(&bufSzMB, []string{"#bufsz", "-bufsz"}, 8, "capture buffer size in MB")
 	mflag.StringVar(&httpAddr, []string{"#httpaddr", "#-httpaddr", "-http-addr"}, "", "address to bind HTTP interface to (disabled if blank, absolute path indicates unix domain socket)")
+	mflag.StringVar(&ipamConfig.Mode, []string{"-ipalloc-init"}, "", "allocator initialisation strategy (consensus, seed or observer)")
 	mflag.StringVar(&ipamConfig.IPRangeCIDR, []string{"#iprange", "#-iprange", "-ipalloc-range"}, "", "IP address range reserved for automatic allocation, in CIDR notation")
 	mflag.StringVar(&ipamConfig.IPSubnetCIDR, []string{"#ipsubnet", "#-ipsubnet", "-ipalloc-default-subnet"}, "", "subnet to allocate within by default, in CIDR notation")
 	mflag.IntVar(&ipamConfig.PeerCount, []string{"#initpeercount", "#-initpeercount", "-init-peer-count"}, 0, "number of peers in network (for IP address allocation)")
-	mflag.BoolVar(&ipamConfig.Observer, []string{"-observer"}, false, "initialise IP address allocation by observing other peers")
-	mflag.StringVar(&ipamConfig.Seed, []string{"-ipam-seed"}, "", "comma-separated list of peer names amongst which address space is shared initially")
 	mflag.StringVar(&dockerAPI, []string{"#api", "#-api", "-docker-api"}, defaultDockerHost, "Docker API endpoint")
 	mflag.BoolVar(&noDNS, []string{"-no-dns"}, false, "disable DNS server")
 	mflag.StringVar(&dnsConfig.Domain, []string{"-dns-domain"}, nameserver.DefaultDomain, "local domain to server requests for")
@@ -389,7 +428,6 @@ func parseAndCheckCIDR(cidrStr string) address.CIDR {
 
 func createAllocator(router *weave.NetworkRouter, config ipamConfig, db db.DB, isKnownPeer func(mesh.PeerName) bool) (*ipam.Allocator, address.CIDR) {
 	ipRange := parseAndCheckCIDR(config.IPRangeCIDR)
-	seed := parseIPAMSeed(config.Seed)
 	defaultSubnet := ipRange
 	if config.IPSubnetCIDR != "" {
 		defaultSubnet = parseAndCheckCIDR(config.IPSubnetCIDR)
@@ -402,7 +440,7 @@ func createAllocator(router *weave.NetworkRouter, config ipamConfig, db db.DB, i
 		OurName:     router.Ourself.Peer.Name,
 		OurUID:      router.Ourself.Peer.UID,
 		OurNickname: router.Ourself.Peer.NickName,
-		Seed:        seed,
+		Seed:        config.SeedPeerNames,
 		Universe:    ipRange.Range(),
 		IsObserver:  config.Observer,
 		Quorum:      func() uint { return determineQuorum(config.PeerCount, router) },
@@ -497,21 +535,21 @@ func parseTrustedSubnets(trustedSubnetStr string) []*net.IPNet {
 	return trustedSubnets
 }
 
-func parseIPAMSeed(ipamSeed string) []mesh.PeerName {
+func parsePeerNames(s string) ([]mesh.PeerName, error) {
 	peerNames := []mesh.PeerName{}
-	if ipamSeed == "" {
-		return peerNames
+	if s == "" {
+		return peerNames, nil
 	}
 
-	for _, peerNameStr := range strings.Split(ipamSeed, ",") {
+	for _, peerNameStr := range strings.Split(s, ",") {
 		peerName, err := mesh.PeerNameFromUserInput(peerNameStr)
 		if err != nil {
-			Log.Fatal("Unable to parse IPAM seed: ", err)
+			return nil, fmt.Errorf("error parsing peer names: %s", err)
 		}
 		peerNames = append(peerNames, peerName)
 	}
 
-	return peerNames
+	return peerNames, nil
 }
 
 func listenAndServeHTTP(httpAddr string, muxRouter *mux.Router) {

site/ipam.md

@@ -14,7 +14,7 @@ addresses in multiple subnets.
 The following automatic IP address management topics are discussed:
 
  * [Initializing Peers on a Weave Network](#initialization)
- * [`--init-peer-count` and How Quorum is Achieved](#quorum)
+ * [`--ipalloc-init:consensus` and How Quorum is Achieved](#quorum)
  * [Forcing Consensus](#forcing-consensus)
  * [Choosing an Allocation Range](#range)
 
@@ -22,21 +22,20 @@ The following automatic IP address management topics are discussed:
 
 ### <a name="initialization"></a>Initializing Peers on a Weave Network
 
-There are two options for initializing IPAM - with a statically
-configured seed or automatically via one-off consensus. The two
-options have different tradeoffs, so pick the one that suits your
-deployment best.
+Three initialization strategies are available: seed, consensus and
+observer. These options have different tradeoffs, so pick the one that
+suits your deployment best.
 
 #### <a name="seed"></a>Via Seed
 
 Configuration via seed requires you to provide a list of peer names
-(via the `--ipam-seed` parameter) amongst which the address space will
-be shared initially - this means that you will also have to specify
-peer names explicitly on launch:
+(via the `--ipalloc-init seed=` parameter) amongst which the address
+space will be shared initially - this means that you will also have to
+specify peer names explicitly on launch:
 
-    host1$ weave launch --name ::1 --ipam-seed ::1,::2,::3
-    host2$ weave launch --name ::2 --ipam-seed ::1,::2,::3
-    host3$ weave launch --name ::3 --ipam-seed ::1,::2,::3
+    host1$ weave launch --name ::1 --ipalloc-init seed=::1,::2,::3
+    host2$ weave launch --name ::2 --ipalloc-init seed=::1,::2,::3
+    host3$ weave launch --name ::3 --ipalloc-init seed=::1,::2,::3
 
 In this configuration each peer knows in advance how the address space
 has been divided up, and will be able to perform allocations from the
@@ -52,7 +51,7 @@ for you as well.
 However, in order for Weave Net to form a single consensus
 reliably you must now instead tell each peer how many peers there are
 in total either by listing them as target peers or using the
-`--init-peer-count` parameter.
+`--ipalloc-init consensus=` parameter.
 
 Just once, when the first automatic IP address allocation is requested
 in the whole network, Weave Net needs a majority of peers to be present in
@@ -61,7 +60,7 @@ inconsistency, for example, the same IP address being allocated to two
 different containers.
 
 Therefore, you must either supply the list of all peers in the network at `weave launch` or add the
-`--init-peer-count` flag to specify how many peers there will be.
+`--ipalloc-init consensus=` flag to specify how many peers there will be.
 
 To illustrate, suppose you have three hosts, accessible to each other
 as `$HOST1`, `$HOST2` and `$HOST3`. You can start Weave Net on those three
@@ -74,11 +73,11 @@ hosts using these three commands:
 Or, if it is not convenient to name all the other hosts at launch
 time, you can pass the number of peers like this:
 
-    host1$ weave launch --init-peer-count 3
-    host2$ weave launch --init-peer-count 3 $HOST3
-    host3$ weave launch --init-peer-count 3 $HOST2
+    host1$ weave launch --ipalloc-init consensus=3
+    host2$ weave launch --ipalloc-init consensus=3 $HOST3
+    host3$ weave launch --ipalloc-init consensus=3 $HOST2
 
-The consensus mechanism used to determine a majority, transitions
+The consensus mechanism used to determine a majority transitions
 through three states: 'deferred', 'waiting' and 'achieved':
 
 * 'deferred' - no allocation requests or claims have been made yet;
@@ -89,21 +88,23 @@ through three states: 'deferred', 'waiting' and 'achieved':
   themselves successfully
 * 'achieved' - consensus achieved; allocations proceed normally
 
-Finally, some (but never all) peers can be launched as consensus
-observers rather than participants by specifying the `--observer`
-option:
+#### <a name="observer"></a>Via Observation
 
-    host4$ weave launch --observer $HOST3
+Finally, some (but never all) peers can be launched as observers
+by specifying the `--ipalloc-init observer` option:
 
-You do not need to specify an initial peer count to such peers. This
-can be useful to add ephemeral peers to an existing fixed cluster (for
-example in response to a scale-out event) without worrying about
+    host4$ weave launch --ipalloc-init observer $HOST3
+
+You do not need to specify an initial peer count or seed to such
+peers. This can be useful to add peers to an existing fixed cluster
+(for example in response to a scale-out event) without worrying about
 adjusting initial peer counts accordingly.
 
-####<a name="quorum"></a> `--init-peer-count` and How Quorum is Achieved
+####<a name="quorum"></a> `--ipalloc-init consensus=` and How Quorum is Achieved
 
-Normally it isn't a problem to over-estimate `--init-peer-count`, but if you supply
-a number that is too small, then multiple independent groups may form.
+Normally it isn't a problem to over-estimate the value supplied to
+`--ipalloc-init consensus=`, but if you supply a number that is too
+small, then multiple independent groups may form.
 
 Weave Net uses the estimate of the number of peers at initialization to
 compute a majority or quorum number - specifically floor(n/2) + 1.
@@ -147,7 +148,7 @@ existing cluster) it is desirable to ensure that a node has
 successfully joined and is ready to allocate IP addresses. An
 administrative command is provided for this purpose:
 
-    host1$ weave consense
+    host1$ weave prime
 
 This operation will block until the node on which it is run has joined
 successfully.

test/335_rmpeer_3_test.sh

@@ -9,9 +9,9 @@ weave_on $HOST1 launch
 PEER1=$(weave_on $HOST1 report -f '{{.Router.Name}}')
 start_container $HOST1
 weave_on $HOST2 launch $HOST1
-weave_on $HOST2 consense
+weave_on $HOST2 prime
 weave_on $HOST3 launch $HOST1
-weave_on $HOST3 consense
+weave_on $HOST3 prime
 
 # nuke 1st peer
 docker_on $HOST1 kill weave

test/340_observers_cannot_vote_3_test.sh

@@ -5,14 +5,14 @@
 start_suite "Observers can't vote"
 
 # Start network that doesn't have enough peers for consensus
-weave_on $HOST1 launch --init-peer-count=2
-weave_on $HOST2 launch --observer $HOST1
+weave_on $HOST1 launch --ipalloc-init consensus=2
+weave_on $HOST2 launch --ipalloc-init observer $HOST1
 
 # Check that an allocation times out
 assert_raises "timeout 2 cat <( start_container $HOST1 )" 124
 
 # Add another elector
-weave_on $HOST3	launch --init-peer-count=2 $HOST2
+weave_on $HOST3	launch --ipalloc-init consensus=2 $HOST2
 
 # Check allocation succeeds
 assert_raises "timeout 10 cat <( start_container $HOST1 )"

test/350_observers_can_call_election_3_test.sh

@@ -5,11 +5,11 @@
 start_suite "Observers can call election"
 
 # Start network with sufficient voters for consensus
-weave_on $HOST1 launch --init-peer-count=2
-weave_on $HOST2 launch --init-peer-count=2 $HOST1
+weave_on $HOST1 launch --ipalloc-init consensus=2
+weave_on $HOST2 launch --ipalloc-init consensus=2 $HOST1
 
 # Add an observer
-weave_on $HOST3 launch --observer $HOST2
+weave_on $HOST3 launch --ipalloc-init observer $HOST2
 
 # Wait for network to settle; ensure IPAM is idle (blank output = no consensus)
 sleep 5

test/380_ipam_seed_2_test.sh

@@ -5,8 +5,8 @@
 start_suite "Specify initial IPAM seed"
 
 # Launch two disconnected routers
-weave_on $HOST1 launch --name ::1 --ipam-seed ::1,::2
-weave_on $HOST2 launch --name ::2 --ipam-seed ::1,::2
+weave_on $HOST1 launch --name ::1 --ipalloc-init seed=::1,::2
+weave_on $HOST2 launch --name ::2 --ipalloc-init seed=::1,::2
 
 # Ensure allocations can proceed
 assert_raises "timeout 10 cat <( start_container $HOST1 --name c1)"

test/510_detach_hide_no_ipalloc.sh

@@ -11,7 +11,7 @@ check() {
 
 start_suite "'detach' and 'hide' do not require IP allocation"
 
-weave_on $HOST1 launch --ipalloc-range $UNIVERSE --init-peer-count=2
+weave_on $HOST1 launch --ipalloc-range $UNIVERSE --ipalloc-init consensus=2
 
 start_container $HOST1 10.2.1.1/24 --name=c1
 check detach           10.2.1.1/24 c1

weave

@@ -41,9 +41,9 @@ weave --help | help
 
 weave launch        <same arguments as 'weave launch-router'>
       launch-router [--password <pass>] [--name <mac>] [--nickname <nickname>]
-                      [--ipam-seed <mac>,...] [--observer]
+                      [--ipalloc-init <mode>]
                       [--ipalloc-range <cidr> [--ipalloc-default-subnet <cidr>]]
-                      [--init-peer-count <count>] [--no-discovery] [--no-dns]
+                      [--no-discovery] [--no-dns]
                       [--trusted-subnets <cidr>,...] <peer> ...
       launch-proxy  [-H <endpoint>] [--without-dns] [--no-multicast-route]
                       [--no-rewrite-hosts] [--no-default-ipalloc]
@@ -53,7 +53,7 @@ weave launch        <same arguments as 'weave launch-router'>
                       [--rewrite-inspect]
       launch-plugin [--no-multicast-route]
 
-weave consense
+weave prime
 
 weave env           [--restore]
       config
@@ -96,6 +96,7 @@ where <peer>     = <ip_address_or_fqdn>[:<port>]
       <addr>     = [ip:]<cidr> | net:<cidr> | net:default
       <endpoint> = [tcp://][<ip_address>]:<port> | [unix://]/path/to/socket
       <peer_id>  = <nickname> | <weave internal peer ID>
+      <mode>     = consensus[=<count>] | seed=<mac>,... | observer
 EOF
 }
 
@@ -1875,10 +1876,12 @@ deprecation_warnings() {
                 shift
                 ;;
             -initpeercount|--initpeercount|-initpeercount=*|--initpeercount=*)
-                deprecation_warning $1 "--init-peer-count"
+                deprecation_warning $1 "--ipalloc-init"
                 [ ${1#--} = "initpeercount" ] && shift
                 ;;
-            --init-peer-count)
+            --init-peer-count|--init-peer-count=*)
+                deprecation_warning $1 "--ipalloc-init"
+                [ ${1#--} = "init-peer-count" ] && shift
                 shift
                 ;;
             -no-default-ipam|--no-default-ipam)
@@ -2263,8 +2266,8 @@ EOF
         echo "The 'stop-dns command has been removed; DNS is stopped as part of 'stop' and 'stop-router'." >&2
         exit 0
         ;;
-    consense)
-        call_weave GET /consensus
+    prime)
+        call_weave GET /ring
         ;;
     *)
         echo "Unknown weave command '$COMMAND'" >&2