From a09986baa2b3d6bc3d463fc0524114822fc7eb15 Mon Sep 17 00:00:00 2001 From: Adam Harrison Date: Tue, 12 Apr 2016 13:31:30 +0100 Subject: [PATCH 1/5] Rename consense -> prime --- ipam/allocator.go | 16 ++++++++-------- ipam/http.go | 4 ++-- ipam/{consense.go => prime.go} | 8 ++++---- site/ipam.md | 2 +- test/335_rmpeer_3_test.sh | 4 ++-- weave | 6 +++--- 6 files changed, 20 insertions(+), 20 deletions(-) rename ipam/{consense.go => prime.go} (55%) diff --git a/ipam/allocator.go b/ipam/allocator.go index 51e42a5b..b7aaae1c 100644 --- a/ipam/allocator.go +++ b/ipam/allocator.go @@ -56,7 +56,7 @@ type Allocator struct { nicknames map[mesh.PeerName]string // so we can map nicknames for rmpeer pendingAllocates []operation // held until we get some free space pendingClaims []operation // held until we know who owns the space - pendingConsenses []operation // held until consensus achieved + pendingPrimes []operation // held while our ring is empty dead map[string]time.Time // containers we heard were dead, and when db db.DB // persistence gossip mesh.Gossip // our link to the outside world for sending messages @@ -193,8 +193,8 @@ func (alloc *Allocator) tryOps(ops *[]operation) { // Try all pending operations func (alloc *Allocator) tryPendingOps() { - // Unblock pending consenses first - alloc.tryOps(&alloc.pendingConsenses) + // Unblock pending primes first + alloc.tryOps(&alloc.pendingPrimes) // Process existing claims before servicing new allocations alloc.tryOps(&alloc.pendingClaims) alloc.tryOps(&alloc.pendingAllocates) @@ -223,11 +223,11 @@ func (e *errorCancelled) Error() string { // Actor client API -// Consense (Sync) - wait for consensus -func (alloc *Allocator) Consense() { +// Prime (Sync) - wait for consensus +func (alloc *Allocator) Prime() { resultChan := make(chan struct{}) - op := &consense{resultChan: resultChan} - alloc.doOperation(op, &alloc.pendingConsenses) + op := &prime{resultChan: resultChan} + alloc.doOperation(op, &alloc.pendingPrimes) <-resultChan } @@ -381,7 +381,7 @@ func (alloc *Allocator) Shutdown() { alloc.shuttingDown = true alloc.cancelOps(&alloc.pendingClaims) alloc.cancelOps(&alloc.pendingAllocates) - alloc.cancelOps(&alloc.pendingConsenses) + alloc.cancelOps(&alloc.pendingPrimes) if heir := alloc.pickPeerForTransfer(); heir != mesh.UnknownPeerName { alloc.ring.Transfer(alloc.ourName, heir) alloc.space.Clear() diff --git a/ipam/http.go b/ipam/http.go index 7330b96c..6fd6a528 100644 --- a/ipam/http.go +++ b/ipam/http.go @@ -82,8 +82,8 @@ func (alloc *Allocator) HandleHTTP(router *mux.Router, defaultSubnet address.CID } }) - router.Methods("GET").Path("/consensus").HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - alloc.Consense() + router.Methods("GET").Path("/ring").HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + alloc.Prime() }) router.Methods("GET").Path("/ip/{id}/{ip}/{prefixlen}").HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/ipam/consense.go b/ipam/prime.go similarity index 55% rename from ipam/consense.go rename to ipam/prime.go index a4e36c15..6b9cff4d 100644 --- a/ipam/consense.go +++ b/ipam/prime.go @@ -1,10 +1,10 @@ package ipam -type consense struct { +type prime struct { resultChan chan<- struct{} } -func (c *consense) Try(alloc *Allocator) bool { +func (c *prime) Try(alloc *Allocator) bool { if !alloc.ring.Empty() { close(c.resultChan) return true @@ -15,10 +15,10 @@ func (c *consense) Try(alloc *Allocator) bool { return false } -func (c *consense) Cancel() { +func (c *prime) Cancel() { close(c.resultChan) } -func (c *consense) ForContainer(ident string) bool { +func (c *prime) ForContainer(ident string) bool { return false } diff --git a/site/ipam.md b/site/ipam.md index 829c29c9..4dfca0ae 100644 --- a/site/ipam.md +++ b/site/ipam.md @@ -147,7 +147,7 @@ existing cluster) it is desirable to ensure that a node has successfully joined and is ready to allocate IP addresses. An administrative command is provided for this purpose: - host1$ weave consense + host1$ weave prime This operation will block until the node on which it is run has joined successfully. diff --git a/test/335_rmpeer_3_test.sh b/test/335_rmpeer_3_test.sh index bcd7adbc..fddd8253 100755 --- a/test/335_rmpeer_3_test.sh +++ b/test/335_rmpeer_3_test.sh @@ -9,9 +9,9 @@ weave_on $HOST1 launch PEER1=$(weave_on $HOST1 report -f '{{.Router.Name}}') start_container $HOST1 weave_on $HOST2 launch $HOST1 -weave_on $HOST2 consense +weave_on $HOST2 prime weave_on $HOST3 launch $HOST1 -weave_on $HOST3 consense +weave_on $HOST3 prime # nuke 1st peer docker_on $HOST1 kill weave diff --git a/weave b/weave index 964a666a..f51e02e8 100755 --- a/weave +++ b/weave @@ -52,7 +52,7 @@ weave launch [--rewrite-inspect] launch-plugin [--no-multicast-route] -weave consense +weave prime weave env [--restore] config @@ -2252,8 +2252,8 @@ EOF echo "The 'stop-dns command has been removed; DNS is stopped as part of 'stop' and 'stop-router'." >&2 exit 0 ;; - consense) - call_weave GET /consensus + prime) + call_weave GET /ring ;; *) echo "Unknown weave command '$COMMAND'" >&2 From c3ecbc9d48f5cb05e6829e2673e0ff657afa8ca9 Mon Sep 17 00:00:00 2001 From: Adam Harrison Date: Tue, 12 Apr 2016 15:01:15 +0100 Subject: [PATCH 2/5] Implement unified --ipalloc-init parameter --- prog/weaver/main.go | 80 +++++++++++++++++++++++++++++++++------------ weave | 3 +- 2 files changed, 61 insertions(+), 22 deletions(-) diff --git a/prog/weaver/main.go b/prog/weaver/main.go index 2f6a86b4..90bb57c8 100644 --- a/prog/weaver/main.go +++ b/prog/weaver/main.go @@ -7,6 +7,7 @@ import ( _ "net/http/pprof" "os" "runtime" + "strconv" "strings" "sync/atomic" "time" @@ -34,11 +35,12 @@ var newVersion atomic.Value var Log = common.Log type ipamConfig struct { - IPRangeCIDR string - IPSubnetCIDR string - PeerCount int - Observer bool - Seed string + IPRangeCIDR string + IPSubnetCIDR string + PeerCount int + Mode string + Observer bool + SeedPeerNames []mesh.PeerName } type dnsConfig struct { @@ -77,26 +79,64 @@ func checkForUpdates() { checker = checkpoint.CheckInterval(¶ms, updateCheckPeriod, handleResponse) } -func (c ipamConfig) Enabled() bool { +func (c *ipamConfig) Enabled() bool { var ( hasPeerCount = c.PeerCount > 0 - hasSeed = len(c.Seed) > 0 + hasMode = len(c.Mode) > 0 hasRange = c.IPRangeCIDR != "" hasSubnet = c.IPSubnetCIDR != "" ) switch { - case !(c.Observer || hasPeerCount || hasSeed || hasRange || hasSubnet): + case !(hasPeerCount || hasMode || hasRange || hasSubnet): return false case !hasRange && hasSubnet: Log.Fatal("--ipalloc-default-subnet specified without --ipalloc-range.") case !hasRange: - Log.Fatal("--observer, --ipam-seed or --init-peer-count specified without --ipalloc-range.") - case c.Observer && hasPeerCount || hasPeerCount && hasSeed || hasSeed && c.Observer: - Log.Fatal("At most one of --observer, --ipam-seed or --init-peer-count may be specified.") + Log.Fatal("--ipalloc-init or --init-peer-count specified without --ipalloc-range.") + case hasMode && hasPeerCount: + Log.Fatal("At most one of --ipalloc-init or --init-peer-count may be specified.") + } + if hasMode { + if err := c.parseMode(); err != nil { + Log.Fatalf("Unable to parse --ipalloc-init: %s", err) + } } return true } +func (c *ipamConfig) parseMode() error { + modeAndParam := strings.SplitN(c.Mode, "=", 2) + + switch modeAndParam[0] { + case "consensus": + if len(modeAndParam) == 2 { + peerCount, err := strconv.Atoi(modeAndParam[1]) + if err != nil { + return fmt.Errorf("bad consensus parameter: %s", err) + } + c.PeerCount = peerCount + } + case "seed": + if len(modeAndParam) != 2 { + return fmt.Errorf("seed mode requires peer name list") + } + seedPeerNames, err := parsePeerNames(modeAndParam[1]) + if err != nil { + return fmt.Errorf("bad seed parameter: %s", err) + } + c.SeedPeerNames = seedPeerNames + case "observer": + if len(modeAndParam) != 1 { + return fmt.Errorf("observer mode takes no parameter") + } + c.Observer = true + default: + return fmt.Errorf("unknown mode: %s", modeAndParam[0]) + } + + return nil +} + func main() { procs := runtime.NumCPU() // packet sniffing can block an OS thread, so we need one thread @@ -152,11 +192,10 @@ func main() { mflag.BoolVar(&noDiscovery, []string{"#nodiscovery", "#-nodiscovery", "-no-discovery"}, false, "disable peer discovery") mflag.IntVar(&bufSzMB, []string{"#bufsz", "-bufsz"}, 8, "capture buffer size in MB") mflag.StringVar(&httpAddr, []string{"#httpaddr", "#-httpaddr", "-http-addr"}, "", "address to bind HTTP interface to (disabled if blank, absolute path indicates unix domain socket)") + mflag.StringVar(&ipamConfig.Mode, []string{"-ipalloc-init"}, "", "allocator initialisation strategy (consensus, seed or observer)") mflag.StringVar(&ipamConfig.IPRangeCIDR, []string{"#iprange", "#-iprange", "-ipalloc-range"}, "", "IP address range reserved for automatic allocation, in CIDR notation") mflag.StringVar(&ipamConfig.IPSubnetCIDR, []string{"#ipsubnet", "#-ipsubnet", "-ipalloc-default-subnet"}, "", "subnet to allocate within by default, in CIDR notation") mflag.IntVar(&ipamConfig.PeerCount, []string{"#initpeercount", "#-initpeercount", "-init-peer-count"}, 0, "number of peers in network (for IP address allocation)") - mflag.BoolVar(&ipamConfig.Observer, []string{"-observer"}, false, "initialise IP address allocation by observing other peers") - mflag.StringVar(&ipamConfig.Seed, []string{"-ipam-seed"}, "", "comma-separated list of peer names amongst which address space is shared initially") mflag.StringVar(&dockerAPI, []string{"#api", "#-api", "-docker-api"}, defaultDockerHost, "Docker API endpoint") mflag.BoolVar(&noDNS, []string{"-no-dns"}, false, "disable DNS server") mflag.StringVar(&dnsConfig.Domain, []string{"-dns-domain"}, nameserver.DefaultDomain, "local domain to server requests for") @@ -389,7 +428,6 @@ func parseAndCheckCIDR(cidrStr string) address.CIDR { func createAllocator(router *weave.NetworkRouter, config ipamConfig, db db.DB, isKnownPeer func(mesh.PeerName) bool) (*ipam.Allocator, address.CIDR) { ipRange := parseAndCheckCIDR(config.IPRangeCIDR) - seed := parseIPAMSeed(config.Seed) defaultSubnet := ipRange if config.IPSubnetCIDR != "" { defaultSubnet = parseAndCheckCIDR(config.IPSubnetCIDR) @@ -402,7 +440,7 @@ func createAllocator(router *weave.NetworkRouter, config ipamConfig, db db.DB, i OurName: router.Ourself.Peer.Name, OurUID: router.Ourself.Peer.UID, OurNickname: router.Ourself.Peer.NickName, - Seed: seed, + Seed: config.SeedPeerNames, Universe: ipRange.Range(), IsObserver: config.Observer, Quorum: func() uint { return determineQuorum(config.PeerCount, router) }, @@ -497,21 +535,21 @@ func parseTrustedSubnets(trustedSubnetStr string) []*net.IPNet { return trustedSubnets } -func parseIPAMSeed(ipamSeed string) []mesh.PeerName { +func parsePeerNames(s string) ([]mesh.PeerName, error) { peerNames := []mesh.PeerName{} - if ipamSeed == "" { - return peerNames + if s == "" { + return peerNames, nil } - for _, peerNameStr := range strings.Split(ipamSeed, ",") { + for _, peerNameStr := range strings.Split(s, ",") { peerName, err := mesh.PeerNameFromUserInput(peerNameStr) if err != nil { - Log.Fatal("Unable to parse IPAM seed: ", err) + return nil, fmt.Errorf("error parsing peer names: %s", err) } peerNames = append(peerNames, peerName) } - return peerNames + return peerNames, nil } func listenAndServeHTTP(httpAddr string, muxRouter *mux.Router) { diff --git a/weave b/weave index f51e02e8..4d32afe2 100755 --- a/weave +++ b/weave @@ -40,7 +40,7 @@ weave --help | help weave launch launch-router [--password ] [--name ] [--nickname ] - [--ipam-seed ,...] [--observer] + [--ipalloc-init ] [--ipalloc-range [--ipalloc-default-subnet ]] [--init-peer-count ] [--no-discovery] [--no-dns] [--trusted-subnets ,...] ... @@ -95,6 +95,7 @@ where = [:] = [ip:] | net: | net:default = [tcp://][]: | [unix://]/path/to/socket = | + = consensus[=] | seed=,... | observer EOF } From af8f5906469757c26cd0fa70aaf5d70f16090f5a Mon Sep 17 00:00:00 2001 From: Adam Harrison Date: Tue, 12 Apr 2016 15:04:14 +0100 Subject: [PATCH 3/5] Deprecate --init-peer-count --- weave | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/weave b/weave index 4d32afe2..a2f4bd72 100755 --- a/weave +++ b/weave @@ -42,7 +42,7 @@ weave launch launch-router [--password ] [--name ] [--nickname ] [--ipalloc-init ] [--ipalloc-range [--ipalloc-default-subnet ]] - [--init-peer-count ] [--no-discovery] [--no-dns] + [--no-discovery] [--no-dns] [--trusted-subnets ,...] ... launch-proxy [-H ] [--without-dns] [--no-multicast-route] [--no-rewrite-hosts] [--no-default-ipalloc] @@ -1865,10 +1865,12 @@ deprecation_warnings() { shift ;; -initpeercount|--initpeercount|-initpeercount=*|--initpeercount=*) - deprecation_warning $1 "--init-peer-count" + deprecation_warning $1 "--ipalloc-init" [ ${1#--} = "initpeercount" ] && shift ;; - --init-peer-count) + --init-peer-count|--init-peer-count=*) + deprecation_warning $1 "--ipalloc-init" + [ ${1#--} = "init-peer-count" ] && shift shift ;; -no-default-ipam|--no-default-ipam) From e355e50c0d660e0f386b7d044261fd1e11c0f15c Mon Sep 17 00:00:00 2001 From: Adam Harrison Date: Tue, 12 Apr 2016 15:15:35 +0100 Subject: [PATCH 4/5] Tests prefer --ipalloc-init --- test/340_observers_cannot_vote_3_test.sh | 6 +++--- test/350_observers_can_call_election_3_test.sh | 6 +++--- test/380_ipam_seed_2_test.sh | 4 ++-- test/510_detach_hide_no_ipalloc.sh | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/test/340_observers_cannot_vote_3_test.sh b/test/340_observers_cannot_vote_3_test.sh index 08d1b06f..2670b407 100755 --- a/test/340_observers_cannot_vote_3_test.sh +++ b/test/340_observers_cannot_vote_3_test.sh @@ -5,14 +5,14 @@ start_suite "Observers can't vote" # Start network that doesn't have enough peers for consensus -weave_on $HOST1 launch --init-peer-count=2 -weave_on $HOST2 launch --observer $HOST1 +weave_on $HOST1 launch --ipalloc-init consensus=2 +weave_on $HOST2 launch --ipalloc-init observer $HOST1 # Check that an allocation times out assert_raises "timeout 2 cat <( start_container $HOST1 )" 124 # Add another elector -weave_on $HOST3 launch --init-peer-count=2 $HOST2 +weave_on $HOST3 launch --ipalloc-init consensus=2 $HOST2 # Check allocation succeeds assert_raises "timeout 10 cat <( start_container $HOST1 )" diff --git a/test/350_observers_can_call_election_3_test.sh b/test/350_observers_can_call_election_3_test.sh index e6604e2b..af058e62 100755 --- a/test/350_observers_can_call_election_3_test.sh +++ b/test/350_observers_can_call_election_3_test.sh @@ -5,11 +5,11 @@ start_suite "Observers can call election" # Start network with sufficient voters for consensus -weave_on $HOST1 launch --init-peer-count=2 -weave_on $HOST2 launch --init-peer-count=2 $HOST1 +weave_on $HOST1 launch --ipalloc-init consensus=2 +weave_on $HOST2 launch --ipalloc-init consensus=2 $HOST1 # Add an observer -weave_on $HOST3 launch --observer $HOST2 +weave_on $HOST3 launch --ipalloc-init observer $HOST2 # Wait for network to settle; ensure IPAM is idle (blank output = no consensus) sleep 5 diff --git a/test/380_ipam_seed_2_test.sh b/test/380_ipam_seed_2_test.sh index f380a0c0..3f18b41b 100755 --- a/test/380_ipam_seed_2_test.sh +++ b/test/380_ipam_seed_2_test.sh @@ -5,8 +5,8 @@ start_suite "Specify initial IPAM seed" # Launch two disconnected routers -weave_on $HOST1 launch --name ::1 --ipam-seed ::1,::2 -weave_on $HOST2 launch --name ::2 --ipam-seed ::1,::2 +weave_on $HOST1 launch --name ::1 --ipalloc-init seed=::1,::2 +weave_on $HOST2 launch --name ::2 --ipalloc-init seed=::1,::2 # Ensure allocations can proceed assert_raises "timeout 10 cat <( start_container $HOST1 --name c1)" diff --git a/test/510_detach_hide_no_ipalloc.sh b/test/510_detach_hide_no_ipalloc.sh index 748e87bc..720e9eb3 100755 --- a/test/510_detach_hide_no_ipalloc.sh +++ b/test/510_detach_hide_no_ipalloc.sh @@ -11,7 +11,7 @@ check() { start_suite "'detach' and 'hide' do not require IP allocation" -weave_on $HOST1 launch --ipalloc-range $UNIVERSE --init-peer-count=2 +weave_on $HOST1 launch --ipalloc-range $UNIVERSE --ipalloc-init consensus=2 start_container $HOST1 10.2.1.1/24 --name=c1 check detach 10.2.1.1/24 c1 From a657e25a81ce322bc7cbbd0b2bc7cf30701f1dd7 Mon Sep 17 00:00:00 2001 From: Adam Harrison Date: Tue, 12 Apr 2016 15:52:34 +0100 Subject: [PATCH 5/5] Update documentation --- site/ipam.md | 55 ++++++++++++++++++++++++++-------------------------- 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/site/ipam.md b/site/ipam.md index 4dfca0ae..03895800 100644 --- a/site/ipam.md +++ b/site/ipam.md @@ -14,7 +14,7 @@ addresses in multiple subnets. The following automatic IP address management topics are discussed: * [Initializing Peers on a Weave Network](#initialization) - * [`--init-peer-count` and How Quorum is Achieved](#quorum) + * [`--ipalloc-init:consensus` and How Quorum is Achieved](#quorum) * [Forcing Consensus](#forcing-consensus) * [Choosing an Allocation Range](#range) @@ -22,21 +22,20 @@ The following automatic IP address management topics are discussed: ### Initializing Peers on a Weave Network -There are two options for initializing IPAM - with a statically -configured seed or automatically via one-off consensus. The two -options have different tradeoffs, so pick the one that suits your -deployment best. +Three initialization strategies are available: seed, consensus and +observer. These options have different tradeoffs, so pick the one that +suits your deployment best. #### Via Seed Configuration via seed requires you to provide a list of peer names -(via the `--ipam-seed` parameter) amongst which the address space will -be shared initially - this means that you will also have to specify -peer names explicitly on launch: +(via the `--ipalloc-init seed=` parameter) amongst which the address +space will be shared initially - this means that you will also have to +specify peer names explicitly on launch: - host1$ weave launch --name ::1 --ipam-seed ::1,::2,::3 - host2$ weave launch --name ::2 --ipam-seed ::1,::2,::3 - host3$ weave launch --name ::3 --ipam-seed ::1,::2,::3 + host1$ weave launch --name ::1 --ipalloc-init seed=::1,::2,::3 + host2$ weave launch --name ::2 --ipalloc-init seed=::1,::2,::3 + host3$ weave launch --name ::3 --ipalloc-init seed=::1,::2,::3 In this configuration each peer knows in advance how the address space has been divided up, and will be able to perform allocations from the @@ -52,7 +51,7 @@ for you as well. However, in order for Weave Net to form a single consensus reliably you must now instead tell each peer how many peers there are in total either by listing them as target peers or using the -`--init-peer-count` parameter. +`--ipalloc-init consensus=` parameter. Just once, when the first automatic IP address allocation is requested in the whole network, Weave Net needs a majority of peers to be present in @@ -61,7 +60,7 @@ inconsistency, for example, the same IP address being allocated to two different containers. Therefore, you must either supply the list of all peers in the network at `weave launch` or add the -`--init-peer-count` flag to specify how many peers there will be. +`--ipalloc-init consensus=` flag to specify how many peers there will be. To illustrate, suppose you have three hosts, accessible to each other as `$HOST1`, `$HOST2` and `$HOST3`. You can start Weave Net on those three @@ -74,11 +73,11 @@ hosts using these three commands: Or, if it is not convenient to name all the other hosts at launch time, you can pass the number of peers like this: - host1$ weave launch --init-peer-count 3 - host2$ weave launch --init-peer-count 3 $HOST3 - host3$ weave launch --init-peer-count 3 $HOST2 + host1$ weave launch --ipalloc-init consensus=3 + host2$ weave launch --ipalloc-init consensus=3 $HOST3 + host3$ weave launch --ipalloc-init consensus=3 $HOST2 -The consensus mechanism used to determine a majority, transitions +The consensus mechanism used to determine a majority transitions through three states: 'deferred', 'waiting' and 'achieved': * 'deferred' - no allocation requests or claims have been made yet; @@ -89,21 +88,23 @@ through three states: 'deferred', 'waiting' and 'achieved': themselves successfully * 'achieved' - consensus achieved; allocations proceed normally -Finally, some (but never all) peers can be launched as consensus -observers rather than participants by specifying the `--observer` -option: +#### Via Observation - host4$ weave launch --observer $HOST3 +Finally, some (but never all) peers can be launched as observers +by specifying the `--ipalloc-init observer` option: -You do not need to specify an initial peer count to such peers. This -can be useful to add ephemeral peers to an existing fixed cluster (for -example in response to a scale-out event) without worrying about + host4$ weave launch --ipalloc-init observer $HOST3 + +You do not need to specify an initial peer count or seed to such +peers. This can be useful to add peers to an existing fixed cluster +(for example in response to a scale-out event) without worrying about adjusting initial peer counts accordingly. -#### `--init-peer-count` and How Quorum is Achieved +#### `--ipalloc-init consensus=` and How Quorum is Achieved -Normally it isn't a problem to over-estimate `--init-peer-count`, but if you supply -a number that is too small, then multiple independent groups may form. +Normally it isn't a problem to over-estimate the value supplied to +`--ipalloc-init consensus=`, but if you supply a number that is too +small, then multiple independent groups may form. Weave Net uses the estimate of the number of peers at initialization to compute a majority or quorum number - specifically floor(n/2) + 1.