Merge pull request #2134 from weaveworks/issues/2063-observer-status

Better log and status messages for Paxos observer

ipam/allocator.go

@@ -767,7 +767,7 @@ func (alloc *Allocator) update(sender mesh.PeerName, msg []byte) error {
 	//   (if we did so on periodic gossip we would force consensus unnecessarily)
 	// * If we are an elector (to avoid a broadcast storm of ring request messages)
 	default:
-		if _, ok := alloc.paxos.(*paxos.Node); ok && sender != mesh.UnknownPeerName {
+		if alloc.paxos.IsElector() && sender != mesh.UnknownPeerName {
 			alloc.establishRing()
 		}
 	}
@@ -887,8 +887,10 @@ func (alloc *Allocator) loadPersistedData() {
 	if len(alloc.seed) != 0 {
 		alloc.infof("Initialising with supplied IPAM seed")
 		alloc.createRing(alloc.seed)
-	} else {
+	} else if alloc.paxos.IsElector() {
 		alloc.infof("Initialising via deferred consensus")
+	} else {
+		alloc.infof("Initialising as observer - awaiting IPAM data from another peer")
 	}
 
 }

ipam/paxos/participant.go

@@ -7,6 +7,7 @@ type Participant interface {
 	Propose()
 	Think() bool
 	Consensus() (bool, AcceptedValue)
+	IsElector() bool
 }
 
 type Observer struct {
@@ -37,3 +38,7 @@ func (observer *Observer) Think() bool {
 func (observer *Observer) Consensus() (bool, AcceptedValue) {
 	return false, AcceptedValue{}
 }
+
+func (observer *Observer) IsElector() bool {
+	return false
+}

ipam/paxos/paxos.go

@@ -257,14 +257,16 @@ func (node *Node) Consensus() (bool, AcceptedValue) {
 	return false, AcceptedValue{}
 }
 
+func (node *Node) IsElector() bool {
+	return true
+}
+
 type Status struct {
+	Elector    bool
 	KnownNodes int
 	Quorum     uint
 }
 
 func NewStatus(node *Node) *Status {
-	if node == nil {
-		return nil
-	}
-	return &Status{len(node.knows), node.quorum}
+	return &Status{true, len(node.knows), node.quorum}
 }

ipam/status.go

@@ -37,8 +37,12 @@ func NewStatus(allocator *Allocator, defaultSubnet address.CIDR) *Status {
 
 	var paxosStatus *paxos.Status
 	if allocator.awaitingConsensus && allocator.paxos != nil {
-		if node, ok := allocator.paxos.(*paxos.Node); ok {
-			paxosStatus = paxos.NewStatus(node)
+		if allocator.paxos.IsElector() {
+			if node, ok := allocator.paxos.(*paxos.Node); ok {
+				paxosStatus = paxos.NewStatus(node)
+			}
+		} else {
+			paxosStatus = &paxos.Status{Elector: false}
 		}
 	}
 

prog/weaver/http.go

@@ -166,7 +166,11 @@ var statusTemplate = defTemplate("status", `\
          Status: ready
 {{end}}\
 {{else if .IPAM.Paxos}}\
+{{if .IPAM.Paxos.Elector}}\
          Status: awaiting consensus (quorum: {{.IPAM.Paxos.Quorum}}, known: {{.IPAM.Paxos.KnownNodes}})
+{{else}}\
+         Status: awaiting consensus (observer)
+{{end}}\
 {{else}}\
          Status: idle
 {{end}}\

site/ipam/troubleshooting-ipam.md

@@ -31,7 +31,9 @@ the total allocation range and default subnet. Columns are as follows:
     * 'awaiting consensus' - an attempt to achieve consensus is
       ongoing, triggered by an allocation or claim request;
       allocations will block.  This state persists until a quorum of
-      peers are able to communicate amongst themselves successfully
+      peers are able to communicate amongst themselves successfully.
+      The Status may also show '(observer)' instead of the quorum size,
+      if this peer has been started with the `--observer` option.
     * 'ready' - consensus achieved; allocations proceed normally
     * 'waiting for IP range grant from peers' - peer has exhausted its
       agreed portion of the range and is waiting to be granted some