Merge pull request #1702 from weaveworks/issues/1639-ipam-diagnostics

Show some IPAM stats with `weave status ipam`

ipam/ring/ring.go

@@ -316,6 +316,25 @@ func (r *Ring) OwnedRanges() (result []address.Range) {
 	return r.splitRangesOverZero(result)
 }
 
+// For printing status
+type RangeInfo struct {
+	Peer mesh.PeerName
+	address.Range
+	Version uint32
+}
+
+func (r *Ring) AllRangeInfo() (result []RangeInfo) {
+	for i, entry := range r.Entries {
+		nextEntry := r.Entries.entry(i + 1)
+		ranges := []address.Range{{Start: entry.Token, End: nextEntry.Token}}
+		ranges = r.splitRangesOverZero(ranges)
+		for _, r := range ranges {
+			result = append(result, RangeInfo{entry.Peer, r, entry.Version})
+		}
+	}
+	return
+}
+
 // ClaimForPeers claims the entire ring for the array of peers passed
 // in.  Only works for empty rings.
 func (r *Ring) ClaimForPeers(peers []mesh.PeerName) {

ipam/status.go

@@ -1,6 +1,7 @@
 package ipam
 
 import (
+	"fmt"
 	"github.com/weaveworks/weave/ipam/paxos"
 	"github.com/weaveworks/weave/net/address"
 )
@@ -8,6 +9,7 @@ import (
 type Status struct {
 	Paxos            *paxos.Status
 	Range            string
+	RangeNumIPs      int
 	DefaultSubnet    string
 	Entries          []EntryStatus
 	PendingClaims    []ClaimStatus
@@ -15,9 +17,12 @@ type Status struct {
 }
 
 type EntryStatus struct {
-	Token   string
-	Peer    string
-	Version uint32
+	Token       string
+	Size        uint32
+	Peer        string
+	Nickname    string
+	IsKnownPeer bool
+	Version     uint32
 }
 
 type ClaimStatus struct {
@@ -40,6 +45,7 @@ func NewStatus(allocator *Allocator, defaultSubnet address.CIDR) *Status {
 		resultChan <- &Status{
 			paxosStatus,
 			allocator.universe.String(),
+			int(allocator.universe.Size()),
 			defaultSubnet.String(),
 			newEntryStatusSlice(allocator),
 			newClaimStatusSlice(allocator),
@@ -56,8 +62,15 @@ func newEntryStatusSlice(allocator *Allocator) []EntryStatus {
 		return slice
 	}
 
-	for _, entry := range allocator.ring.Entries {
-		slice = append(slice, EntryStatus{entry.Token.String(), entry.Peer.String(), entry.Version})
+	for _, r := range allocator.ring.AllRangeInfo() {
+		slice = append(slice, EntryStatus{
+			Token:       r.Start.String(),
+			Size:        uint32(r.Size()),
+			Peer:        r.Peer.String(),
+			Nickname:    allocator.nicknames[r.Peer],
+			IsKnownPeer: allocator.isKnownPeer(r.Peer),
+			Version:     r.Version,
+		})
 	}
 
 	return slice
@@ -76,7 +89,7 @@ func newAllocateIdentSlice(allocator *Allocator) []string {
 	var slice []string
 	for _, op := range allocator.pendingAllocates {
 		allocate := op.(*allocate)
-		slice = append(slice, allocate.ident)
+		slice = append(slice, fmt.Sprintf("%s %s", allocate.ident, allocate.r.String()))
 	}
 	return slice
 }

prog/weaver/http.go

@@ -1,8 +1,13 @@
 package main
 
 import (
+	"bytes"
 	"encoding/json"
 	"fmt"
+	"net/http"
+	"strings"
+	"text/template"
+
 	"github.com/gorilla/mux"
 	. "github.com/weaveworks/weave/common"
 	"github.com/weaveworks/weave/ipam"
@@ -10,9 +15,6 @@ import (
 	"github.com/weaveworks/weave/nameserver"
 	"github.com/weaveworks/weave/net/address"
 	weave "github.com/weaveworks/weave/router"
-	"net/http"
-	"strings"
-	"text/template"
 )
 
 var rootTemplate = template.New("root").Funcs(map[string]interface{}{
@@ -31,6 +33,60 @@ var rootTemplate = template.New("root").Funcs(map[string]interface{}{
 		}
 		return strings.Join(servers, ", ")
 	},
+	"printIPAMRanges": func(router weave.NetworkRouterStatus, status ipam.Status) string {
+		var buffer bytes.Buffer
+
+		type stats struct {
+			ips       uint32
+			nickname  string
+			reachable bool
+		}
+
+		peerStats := make(map[string]*stats)
+
+		for _, entry := range status.Entries {
+			s, found := peerStats[entry.Peer]
+			if !found {
+				s = &stats{nickname: entry.Nickname, reachable: entry.IsKnownPeer}
+				peerStats[entry.Peer] = s
+			}
+			s.ips += entry.Size
+		}
+
+		printOwned := func(name string, nickName string, reachable bool, ips uint32) {
+			reachableStr := ""
+			if !reachable {
+				reachableStr = "- unreachable!"
+			}
+			percentageRanges := float32(ips) * 100.0 / float32(status.RangeNumIPs)
+
+			displayName := name + "(" + nickName + ")"
+			fmt.Fprintf(&buffer, "%-37v %8d IPs (%04.1f%% of total) %s\n",
+				displayName, ips, percentageRanges, reachableStr)
+		}
+
+		// print the local info first
+		if ourStats := peerStats[router.Name]; ourStats != nil {
+			printOwned(router.Name, ourStats.nickname, true, ourStats.ips)
+		}
+
+		// and then the rest
+		for peer, stats := range peerStats {
+			if peer != router.Name {
+				printOwned(peer, stats.nickname, stats.reachable, stats.ips)
+			}
+		}
+
+		return buffer.String()
+	},
+	"allIPAMOwnersUnreachable": func(status ipam.Status) bool {
+		for _, entry := range status.Entries {
+			if entry.Size > 0 && entry.IsKnownPeer {
+				return false
+			}
+		}
+		return true
+	},
 	"printConnectionCounts": func(conns []mesh.LocalConnectionStatus) string {
 		counts := make(map[string]int)
 		for _, conn := range conns {
@@ -101,11 +157,17 @@ var statusTemplate = defTemplate("status", `\
 
        Service: ipam
 {{if .IPAM.Entries}}\
-     Consensus: achieved
-{{else if .IPAM.Paxos}}\
-     Consensus: waiting (quorum: {{.IPAM.Paxos.Quorum}}, known: {{.IPAM.Paxos.KnownNodes}})
+{{if allIPAMOwnersUnreachable .IPAM}}\
+        Status: all IP ranges owned by unreachable peers - use 'rmpeer' if they are dead
+{{else if len .IPAM.PendingAllocates}}\
+        Status: waiting for IP range grant from peers
 {{else}}\
-     Consensus: deferred
+        Status: ready
+{{end}}\
+{{else if .IPAM.Paxos}}\
+        Status: awaiting consensus (quorum: {{.IPAM.Paxos.Quorum}}, known: {{.IPAM.Paxos.KnownNodes}})
+{{else}}\
+        Status: idle
 {{end}}\
          Range: {{.IPAM.Range}}
  DefaultSubnet: {{.IPAM.DefaultSubnet}}
@@ -136,7 +198,7 @@ var peersTemplate = defTemplate("peers", `\
 {{.Name}}({{.NickName}})
 {{range .Connections}}\
    {{if .Outbound}}->{{else}}<-{{end}} {{printf "%-21v" .Address}} \
-{{$nameNickName := printf "%v(%v)" .Name .NickName}}{{printf "%-32v" $nameNickName}} \
+{{$nameNickName := printf "%v(%v)" .Name .NickName}}{{printf "%-37v" $nameNickName}} \
 {{if .Established}}established{{else}}pending{{end}}
 {{end}}\
 {{end}}\
@@ -152,6 +214,8 @@ var dnsEntriesTemplate = defTemplate("dnsEntries", `\
 {{end}}\
 `)
 
+var ipamTemplate = defTemplate("ipamTemplate", `{{printIPAMRanges .Router .IPAM}}`)
+
 type WeaveStatus struct {
 	Version string
 	Router  *weave.NetworkRouterStatus `json:"Router,omitempty"`
@@ -207,5 +271,6 @@ func HandleHTTP(muxRouter *mux.Router, version string, router *weave.NetworkRout
 	defHandler("/status/connections", connectionsTemplate)
 	defHandler("/status/peers", peersTemplate)
 	defHandler("/status/dns", dnsEntriesTemplate)
+	defHandler("/status/ipam", ipamTemplate)
 
 }