From b8d619dde7622faf0f48ef9e68ac4bcfe1a2ca30 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Thu, 20 Oct 2016 16:23:49 +0100 Subject: [PATCH] Add Prometheus http handler for metrics Metrics are: connection counts, active IP addresses, DNS addresses, FastDP flow metrics, connection termination count --- .gitmodules | 21 +++ prog/weaver/http.go | 12 ++ prog/weaver/metrics.go | 129 ++++++++++++++++++ vendor/github.com/beorn7/perks | 1 + vendor/github.com/golang/protobuf | 1 + .../matttproud/golang_protobuf_extensions | 1 + vendor/github.com/prometheus/client_golang | 1 + vendor/github.com/prometheus/client_model | 1 + vendor/github.com/prometheus/common | 1 + vendor/github.com/prometheus/procfs | 1 + vendor/github.com/weaveworks/mesh | 2 +- 11 files changed, 170 insertions(+), 1 deletion(-) create mode 100644 prog/weaver/metrics.go create mode 160000 vendor/github.com/beorn7/perks create mode 160000 vendor/github.com/golang/protobuf create mode 160000 vendor/github.com/matttproud/golang_protobuf_extensions create mode 160000 vendor/github.com/prometheus/client_golang create mode 160000 vendor/github.com/prometheus/client_model create mode 160000 vendor/github.com/prometheus/common create mode 160000 vendor/github.com/prometheus/procfs diff --git a/.gitmodules b/.gitmodules index 6315c8bb..7ed8985c 100644 --- a/.gitmodules +++ b/.gitmodules @@ -82,3 +82,24 @@ [submodule "vendor/github.com/weaveworks/docker"] path = vendor/github.com/weaveworks/docker url = https://github.com/weaveworks/docker +[submodule "vendor/github.com/beorn7/perks"] + path = vendor/github.com/beorn7/perks + url = https://github.com/beorn7/perks +[submodule "vendor/github.com/prometheus/procfs"] + path = vendor/github.com/prometheus/procfs + url = https://github.com/prometheus/procfs +[submodule "vendor/github.com/prometheus/client_golang"] + path = vendor/github.com/prometheus/client_golang + url = https://github.com/prometheus/client_golang +[submodule "vendor/github.com/prometheus/client_model"] + path = vendor/github.com/prometheus/client_model + url = https://github.com/prometheus/client_model +[submodule "vendor/github.com/prometheus/common"] + path = vendor/github.com/prometheus/common + url = https://github.com/prometheus/common +[submodule "vendor/github.com/golang/protobuf"] + path = vendor/github.com/golang/protobuf + url = https://github.com/golang/protobuf +[submodule "vendor/github.com/matttproud/golang_protobuf_extensions"] + path = vendor/github.com/matttproud/golang_protobuf_extensions + url = https://github.com/matttproud/golang_protobuf_extensions diff --git a/prog/weaver/http.go b/prog/weaver/http.go index f539ad12..8ce73227 100644 --- a/prog/weaver/http.go +++ b/prog/weaver/http.go @@ -121,6 +121,16 @@ func countDNSEntries(entries []nameserver.EntryStatus) int { return count } +func countDNSEntriesForPeer(peername string, entries []nameserver.EntryStatus) int { + count := 0 + for _, entry := range entries { + if entry.Tombstone == 0 && entry.Origin == peername { + count++ + } + } + return count +} + // Print counts in a specified order func printCounts(counts map[string]int, keys []string) string { var stringCounts []string @@ -322,4 +332,6 @@ func HandleHTTP(muxRouter *mux.Router, version string, router *weave.NetworkRout defHandler("/status/peers", peersTemplate) defHandler("/status/dns", dnsEntriesTemplate) defHandler("/status/ipam", ipamTemplate) + + muxRouter.Methods("GET").Path("/metrics").Handler(metricsHandler(router, allocator, ns)) } diff --git a/prog/weaver/metrics.go b/prog/weaver/metrics.go new file mode 100644 index 00000000..f5198a48 --- /dev/null +++ b/prog/weaver/metrics.go @@ -0,0 +1,129 @@ +package main + +import ( + "net/http" + "os" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + + "github.com/weaveworks/weave/ipam" + "github.com/weaveworks/weave/nameserver" + "github.com/weaveworks/weave/net/address" + weave "github.com/weaveworks/weave/router" +) + +func metricsHandler(router *weave.NetworkRouter, allocator *ipam.Allocator, ns *nameserver.Nameserver, dnsserver *nameserver.DNSServer) http.Handler { + reg := prometheus.NewRegistry() + reg.MustRegister(prometheus.NewProcessCollector(os.Getpid(), "")) + reg.MustRegister(newMetrics(router, allocator, ns, dnsserver)) + return promhttp.HandlerFor(reg, promhttp.HandlerOpts{}) +} + +type collector struct { + router *weave.NetworkRouter + allocator *ipam.Allocator + ns *nameserver.Nameserver + dnsserver *nameserver.DNSServer +} + +type metric struct { + *prometheus.Desc + Collect func(WeaveStatus, *prometheus.Desc, chan<- prometheus.Metric) +} + +func desc(fqName, help string, variableLabels ...string) *prometheus.Desc { + return prometheus.NewDesc(fqName, help, variableLabels, prometheus.Labels{}) +} + +func intGauge(desc *prometheus.Desc, val int, labels ...string) prometheus.Metric { + return prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, float64(val), labels...) +} +func uint64Counter(desc *prometheus.Desc, val uint64, labels ...string) prometheus.Metric { + return prometheus.MustNewConstMetric(desc, prometheus.CounterValue, float64(val), labels...) +} + +var metrics []metric = []metric{ + {desc("weave_connections", "Number of peer-to-peer connections.", "state"), + func(s WeaveStatus, desc *prometheus.Desc, ch chan<- prometheus.Metric) { + established := 0 + for _, conn := range s.Router.Connections { + if conn.State == "established" { + established++ + } + } + + ch <- intGauge(desc, len(s.Router.Connections)-established, "non-established") + ch <- intGauge(desc, established, "established") + }}, + {desc("weave_connection_termination_count", "Number of peer-to-peer connections terminated."), + func(s WeaveStatus, desc *prometheus.Desc, ch chan<- prometheus.Metric) { + ch <- uint64Counter(desc, uint64(s.Router.TerminationCount)) + }}, + {desc("weave_ips", "Number of IP addresses.", "state"), + func(s WeaveStatus, desc *prometheus.Desc, ch chan<- prometheus.Metric) { + ch <- intGauge(desc, s.IPAM.RangeNumIPs, "total") + ch <- intGauge(desc, s.IPAM.ActiveIPs, "local-used") + }}, + {desc("weave_dns_entries", "Number of DNS entries.", "state"), + func(s WeaveStatus, desc *prometheus.Desc, ch chan<- prometheus.Metric) { + ch <- intGauge(desc, countDNSEntries(s.DNS.Entries), "total") + ch <- intGauge(desc, countDNSEntriesForPeer(s.Router.Name, s.DNS.Entries), "local") + }}, + {desc("weave_flows", "Number of FastDP flows."), + func(s WeaveStatus, desc *prometheus.Desc, ch chan<- prometheus.Metric) { + if metrics := fastDPMetrics(s); metrics != nil { + ch <- intGauge(desc, metrics.Flows) + } + }}, + {desc("weave_packets_total", "Number of packets transferred.", "flow"), + func(s WeaveStatus, desc *prometheus.Desc, ch chan<- prometheus.Metric) { + if metrics := fastDPMetrics(s); metrics != nil { + ch <- uint64Counter(desc, metrics.TotalPackets, "total") + } + }}, + {desc("weave_bytes_total", "Number of bytes transferred.", "flow"), + func(s WeaveStatus, desc *prometheus.Desc, ch chan<- prometheus.Metric) { + if metrics := fastDPMetrics(s); metrics != nil { + ch <- uint64Counter(desc, metrics.TotalBytes, "total") + } + }}, +} + +func fastDPMetrics(s WeaveStatus) *weave.FastDPMetrics { + if reporter, ok := s.Router.OverlayDiagnostics.(weave.MetricsReporter); ok { + if metrics := reporter.Metrics(); metrics != nil { + if fastdpMetrics, ok := metrics.(*weave.FastDPMetrics); ok { + return fastdpMetrics + } + } + } + return nil +} + +func newMetrics(router *weave.NetworkRouter, allocator *ipam.Allocator, ns *nameserver.Nameserver, dnsserver *nameserver.DNSServer) *collector { + return &collector{ + router: router, + allocator: allocator, + ns: ns, + dnsserver: dnsserver, + } +} + +func (m *collector) Collect(ch chan<- prometheus.Metric) { + + status := WeaveStatus{"", nil, + weave.NewNetworkRouterStatus(m.router), + ipam.NewStatus(m.allocator, address.CIDR{}), + nameserver.NewStatus(m.ns, m.dnsserver)} + + for _, metric := range metrics { + metric.Collect(status, metric.Desc, ch) + } +} + +func (m *collector) Describe(ch chan<- *prometheus.Desc) { + for _, metric := range metrics { + ch <- metric.Desc + } +} diff --git a/vendor/github.com/beorn7/perks b/vendor/github.com/beorn7/perks new file mode 160000 index 00000000..4c0e8459 --- /dev/null +++ b/vendor/github.com/beorn7/perks @@ -0,0 +1 @@ +Subproject commit 4c0e84591b9aa9e6dcfdf3e020114cd81f89d5f9 diff --git a/vendor/github.com/golang/protobuf b/vendor/github.com/golang/protobuf new file mode 160000 index 00000000..df1d3ca0 --- /dev/null +++ b/vendor/github.com/golang/protobuf @@ -0,0 +1 @@ +Subproject commit df1d3ca07d2d07bba352d5b73c4313b4e2a6203e diff --git a/vendor/github.com/matttproud/golang_protobuf_extensions b/vendor/github.com/matttproud/golang_protobuf_extensions new file mode 160000 index 00000000..3247c845 --- /dev/null +++ b/vendor/github.com/matttproud/golang_protobuf_extensions @@ -0,0 +1 @@ +Subproject commit 3247c84500bff8d9fb6d579d800f20b3e091582c diff --git a/vendor/github.com/prometheus/client_golang b/vendor/github.com/prometheus/client_golang new file mode 160000 index 00000000..c5b7fccd --- /dev/null +++ b/vendor/github.com/prometheus/client_golang @@ -0,0 +1 @@ +Subproject commit c5b7fccd204277076155f10851dad72b76a49317 diff --git a/vendor/github.com/prometheus/client_model b/vendor/github.com/prometheus/client_model new file mode 160000 index 00000000..fa8ad6fe --- /dev/null +++ b/vendor/github.com/prometheus/client_model @@ -0,0 +1 @@ +Subproject commit fa8ad6fec33561be4280a8f0514318c79d7f6cb6 diff --git a/vendor/github.com/prometheus/common b/vendor/github.com/prometheus/common new file mode 160000 index 00000000..85637ea6 --- /dev/null +++ b/vendor/github.com/prometheus/common @@ -0,0 +1 @@ +Subproject commit 85637ea67b04b5c3bb25e671dacded2977f8f9f6 diff --git a/vendor/github.com/prometheus/procfs b/vendor/github.com/prometheus/procfs new file mode 160000 index 00000000..abf152e5 --- /dev/null +++ b/vendor/github.com/prometheus/procfs @@ -0,0 +1 @@ +Subproject commit abf152e5f3e97f2fafac028d2cc06c1feb87ffa5 diff --git a/vendor/github.com/weaveworks/mesh b/vendor/github.com/weaveworks/mesh index 3e1ae95e..80411e92 160000 --- a/vendor/github.com/weaveworks/mesh +++ b/vendor/github.com/weaveworks/mesh @@ -1 +1 @@ -Subproject commit 3e1ae95e2359c0828318fc0a593f2d08e3091c96 +Subproject commit 80411e92f3acb085a75a9eba545550e8f79de34c