Package list golang-github-go-kit-kit / 49d9075
Add support for metrics.Histogram distribution Histograms gain a method to read the distribution (slice of buckets) that has been observed so far. In this PR, the only implementation that supports Distribution is expvar (via codahale/hdrhistogram). Prometheus support is possible and planned. - Each metrics type gains a Name() method - metrics.Histogram gains Distribution() - metrics package gains PrintDistribution() - Minor updates to README Peter Bourgon 5 years ago
10 changed file(s) with 258 addition(s) and 62 deletion(s). Raw diff Collapse all Expand all
1212
1313 ## Rationale
1414
15 TODO
15 Code instrumentation is absolutely essential to achieve [observability][] into a distributed system.
16 Metrics and instrumentation tools have coalesced around a few well-defined idioms.
17 `package metrics` provides a common, minimal interface those idioms for service authors.
18
19 [observability]: https://speakerdeck.com/mattheath/observability-in-micro-service-architectures
1620
1721 ## Usage
1822
5256 ```
5357
5458 A gauge for the number of goroutines currently running, exported via statsd.
59
5560 ```go
5661 import (
5762 "net"
6570 func main() {
6671 statsdWriter, err := net.Dial("udp", "127.0.0.1:8126")
6772 if err != nil {
68 os.Exit(1)
73 panic(err)
6974 }
7075
71 reportingDuration := 5 * time.Second
72 goroutines := statsd.NewGauge(statsdWriter, "total_goroutines", reportingDuration)
73 for range time.Tick(reportingDuration) {
76 reportInterval := 5 * time.Second
77 goroutines := statsd.NewGauge(statsdWriter, "total_goroutines", reportInterval)
78 for range time.Tick(reportInterval) {
7479 goroutines.Set(float64(runtime.NumGoroutine()))
7580 }
7681 }
77
7882 ```
2828 )
2929
3030 type counter struct {
31 v *expvar.Int
31 name string
32 v *expvar.Int
3233 }
3334
3435 // NewCounter returns a new Counter backed by an expvar with the given name.
3536 // Fields are ignored.
3637 func NewCounter(name string) metrics.Counter {
37 return &counter{expvar.NewInt(name)}
38 return &counter{
39 name: name,
40 v: expvar.NewInt(name),
41 }
3842 }
3943
44 func (c *counter) Name() string { return c.name }
4045 func (c *counter) With(metrics.Field) metrics.Counter { return c }
4146 func (c *counter) Add(delta uint64) { c.v.Add(int64(delta)) }
4247
4348 type gauge struct {
44 v *expvar.Float
49 name string
50 v *expvar.Float
4551 }
4652
4753 // NewGauge returns a new Gauge backed by an expvar with the given name. It
4854 // should be updated manually; for a callback-based approach, see
4955 // PublishCallbackGauge. Fields are ignored.
5056 func NewGauge(name string) metrics.Gauge {
51 return &gauge{expvar.NewFloat(name)}
57 return &gauge{
58 name: name,
59 v: expvar.NewFloat(name),
60 }
5261 }
5362
63 func (g *gauge) Name() string { return g.name }
5464 func (g *gauge) With(metrics.Field) metrics.Gauge { return g }
55
56 func (g *gauge) Add(delta float64) { g.v.Add(delta) }
57
58 func (g *gauge) Set(value float64) { g.v.Set(value) }
65 func (g *gauge) Add(delta float64) { g.v.Add(delta) }
66 func (g *gauge) Set(value float64) { g.v.Set(value) }
5967
6068 // PublishCallbackGauge publishes a Gauge as an expvar with the given name,
6169 // whose value is determined at collect time by the passed callback function.
100108 return h
101109 }
102110
111 func (h *histogram) Name() string { return h.name }
103112 func (h *histogram) With(metrics.Field) metrics.Histogram { return h }
104113
105114 func (h *histogram) Observe(value int64) {
116125 }
117126 }
118127
128 func (h *histogram) Distribution() []metrics.Bucket {
129 bars := h.hist.Current.Distribution()
130 buckets := make([]metrics.Bucket, len(bars))
131 for i, bar := range bars {
132 buckets[i] = metrics.Bucket{
133 From: bar.From,
134 To: bar.To,
135 Count: bar.Count,
136 }
137 }
138 return buckets
139 }
140
119141 func (h *histogram) rotateLoop(d time.Duration) {
120142 for range time.Tick(d) {
121143 h.mu.Lock()
1111
1212 func TestHistogramQuantiles(t *testing.T) {
1313 var (
14 name = "test_histogram"
14 name = "test_histogram_quantiles"
1515 quantiles = []int{50, 90, 95, 99}
1616 h = expvar.NewHistogram(name, 0, 100, 3, quantiles...).With(metrics.Field{Key: "ignored", Value: "field"})
1717 )
88 // between measurements of a counter over intervals of time, an aggregation
99 // layer can derive rates, acceleration, etc.
1010 type Counter interface {
11 Name() string
1112 With(Field) Counter
1213 Add(delta uint64)
1314 }
1516 // Gauge captures instantaneous measurements of something using signed, 64-bit
1617 // floats. The value does not need to be monotonic.
1718 type Gauge interface {
19 Name() string
1820 With(Field) Gauge
1921 Set(value float64)
2022 Add(delta float64)
2426 // milliseconds it takes to handle requests). Implementations may choose to
2527 // add gauges for values at meaningful quantiles.
2628 type Histogram interface {
29 Name() string
2730 With(Field) Histogram
2831 Observe(value int64)
32 Distribution() []Bucket
2933 }
3034
3135 // Field is a key/value pair associated with an observation for a specific
3438 Key string
3539 Value string
3640 }
41
42 // Bucket is a range in a histogram which aggregates observations.
43 type Bucket struct {
44 From int64
45 To int64
46 Count int64
47 }
00 package metrics
11
2 type multiCounter []Counter
2 type multiCounter struct {
3 name string
4 a []Counter
5 }
36
47 // NewMultiCounter returns a wrapper around multiple Counters.
5 func NewMultiCounter(counters ...Counter) Counter {
6 c := make(multiCounter, 0, len(counters))
7 return append(c, counters...)
8 func NewMultiCounter(name string, counters ...Counter) Counter {
9 return &multiCounter{
10 name: name,
11 a: counters,
12 }
813 }
914
15 func (c multiCounter) Name() string { return c.name }
16
1017 func (c multiCounter) With(f Field) Counter {
11 next := make(multiCounter, len(c))
12 for i, counter := range c {
13 next[i] = counter.With(f)
18 next := &multiCounter{
19 name: c.name,
20 a: make([]Counter, len(c.a)),
21 }
22 for i, counter := range c.a {
23 next.a[i] = counter.With(f)
1424 }
1525 return next
1626 }
1727
1828 func (c multiCounter) Add(delta uint64) {
19 for _, counter := range c {
29 for _, counter := range c.a {
2030 counter.Add(delta)
2131 }
2232 }
2333
24 type multiGauge []Gauge
34 type multiGauge struct {
35 name string
36 a []Gauge
37 }
38
39 func (g multiGauge) Name() string { return g.name }
2540
2641 // NewMultiGauge returns a wrapper around multiple Gauges.
27 func NewMultiGauge(gauges ...Gauge) Gauge {
28 g := make(multiGauge, 0, len(gauges))
29 return append(g, gauges...)
42 func NewMultiGauge(name string, gauges ...Gauge) Gauge {
43 return &multiGauge{
44 name: name,
45 a: gauges,
46 }
3047 }
3148
3249 func (g multiGauge) With(f Field) Gauge {
33 next := make(multiGauge, len(g))
34 for i, gauge := range g {
35 next[i] = gauge.With(f)
50 next := &multiGauge{
51 name: g.name,
52 a: make([]Gauge, len(g.a)),
53 }
54 for i, gauge := range g.a {
55 next.a[i] = gauge.With(f)
3656 }
3757 return next
3858 }
3959
4060 func (g multiGauge) Set(value float64) {
41 for _, gauge := range g {
61 for _, gauge := range g.a {
4262 gauge.Set(value)
4363 }
4464 }
4565
4666 func (g multiGauge) Add(delta float64) {
47 for _, gauge := range g {
67 for _, gauge := range g.a {
4868 gauge.Add(delta)
4969 }
5070 }
5171
52 type multiHistogram []Histogram
72 type multiHistogram struct {
73 name string
74 a []Histogram
75 }
5376
5477 // NewMultiHistogram returns a wrapper around multiple Histograms.
55 func NewMultiHistogram(histograms ...Histogram) Histogram {
56 h := make(multiHistogram, 0, len(histograms))
57 return append(h, histograms...)
78 func NewMultiHistogram(name string, histograms ...Histogram) Histogram {
79 return &multiHistogram{
80 name: name,
81 a: histograms,
82 }
5883 }
5984
85 func (h multiHistogram) Name() string { return h.name }
86
6087 func (h multiHistogram) With(f Field) Histogram {
61 next := make(multiHistogram, len(h))
62 for i, histogram := range h {
63 next[i] = histogram.With(f)
88 next := &multiHistogram{
89 name: h.name,
90 a: make([]Histogram, len(h.a)),
91 }
92 for i, histogram := range h.a {
93 next.a[i] = histogram.With(f)
6494 }
6595 return next
6696 }
6797
6898 func (h multiHistogram) Observe(value int64) {
69 for _, histogram := range h {
99 for _, histogram := range h.a {
70100 histogram.Observe(value)
71101 }
72102 }
103
104 func (h multiHistogram) Distribution() []Bucket {
105 return []Bucket{} // TODO(pb): can this be statistically valid?
106 }
2121
2222 func TestMultiWith(t *testing.T) {
2323 c := metrics.NewMultiCounter(
24 "multifoo",
2425 expvar.NewCounter("foo"),
2526 prometheus.NewCounter(stdprometheus.CounterOpts{
2627 Namespace: "test",
4647
4748 func TestMultiCounter(t *testing.T) {
4849 metrics.NewMultiCounter(
50 "multialpha",
4951 expvar.NewCounter("alpha"),
5052 prometheus.NewCounter(stdprometheus.CounterOpts{
5153 Namespace: "test",
7072
7173 func TestMultiGauge(t *testing.T) {
7274 g := metrics.NewMultiGauge(
75 "multidelta",
7376 expvar.NewGauge("delta"),
7477 prometheus.NewGauge(stdprometheus.GaugeOpts{
7578 Namespace: "test",
110113 func TestMultiHistogram(t *testing.T) {
111114 quantiles := []int{50, 90, 99}
112115 h := metrics.NewMultiHistogram(
116 "multiomicron",
113117 expvar.NewHistogram("omicron", 0, 100, 3, quantiles...),
114118 prometheus.NewSummary(stdprometheus.SummaryOpts{
115119 Namespace: "test",
0 package metrics
1
2 import (
3 "fmt"
4 "io"
5 "text/tabwriter"
6 )
7
8 const (
9 bs = "####################################################################################################"
10 bsz = float64(len(bs))
11 )
12
13 // PrintDistribution writes a human-readable graph of the distribution to the
14 // passed writer.
15 func PrintDistribution(w io.Writer, name string, buckets []Bucket) {
16 fmt.Fprintf(w, "name: %v\n", name)
17
18 var total float64
19 for _, bucket := range buckets {
20 total += float64(bucket.Count)
21 }
22
23 tw := tabwriter.NewWriter(w, 0, 2, 2, ' ', 0)
24 fmt.Fprintf(tw, "From\tTo\tCount\tProb\tBar\n")
25
26 axis := "|"
27 for _, bucket := range buckets {
28 if bucket.Count > 0 {
29 p := float64(bucket.Count) / total
30 fmt.Fprintf(tw, "%d\t%d\t%d\t%.4f\t%s%s\n", bucket.From, bucket.To, bucket.Count, p, axis, bs[:int(p*bsz)])
31 axis = "|"
32 } else {
33 axis = ":" // show that some bars were skipped
34 }
35 }
36
37 tw.Flush() // to buf
38 }
0 package metrics_test
1
2 import (
3 "os"
4 "testing"
5
6 "github.com/go-kit/kit/metrics"
7 "github.com/go-kit/kit/metrics/expvar"
8 "github.com/go-kit/kit/metrics/teststat"
9 )
10
11 func TestPrintDistribution(t *testing.T) {
12 var (
13 name = "foobar"
14 quantiles = []int{50, 90, 95, 99}
15 h = expvar.NewHistogram("test_print_distribution", 1, 10, 3, quantiles...)
16 seed = int64(555)
17 mean = int64(5)
18 stdev = int64(1)
19 )
20 teststat.PopulateNormalHistogram(t, h, seed, mean, stdev)
21 metrics.PrintDistribution(os.Stdout, name, h.Distribution())
22 }
1515
1616 type prometheusCounter struct {
1717 *prometheus.CounterVec
18 name string
1819 Pairs map[string]string
1920 }
2021
2930 }
3031 return prometheusCounter{
3132 CounterVec: m,
33 name: opts.Name,
3234 Pairs: p,
3335 }
3436 }
3537
38 func (c prometheusCounter) Name() string { return c.name }
39
3640 func (c prometheusCounter) With(f metrics.Field) metrics.Counter {
3741 return prometheusCounter{
3842 CounterVec: c.CounterVec,
43 name: c.name,
3944 Pairs: merge(c.Pairs, f),
4045 }
4146 }
4651
4752 type prometheusGauge struct {
4853 *prometheus.GaugeVec
54 name string
4955 Pairs map[string]string
5056 }
5157
5662 prometheus.MustRegister(m)
5763 return prometheusGauge{
5864 GaugeVec: m,
65 name: opts.Name,
5966 Pairs: pairsFrom(fieldKeys),
6067 }
6168 }
6269
70 func (g prometheusGauge) Name() string { return g.name }
71
6372 func (g prometheusGauge) With(f metrics.Field) metrics.Gauge {
6473 return prometheusGauge{
6574 GaugeVec: g.GaugeVec,
75 name: g.name,
6676 Pairs: merge(g.Pairs, f),
6777 }
6878 }
8595
8696 type prometheusSummary struct {
8797 *prometheus.SummaryVec
98 name string
8899 Pairs map[string]string
89100 }
90101
98109 prometheus.MustRegister(m)
99110 return prometheusSummary{
100111 SummaryVec: m,
112 name: opts.Name,
101113 Pairs: pairsFrom(fieldKeys),
102114 }
103115 }
104116
117 func (s prometheusSummary) Name() string { return s.name }
118
105119 func (s prometheusSummary) With(f metrics.Field) metrics.Histogram {
106120 return prometheusSummary{
107121 SummaryVec: s.SummaryVec,
122 name: s.name,
108123 Pairs: merge(s.Pairs, f),
109124 }
110125 }
113128 s.SummaryVec.With(prometheus.Labels(s.Pairs)).Observe(float64(value))
114129 }
115130
131 func (s prometheusSummary) Distribution() []metrics.Bucket {
132 // TODO(pb): see https://github.com/prometheus/client_golang/issues/58
133 return []metrics.Bucket{}
134 }
135
116136 type prometheusHistogram struct {
117137 *prometheus.HistogramVec
138 name string
118139 Pairs map[string]string
119140 }
120141
128149 prometheus.MustRegister(m)
129150 return prometheusHistogram{
130151 HistogramVec: m,
152 name: opts.Name,
131153 Pairs: pairsFrom(fieldKeys),
132154 }
133155 }
134156
157 func (h prometheusHistogram) Name() string { return h.name }
158
135159 func (h prometheusHistogram) With(f metrics.Field) metrics.Histogram {
136160 return prometheusHistogram{
137161 HistogramVec: h.HistogramVec,
162 name: h.name,
138163 Pairs: merge(h.Pairs, f),
139164 }
140165 }
141166
142167 func (h prometheusHistogram) Observe(value int64) {
143168 h.HistogramVec.With(prometheus.Labels(h.Pairs)).Observe(float64(value))
169 }
170
171 func (h prometheusHistogram) Distribution() []metrics.Bucket {
172 // TODO(pb): see https://github.com/prometheus/client_golang/issues/58
173 return []metrics.Bucket{}
144174 }
145175
146176 func pairsFrom(fieldKeys []string) map[string]string {
2626
2727 const maxBufferSize = 1400 // bytes
2828
29 type statsdCounter chan string
29 type statsdCounter struct {
30 key string
31 c chan string
32 }
3033
3134 // NewCounter returns a Counter that emits observations in the statsd protocol
3235 // to the passed writer. Observations are buffered for the report interval or
3538 //
3639 // TODO: support for sampling.
3740 func NewCounter(w io.Writer, key string, reportInterval time.Duration) metrics.Counter {
38 c := make(chan string)
39 go fwd(w, key, reportInterval, c)
40 return statsdCounter(c)
41 c := &statsdCounter{
42 key: key,
43 c: make(chan string),
44 }
45 go fwd(w, key, reportInterval, c.c)
46 return c
4147 }
4248
43 func (c statsdCounter) With(metrics.Field) metrics.Counter { return c }
49 func (c *statsdCounter) Name() string { return c.key }
4450
45 func (c statsdCounter) Add(delta uint64) { c <- fmt.Sprintf("%d|c", delta) }
51 func (c *statsdCounter) With(metrics.Field) metrics.Counter { return c }
4652
47 type statsdGauge chan string
53 func (c *statsdCounter) Add(delta uint64) { c.c <- fmt.Sprintf("%d|c", delta) }
54
55 type statsdGauge struct {
56 key string
57 g chan string
58 }
4859
4960 // NewGauge returns a Gauge that emits values in the statsd protocol to the
5061 // passed writer. Values are buffered for the report interval or until the
5364 //
5465 // TODO: support for sampling.
5566 func NewGauge(w io.Writer, key string, reportInterval time.Duration) metrics.Gauge {
56 g := make(chan string)
57 go fwd(w, key, reportInterval, g)
58 return statsdGauge(g)
67 g := &statsdGauge{
68 key: key,
69 g: make(chan string),
70 }
71 go fwd(w, key, reportInterval, g.g)
72 return g
5973 }
6074
61 func (g statsdGauge) With(metrics.Field) metrics.Gauge { return g }
75 func (g *statsdGauge) Name() string { return g.key }
6276
63 func (g statsdGauge) Add(delta float64) {
77 func (g *statsdGauge) With(metrics.Field) metrics.Gauge { return g }
78
79 func (g *statsdGauge) Add(delta float64) {
6480 // https://github.com/etsy/statsd/blob/master/docs/metric_types.md#gauges
6581 sign := "+"
6682 if delta < 0 {
6783 sign, delta = "-", -delta
6884 }
69 g <- fmt.Sprintf("%s%f|g", sign, delta)
85 g.g <- fmt.Sprintf("%s%f|g", sign, delta)
7086 }
7187
72 func (g statsdGauge) Set(value float64) {
73 g <- fmt.Sprintf("%f|g", value)
88 func (g *statsdGauge) Set(value float64) {
89 g.g <- fmt.Sprintf("%f|g", value)
7490 }
7591
7692 // NewCallbackGauge emits values in the statsd protocol to the passed writer.
93109 return c
94110 }
95111
96 type statsdHistogram chan string
112 type statsdHistogram struct {
113 key string
114 h chan string
115 }
97116
98117 // NewHistogram returns a Histogram that emits observations in the statsd
99118 // protocol to the passed writer. Observations are buffered for the reporting
113132 //
114133 // TODO: support for sampling.
115134 func NewHistogram(w io.Writer, key string, reportInterval time.Duration) metrics.Histogram {
116 h := make(chan string)
117 go fwd(w, key, reportInterval, h)
118 return statsdHistogram(h)
135 h := &statsdHistogram{
136 key: key,
137 h: make(chan string),
138 }
139 go fwd(w, key, reportInterval, h.h)
140 return h
119141 }
120142
121 func (h statsdHistogram) With(metrics.Field) metrics.Histogram { return h }
143 func (h *statsdHistogram) Name() string { return h.key }
122144
123 func (h statsdHistogram) Observe(value int64) {
124 h <- fmt.Sprintf("%d|ms", value)
145 func (h *statsdHistogram) With(metrics.Field) metrics.Histogram { return h }
146
147 func (h *statsdHistogram) Observe(value int64) {
148 h.h <- fmt.Sprintf("%d|ms", value)
149 }
150
151 func (h *statsdHistogram) Distribution() []metrics.Bucket {
152 // TODO(pb): no way to do this without introducing e.g. codahale/hdrhistogram
153 return []metrics.Bucket{}
125154 }
126155
127156 var tick = time.Tick