Package list golang-github-go-kit-kit / 4928fe6
package metrics RFC + first draft implementation Peter Bourgon 6 years ago
15 changed file(s) with 1163 addition(s) and 2 deletion(s). Raw diff Collapse all Expand all
2222 - Having opinions on deployment, orchestration, process supervision
2323 - Having opinions on configuration passing -- flags vs. env vars vs. files vs. ...
2424 - _more TODO_
25
26 ## Contributing
27
28 At this stage, we're still developing the initial drafts of all of the
29 packages, using an
30 [RFC workflow](https://github.com/peterbourgon/gokit/tree/master/rfc).
31 Before submitting major changes, please write to
32 [the mailing list](groups.google.com/forum/#!forum/go-kit)
33 to register your interest, and check the
34 [open issues](https://github.com/peterbourgon/gokit/issues) and
35 [pull requests](https://github.com/peterbourgon/gokit/pulls)
36 for existing discussions.
37
38 ### Dependency management
39
40 Users who import gokit into their `package main` are responsible to organize
41 and maintain all of their dependencies to ensure code compatibility and build
42 reproducibility. Gokit makes no direct use of dependency management tools like
43 [Godep](https://github.com/tools/godep).
44
45 We will use a variety of continuous integration providers to find and fix
46 compatibility problems as soon as they occur.
2547
2648 ## Related projects
2749
0 // Package expvar implements an expvar backend for package metrics.
1 //
2 // The current implementation ignores fields. In the future, it would be good
3 // to have an implementation that accepted a set of predeclared field names at
4 // construction time, and used field values to produce delimiter-separated
5 // bucket (key) names. That is,
6 //
7 // c := NewFieldedCounter(..., "path", "status")
8 // c.Add(1) // "myprefix_unknown_unknown" += 1
9 // c2 := c.With("path", "foo").With("status": "200")
10 // c2.Add(1) // "myprefix_foo_status" += 1
11 //
12 // It would also be possible to have an implementation that generated more
13 // sophisticated expvar.Values. For example, a Counter could be implemented as
14 // a map, representing a tree of key/value pairs whose leaves were the actual
15 // expvar.Ints.
16 package expvar
17
18 import (
19 "expvar"
20 "fmt"
21 "sync"
22 "time"
23
24 "github.com/peterbourgon/gokit/metrics"
25
26 "github.com/codahale/hdrhistogram"
27 )
28
29 type counter struct {
30 v *expvar.Int
31 }
32
33 // NewCounter returns a new Counter backed by an expvar with the given name.
34 // Fields are ignored.
35 func NewCounter(name string) metrics.Counter {
36 return &counter{expvar.NewInt(name)}
37 }
38
39 func (c *counter) With(metrics.Field) metrics.Counter { return c }
40
41 func (c *counter) Add(delta uint64) { c.v.Add(int64(delta)) }
42
43 type gauge struct {
44 v *expvar.Int
45 }
46
47 // NewGauge returns a new Gauge backed by an expvar with the given name.
48 // Fields are ignored.
49 func NewGauge(name string) metrics.Gauge {
50 return &gauge{expvar.NewInt(name)}
51 }
52
53 func (g *gauge) With(metrics.Field) metrics.Gauge { return g }
54
55 func (g *gauge) Add(delta int64) { g.v.Add(delta) }
56
57 func (g *gauge) Set(value int64) { g.v.Set(value) }
58
59 type histogram struct {
60 mu sync.Mutex
61 hist *hdrhistogram.WindowedHistogram
62
63 name string
64 gauges map[int]metrics.Gauge
65 }
66
67 // NewHistogram is taken from http://github.com/codahale/metrics. It returns a
68 // windowed HDR histogram which drops data older than five minutes.
69 //
70 // The histogram exposes metrics for each passed quantile as gauges. Quantiles
71 // should be integers in the range 1..99. The gauge names are assigned by
72 // using the passed name as a prefix and appending "_pNN" e.g. "_p50".
73 func NewHistogram(name string, minValue, maxValue int64, sigfigs int, quantiles ...int) metrics.Histogram {
74 gauges := map[int]metrics.Gauge{}
75 for _, quantile := range quantiles {
76 if quantile <= 0 || quantile >= 100 {
77 panic(fmt.Sprintf("invalid quantile %d", quantile))
78 }
79 gauges[quantile] = NewGauge(fmt.Sprintf("%s_p%02d", name, quantile))
80 }
81 h := &histogram{
82 hist: hdrhistogram.NewWindowed(5, minValue, maxValue, sigfigs),
83 name: name,
84 gauges: gauges,
85 }
86 go h.rotateLoop(1 * time.Minute)
87 return h
88 }
89
90 func (h *histogram) With(metrics.Field) metrics.Histogram { return h }
91
92 func (h *histogram) Observe(value int64) {
93 h.mu.Lock()
94 err := h.hist.Current.RecordValue(value)
95 h.mu.Unlock()
96
97 if err != nil {
98 panic(err.Error())
99 }
100
101 for q, gauge := range h.gauges {
102 gauge.Set(h.hist.Current.ValueAtQuantile(float64(q)))
103 }
104 }
105
106 func (h *histogram) rotateLoop(d time.Duration) {
107 for _ = range time.Tick(d) {
108 h.mu.Lock()
109 h.hist.Rotate()
110 h.mu.Unlock()
111 }
112 }
0 package expvar_test
1
2 import (
3 stdexpvar "expvar"
4 "fmt"
5 "math"
6 "math/rand"
7 "strconv"
8 "testing"
9
10 "github.com/peterbourgon/gokit/metrics"
11 "github.com/peterbourgon/gokit/metrics/expvar"
12 )
13
14 func TestHistogramQuantiles(t *testing.T) {
15 metricName := "test_histogram"
16 quantiles := []int{50, 90, 95, 99}
17 h := expvar.NewHistogram(metricName, 0, 100, 3, quantiles...)
18
19 const seed, mean, stdev int64 = 424242, 50, 10
20 populateNormalHistogram(t, h, seed, mean, stdev)
21 assertNormalHistogram(t, metricName, mean, stdev, quantiles)
22 }
23
24 func populateNormalHistogram(t *testing.T, h metrics.Histogram, seed int64, mean, stdev int64) {
25 rand.Seed(seed)
26 for i := 0; i < 1234; i++ {
27 sample := int64(rand.NormFloat64()*float64(stdev) + float64(mean))
28 h.Observe(sample)
29 }
30 }
31
32 func assertNormalHistogram(t *testing.T, metricName string, mean, stdev int64, quantiles []int) {
33 const tolerance int = 2
34 for _, quantile := range quantiles {
35 want := normalValueAtQuantile(mean, stdev, quantile)
36 s := stdexpvar.Get(fmt.Sprintf("%s_p%02d", metricName, quantile)).String()
37 have, err := strconv.Atoi(s)
38 if err != nil {
39 t.Fatal(err)
40 }
41 if int(math.Abs(float64(want)-float64(have))) > tolerance {
42 t.Errorf("quantile %d: want %d, have %d", quantile, want, have)
43 }
44 }
45 }
46
47 // https://en.wikipedia.org/wiki/Normal_distribution#Quantile_function
48 func normalValueAtQuantile(mean, stdev int64, quantile int) int64 {
49 return int64(float64(mean) + float64(stdev)*math.Sqrt2*erfinv(2*(float64(quantile)/100)-1))
50 }
51
52 // https://stackoverflow.com/questions/5971830/need-code-for-inverse-error-function
53 func erfinv(y float64) float64 {
54 if y < -1.0 || y > 1.0 {
55 panic("invalid input")
56 }
57
58 var (
59 a = [4]float64{0.886226899, -1.645349621, 0.914624893, -0.140543331}
60 b = [4]float64{-2.118377725, 1.442710462, -0.329097515, 0.012229801}
61 c = [4]float64{-1.970840454, -1.624906493, 3.429567803, 1.641345311}
62 d = [2]float64{3.543889200, 1.637067800}
63 )
64
65 const y0 = 0.7
66 var x, z float64
67
68 if math.Abs(y) == 1.0 {
69 x = -y * math.Log(0.0)
70 } else if y < -y0 {
71 z = math.Sqrt(-math.Log((1.0 + y) / 2.0))
72 x = -(((c[3]*z+c[2])*z+c[1])*z + c[0]) / ((d[1]*z+d[0])*z + 1.0)
73 } else {
74 if y < y0 {
75 z = y * y
76 x = y * (((a[3]*z+a[2])*z+a[1])*z + a[0]) / ((((b[3]*z+b[3])*z+b[1])*z+b[0])*z + 1.0)
77 } else {
78 z = math.Sqrt(-math.Log((1.0 - y) / 2.0))
79 x = (((c[3]*z+c[2])*z+c[1])*z + c[0]) / ((d[1]*z+d[0])*z + 1.0)
80 }
81 x = x - (math.Erf(x)-y)/(2.0/math.SqrtPi*math.Exp(-x*x))
82 x = x - (math.Erf(x)-y)/(2.0/math.SqrtPi*math.Exp(-x*x))
83 }
84
85 return x
86 }
0 // Package metrics provides an extensible framework to instrument your
1 // application. All metrics are safe for concurrent use. Considerable design
2 // influence has been taken from https://github.com/codahale/metrics and
3 // https://prometheus.io.
4 package metrics
5
6 // Counter is a monotonically-increasing, unsigned, 64-bit integer used to
7 // capture the number of times an event has occurred. By tracking the deltas
8 // between measurements of a counter over intervals of time, an aggregation
9 // layer can derive rates, acceleration, etc.
10 type Counter interface {
11 With(Field) Counter
12 Add(delta uint64)
13 }
14
15 // Gauge captures instantaneous measurements of something using signed, 64-bit
16 // integers. The value does not need to be monotonic.
17 type Gauge interface {
18 With(Field) Gauge
19 Set(value int64)
20 Add(delta int64)
21 }
22
23 // Histogram tracks the distribution of a stream of values (e.g. the number of
24 // milliseconds it takes to handle requests). Implementations may choose to
25 // add gauges for values at meaningful quantiles.
26 type Histogram interface {
27 With(Field) Histogram
28 Observe(value int64)
29 }
30
31 // Field is a key/value pair associated with an observation for a specific
32 // metric. Fields may be ignored by implementations.
33 type Field struct {
34 Key string
35 Value string
36 }
0 package metrics
1
2 type multiCounter []Counter
3
4 // NewMultiCounter returns a wrapper around multiple Counters.
5 func NewMultiCounter(counters ...Counter) Counter {
6 c := make(multiCounter, 0, len(counters))
7 for _, counter := range counters {
8 c = append(c, counter)
9 }
10 return c
11 }
12
13 func (c multiCounter) With(f Field) Counter {
14 next := make(multiCounter, len(c))
15 for i, counter := range c {
16 next[i] = counter.With(f)
17 }
18 return next
19 }
20
21 func (c multiCounter) Add(delta uint64) {
22 for _, counter := range c {
23 counter.Add(delta)
24 }
25 }
26
27 type multiGauge []Gauge
28
29 // NewMultiGauge returns a wrapper around multiple Gauges.
30 func NewMultiGauge(gauges ...Gauge) Gauge {
31 g := make(multiGauge, 0, len(gauges))
32 for _, gauge := range gauges {
33 g = append(g, gauge)
34 }
35 return g
36 }
37
38 func (g multiGauge) With(f Field) Gauge {
39 next := make(multiGauge, len(g))
40 for i, gauge := range g {
41 next[i] = gauge.With(f)
42 }
43 return next
44 }
45
46 func (g multiGauge) Set(value int64) {
47 for _, gauge := range g {
48 gauge.Set(value)
49 }
50 }
51
52 func (g multiGauge) Add(delta int64) {
53 for _, gauge := range g {
54 gauge.Add(delta)
55 }
56 }
57
58 type multiHistogram []Histogram
59
60 // NewMultiHistogram returns a wrapper around multiple Histograms.
61 func NewMultiHistogram(histograms ...Histogram) Histogram {
62 h := make(multiHistogram, 0, len(histograms))
63 for _, histogram := range histograms {
64 h = append(h, histogram)
65 }
66 return h
67 }
68
69 func (h multiHistogram) With(f Field) Histogram {
70 next := make(multiHistogram, len(h))
71 for i, histogram := range h {
72 next[i] = histogram.With(f)
73 }
74 return next
75 }
76
77 func (h multiHistogram) Observe(value int64) {
78 for _, histogram := range h {
79 histogram.Observe(value)
80 }
81 }
0 package metrics_test
1
2 import (
3 "expvar"
4 "strings"
5 "testing"
6
7 "github.com/peterbourgon/gokit/metrics"
8 )
9
10 func TestMultiWith(t *testing.T) {
11 c := metrics.NewMultiCounter(
12 metrics.NewExpvarCounter("foo"),
13 metrics.NewPrometheusCounter("test", "multi_with", "bar", "Bar counter.", []string{"a"}),
14 )
15
16 c.Add(1)
17 c.With(metrics.Field{Key: "a", Value: "1"}).Add(2)
18 c.Add(3)
19
20 if want, have := strings.Join([]string{
21 `# HELP test_multi_with_bar Bar counter.`,
22 `# TYPE test_multi_with_bar counter`,
23 `test_multi_with_bar{a="1"} 2`,
24 `test_multi_with_bar{a="unknown"} 4`,
25 }, "\n"), scrapePrometheus(t); !strings.Contains(have, want) {
26 t.Errorf("Prometheus metric stanza not found or incorrect\n%s", have)
27 }
28 }
29
30 func TestMultiCounter(t *testing.T) {
31 metrics.NewMultiCounter(
32 metrics.NewExpvarCounter("alpha"),
33 metrics.NewPrometheusCounter("test", "multi_counter", "beta", "Beta counter.", []string{}),
34 ).Add(123)
35
36 if want, have := "123", expvar.Get("alpha").String(); want != have {
37 t.Errorf("expvar: want %q, have %q", want, have)
38 }
39
40 if want, have := strings.Join([]string{
41 `# HELP test_multi_counter_beta Beta counter.`,
42 `# TYPE test_multi_counter_beta counter`,
43 `test_multi_counter_beta 123`,
44 }, "\n"), scrapePrometheus(t); !strings.Contains(have, want) {
45 t.Errorf("Prometheus metric stanza not found or incorrect\n%s", have)
46 }
47 }
48
49 func TestMultiGauge(t *testing.T) {
50 g := metrics.NewMultiGauge(
51 metrics.NewExpvarGauge("delta"),
52 metrics.NewPrometheusGauge("test", "multi_gauge", "kappa", "Kappa gauge.", []string{}),
53 )
54
55 g.Set(34)
56
57 if want, have := "34", expvar.Get("delta").String(); want != have {
58 t.Errorf("expvar: want %q, have %q", want, have)
59 }
60 if want, have := strings.Join([]string{
61 `# HELP test_multi_gauge_kappa Kappa gauge.`,
62 `# TYPE test_multi_gauge_kappa gauge`,
63 `test_multi_gauge_kappa 34`,
64 }, "\n"), scrapePrometheus(t); !strings.Contains(have, want) {
65 t.Errorf("Prometheus metric stanza not found or incorrect\n%s", have)
66 }
67
68 g.Add(-40)
69
70 if want, have := "-6", expvar.Get("delta").String(); want != have {
71 t.Errorf("expvar: want %q, have %q", want, have)
72 }
73 if want, have := strings.Join([]string{
74 `# HELP test_multi_gauge_kappa Kappa gauge.`,
75 `# TYPE test_multi_gauge_kappa gauge`,
76 `test_multi_gauge_kappa -6`,
77 }, "\n"), scrapePrometheus(t); !strings.Contains(have, want) {
78 t.Errorf("Prometheus metric stanza not found or incorrect\n%s", have)
79 }
80 }
81
82 func TestMultiHistogram(t *testing.T) {
83 quantiles := []int{50, 90, 99}
84 h := metrics.NewMultiHistogram(
85 metrics.NewExpvarHistogram("omicron", 0, 100, 3, quantiles...),
86 metrics.NewPrometheusHistogram("test", "multi_histogram", "nu", "Nu histogram.", []string{}),
87 )
88
89 const seed, mean, stdev int64 = 123, 50, 10
90 populateNormalHistogram(t, h, seed, mean, stdev)
91 assertExpvarNormalHistogram(t, "omicron", mean, stdev, quantiles)
92 assertPrometheusNormalHistogram(t, "test_multi_histogram_nu", mean, stdev)
93 }
0 // Package prometheus implements a Prometheus backend for package metrics.
1 package prometheus
2
3 import (
4 "time"
5
6 "github.com/prometheus/client_golang/prometheus"
7
8 "github.com/peterbourgon/gokit/metrics"
9 )
10
11 // Prometheus has strong opinions about the dimensionality of fields. Users
12 // must predeclare every field key they intend to use. On every observation,
13 // fields with keys that haven't been predeclared will be silently dropped,
14 // and predeclared field keys without values will receive the value
15 // PrometheusLabelValueUnknown.
16 var PrometheusLabelValueUnknown = "unknown"
17
18 type prometheusCounter struct {
19 *prometheus.CounterVec
20 Pairs map[string]string
21 }
22
23 // NewCounter returns a new Counter backed by a Prometheus metric. The counter
24 // is automatically registered via prometheus.Register.
25 func NewCounter(namespace, subsystem, name, help string, fieldKeys []string) metrics.Counter {
26 return NewCounterWithLabels(namespace, subsystem, name, help, fieldKeys, prometheus.Labels{})
27 }
28
29 // NewCounterWithLabels is the same as NewCounter, but attaches a set of const
30 // label pairs to the metric.
31 func NewCounterWithLabels(namespace, subsystem, name, help string, fieldKeys []string, constLabels prometheus.Labels) metrics.Counter {
32 m := prometheus.NewCounterVec(
33 prometheus.CounterOpts{
34 Namespace: namespace,
35 Subsystem: subsystem,
36 Name: name,
37 Help: help,
38 ConstLabels: constLabels,
39 },
40 fieldKeys,
41 )
42 prometheus.MustRegister(m)
43
44 p := map[string]string{}
45 for _, fieldName := range fieldKeys {
46 p[fieldName] = PrometheusLabelValueUnknown
47 }
48
49 return prometheusCounter{
50 CounterVec: m,
51 Pairs: p,
52 }
53 }
54
55 func (c prometheusCounter) With(f metrics.Field) metrics.Counter {
56 return prometheusCounter{
57 CounterVec: c.CounterVec,
58 Pairs: merge(c.Pairs, f),
59 }
60 }
61
62 func (c prometheusCounter) Add(delta uint64) {
63 c.CounterVec.With(prometheus.Labels(c.Pairs)).Add(float64(delta))
64 }
65
66 type prometheusGauge struct {
67 *prometheus.GaugeVec
68 Pairs map[string]string
69 }
70
71 // NewGauge returns a new Gauge backed by a Prometheus metric. The gauge is
72 // automatically registered via prometheus.Register.
73 func NewGauge(namespace, subsystem, name, help string, fieldKeys []string) metrics.Gauge {
74 return NewGaugeWithLabels(namespace, subsystem, name, help, fieldKeys, prometheus.Labels{})
75 }
76
77 // NewGaugeWithLabels is the same as NewGauge, but attaches a set of const
78 // label pairs to the metric.
79 func NewGaugeWithLabels(namespace, subsystem, name, help string, fieldKeys []string, constLabels prometheus.Labels) metrics.Gauge {
80 m := prometheus.NewGaugeVec(
81 prometheus.GaugeOpts{
82 Namespace: namespace,
83 Subsystem: subsystem,
84 Name: name,
85 Help: help,
86 ConstLabels: constLabels,
87 },
88 fieldKeys,
89 )
90 prometheus.MustRegister(m)
91
92 return prometheusGauge{
93 GaugeVec: m,
94 Pairs: pairsFrom(fieldKeys),
95 }
96 }
97
98 func (g prometheusGauge) With(f metrics.Field) metrics.Gauge {
99 return prometheusGauge{
100 GaugeVec: g.GaugeVec,
101 Pairs: merge(g.Pairs, f),
102 }
103 }
104
105 func (g prometheusGauge) Set(value int64) {
106 g.GaugeVec.With(prometheus.Labels(g.Pairs)).Set(float64(value))
107 }
108
109 func (g prometheusGauge) Add(delta int64) {
110 g.GaugeVec.With(prometheus.Labels(g.Pairs)).Add(float64(delta))
111 }
112
113 type prometheusHistogram struct {
114 *prometheus.SummaryVec
115 Pairs map[string]string
116 }
117
118 // NewHistogram returns a new Histogram backed by a Prometheus summary. It
119 // uses a 10-second max age for bucketing. The histogram is automatically
120 // registered via prometheus.Register.
121 func NewHistogram(namespace, subsystem, name, help string, fieldKeys []string) metrics.Histogram {
122 return NewHistogramWithLabels(namespace, subsystem, name, help, fieldKeys, prometheus.Labels{})
123 }
124
125 // NewHistogramWithLabels is the same as NewHistogram, but attaches a set of
126 // const label pairs to the metric.
127 func NewHistogramWithLabels(namespace, subsystem, name, help string, fieldKeys []string, constLabels prometheus.Labels) metrics.Histogram {
128 m := prometheus.NewSummaryVec(
129 prometheus.SummaryOpts{
130 Namespace: namespace,
131 Subsystem: subsystem,
132 Name: name,
133 Help: help,
134 ConstLabels: constLabels,
135 MaxAge: 10 * time.Second,
136 },
137 fieldKeys,
138 )
139 prometheus.MustRegister(m)
140
141 return prometheusHistogram{
142 SummaryVec: m,
143 Pairs: pairsFrom(fieldKeys),
144 }
145 }
146
147 func (h prometheusHistogram) With(f metrics.Field) metrics.Histogram {
148 return prometheusHistogram{
149 SummaryVec: h.SummaryVec,
150 Pairs: merge(h.Pairs, f),
151 }
152 }
153
154 func (h prometheusHistogram) Observe(value int64) {
155 h.SummaryVec.With(prometheus.Labels(h.Pairs)).Observe(float64(value))
156 }
157
158 func pairsFrom(fieldKeys []string) map[string]string {
159 p := map[string]string{}
160 for _, fieldName := range fieldKeys {
161 p[fieldName] = PrometheusLabelValueUnknown
162 }
163 return p
164 }
165
166 func merge(orig map[string]string, f metrics.Field) map[string]string {
167 if _, ok := orig[f.Key]; !ok {
168 return orig
169 }
170
171 newPairs := make(map[string]string, len(orig))
172 for k, v := range orig {
173 newPairs[k] = v
174 }
175
176 newPairs[f.Key] = f.Value
177 return newPairs
178 }
0 package prometheus_test
1
2 import (
3 "io/ioutil"
4 "math"
5 "math/rand"
6 "net/http"
7 "net/http/httptest"
8 "regexp"
9 "strconv"
10 "strings"
11 "testing"
12
13 stdprometheus "github.com/prometheus/client_golang/prometheus"
14
15 "github.com/peterbourgon/gokit/metrics"
16 "github.com/peterbourgon/gokit/metrics/prometheus"
17 )
18
19 func TestPrometheusLabelBehavior(t *testing.T) {
20 c := prometheus.NewCounter("test", "prometheus_label_behavior", "foobar", "Abc def.", []string{"used_key", "unused_key"})
21 c.With(metrics.Field{Key: "used_key", Value: "declared"}).Add(1)
22 c.Add(1)
23
24 if want, have := strings.Join([]string{
25 `# HELP test_prometheus_label_behavior_foobar Abc def.`,
26 `# TYPE test_prometheus_label_behavior_foobar counter`,
27 `test_prometheus_label_behavior_foobar{unused_key="unknown",used_key="declared"} 1`,
28 `test_prometheus_label_behavior_foobar{unused_key="unknown",used_key="unknown"} 1`,
29 }, "\n"), scrapePrometheus(t); !strings.Contains(have, want) {
30 t.Errorf("metric stanza not found or incorrect\n%s", have)
31 }
32 }
33
34 func TestPrometheusCounter(t *testing.T) {
35 c := prometheus.NewCounter("test", "prometheus_counter", "foobar", "Lorem ipsum.", []string{})
36 c.Add(1)
37 c.Add(2)
38 if want, have := strings.Join([]string{
39 `# HELP test_prometheus_counter_foobar Lorem ipsum.`,
40 `# TYPE test_prometheus_counter_foobar counter`,
41 `test_prometheus_counter_foobar 3`,
42 }, "\n"), scrapePrometheus(t); !strings.Contains(have, want) {
43 t.Errorf("metric stanza not found or incorrect\n%s", have)
44 }
45 c.Add(3)
46 c.Add(4)
47 if want, have := strings.Join([]string{
48 `# HELP test_prometheus_counter_foobar Lorem ipsum.`,
49 `# TYPE test_prometheus_counter_foobar counter`,
50 `test_prometheus_counter_foobar 10`,
51 }, "\n"), scrapePrometheus(t); !strings.Contains(have, want) {
52 t.Errorf("metric stanza not found or incorrect\n%s", have)
53 }
54 }
55
56 func TestPrometheusGauge(t *testing.T) {
57 c := prometheus.NewGauge("test", "prometheus_gauge", "foobar", "Dolor sit.", []string{})
58 c.Set(42)
59 if want, have := strings.Join([]string{
60 `# HELP test_prometheus_gauge_foobar Dolor sit.`,
61 `# TYPE test_prometheus_gauge_foobar gauge`,
62 `test_prometheus_gauge_foobar 42`,
63 }, "\n"), scrapePrometheus(t); !strings.Contains(have, want) {
64 t.Errorf("metric stanza not found or incorrect\n%s", have)
65 }
66 c.Add(-43)
67 if want, have := strings.Join([]string{
68 `# HELP test_prometheus_gauge_foobar Dolor sit.`,
69 `# TYPE test_prometheus_gauge_foobar gauge`,
70 `test_prometheus_gauge_foobar -1`,
71 }, "\n"), scrapePrometheus(t); !strings.Contains(have, want) {
72 t.Errorf("metric stanza not found or incorrect\n%s", have)
73 }
74 }
75
76 func TestPrometheusHistogram(t *testing.T) {
77 h := prometheus.NewHistogram("test", "prometheus_histogram", "foobar", "Qwerty asdf.", []string{})
78
79 const mean, stdev int64 = 50, 10
80 populateNormalHistogram(t, h, 34, mean, stdev)
81 assertNormalHistogram(t, "test_prometheus_histogram_foobar", mean, stdev)
82 }
83
84 func populateNormalHistogram(t *testing.T, h metrics.Histogram, seed int64, mean, stdev int64) {
85 rand.Seed(seed)
86 for i := 0; i < 1234; i++ {
87 sample := int64(rand.NormFloat64()*float64(stdev) + float64(mean))
88 h.Observe(sample)
89 }
90 }
91
92 func assertNormalHistogram(t *testing.T, metricName string, mean, stdev int64) {
93 scrape := scrapePrometheus(t)
94 const tolerance int = 5 // Prometheus approximates higher quantiles badly -_-;
95 for quantileInt, quantileStr := range map[int]string{50: "0.5", 90: "0.9", 99: "0.99"} {
96 want := normalValueAtQuantile(mean, stdev, quantileInt)
97 have := getPrometheusQuantile(t, scrape, metricName, quantileStr)
98 if int(math.Abs(float64(want)-float64(have))) > tolerance {
99 t.Errorf("%q: want %d, have %d", quantileStr, want, have)
100 }
101 }
102 }
103
104 // https://en.wikipedia.org/wiki/Normal_distribution#Quantile_function
105 func normalValueAtQuantile(mean, stdev int64, quantile int) int64 {
106 return int64(float64(mean) + float64(stdev)*math.Sqrt2*erfinv(2*(float64(quantile)/100)-1))
107 }
108
109 // https://stackoverflow.com/questions/5971830/need-code-for-inverse-error-function
110 func erfinv(y float64) float64 {
111 if y < -1.0 || y > 1.0 {
112 panic("invalid input")
113 }
114
115 var (
116 a = [4]float64{0.886226899, -1.645349621, 0.914624893, -0.140543331}
117 b = [4]float64{-2.118377725, 1.442710462, -0.329097515, 0.012229801}
118 c = [4]float64{-1.970840454, -1.624906493, 3.429567803, 1.641345311}
119 d = [2]float64{3.543889200, 1.637067800}
120 )
121
122 const y0 = 0.7
123 var x, z float64
124
125 if math.Abs(y) == 1.0 {
126 x = -y * math.Log(0.0)
127 } else if y < -y0 {
128 z = math.Sqrt(-math.Log((1.0 + y) / 2.0))
129 x = -(((c[3]*z+c[2])*z+c[1])*z + c[0]) / ((d[1]*z+d[0])*z + 1.0)
130 } else {
131 if y < y0 {
132 z = y * y
133 x = y * (((a[3]*z+a[2])*z+a[1])*z + a[0]) / ((((b[3]*z+b[3])*z+b[1])*z+b[0])*z + 1.0)
134 } else {
135 z = math.Sqrt(-math.Log((1.0 - y) / 2.0))
136 x = (((c[3]*z+c[2])*z+c[1])*z + c[0]) / ((d[1]*z+d[0])*z + 1.0)
137 }
138 x = x - (math.Erf(x)-y)/(2.0/math.SqrtPi*math.Exp(-x*x))
139 x = x - (math.Erf(x)-y)/(2.0/math.SqrtPi*math.Exp(-x*x))
140 }
141
142 return x
143 }
144
145 func scrapePrometheus(t *testing.T) string {
146 server := httptest.NewServer(stdprometheus.UninstrumentedHandler())
147 defer server.Close()
148
149 resp, err := http.Get(server.URL)
150 if err != nil {
151 t.Fatal(err)
152 }
153 defer resp.Body.Close()
154
155 buf, err := ioutil.ReadAll(resp.Body)
156 if err != nil {
157 t.Fatal(err)
158 }
159
160 return strings.TrimSpace(string(buf))
161 }
162
163 func getPrometheusQuantile(t *testing.T, scrape, name, quantileStr string) int {
164 matches := regexp.MustCompile(name+`{quantile="`+quantileStr+`"} ([0-9]+)`).FindAllStringSubmatch(scrape, -1)
165 if len(matches) < 1 {
166 t.Fatalf("%q: quantile %q not found in scrape", name, quantileStr)
167 }
168 if len(matches[0]) < 2 {
169 t.Fatalf("%q: quantile %q not found in scrape", name, quantileStr)
170 }
171 i, err := strconv.Atoi(matches[0][1])
172 if err != nil {
173 t.Fatal(err)
174 }
175 return i
176 }
0 package metrics
1
2 type scaledHistogram struct {
3 Histogram
4 scale int64
5 }
6
7 // NewScaledHistogram returns a Histogram whose observed values are downscaled
8 // (divided) by scale.
9 func NewScaledHistogram(h Histogram, scale int64) Histogram {
10 return scaledHistogram{h, scale}
11 }
12
13 func (h scaledHistogram) With(f Field) Histogram {
14 return scaledHistogram{
15 Histogram: h.Histogram.With(f),
16 scale: h.scale,
17 }
18 }
19
20 func (h scaledHistogram) Observe(value int64) {
21 h.Histogram.Observe(value / h.scale)
22 }
0 package metrics_test
1
2 import (
3 "testing"
4
5 "github.com/peterbourgon/gokit/metrics"
6 )
7
8 func TestScaledHistogram(t *testing.T) {
9 quantiles := []int{50, 90, 99}
10 scale := int64(10)
11 metricName := "test_scaled_histogram"
12
13 var h metrics.Histogram
14 h = metrics.NewExpvarHistogram(metricName, 0, 1000, 3, quantiles...)
15 h = metrics.NewScaledHistogram(h, scale)
16
17 const seed, mean, stdev = 333, 500, 100 // input values
18 populateNormalHistogram(t, h, seed, mean, stdev) // will be scaled down
19 assertExpvarNormalHistogram(t, metricName, mean/scale, stdev/scale, quantiles)
20 }
0 // Package statsd implements a statsd backend for package metrics.
1 //
2 // The current implementation ignores fields. In the future, it would be good
3 // to have an implementation that accepted a set of predeclared field names at
4 // construction time, and used field values to produce delimiter-separated
5 // bucket (key) names. That is,
6 //
7 // c := NewFieldedCounter(..., "path", "status")
8 // c.Add(1) // "myprefix.unknown.unknown:1|c\n"
9 // c2 := c.With("path", "foo").With("status": "200")
10 // c2.Add(1) // "myprefix.foo.status:1|c\n"
11 //
12 package statsd
13
14 import (
15 "bytes"
16 "fmt"
17 "io"
18 "log"
19 "time"
20
21 "github.com/peterbourgon/gokit/metrics"
22 )
23
24 // statsd metrics take considerable influence from
25 // https://github.com/streadway/handy package statsd.
26
27 const maxBufferSize = 1400 // bytes
28
29 type statsdCounter chan string
30
31 // NewCounter returns a Counter that emits observations in the statsd protocol
32 // to the passed writer. Observations are buffered for the reporting interval
33 // or until the buffer exceeds a max packet size, whichever comes first.
34 // Fields are ignored.
35 //
36 // TODO: support for sampling.
37 func NewCounter(w io.Writer, key string, interval time.Duration) metrics.Counter {
38 c := make(chan string)
39 go fwd(w, key, interval, c)
40 return statsdCounter(c)
41 }
42
43 func (c statsdCounter) With(metrics.Field) metrics.Counter { return c }
44
45 func (c statsdCounter) Add(delta uint64) { c <- fmt.Sprintf("%d|c", delta) }
46
47 type statsdGauge chan string
48
49 // NewGauge returns a Gauge that emits values in the statsd protocol to the
50 // passed writer. Values are buffered for the reporting interval or until the
51 // buffer exceeds a max packet size, whichever comes first. Fields are
52 // ignored.
53 //
54 // TODO: support for sampling.
55 func NewGauge(w io.Writer, key string, interval time.Duration) metrics.Gauge {
56 g := make(chan string)
57 go fwd(w, key, interval, g)
58 return statsdGauge(g)
59 }
60
61 func (g statsdGauge) With(metrics.Field) metrics.Gauge { return g }
62
63 func (g statsdGauge) Add(delta int64) {
64 // https://github.com/etsy/statsd/blob/master/docs/metric_types.md#gauges
65 sign := "+"
66 if delta < 0 {
67 sign, delta = "-", -delta
68 }
69 g <- fmt.Sprintf("%s%d|g", sign, delta)
70 }
71
72 func (g statsdGauge) Set(value int64) {
73 g <- fmt.Sprintf("%d|g", value)
74 }
75
76 type statsdHistogram chan string
77
78 // NewHistogram returns a Histogram that emits observations in the statsd
79 // protocol to the passed writer. Observations are buffered for the reporting
80 // interval or until the buffer exceeds a max packet size, whichever comes
81 // first. Fields are ignored.
82 //
83 // NewHistogram is mapped to a statsd Timing, so observations should represent
84 // milliseconds. If you observe in units of nanoseconds, you can make the
85 // translation with a ScaledHistogram:
86 //
87 // NewScaledHistogram(statsdHistogram, time.Millisecond)
88 //
89 // You can also enforce the constraint in a typesafe way with a millisecond
90 // TimeHistogram:
91 //
92 // NewTimeHistogram(statsdHistogram, time.Millisecond)
93 //
94 // TODO: support for sampling.
95 func NewHistogram(w io.Writer, key string, interval time.Duration) metrics.Histogram {
96 h := make(chan string)
97 go fwd(w, key, interval, h)
98 return statsdHistogram(h)
99 }
100
101 func (h statsdHistogram) With(metrics.Field) metrics.Histogram { return h }
102
103 func (h statsdHistogram) Observe(value int64) {
104 h <- fmt.Sprintf("%d|ms", value)
105 }
106
107 var tick = time.Tick
108
109 func fwd(w io.Writer, key string, interval time.Duration, c chan string) {
110 buf := &bytes.Buffer{}
111 tick := tick(interval)
112 for {
113 select {
114 case s := <-c:
115 fmt.Fprintf(buf, "%s:%s\n", key, s)
116 if buf.Len() > maxBufferSize {
117 flush(w, buf)
118 }
119
120 case <-tick:
121 flush(w, buf)
122 }
123 }
124 }
125
126 func flush(w io.Writer, buf *bytes.Buffer) {
127 if buf.Len() <= 0 {
128 return
129 }
130 if _, err := w.Write(buf.Bytes()); err != nil {
131 log.Printf("error: could not write to statsd: %v", err)
132 }
133 buf.Reset()
134 }
0 package statsd
1
2 // In package metrics so we can stub tick.
3
4 import (
5 "bytes"
6 "runtime"
7 "testing"
8 "time"
9 )
10
11 func TestCounter(t *testing.T) {
12 ch := make(chan time.Time)
13 tick = func(time.Duration) <-chan time.Time { return ch }
14 defer func() { tick = time.Tick }()
15
16 buf := &bytes.Buffer{}
17 c := NewCounter(buf, "test_statsd_counter", time.Second)
18
19 c.Add(1)
20 c.Add(2)
21 ch <- time.Now()
22
23 for i := 0; i < 10 && buf.Len() == 0; i++ {
24 time.Sleep(time.Millisecond)
25 }
26
27 if want, have := "test_statsd_counter:1|c\ntest_statsd_counter:2|c\n", buf.String(); want != have {
28 t.Errorf("want %q, have %q", want, have)
29 }
30 }
31
32 func TestGauge(t *testing.T) {
33 ch := make(chan time.Time)
34 tick = func(time.Duration) <-chan time.Time { return ch }
35 defer func() { tick = time.Tick }()
36
37 buf := &bytes.Buffer{}
38 g := NewGauge(buf, "test_statsd_gauge", time.Second)
39
40 g.Add(1) // send command
41 runtime.Gosched() // yield to buffer write
42 ch <- time.Now() // signal flush
43 runtime.Gosched() // yield to flush
44 if want, have := "test_statsd_gauge:+1|g\n", buf.String(); want != have {
45 t.Errorf("want %q, have %q", want, have)
46 }
47
48 buf.Reset()
49
50 g.Add(-2)
51 runtime.Gosched()
52 ch <- time.Now()
53 runtime.Gosched()
54 if want, have := "test_statsd_gauge:-2|g\n", buf.String(); want != have {
55 t.Errorf("want %q, have %q", want, have)
56 }
57
58 buf.Reset()
59
60 g.Set(3)
61 runtime.Gosched()
62 ch <- time.Now()
63 runtime.Gosched()
64 if want, have := "test_statsd_gauge:3|g\n", buf.String(); want != have {
65 t.Errorf("want %q, have %q", want, have)
66 }
67 }
68
69 func TestHistogram(t *testing.T) {
70 ch := make(chan time.Time)
71 tick = func(time.Duration) <-chan time.Time { return ch }
72 defer func() { tick = time.Tick }()
73
74 buf := &bytes.Buffer{}
75 h := NewHistogram(buf, "test_statsd_histogram", time.Second)
76
77 h.Observe(123)
78
79 runtime.Gosched()
80 ch <- time.Now()
81 runtime.Gosched()
82 if want, have := "test_statsd_histogram:123|ms\n", buf.String(); want != have {
83 t.Errorf("want %q, have %q", want, have)
84 }
85 }
0 package metrics
1
2 import "time"
3
4 // TimeHistogram is a convenience wrapper for a Histogram of time.Durations.
5 type TimeHistogram interface {
6 With(Field) TimeHistogram
7 Observe(time.Duration)
8 }
9
10 type timeHistogram struct {
11 Histogram
12 unit time.Duration
13 }
14
15 // NewTimeHistogram returns a TimeHistogram wrapper around the passed
16 // Histogram, in units of unit.
17 func NewTimeHistogram(h Histogram, unit time.Duration) TimeHistogram {
18 return &timeHistogram{
19 Histogram: h,
20 unit: unit,
21 }
22 }
23
24 func (h *timeHistogram) With(f Field) TimeHistogram {
25 return &timeHistogram{
26 Histogram: h.Histogram.With(f),
27 unit: h.unit,
28 }
29 }
30
31 func (h *timeHistogram) Observe(d time.Duration) {
32 h.Histogram.Observe(int64(d / h.unit))
33 }
0 package metrics_test
1
2 import (
3 "math/rand"
4 "testing"
5 "time"
6
7 "github.com/peterbourgon/gokit/metrics"
8 )
9
10 func TestTimeHistogram(t *testing.T) {
11 const metricName string = "test_time_histogram"
12 quantiles := []int{50, 90, 99}
13 h0 := metrics.NewExpvarHistogram(metricName, 0, 200, 3, quantiles...)
14 h := metrics.NewTimeHistogram(h0, time.Millisecond)
15 const seed, mean, stdev int64 = 321, 100, 20
16
17 for i := 0; i < 4321; i++ {
18 sample := time.Duration(rand.NormFloat64()*float64(stdev)+float64(mean)) * time.Millisecond
19 h.Observe(sample)
20 }
21
22 assertExpvarNormalHistogram(t, metricName, mean, stdev, quantiles)
23 }
1111
1212 ## Scope
1313
14 To be defined.
14 - Package metrics SHALL implement Gauges, Counters, and Histograms.
15
16 - Each metric type SHALL allow observations with an unlimited number of key/value field pairs,
17 similar to [package log](https://github.com/peterbourgon/gokit/blob/master/rfc/rfc004-package-log.md).
18
19 - Counter SHALL be an increment-only counter of type uint64.
20
21 - Gauge SHALL be an arbitrarily-settable register of type int64.
22
23 - Histogram SHALL collect observations of type int64.
24
25 - These interfaces SHALL be the primary and exclusive API for metrics.
26
27 - We SHALL provide a variety of implementations of each interface that act as a
28 bridge to different backends: expvar, Graphite, statsd, Prometheus, etc.
29
30 - Each metric backend MAY provide additional value-add behaviors. For example,
31 a backend for Histogram may bucket observations according to quantile and
32 calculate additional, derived statistics.
33
1534
1635 ## Implementation
1736
18 To be defined.
37 https://github.com/peterbourgon/gokit/tree/master/metrics
38
39 ### Gauge
40
41 ```go
42 type Gauge interface {
43 With(Field) Gauge
44 Set(value int64)
45 Add(delta int64)
46 }
47 ```
48
49 ### Counter
50
51 ```go
52 type Counter interface {
53 With(Field) Counter
54 Add(delta uint64)
55 }
56 ```
57
58 ### Histogram
59
60 ```go
61 type Histogram interface {
62 With(Field) Histogram
63 Observe(int64)
64 }
65 ```