metrics: prometheus: allow Summary and Histogram
Peter Bourgon
8 years ago
95 | 95 | quantiles := []int{50, 90, 99} |
96 | 96 | h := metrics.NewMultiHistogram( |
97 | 97 | expvar.NewHistogram("omicron", 0, 100, 3, quantiles...), |
98 | prometheus.NewHistogram("test", "multi_histogram", "nu", "Nu histogram.", []string{}), | |
98 | prometheus.NewSummary("test", "multi_histogram", "nu", "Nu histogram.", []string{}), | |
99 | 99 | ) |
100 | 100 | |
101 | 101 | const seed, mean, stdev int64 = 123, 50, 10 |
133 | 133 | )) |
134 | 134 | } |
135 | 135 | |
136 | type prometheusHistogram struct { | |
136 | type prometheusSummary struct { | |
137 | 137 | *prometheus.SummaryVec |
138 | 138 | Pairs map[string]string |
139 | 139 | } |
140 | 140 | |
141 | // NewHistogram returns a new Histogram backed by a Prometheus summary. It | |
142 | // uses a 10-second max age for bucketing. The histogram is automatically | |
143 | // registered via prometheus.Register. | |
144 | func NewHistogram(namespace, subsystem, name, help string, fieldKeys []string) metrics.Histogram { | |
145 | return NewHistogramWithLabels(namespace, subsystem, name, help, fieldKeys, prometheus.Labels{}) | |
146 | } | |
147 | ||
148 | // NewHistogramWithLabels is the same as NewHistogram, but attaches a set of | |
149 | // const label pairs to the metric. | |
150 | func NewHistogramWithLabels(namespace, subsystem, name, help string, fieldKeys []string, constLabels prometheus.Labels) metrics.Histogram { | |
141 | // NewSummary returns a new Histogram backed by a Prometheus summary. It uses | |
142 | // a 10-second max age for bucketing, emulating statsd. The histogram is | |
143 | // automatically registered via prometheus.Register. | |
144 | func NewSummary(namespace, subsystem, name, help string, fieldKeys []string) metrics.Histogram { | |
145 | return NewSummaryWithLabels(namespace, subsystem, name, help, fieldKeys, prometheus.Labels{}) | |
146 | } | |
147 | ||
148 | // NewSummaryWithLabels is the same as NewSummary, but attaches a set of const | |
149 | // label pairs to the metric. | |
150 | func NewSummaryWithLabels(namespace, subsystem, name, help string, fieldKeys []string, constLabels prometheus.Labels) metrics.Histogram { | |
151 | 151 | m := prometheus.NewSummaryVec( |
152 | 152 | prometheus.SummaryOpts{ |
153 | 153 | Namespace: namespace, |
161 | 161 | ) |
162 | 162 | prometheus.MustRegister(m) |
163 | 163 | |
164 | return prometheusHistogram{ | |
164 | return prometheusSummary{ | |
165 | 165 | SummaryVec: m, |
166 | 166 | Pairs: pairsFrom(fieldKeys), |
167 | 167 | } |
168 | 168 | } |
169 | 169 | |
170 | func (s prometheusSummary) With(f metrics.Field) metrics.Histogram { | |
171 | return prometheusSummary{ | |
172 | SummaryVec: s.SummaryVec, | |
173 | Pairs: merge(s.Pairs, f), | |
174 | } | |
175 | } | |
176 | ||
177 | func (s prometheusSummary) Observe(value int64) { | |
178 | s.SummaryVec.With(prometheus.Labels(s.Pairs)).Observe(float64(value)) | |
179 | } | |
180 | ||
181 | type prometheusHistogram struct { | |
182 | *prometheus.HistogramVec | |
183 | Pairs map[string]string | |
184 | } | |
185 | ||
186 | // NewHistogram returns a new Histogram backed by a Prometheus Histogram. | |
187 | // Observations are counted into buckets; see Prometheus documentation for | |
188 | // details. The histogram is automatically registered via prometheus.Register. | |
189 | func NewHistogram(namespace, subsystem, name, help string, fieldKeys []string, buckets []float64) metrics.Histogram { | |
190 | return NewHistogramWithLabels(namespace, subsystem, name, help, fieldKeys, buckets, prometheus.Labels{}) | |
191 | } | |
192 | ||
193 | // NewHistogramWithLabels is the same as NewHistogram, but attaches a set of const | |
194 | // label pairs to the metric. | |
195 | func NewHistogramWithLabels(namespace, subsystem, name, help string, fieldKeys []string, buckets []float64, constLabels prometheus.Labels) metrics.Histogram { | |
196 | m := prometheus.NewHistogramVec( | |
197 | prometheus.HistogramOpts{ | |
198 | Namespace: namespace, | |
199 | Subsystem: subsystem, | |
200 | Name: name, | |
201 | Help: help, | |
202 | ConstLabels: constLabels, | |
203 | Buckets: buckets, | |
204 | }, | |
205 | fieldKeys, | |
206 | ) | |
207 | prometheus.MustRegister(m) | |
208 | ||
209 | return prometheusHistogram{ | |
210 | HistogramVec: m, | |
211 | Pairs: pairsFrom(fieldKeys), | |
212 | } | |
213 | } | |
214 | ||
170 | 215 | func (h prometheusHistogram) With(f metrics.Field) metrics.Histogram { |
171 | 216 | return prometheusHistogram{ |
172 | SummaryVec: h.SummaryVec, | |
173 | Pairs: merge(h.Pairs, f), | |
217 | HistogramVec: h.HistogramVec, | |
218 | Pairs: merge(h.Pairs, f), | |
174 | 219 | } |
175 | 220 | } |
176 | 221 | |
177 | 222 | func (h prometheusHistogram) Observe(value int64) { |
178 | h.SummaryVec.With(prometheus.Labels(h.Pairs)).Observe(float64(value)) | |
223 | h.HistogramVec.With(prometheus.Labels(h.Pairs)).Observe(float64(value)) | |
179 | 224 | } |
180 | 225 | |
181 | 226 | func pairsFrom(fieldKeys []string) map[string]string { |
78 | 78 | } |
79 | 79 | } |
80 | 80 | |
81 | func TestPrometheusHistogram(t *testing.T) { | |
82 | h := prometheus.NewHistogram("test", "prometheus_histogram", "foobar", "Qwerty asdf.", []string{}) | |
81 | func TestPrometheusSummary(t *testing.T) { | |
82 | h := prometheus.NewSummary("test", "prometheus_summary_histogram", "foobar", "Qwerty asdf.", []string{}) | |
83 | 83 | |
84 | 84 | const mean, stdev int64 = 50, 10 |
85 | 85 | teststat.PopulateNormalHistogram(t, h, 34, mean, stdev) |
86 | teststat.AssertPrometheusNormalHistogram(t, "test_prometheus_histogram_foobar", mean, stdev) | |
86 | teststat.AssertPrometheusNormalSummary(t, "test_prometheus_summary_histogram_foobar", mean, stdev) | |
87 | 87 | } |
88 | ||
89 | func TestPrometheusHistogram(t *testing.T) { | |
90 | buckets := []float64{20, 40, 60, 80, 100} | |
91 | h := prometheus.NewHistogram("test", "prometheus_histogram_histogram", "quux", "Qwerty asdf.", []string{}, buckets) | |
92 | ||
93 | const mean, stdev int64 = 50, 10 | |
94 | teststat.PopulateNormalHistogram(t, h, 34, mean, stdev) | |
95 | teststat.AssertPrometheusBucketedHistogram(t, "test_prometheus_histogram_histogram_quux_bucket", mean, stdev, buckets) | |
96 | } |
9 | 9 | "github.com/peterbourgon/gokit/metrics" |
10 | 10 | ) |
11 | 11 | |
12 | const population = 1234 | |
13 | ||
12 | 14 | // PopulateNormalHistogram populates the Histogram with a normal distribution |
13 | 15 | // of observations. |
14 | 16 | func PopulateNormalHistogram(t *testing.T, h metrics.Histogram, seed int64, mean, stdev int64) { |
15 | 17 | rand.Seed(seed) |
16 | for i := 0; i < 1234; i++ { | |
18 | for i := 0; i < population; i++ { | |
17 | 19 | sample := int64(rand.NormFloat64()*float64(stdev) + float64(mean)) |
18 | 20 | h.Observe(sample) |
19 | 21 | } |
22 | 24 | // https://en.wikipedia.org/wiki/Normal_distribution#Quantile_function |
23 | 25 | func normalValueAtQuantile(mean, stdev int64, quantile int) int64 { |
24 | 26 | return int64(float64(mean) + float64(stdev)*math.Sqrt2*erfinv(2*(float64(quantile)/100)-1)) |
27 | } | |
28 | ||
29 | // https://code.google.com/p/gostat/source/browse/stat/normal.go | |
30 | func observationsLessThan(mean, stdev int64, x float64, total int) int { | |
31 | cdf := ((1.0 / 2.0) * (1 + math.Erf((x-float64(mean))/(float64(stdev)*math.Sqrt2)))) | |
32 | return int(cdf * float64(total)) | |
25 | 33 | } |
26 | 34 | |
27 | 35 | // https://stackoverflow.com/questions/5971830/need-code-for-inverse-error-function |
32 | 32 | return strings.TrimSpace(string(buf)) |
33 | 33 | } |
34 | 34 | |
35 | // AssertPrometheusNormalHistogram ensures the Prometheus Histogram referenced | |
36 | // by metricName abides a normal distribution. | |
37 | func AssertPrometheusNormalHistogram(t *testing.T, metricName string, mean, stdev int64) { | |
35 | // AssertPrometheusNormalSummary ensures the Prometheus Summary referenced by | |
36 | // name abides a normal distribution. | |
37 | func AssertPrometheusNormalSummary(t *testing.T, metricName string, mean, stdev int64) { | |
38 | 38 | scrape := ScrapePrometheus(t) |
39 | 39 | const tolerance int = 5 // Prometheus approximates higher quantiles badly -_-; |
40 | 40 | for quantileInt, quantileStr := range map[int]string{50: "0.5", 90: "0.9", 99: "0.99"} { |
42 | 42 | have := getPrometheusQuantile(t, scrape, metricName, quantileStr) |
43 | 43 | if int(math.Abs(float64(want)-float64(have))) > tolerance { |
44 | 44 | t.Errorf("%q: want %d, have %d", quantileStr, want, have) |
45 | } | |
46 | } | |
47 | } | |
48 | ||
49 | // AssertPrometheusBucketedHistogram ensures the Prometheus Histogram | |
50 | // referenced by name has observations in the expected quantity and bucket. | |
51 | func AssertPrometheusBucketedHistogram(t *testing.T, metricName string, mean, stdev int64, buckets []float64) { | |
52 | scrape := ScrapePrometheus(t) | |
53 | const tolerance int = population / 50 // pretty coarse-grained | |
54 | for _, bucket := range buckets { | |
55 | want := observationsLessThan(mean, stdev, bucket, population) | |
56 | have := getPrometheusLessThan(t, scrape, metricName, strconv.FormatFloat(bucket, 'f', 0, 64)) | |
57 | if int(math.Abs(float64(want)-float64(have))) > tolerance { | |
58 | t.Errorf("%.0f: want %d, have %d", bucket, want, have) | |
45 | 59 | } |
46 | 60 | } |
47 | 61 | } |
60 | 74 | } |
61 | 75 | return i |
62 | 76 | } |
77 | ||
78 | func getPrometheusLessThan(t *testing.T, scrape, name, target string) int { | |
79 | matches := regexp.MustCompile(name+`{le="`+target+`"} ([0-9]+)`).FindAllStringSubmatch(scrape, -1) | |
80 | if len(matches) < 1 { | |
81 | t.Logf(">>>\n%s\n", scrape) | |
82 | t.Fatalf("%q: bucket %q not found in scrape", name, target) | |
83 | } | |
84 | if len(matches[0]) < 2 { | |
85 | t.Fatalf("%q: bucket %q not found in scrape", name, target) | |
86 | } | |
87 | i, err := strconv.Atoi(matches[0][1]) | |
88 | if err != nil { | |
89 | t.Fatal(err) | |
90 | } | |
91 | return i | |
92 | } |