Commit 4928fe6b444b27b24a1db2747a2c35e76aa3a6da - golang-github-go-kit-kit

+22

-0

README.md less more

22	22	- Having opinions on deployment, orchestration, process supervision
23	23	- Having opinions on configuration passing -- flags vs. env vars vs. files vs. ...
24	24	- _more TODO_
	25
	26	## Contributing
	27
	28	At this stage, we're still developing the initial drafts of all of the
	29	packages, using an
	30	[RFC workflow](https://github.com/peterbourgon/gokit/tree/master/rfc).
	31	Before submitting major changes, please write to
	32	[the mailing list](groups.google.com/forum/#!forum/go-kit)
	33	to register your interest, and check the
	34	[open issues](https://github.com/peterbourgon/gokit/issues) and
	35	[pull requests](https://github.com/peterbourgon/gokit/pulls)
	36	for existing discussions.
	37
	38	### Dependency management
	39
	40	Users who import gokit into their `package main` are responsible to organize
	41	and maintain all of their dependencies to ensure code compatibility and build
	42	reproducibility. Gokit makes no direct use of dependency management tools like
	43	[Godep](https://github.com/tools/godep).
	44
	45	We will use a variety of continuous integration providers to find and fix
	46	compatibility problems as soon as they occur.
25	47
26	48	## Related projects
27	49

+113

-0

metrics/expvar/expvar.go less more

	0	// Package expvar implements an expvar backend for package metrics.
	1	//
	2	// The current implementation ignores fields. In the future, it would be good
	3	// to have an implementation that accepted a set of predeclared field names at
	4	// construction time, and used field values to produce delimiter-separated
	5	// bucket (key) names. That is,
	6	//
	7	// c := NewFieldedCounter(..., "path", "status")
	8	// c.Add(1) // "myprefix_unknown_unknown" += 1
	9	// c2 := c.With("path", "foo").With("status": "200")
	10	// c2.Add(1) // "myprefix_foo_status" += 1
	11	//
	12	// It would also be possible to have an implementation that generated more
	13	// sophisticated expvar.Values. For example, a Counter could be implemented as
	14	// a map, representing a tree of key/value pairs whose leaves were the actual
	15	// expvar.Ints.
	16	package expvar
	17
	18	import (
	19	"expvar"
	20	"fmt"
	21	"sync"
	22	"time"
	23
	24	"github.com/peterbourgon/gokit/metrics"
	25
	26	"github.com/codahale/hdrhistogram"
	27	)
	28
	29	type counter struct {
	30	v *expvar.Int
	31	}
	32
	33	// NewCounter returns a new Counter backed by an expvar with the given name.
	34	// Fields are ignored.
	35	func NewCounter(name string) metrics.Counter {
	36	return &counter{expvar.NewInt(name)}
	37	}
	38
	39	func (c *counter) With(metrics.Field) metrics.Counter { return c }
	40
	41	func (c *counter) Add(delta uint64) { c.v.Add(int64(delta)) }
	42
	43	type gauge struct {
	44	v *expvar.Int
	45	}
	46
	47	// NewGauge returns a new Gauge backed by an expvar with the given name.
	48	// Fields are ignored.
	49	func NewGauge(name string) metrics.Gauge {
	50	return &gauge{expvar.NewInt(name)}
	51	}
	52
	53	func (g *gauge) With(metrics.Field) metrics.Gauge { return g }
	54
	55	func (g *gauge) Add(delta int64) { g.v.Add(delta) }
	56
	57	func (g *gauge) Set(value int64) { g.v.Set(value) }
	58
	59	type histogram struct {
	60	mu sync.Mutex
	61	hist *hdrhistogram.WindowedHistogram
	62
	63	name string
	64	gauges map[int]metrics.Gauge
	65	}
	66
	67	// NewHistogram is taken from http://github.com/codahale/metrics. It returns a
	68	// windowed HDR histogram which drops data older than five minutes.
	69	//
	70	// The histogram exposes metrics for each passed quantile as gauges. Quantiles
	71	// should be integers in the range 1..99. The gauge names are assigned by
	72	// using the passed name as a prefix and appending "_pNN" e.g. "_p50".
	73	func NewHistogram(name string, minValue, maxValue int64, sigfigs int, quantiles ...int) metrics.Histogram {
	74	gauges := map[int]metrics.Gauge{}
	75	for _, quantile := range quantiles {
	76	if quantile <= 0 \|\| quantile >= 100 {
	77	panic(fmt.Sprintf("invalid quantile %d", quantile))
	78	}
	79	gauges[quantile] = NewGauge(fmt.Sprintf("%s_p%02d", name, quantile))
	80	}
	81	h := &histogram{
	82	hist: hdrhistogram.NewWindowed(5, minValue, maxValue, sigfigs),
	83	name: name,
	84	gauges: gauges,
	85	}
	86	go h.rotateLoop(1 * time.Minute)
	87	return h
	88	}
	89
	90	func (h *histogram) With(metrics.Field) metrics.Histogram { return h }
	91
	92	func (h *histogram) Observe(value int64) {
	93	h.mu.Lock()
	94	err := h.hist.Current.RecordValue(value)
	95	h.mu.Unlock()
	96
	97	if err != nil {
	98	panic(err.Error())
	99	}
	100
	101	for q, gauge := range h.gauges {
	102	gauge.Set(h.hist.Current.ValueAtQuantile(float64(q)))
	103	}
	104	}
	105
	106	func (h *histogram) rotateLoop(d time.Duration) {
	107	for _ = range time.Tick(d) {
	108	h.mu.Lock()
	109	h.hist.Rotate()
	110	h.mu.Unlock()
	111	}
	112	}

+87

-0

metrics/expvar/expvar_test.go less more

	0	package expvar_test
	1
	2	import (
	3	stdexpvar "expvar"
	4	"fmt"
	5	"math"
	6	"math/rand"
	7	"strconv"
	8	"testing"
	9
	10	"github.com/peterbourgon/gokit/metrics"
	11	"github.com/peterbourgon/gokit/metrics/expvar"
	12	)
	13
	14	func TestHistogramQuantiles(t *testing.T) {
	15	metricName := "test_histogram"
	16	quantiles := []int{50, 90, 95, 99}
	17	h := expvar.NewHistogram(metricName, 0, 100, 3, quantiles...)
	18
	19	const seed, mean, stdev int64 = 424242, 50, 10
	20	populateNormalHistogram(t, h, seed, mean, stdev)
	21	assertNormalHistogram(t, metricName, mean, stdev, quantiles)
	22	}
	23
	24	func populateNormalHistogram(t *testing.T, h metrics.Histogram, seed int64, mean, stdev int64) {
	25	rand.Seed(seed)
	26	for i := 0; i < 1234; i++ {
	27	sample := int64(rand.NormFloat64()*float64(stdev) + float64(mean))
	28	h.Observe(sample)
	29	}
	30	}
	31
	32	func assertNormalHistogram(t *testing.T, metricName string, mean, stdev int64, quantiles []int) {
	33	const tolerance int = 2
	34	for _, quantile := range quantiles {
	35	want := normalValueAtQuantile(mean, stdev, quantile)
	36	s := stdexpvar.Get(fmt.Sprintf("%s_p%02d", metricName, quantile)).String()
	37	have, err := strconv.Atoi(s)
	38	if err != nil {
	39	t.Fatal(err)
	40	}
	41	if int(math.Abs(float64(want)-float64(have))) > tolerance {
	42	t.Errorf("quantile %d: want %d, have %d", quantile, want, have)
	43	}
	44	}
	45	}
	46
	47	// https://en.wikipedia.org/wiki/Normal_distribution#Quantile_function
	48	func normalValueAtQuantile(mean, stdev int64, quantile int) int64 {
	49	return int64(float64(mean) + float64(stdev)math.Sqrt2erfinv(2*(float64(quantile)/100)-1))
	50	}
	51
	52	// https://stackoverflow.com/questions/5971830/need-code-for-inverse-error-function
	53	func erfinv(y float64) float64 {
	54	if y < -1.0 \|\| y > 1.0 {
	55	panic("invalid input")
	56	}
	57
	58	var (
	59	a = [4]float64{0.886226899, -1.645349621, 0.914624893, -0.140543331}
	60	b = [4]float64{-2.118377725, 1.442710462, -0.329097515, 0.012229801}
	61	c = [4]float64{-1.970840454, -1.624906493, 3.429567803, 1.641345311}
	62	d = [2]float64{3.543889200, 1.637067800}
	63	)
	64
	65	const y0 = 0.7
	66	var x, z float64
	67
	68	if math.Abs(y) == 1.0 {
	69	x = -y * math.Log(0.0)
	70	} else if y < -y0 {
	71	z = math.Sqrt(-math.Log((1.0 + y) / 2.0))
	72	x = -(((c[3]z+c[2])z+c[1])z + c[0]) / ((d[1]z+d[0])*z + 1.0)
	73	} else {
	74	if y < y0 {
	75	z = y * y
	76	x = y * (((a[3]z+a[2])z+a[1])z + a[0]) / ((((b[3]z+b[3])z+b[1])z+b[0])*z + 1.0)
	77	} else {
	78	z = math.Sqrt(-math.Log((1.0 - y) / 2.0))
	79	x = (((c[3]z+c[2])z+c[1])z + c[0]) / ((d[1]z+d[0])*z + 1.0)
	80	}
	81	x = x - (math.Erf(x)-y)/(2.0/math.SqrtPimath.Exp(-xx))
	82	x = x - (math.Erf(x)-y)/(2.0/math.SqrtPimath.Exp(-xx))
	83	}
	84
	85	return x
	86	}

+37

-0

metrics/metrics.go less more

	0	// Package metrics provides an extensible framework to instrument your
	1	// application. All metrics are safe for concurrent use. Considerable design
	2	// influence has been taken from https://github.com/codahale/metrics and
	3	// https://prometheus.io.
	4	package metrics
	5
	6	// Counter is a monotonically-increasing, unsigned, 64-bit integer used to
	7	// capture the number of times an event has occurred. By tracking the deltas
	8	// between measurements of a counter over intervals of time, an aggregation
	9	// layer can derive rates, acceleration, etc.
	10	type Counter interface {
	11	With(Field) Counter
	12	Add(delta uint64)
	13	}
	14
	15	// Gauge captures instantaneous measurements of something using signed, 64-bit
	16	// integers. The value does not need to be monotonic.
	17	type Gauge interface {
	18	With(Field) Gauge
	19	Set(value int64)
	20	Add(delta int64)
	21	}
	22
	23	// Histogram tracks the distribution of a stream of values (e.g. the number of
	24	// milliseconds it takes to handle requests). Implementations may choose to
	25	// add gauges for values at meaningful quantiles.
	26	type Histogram interface {
	27	With(Field) Histogram
	28	Observe(value int64)
	29	}
	30
	31	// Field is a key/value pair associated with an observation for a specific
	32	// metric. Fields may be ignored by implementations.
	33	type Field struct {
	34	Key string
	35	Value string
	36	}

+82

-0

metrics/multi.go less more

	0	package metrics
	1
	2	type multiCounter []Counter
	3
	4	// NewMultiCounter returns a wrapper around multiple Counters.
	5	func NewMultiCounter(counters ...Counter) Counter {
	6	c := make(multiCounter, 0, len(counters))
	7	for _, counter := range counters {
	8	c = append(c, counter)
	9	}
	10	return c
	11	}
	12
	13	func (c multiCounter) With(f Field) Counter {
	14	next := make(multiCounter, len(c))
	15	for i, counter := range c {
	16	next[i] = counter.With(f)
	17	}
	18	return next
	19	}
	20
	21	func (c multiCounter) Add(delta uint64) {
	22	for _, counter := range c {
	23	counter.Add(delta)
	24	}
	25	}
	26
	27	type multiGauge []Gauge
	28
	29	// NewMultiGauge returns a wrapper around multiple Gauges.
	30	func NewMultiGauge(gauges ...Gauge) Gauge {
	31	g := make(multiGauge, 0, len(gauges))
	32	for _, gauge := range gauges {
	33	g = append(g, gauge)
	34	}
	35	return g
	36	}
	37
	38	func (g multiGauge) With(f Field) Gauge {
	39	next := make(multiGauge, len(g))
	40	for i, gauge := range g {
	41	next[i] = gauge.With(f)
	42	}
	43	return next
	44	}
	45
	46	func (g multiGauge) Set(value int64) {
	47	for _, gauge := range g {
	48	gauge.Set(value)
	49	}
	50	}
	51
	52	func (g multiGauge) Add(delta int64) {
	53	for _, gauge := range g {
	54	gauge.Add(delta)
	55	}
	56	}
	57
	58	type multiHistogram []Histogram
	59
	60	// NewMultiHistogram returns a wrapper around multiple Histograms.
	61	func NewMultiHistogram(histograms ...Histogram) Histogram {
	62	h := make(multiHistogram, 0, len(histograms))
	63	for _, histogram := range histograms {
	64	h = append(h, histogram)
	65	}
	66	return h
	67	}
	68
	69	func (h multiHistogram) With(f Field) Histogram {
	70	next := make(multiHistogram, len(h))
	71	for i, histogram := range h {
	72	next[i] = histogram.With(f)
	73	}
	74	return next
	75	}
	76
	77	func (h multiHistogram) Observe(value int64) {
	78	for _, histogram := range h {
	79	histogram.Observe(value)
	80	}
	81	}

+94

-0

metrics/multi_test.go less more

	0	package metrics_test
	1
	2	import (
	3	"expvar"
	4	"strings"
	5	"testing"
	6
	7	"github.com/peterbourgon/gokit/metrics"
	8	)
	9
	10	func TestMultiWith(t *testing.T) {
	11	c := metrics.NewMultiCounter(
	12	metrics.NewExpvarCounter("foo"),
	13	metrics.NewPrometheusCounter("test", "multi_with", "bar", "Bar counter.", []string{"a"}),
	14	)
	15
	16	c.Add(1)
	17	c.With(metrics.Field{Key: "a", Value: "1"}).Add(2)
	18	c.Add(3)
	19
	20	if want, have := strings.Join([]string{
	21	`# HELP test_multi_with_bar Bar counter.`,
	22	`# TYPE test_multi_with_bar counter`,
	23	`test_multi_with_bar{a="1"} 2`,
	24	`test_multi_with_bar{a="unknown"} 4`,
	25	}, "\n"), scrapePrometheus(t); !strings.Contains(have, want) {
	26	t.Errorf("Prometheus metric stanza not found or incorrect\n%s", have)
	27	}
	28	}
	29
	30	func TestMultiCounter(t *testing.T) {
	31	metrics.NewMultiCounter(
	32	metrics.NewExpvarCounter("alpha"),
	33	metrics.NewPrometheusCounter("test", "multi_counter", "beta", "Beta counter.", []string{}),
	34	).Add(123)
	35
	36	if want, have := "123", expvar.Get("alpha").String(); want != have {
	37	t.Errorf("expvar: want %q, have %q", want, have)
	38	}
	39
	40	if want, have := strings.Join([]string{
	41	`# HELP test_multi_counter_beta Beta counter.`,
	42	`# TYPE test_multi_counter_beta counter`,
	43	`test_multi_counter_beta 123`,
	44	}, "\n"), scrapePrometheus(t); !strings.Contains(have, want) {
	45	t.Errorf("Prometheus metric stanza not found or incorrect\n%s", have)
	46	}
	47	}
	48
	49	func TestMultiGauge(t *testing.T) {
	50	g := metrics.NewMultiGauge(
	51	metrics.NewExpvarGauge("delta"),
	52	metrics.NewPrometheusGauge("test", "multi_gauge", "kappa", "Kappa gauge.", []string{}),
	53	)
	54
	55	g.Set(34)
	56
	57	if want, have := "34", expvar.Get("delta").String(); want != have {
	58	t.Errorf("expvar: want %q, have %q", want, have)
	59	}
	60	if want, have := strings.Join([]string{
	61	`# HELP test_multi_gauge_kappa Kappa gauge.`,
	62	`# TYPE test_multi_gauge_kappa gauge`,
	63	`test_multi_gauge_kappa 34`,
	64	}, "\n"), scrapePrometheus(t); !strings.Contains(have, want) {
	65	t.Errorf("Prometheus metric stanza not found or incorrect\n%s", have)
	66	}
	67
	68	g.Add(-40)
	69
	70	if want, have := "-6", expvar.Get("delta").String(); want != have {
	71	t.Errorf("expvar: want %q, have %q", want, have)
	72	}
	73	if want, have := strings.Join([]string{
	74	`# HELP test_multi_gauge_kappa Kappa gauge.`,
	75	`# TYPE test_multi_gauge_kappa gauge`,
	76	`test_multi_gauge_kappa -6`,
	77	}, "\n"), scrapePrometheus(t); !strings.Contains(have, want) {
	78	t.Errorf("Prometheus metric stanza not found or incorrect\n%s", have)
	79	}
	80	}
	81
	82	func TestMultiHistogram(t *testing.T) {
	83	quantiles := []int{50, 90, 99}
	84	h := metrics.NewMultiHistogram(
	85	metrics.NewExpvarHistogram("omicron", 0, 100, 3, quantiles...),
	86	metrics.NewPrometheusHistogram("test", "multi_histogram", "nu", "Nu histogram.", []string{}),
	87	)
	88
	89	const seed, mean, stdev int64 = 123, 50, 10
	90	populateNormalHistogram(t, h, seed, mean, stdev)
	91	assertExpvarNormalHistogram(t, "omicron", mean, stdev, quantiles)
	92	assertPrometheusNormalHistogram(t, "test_multi_histogram_nu", mean, stdev)
	93	}

+179

-0

metrics/prometheus/prometheus.go less more

	0	// Package prometheus implements a Prometheus backend for package metrics.
	1	package prometheus
	2
	3	import (
	4	"time"
	5
	6	"github.com/prometheus/client_golang/prometheus"
	7
	8	"github.com/peterbourgon/gokit/metrics"
	9	)
	10
	11	// Prometheus has strong opinions about the dimensionality of fields. Users
	12	// must predeclare every field key they intend to use. On every observation,
	13	// fields with keys that haven't been predeclared will be silently dropped,
	14	// and predeclared field keys without values will receive the value
	15	// PrometheusLabelValueUnknown.
	16	var PrometheusLabelValueUnknown = "unknown"
	17
	18	type prometheusCounter struct {
	19	*prometheus.CounterVec
	20	Pairs map[string]string
	21	}
	22
	23	// NewCounter returns a new Counter backed by a Prometheus metric. The counter
	24	// is automatically registered via prometheus.Register.
	25	func NewCounter(namespace, subsystem, name, help string, fieldKeys []string) metrics.Counter {
	26	return NewCounterWithLabels(namespace, subsystem, name, help, fieldKeys, prometheus.Labels{})
	27	}
	28
	29	// NewCounterWithLabels is the same as NewCounter, but attaches a set of const
	30	// label pairs to the metric.
	31	func NewCounterWithLabels(namespace, subsystem, name, help string, fieldKeys []string, constLabels prometheus.Labels) metrics.Counter {
	32	m := prometheus.NewCounterVec(
	33	prometheus.CounterOpts{
	34	Namespace: namespace,
	35	Subsystem: subsystem,
	36	Name: name,
	37	Help: help,
	38	ConstLabels: constLabels,
	39	},
	40	fieldKeys,
	41	)
	42	prometheus.MustRegister(m)
	43
	44	p := map[string]string{}
	45	for _, fieldName := range fieldKeys {
	46	p[fieldName] = PrometheusLabelValueUnknown
	47	}
	48
	49	return prometheusCounter{
	50	CounterVec: m,
	51	Pairs: p,
	52	}
	53	}
	54
	55	func (c prometheusCounter) With(f metrics.Field) metrics.Counter {
	56	return prometheusCounter{
	57	CounterVec: c.CounterVec,
	58	Pairs: merge(c.Pairs, f),
	59	}
	60	}
	61
	62	func (c prometheusCounter) Add(delta uint64) {
	63	c.CounterVec.With(prometheus.Labels(c.Pairs)).Add(float64(delta))
	64	}
	65
	66	type prometheusGauge struct {
	67	*prometheus.GaugeVec
	68	Pairs map[string]string
	69	}
	70
	71	// NewGauge returns a new Gauge backed by a Prometheus metric. The gauge is
	72	// automatically registered via prometheus.Register.
	73	func NewGauge(namespace, subsystem, name, help string, fieldKeys []string) metrics.Gauge {
	74	return NewGaugeWithLabels(namespace, subsystem, name, help, fieldKeys, prometheus.Labels{})
	75	}
	76
	77	// NewGaugeWithLabels is the same as NewGauge, but attaches a set of const
	78	// label pairs to the metric.
	79	func NewGaugeWithLabels(namespace, subsystem, name, help string, fieldKeys []string, constLabels prometheus.Labels) metrics.Gauge {
	80	m := prometheus.NewGaugeVec(
	81	prometheus.GaugeOpts{
	82	Namespace: namespace,
	83	Subsystem: subsystem,
	84	Name: name,
	85	Help: help,
	86	ConstLabels: constLabels,
	87	},
	88	fieldKeys,
	89	)
	90	prometheus.MustRegister(m)
	91
	92	return prometheusGauge{
	93	GaugeVec: m,
	94	Pairs: pairsFrom(fieldKeys),
	95	}
	96	}
	97
	98	func (g prometheusGauge) With(f metrics.Field) metrics.Gauge {
	99	return prometheusGauge{
	100	GaugeVec: g.GaugeVec,
	101	Pairs: merge(g.Pairs, f),
	102	}
	103	}
	104
	105	func (g prometheusGauge) Set(value int64) {
	106	g.GaugeVec.With(prometheus.Labels(g.Pairs)).Set(float64(value))
	107	}
	108
	109	func (g prometheusGauge) Add(delta int64) {
	110	g.GaugeVec.With(prometheus.Labels(g.Pairs)).Add(float64(delta))
	111	}
	112
	113	type prometheusHistogram struct {
	114	*prometheus.SummaryVec
	115	Pairs map[string]string
	116	}
	117
	118	// NewHistogram returns a new Histogram backed by a Prometheus summary. It
	119	// uses a 10-second max age for bucketing. The histogram is automatically
	120	// registered via prometheus.Register.
	121	func NewHistogram(namespace, subsystem, name, help string, fieldKeys []string) metrics.Histogram {
	122	return NewHistogramWithLabels(namespace, subsystem, name, help, fieldKeys, prometheus.Labels{})
	123	}
	124
	125	// NewHistogramWithLabels is the same as NewHistogram, but attaches a set of
	126	// const label pairs to the metric.
	127	func NewHistogramWithLabels(namespace, subsystem, name, help string, fieldKeys []string, constLabels prometheus.Labels) metrics.Histogram {
	128	m := prometheus.NewSummaryVec(
	129	prometheus.SummaryOpts{
	130	Namespace: namespace,
	131	Subsystem: subsystem,
	132	Name: name,
	133	Help: help,
	134	ConstLabels: constLabels,
	135	MaxAge: 10 * time.Second,
	136	},
	137	fieldKeys,
	138	)
	139	prometheus.MustRegister(m)
	140
	141	return prometheusHistogram{
	142	SummaryVec: m,
	143	Pairs: pairsFrom(fieldKeys),
	144	}
	145	}
	146
	147	func (h prometheusHistogram) With(f metrics.Field) metrics.Histogram {
	148	return prometheusHistogram{
	149	SummaryVec: h.SummaryVec,
	150	Pairs: merge(h.Pairs, f),
	151	}
	152	}
	153
	154	func (h prometheusHistogram) Observe(value int64) {
	155	h.SummaryVec.With(prometheus.Labels(h.Pairs)).Observe(float64(value))
	156	}
	157
	158	func pairsFrom(fieldKeys []string) map[string]string {
	159	p := map[string]string{}
	160	for _, fieldName := range fieldKeys {
	161	p[fieldName] = PrometheusLabelValueUnknown
	162	}
	163	return p
	164	}
	165
	166	func merge(orig map[string]string, f metrics.Field) map[string]string {
	167	if _, ok := orig[f.Key]; !ok {
	168	return orig
	169	}
	170
	171	newPairs := make(map[string]string, len(orig))
	172	for k, v := range orig {
	173	newPairs[k] = v
	174	}
	175
	176	newPairs[f.Key] = f.Value
	177	return newPairs
	178	}

+177

-0

metrics/prometheus/prometheus_test.go less more

	0	package prometheus_test
	1
	2	import (
	3	"io/ioutil"
	4	"math"
	5	"math/rand"
	6	"net/http"
	7	"net/http/httptest"
	8	"regexp"
	9	"strconv"
	10	"strings"
	11	"testing"
	12
	13	stdprometheus "github.com/prometheus/client_golang/prometheus"
	14
	15	"github.com/peterbourgon/gokit/metrics"
	16	"github.com/peterbourgon/gokit/metrics/prometheus"
	17	)
	18
	19	func TestPrometheusLabelBehavior(t *testing.T) {
	20	c := prometheus.NewCounter("test", "prometheus_label_behavior", "foobar", "Abc def.", []string{"used_key", "unused_key"})
	21	c.With(metrics.Field{Key: "used_key", Value: "declared"}).Add(1)
	22	c.Add(1)
	23
	24	if want, have := strings.Join([]string{
	25	`# HELP test_prometheus_label_behavior_foobar Abc def.`,
	26	`# TYPE test_prometheus_label_behavior_foobar counter`,
	27	`test_prometheus_label_behavior_foobar{unused_key="unknown",used_key="declared"} 1`,
	28	`test_prometheus_label_behavior_foobar{unused_key="unknown",used_key="unknown"} 1`,
	29	}, "\n"), scrapePrometheus(t); !strings.Contains(have, want) {
	30	t.Errorf("metric stanza not found or incorrect\n%s", have)
	31	}
	32	}
	33
	34	func TestPrometheusCounter(t *testing.T) {
	35	c := prometheus.NewCounter("test", "prometheus_counter", "foobar", "Lorem ipsum.", []string{})
	36	c.Add(1)
	37	c.Add(2)
	38	if want, have := strings.Join([]string{
	39	`# HELP test_prometheus_counter_foobar Lorem ipsum.`,
	40	`# TYPE test_prometheus_counter_foobar counter`,
	41	`test_prometheus_counter_foobar 3`,
	42	}, "\n"), scrapePrometheus(t); !strings.Contains(have, want) {
	43	t.Errorf("metric stanza not found or incorrect\n%s", have)
	44	}
	45	c.Add(3)
	46	c.Add(4)
	47	if want, have := strings.Join([]string{
	48	`# HELP test_prometheus_counter_foobar Lorem ipsum.`,
	49	`# TYPE test_prometheus_counter_foobar counter`,
	50	`test_prometheus_counter_foobar 10`,
	51	}, "\n"), scrapePrometheus(t); !strings.Contains(have, want) {
	52	t.Errorf("metric stanza not found or incorrect\n%s", have)
	53	}
	54	}
	55
	56	func TestPrometheusGauge(t *testing.T) {
	57	c := prometheus.NewGauge("test", "prometheus_gauge", "foobar", "Dolor sit.", []string{})
	58	c.Set(42)
	59	if want, have := strings.Join([]string{
	60	`# HELP test_prometheus_gauge_foobar Dolor sit.`,
	61	`# TYPE test_prometheus_gauge_foobar gauge`,
	62	`test_prometheus_gauge_foobar 42`,
	63	}, "\n"), scrapePrometheus(t); !strings.Contains(have, want) {
	64	t.Errorf("metric stanza not found or incorrect\n%s", have)
	65	}
	66	c.Add(-43)
	67	if want, have := strings.Join([]string{
	68	`# HELP test_prometheus_gauge_foobar Dolor sit.`,
	69	`# TYPE test_prometheus_gauge_foobar gauge`,
	70	`test_prometheus_gauge_foobar -1`,
	71	}, "\n"), scrapePrometheus(t); !strings.Contains(have, want) {
	72	t.Errorf("metric stanza not found or incorrect\n%s", have)
	73	}
	74	}
	75
	76	func TestPrometheusHistogram(t *testing.T) {
	77	h := prometheus.NewHistogram("test", "prometheus_histogram", "foobar", "Qwerty asdf.", []string{})
	78
	79	const mean, stdev int64 = 50, 10
	80	populateNormalHistogram(t, h, 34, mean, stdev)
	81	assertNormalHistogram(t, "test_prometheus_histogram_foobar", mean, stdev)
	82	}
	83
	84	func populateNormalHistogram(t *testing.T, h metrics.Histogram, seed int64, mean, stdev int64) {
	85	rand.Seed(seed)
	86	for i := 0; i < 1234; i++ {
	87	sample := int64(rand.NormFloat64()*float64(stdev) + float64(mean))
	88	h.Observe(sample)
	89	}
	90	}
	91
	92	func assertNormalHistogram(t *testing.T, metricName string, mean, stdev int64) {
	93	scrape := scrapePrometheus(t)
	94	const tolerance int = 5 // Prometheus approximates higher quantiles badly -_-;
	95	for quantileInt, quantileStr := range map[int]string{50: "0.5", 90: "0.9", 99: "0.99"} {
	96	want := normalValueAtQuantile(mean, stdev, quantileInt)
	97	have := getPrometheusQuantile(t, scrape, metricName, quantileStr)
	98	if int(math.Abs(float64(want)-float64(have))) > tolerance {
	99	t.Errorf("%q: want %d, have %d", quantileStr, want, have)
	100	}
	101	}
	102	}
	103
	104	// https://en.wikipedia.org/wiki/Normal_distribution#Quantile_function
	105	func normalValueAtQuantile(mean, stdev int64, quantile int) int64 {
	106	return int64(float64(mean) + float64(stdev)math.Sqrt2erfinv(2*(float64(quantile)/100)-1))
	107	}
	108
	109	// https://stackoverflow.com/questions/5971830/need-code-for-inverse-error-function
	110	func erfinv(y float64) float64 {
	111	if y < -1.0 \|\| y > 1.0 {
	112	panic("invalid input")
	113	}
	114
	115	var (
	116	a = [4]float64{0.886226899, -1.645349621, 0.914624893, -0.140543331}
	117	b = [4]float64{-2.118377725, 1.442710462, -0.329097515, 0.012229801}
	118	c = [4]float64{-1.970840454, -1.624906493, 3.429567803, 1.641345311}
	119	d = [2]float64{3.543889200, 1.637067800}
	120	)
	121
	122	const y0 = 0.7
	123	var x, z float64
	124
	125	if math.Abs(y) == 1.0 {
	126	x = -y * math.Log(0.0)
	127	} else if y < -y0 {
	128	z = math.Sqrt(-math.Log((1.0 + y) / 2.0))
	129	x = -(((c[3]z+c[2])z+c[1])z + c[0]) / ((d[1]z+d[0])*z + 1.0)
	130	} else {
	131	if y < y0 {
	132	z = y * y
	133	x = y * (((a[3]z+a[2])z+a[1])z + a[0]) / ((((b[3]z+b[3])z+b[1])z+b[0])*z + 1.0)
	134	} else {
	135	z = math.Sqrt(-math.Log((1.0 - y) / 2.0))
	136	x = (((c[3]z+c[2])z+c[1])z + c[0]) / ((d[1]z+d[0])*z + 1.0)
	137	}
	138	x = x - (math.Erf(x)-y)/(2.0/math.SqrtPimath.Exp(-xx))
	139	x = x - (math.Erf(x)-y)/(2.0/math.SqrtPimath.Exp(-xx))
	140	}
	141
	142	return x
	143	}
	144
	145	func scrapePrometheus(t *testing.T) string {
	146	server := httptest.NewServer(stdprometheus.UninstrumentedHandler())
	147	defer server.Close()
	148
	149	resp, err := http.Get(server.URL)
	150	if err != nil {
	151	t.Fatal(err)
	152	}
	153	defer resp.Body.Close()
	154
	155	buf, err := ioutil.ReadAll(resp.Body)
	156	if err != nil {
	157	t.Fatal(err)
	158	}
	159
	160	return strings.TrimSpace(string(buf))
	161	}
	162
	163	func getPrometheusQuantile(t *testing.T, scrape, name, quantileStr string) int {
	164	matches := regexp.MustCompile(name+`{quantile="`+quantileStr+`"} ([0-9]+)`).FindAllStringSubmatch(scrape, -1)
	165	if len(matches) < 1 {
	166	t.Fatalf("%q: quantile %q not found in scrape", name, quantileStr)
	167	}
	168	if len(matches[0]) < 2 {
	169	t.Fatalf("%q: quantile %q not found in scrape", name, quantileStr)
	170	}
	171	i, err := strconv.Atoi(matches[0][1])
	172	if err != nil {
	173	t.Fatal(err)
	174	}
	175	return i
	176	}

+23

-0

metrics/scaled_histogram.go less more

	0	package metrics
	1
	2	type scaledHistogram struct {
	3	Histogram
	4	scale int64
	5	}
	6
	7	// NewScaledHistogram returns a Histogram whose observed values are downscaled
	8	// (divided) by scale.
	9	func NewScaledHistogram(h Histogram, scale int64) Histogram {
	10	return scaledHistogram{h, scale}
	11	}
	12
	13	func (h scaledHistogram) With(f Field) Histogram {
	14	return scaledHistogram{
	15	Histogram: h.Histogram.With(f),
	16	scale: h.scale,
	17	}
	18	}
	19
	20	func (h scaledHistogram) Observe(value int64) {
	21	h.Histogram.Observe(value / h.scale)
	22	}

+21

-0

metrics/scaled_histogram_test.go less more

	0	package metrics_test
	1
	2	import (
	3	"testing"
	4
	5	"github.com/peterbourgon/gokit/metrics"
	6	)
	7
	8	func TestScaledHistogram(t *testing.T) {
	9	quantiles := []int{50, 90, 99}
	10	scale := int64(10)
	11	metricName := "test_scaled_histogram"
	12
	13	var h metrics.Histogram
	14	h = metrics.NewExpvarHistogram(metricName, 0, 1000, 3, quantiles...)
	15	h = metrics.NewScaledHistogram(h, scale)
	16
	17	const seed, mean, stdev = 333, 500, 100 // input values
	18	populateNormalHistogram(t, h, seed, mean, stdev) // will be scaled down
	19	assertExpvarNormalHistogram(t, metricName, mean/scale, stdev/scale, quantiles)
	20	}

+135

-0

metrics/statsd/statsd.go less more

	0	// Package statsd implements a statsd backend for package metrics.
	1	//
	2	// The current implementation ignores fields. In the future, it would be good
	3	// to have an implementation that accepted a set of predeclared field names at
	4	// construction time, and used field values to produce delimiter-separated
	5	// bucket (key) names. That is,
	6	//
	7	// c := NewFieldedCounter(..., "path", "status")
	8	// c.Add(1) // "myprefix.unknown.unknown:1\|c\n"
	9	// c2 := c.With("path", "foo").With("status": "200")
	10	// c2.Add(1) // "myprefix.foo.status:1\|c\n"
	11	//
	12	package statsd
	13
	14	import (
	15	"bytes"
	16	"fmt"
	17	"io"
	18	"log"
	19	"time"
	20
	21	"github.com/peterbourgon/gokit/metrics"
	22	)
	23
	24	// statsd metrics take considerable influence from
	25	// https://github.com/streadway/handy package statsd.
	26
	27	const maxBufferSize = 1400 // bytes
	28
	29	type statsdCounter chan string
	30
	31	// NewCounter returns a Counter that emits observations in the statsd protocol
	32	// to the passed writer. Observations are buffered for the reporting interval
	33	// or until the buffer exceeds a max packet size, whichever comes first.
	34	// Fields are ignored.
	35	//
	36	// TODO: support for sampling.
	37	func NewCounter(w io.Writer, key string, interval time.Duration) metrics.Counter {
	38	c := make(chan string)
	39	go fwd(w, key, interval, c)
	40	return statsdCounter(c)
	41	}
	42
	43	func (c statsdCounter) With(metrics.Field) metrics.Counter { return c }
	44
	45	func (c statsdCounter) Add(delta uint64) { c <- fmt.Sprintf("%d\|c", delta) }
	46
	47	type statsdGauge chan string
	48
	49	// NewGauge returns a Gauge that emits values in the statsd protocol to the
	50	// passed writer. Values are buffered for the reporting interval or until the
	51	// buffer exceeds a max packet size, whichever comes first. Fields are
	52	// ignored.
	53	//
	54	// TODO: support for sampling.
	55	func NewGauge(w io.Writer, key string, interval time.Duration) metrics.Gauge {
	56	g := make(chan string)
	57	go fwd(w, key, interval, g)
	58	return statsdGauge(g)
	59	}
	60
	61	func (g statsdGauge) With(metrics.Field) metrics.Gauge { return g }
	62
	63	func (g statsdGauge) Add(delta int64) {
	64	// https://github.com/etsy/statsd/blob/master/docs/metric_types.md#gauges
	65	sign := "+"
	66	if delta < 0 {
	67	sign, delta = "-", -delta
	68	}
	69	g <- fmt.Sprintf("%s%d\|g", sign, delta)
	70	}
	71
	72	func (g statsdGauge) Set(value int64) {
	73	g <- fmt.Sprintf("%d\|g", value)
	74	}
	75
	76	type statsdHistogram chan string
	77
	78	// NewHistogram returns a Histogram that emits observations in the statsd
	79	// protocol to the passed writer. Observations are buffered for the reporting
	80	// interval or until the buffer exceeds a max packet size, whichever comes
	81	// first. Fields are ignored.
	82	//
	83	// NewHistogram is mapped to a statsd Timing, so observations should represent
	84	// milliseconds. If you observe in units of nanoseconds, you can make the
	85	// translation with a ScaledHistogram:
	86	//
	87	// NewScaledHistogram(statsdHistogram, time.Millisecond)
	88	//
	89	// You can also enforce the constraint in a typesafe way with a millisecond
	90	// TimeHistogram:
	91	//
	92	// NewTimeHistogram(statsdHistogram, time.Millisecond)
	93	//
	94	// TODO: support for sampling.
	95	func NewHistogram(w io.Writer, key string, interval time.Duration) metrics.Histogram {
	96	h := make(chan string)
	97	go fwd(w, key, interval, h)
	98	return statsdHistogram(h)
	99	}
	100
	101	func (h statsdHistogram) With(metrics.Field) metrics.Histogram { return h }
	102
	103	func (h statsdHistogram) Observe(value int64) {
	104	h <- fmt.Sprintf("%d\|ms", value)
	105	}
	106
	107	var tick = time.Tick
	108
	109	func fwd(w io.Writer, key string, interval time.Duration, c chan string) {
	110	buf := &bytes.Buffer{}
	111	tick := tick(interval)
	112	for {
	113	select {
	114	case s := <-c:
	115	fmt.Fprintf(buf, "%s:%s\n", key, s)
	116	if buf.Len() > maxBufferSize {
	117	flush(w, buf)
	118	}
	119
	120	case <-tick:
	121	flush(w, buf)
	122	}
	123	}
	124	}
	125
	126	func flush(w io.Writer, buf *bytes.Buffer) {
	127	if buf.Len() <= 0 {
	128	return
	129	}
	130	if _, err := w.Write(buf.Bytes()); err != nil {
	131	log.Printf("error: could not write to statsd: %v", err)
	132	}
	133	buf.Reset()
	134	}

+86

-0

metrics/statsd/statsd_test.go less more

	0	package statsd
	1
	2	// In package metrics so we can stub tick.
	3
	4	import (
	5	"bytes"
	6	"runtime"
	7	"testing"
	8	"time"
	9	)
	10
	11	func TestCounter(t *testing.T) {
	12	ch := make(chan time.Time)
	13	tick = func(time.Duration) <-chan time.Time { return ch }
	14	defer func() { tick = time.Tick }()
	15
	16	buf := &bytes.Buffer{}
	17	c := NewCounter(buf, "test_statsd_counter", time.Second)
	18
	19	c.Add(1)
	20	c.Add(2)
	21	ch <- time.Now()
	22
	23	for i := 0; i < 10 && buf.Len() == 0; i++ {
	24	time.Sleep(time.Millisecond)
	25	}
	26
	27	if want, have := "test_statsd_counter:1\|c\ntest_statsd_counter:2\|c\n", buf.String(); want != have {
	28	t.Errorf("want %q, have %q", want, have)
	29	}
	30	}
	31
	32	func TestGauge(t *testing.T) {
	33	ch := make(chan time.Time)
	34	tick = func(time.Duration) <-chan time.Time { return ch }
	35	defer func() { tick = time.Tick }()
	36
	37	buf := &bytes.Buffer{}
	38	g := NewGauge(buf, "test_statsd_gauge", time.Second)
	39
	40	g.Add(1) // send command
	41	runtime.Gosched() // yield to buffer write
	42	ch <- time.Now() // signal flush
	43	runtime.Gosched() // yield to flush
	44	if want, have := "test_statsd_gauge:+1\|g\n", buf.String(); want != have {
	45	t.Errorf("want %q, have %q", want, have)
	46	}
	47
	48	buf.Reset()
	49
	50	g.Add(-2)
	51	runtime.Gosched()
	52	ch <- time.Now()
	53	runtime.Gosched()
	54	if want, have := "test_statsd_gauge:-2\|g\n", buf.String(); want != have {
	55	t.Errorf("want %q, have %q", want, have)
	56	}
	57
	58	buf.Reset()
	59
	60	g.Set(3)
	61	runtime.Gosched()
	62	ch <- time.Now()
	63	runtime.Gosched()
	64	if want, have := "test_statsd_gauge:3\|g\n", buf.String(); want != have {
	65	t.Errorf("want %q, have %q", want, have)
	66	}
	67	}
	68
	69	func TestHistogram(t *testing.T) {
	70	ch := make(chan time.Time)
	71	tick = func(time.Duration) <-chan time.Time { return ch }
	72	defer func() { tick = time.Tick }()
	73
	74	buf := &bytes.Buffer{}
	75	h := NewHistogram(buf, "test_statsd_histogram", time.Second)
	76
	77	h.Observe(123)
	78
	79	runtime.Gosched()
	80	ch <- time.Now()
	81	runtime.Gosched()
	82	if want, have := "test_statsd_histogram:123\|ms\n", buf.String(); want != have {
	83	t.Errorf("want %q, have %q", want, have)
	84	}
	85	}

+34

-0

metrics/time_histogram.go less more

	0	package metrics
	1
	2	import "time"
	3
	4	// TimeHistogram is a convenience wrapper for a Histogram of time.Durations.
	5	type TimeHistogram interface {
	6	With(Field) TimeHistogram
	7	Observe(time.Duration)
	8	}
	9
	10	type timeHistogram struct {
	11	Histogram
	12	unit time.Duration
	13	}
	14
	15	// NewTimeHistogram returns a TimeHistogram wrapper around the passed
	16	// Histogram, in units of unit.
	17	func NewTimeHistogram(h Histogram, unit time.Duration) TimeHistogram {
	18	return &timeHistogram{
	19	Histogram: h,
	20	unit: unit,
	21	}
	22	}
	23
	24	func (h *timeHistogram) With(f Field) TimeHistogram {
	25	return &timeHistogram{
	26	Histogram: h.Histogram.With(f),
	27	unit: h.unit,
	28	}
	29	}
	30
	31	func (h *timeHistogram) Observe(d time.Duration) {
	32	h.Histogram.Observe(int64(d / h.unit))
	33	}

+24

-0

metrics/time_histogram_test.go less more

	0	package metrics_test
	1
	2	import (
	3	"math/rand"
	4	"testing"
	5	"time"
	6
	7	"github.com/peterbourgon/gokit/metrics"
	8	)
	9
	10	func TestTimeHistogram(t *testing.T) {
	11	const metricName string = "test_time_histogram"
	12	quantiles := []int{50, 90, 99}
	13	h0 := metrics.NewExpvarHistogram(metricName, 0, 200, 3, quantiles...)
	14	h := metrics.NewTimeHistogram(h0, time.Millisecond)
	15	const seed, mean, stdev int64 = 321, 100, 20
	16
	17	for i := 0; i < 4321; i++ {
	18	sample := time.Duration(rand.NormFloat64()float64(stdev)+float64(mean)) time.Millisecond
	19	h.Observe(sample)
	20	}
	21
	22	assertExpvarNormalHistogram(t, metricName, mean, stdev, quantiles)
	23	}

+49

-2

rfc/rfc003-package-metrics.md less more

11	11
12	12	## Scope
13	13
14		To be defined.
	14	- Package metrics SHALL implement Gauges, Counters, and Histograms.
	15
	16	- Each metric type SHALL allow observations with an unlimited number of key/value field pairs,
	17	similar to [package log](https://github.com/peterbourgon/gokit/blob/master/rfc/rfc004-package-log.md).
	18
	19	- Counter SHALL be an increment-only counter of type uint64.
	20
	21	- Gauge SHALL be an arbitrarily-settable register of type int64.
	22
	23	- Histogram SHALL collect observations of type int64.
	24
	25	- These interfaces SHALL be the primary and exclusive API for metrics.
	26
	27	- We SHALL provide a variety of implementations of each interface that act as a
	28	bridge to different backends: expvar, Graphite, statsd, Prometheus, etc.
	29
	30	- Each metric backend MAY provide additional value-add behaviors. For example,
	31	a backend for Histogram may bucket observations according to quantile and
	32	calculate additional, derived statistics.
	33
15	34
16	35	## Implementation
17	36
18		To be defined.⏎
	37	https://github.com/peterbourgon/gokit/tree/master/metrics
	38
	39	### Gauge
	40
	41	```go
	42	type Gauge interface {
	43	With(Field) Gauge
	44	Set(value int64)
	45	Add(delta int64)
	46	}
	47	```
	48
	49	### Counter
	50
	51	```go
	52	type Counter interface {
	53	With(Field) Counter
	54	Add(delta uint64)
	55	}
	56	```
	57
	58	### Histogram
	59
	60	```go
	61	type Histogram interface {
	62	With(Field) Histogram
	63	Observe(int64)
	64	}
	65	```