0 | |
// Package graphite implements a graphite backend for package metrics.
|
|
0 |
// Package graphite implements a Graphite backend for package metrics. Metrics
|
|
1 |
// will be emitted to a Graphite server in the plaintext protocol
|
|
2 |
// (http://graphite.readthedocs.io/en/latest/feeding-carbon.html#the-plaintext-protocol)
|
|
3 |
// which looks like:
|
|
4 |
// "<metric path> <metric value> <metric timestamp>"
|
1 | 5 |
//
|
2 | 6 |
// The current implementation ignores fields.
|
3 | 7 |
package graphite
|
|
6 | 10 |
"bufio"
|
7 | 11 |
"fmt"
|
8 | 12 |
"io"
|
9 | |
"log"
|
10 | 13 |
"math"
|
11 | 14 |
"net"
|
12 | 15 |
"sort"
|
|
15 | 18 |
"time"
|
16 | 19 |
|
17 | 20 |
"github.com/codahale/hdrhistogram"
|
|
21 |
"github.com/go-kit/kit/log"
|
18 | 22 |
"github.com/go-kit/kit/metrics"
|
19 | 23 |
)
|
20 | 24 |
|
21 | 25 |
// Emitter will keep track of all metrics and, once started,
|
22 | 26 |
// will emit the metrics via the Flush method to the given address.
|
23 | 27 |
type Emitter interface {
|
24 | |
NewCounter(string) metrics.Counter
|
25 | |
NewHistogram(string, int64, int64, int, ...int) metrics.Histogram
|
26 | |
NewTimeHistogram(string, time.Duration, int64, int64, int, ...int) metrics.TimeHistogram
|
27 | |
NewGauge(string) metrics.Gauge
|
28 | |
|
29 | |
Start(time.Duration)
|
30 | |
Flush() error
|
|
28 |
NewCounter(name string) metrics.Counter
|
|
29 |
NewHistogram(name string, min int64, max int64, sigfigs int, quantiles ...int) (metrics.Histogram, error)
|
|
30 |
NewTimeHistogram(name string, unit time.Duration, min int64, max int64, sigfigs int, quantiles ...int) (metrics.TimeHistogram, error)
|
|
31 |
NewGauge(name string) metrics.Gauge
|
|
32 |
|
|
33 |
Start(reportInvterval time.Duration) error
|
|
34 |
Flush()
|
31 | 35 |
}
|
32 | 36 |
|
33 | 37 |
type emitter struct {
|
34 | |
addr *net.TCPAddr
|
35 | 38 |
prefix string
|
36 | 39 |
|
37 | |
metricMu *sync.Mutex
|
|
40 |
addr string
|
|
41 |
tcp bool
|
|
42 |
conn net.Conn
|
|
43 |
|
|
44 |
mtx sync.Mutex
|
38 | 45 |
counters []*counter
|
39 | 46 |
histograms []*windowedHistogram
|
40 | 47 |
gauges []*gauge
|
|
48 |
|
|
49 |
logger log.Logger
|
41 | 50 |
}
|
42 | 51 |
|
43 | 52 |
// NewEmitter will return an Emitter that will prefix all
|
44 | 53 |
// metrics names with the given prefix. Once started, it will attempt to create
|
45 | |
// a TCP connection with the given address and periodically post metrics to the
|
46 | |
// connection in a Graphite-compatible format.
|
47 | |
func NewEmitter(addr *net.TCPAddr, prefix string) Emitter {
|
48 | |
e := &emitter{
|
49 | |
addr, prefix, &sync.Mutex{},
|
50 | |
[]*counter{}, []*windowedHistogram{}, []*gauge{},
|
51 | |
}
|
52 | |
|
53 | |
return e
|
|
54 |
// a TCP or a UDP connection with the given address and periodically post
|
|
55 |
// metrics to the connection in the Graphite plaintext protocol.
|
|
56 |
// If the provided `tcp` parameter is false, a UDP connection will be used.
|
|
57 |
func NewEmitter(addr string, tcp bool, metricsPrefix string, logger log.Logger) Emitter {
|
|
58 |
return &emitter{
|
|
59 |
addr: addr,
|
|
60 |
tcp: tcp,
|
|
61 |
prefix: metricsPrefix,
|
|
62 |
logger: logger,
|
|
63 |
}
|
54 | 64 |
}
|
55 | 65 |
|
56 | 66 |
// NewCounter returns a Counter whose value will be periodically emitted in
|
57 | 67 |
// a Graphite-compatible format once the Emitter is started. Fields are ignored.
|
58 | 68 |
func (e *emitter) NewCounter(name string) metrics.Counter {
|
59 | 69 |
// only one flush at a time
|
60 | |
e.metricMu.Lock()
|
61 | |
defer e.metricMu.Unlock()
|
62 | 70 |
c := &counter{name, 0}
|
|
71 |
e.mtx.Lock()
|
63 | 72 |
e.counters = append(e.counters, c)
|
|
73 |
e.mtx.Unlock()
|
64 | 74 |
return c
|
65 | 75 |
}
|
66 | 76 |
|
|
73 | 83 |
//
|
74 | 84 |
// The values of this histogram will be periodically emitted in a Graphite-compatible
|
75 | 85 |
// format once the Emitter is started. Fields are ignored.
|
76 | |
func (e *emitter) NewHistogram(name string, minValue, maxValue int64, sigfigs int, quantiles ...int) metrics.Histogram {
|
77 | |
// only one flush at a time
|
78 | |
e.metricMu.Lock()
|
79 | |
defer e.metricMu.Unlock()
|
80 | |
|
|
86 |
func (e *emitter) NewHistogram(name string, minValue, maxValue int64, sigfigs int, quantiles ...int) (metrics.Histogram, error) {
|
81 | 87 |
gauges := map[int]metrics.Gauge{}
|
82 | 88 |
for _, quantile := range quantiles {
|
83 | 89 |
if quantile <= 0 || quantile >= 100 {
|
84 | |
panic(fmt.Sprintf("invalid quantile %d", quantile))
|
|
90 |
return nil, fmt.Errorf("invalid quantile %d", quantile)
|
85 | 91 |
}
|
86 | 92 |
gauges[quantile] = e.gauge(fmt.Sprintf("%s_p%02d", name, quantile))
|
87 | 93 |
}
|
88 | |
h := newWindowedHistogram(name, minValue, maxValue, sigfigs, gauges)
|
|
94 |
h := newWindowedHistogram(name, minValue, maxValue, sigfigs, gauges, e.logger)
|
|
95 |
|
|
96 |
e.mtx.Lock()
|
89 | 97 |
e.histograms = append(e.histograms, h)
|
90 | |
return h
|
|
98 |
e.mtx.Unlock()
|
|
99 |
return h, nil
|
91 | 100 |
}
|
92 | 101 |
|
93 | 102 |
// NewTimeHistogram returns a TimeHistogram wrapper around the windowed
|
94 | 103 |
// HDR histrogram provided by this package.
|
95 | |
func (e *emitter) NewTimeHistogram(name string, unit time.Duration, minValue, maxValue int64, sigfigs int, quantiles ...int) metrics.TimeHistogram {
|
96 | |
h := e.NewHistogram(name, minValue, maxValue, sigfigs, quantiles...)
|
97 | |
return metrics.NewTimeHistogram(unit, h)
|
|
104 |
func (e *emitter) NewTimeHistogram(name string, unit time.Duration, minValue, maxValue int64, sigfigs int, quantiles ...int) (metrics.TimeHistogram, error) {
|
|
105 |
h, err := e.NewHistogram(name, minValue, maxValue, sigfigs, quantiles...)
|
|
106 |
if err != nil {
|
|
107 |
return nil, err
|
|
108 |
}
|
|
109 |
return metrics.NewTimeHistogram(unit, h), nil
|
98 | 110 |
}
|
99 | 111 |
|
100 | 112 |
// NewGauge returns a Gauge whose value will be periodically emitted in
|
101 | 113 |
// a Graphite-compatible format once the Emitter is started. Fields are ignored.
|
102 | 114 |
func (e *emitter) NewGauge(name string) metrics.Gauge {
|
103 | |
// only one flush at a time
|
104 | |
e.metricMu.Lock()
|
105 | |
defer e.metricMu.Unlock()
|
|
115 |
e.mtx.Lock()
|
|
116 |
defer e.mtx.Unlock()
|
106 | 117 |
return e.gauge(name)
|
107 | 118 |
}
|
108 | 119 |
|
|
112 | 123 |
return g
|
113 | 124 |
}
|
114 | 125 |
|
|
126 |
func (e *emitter) dial() error {
|
|
127 |
if e.tcp {
|
|
128 |
tAddr, err := net.ResolveTCPAddr("tcp", e.addr)
|
|
129 |
if err != nil {
|
|
130 |
return err
|
|
131 |
}
|
|
132 |
e.conn, err = net.DialTCP("tcp", nil, tAddr)
|
|
133 |
if err != nil {
|
|
134 |
return err
|
|
135 |
}
|
|
136 |
} else {
|
|
137 |
uAddr, err := net.ResolveUDPAddr("udp", e.addr)
|
|
138 |
if err != nil {
|
|
139 |
return err
|
|
140 |
}
|
|
141 |
e.conn, err = net.DialUDP("udp", nil, uAddr)
|
|
142 |
if err != nil {
|
|
143 |
return err
|
|
144 |
}
|
|
145 |
}
|
|
146 |
return nil
|
|
147 |
}
|
|
148 |
|
115 | 149 |
// Start will kick off a background goroutine to
|
116 | 150 |
// call Flush once every interval.
|
117 | |
func (e *emitter) Start(interval time.Duration) {
|
|
151 |
func (e *emitter) Start(interval time.Duration) error {
|
|
152 |
err := e.dial()
|
|
153 |
if err != nil {
|
|
154 |
return err
|
|
155 |
}
|
118 | 156 |
go func() {
|
119 | |
t := time.Tick(interval)
|
120 | |
for range t {
|
121 | |
err := e.Flush()
|
122 | |
if err != nil {
|
123 | |
log.Print("error: could not dial graphite host: ", err)
|
124 | |
continue
|
125 | |
}
|
|
157 |
for range time.Tick(interval) {
|
|
158 |
e.Flush()
|
126 | 159 |
}
|
127 | 160 |
}()
|
|
161 |
return nil
|
128 | 162 |
}
|
129 | 163 |
|
130 | 164 |
// Flush will attempt to create a connection with the given address
|
131 | |
// and write the current metrics to it in a Graphite-compatible format.
|
|
165 |
// and write the current metrics to it in the Graphite plaintext protocol.
|
132 | 166 |
//
|
133 | 167 |
// Users can call this method on process shutdown to ensure
|
134 | 168 |
// the current metrics are pushed to Graphite.
|
135 | |
func (e *emitter) Flush() error {
|
136 | |
// open connection
|
137 | |
conn, err := net.DialTCP("tcp", nil, e.addr)
|
138 | |
if err != nil {
|
139 | |
return err
|
140 | |
}
|
141 | |
|
142 | |
// flush stats to connection
|
143 | |
e.flush(conn)
|
144 | |
|
145 | |
// close connection
|
146 | |
conn.Close()
|
147 | |
return nil
|
148 | |
}
|
|
169 |
func (e *emitter) Flush() { e.flush(e.conn) }
|
149 | 170 |
|
150 | 171 |
func (e *emitter) flush(conn io.Writer) {
|
151 | 172 |
// only one flush at a time
|
152 | |
e.metricMu.Lock()
|
153 | |
defer e.metricMu.Unlock()
|
|
173 |
e.mtx.Lock()
|
|
174 |
defer e.mtx.Unlock()
|
154 | 175 |
|
155 | 176 |
// buffer the writer and make sure to flush it
|
156 | 177 |
w := bufio.NewWriter(conn)
|
157 | 178 |
defer w.Flush()
|
158 | 179 |
|
159 | |
now := time.Now().Unix()
|
160 | |
|
161 | 180 |
// emit counter stats
|
162 | 181 |
for _, c := range e.counters {
|
163 | |
fmt.Fprintf(w, "%s.%s.count %d %d\n", e.prefix, c.Name(), c.count, now)
|
|
182 |
fmt.Fprintf(w, "%s.%s.count %d %d\n", e.prefix, c.Name(), c.count, time.Now().Unix())
|
164 | 183 |
}
|
165 | 184 |
|
166 | 185 |
// emit histogram specific stats
|
167 | 186 |
for _, h := range e.histograms {
|
168 | 187 |
hist := h.hist.Merge()
|
|
188 |
now := time.Now().Unix()
|
169 | 189 |
fmt.Fprintf(w, "%s.%s.count %d %d\n", e.prefix, h.Name(), hist.TotalCount(), now)
|
170 | 190 |
fmt.Fprintf(w, "%s.%s.min %d %d\n", e.prefix, h.Name(), hist.Min(), now)
|
171 | 191 |
fmt.Fprintf(w, "%s.%s.max %d %d\n", e.prefix, h.Name(), hist.Max(), now)
|
|
175 | 195 |
|
176 | 196 |
// emit gauge stats (which can include some histogram quantiles)
|
177 | 197 |
for _, g := range e.gauges {
|
178 | |
fmt.Fprintf(w, "%s.%s %.2f %d\n", e.prefix, g.Name(), g.Get(), now)
|
|
198 |
fmt.Fprintf(w, "%s.%s %.2f %d\n", e.prefix, g.Name(), g.Get(), time.Now().Unix())
|
179 | 199 |
}
|
180 | 200 |
}
|
181 | 201 |
|
|
218 | 238 |
}
|
219 | 239 |
|
220 | 240 |
type windowedHistogram struct {
|
221 | |
mu sync.Mutex
|
|
241 |
mtx sync.Mutex
|
222 | 242 |
hist *hdrhistogram.WindowedHistogram
|
223 | 243 |
|
224 | 244 |
name string
|
225 | 245 |
gauges map[int]metrics.Gauge
|
|
246 |
logger log.Logger
|
226 | 247 |
}
|
227 | 248 |
|
228 | 249 |
// newWindowedHistogram is taken from http://github.com/codahale/metrics. It returns a
|
|
231 | 252 |
// The histogram exposes metrics for each passed quantile as gauges. Users are expected
|
232 | 253 |
// to provide their own set of Gauges for quantiles to make this Histogram work across multiple
|
233 | 254 |
// metrics providers.
|
234 | |
func newWindowedHistogram(name string, minValue, maxValue int64, sigfigs int, quantiles map[int]metrics.Gauge) *windowedHistogram {
|
|
255 |
func newWindowedHistogram(name string, minValue, maxValue int64, sigfigs int, quantiles map[int]metrics.Gauge, logger log.Logger) *windowedHistogram {
|
235 | 256 |
h := &windowedHistogram{
|
236 | 257 |
hist: hdrhistogram.NewWindowed(5, minValue, maxValue, sigfigs),
|
237 | 258 |
name: name,
|
238 | 259 |
gauges: quantiles,
|
|
260 |
logger: logger,
|
239 | 261 |
}
|
240 | 262 |
go h.rotateLoop(1 * time.Minute)
|
241 | 263 |
return h
|
|
245 | 267 |
func (h *windowedHistogram) With(metrics.Field) metrics.Histogram { return h }
|
246 | 268 |
|
247 | 269 |
func (h *windowedHistogram) Observe(value int64) {
|
248 | |
h.mu.Lock()
|
|
270 |
h.mtx.Lock()
|
249 | 271 |
err := h.hist.Current.RecordValue(value)
|
250 | |
h.mu.Unlock()
|
|
272 |
h.mtx.Unlock()
|
251 | 273 |
|
252 | 274 |
if err != nil {
|
253 | |
panic(err.Error())
|
|
275 |
h.logger.Log("err", err, "msg", "unable to record histogram value")
|
|
276 |
return
|
254 | 277 |
}
|
255 | 278 |
|
256 | 279 |
for q, gauge := range h.gauges {
|
|
281 | 304 |
|
282 | 305 |
func (h *windowedHistogram) rotateLoop(d time.Duration) {
|
283 | 306 |
for range time.Tick(d) {
|
284 | |
h.mu.Lock()
|
|
307 |
h.mtx.Lock()
|
285 | 308 |
h.hist.Rotate()
|
286 | |
h.mu.Unlock()
|
|
309 |
h.mtx.Unlock()
|
287 | 310 |
}
|
288 | 311 |
}
|
289 | 312 |
|