Add backoff package and fix Consul CPU usage (#635)
* Add backoff package
Justification for jitter and growth factor:
https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/.
Add backoff to the Consul instancer loop.
Fixes https://github.com/go-kit/kit/issues/627.
* Revert "Add backoff package"
This reverts commit 924501ae1fcfadaa27593e9c019283412c513928.
* Get rid of external package and update exponential
* Add instancer backoff
* Fix old exponential name
* Add doc comment
* Fixup & respond to review
Nico Tonozzi authored 4 years ago
Peter Bourgon committed 4 years ago
| 2 | 2 |
import (
|
| 3 | 3 |
"fmt"
|
| 4 | 4 |
"io"
|
|
5 |
"time"
|
| 5 | 6 |
|
| 6 | 7 |
consul "github.com/hashicorp/consul/api"
|
| 7 | 8 |
|
| 8 | 9 |
"github.com/go-kit/kit/log"
|
| 9 | 10 |
"github.com/go-kit/kit/sd"
|
| 10 | 11 |
"github.com/go-kit/kit/sd/internal/instance"
|
|
12 |
"github.com/go-kit/kit/util/conn"
|
| 11 | 13 |
)
|
| 12 | 14 |
|
| 13 | 15 |
const defaultIndex = 0
|
|
| 58 | 60 |
var (
|
| 59 | 61 |
instances []string
|
| 60 | 62 |
err error
|
|
63 |
d time.Duration = 10 * time.Millisecond
|
| 61 | 64 |
)
|
| 62 | 65 |
for {
|
| 63 | 66 |
instances, lastIndex, err = s.getInstances(lastIndex, s.quitc)
|
|
| 66 | 69 |
return // stopped via quitc
|
| 67 | 70 |
case err != nil:
|
| 68 | 71 |
s.logger.Log("err", err)
|
|
72 |
time.Sleep(d)
|
|
73 |
d = conn.Exponential(d)
|
| 69 | 74 |
s.cache.Update(sd.Event{Err: err})
|
| 70 | 75 |
default:
|
| 71 | 76 |
s.cache.Update(sd.Event{Instances: instances})
|
|
77 |
d = 10 * time.Millisecond
|
| 72 | 78 |
}
|
| 73 | 79 |
}
|
| 74 | 80 |
}
|
| 1 | 1 |
|
| 2 | 2 |
import (
|
| 3 | 3 |
"errors"
|
|
4 |
"math/rand"
|
| 4 | 5 |
"net"
|
| 5 | 6 |
"time"
|
| 6 | 7 |
|
|
| 102 | 103 |
case conn = <-connc:
|
| 103 | 104 |
if conn == nil {
|
| 104 | 105 |
// didn't work
|
| 105 | |
backoff = exponential(backoff) // wait longer
|
|
106 |
backoff = Exponential(backoff) // wait longer
|
| 106 | 107 |
reconnectc = m.after(backoff) // try again
|
| 107 | 108 |
} else {
|
| 108 | 109 |
// worked!
|
|
| 131 | 132 |
return conn
|
| 132 | 133 |
}
|
| 133 | 134 |
|
| 134 | |
func exponential(d time.Duration) time.Duration {
|
|
135 |
// Exponential takes a duration and returns another one that is twice as long, +/- 50%. It is
|
|
136 |
// used to provide backoff for operations that may fail and should avoid thundering herds.
|
|
137 |
// See https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/ for rationale
|
|
138 |
func Exponential(d time.Duration) time.Duration {
|
| 135 | 139 |
d *= 2
|
|
140 |
jitter := rand.Float64() + 0.5
|
|
141 |
d = time.Duration(int64(float64(d.Nanoseconds()) * jitter))
|
| 136 | 142 |
if d > time.Minute {
|
| 137 | 143 |
d = time.Minute
|
| 138 | 144 |
}
|
| 139 | 145 |
return d
|
|
146 |
|
| 140 | 147 |
}
|
| 141 | 148 |
|
| 142 | 149 |
// ErrConnectionUnavailable is returned by the Manager's Write method when the
|