Package list golang-github-go-kit-kit / e8ced30
circuitbreaker: attempt to fix flaky Hystrix test (again) Peter Bourgon 5 years ago
5 changed file(s) with 28 addition(s) and 25 deletion(s). Raw diff Collapse all Expand all
1414 shouldPass = func(n int) bool { return n <= 5 } // https://github.com/sony/gobreaker/blob/bfa846d/gobreaker.go#L76
1515 circuitOpenError = "circuit breaker is open"
1616 )
17 testFailingEndpoint(t, breaker, primeWith, shouldPass, circuitOpenError)
17 testFailingEndpoint(t, breaker, primeWith, shouldPass, 0, circuitOpenError)
1818 }
1515 shouldPass = func(n int) bool { return (float64(n) / float64(primeWith+n)) <= failureRatio }
1616 openCircuitError = handybreaker.ErrCircuitOpen.Error()
1717 )
18 testFailingEndpoint(t, breaker, primeWith, shouldPass, openCircuitError)
18 testFailingEndpoint(t, breaker, primeWith, shouldPass, 0, openCircuitError)
1919 }
1616 func Hystrix(commandName string) endpoint.Middleware {
1717 return func(next endpoint.Endpoint) endpoint.Endpoint {
1818 return func(ctx context.Context, request interface{}) (response interface{}, err error) {
19 output := make(chan interface{}, 1)
20 errors := hystrix.Go(commandName, func() error {
21 resp, err := next(ctx, request)
22 if err == nil {
23 output <- resp
24 }
19 var resp interface{}
20 if err := hystrix.Do(commandName, func() (err error) {
21 resp, err = next(ctx, request)
2522 return err
26 }, nil)
27
28 select {
29 case resp := <-output:
30 return resp, nil
31 case err := <-errors:
23 }, nil); err != nil {
3224 return nil, err
3325 }
26 return resp, nil
3427 }
3528 }
3629 }
00 package circuitbreaker_test
11
22 import (
3 "io/ioutil"
34 stdlog "log"
4 "os"
55 "testing"
6 "time"
67
78 "github.com/afex/hystrix-go/hystrix"
89
910 "github.com/go-kit/kit/circuitbreaker"
10 kitlog "github.com/go-kit/kit/log"
1111 )
1212
1313 func TestHystrix(t *testing.T) {
14 logger := kitlog.NewLogfmtLogger(os.Stderr)
15 stdlog.SetOutput(kitlog.NewStdlibAdapter(logger))
14 stdlog.SetOutput(ioutil.Discard)
1615
1716 const (
1817 commandName = "my-endpoint"
3029 shouldPass = func(n int) bool { return (float64(n) / float64(primeWith+n)) <= (float64(errorPercent-1) / 100.0) }
3130 openCircuitError = hystrix.ErrCircuitOpen.Error()
3231 )
33 testFailingEndpoint(t, breaker, primeWith, shouldPass, openCircuitError)
32
33 // hystrix-go uses buffered channels to receive reports on request success/failure,
34 // and so is basically impossible to test deterministically. We have to make sure
35 // the report buffer is emptied, by injecting a sleep between each invocation.
36 requestDelay := 5 * time.Millisecond
37
38 testFailingEndpoint(t, breaker, primeWith, shouldPass, requestDelay, openCircuitError)
3439 }
1212 "github.com/go-kit/kit/endpoint"
1313 )
1414
15 func testFailingEndpoint(t *testing.T, breaker endpoint.Middleware, primeWith int, shouldPass func(int) bool, openCircuitError string) {
15 func testFailingEndpoint(
16 t *testing.T,
17 breaker endpoint.Middleware,
18 primeWith int,
19 shouldPass func(int) bool,
20 requestDelay time.Duration,
21 openCircuitError string,
22 ) {
1623 _, file, line, _ := runtime.Caller(1)
1724 caller := fmt.Sprintf("%s:%d", filepath.Base(file), line)
1825
2734 if _, err := e(context.Background(), struct{}{}); err != nil {
2835 t.Fatalf("%s: during priming, got error: %v", caller, err)
2936 }
37 time.Sleep(requestDelay)
3038 }
3139
3240 // Switch the endpoint to start throwing errors.
3846 if _, err := e(context.Background(), struct{}{}); err != m.err {
3947 t.Fatalf("%s: want %v, have %v", caller, m.err, err)
4048 }
49 time.Sleep(requestDelay)
4150 }
4251 thru := m.thru
43
44 // Adding the sleep due to https://github.com/afex/hystrix-go/issues/41
45 // Increasing the sleep due to https://github.com/go-kit/kit/issues/169
46 // And increasing again for the same reason.
47 time.Sleep(50 * time.Millisecond)
4852
4953 // But the rest should be blocked by an open circuit.
5054 for i := 0; i < 10; i++ {
5155 if _, err := e(context.Background(), struct{}{}); err.Error() != openCircuitError {
5256 t.Fatalf("%s: want %q, have %q", caller, openCircuitError, err.Error())
5357 }
58 time.Sleep(requestDelay)
5459 }
5560
5661 // Make sure none of those got through.