circuitbreaker: attempt to fix flaky Hystrix test (again)
Peter Bourgon
8 years ago
14 | 14 | shouldPass = func(n int) bool { return n <= 5 } // https://github.com/sony/gobreaker/blob/bfa846d/gobreaker.go#L76 |
15 | 15 | circuitOpenError = "circuit breaker is open" |
16 | 16 | ) |
17 | testFailingEndpoint(t, breaker, primeWith, shouldPass, circuitOpenError) | |
17 | testFailingEndpoint(t, breaker, primeWith, shouldPass, 0, circuitOpenError) | |
18 | 18 | } |
15 | 15 | shouldPass = func(n int) bool { return (float64(n) / float64(primeWith+n)) <= failureRatio } |
16 | 16 | openCircuitError = handybreaker.ErrCircuitOpen.Error() |
17 | 17 | ) |
18 | testFailingEndpoint(t, breaker, primeWith, shouldPass, openCircuitError) | |
18 | testFailingEndpoint(t, breaker, primeWith, shouldPass, 0, openCircuitError) | |
19 | 19 | } |
16 | 16 | func Hystrix(commandName string) endpoint.Middleware { |
17 | 17 | return func(next endpoint.Endpoint) endpoint.Endpoint { |
18 | 18 | return func(ctx context.Context, request interface{}) (response interface{}, err error) { |
19 | output := make(chan interface{}, 1) | |
20 | errors := hystrix.Go(commandName, func() error { | |
21 | resp, err := next(ctx, request) | |
22 | if err == nil { | |
23 | output <- resp | |
24 | } | |
19 | var resp interface{} | |
20 | if err := hystrix.Do(commandName, func() (err error) { | |
21 | resp, err = next(ctx, request) | |
25 | 22 | return err |
26 | }, nil) | |
27 | ||
28 | select { | |
29 | case resp := <-output: | |
30 | return resp, nil | |
31 | case err := <-errors: | |
23 | }, nil); err != nil { | |
32 | 24 | return nil, err |
33 | 25 | } |
26 | return resp, nil | |
34 | 27 | } |
35 | 28 | } |
36 | 29 | } |
0 | 0 | package circuitbreaker_test |
1 | 1 | |
2 | 2 | import ( |
3 | "io/ioutil" | |
3 | 4 | stdlog "log" |
4 | "os" | |
5 | 5 | "testing" |
6 | "time" | |
6 | 7 | |
7 | 8 | "github.com/afex/hystrix-go/hystrix" |
8 | 9 | |
9 | 10 | "github.com/go-kit/kit/circuitbreaker" |
10 | kitlog "github.com/go-kit/kit/log" | |
11 | 11 | ) |
12 | 12 | |
13 | 13 | func TestHystrix(t *testing.T) { |
14 | logger := kitlog.NewLogfmtLogger(os.Stderr) | |
15 | stdlog.SetOutput(kitlog.NewStdlibAdapter(logger)) | |
14 | stdlog.SetOutput(ioutil.Discard) | |
16 | 15 | |
17 | 16 | const ( |
18 | 17 | commandName = "my-endpoint" |
30 | 29 | shouldPass = func(n int) bool { return (float64(n) / float64(primeWith+n)) <= (float64(errorPercent-1) / 100.0) } |
31 | 30 | openCircuitError = hystrix.ErrCircuitOpen.Error() |
32 | 31 | ) |
33 | testFailingEndpoint(t, breaker, primeWith, shouldPass, openCircuitError) | |
32 | ||
33 | // hystrix-go uses buffered channels to receive reports on request success/failure, | |
34 | // and so is basically impossible to test deterministically. We have to make sure | |
35 | // the report buffer is emptied, by injecting a sleep between each invocation. | |
36 | requestDelay := 5 * time.Millisecond | |
37 | ||
38 | testFailingEndpoint(t, breaker, primeWith, shouldPass, requestDelay, openCircuitError) | |
34 | 39 | } |
12 | 12 | "github.com/go-kit/kit/endpoint" |
13 | 13 | ) |
14 | 14 | |
15 | func testFailingEndpoint(t *testing.T, breaker endpoint.Middleware, primeWith int, shouldPass func(int) bool, openCircuitError string) { | |
15 | func testFailingEndpoint( | |
16 | t *testing.T, | |
17 | breaker endpoint.Middleware, | |
18 | primeWith int, | |
19 | shouldPass func(int) bool, | |
20 | requestDelay time.Duration, | |
21 | openCircuitError string, | |
22 | ) { | |
16 | 23 | _, file, line, _ := runtime.Caller(1) |
17 | 24 | caller := fmt.Sprintf("%s:%d", filepath.Base(file), line) |
18 | 25 | |
27 | 34 | if _, err := e(context.Background(), struct{}{}); err != nil { |
28 | 35 | t.Fatalf("%s: during priming, got error: %v", caller, err) |
29 | 36 | } |
37 | time.Sleep(requestDelay) | |
30 | 38 | } |
31 | 39 | |
32 | 40 | // Switch the endpoint to start throwing errors. |
38 | 46 | if _, err := e(context.Background(), struct{}{}); err != m.err { |
39 | 47 | t.Fatalf("%s: want %v, have %v", caller, m.err, err) |
40 | 48 | } |
49 | time.Sleep(requestDelay) | |
41 | 50 | } |
42 | 51 | thru := m.thru |
43 | ||
44 | // Adding the sleep due to https://github.com/afex/hystrix-go/issues/41 | |
45 | // Increasing the sleep due to https://github.com/go-kit/kit/issues/169 | |
46 | // And increasing again for the same reason. | |
47 | time.Sleep(50 * time.Millisecond) | |
48 | 52 | |
49 | 53 | // But the rest should be blocked by an open circuit. |
50 | 54 | for i := 0; i < 10; i++ { |
51 | 55 | if _, err := e(context.Background(), struct{}{}); err.Error() != openCircuitError { |
52 | 56 | t.Fatalf("%s: want %q, have %q", caller, openCircuitError, err.Error()) |
53 | 57 | } |
58 | time.Sleep(requestDelay) | |
54 | 59 | } |
55 | 60 | |
56 | 61 | // Make sure none of those got through. |