diff --git a/README.md b/README.md index af67575..11ea1ed 100644 --- a/README.md +++ b/README.md @@ -142,6 +142,41 @@ fmt.Println("This is a hystrix configured circuit", c.Name()) // Output: This is a hystrix configured circuit hystrix-circuit ``` +## Adaptive Hystrix (`closers/hystrix-adaptive`) + +The [hystrix](https://pkg.go.dev/github.com/cep21/circuit/v4/closers/hystrix) opener trips the breaker when enough requests fail. That is what you want when the dependency is actually unhealthy—but when *everything* is simply a bit slow, many calls may time out and look like total failure even though nothing is uniquely broken. Package **`hystrixadaptive`** (`import "github.com/cep21/circuit/v4/closers/hystrix-adaptive"`) layers on top of the usual Hystrix opener: it can be more patient in that situation (mostly timeouts, not hard errors), give a little slack while the outage pattern looks like blanket slowness, and tighten again when fast successes return. You tune how much slack is allowed and how quickly it grows or shrinks. It only affects **when** the breaker opens; **per-call** deadlines are still whatever you set on **`Execution.Timeout`**. + +```go +configuration := hystrixadaptive.Factory{ + Factory: hystrix.Factory{ + ConfigureCloser: hystrix.ConfigureCloser{ + // Same half-open / sleep window behavior as plain Hystrix + }, + }, + ConfigureAdaptive: hystrixadaptive.ConfigureAdaptive{ + ConfigureOpener: hystrix.ConfigureOpener{ + RequestVolumeThreshold: 10, + }, + // Expected healthy latency; "fast" successes decay headroom below this. + BaselineLatency: 100 * time.Millisecond, + // Cap on accumulated headroom (e.g. allow up to baseline + 200ms conceptually). + MaxExtraLatency: 200 * time.Millisecond, + // Added on each timeout and on successes slower than baseline+headroom. + IncreaseExtra: 10 * time.Millisecond, + // Subtracted when a success finishes faster than BaselineLatency. + DecreaseExtra: 10 * time.Millisecond, + // While headroom > 0, defer opening if timeouts/(timeouts+failures) is at least this ratio. + MinTimeoutRatioToDefer: 0.85, + }, +} +h := circuit.Manager{ + DefaultCircuitProperties: []circuit.CommandPropertiesConstructor{configuration.Configure}, +} +c := h.MustCreateCircuit("adaptive-hystrix") +fmt.Println("This circuit uses the adaptive Hystrix opener", c.Name()) +// Output: This circuit uses the adaptive Hystrix opener adaptive-hystrix +``` + ## [Enable dashboard metrics](https://godoc.org/github.com/cep21/circuit/metriceventstream#example-MetricEventStream) Dashboard metrics can be enabled with the MetricEventStream object. This example creates an event stream handler, diff --git a/closers/hystrix-adaptive/config.go b/closers/hystrix-adaptive/config.go new file mode 100644 index 0000000..b71c355 --- /dev/null +++ b/closers/hystrix-adaptive/config.go @@ -0,0 +1,80 @@ +package hystrixadaptive + +import ( + "time" + + "github.com/cep21/circuit/v4" + "github.com/cep21/circuit/v4/closers/hystrix" +) + +// ConfigureAdaptive holds adaptive policy and embeds hystrix.ConfigureOpener; it does not set circuit execution timeouts +type ConfigureAdaptive struct { + hystrix.ConfigureOpener + + // BaselineLatency is the nominal fast path; successes below it decrease extra + BaselineLatency time.Duration + // MaxExtraLatency caps extra; slow-success threshold is roughly baseline+extra; at the cap, timeout deferral ends if inner ShouldOpen + MaxExtraLatency time.Duration + // IncreaseExtra added to extra on each timeout and on each success slower than baseline+current extra + IncreaseExtra time.Duration + // DecreaseExtra subtracted from extra when a success finishes faster than BaselineLatency + DecreaseExtra time.Duration + // MinTimeoutRatioToDefer is the rolling timeouts/(timeouts+failures) above which ShouldOpen may defer while 0 < extra < MaxExtraLatency + MinTimeoutRatioToDefer float64 +} + +// Merge copies missing fields from other +func (c *ConfigureAdaptive) Merge(other ConfigureAdaptive) { + c.ConfigureOpener.Merge(other.ConfigureOpener) + if c.BaselineLatency == 0 { + c.BaselineLatency = other.BaselineLatency + } + if c.MaxExtraLatency == 0 { + c.MaxExtraLatency = other.MaxExtraLatency + } + if c.IncreaseExtra == 0 { + c.IncreaseExtra = other.IncreaseExtra + } + if c.DecreaseExtra == 0 { + c.DecreaseExtra = other.DecreaseExtra + } + if c.MinTimeoutRatioToDefer == 0 { + c.MinTimeoutRatioToDefer = other.MinTimeoutRatioToDefer + } +} + +// defaultConfigureAdaptive is the default adaptive configuration +var defaultConfigureAdaptive = ConfigureAdaptive{ + ConfigureOpener: hystrix.ConfigureOpener{ + RequestVolumeThreshold: 20, + ErrorThresholdPercentage: 50, + Now: time.Now, + NumBuckets: 10, + RollingDuration: 10 * time.Second, + }, + BaselineLatency: 100 * time.Millisecond, + MaxExtraLatency: 200 * time.Millisecond, + IncreaseExtra: 10 * time.Millisecond, + DecreaseExtra: 10 * time.Millisecond, + MinTimeoutRatioToDefer: 0.85, +} + +// Factory merges hystrix.Factory with adaptive configuration +type Factory struct { + hystrix.Factory + + ConfigureAdaptive ConfigureAdaptive + CreateConfigureAdaptive []func(circuitName string) ConfigureAdaptive +} + +// Configure returns circuit.Config with adaptive ClosedToOpen from this factory +func (f *Factory) Configure(circuitName string) circuit.Config { + cfg := f.Factory.Configure(circuitName) + adaptiveCfg := ConfigureAdaptive{} + for i := len(f.CreateConfigureAdaptive) - 1; i >= 0; i-- { + adaptiveCfg.Merge(f.CreateConfigureAdaptive[i](circuitName)) + } + adaptiveCfg.Merge(f.ConfigureAdaptive) + cfg.General.ClosedToOpenFactory = OpenerFactory(adaptiveCfg) + return cfg +} diff --git a/closers/hystrix-adaptive/doc.go b/closers/hystrix-adaptive/doc.go new file mode 100644 index 0000000..2482599 --- /dev/null +++ b/closers/hystrix-adaptive/doc.go @@ -0,0 +1,9 @@ +/* +Package hystrixadaptive implements ClosedToOpen by wrapping the Hystrix opener and +deferring trips when failures are mostly timeouts while headroom is below its cap + +Additive headroom "extra" sits on top of BaselineLatency; timeouts and slow successes +raise it (capped by MaxExtraLatency); successes faster than BaselineLatency lower it; +set per-request deadlines on circuit.Execution, not in this package +*/ +package hystrixadaptive diff --git a/closers/hystrix-adaptive/example_test.go b/closers/hystrix-adaptive/example_test.go new file mode 100644 index 0000000..88b2b66 --- /dev/null +++ b/closers/hystrix-adaptive/example_test.go @@ -0,0 +1,67 @@ +package hystrixadaptive_test + +import ( + "fmt" + "time" + + "github.com/cep21/circuit/v4" + "github.com/cep21/circuit/v4/closers/hystrix" + hystrixadaptive "github.com/cep21/circuit/v4/closers/hystrix-adaptive" +) + +// ExampleFactory wires circuit.Manager with adaptive ClosedToOpen on top of hystrix.Factory +func ExampleFactory() { + configuration := hystrixadaptive.Factory{ + Factory: hystrix.Factory{ + ConfigureOpener: hystrix.ConfigureOpener{ + RequestVolumeThreshold: 10, + }, + ConfigureCloser: hystrix.ConfigureCloser{}, + }, + ConfigureAdaptive: hystrixadaptive.ConfigureAdaptive{ + BaselineLatency: 100 * time.Millisecond, + MaxExtraLatency: 200 * time.Millisecond, + IncreaseExtra: 10 * time.Millisecond, + DecreaseExtra: 10 * time.Millisecond, + MinTimeoutRatioToDefer: 0.85, + }, + } + h := circuit.Manager{ + DefaultCircuitProperties: []circuit.CommandPropertiesConstructor{configuration.Configure}, + } + c := h.MustCreateCircuit("adaptive-hystrix") + fmt.Println("circuit:", c.Name()) + // Output: + // circuit: adaptive-hystrix +} + +// ExampleOpenerFactory builds a [circuit.Config] with OpenerFactory and a Hystrix closer +func ExampleOpenerFactory() { + cfg := circuit.Config{ + General: circuit.GeneralConfig{ + ClosedToOpenFactory: hystrixadaptive.OpenerFactory(hystrixadaptive.ConfigureAdaptive{ + ConfigureOpener: hystrix.ConfigureOpener{ + RequestVolumeThreshold: 10, + }, + }), + OpenToClosedFactory: hystrix.CloserFactory(hystrix.ConfigureCloser{}), + }, + } + c := circuit.NewCircuitFromConfig("custom-opener", cfg) + fmt.Println("circuit:", c.Name()) + // Output: + // circuit: custom-opener +} + +// ExampleOpener_SetConfigThreadSafe updates adaptive fields via SetConfigThreadSafe; use NewOpener for a concrete *Opener +func ExampleOpener_SetConfigThreadSafe() { + ao := hystrixadaptive.NewOpener(hystrixadaptive.ConfigureAdaptive{}) + fmt.Println("default baseline:", ao.Config().BaselineLatency) + ao.SetConfigThreadSafe(hystrixadaptive.ConfigureAdaptive{ + BaselineLatency: 50 * time.Millisecond, + }) + fmt.Println("new baseline:", ao.Config().BaselineLatency) + // Output: + // default baseline: 100ms + // new baseline: 50ms +} diff --git a/closers/hystrix-adaptive/opener.go b/closers/hystrix-adaptive/opener.go new file mode 100644 index 0000000..bf4a613 --- /dev/null +++ b/closers/hystrix-adaptive/opener.go @@ -0,0 +1,235 @@ +package hystrixadaptive + +import ( + "context" + "encoding/json" + "sync" + "time" + + "github.com/cep21/circuit/v4" + "github.com/cep21/circuit/v4/closers/hystrix" + "github.com/cep21/circuit/v4/faststats" +) + +// Opener wraps hystrix.Opener and may defer ShouldOpen when failures are timeout-heavy and extra is under the cap +type Opener struct { + Opener *hystrix.Opener + + mu sync.Mutex + config ConfigureAdaptive + + // extra is added to BaselineLatency for slow-success detection and ShouldOpen + extra time.Duration + + timeoutCount faststats.RollingCounter + failureCount faststats.RollingCounter +} + +var ( + _ circuit.ClosedToOpen = (*Opener)(nil) + _ json.Marshaler = (*Opener)(nil) +) + +// NewOpener merges defaults into config and returns a new Opener (same as OpenerFactory(config)()) +func NewOpener(config ConfigureAdaptive) *Opener { + cfg := config + cfg.Merge(defaultConfigureAdaptive) + inner := hystrix.OpenerFactory(cfg.ConfigureOpener)().(*hystrix.Opener) + a := &Opener{Opener: inner} + a.setConfigNotThreadSafeLocked(cfg) + return a +} + +// OpenerFactory wraps hystrix.OpenerFactory with adaptive behavior +func OpenerFactory(config ConfigureAdaptive) func() circuit.ClosedToOpen { + return func() circuit.ClosedToOpen { + return NewOpener(config) + } +} + +// ShouldOpen defers only if inner wants open, extra is in (0, MaxExtraLatency), and timeout ratio is high enough +func (a *Opener) ShouldOpen(ctx context.Context, now time.Time) bool { + if !a.Opener.ShouldOpen(ctx, now) { + return false + } + a.mu.Lock() + defer a.mu.Unlock() + + if a.extra <= 0 { + return true + } + if a.config.MaxExtraLatency > 0 && a.extra >= a.config.MaxExtraLatency { + return true + } + + t := a.timeoutCount.RollingSumAt(now) + f := a.failureCount.RollingSumAt(now) + + if t+f == 0 { + return true + } + + ratio := float64(t) / float64(t+f) + if ratio >= a.config.MinTimeoutRatioToDefer { + return false + } + + return true +} + +// Prevent forwards to the inner opener +func (a *Opener) Prevent(ctx context.Context, now time.Time) bool { + return a.Opener.Prevent(ctx, now) +} + +// Closed resets adaptive state and forwards to the inner opener +func (a *Opener) Closed(ctx context.Context, now time.Time) { + a.Opener.Closed(ctx, now) + a.resetAdaptive(now) +} + +// Opened resets adaptive state and forwards to the inner opener +func (a *Opener) Opened(ctx context.Context, now time.Time) { + a.Opener.Opened(ctx, now) + a.resetAdaptive(now) +} + +// Success adjusts extra and forwards to the inner opener +func (a *Opener) Success(ctx context.Context, now time.Time, d time.Duration) { + a.Opener.Success(ctx, now, d) + a.mu.Lock() + defer a.mu.Unlock() + a.adjustExtraOnSuccessLocked(d) +} + +// ErrBadRequest forwards to the inner opener +func (a *Opener) ErrBadRequest(ctx context.Context, now time.Time, d time.Duration) { + a.Opener.ErrBadRequest(ctx, now, d) +} + +// ErrInterrupt forwards to the inner opener +func (a *Opener) ErrInterrupt(ctx context.Context, now time.Time, d time.Duration) { + a.Opener.ErrInterrupt(ctx, now, d) +} + +// ErrFailure increments adaptive failure tally and forwards to the inner opener +func (a *Opener) ErrFailure(ctx context.Context, now time.Time, d time.Duration) { + a.Opener.ErrFailure(ctx, now, d) + a.mu.Lock() + defer a.mu.Unlock() + a.failureCount.Inc(now) +} + +// ErrTimeout increments adaptive timeout tally, bumps extra, and forwards to the inner opener +func (a *Opener) ErrTimeout(ctx context.Context, now time.Time, d time.Duration) { + a.Opener.ErrTimeout(ctx, now, d) + a.mu.Lock() + defer a.mu.Unlock() + a.timeoutCount.Inc(now) + a.bumpExtraLocked() +} + +// ErrConcurrencyLimitReject forwards to the inner opener +func (a *Opener) ErrConcurrencyLimitReject(ctx context.Context, now time.Time) { + a.Opener.ErrConcurrencyLimitReject(ctx, now) +} + +// ErrShortCircuit forwards to the inner opener +func (a *Opener) ErrShortCircuit(ctx context.Context, now time.Time) { + a.Opener.ErrShortCircuit(ctx, now) +} + +func (a *Opener) adjustExtraOnSuccessLocked(d time.Duration) { + base := a.config.BaselineLatency + maxE := a.config.MaxExtraLatency + effectiveSlow := base + a.extra + switch { + case d > effectiveSlow: + a.extra += a.config.IncreaseExtra + if a.extra > maxE { + a.extra = maxE + } + case d < base: + a.extra -= a.config.DecreaseExtra + if a.extra < 0 { + a.extra = 0 + } + } +} + +func (a *Opener) bumpExtraLocked() { + maxE := a.config.MaxExtraLatency + a.extra += a.config.IncreaseExtra + if a.extra > maxE { + a.extra = maxE + } +} + +func (a *Opener) resetAdaptive(now time.Time) { + a.mu.Lock() + defer a.mu.Unlock() + a.extra = 0 + a.timeoutCount.Reset(now) + a.failureCount.Reset(now) +} + +// SetConfigThreadSafe updates adaptive and Hystrix fields without rebuilding adaptive rolling counters +func (a *Opener) SetConfigThreadSafe(props ConfigureAdaptive) { + props.Merge(defaultConfigureAdaptive) + a.mu.Lock() + a.config = props + a.mu.Unlock() + a.Opener.SetConfigThreadSafe(props.ConfigureOpener) +} + +// SetConfigNotThreadSafe rebuilds rolling windows and resets extra; prefer when rolling parameters change +func (a *Opener) SetConfigNotThreadSafe(props ConfigureAdaptive) { + a.setConfigNotThreadSafeLocked(props) +} + +func (a *Opener) setConfigNotThreadSafeLocked(props ConfigureAdaptive) { + props.Merge(defaultConfigureAdaptive) + a.mu.Lock() + a.config = props + a.extra = 0 + a.mu.Unlock() + a.Opener.SetConfigNotThreadSafe(props.ConfigureOpener) + ho := props.ConfigureOpener + nowFn := ho.Now + if nowFn == nil { + nowFn = time.Now + } + t := nowFn() + rollingCounterBucketWidth := time.Duration(ho.RollingDuration.Nanoseconds() / int64(ho.NumBuckets)) + a.mu.Lock() + a.timeoutCount = faststats.NewRollingCounter(rollingCounterBucketWidth, ho.NumBuckets, t) + a.failureCount = faststats.NewRollingCounter(rollingCounterBucketWidth, ho.NumBuckets, t) + a.mu.Unlock() +} + +// Config returns the merged ConfigureAdaptive (including embedded hystrix config) +func (a *Opener) Config() ConfigureAdaptive { + a.mu.Lock() + defer a.mu.Unlock() + return a.config +} + +// ExtraLatency returns current extra headroom above BaselineLatency +func (a *Opener) ExtraLatency() time.Duration { + a.mu.Lock() + defer a.mu.Unlock() + return a.extra +} + +// MarshalJSON is for debugging +func (a *Opener) MarshalJSON() ([]byte, error) { + a.mu.Lock() + defer a.mu.Unlock() + return json.Marshal(map[string]interface{}{ + "hystrix": a.Opener, + "config": a.config, + "extra_latency_ns": int64(a.extra), + "timeouts": &a.timeoutCount, + "failures": &a.failureCount, + }) +} diff --git a/closers/hystrix-adaptive/opener_test.go b/closers/hystrix-adaptive/opener_test.go new file mode 100644 index 0000000..e519dc2 --- /dev/null +++ b/closers/hystrix-adaptive/opener_test.go @@ -0,0 +1,220 @@ +package hystrixadaptive + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/cep21/circuit/v4" + "github.com/cep21/circuit/v4/closers/hystrix" +) + +func TestOpener_TimeoutHeavyDefersOpen(t *testing.T) { + ctx := context.Background() + o := OpenerFactory(ConfigureAdaptive{ + ConfigureOpener: hystrix.ConfigureOpener{ + RequestVolumeThreshold: 3, + ErrorThresholdPercentage: 50, + NumBuckets: 10, + RollingDuration: 10 * time.Second, + }, + MinTimeoutRatioToDefer: 0.85, + })().(*Opener) + // Timestamps must be >= rolling window StartTime (set when the opener is constructed) + now := time.Now() + + if o.ShouldOpen(ctx, now) { + t.Fatal("should not open with no traffic") + } + for i := 0; i < 3; i++ { + o.ErrTimeout(ctx, now, 100*time.Millisecond) + } + if o.ExtraLatency() <= 0 { + t.Fatal("expected non-zero extra headroom after timeouts") + } + if !o.Opener.ShouldOpen(ctx, now) { + t.Fatal("inner hystrix opener should want to open") + } + if o.ShouldOpen(ctx, now) { + t.Fatal("adaptive layer should defer open when failures are mostly timeouts with headroom") + } +} + +func TestOpener_MaxExtraLatencySaturatedAllowsOpen(t *testing.T) { + ctx := context.Background() + o := OpenerFactory(ConfigureAdaptive{ + ConfigureOpener: hystrix.ConfigureOpener{ + RequestVolumeThreshold: 3, + ErrorThresholdPercentage: 50, + NumBuckets: 10, + RollingDuration: 10 * time.Second, + }, + MinTimeoutRatioToDefer: 0.85, + MaxExtraLatency: 30 * time.Millisecond, + IncreaseExtra: 10 * time.Millisecond, + })().(*Opener) + now := time.Now() + for i := 0; i < 3; i++ { + o.ErrTimeout(ctx, now, time.Millisecond) + } + if got := o.ExtraLatency(); got != 30*time.Millisecond { + t.Fatalf("extra headroom: got %v want 30ms", got) + } + if !o.Opener.ShouldOpen(ctx, now) { + t.Fatal("inner opener should want to open") + } + if !o.ShouldOpen(ctx, now) { + t.Fatal("expected open when extra is saturated at MaxExtraLatency (timeout defer ends)") + } +} + +func TestOpener_FailuresStillOpen(t *testing.T) { + ctx := context.Background() + o := OpenerFactory(ConfigureAdaptive{ + ConfigureOpener: hystrix.ConfigureOpener{ + RequestVolumeThreshold: 3, + ErrorThresholdPercentage: 50, + NumBuckets: 10, + RollingDuration: 10 * time.Second, + }, + })().(*Opener) + now := time.Now() + + for i := 0; i < 3; i++ { + o.ErrFailure(ctx, now, time.Millisecond) + } + if !o.ShouldOpen(ctx, now) { + t.Fatal("expected open when rolling window is failure-dominated") + } +} + +func TestOpener_MarshalJSON(t *testing.T) { + o := OpenerFactory(ConfigureAdaptive{})().(*Opener) + ctx := context.Background() + now := time.Now() + o.ErrTimeout(ctx, now, time.Second) + b, err := o.MarshalJSON() + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(b), "timeouts") { + t.Fatalf("expected timeouts in json: %s", b) + } +} + +func TestFactoryConfigure(t *testing.T) { + f := Factory{ + ConfigureAdaptive: ConfigureAdaptive{ + ConfigureOpener: hystrix.ConfigureOpener{ + RequestVolumeThreshold: 7, + }, + }, + } + cfg := f.Configure("x") + ao := cfg.General.ClosedToOpenFactory().(*Opener) + if ao.Config().ConfigureOpener.RequestVolumeThreshold != 7 { + t.Fatalf("got threshold %d", ao.Config().ConfigureOpener.RequestVolumeThreshold) + } +} + +func TestOpener_FastSuccessClearsExtraHeadroom(t *testing.T) { + ctx := context.Background() + o := OpenerFactory(ConfigureAdaptive{ + ConfigureOpener: hystrix.ConfigureOpener{ + RequestVolumeThreshold: 3, + ErrorThresholdPercentage: 50, + NumBuckets: 10, + RollingDuration: 10 * time.Second, + }, + BaselineLatency: 100 * time.Millisecond, + IncreaseExtra: 10 * time.Millisecond, + DecreaseExtra: 10 * time.Millisecond, + MaxExtraLatency: 200 * time.Millisecond, + })().(*Opener) + now := time.Now() + + o.ErrTimeout(ctx, now, 100*time.Millisecond) + if got := o.ExtraLatency(); got != 10*time.Millisecond { + t.Fatalf("after one timeout, extra = %v, want 10ms", got) + } + o.Success(ctx, now, 5*time.Millisecond) + if got := o.ExtraLatency(); got != 0 { + t.Fatalf("after fast success below baseline, extra = %v, want 0", got) + } +} + +func TestOpener_ClosedResetsAdaptiveState(t *testing.T) { + ctx := context.Background() + o := OpenerFactory(ConfigureAdaptive{ + ConfigureOpener: hystrix.ConfigureOpener{ + RequestVolumeThreshold: 3, + ErrorThresholdPercentage: 50, + NumBuckets: 10, + RollingDuration: 10 * time.Second, + }, + })().(*Opener) + now := time.Now() + o.ErrTimeout(ctx, now, time.Millisecond) + if o.ExtraLatency() <= 0 { + t.Fatal("expected extra after timeout") + } + o.Closed(ctx, now) + if o.ExtraLatency() != 0 { + t.Fatalf("Closed should reset extra, got %v", o.ExtraLatency()) + } +} + +// TestCircuit_AdaptiveVsPlainHystrix_OpenerBehavior compares Execute timeout bursts: plain Hystrix opens, adaptive defers +func TestCircuit_AdaptiveVsPlainHystrix_OpenerBehavior(t *testing.T) { + ctx := context.Background() + opener := hystrix.ConfigureOpener{ + RequestVolumeThreshold: 3, + ErrorThresholdPercentage: 50, + NumBuckets: 10, + RollingDuration: 10 * time.Second, + } + + runTimeoutBurst := func(factory func() circuit.ClosedToOpen) bool { + cfg := circuit.Config{ + General: circuit.GeneralConfig{ + ClosedToOpenFactory: factory, + OpenToClosedFactory: hystrix.CloserFactory(hystrix.ConfigureCloser{ + SleepWindow: time.Hour, + }), + }, + Execution: circuit.ExecutionConfig{ + Timeout: 5 * time.Millisecond, + }, + } + c := circuit.NewCircuitFromConfig("opener-behavior", cfg) + for i := 0; i < 3; i++ { + _ = c.Execute(ctx, func(ctx context.Context) error { + select { + case <-time.After(50 * time.Millisecond): + return nil + case <-ctx.Done(): + return ctx.Err() + } + }, nil) + } + return c.IsOpen() + } + + t.Run("plainHystrixOpens", func(t *testing.T) { + opened := runTimeoutBurst(hystrix.OpenerFactory(opener)) + if !opened { + t.Fatal("expected plain Hystrix opener to open the circuit after three timeouts with volume=3 and 100% errors") + } + }) + + t.Run("adaptiveStaysClosed", func(t *testing.T) { + opened := runTimeoutBurst(OpenerFactory(ConfigureAdaptive{ + ConfigureOpener: opener, + MinTimeoutRatioToDefer: 0.85, + })) + if opened { + t.Fatal("expected adaptive opener to defer opening while failures are timeout-heavy and headroom is non-zero") + } + }) +}