diff --git a/docs/ActiveActive.md b/docs/ActiveActive.md index 1217f35fa..e49bbf88a 100644 --- a/docs/ActiveActive.md +++ b/docs/ActiveActive.md @@ -217,7 +217,7 @@ var healthCheck = new HealthCheck var options = new MultiGroupOptions { - DefaultHealthCheck = healthCheck + HealthCheck = healthCheck }; ConnectionGroupMember[] members = [ @@ -239,7 +239,7 @@ await using var conn = await ConnectionMultiplexer.ConnectGroupAsync(members); // Equivalent to: var options = new MultiGroupOptions { - DefaultHealthCheck = HealthCheck.Default + HealthCheck = HealthCheck.Default }; await using var conn = await ConnectionMultiplexer.ConnectGroupAsync(members, options); ``` @@ -253,7 +253,7 @@ customHealthCheck.ProbeCount = 5; var options = new MultiGroupOptions { - DefaultHealthCheck = customHealthCheck + HealthCheck = customHealthCheck }; ``` @@ -434,7 +434,7 @@ var healthCheck = new HealthCheck var options = new MultiGroupOptions { - DefaultHealthCheck = healthCheck + HealthCheck = healthCheck }; ``` @@ -457,6 +457,133 @@ When a health check fails for a member: 5. **Set reasonable timeouts**: Ensure `ProbeTimeout` accounts for network latency to your Redis servers 6. **Consider replica behavior**: Write-based probes automatically skip replicas to avoid false negatives +## Circuit Breakers + +Where health checks *actively* probe each member on a timer, a **circuit breaker** works *passively*: it observes the outcome of the normal traffic already flowing over a connection, and tears that connection down as soon as it decides the connection has become unstable. The two are complementary: + +- A **health check** answers *"is this member reachable?"* by sending its own probes every `Interval`. +- A **circuit breaker** answers *"is the traffic I'm already sending actually succeeding?"* with no extra round-trips, reacting the instant real commands start failing. + +When a circuit breaker trips, it shuts the underlying physical connection down with `ConnectionFailureType.CircuitBreaker`. The connection then reconnects as usual, and — in an Active:Active group — the health check and member-selection logic route traffic to other members until the affected member recovers. A circuit breaker is evaluated **per connection**, so +its state is scoped to exactly the connection whose health it is measuring; a reconnect starts from a clean slate - but breaking the connection is sufficient for Active:Active to notice non-availability. + +### Configuring a Circuit Breaker for a Group + +Circuit breakers are configured globally for all members via `MultiGroupOptions`, alongside the health check. The setting flows into every member connection: + +```csharp +var options = new MultiGroupOptions +{ + CircuitBreaker = CircuitBreaker.Default +}; + +ConnectionGroupMember[] members = [ + new("us-east.redis.example.com:6379", name: "US East") { Weight = 10 }, + new("us-west.redis.example.com:6379", name: "US West") { Weight = 5 } +]; + +await using var conn = await ConnectionMultiplexer.ConnectGroupAsync(members, options); +``` + +If you don't specify one, `CircuitBreaker.Default` is used automatically: + +```csharp +// these are equivalent +await using var conn = await ConnectionMultiplexer.ConnectGroupAsync(members); + +var options = new MultiGroupOptions { CircuitBreaker = CircuitBreaker.Default }; +await using var conn = await ConnectionMultiplexer.ConnectGroupAsync(members, options); +``` + +### Tuning the Default Circuit Breaker + +The default circuit breaker uses a rolling time-window: it counts successes and failures over a short window and trips once the failure rate crosses a +threshold, provided enough failures have been seen to be statistically meaningful. Use `CircuitBreaker.Builder` to tune it: + +```csharp +var options = new MultiGroupOptions +{ + CircuitBreaker = new CircuitBreaker.Builder + { + FailureRateThreshold = 25, // trip above 25% failures + MinimumNumberOfFailures = 100, // ...but only after 100 failures in the window + MetricsWindowSize = TimeSpan.FromSeconds(5), // rolling window to measure over + TrackedExceptions = [typeof(RedisTimeoutException)], // which faults count as failures + } +}; +``` + +A `Builder` converts implicitly to a `CircuitBreaker`, so it can be assigned directly as above; calling `.Create()` explicitly is equivalent. + +| Property | Default | Description | +|----------|---------|-------------| +| `FailureRateThreshold` | 10 | Percentage of failures within the window that trips the breaker | +| `MinimumNumberOfFailures` | 1000 | Minimum tracked failures in the window before the breaker can trip (avoids acting on tiny samples) | +| `MetricsWindowSize` | 2 seconds | Rolling window over which successes and failures are counted | +| `TrackedExceptions` | `null` | Exception types that count as failures; when `null`, `RedisConnectionException` and `RedisTimeoutException` are tracked | + +`TrackedExceptions` controls which faults count against the breaker; anything not tracked is treated as a success for circuit-breaking purposes (for example, a `RedisServerException` for a bad command is not a *connection* problem). Passing an array containing `typeof(Exception)` tracks everything; passing an empty array tracks nothing. + +### Disabling the Circuit Breaker + +Use `CircuitBreaker.None` to opt out entirely: + +```csharp +var options = new MultiGroupOptions +{ + CircuitBreaker = CircuitBreaker.None +}; +``` + +### Custom Circuit Breakers (Advanced) + +> This is an advanced extension point; most applications should use `CircuitBreaker.Default` (optionally tuned via `CircuitBreaker.Builder`) or `CircuitBreaker.None`. + +You can implement your own policy by extending `CircuitBreaker` and its `Accumulator`. `CreateAccumulator()` is called once per underlying connection, so each accumulator holds the state for a single connection. +For every completed message the connection calls `ObserveResult`, passing a `CircuitBreakerContext` that exposes whether the operation succeeded (`Success`) and the associated `Fault` (if any). +Return `true` from `IsHealthy()` while the connection should be considered **healthy**, or `false` to **trip** the breaker and tear the connection down: + +```csharp +public sealed class ConsecutiveFailureBreaker(int limit) : CircuitBreaker +{ + public override Accumulator CreateAccumulator() => new Acc(limit); + + private sealed class Acc(int limit) : Accumulator + { + private int _consecutiveFailures; + + public override bool ObserveResult(in CircuitBreakerContext context) + { + if (context.Success) + { + _consecutiveFailures = 0; + } + else + { + _consecutiveFailures++; + } + + // if Evaluate is specified, we should also compute IsHealthy(), + // often this can reuse local state needed by ObserveResult + return context.Evaluate ? IsHealthy() : true; + } + + // healthy until we hit the configured run of consecutive failures + public override bool IsHealthy() => _consecutiveFailures < limit; + + // called if the connection wants to discard accumulated history + public override void Reset() => _consecutiveFailures = 0; + } +} + +var options = new MultiGroupOptions +{ + CircuitBreaker = new ConsecutiveFailureBreaker(limit: 5) +}; +``` + +Keep `ObserveResult` cheap and thread-safe: it runs on the hot path for every completed message, and may be called concurrently. + ## Manual Failover In some scenarios, you may need to manually control which member is actively serving traffic, overriding the automatic selection based on weight and latency. The `TryFailoverTo` method allows you to explicitly switch to a specific member or restore automatic selection. diff --git a/src/StackExchange.Redis/Availability/CircuitBreaker.cs b/src/StackExchange.Redis/Availability/CircuitBreaker.cs new file mode 100644 index 000000000..feaa65399 --- /dev/null +++ b/src/StackExchange.Redis/Availability/CircuitBreaker.cs @@ -0,0 +1,451 @@ +using System; +#if !NET8_0_OR_GREATER +using System.Diagnostics; +#endif +using System.Diagnostics.CodeAnalysis; +using System.Runtime.CompilerServices; +using System.Threading; +using RESPite; + +namespace StackExchange.Redis.Availability; + +/// +/// Reports connection health by responding to observed success and failure conditions of processed messages. +/// +[Experimental(Experiments.ActiveActive, UrlFormat = Experiments.UrlFormat)] +public abstract class CircuitBreaker +{ + /// + /// Default circuit-breaker logic. + /// + public static CircuitBreaker Default => Builder.DefaultInstance; + + /// + /// No circuit-breaker logic is applied. + /// + public static CircuitBreaker None => NulCircuitBreaker.Instance; + + /// + /// Allows configuration of the default implementation. + /// + public class Builder + { + private const double DefaultFailureRateThreshold = 10; + private const int DefaultMinimumNumberOfFailures = 1000; + private static readonly TimeSpan DefaultMetricsWindowSize = TimeSpan.FromSeconds(2); + + internal static CircuitBreaker DefaultInstance = new DefaultCircuitBreaker( + DefaultFailureRateThreshold, + DefaultMinimumNumberOfFailures, + DefaultMetricsWindowSize, +#if NET8_0_OR_GREATER + timeProvider: null, +#endif + trackedExceptions: null); + + /// + /// Percentage of failures to trigger circuit breaker. + /// + /// Failures are only included if they are of tracked exception types. + public double FailureRateThreshold { get; set; } = DefaultFailureRateThreshold; + + /// + /// Minimum failures before circuit breaker can open. + /// + public int MinimumNumberOfFailures { get; set; } = DefaultMinimumNumberOfFailures; + + /// + /// Time window for collecting metrics. + /// + public TimeSpan MetricsWindowSize { get; set; } = DefaultMetricsWindowSize; + +#if NET8_0_OR_GREATER + /// + /// Time source used to drive the metrics window; when null, the system clock is used. + /// Intended for testing, to make the time-windowed logic deterministic. + /// + internal TimeProvider? TimeProvider { get; set; } +#endif + + /// + /// Create a new circuit-breaker instance. + /// + public CircuitBreaker Create() + { + if ((FailureRateThreshold is DefaultFailureRateThreshold + & MinimumNumberOfFailures is DefaultMinimumNumberOfFailures +#if NET8_0_OR_GREATER + & TimeProvider is null +#endif + & TrackedExceptions is null) + && MetricsWindowSize == DefaultMetricsWindowSize) + return DefaultInstance; + + return new DefaultCircuitBreaker( + FailureRateThreshold, + MinimumNumberOfFailures, + MetricsWindowSize, +#if NET8_0_OR_GREATER + TimeProvider, +#endif + TrackedExceptions); + } + + /// + /// Create a new circuit-breaker instance. + /// + public static implicit operator CircuitBreaker(Builder builder) => builder.Create(); + + /// + /// Exceptions that count as failures. When null, + /// and are assumed. + /// + public Type[]? TrackedExceptions { get; set; } + } + + /// + /// Create an object to collate observations for a connection. + /// + public abstract Accumulator CreateAccumulator(); + + /// + /// Collates observations for a connection. + /// + public abstract class Accumulator + { + /// + /// Record a message outcome, and indicate whether the connection is considered healthy. + /// + /// True if the connection is still considered healthy. + /// struct arg here is in case we want to add more things later + public abstract bool ObserveResult(in CircuitBreakerContext context); + + internal bool ObserveResult(Exception? fault) + { + // only evaluate state upon failure; don't pay that overhead for success, just increment the counters + bool evaluate = fault is not null; + var ctx = new CircuitBreakerContext(fault, evaluate: evaluate); + bool healthy = ObserveResult(in ctx); + + // when we didn't ask for an evaluation, the returned verdict is meaningless: a custom + // implementation might return default(false) without having actually computed anything. + // never treat a non-evaluating observation as unhealthy - only a genuine evaluation may trip. + return !evaluate || healthy; + } + + /// + /// Indicate whether the connection is currently considered healthy, without recording an observation. + /// + /// True if the connection is considered healthy. + public abstract bool IsHealthy(); + + /// + /// Discard all accumulated observations, returning to a clean state. + /// + public abstract void Reset(); + } + + /// + /// Provides information about a circuit-breaker test. + /// + public readonly struct CircuitBreakerContext(Exception? fault, bool evaluate = true) + { + /// + /// Was the operation a success. + /// + [MemberNotNullWhen(false, nameof(Fault))] + public bool Success => fault is null; + + /// + /// The fault associated with the operation. + /// + public Exception? Fault => fault; + + internal bool Evaluate => evaluate; + } + + private enum ExceptionStrategy + { + Default, + Any, + None, + CustomOpen, + CustomSealed, + } + + private sealed class NulCircuitBreaker : CircuitBreaker + { + public static readonly NulCircuitBreaker Instance = new(); + private NulCircuitBreaker() { } + public override Accumulator CreateAccumulator() => NulAccumulator.AccumulatorInstance; + + private sealed class NulAccumulator : Accumulator + { + public static readonly NulAccumulator AccumulatorInstance = new(); + private NulAccumulator() { } + public override bool ObserveResult(in CircuitBreakerContext context) => true; + public override bool IsHealthy() => true; + public override void Reset() { } + } + } + private sealed class DefaultCircuitBreaker : CircuitBreaker + { + private readonly ExceptionStrategy _trackingStrategy; + private readonly Type[] _trackedExceptions; + private readonly double _failureRateThreshold; + private readonly int _minimumNumberOfFailures; + + // the metrics window is divided into a fixed number of equal time-slices ("buckets"); + // this lets us keep a rolling count cheaply, evicting whole buckets as they age out, + // rather than storing (and pruning) a timestamp per event + private const int BucketCount = 10; + private readonly long _bucketTicks; // width of one bucket, in high-resolution ticks + +#if NET8_0_OR_GREATER + private readonly TimeProvider _time; +#endif + + // which time-slice ("epoch") are we in right now: the high-resolution timestamp (from + // TimeProvider where available, so tests can drive it; otherwise the Stopwatch clock) + // divided down to bucket width + private long GetEpoch() => +#if NET8_0_OR_GREATER + _time.GetTimestamp() +#else + Stopwatch.GetTimestamp() +#endif + / _bucketTicks; + + public DefaultCircuitBreaker( + double failureRateThreshold, + int minimumNumberOfFailures, + TimeSpan metricsWindowSize, +#if NET8_0_OR_GREATER + TimeProvider? timeProvider, +#endif + Type[]? trackedExceptions) + { + _trackedExceptions = CheckExceptions(trackedExceptions, out _trackingStrategy); + _failureRateThreshold = failureRateThreshold; + _minimumNumberOfFailures = minimumNumberOfFailures; + +#if NET8_0_OR_GREATER + _time = timeProvider ?? TimeProvider.System; + long frequency = _time.TimestampFrequency; +#else + long frequency = Stopwatch.Frequency; +#endif + long windowTicks = (long)(metricsWindowSize.TotalSeconds * frequency); + _bucketTicks = Math.Max(1, windowTicks / BucketCount); + } + + public override Accumulator CreateAccumulator() => new DefaultAccumulator(this); + + private sealed class DefaultAccumulator(DefaultCircuitBreaker breaker) : Accumulator + { + // ring of buckets; each holds the counts for a single time-slice, tagged with the + // slice ("epoch") it currently represents so we can tell live buckets from stale ones +#if NET8_0_OR_GREATER + // inline it directly into the accumulator + private BucketRing _buckets; // cannot be "readonly", else the indexer is "ref readonly" + [InlineArray(BucketCount)] + private struct BucketRing + { + // beware init rules; we're OK in this case because _buckets is in a field on a heap object, + // but if BucketRing was ever used as a local: [SkipLocalsInit] rules can apply. + private Bucket _element0; + } +#else + // fallback to a separate heap array + private readonly Bucket[] _buckets = new Bucket[BucketCount]; +#endif + + private struct Bucket + { + private long _epoch; + private volatile int _success, _failure; + + // note that Volatile guarantees atomicity (even on x86), so no torn values here + // see https://learn.microsoft.com/dotnet/api/system.threading.volatile + public long Epoch => Volatile.Read(ref _epoch); + + public int Success => _success; + public int Failure => _failure; + + public void Count(long epoch, bool success) + { + if (epoch != Epoch) + { + // epoch rollover; clear the counts first, to prevent anyone over-counting + // in their count loop (under-counting is fine); dropped counts are self-correcting + // if there's an actual problem, stale data misread as current: is not. + _success = _failure = 0; + Volatile.Write(ref _epoch, epoch); + + // if we want to get *super* accurate, we could use bit-packing here and use + // CAS over a 64-bit value, but we'd need to compromise on count upper bounds + // *and* we'd need to consider the max epoch problem - maybe 32 bits for epoch + // and 16 bits for each count, but... let's keep things simple and accept a + // few dropped counts instead, and luxuriate in unreasonably fat epochs and counts. + } + // ReSharper disable ByRefArgumentIsVolatileField + Interlocked.Increment(ref success ? ref _success : ref _failure); + // ReSharper restore ByRefArgumentIsVolatileField + } + + public void Reset() + { + // clear the counts first (as in Count), so a concurrent count-loop reader never + // attributes these stale counts to the epoch; then blank the epoch back to "unused" + _success = _failure = 0; + Volatile.Write(ref _epoch, 0); + } + } + + public override bool ObserveResult(in CircuitBreakerContext context) + { + // not-tracked failures (based on exception type) count as "success" for the purposes of circuit breaking + bool countAsSuccess = context.Success || !breaker.IsTracked(context.Fault); + + // which time-slice are we in, and where does it live in the ring? + long epoch = breaker.GetEpoch(); + int index = (int)(epoch % BucketCount); + + // note: to avoid concurrency problems, we're going lock-free here; *technically* this + // might mean we see race oddities during epoch rollovers, but in general that means we're + // miscounting by a tiny amount during intervals when there's enough load to get a race, + // in which case: we're probably fine; also, note that when tracking by endpoint, we're + // only getting results one at a time, so we shouldn't be over-stomping much *anyway* + Span buckets = _buckets; // *not* a payload copy; this is in-place over the data + ref Bucket bucket = ref buckets[index]; + bucket.Count(epoch, countAsSuccess); + + return !context.Evaluate || Evaluate(epoch); + } + + public override bool IsHealthy() => Evaluate(breaker.GetEpoch()); + + // evaluate health from the buckets still inside the window ending at the given epoch, + // without recording anything; shared by ObserveResult and IsHealthy + private bool Evaluate(long epoch) + { + // sum only the buckets still inside the window; anything older is ignored + // (and contributes nothing even if left un-recycled). empty/never-used buckets + // add zero, so no explicit "unused" sentinel is required + long oldest = epoch - BucketCount + 1; + int failures, total = failures = 0; + Span buckets = _buckets; // *not* a payload copy; this is in-place over the data + foreach (ref readonly Bucket b in buckets) + { + if (b.Epoch < oldest) continue; + + int failure; + failures += failure = b.Failure; // capture to avoid double-read (think epoch rollover) + total += b.Success + failure; + } + + // don't act until we've seen enough failures to be statistically meaningful + if (total is 0 | failures < breaker._minimumNumberOfFailures) + { + return true; + } + double failureRate = (failures * 100d) / total; + return failureRate < breaker._failureRateThreshold; + } + + public override void Reset() + { + Span buckets = _buckets; // in-place over the data, not a copy + foreach (ref Bucket b in buckets) + { + b.Reset(); + } + } + } + + private static Type[] CheckExceptions(Type[]? tracked, out ExceptionStrategy strategy) + { + if (tracked is null) + { + strategy = ExceptionStrategy.Default; + return []; + } + if (tracked.Length is 0) + { + strategy = ExceptionStrategy.None; + return []; + } + strategy = ExceptionStrategy.CustomOpen; + + static bool Contains(Type[] array, Type type) + { + for (int i = 0; i < array.Length; i++) + { + if (array[i] == type) return true; + } + + return false; + } + + // if we have Exception anywhere: we'll track everything + if (Contains(tracked, typeof(Exception))) + { + strategy = ExceptionStrategy.Any; + return []; + } + + if (tracked.Length is 2 + && Contains(tracked, typeof(RedisConnectionException)) + && Contains(tracked, typeof(RedisTimeoutException))) + { + strategy = ExceptionStrategy.Default; + return []; + } + + // finally, see if they're all sealed types + strategy = ExceptionStrategy.CustomSealed; + foreach (var exception in tracked) + { + if (!exception.IsSealed) + { + strategy = ExceptionStrategy.CustomOpen; + break; + } + } + return tracked; + } + + private bool IsTracked(Exception? fault) + { + if (fault is null) return false; + switch (_trackingStrategy) + { + case ExceptionStrategy.Any: + return true; + case ExceptionStrategy.Default: + return fault is RedisTimeoutException or RedisConnectionException; + case ExceptionStrategy.None: + return false; + } + + var span = _trackedExceptions.AsSpan(); + var actualType = fault.GetType(); + // check for exact matches + foreach (var testType in span) + { + if (ReferenceEquals(testType, actualType)) return true; + } + + if (_trackingStrategy is ExceptionStrategy.CustomOpen) + { + // we need to check for subclasses (more expensive) + foreach (var testType in span) + { + if (!testType.IsSealed && testType.IsAssignableFrom(actualType)) return true; + } + } + + return false; + } + } +} diff --git a/src/StackExchange.Redis/HealthCheck.ConnectedProbe.cs b/src/StackExchange.Redis/Availability/HealthCheck.ConnectedProbe.cs similarity index 93% rename from src/StackExchange.Redis/HealthCheck.ConnectedProbe.cs rename to src/StackExchange.Redis/Availability/HealthCheck.ConnectedProbe.cs index 0a669324b..3850aaa61 100644 --- a/src/StackExchange.Redis/HealthCheck.ConnectedProbe.cs +++ b/src/StackExchange.Redis/Availability/HealthCheck.ConnectedProbe.cs @@ -1,7 +1,6 @@ -using System.Net; using System.Threading.Tasks; -namespace StackExchange.Redis; +namespace StackExchange.Redis.Availability; public sealed partial class HealthCheck { diff --git a/src/StackExchange.Redis/HealthCheck.Execute.cs b/src/StackExchange.Redis/Availability/HealthCheck.Execute.cs similarity index 99% rename from src/StackExchange.Redis/HealthCheck.Execute.cs rename to src/StackExchange.Redis/Availability/HealthCheck.Execute.cs index 29070233b..968b4e81e 100644 --- a/src/StackExchange.Redis/HealthCheck.Execute.cs +++ b/src/StackExchange.Redis/Availability/HealthCheck.Execute.cs @@ -1,11 +1,9 @@ using System; -using System.Buffers; using System.Diagnostics; -using System.Net; using System.Threading; using System.Threading.Tasks; -namespace StackExchange.Redis; +namespace StackExchange.Redis.Availability; public sealed partial class HealthCheck { diff --git a/src/StackExchange.Redis/HealthCheck.HealthCheckProbe.cs b/src/StackExchange.Redis/Availability/HealthCheck.HealthCheckProbe.cs similarity index 98% rename from src/StackExchange.Redis/HealthCheck.HealthCheckProbe.cs rename to src/StackExchange.Redis/Availability/HealthCheck.HealthCheckProbe.cs index 717863150..6789bc891 100644 --- a/src/StackExchange.Redis/HealthCheck.HealthCheckProbe.cs +++ b/src/StackExchange.Redis/Availability/HealthCheck.HealthCheckProbe.cs @@ -1,10 +1,9 @@ using System.Diagnostics; using System.Diagnostics.CodeAnalysis; -using System.Net; using System.Threading.Tasks; using RESPite; -namespace StackExchange.Redis; +namespace StackExchange.Redis.Availability; public sealed partial class HealthCheck { diff --git a/src/StackExchange.Redis/HealthCheck.HealthCheckProbeContext.cs b/src/StackExchange.Redis/Availability/HealthCheck.HealthCheckProbeContext.cs similarity index 95% rename from src/StackExchange.Redis/HealthCheck.HealthCheckProbeContext.cs rename to src/StackExchange.Redis/Availability/HealthCheck.HealthCheckProbeContext.cs index a0e6a58a3..82062e830 100644 --- a/src/StackExchange.Redis/HealthCheck.HealthCheckProbeContext.cs +++ b/src/StackExchange.Redis/Availability/HealthCheck.HealthCheckProbeContext.cs @@ -1,9 +1,8 @@ using System; using System.Diagnostics.CodeAnalysis; -using System.Runtime.CompilerServices; using RESPite; -namespace StackExchange.Redis; +namespace StackExchange.Redis.Availability; public sealed partial class HealthCheck { diff --git a/src/StackExchange.Redis/HealthCheck.HealthCheckProbePolicy.cs b/src/StackExchange.Redis/Availability/HealthCheck.HealthCheckProbePolicy.cs similarity index 98% rename from src/StackExchange.Redis/HealthCheck.HealthCheckProbePolicy.cs rename to src/StackExchange.Redis/Availability/HealthCheck.HealthCheckProbePolicy.cs index c303bf8ed..5e465dae2 100644 --- a/src/StackExchange.Redis/HealthCheck.HealthCheckProbePolicy.cs +++ b/src/StackExchange.Redis/Availability/HealthCheck.HealthCheckProbePolicy.cs @@ -2,7 +2,7 @@ using System.Diagnostics.CodeAnalysis; using RESPite; -namespace StackExchange.Redis; +namespace StackExchange.Redis.Availability; public sealed partial class HealthCheck { diff --git a/src/StackExchange.Redis/HealthCheck.PingProbe.cs b/src/StackExchange.Redis/Availability/HealthCheck.PingProbe.cs similarity index 93% rename from src/StackExchange.Redis/HealthCheck.PingProbe.cs rename to src/StackExchange.Redis/Availability/HealthCheck.PingProbe.cs index 37257e52d..594a1bbeb 100644 --- a/src/StackExchange.Redis/HealthCheck.PingProbe.cs +++ b/src/StackExchange.Redis/Availability/HealthCheck.PingProbe.cs @@ -1,7 +1,6 @@ -using System.Net; using System.Threading.Tasks; -namespace StackExchange.Redis; +namespace StackExchange.Redis.Availability; public sealed partial class HealthCheck { diff --git a/src/StackExchange.Redis/HealthCheck.StringSetProbe.cs b/src/StackExchange.Redis/Availability/HealthCheck.StringSetProbe.cs similarity index 97% rename from src/StackExchange.Redis/HealthCheck.StringSetProbe.cs rename to src/StackExchange.Redis/Availability/HealthCheck.StringSetProbe.cs index fb12baf4f..1a6bd2d2d 100644 --- a/src/StackExchange.Redis/HealthCheck.StringSetProbe.cs +++ b/src/StackExchange.Redis/Availability/HealthCheck.StringSetProbe.cs @@ -2,7 +2,7 @@ using System.Buffers; using System.Threading.Tasks; -namespace StackExchange.Redis; +namespace StackExchange.Redis.Availability; public sealed partial class HealthCheck { diff --git a/src/StackExchange.Redis/HealthCheck.cs b/src/StackExchange.Redis/Availability/HealthCheck.cs similarity index 98% rename from src/StackExchange.Redis/HealthCheck.cs rename to src/StackExchange.Redis/Availability/HealthCheck.cs index 4cae83f6e..7b1352034 100644 --- a/src/StackExchange.Redis/HealthCheck.cs +++ b/src/StackExchange.Redis/Availability/HealthCheck.cs @@ -4,7 +4,7 @@ using System.Threading; using RESPite; -namespace StackExchange.Redis; +namespace StackExchange.Redis.Availability; /// /// Describes a health check to perform against instances. diff --git a/src/StackExchange.Redis/MultiGroupDatabase.Arrays.Async.cs b/src/StackExchange.Redis/Availability/MultiGroupDatabase.Arrays.Async.cs similarity index 99% rename from src/StackExchange.Redis/MultiGroupDatabase.Arrays.Async.cs rename to src/StackExchange.Redis/Availability/MultiGroupDatabase.Arrays.Async.cs index 01951451f..299159b43 100644 --- a/src/StackExchange.Redis/MultiGroupDatabase.Arrays.Async.cs +++ b/src/StackExchange.Redis/Availability/MultiGroupDatabase.Arrays.Async.cs @@ -1,6 +1,6 @@ using System.Threading.Tasks; -namespace StackExchange.Redis; +namespace StackExchange.Redis.Availability; internal sealed partial class MultiGroupDatabase { diff --git a/src/StackExchange.Redis/MultiGroupDatabase.Arrays.cs b/src/StackExchange.Redis/Availability/MultiGroupDatabase.Arrays.cs similarity index 99% rename from src/StackExchange.Redis/MultiGroupDatabase.Arrays.cs rename to src/StackExchange.Redis/Availability/MultiGroupDatabase.Arrays.cs index 6f5688ddc..45405e052 100644 --- a/src/StackExchange.Redis/MultiGroupDatabase.Arrays.cs +++ b/src/StackExchange.Redis/Availability/MultiGroupDatabase.Arrays.cs @@ -1,4 +1,4 @@ -namespace StackExchange.Redis; +namespace StackExchange.Redis.Availability; internal sealed partial class MultiGroupDatabase { diff --git a/src/StackExchange.Redis/MultiGroupDatabase.Async.cs b/src/StackExchange.Redis/Availability/MultiGroupDatabase.Async.cs similarity index 99% rename from src/StackExchange.Redis/MultiGroupDatabase.Async.cs rename to src/StackExchange.Redis/Availability/MultiGroupDatabase.Async.cs index b1e70bf34..1e8dbc05c 100644 --- a/src/StackExchange.Redis/MultiGroupDatabase.Async.cs +++ b/src/StackExchange.Redis/Availability/MultiGroupDatabase.Async.cs @@ -2,7 +2,7 @@ using System.Net; using System.Threading.Tasks; -namespace StackExchange.Redis; +namespace StackExchange.Redis.Availability; internal sealed partial class MultiGroupDatabase { diff --git a/src/StackExchange.Redis/MultiGroupDatabase.Geo.Async.cs b/src/StackExchange.Redis/Availability/MultiGroupDatabase.Geo.Async.cs similarity index 98% rename from src/StackExchange.Redis/MultiGroupDatabase.Geo.Async.cs rename to src/StackExchange.Redis/Availability/MultiGroupDatabase.Geo.Async.cs index 422ff59fc..32965a40e 100644 --- a/src/StackExchange.Redis/MultiGroupDatabase.Geo.Async.cs +++ b/src/StackExchange.Redis/Availability/MultiGroupDatabase.Geo.Async.cs @@ -1,6 +1,6 @@ using System.Threading.Tasks; -namespace StackExchange.Redis; +namespace StackExchange.Redis.Availability; internal sealed partial class MultiGroupDatabase { diff --git a/src/StackExchange.Redis/MultiGroupDatabase.Geo.cs b/src/StackExchange.Redis/Availability/MultiGroupDatabase.Geo.cs similarity index 98% rename from src/StackExchange.Redis/MultiGroupDatabase.Geo.cs rename to src/StackExchange.Redis/Availability/MultiGroupDatabase.Geo.cs index c74982c83..662c4de3f 100644 --- a/src/StackExchange.Redis/MultiGroupDatabase.Geo.cs +++ b/src/StackExchange.Redis/Availability/MultiGroupDatabase.Geo.cs @@ -1,4 +1,4 @@ -namespace StackExchange.Redis; +namespace StackExchange.Redis.Availability; internal sealed partial class MultiGroupDatabase { diff --git a/src/StackExchange.Redis/MultiGroupDatabase.Hashes.Async.cs b/src/StackExchange.Redis/Availability/MultiGroupDatabase.Hashes.Async.cs similarity index 99% rename from src/StackExchange.Redis/MultiGroupDatabase.Hashes.Async.cs rename to src/StackExchange.Redis/Availability/MultiGroupDatabase.Hashes.Async.cs index 15e8111e8..bba401aa8 100644 --- a/src/StackExchange.Redis/MultiGroupDatabase.Hashes.Async.cs +++ b/src/StackExchange.Redis/Availability/MultiGroupDatabase.Hashes.Async.cs @@ -1,7 +1,7 @@ using System; using System.Threading.Tasks; -namespace StackExchange.Redis; +namespace StackExchange.Redis.Availability; internal sealed partial class MultiGroupDatabase { diff --git a/src/StackExchange.Redis/MultiGroupDatabase.Hashes.cs b/src/StackExchange.Redis/Availability/MultiGroupDatabase.Hashes.cs similarity index 99% rename from src/StackExchange.Redis/MultiGroupDatabase.Hashes.cs rename to src/StackExchange.Redis/Availability/MultiGroupDatabase.Hashes.cs index 312a4dcb0..3adf40c7d 100644 --- a/src/StackExchange.Redis/MultiGroupDatabase.Hashes.cs +++ b/src/StackExchange.Redis/Availability/MultiGroupDatabase.Hashes.cs @@ -1,6 +1,6 @@ using System; -namespace StackExchange.Redis; +namespace StackExchange.Redis.Availability; internal sealed partial class MultiGroupDatabase { diff --git a/src/StackExchange.Redis/MultiGroupDatabase.HyperLogLog.Async.cs b/src/StackExchange.Redis/Availability/MultiGroupDatabase.HyperLogLog.Async.cs similarity index 96% rename from src/StackExchange.Redis/MultiGroupDatabase.HyperLogLog.Async.cs rename to src/StackExchange.Redis/Availability/MultiGroupDatabase.HyperLogLog.Async.cs index d57166885..933d0131e 100644 --- a/src/StackExchange.Redis/MultiGroupDatabase.HyperLogLog.Async.cs +++ b/src/StackExchange.Redis/Availability/MultiGroupDatabase.HyperLogLog.Async.cs @@ -1,6 +1,6 @@ using System.Threading.Tasks; -namespace StackExchange.Redis; +namespace StackExchange.Redis.Availability; internal sealed partial class MultiGroupDatabase { diff --git a/src/StackExchange.Redis/MultiGroupDatabase.HyperLogLog.cs b/src/StackExchange.Redis/Availability/MultiGroupDatabase.HyperLogLog.cs similarity index 96% rename from src/StackExchange.Redis/MultiGroupDatabase.HyperLogLog.cs rename to src/StackExchange.Redis/Availability/MultiGroupDatabase.HyperLogLog.cs index 3312b4fc0..b222deea1 100644 --- a/src/StackExchange.Redis/MultiGroupDatabase.HyperLogLog.cs +++ b/src/StackExchange.Redis/Availability/MultiGroupDatabase.HyperLogLog.cs @@ -1,4 +1,4 @@ -namespace StackExchange.Redis; +namespace StackExchange.Redis.Availability; internal sealed partial class MultiGroupDatabase { diff --git a/src/StackExchange.Redis/MultiGroupDatabase.Keys.Async.cs b/src/StackExchange.Redis/Availability/MultiGroupDatabase.Keys.Async.cs similarity index 98% rename from src/StackExchange.Redis/MultiGroupDatabase.Keys.Async.cs rename to src/StackExchange.Redis/Availability/MultiGroupDatabase.Keys.Async.cs index 76c7798be..a944d57f4 100644 --- a/src/StackExchange.Redis/MultiGroupDatabase.Keys.Async.cs +++ b/src/StackExchange.Redis/Availability/MultiGroupDatabase.Keys.Async.cs @@ -1,7 +1,7 @@ using System; using System.Threading.Tasks; -namespace StackExchange.Redis; +namespace StackExchange.Redis.Availability; internal sealed partial class MultiGroupDatabase { diff --git a/src/StackExchange.Redis/MultiGroupDatabase.Keys.cs b/src/StackExchange.Redis/Availability/MultiGroupDatabase.Keys.cs similarity index 98% rename from src/StackExchange.Redis/MultiGroupDatabase.Keys.cs rename to src/StackExchange.Redis/Availability/MultiGroupDatabase.Keys.cs index 2ee6bdce0..119536084 100644 --- a/src/StackExchange.Redis/MultiGroupDatabase.Keys.cs +++ b/src/StackExchange.Redis/Availability/MultiGroupDatabase.Keys.cs @@ -1,6 +1,6 @@ using System; -namespace StackExchange.Redis; +namespace StackExchange.Redis.Availability; internal sealed partial class MultiGroupDatabase { diff --git a/src/StackExchange.Redis/MultiGroupDatabase.Lists.Async.cs b/src/StackExchange.Redis/Availability/MultiGroupDatabase.Lists.Async.cs similarity index 99% rename from src/StackExchange.Redis/MultiGroupDatabase.Lists.Async.cs rename to src/StackExchange.Redis/Availability/MultiGroupDatabase.Lists.Async.cs index d284dfee9..74cf09843 100644 --- a/src/StackExchange.Redis/MultiGroupDatabase.Lists.Async.cs +++ b/src/StackExchange.Redis/Availability/MultiGroupDatabase.Lists.Async.cs @@ -1,6 +1,6 @@ using System.Threading.Tasks; -namespace StackExchange.Redis; +namespace StackExchange.Redis.Availability; internal sealed partial class MultiGroupDatabase { diff --git a/src/StackExchange.Redis/MultiGroupDatabase.Lists.cs b/src/StackExchange.Redis/Availability/MultiGroupDatabase.Lists.cs similarity index 99% rename from src/StackExchange.Redis/MultiGroupDatabase.Lists.cs rename to src/StackExchange.Redis/Availability/MultiGroupDatabase.Lists.cs index 58ffc7a1b..fad08222d 100644 --- a/src/StackExchange.Redis/MultiGroupDatabase.Lists.cs +++ b/src/StackExchange.Redis/Availability/MultiGroupDatabase.Lists.cs @@ -1,4 +1,4 @@ -namespace StackExchange.Redis; +namespace StackExchange.Redis.Availability; internal sealed partial class MultiGroupDatabase { diff --git a/src/StackExchange.Redis/MultiGroupDatabase.Sets.Async.cs b/src/StackExchange.Redis/Availability/MultiGroupDatabase.Sets.Async.cs similarity index 98% rename from src/StackExchange.Redis/MultiGroupDatabase.Sets.Async.cs rename to src/StackExchange.Redis/Availability/MultiGroupDatabase.Sets.Async.cs index 7d8eddf54..90341cb36 100644 --- a/src/StackExchange.Redis/MultiGroupDatabase.Sets.Async.cs +++ b/src/StackExchange.Redis/Availability/MultiGroupDatabase.Sets.Async.cs @@ -1,6 +1,6 @@ using System.Threading.Tasks; -namespace StackExchange.Redis; +namespace StackExchange.Redis.Availability; internal sealed partial class MultiGroupDatabase { diff --git a/src/StackExchange.Redis/MultiGroupDatabase.Sets.cs b/src/StackExchange.Redis/Availability/MultiGroupDatabase.Sets.cs similarity index 98% rename from src/StackExchange.Redis/MultiGroupDatabase.Sets.cs rename to src/StackExchange.Redis/Availability/MultiGroupDatabase.Sets.cs index f04df3eeb..443039b48 100644 --- a/src/StackExchange.Redis/MultiGroupDatabase.Sets.cs +++ b/src/StackExchange.Redis/Availability/MultiGroupDatabase.Sets.cs @@ -1,4 +1,4 @@ -namespace StackExchange.Redis; +namespace StackExchange.Redis.Availability; internal sealed partial class MultiGroupDatabase { diff --git a/src/StackExchange.Redis/MultiGroupDatabase.SortedSets.Async.cs b/src/StackExchange.Redis/Availability/MultiGroupDatabase.SortedSets.Async.cs similarity index 99% rename from src/StackExchange.Redis/MultiGroupDatabase.SortedSets.Async.cs rename to src/StackExchange.Redis/Availability/MultiGroupDatabase.SortedSets.Async.cs index 9f79cd423..e89cb3ae6 100644 --- a/src/StackExchange.Redis/MultiGroupDatabase.SortedSets.Async.cs +++ b/src/StackExchange.Redis/Availability/MultiGroupDatabase.SortedSets.Async.cs @@ -1,6 +1,6 @@ using System.Threading.Tasks; -namespace StackExchange.Redis; +namespace StackExchange.Redis.Availability; internal sealed partial class MultiGroupDatabase { diff --git a/src/StackExchange.Redis/MultiGroupDatabase.SortedSets.cs b/src/StackExchange.Redis/Availability/MultiGroupDatabase.SortedSets.cs similarity index 99% rename from src/StackExchange.Redis/MultiGroupDatabase.SortedSets.cs rename to src/StackExchange.Redis/Availability/MultiGroupDatabase.SortedSets.cs index ad1a85aab..2e237dae8 100644 --- a/src/StackExchange.Redis/MultiGroupDatabase.SortedSets.cs +++ b/src/StackExchange.Redis/Availability/MultiGroupDatabase.SortedSets.cs @@ -1,6 +1,4 @@ -using System; - -namespace StackExchange.Redis; +namespace StackExchange.Redis.Availability; internal sealed partial class MultiGroupDatabase { diff --git a/src/StackExchange.Redis/MultiGroupDatabase.Streams.Async.cs b/src/StackExchange.Redis/Availability/MultiGroupDatabase.Streams.Async.cs similarity index 99% rename from src/StackExchange.Redis/MultiGroupDatabase.Streams.Async.cs rename to src/StackExchange.Redis/Availability/MultiGroupDatabase.Streams.Async.cs index 584cabd0f..bef1a96cd 100644 --- a/src/StackExchange.Redis/MultiGroupDatabase.Streams.Async.cs +++ b/src/StackExchange.Redis/Availability/MultiGroupDatabase.Streams.Async.cs @@ -1,7 +1,7 @@ using System; using System.Threading.Tasks; -namespace StackExchange.Redis; +namespace StackExchange.Redis.Availability; internal sealed partial class MultiGroupDatabase { diff --git a/src/StackExchange.Redis/MultiGroupDatabase.Streams.cs b/src/StackExchange.Redis/Availability/MultiGroupDatabase.Streams.cs similarity index 99% rename from src/StackExchange.Redis/MultiGroupDatabase.Streams.cs rename to src/StackExchange.Redis/Availability/MultiGroupDatabase.Streams.cs index a9061464f..e10c847fc 100644 --- a/src/StackExchange.Redis/MultiGroupDatabase.Streams.cs +++ b/src/StackExchange.Redis/Availability/MultiGroupDatabase.Streams.cs @@ -1,6 +1,6 @@ using System; -namespace StackExchange.Redis; +namespace StackExchange.Redis.Availability; internal sealed partial class MultiGroupDatabase { diff --git a/src/StackExchange.Redis/MultiGroupDatabase.Strings.Async.cs b/src/StackExchange.Redis/Availability/MultiGroupDatabase.Strings.Async.cs similarity index 99% rename from src/StackExchange.Redis/MultiGroupDatabase.Strings.Async.cs rename to src/StackExchange.Redis/Availability/MultiGroupDatabase.Strings.Async.cs index 044fffb2f..5e4447b3b 100644 --- a/src/StackExchange.Redis/MultiGroupDatabase.Strings.Async.cs +++ b/src/StackExchange.Redis/Availability/MultiGroupDatabase.Strings.Async.cs @@ -1,7 +1,7 @@ using System; using System.Threading.Tasks; -namespace StackExchange.Redis; +namespace StackExchange.Redis.Availability; internal sealed partial class MultiGroupDatabase { diff --git a/src/StackExchange.Redis/MultiGroupDatabase.Strings.cs b/src/StackExchange.Redis/Availability/MultiGroupDatabase.Strings.cs similarity index 99% rename from src/StackExchange.Redis/MultiGroupDatabase.Strings.cs rename to src/StackExchange.Redis/Availability/MultiGroupDatabase.Strings.cs index 98e3d9ccc..b29d4a771 100644 --- a/src/StackExchange.Redis/MultiGroupDatabase.Strings.cs +++ b/src/StackExchange.Redis/Availability/MultiGroupDatabase.Strings.cs @@ -1,6 +1,6 @@ using System; -namespace StackExchange.Redis; +namespace StackExchange.Redis.Availability; internal sealed partial class MultiGroupDatabase { diff --git a/src/StackExchange.Redis/MultiGroupDatabase.VectorSets.Async.cs b/src/StackExchange.Redis/Availability/MultiGroupDatabase.VectorSets.Async.cs similarity index 98% rename from src/StackExchange.Redis/MultiGroupDatabase.VectorSets.Async.cs rename to src/StackExchange.Redis/Availability/MultiGroupDatabase.VectorSets.Async.cs index f4c11eb24..b72b6ced1 100644 --- a/src/StackExchange.Redis/MultiGroupDatabase.VectorSets.Async.cs +++ b/src/StackExchange.Redis/Availability/MultiGroupDatabase.VectorSets.Async.cs @@ -1,6 +1,6 @@ using System.Threading.Tasks; -namespace StackExchange.Redis; +namespace StackExchange.Redis.Availability; internal sealed partial class MultiGroupDatabase { diff --git a/src/StackExchange.Redis/MultiGroupDatabase.VectorSets.cs b/src/StackExchange.Redis/Availability/MultiGroupDatabase.VectorSets.cs similarity index 98% rename from src/StackExchange.Redis/MultiGroupDatabase.VectorSets.cs rename to src/StackExchange.Redis/Availability/MultiGroupDatabase.VectorSets.cs index 64470da4e..e7a4164db 100644 --- a/src/StackExchange.Redis/MultiGroupDatabase.VectorSets.cs +++ b/src/StackExchange.Redis/Availability/MultiGroupDatabase.VectorSets.cs @@ -1,4 +1,4 @@ -namespace StackExchange.Redis; +namespace StackExchange.Redis.Availability; internal sealed partial class MultiGroupDatabase { diff --git a/src/StackExchange.Redis/MultiGroupDatabase.cs b/src/StackExchange.Redis/Availability/MultiGroupDatabase.cs similarity index 99% rename from src/StackExchange.Redis/MultiGroupDatabase.cs rename to src/StackExchange.Redis/Availability/MultiGroupDatabase.cs index 64728438f..77e0fe4d5 100644 --- a/src/StackExchange.Redis/MultiGroupDatabase.cs +++ b/src/StackExchange.Redis/Availability/MultiGroupDatabase.cs @@ -1,6 +1,6 @@ using System; -namespace StackExchange.Redis; +namespace StackExchange.Redis.Availability; internal sealed partial class MultiGroupDatabase(MultiGroupMultiplexer parent, int database, object? asyncState) : IDatabase diff --git a/src/StackExchange.Redis/Availability/MultiGroupMultiplexer.cs b/src/StackExchange.Redis/Availability/MultiGroupMultiplexer.cs new file mode 100644 index 000000000..b4e47fcce --- /dev/null +++ b/src/StackExchange.Redis/Availability/MultiGroupMultiplexer.cs @@ -0,0 +1,1230 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using System.Linq; +using System.Net; +using System.Threading; +using System.Threading.Tasks; +using RESPite; +using StackExchange.Redis.Availability; +using StackExchange.Redis.Maintenance; +using StackExchange.Redis.Profiling; + +namespace StackExchange.Redis +{ + public partial class ConnectionMultiplexer + { + /// + /// Creates a new instance that manages connections to multiple + /// redundant configurations, based on their availability and relative . + /// + /// The initial configurations to connect to. + /// Additional options for configuring this group. + /// The to log to. +#pragma warning disable RS0026 + [Experimental(Experiments.ActiveActive, UrlFormat = Experiments.UrlFormat)] + public static Task ConnectGroupAsync( + ConnectionGroupMember[] members, + MultiGroupOptions? options = null, + TextWriter? log = null) +#pragma warning restore RS0026 + { + // create a defensive copy of the array; we don't want callers being able to radically swap things! + members = (ConnectionGroupMember[])members.Clone(); + options ??= MultiGroupOptions.Default; + options.Freeze(); + return MultiGroupMultiplexer.ConnectAsync(members, options, log); + } + + /// + /// Creates a new instance that manages connections to multiple + /// redundant configurations, based on their availability and relative . + /// + /// An initial configuration to connect to. + /// An additional initial configuration to connect to. + /// Additional options for configuring this group. + /// The to log to. + [Experimental(Experiments.ActiveActive, UrlFormat = Experiments.UrlFormat)] +#pragma warning disable RS0026 + public static Task ConnectGroupAsync( + ConnectionGroupMember member0, + ConnectionGroupMember member1, + MultiGroupOptions? options = null, + TextWriter? log = null) +#pragma warning restore RS0026 + { + options ??= MultiGroupOptions.Default; + options.Freeze(); + return MultiGroupMultiplexer.ConnectAsync([member0, member1], options, log); + } + } + +#pragma warning disable SA1403 + namespace Availability +#pragma warning restore SA1403 + { + /// + /// A configured member of a . + /// + [Experimental(Experiments.ActiveActive, UrlFormat = Experiments.UrlFormat)] +#pragma warning disable RS0016, RS0026 + public sealed partial class ConnectionGroupMember(ConfigurationOptions configuration, string name = "") +#pragma warning restore RS0016, RS0026 + { + /// + /// Create a new from a configuration string. + /// +#pragma warning disable RS0016, RS0026 + public ConnectionGroupMember(string configuration, string name = "") : this( + ConfigurationOptions.Parse(configuration)) +#pragma warning restore RS0016, RS0026 + { + } + + internal ConfigurationOptions Configuration => configuration; + + private int _activated; // each member can only be activated once + + /// + public override string ToString() => Name; + + private ConnectionMultiplexer? _muxer; + + internal ConnectionMultiplexer Multiplexer => _muxer ?? ThrowNoMuxer(); + + [DoesNotReturn] + private static ConnectionMultiplexer ThrowNoMuxer() => + throw new InvalidOperationException("Member is not connected."); + + // ReSharper disable once NullCoalescingConditionIsAlwaysNotNullAccordingToAPIContract + internal void SetMultiplexer(ConnectionMultiplexer muxer) + => Interlocked.Exchange(ref _muxer, muxer ?? ThrowNoMuxer()); + + internal ConnectionMultiplexer? ClearMultiplexer() => Interlocked.Exchange(ref _muxer, null); + + internal void Init(int index) + { + // add a name if not provided + if (string.IsNullOrWhiteSpace(Name)) + { + var ep = Configuration.EndPoints.FirstOrDefault(); + if (ep is null) + { + Name = index.ToString(); + } + else + { + Name = Format.ToString(ep); + } + } + + // check not already attached + if (Interlocked.CompareExchange(ref _activated, 1, 0) != 0) + { + throw new InvalidOperationException( + $"Member '{Name}' is already associated with a group, and cannot be reused."); + } + } + + /// + /// Indicates whether this group is currently connected. + /// + public bool IsConnected { get; private set; } + + /// + /// The name of this group member. + /// + public string Name { get; private set; } = name; + + /// + /// The relative weight of this group member; higher is preferred. + /// + public double Weight + { + // avoid "tearing", since we can't rule out this being updated concurrently, and the runtime + // only guarantees atomicity for 32-bit reads/writes + get => BitConverter.Int64BitsToDouble(Interlocked.Read(ref _weight64)); + set => Interlocked.Exchange(ref _weight64, BitConverter.DoubleToInt64Bits(value)); + } + + internal bool ExplicitOverride { get; set; } + + private long _weight64 = BitConverter.DoubleToInt64Bits(1.0); + + /// + /// The measured latency to this member. + /// + public TimeSpan Latency => + _latencyTicks is uint.MaxValue ? TimeSpan.MaxValue : TimeSpan.FromTicks(_latencyTicks); + + private uint _latencyTicks = uint.MaxValue; + + internal void SetLatency(uint ticks) => _latencyTicks = ticks; + + internal static uint ToLatencyTicks(TimeSpan latency) + { + long ticks = latency.Ticks; + if (ticks <= 0) + { + return 0; + } + + return ticks > uint.MaxValue ? uint.MaxValue : (uint)ticks; + } + + internal void SetLatency(TimeSpan latency) => SetLatency(ToLatencyTicks(latency)); + + internal static ConnectionGroupMember? Select(ConnectionGroupMember? x, ConnectionGroupMember? y) + { + if (x is null) return y; + if (y is null) return x; + + // always prefer a connected endpoint + bool xc = x.IsConnected, yc = y.IsConnected; + if (xc != yc) return xc ? x : y; + + // prefer manual override if only one is overridden + xc = x.ExplicitOverride; + yc = y.ExplicitOverride; + if (xc != yc) return xc ? x : y; + + // prefer higher weight + double xw = x.Weight, yw = y.Weight; + // ReSharper disable once CompareOfFloatsByEqualityOperator + if (xw != yw) return xw > yw ? x : y; + + // then by latency + uint xl = x._latencyTicks, yl = y._latencyTicks; + return xl <= yl ? x : y; + } + + internal GroupConnectionChangedEventArgs.ChangeType UpdateState(HealthCheck.HealthCheckResult result) + { + bool isConnected; + if (_muxer is { IsConnected: true } muxer) + { + isConnected = result is not HealthCheck.HealthCheckResult.Unhealthy; + SetLatency(muxer.UpdateLatency()); + } + else + { + isConnected = false; + } + + var oldConnected = IsConnected; + IsConnected = isConnected; + + return isConnected == oldConnected ? GroupConnectionChangedEventArgs.ChangeType.Unknown + : isConnected ? GroupConnectionChangedEventArgs.ChangeType.Reconnected + : GroupConnectionChangedEventArgs.ChangeType.Disconnected; + } + + internal void UpdateLatency() + { + if (_muxer is { } muxer) SetLatency(muxer.UpdateLatency()); + } + } + + internal sealed partial class MultiGroupMultiplexer : IConnectionGroup + { + private ConnectionMultiplexer? _active; + private ConnectionGroupMember[] _members; + + public override string ToString() + { + var muxer = _active; + ConnectionGroupMember? member = null; + if (muxer is not null) + { + foreach (var m in _members) + { + if (ReferenceEquals(muxer, m.Multiplexer)) + { + member = m; + break; + } + } + } + + return member is null ? "No active connection" : $"Connected to {member.Name}"; + } + + public ReadOnlySpan GetMembers() => _members; + + internal ConnectionMultiplexer Active + { + get + { + return _active ?? Throw(); + + [DoesNotReturn] + static ConnectionMultiplexer Throw() => + throw new InvalidOperationException("All connections are unavailable."); + } + } + + // non-throwing twin of Active, for callers that have a trivial answer when the group is fully down + internal ConnectionMultiplexer? TryGetActive() => _active; + + // a completed "no endpoint" result, shared by the database/subscriber facades when the group is fully down + internal static readonly Task NoEndpoint = Task.FromResult(null); + + private ConnectionGroupMember? GetActiveMember() + { + var active = _active; + foreach (var member in _members) + { + if (ReferenceEquals(active, member.Multiplexer)) + { + return member; + } + } + + return null; + } + + ConnectionGroupMember? IConnectionGroup.ActiveMember => GetActiveMember(); + + internal ConnectionGroupMember ActiveMember + { + get + { + return GetActiveMember() ?? Throw(); + + [DoesNotReturn] + static ConnectionGroupMember Throw() => + throw new InvalidOperationException("All connections are unavailable."); + } + } + + internal static async Task ConnectAsync( + ConnectionGroupMember[] members, + MultiGroupOptions options, + TextWriter? log) + { + for (int i = 0; i < members.Length; i++) + { + members[i].Init(i); + } + + var pending = new Task[members.Length]; + for (int i = 0; i < members.Length; i++) + { + var config = members[i].Configuration; + config.AbortOnConnectFail = false; + config.HeartbeatConsistencyChecks = true; + config.CircuitBreaker ??= options.CircuitBreaker; // AA options flow into the children + pending[i] = ConnectionMultiplexer.ConnectAsync(config, log); + } + + for (int i = 0; i < pending.Length; i++) + { + var muxer = await pending[i].ConfigureAwait(false); + members[i].SetMultiplexer(muxer); + } + + // run initial healthcheck and begin + var result = new MultiGroupMultiplexer(members, options); + await TryHealthCheckAndSelectPreferredGroupAsync(result).ForAwait(); + result.StartPolling(); + return result; + } + + private readonly MultiGroupOptions _options; + + private MultiGroupMultiplexer(ConnectionGroupMember[] members, MultiGroupOptions options) + { + _options = options; + _members = members; + _active = null; + + _connectionFailedWithCircuitBreaker = OnMemberConnectionFailed; + // multiplexers should already be attached (ConnectAsync sets them before constructing us) + foreach (var member in members) + { + member.Multiplexer?.ConnectionFailed += _connectionFailedWithCircuitBreaker; + } + } + + internal static async Task TryHealthCheckAndSelectPreferredGroupAsync(object? target) + { + if (target is MultiGroupMultiplexer typed) + { + if (typed.IsDisposed) return false; + + // serialize health-check + select: the poll loop and the circuit-breaker fast-path + // can both land here, and we must not run two overlapping check/select passes (they + // would race _active and emit duplicate change events). If a pass is already running, + // skip - it will select on our behalf, and we converge on the next tick regardless. + if (Interlocked.CompareExchange(ref typed._healthCheckGate, 1, 0) is not 0) + { + return true; // still a live target; keep polling + } + + try + { + await typed.RunHealthCheckAsync().ForAwait(); + typed.SelectPreferredGroup(); + } + catch (Exception ex) + { + typed.OnInternalError(ex, origin: "update group"); + } + finally + { + Volatile.Write(ref typed._healthCheckGate, 0); + } + + return true; // even if we fault: try again + } + + return false; + } + + private void StartPolling() + { + // use a weak-ref to avoid the loop keeping the object alive; capturing the token (rather than + // the muxer) lets us break out of the delay promptly on dispose without resurrecting the target + var cancellationToken = _pollCancellation.Token; + _ = Task.Run(() => PollAsync(new(this), cancellationToken)); + + static async Task PollAsync(WeakReference weakRef, CancellationToken cancellationToken) + { + while (TryGetHealthCheck(weakRef.Target, out var interval)) + { + try + { + await Task.Delay(interval, cancellationToken).ForAwait(); + } + catch (OperationCanceledException) + { + break; // disposed; stop polling without waiting out the interval + } + + if (!await TryHealthCheckAndSelectPreferredGroupAsync(weakRef.Target).ForAwait()) break; + } + } + + static bool TryGetHealthCheck(object? target, out TimeSpan interval) + { + if (target is MultiGroupMultiplexer typed + && typed._options.HealthCheck is { } healthCheck) + { + interval = healthCheck.Interval; + return interval > TimeSpan.Zero & interval != TimeSpan.MaxValue; + } + + interval = TimeSpan.Zero; + return false; + } + } + + internal bool IsDisposed => _disposed; + + private Task[]? _reusableHealthCheckBuffer; + private int _healthCheckGate; // 0 = idle, 1 = a check/select pass is in flight (see TryHealthCheckAndSelectPreferredGroupAsync) + + internal async Task RunHealthCheckAsync() + { + if (_disposed) return; + var healthCheck = _options.HealthCheck; + var members = _members; + var pending = HealthCheck.GetReusablePending(ref _reusableHealthCheckBuffer, members.Length); + for (int i = 0; i < members.Length; i++) + { + pending[i] = healthCheck.CheckHealthAsync(members[i].Multiplexer); + } + + await Task.WhenAll(pending).TimeoutAfter(healthCheck.TotalTimeoutMillis()).ForAwait(); + for (int i = 0; i < pending.Length; i++) + { + HealthCheck.HealthCheckResult result; + if (pending[i].IsCompletedSuccessfully) + { + result = await pending[i].ForAwait(); + } + else + { + _ = pending[i].ObserveErrors(); + result = HealthCheck.HealthCheckResult.Unhealthy; + } + + var delta = members[i].UpdateState(result); + if (delta != GroupConnectionChangedEventArgs.ChangeType.Unknown) + { + OnConnectionChanged(delta, members[i]); + } + } + + HealthCheck.PutReusablePending(ref _reusableHealthCheckBuffer, ref pending); + } + + internal void SelectPreferredGroup() + { + if (_disposed) return; + var previousMuxer = _active; + ConnectionGroupMember? preferredMember = null, previousMember = null; + var members = _members; + foreach (var member in members) + { + if (previousMember is null && ReferenceEquals(member.Multiplexer, previousMuxer)) + { + previousMember = member; + } + + if (member.IsConnected) + { + member.UpdateLatency(); // this can change passively + preferredMember = ConnectionGroupMember.Select(preferredMember, member); + } + } + + _active = preferredMember?.Multiplexer; + + if (preferredMember is not null && !ReferenceEquals(preferredMember, previousMember)) + { + OnConnectionChanged( + GroupConnectionChangedEventArgs.ChangeType.ActiveChanged, + preferredMember, + previousMember); + } + } + + private readonly CancellationTokenSource _pollCancellation = new(); + + private List DropAll() + { + _pollCancellation.Cancel(); // stop the polling loop promptly (idempotent) + _active = null; + var members = Interlocked.Exchange(ref _members, []); + if (members.Length is 0) return []; + var muxers = new List(members.Length); + foreach (var member in members) + { + var muxer = member.ClearMultiplexer(); + if (muxer is not null) muxers.Add(muxer); + } + + return muxers; + } + + private bool _disposed; + + public void Dispose() + { + _disposed = true; + foreach (var muxer in DropAll()) + { + muxer.Dispose(); + } + } + + public async ValueTask DisposeAsync() + { + _disposed = true; + foreach (var muxer in DropAll()) + { + await muxer.DisposeAsync(); + } + } + + public string ClientName => Active.ClientName; + public string Configuration => Active.Configuration; + public int TimeoutMilliseconds => Active.TimeoutMilliseconds; + + public long OperationCount + { + get + { + long count = 0; + foreach (var member in _members) + { + count += member.Multiplexer.OperationCount; + } + + return count; + } + } + + [Obsolete] + public bool PreserveAsyncOrder + { + get => Active.PreserveAsyncOrder; + set => Active.PreserveAsyncOrder = value; + } + + // Unlike most members, these intentionally do *not* go via Active (which throws when no member is + // available); callers routinely use IsConnected/IsConnecting as a pre-flight check and expect a + // 'false' result - not an exception - when the entire group is down. + public bool IsConnected => _active?.IsConnected ?? false; + public bool IsConnecting => _active?.IsConnecting ?? false; + + [Obsolete] + public bool IncludeDetailInExceptions + { + get => Active.IncludeDetailInExceptions; + set => Active.IncludeDetailInExceptions = value; + } + + public int StormLogThreshold + { + get => Active.StormLogThreshold; + set => Active.StormLogThreshold = value; + } + + private Func? _profilingSessionProvider; + + public void RegisterProfiler(Func profilingSessionProvider) + { + _profilingSessionProvider = profilingSessionProvider; + foreach (var member in _members) + { + member.Multiplexer.RegisterProfiler(profilingSessionProvider); + } + } + + public ServerCounters GetCounters() => Active.GetCounters(); + + private EventHandler? _errorMessage; + + public event EventHandler? ErrorMessage + { + add + { + if (AddHandler(ref _errorMessage, value)) + { + foreach (var member in _members) + { + member.Multiplexer.ErrorMessage += value; + } + } + } + remove + { + if (RemoveHandler(ref _errorMessage, value)) + { + foreach (var member in _members) + { + member.Multiplexer.ErrorMessage -= value; + } + } + } + } + + /// + /// Add a handler, and return true if this is the *first* handler, which means we should subscribe the dependents. + /// + private static bool AddHandler(ref T? field, T? value) where T : Delegate + { + if (value is null) return false; + while (true) // loop until we win (competition) + { + var oldValue = field; + var newValue = oldValue is null ? value : (T)Delegate.Combine(oldValue, value); + + if (ReferenceEquals(Interlocked.CompareExchange(ref field, newValue, oldValue), oldValue)) + { + return oldValue is null; + } + } + } + + /// + /// Remove a handler, and return true if this is the *last* handler, which means we should unsubscribe the dependents. + /// + private static bool RemoveHandler(ref T? field, T? value) where T : Delegate + { + if (value is null) return false; + while (true) // loop until we win (competition) + { + var oldValue = field; + var newValue = oldValue is null ? null : (T?)Delegate.Remove(oldValue, value); + + if (ReferenceEquals(Interlocked.CompareExchange(ref field, newValue, oldValue), oldValue)) + { + return newValue is null; + } + } + } + + private int _circuitBreakerDebounce = 0; + private void OnMemberConnectionFailed(object? sender, ConnectionFailedEventArgs e) + { + // deliberately scoped to this one failure type; re-probe and re-select promptly rather + // than waiting for the next poll tick, so traffic routes away from the dropped member + if (e.FailureType is ConnectionFailureType.CircuitBreaker + && Interlocked.CompareExchange(ref _circuitBreakerDebounce, 1, 0) is 0) + { + _ = Task.Run(async () => + { + try + { + await TryHealthCheckAndSelectPreferredGroupAsync(this); + } + catch (Exception ex) + { + Debug.WriteLine(ex.Message); + } + finally + { + Volatile.Write(ref _circuitBreakerDebounce, 0); + } + }); + } + + // invoke any custom logic from consumers subscribing to the public events + _connectionFailedExternal?.Invoke(sender, e); + } + + /// + /// Subscribe a child multiplexer to all local event handlers that have subscribers. + /// + private void AddEventHandlers(ConnectionMultiplexer muxer) + { + muxer.ErrorMessage += _errorMessage; + muxer.ConnectionFailed += _connectionFailedWithCircuitBreaker; // always non-null + muxer.InternalError += _internalError; + muxer.ConnectionRestored += _connectionRestored; + muxer.ConfigurationChanged += _configurationChanged; + muxer.ConfigurationChangedBroadcast += _configurationChangedBroadcast; + muxer.ServerMaintenanceEvent += _serverMaintenanceEvent; + muxer.HashSlotMoved += _hashSlotMoved; + } + + /// + /// Unsubscribe a child multiplexer from all local event handlers. + /// + private void RemoveEventHandlers(ConnectionMultiplexer? muxer) + { + if (muxer is null) return; + muxer.ErrorMessage -= _errorMessage; + muxer.ConnectionFailed -= _connectionFailedWithCircuitBreaker; // always non-null + muxer.InternalError -= _internalError; + muxer.ConnectionRestored -= _connectionRestored; + muxer.ConfigurationChanged -= _configurationChanged; + muxer.ConfigurationChangedBroadcast -= _configurationChangedBroadcast; + muxer.ServerMaintenanceEvent -= _serverMaintenanceEvent; + muxer.HashSlotMoved -= _hashSlotMoved; + } + + private readonly EventHandler _connectionFailedWithCircuitBreaker; + + private EventHandler? _connectionFailedExternal; // from public callers + + public event EventHandler? ConnectionFailed + { + // note we *do not* need to hook/unhook the inner connection - we always subscribe to that + add => AddHandler(ref _connectionFailedExternal, value); + remove => RemoveHandler(ref _connectionFailedExternal, value); + } + + private EventHandler? _internalError; + + public event EventHandler? InternalError + { + add + { + if (AddHandler(ref _internalError, value)) + { + foreach (var member in _members) + { + member.Multiplexer.InternalError += value; + } + } + } + remove + { + if (RemoveHandler(ref _internalError, value)) + { + foreach (var member in _members) + { + member.Multiplexer.InternalError -= value; + } + } + } + } + + private EventHandler? _connectionRestored; + + public event EventHandler? ConnectionRestored + { + add + { + if (AddHandler(ref _connectionRestored, value)) + { + foreach (var member in _members) + { + member.Multiplexer.ConnectionRestored += value; + } + } + } + remove + { + if (RemoveHandler(ref _connectionRestored, value)) + { + foreach (var member in _members) + { + member.Multiplexer.ConnectionRestored -= value; + } + } + } + } + + private EventHandler? _configurationChanged; + + public event EventHandler? ConfigurationChanged + { + add + { + if (AddHandler(ref _configurationChanged, value)) + { + foreach (var member in _members) + { + member.Multiplexer.ConfigurationChanged += value; + } + } + } + remove + { + if (RemoveHandler(ref _configurationChanged, value)) + { + foreach (var member in _members) + { + member.Multiplexer.ConfigurationChanged -= value; + } + } + } + } + + private EventHandler? _configurationChangedBroadcast; + + public event EventHandler? ConfigurationChangedBroadcast + { + add + { + if (AddHandler(ref _configurationChangedBroadcast, value)) + { + foreach (var member in _members) + { + member.Multiplexer.ConfigurationChangedBroadcast += value; + } + } + } + remove + { + if (RemoveHandler(ref _configurationChangedBroadcast, value)) + { + foreach (var member in _members) + { + member.Multiplexer.ConfigurationChangedBroadcast -= value; + } + } + } + } + + private EventHandler? _serverMaintenanceEvent; + + public event EventHandler? ServerMaintenanceEvent + { + add + { + if (AddHandler(ref _serverMaintenanceEvent, value)) + { + foreach (var member in _members) + { + member.Multiplexer.ServerMaintenanceEvent += value; + } + } + } + remove + { + if (RemoveHandler(ref _serverMaintenanceEvent, value)) + { + foreach (var member in _members) + { + member.Multiplexer.ServerMaintenanceEvent -= value; + } + } + } + } + + public EndPoint[] GetEndPoints(bool configuredOnly = false) => Active.GetEndPoints(configuredOnly); + + public void Wait(Task task) => Active.Wait(task); + + public T Wait(Task task) => Active.Wait(task); + + public void WaitAll(params Task[] tasks) => Active.WaitAll(tasks); + + private EventHandler? _hashSlotMoved; + + public event EventHandler? HashSlotMoved + { + add + { + if (AddHandler(ref _hashSlotMoved, value)) + { + foreach (var member in _members) + { + member.Multiplexer.HashSlotMoved += value; + } + } + } + remove + { + if (RemoveHandler(ref _hashSlotMoved, value)) + { + foreach (var member in _members) + { + member.Multiplexer.HashSlotMoved -= value; + } + } + } + } + + public int HashSlot(RedisKey key) => Active.HashSlot(key); + + private ISubscriber? _defaultSubscriber; + + public ISubscriber GetSubscriber(object? asyncState = null) + { + if (asyncState is null) return _defaultSubscriber ??= new MultiGroupSubscriber(this, null); + return new MultiGroupSubscriber(this, asyncState); + } + + public IDatabase GetDatabase(int db = -1, object? asyncState = null) + { + if (asyncState is null & db >= -1 & db <= ConnectionMultiplexer.MaxCachedDatabaseInstance) + { + return _databases[db + 1] ??= new MultiGroupDatabase(this, db, null); + } + + return new MultiGroupDatabase(this, db, asyncState); + } + + private readonly IDatabase?[] _databases = + new IDatabase?[ConnectionMultiplexer.MaxCachedDatabaseInstance + 2]; + + public IServer GetServer(string host, int port, object? asyncState = null) + { + Exception ex; + try + { + // try "active" first, and preserve the exception + return Active.GetServer(host, port, asyncState); + } + catch (Exception e) + { + ex = e; + } + + foreach (var member in _members) + { + try + { + return member.Multiplexer.GetServer(host, port, asyncState); + } + catch (Exception e) { Debug.WriteLine(e.Message); } + } + + throw ex; + } + + public IServer GetServer(string hostAndPort, object? asyncState = null) + { + Exception ex; + try + { + // try "active" first, and preserve the exception + return Active.GetServer(hostAndPort, asyncState); + } + catch (Exception e) + { + ex = e; + } + + foreach (var member in _members) + { + try + { + return member.Multiplexer.GetServer(hostAndPort, asyncState); + } + catch (Exception e) { Debug.WriteLine(e.Message); } + } + + throw ex; + } + + public IServer GetServer(IPAddress host, int port) + { + Exception ex; + try + { + // try "active" first, and preserve the exception + return Active.GetServer(host, port); + } + catch (Exception e) + { + ex = e; + } + + foreach (var member in _members) + { + try + { + return member.Multiplexer.GetServer(host, port); + } + catch (Exception e) { Debug.WriteLine(e.Message); } + } + + throw ex; + } + + public IServer GetServer(EndPoint endpoint, object? asyncState = null) + { + Exception ex; + try + { + // try "active" first, and preserve the exception + return Active.GetServer(endpoint, asyncState); + } + catch (Exception e) + { + ex = e; + } + + foreach (var member in _members) + { + try + { + return member.Multiplexer.GetServer(endpoint, asyncState); + } + catch (Exception e) { Debug.WriteLine(e.Message); } + } + + throw ex; + } + + public IServer GetServer(RedisKey key, object? asyncState = null, CommandFlags flags = CommandFlags.None) => + Active.GetServer(key, asyncState, flags); + + public IServer[] GetServers() => Active.GetServers(); + + public Task ConfigureAsync(TextWriter? log = null) => Active.ConfigureAsync(log); + + public bool Configure(TextWriter? log = null) => Active.Configure(log); + + public string GetStatus() => Active.GetStatus(); + + public void GetStatus(TextWriter log) => Active.GetStatus(log); + + public void Close(bool allowCommandsToComplete = true) + { + _disposed = true; + foreach (var member in DropAll()) + { + member.Close(allowCommandsToComplete); + } + } + + public async Task CloseAsync(bool allowCommandsToComplete = true) + { + _disposed = true; + foreach (var member in DropAll()) + { + await member.CloseAsync(allowCommandsToComplete); + } + } + + public string? GetStormLog() => Active.GetStormLog(); + + public void ResetStormLog() => Active.ResetStormLog(); + + public long PublishReconfigure(CommandFlags flags = CommandFlags.None) => Active.PublishReconfigure(flags); + + public Task PublishReconfigureAsync(CommandFlags flags = CommandFlags.None) => + Active.PublishReconfigureAsync(flags); + + public int GetHashSlot(RedisKey key) => Active.GetHashSlot(key); + + public void ExportConfiguration(Stream destination, ExportOptions options = ExportOptions.All) => + Active.ExportConfiguration(destination, options); + + private readonly HashSet _suffixes = new(); // in case we need to add to a new muxer + + public void AddLibraryNameSuffix(string suffix) + { + if (string.IsNullOrWhiteSpace(suffix)) return; // trivial + bool isNew; + lock (_suffixes) + { + isNew = _suffixes.Add(suffix); + } + + if (isNew) + { + foreach (var member in _members) + { + member.Multiplexer.AddLibraryNameSuffix(suffix); + } + } + } + + public event EventHandler? ConnectionChanged; + + private void OnConnectionChanged( + GroupConnectionChangedEventArgs.ChangeType changeType, + ConnectionGroupMember member, + ConnectionGroupMember? previousMember = null) + { + var handler = ConnectionChanged; + if (handler is not null) + { + new GroupConnectionChangedEventArgs(changeType, member, previousMember) + .CompleteAsWorker(handler, this); + } + } + + public async Task AddAsync(ConnectionGroupMember member, TextWriter? log = null) + { + // connect + member.Init(_members.Length); + member.Configuration.HeartbeatConsistencyChecks = true; + member.Configuration.CircuitBreaker ??= _options.CircuitBreaker; // AA options flow into the children + var muxer = await ConnectionMultiplexer.ConnectAsync(member.Configuration, log).ConfigureAwait(false); + member.SetMultiplexer(muxer); + var health = await _options.HealthCheck.CheckHealthAsync(muxer).ConfigureAwait(false); + member.UpdateState(health); + + // apply any shared hooks + AddEventHandlers(muxer); // includes circuit-breaker + if (_profilingSessionProvider is not null) muxer.RegisterProfiler(_profilingSessionProvider); + lock (_suffixes) + { + foreach (var suffix in _suffixes) + { + muxer.AddLibraryNameSuffix(suffix); + } + } + + // update the members array + while (true) + { + var arr = _members; + var newArr = new ConnectionGroupMember[arr.Length + 1]; + Array.Copy(arr, 0, newArr, 0, arr.Length); + newArr[arr.Length] = member; + if (Interlocked.CompareExchange(ref _members, newArr, arr) == arr) break; + } + + OnConnectionChanged(GroupConnectionChangedEventArgs.ChangeType.Added, member); + SelectPreferredGroup(); + + // pub/sub + await AddPubSubHandlersAsync(member).ConfigureAwait(false); + } + + public bool TryFailoverTo(ConnectionGroupMember? member) + { + if (member is null) + { + // remove any explicit overrides, returning whether that was an actual change + bool result = false; + foreach (var m in _members) + { + if (m.ExplicitOverride) + { + result = true; // someone was explicitly enabled + m.ExplicitOverride = false; + } + } + + SelectPreferredGroup(); + return result; + } + + var members = _members; + if (!members.Contains(member)) + { + // not one of ours? + return false; + } + + if (!member.IsConnected) + { + // not allowed + return false; + } + + if (member.ExplicitOverride) + { + // already preferred; no change, but report as success + return true; + } + + // otherwise, deselect everyone else, and select this one + foreach (var m in members) + { + m.ExplicitOverride = ReferenceEquals(m, member); + } + + SelectPreferredGroup(); + return true; + } + + public bool Remove(ConnectionGroupMember member) + { + while (true) + { + var arr = _members; + int index = -1; + for (int i = 0; i < arr.Length; i++) + { + if (ReferenceEquals(arr[i], member)) + { + index = i; + break; + } + } + + if (index == -1) return false; + var newArr = new ConnectionGroupMember[arr.Length - 1]; + if (index > 0) Array.Copy(arr, 0, newArr, 0, index); + if (index < newArr.Length) Array.Copy(arr, index + 1, newArr, index, newArr.Length - index); + if (Interlocked.CompareExchange(ref _members, newArr, arr) == arr) break; + } + + var muxer = member.ClearMultiplexer(); + RemoveEventHandlers(muxer); // includes circuit-breaker + OnConnectionChanged(GroupConnectionChangedEventArgs.ChangeType.Removed, member); + SelectPreferredGroup(); + muxer?.Dispose(); + return true; + } + + internal void OnHeartbeat() // for testing, to update latency etc + { + foreach (var member in _members) + { + member.Multiplexer.OnHeartbeat(); + } + } + + internal void OnInternalError( + Exception exception, + EndPoint? endpoint = null, + ConnectionType connectionType = ConnectionType.None, + string? origin = null) + { + var handler = _internalError; + if (handler is not null) + { + InternalErrorEventArgs args = new(handler, this, endpoint, connectionType, exception, origin); + ConnectionMultiplexer.CompleteAsWorker(args); + } + } + } + } +} diff --git a/src/StackExchange.Redis/MultiGroupOptions.cs b/src/StackExchange.Redis/Availability/MultiGroupOptions.cs similarity index 81% rename from src/StackExchange.Redis/MultiGroupOptions.cs rename to src/StackExchange.Redis/Availability/MultiGroupOptions.cs index 8f783a737..3799836e6 100644 --- a/src/StackExchange.Redis/MultiGroupOptions.cs +++ b/src/StackExchange.Redis/Availability/MultiGroupOptions.cs @@ -4,7 +4,7 @@ using System.Threading; using RESPite; -namespace StackExchange.Redis; +namespace StackExchange.Redis.Availability; /// /// Configuration options for controlling connections to multiple groups. @@ -36,6 +36,15 @@ public HealthCheck HealthCheck set => SetField(ref field, value); } + /// + /// The circuit-breaker to use for members of the group when no per-member circuit-breaker is specified. + /// + public CircuitBreaker CircuitBreaker + { + get => field ?? CircuitBreaker.Default; + set => SetField(ref field, value); + } + // ReSharper disable once RedundantAssignment private void SetField(ref T field, T value, [CallerMemberName] string caller = "") { diff --git a/src/StackExchange.Redis/MultiGroupSubscriber.Tracking.cs b/src/StackExchange.Redis/Availability/MultiGroupSubscriber.Tracking.cs similarity index 99% rename from src/StackExchange.Redis/MultiGroupSubscriber.Tracking.cs rename to src/StackExchange.Redis/Availability/MultiGroupSubscriber.Tracking.cs index 98fcf7ddf..eb560004d 100644 --- a/src/StackExchange.Redis/MultiGroupSubscriber.Tracking.cs +++ b/src/StackExchange.Redis/Availability/MultiGroupSubscriber.Tracking.cs @@ -3,7 +3,7 @@ using System.Collections.Generic; using System.Threading.Tasks; -namespace StackExchange.Redis; +namespace StackExchange.Redis.Availability; internal sealed partial class MultiGroupSubscriber { diff --git a/src/StackExchange.Redis/MultiGroupSubscriber.cs b/src/StackExchange.Redis/Availability/MultiGroupSubscriber.cs similarity index 98% rename from src/StackExchange.Redis/MultiGroupSubscriber.cs rename to src/StackExchange.Redis/Availability/MultiGroupSubscriber.cs index 3ce12764d..4dbb8e9e3 100644 --- a/src/StackExchange.Redis/MultiGroupSubscriber.cs +++ b/src/StackExchange.Redis/Availability/MultiGroupSubscriber.cs @@ -2,7 +2,7 @@ using System.Net; using System.Threading.Tasks; -namespace StackExchange.Redis; +namespace StackExchange.Redis.Availability; internal sealed partial class MultiGroupSubscriber(MultiGroupMultiplexer parent, object? asyncState) : ISubscriber { diff --git a/src/StackExchange.Redis/ConfigurationOptions.cs b/src/StackExchange.Redis/ConfigurationOptions.cs index 87ef46740..5a497e825 100644 --- a/src/StackExchange.Redis/ConfigurationOptions.cs +++ b/src/StackExchange.Redis/ConfigurationOptions.cs @@ -16,6 +16,7 @@ using Microsoft.Extensions.Logging; using RESPite; using RESPite.Buffers; +using StackExchange.Redis.Availability; using StackExchange.Redis.Configuration; namespace StackExchange.Redis @@ -978,6 +979,7 @@ public static ConfigurationOptions Parse(string configuration, bool ignoreUnknow _protocol = _protocol, heartbeatInterval = heartbeatInterval, WriteMode = WriteMode, + CircuitBreaker = CircuitBreaker, #if DEBUG OutputLog = OutputLog, #endif @@ -1181,6 +1183,7 @@ private void Clear() Tunnel = null; _protocol = default; WriteMode = default; + CircuitBreaker = null; #if DEBUG OutputLog = null; #endif @@ -1379,6 +1382,11 @@ public RedisProtocol? Protocol } internal BufferedStreamWriter.WriteMode WriteMode { get; set; } + + // circuit-breaker to apply to physical connections; when null, no breaker is used. Not surfaced as + // public API for now (minimizing drift); flows in from MultiGroupOptions when connecting a group. + internal CircuitBreaker? CircuitBreaker { get; set; } + internal bool AllowSimulateConnectionFailure { get => IsSet(OptionFlags.AllowSimulateConnectionFailure); diff --git a/src/StackExchange.Redis/Enums/ConnectionFailureType.cs b/src/StackExchange.Redis/Enums/ConnectionFailureType.cs index 55eeacef6..d771df9db 100644 --- a/src/StackExchange.Redis/Enums/ConnectionFailureType.cs +++ b/src/StackExchange.Redis/Enums/ConnectionFailureType.cs @@ -1,4 +1,7 @@ -namespace StackExchange.Redis +using System.Diagnostics.CodeAnalysis; +using RESPite; + +namespace StackExchange.Redis { /// /// The known types of connection failure. @@ -59,5 +62,11 @@ public enum ConnectionFailureType /// High-integrity mode was enabled, and a failure was detected. /// ResponseIntegrityFailure, + + /// + /// The associated with this connection detected instability. + /// + [Experimental(Experiments.ActiveActive, UrlFormat = Experiments.UrlFormat)] + CircuitBreaker, } } diff --git a/src/StackExchange.Redis/Interfaces/IConnectionGroup.cs b/src/StackExchange.Redis/Interfaces/IConnectionGroup.cs index 1afd73290..7f78328e3 100644 --- a/src/StackExchange.Redis/Interfaces/IConnectionGroup.cs +++ b/src/StackExchange.Redis/Interfaces/IConnectionGroup.cs @@ -4,9 +4,10 @@ using System.Text; using System.Threading.Tasks; using RESPite; +using StackExchange.Redis.Availability; // ReSharper disable once CheckNamespace -namespace StackExchange.Redis; +namespace StackExchange.Redis.Availability; /// /// A group of connections to redis servers, that manages connections to multiple diff --git a/src/StackExchange.Redis/Message.cs b/src/StackExchange.Redis/Message.cs index 17ded6c20..b37995b78 100644 --- a/src/StackExchange.Redis/Message.cs +++ b/src/StackExchange.Redis/Message.cs @@ -483,18 +483,21 @@ public string ToStringCommandOnly() => bool ICompletable.TryComplete(bool isAsync) { - Complete(); + Complete(null); return true; } - public void Complete() + public void Complete(PhysicalConnection? connection) { // Ensure we can never call Complete on the same resultBox from two threads by grabbing it now var currBox = Interlocked.Exchange(ref resultBox, null); // set the completion/performance data performance?.SetCompleted(); - + if (currBox is not null) + { + connection?.ObserveMessageResult(currBox.Fault); + } currBox?.ActivateContinuations(); } @@ -693,7 +696,7 @@ internal bool ComputeResult(PhysicalConnection connection, ref RespReader reader { connection.OnDetailLog($"{ex.GetType().Name}: {ex.Message}"); ex.Data.Add("got", prefix.ToString()); - connection?.BridgeCouldBeNull?.Multiplexer?.OnMessageFaulted(this, ex); + connection.BridgeCouldBeNull?.Multiplexer?.OnMessageFaulted(this, ex); box?.SetException(ex); return box != null; // we still want to pulse/complete } @@ -706,10 +709,10 @@ internal void Fail(ConnectionFailureType failure, Exception? innerException, str resultProcessor?.ConnectionFail(this, failure, innerException, annotation, muxer); } - internal virtual void SetExceptionAndComplete(Exception exception, PhysicalBridge? bridge) + internal virtual void SetExceptionAndComplete(Exception exception, PhysicalConnection? connection) { resultBox?.SetException(exception); - Complete(); + Complete(connection); } internal bool TrySetResult(T value) diff --git a/src/StackExchange.Redis/MultiGroupMultiplexer.cs b/src/StackExchange.Redis/MultiGroupMultiplexer.cs deleted file mode 100644 index 9c2e1d45b..000000000 --- a/src/StackExchange.Redis/MultiGroupMultiplexer.cs +++ /dev/null @@ -1,1163 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Diagnostics.CodeAnalysis; -using System.IO; -using System.Linq; -using System.Net; -using System.Threading; -using System.Threading.Tasks; -using RESPite; -using StackExchange.Redis.Maintenance; -using StackExchange.Redis.Profiling; - -namespace StackExchange.Redis; - -public partial class ConnectionMultiplexer -{ - /// - /// Creates a new instance that manages connections to multiple - /// redundant configurations, based on their availability and relative . - /// - /// The initial configurations to connect to. - /// Additional options for configuring this group. - /// The to log to. -#pragma warning disable RS0026 - [Experimental(Experiments.ActiveActive, UrlFormat = Experiments.UrlFormat)] - public static Task ConnectGroupAsync(ConnectionGroupMember[] members, MultiGroupOptions? options = null, TextWriter? log = null) -#pragma warning restore RS0026 - { - // create a defensive copy of the array; we don't want callers being able to radically swap things! - members = (ConnectionGroupMember[])members.Clone(); - options ??= MultiGroupOptions.Default; - options.Freeze(); - return MultiGroupMultiplexer.ConnectAsync(members, options, log); - } - - /// - /// Creates a new instance that manages connections to multiple - /// redundant configurations, based on their availability and relative . - /// - /// An initial configuration to connect to. - /// An additional initial configuration to connect to. - /// Additional options for configuring this group. - /// The to log to. - [Experimental(Experiments.ActiveActive, UrlFormat = Experiments.UrlFormat)] -#pragma warning disable RS0026 - public static Task ConnectGroupAsync( - ConnectionGroupMember member0, - ConnectionGroupMember member1, - MultiGroupOptions? options = null, - TextWriter? log = null) -#pragma warning restore RS0026 - { - options ??= MultiGroupOptions.Default; - options.Freeze(); - return MultiGroupMultiplexer.ConnectAsync([member0, member1], options, log); - } -} - -/// -/// A configured member of a . -/// -[Experimental(Experiments.ActiveActive, UrlFormat = Experiments.UrlFormat)] -#pragma warning disable RS0016, RS0026 -public sealed partial class ConnectionGroupMember(ConfigurationOptions configuration, string name = "") -#pragma warning restore RS0016, RS0026 -{ - /// - /// Create a new from a configuration string. - /// -#pragma warning disable RS0016, RS0026 - public ConnectionGroupMember(string configuration, string name = "") : this( - ConfigurationOptions.Parse(configuration)) -#pragma warning restore RS0016, RS0026 - { - } - - internal ConfigurationOptions Configuration => configuration; - - private int _activated; // each member can only be activated once - - /// - public override string ToString() => Name; - - private ConnectionMultiplexer? _muxer; - - internal ConnectionMultiplexer Multiplexer => _muxer ?? ThrowNoMuxer(); - - [DoesNotReturn] - private static ConnectionMultiplexer ThrowNoMuxer() => - throw new InvalidOperationException("Member is not connected."); - - // ReSharper disable once NullCoalescingConditionIsAlwaysNotNullAccordingToAPIContract - internal void SetMultiplexer(ConnectionMultiplexer muxer) - => Interlocked.Exchange(ref _muxer, muxer ?? ThrowNoMuxer()); - - internal ConnectionMultiplexer? ClearMultiplexer() => Interlocked.Exchange(ref _muxer, null); - - internal void Init(int index) - { - // add a name if not provided - if (string.IsNullOrWhiteSpace(Name)) - { - var ep = Configuration.EndPoints.FirstOrDefault(); - if (ep is null) - { - Name = index.ToString(); - } - else - { - Name = Format.ToString(ep); - } - } - - // check not already attached - if (Interlocked.CompareExchange(ref _activated, 1, 0) != 0) - { - throw new InvalidOperationException( - $"Member '{Name}' is already associated with a group, and cannot be reused."); - } - } - - /// - /// Indicates whether this group is currently connected. - /// - public bool IsConnected { get; private set; } - - /// - /// The name of this group member. - /// - public string Name { get; private set; } = name; - - /// - /// The relative weight of this group member; higher is preferred. - /// - public double Weight - { - // avoid "tearing", since we can't rule out this being updated concurrently, and the runtime - // only guarantees atomicity for 32-bit reads/writes - get => BitConverter.Int64BitsToDouble(Interlocked.Read(ref _weight64)); - set => Interlocked.Exchange(ref _weight64, BitConverter.DoubleToInt64Bits(value)); - } - - internal bool ExplicitOverride { get; set; } - - private long _weight64 = BitConverter.DoubleToInt64Bits(1.0); - - /// - /// The measured latency to this member. - /// - public TimeSpan Latency => _latencyTicks is uint.MaxValue ? TimeSpan.MaxValue : TimeSpan.FromTicks(_latencyTicks); - - private uint _latencyTicks = uint.MaxValue; - - internal void SetLatency(uint ticks) => _latencyTicks = ticks; - - internal static uint ToLatencyTicks(TimeSpan latency) - { - long ticks = latency.Ticks; - if (ticks <= 0) - { - return 0; - } - return ticks > uint.MaxValue ? uint.MaxValue : (uint)ticks; - } - - internal void SetLatency(TimeSpan latency) => SetLatency(ToLatencyTicks(latency)); - - internal static ConnectionGroupMember? Select(ConnectionGroupMember? x, ConnectionGroupMember? y) - { - if (x is null) return y; - if (y is null) return x; - - // always prefer a connected endpoint - bool xc = x.IsConnected, yc = y.IsConnected; - if (xc != yc) return xc ? x : y; - - // prefer manual override if only one is overridden - xc = x.ExplicitOverride; - yc = y.ExplicitOverride; - if (xc != yc) return xc ? x : y; - - // prefer higher weight - double xw = x.Weight, yw = y.Weight; - // ReSharper disable once CompareOfFloatsByEqualityOperator - if (xw != yw) return xw > yw ? x : y; - - // then by latency - uint xl = x._latencyTicks, yl = y._latencyTicks; - return xl <= yl ? x : y; - } - - internal GroupConnectionChangedEventArgs.ChangeType UpdateState(HealthCheck.HealthCheckResult result) - { - bool isConnected; - if (_muxer is { IsConnected: true } muxer) - { - isConnected = result is not HealthCheck.HealthCheckResult.Unhealthy; - SetLatency(muxer.UpdateLatency()); - } - else - { - isConnected = false; - } - - var oldConnected = IsConnected; - IsConnected = isConnected; - - return isConnected == oldConnected ? GroupConnectionChangedEventArgs.ChangeType.Unknown - : isConnected ? GroupConnectionChangedEventArgs.ChangeType.Reconnected - : GroupConnectionChangedEventArgs.ChangeType.Disconnected; - } - - internal void UpdateLatency() - { - if (_muxer is { } muxer) SetLatency(muxer.UpdateLatency()); - } -} - -internal sealed partial class MultiGroupMultiplexer : IConnectionGroup -{ - private ConnectionMultiplexer? _active; - private ConnectionGroupMember[] _members; - - public override string ToString() - { - var muxer = _active; - ConnectionGroupMember? member = null; - if (muxer is not null) - { - foreach (var m in _members) - { - if (ReferenceEquals(muxer, m.Multiplexer)) - { - member = m; - break; - } - } - } - - return member is null ? "No active connection" : $"Connected to {member.Name}"; - } - - public ReadOnlySpan GetMembers() => _members; - - internal ConnectionMultiplexer Active - { - get - { - return _active ?? Throw(); - - [DoesNotReturn] - static ConnectionMultiplexer Throw() => - throw new InvalidOperationException("All connections are unavailable."); - } - } - - // non-throwing twin of Active, for callers that have a trivial answer when the group is fully down - internal ConnectionMultiplexer? TryGetActive() => _active; - - // a completed "no endpoint" result, shared by the database/subscriber facades when the group is fully down - internal static readonly Task NoEndpoint = Task.FromResult(null); - - private ConnectionGroupMember? GetActiveMember() - { - var active = _active; - foreach (var member in _members) - { - if (ReferenceEquals(active, member.Multiplexer)) - { - return member; - } - } - - return null; - } - ConnectionGroupMember? IConnectionGroup.ActiveMember => GetActiveMember(); - - internal ConnectionGroupMember ActiveMember - { - get - { - return GetActiveMember() ?? Throw(); - - [DoesNotReturn] - static ConnectionGroupMember Throw() => - throw new InvalidOperationException("All connections are unavailable."); - } - } - - internal static async Task ConnectAsync(ConnectionGroupMember[] members, MultiGroupOptions options, TextWriter? log) - { - for (int i = 0; i < members.Length; i++) - { - members[i].Init(i); - } - - var pending = new Task[members.Length]; - for (int i = 0; i < members.Length; i++) - { - var config = members[i].Configuration; - config.AbortOnConnectFail = false; - config.HeartbeatConsistencyChecks = true; - pending[i] = ConnectionMultiplexer.ConnectAsync(config, log); - } - - for (int i = 0; i < pending.Length; i++) - { - var muxer = await pending[i].ConfigureAwait(false); - members[i].SetMultiplexer(muxer); - } - - // run initial healthcheck and begin - var result = new MultiGroupMultiplexer(members, options); - await TryHealthCheckAndSelectPreferredGroupAsync(result).ForAwait(); - result.StartPolling(); - return result; - } - - private readonly MultiGroupOptions _options; - private MultiGroupMultiplexer(ConnectionGroupMember[] members, MultiGroupOptions options) - { - _options = options; - _members = members; - _active = null; - } - - internal static async Task TryHealthCheckAndSelectPreferredGroupAsync(object? target) - { - if (target is MultiGroupMultiplexer typed) - { - try - { - if (typed.IsDisposed) return false; - await typed.RunHealthCheckAsync().ForAwait(); - typed.SelectPreferredGroup(); - } - catch (Exception ex) - { - typed.OnInternalError(ex, origin: "update group"); - } - return true; // even if we fault: try again - } - return false; - } - - private void StartPolling() - { - // use a weak-ref to avoid the loop keeping the object alive; capturing the token (rather than - // the muxer) lets us break out of the delay promptly on dispose without resurrecting the target - var cancellationToken = _pollCancellation.Token; - _ = Task.Run(() => PollAsync(new(this), cancellationToken)); - - static async Task PollAsync(WeakReference weakRef, CancellationToken cancellationToken) - { - while (TryGetHealthCheck(weakRef.Target, out var interval)) - { - try - { - await Task.Delay(interval, cancellationToken).ForAwait(); - } - catch (OperationCanceledException) - { - break; // disposed; stop polling without waiting out the interval - } - if (!await TryHealthCheckAndSelectPreferredGroupAsync(weakRef.Target).ForAwait()) break; - } - } - - static bool TryGetHealthCheck(object? target, out TimeSpan interval) - { - if (target is MultiGroupMultiplexer typed - && typed._options.HealthCheck is { } healthCheck) - { - interval = healthCheck.Interval; - return interval > TimeSpan.Zero & interval != TimeSpan.MaxValue; - } - - interval = TimeSpan.Zero; - return false; - } - } - - internal bool IsDisposed => _disposed; - - private Task[]? _reusableHealthCheckBuffer; - - internal async Task RunHealthCheckAsync() - { - if (_disposed) return; - var healthCheck = _options.HealthCheck; - var members = _members; - var pending = HealthCheck.GetReusablePending(ref _reusableHealthCheckBuffer, members.Length); - for (int i = 0; i < members.Length; i++) - { - pending[i] = healthCheck.CheckHealthAsync(members[i].Multiplexer); - } - - await Task.WhenAll(pending).TimeoutAfter(healthCheck.TotalTimeoutMillis()).ForAwait(); - for (int i = 0; i < pending.Length; i++) - { - HealthCheck.HealthCheckResult result; - if (pending[i].IsCompletedSuccessfully) - { - result = await pending[i].ForAwait(); - } - else - { - _ = pending[i].ObserveErrors(); - result = HealthCheck.HealthCheckResult.Unhealthy; - } - - var delta = members[i].UpdateState(result); - if (delta != GroupConnectionChangedEventArgs.ChangeType.Unknown) - { - OnConnectionChanged(delta, members[i]); - } - } - - HealthCheck.PutReusablePending(ref _reusableHealthCheckBuffer, ref pending); - } - - internal void SelectPreferredGroup() - { - if (_disposed) return; - var previousMuxer = _active; - ConnectionGroupMember? preferredMember = null, previousMember = null; - var members = _members; - foreach (var member in members) - { - if (previousMember is null && ReferenceEquals(member.Multiplexer, previousMuxer)) - { - previousMember = member; - } - - if (member.IsConnected) - { - member.UpdateLatency(); // this can change passively - preferredMember = ConnectionGroupMember.Select(preferredMember, member); - } - } - - _active = preferredMember?.Multiplexer; - - if (preferredMember is not null && !ReferenceEquals(preferredMember, previousMember)) - { - OnConnectionChanged( - GroupConnectionChangedEventArgs.ChangeType.ActiveChanged, - preferredMember, - previousMember); - } - } - - private readonly CancellationTokenSource _pollCancellation = new(); - - private List DropAll() - { - _pollCancellation.Cancel(); // stop the polling loop promptly (idempotent) - _active = null; - var members = Interlocked.Exchange(ref _members, []); - if (members.Length is 0) return []; - var muxers = new List(members.Length); - foreach (var member in members) - { - var muxer = member.ClearMultiplexer(); - if (muxer is not null) muxers.Add(muxer); - } - - return muxers; - } - - private bool _disposed; - public void Dispose() - { - _disposed = true; - foreach (var muxer in DropAll()) - { - muxer.Dispose(); - } - } - - public async ValueTask DisposeAsync() - { - _disposed = true; - foreach (var muxer in DropAll()) - { - await muxer.DisposeAsync(); - } - } - - public string ClientName => Active.ClientName; - public string Configuration => Active.Configuration; - public int TimeoutMilliseconds => Active.TimeoutMilliseconds; - - public long OperationCount - { - get - { - long count = 0; - foreach (var member in _members) - { - count += member.Multiplexer.OperationCount; - } - - return count; - } - } - - [Obsolete] - public bool PreserveAsyncOrder - { - get => Active.PreserveAsyncOrder; - set => Active.PreserveAsyncOrder = value; - } - - // Unlike most members, these intentionally do *not* go via Active (which throws when no member is - // available); callers routinely use IsConnected/IsConnecting as a pre-flight check and expect a - // 'false' result - not an exception - when the entire group is down. - public bool IsConnected => _active?.IsConnected ?? false; - public bool IsConnecting => _active?.IsConnecting ?? false; - - [Obsolete] - public bool IncludeDetailInExceptions - { - get => Active.IncludeDetailInExceptions; - set => Active.IncludeDetailInExceptions = value; - } - - public int StormLogThreshold - { - get => Active.StormLogThreshold; - set => Active.StormLogThreshold = value; - } - - private Func? _profilingSessionProvider; - - public void RegisterProfiler(Func profilingSessionProvider) - { - _profilingSessionProvider = profilingSessionProvider; - foreach (var member in _members) - { - member.Multiplexer.RegisterProfiler(profilingSessionProvider); - } - } - - public ServerCounters GetCounters() => Active.GetCounters(); - - private EventHandler? _errorMessage; - - public event EventHandler? ErrorMessage - { - add - { - if (AddHandler(ref _errorMessage, value)) - { - foreach (var member in _members) - { - member.Multiplexer.ErrorMessage += value; - } - } - } - remove - { - if (RemoveHandler(ref _errorMessage, value)) - { - foreach (var member in _members) - { - member.Multiplexer.ErrorMessage -= value; - } - } - } - } - - /// - /// Add a handler, and return true if this is the *first* handler, which means we should subscribe the dependents. - /// - private static bool AddHandler(ref T? field, T? value) where T : Delegate - { - if (value is null) return false; - while (true) // loop until we win (competition) - { - var oldValue = field; - var newValue = oldValue is null ? value : (T)Delegate.Combine(oldValue, value); - - if (ReferenceEquals(Interlocked.CompareExchange(ref field, newValue, oldValue), oldValue)) - { - return oldValue is null; - } - } - } - - /// - /// Remove a handler, and return true if this is the *last* handler, which means we should unsubscribe the dependents. - /// - private static bool RemoveHandler(ref T? field, T? value) where T : Delegate - { - if (value is null) return false; - while (true) // loop until we win (competition) - { - var oldValue = field; - var newValue = oldValue is null ? null : (T?)Delegate.Remove(oldValue, value); - - if (ReferenceEquals(Interlocked.CompareExchange(ref field, newValue, oldValue), oldValue)) - { - return newValue is null; - } - } - } - - /// - /// Subscribe a child multiplexer to all local event handlers that have subscribers. - /// - private void AddEventHandlers(ConnectionMultiplexer muxer) - { - muxer.ErrorMessage += _errorMessage; - muxer.ConnectionFailed += _connectionFailed; - muxer.InternalError += _internalError; - muxer.ConnectionRestored += _connectionRestored; - muxer.ConfigurationChanged += _configurationChanged; - muxer.ConfigurationChangedBroadcast += _configurationChangedBroadcast; - muxer.ServerMaintenanceEvent += _serverMaintenanceEvent; - muxer.HashSlotMoved += _hashSlotMoved; - } - - /// - /// Unsubscribe a child multiplexer from all local event handlers. - /// - private void RemoveEventHandlers(ConnectionMultiplexer? muxer) - { - if (muxer is null) return; - muxer.ErrorMessage -= _errorMessage; - muxer.ConnectionFailed -= _connectionFailed; - muxer.InternalError -= _internalError; - muxer.ConnectionRestored -= _connectionRestored; - muxer.ConfigurationChanged -= _configurationChanged; - muxer.ConfigurationChangedBroadcast -= _configurationChangedBroadcast; - muxer.ServerMaintenanceEvent -= _serverMaintenanceEvent; - muxer.HashSlotMoved -= _hashSlotMoved; - } - - private EventHandler? _connectionFailed; - - public event EventHandler? ConnectionFailed - { - add - { - if (AddHandler(ref _connectionFailed, value)) - { - foreach (var member in _members) - { - member.Multiplexer.ConnectionFailed += value; - } - } - } - remove - { - if (RemoveHandler(ref _connectionFailed, value)) - { - foreach (var member in _members) - { - member.Multiplexer.ConnectionFailed -= value; - } - } - } - } - - private EventHandler? _internalError; - - public event EventHandler? InternalError - { - add - { - if (AddHandler(ref _internalError, value)) - { - foreach (var member in _members) - { - member.Multiplexer.InternalError += value; - } - } - } - remove - { - if (RemoveHandler(ref _internalError, value)) - { - foreach (var member in _members) - { - member.Multiplexer.InternalError -= value; - } - } - } - } - - private EventHandler? _connectionRestored; - - public event EventHandler? ConnectionRestored - { - add - { - if (AddHandler(ref _connectionRestored, value)) - { - foreach (var member in _members) - { - member.Multiplexer.ConnectionRestored += value; - } - } - } - remove - { - if (RemoveHandler(ref _connectionRestored, value)) - { - foreach (var member in _members) - { - member.Multiplexer.ConnectionRestored -= value; - } - } - } - } - - private EventHandler? _configurationChanged; - - public event EventHandler? ConfigurationChanged - { - add - { - if (AddHandler(ref _configurationChanged, value)) - { - foreach (var member in _members) - { - member.Multiplexer.ConfigurationChanged += value; - } - } - } - remove - { - if (RemoveHandler(ref _configurationChanged, value)) - { - foreach (var member in _members) - { - member.Multiplexer.ConfigurationChanged -= value; - } - } - } - } - - private EventHandler? _configurationChangedBroadcast; - - public event EventHandler? ConfigurationChangedBroadcast - { - add - { - if (AddHandler(ref _configurationChangedBroadcast, value)) - { - foreach (var member in _members) - { - member.Multiplexer.ConfigurationChangedBroadcast += value; - } - } - } - remove - { - if (RemoveHandler(ref _configurationChangedBroadcast, value)) - { - foreach (var member in _members) - { - member.Multiplexer.ConfigurationChangedBroadcast -= value; - } - } - } - } - - private EventHandler? _serverMaintenanceEvent; - - public event EventHandler? ServerMaintenanceEvent - { - add - { - if (AddHandler(ref _serverMaintenanceEvent, value)) - { - foreach (var member in _members) - { - member.Multiplexer.ServerMaintenanceEvent += value; - } - } - } - remove - { - if (RemoveHandler(ref _serverMaintenanceEvent, value)) - { - foreach (var member in _members) - { - member.Multiplexer.ServerMaintenanceEvent -= value; - } - } - } - } - - public EndPoint[] GetEndPoints(bool configuredOnly = false) => Active.GetEndPoints(configuredOnly); - - public void Wait(Task task) => Active.Wait(task); - - public T Wait(Task task) => Active.Wait(task); - - public void WaitAll(params Task[] tasks) => Active.WaitAll(tasks); - - private EventHandler? _hashSlotMoved; - - public event EventHandler? HashSlotMoved - { - add - { - if (AddHandler(ref _hashSlotMoved, value)) - { - foreach (var member in _members) - { - member.Multiplexer.HashSlotMoved += value; - } - } - } - remove - { - if (RemoveHandler(ref _hashSlotMoved, value)) - { - foreach (var member in _members) - { - member.Multiplexer.HashSlotMoved -= value; - } - } - } - } - - public int HashSlot(RedisKey key) => Active.HashSlot(key); - - private ISubscriber? _defaultSubscriber; - - public ISubscriber GetSubscriber(object? asyncState = null) - { - if (asyncState is null) return _defaultSubscriber ??= new MultiGroupSubscriber(this, null); - return new MultiGroupSubscriber(this, asyncState); - } - - public IDatabase GetDatabase(int db = -1, object? asyncState = null) - { - if (asyncState is null & db >= -1 & db <= ConnectionMultiplexer.MaxCachedDatabaseInstance) - { - return _databases[db + 1] ??= new MultiGroupDatabase(this, db, null); - } - - return new MultiGroupDatabase(this, db, asyncState); - } - - private readonly IDatabase?[] _databases = new IDatabase?[ConnectionMultiplexer.MaxCachedDatabaseInstance + 2]; - - public IServer GetServer(string host, int port, object? asyncState = null) - { - Exception ex; - try - { - // try "active" first, and preserve the exception - return Active.GetServer(host, port, asyncState); - } - catch (Exception e) - { - ex = e; - } - - foreach (var member in _members) - { - try - { - return member.Multiplexer.GetServer(host, port, asyncState); - } - catch (Exception e) { Debug.WriteLine(e.Message); } - } - - throw ex; - } - - public IServer GetServer(string hostAndPort, object? asyncState = null) - { - Exception ex; - try - { - // try "active" first, and preserve the exception - return Active.GetServer(hostAndPort, asyncState); - } - catch (Exception e) - { - ex = e; - } - - foreach (var member in _members) - { - try - { - return member.Multiplexer.GetServer(hostAndPort, asyncState); - } - catch (Exception e) { Debug.WriteLine(e.Message); } - } - - throw ex; - } - - public IServer GetServer(IPAddress host, int port) - { - Exception ex; - try - { - // try "active" first, and preserve the exception - return Active.GetServer(host, port); - } - catch (Exception e) - { - ex = e; - } - - foreach (var member in _members) - { - try - { - return member.Multiplexer.GetServer(host, port); - } - catch (Exception e) { Debug.WriteLine(e.Message); } - } - - throw ex; - } - - public IServer GetServer(EndPoint endpoint, object? asyncState = null) - { - Exception ex; - try - { - // try "active" first, and preserve the exception - return Active.GetServer(endpoint, asyncState); - } - catch (Exception e) - { - ex = e; - } - - foreach (var member in _members) - { - try - { - return member.Multiplexer.GetServer(endpoint, asyncState); - } - catch (Exception e) { Debug.WriteLine(e.Message); } - } - - throw ex; - } - - public IServer GetServer(RedisKey key, object? asyncState = null, CommandFlags flags = CommandFlags.None) => - Active.GetServer(key, asyncState, flags); - - public IServer[] GetServers() => Active.GetServers(); - - public Task ConfigureAsync(TextWriter? log = null) => Active.ConfigureAsync(log); - - public bool Configure(TextWriter? log = null) => Active.Configure(log); - - public string GetStatus() => Active.GetStatus(); - - public void GetStatus(TextWriter log) => Active.GetStatus(log); - - public void Close(bool allowCommandsToComplete = true) - { - _disposed = true; - foreach (var member in DropAll()) - { - member.Close(allowCommandsToComplete); - } - } - - public async Task CloseAsync(bool allowCommandsToComplete = true) - { - _disposed = true; - foreach (var member in DropAll()) - { - await member.CloseAsync(allowCommandsToComplete); - } - } - - public string? GetStormLog() => Active.GetStormLog(); - - public void ResetStormLog() => Active.ResetStormLog(); - - public long PublishReconfigure(CommandFlags flags = CommandFlags.None) => Active.PublishReconfigure(flags); - - public Task PublishReconfigureAsync(CommandFlags flags = CommandFlags.None) => - Active.PublishReconfigureAsync(flags); - - public int GetHashSlot(RedisKey key) => Active.GetHashSlot(key); - - public void ExportConfiguration(Stream destination, ExportOptions options = ExportOptions.All) => - Active.ExportConfiguration(destination, options); - - private readonly HashSet _suffixes = new(); // in case we need to add to a new muxer - public void AddLibraryNameSuffix(string suffix) - { - if (string.IsNullOrWhiteSpace(suffix)) return; // trivial - bool isNew; - lock (_suffixes) - { - isNew = _suffixes.Add(suffix); - } - - if (isNew) - { - foreach (var member in _members) - { - member.Multiplexer.AddLibraryNameSuffix(suffix); - } - } - } - - public event EventHandler? ConnectionChanged; - - private void OnConnectionChanged( - GroupConnectionChangedEventArgs.ChangeType changeType, - ConnectionGroupMember member, - ConnectionGroupMember? previousMember = null) - { - var handler = ConnectionChanged; - if (handler is not null) - { - new GroupConnectionChangedEventArgs(changeType, member, previousMember) - .CompleteAsWorker(handler, this); - } - } - - public async Task AddAsync(ConnectionGroupMember member, TextWriter? log = null) - { - // connect - member.Init(_members.Length); - member.Configuration.HeartbeatConsistencyChecks = true; - var muxer = await ConnectionMultiplexer.ConnectAsync(member.Configuration, log).ConfigureAwait(false); - member.SetMultiplexer(muxer); - var health = await _options.HealthCheck.CheckHealthAsync(muxer).ConfigureAwait(false); - member.UpdateState(health); - - // apply any shared hooks - AddEventHandlers(muxer); - if (_profilingSessionProvider is not null) muxer.RegisterProfiler(_profilingSessionProvider); - lock (_suffixes) - { - foreach (var suffix in _suffixes) - { - muxer.AddLibraryNameSuffix(suffix); - } - } - - // update the members array - while (true) - { - var arr = _members; - var newArr = new ConnectionGroupMember[arr.Length + 1]; - Array.Copy(arr, 0, newArr, 0, arr.Length); - newArr[arr.Length] = member; - if (Interlocked.CompareExchange(ref _members, newArr, arr) == arr) break; - } - - OnConnectionChanged(GroupConnectionChangedEventArgs.ChangeType.Added, member); - SelectPreferredGroup(); - - // pub/sub - await AddPubSubHandlersAsync(member).ConfigureAwait(false); - } - - public bool TryFailoverTo(ConnectionGroupMember? member) - { - if (member is null) - { - // remove any explicit overrides, returning whether that was an actual change - bool result = false; - foreach (var m in _members) - { - if (m.ExplicitOverride) - { - result = true; // someone was explicitly enabled - m.ExplicitOverride = false; - } - } - SelectPreferredGroup(); - return result; - } - - var members = _members; - if (!members.Contains(member)) - { - // not one of ours? - return false; - } - - if (!member.IsConnected) - { - // not allowed - return false; - } - - if (member.ExplicitOverride) - { - // already preferred; no change, but report as success - return true; - } - - // otherwise, deselect everyone else, and select this one - foreach (var m in members) - { - m.ExplicitOverride = ReferenceEquals(m, member); - } - SelectPreferredGroup(); - return true; - } - - public bool Remove(ConnectionGroupMember member) - { - while (true) - { - var arr = _members; - int index = -1; - for (int i = 0; i < arr.Length; i++) - { - if (ReferenceEquals(arr[i], member)) - { - index = i; - break; - } - } - - if (index == -1) return false; - var newArr = new ConnectionGroupMember[arr.Length - 1]; - if (index > 0) Array.Copy(arr, 0, newArr, 0, index); - if (index < newArr.Length) Array.Copy(arr, index + 1, newArr, index, newArr.Length - index); - if (Interlocked.CompareExchange(ref _members, newArr, arr) == arr) break; - } - - var muxer = member.ClearMultiplexer(); - RemoveEventHandlers(muxer); - OnConnectionChanged(GroupConnectionChangedEventArgs.ChangeType.Removed, member); - SelectPreferredGroup(); - muxer?.Dispose(); - return true; - } - - internal void OnHeartbeat() // for testing, to update latency etc - { - foreach (var member in _members) - { - member.Multiplexer.OnHeartbeat(); - } - } - - internal void OnInternalError(Exception exception, EndPoint? endpoint = null, ConnectionType connectionType = ConnectionType.None, string? origin = null) - { - var handler = _internalError; - if (handler is not null) - { - InternalErrorEventArgs args = new(handler, this, endpoint, connectionType, exception, origin); - ConnectionMultiplexer.CompleteAsWorker(args); - } - } -} diff --git a/src/StackExchange.Redis/PhysicalBridge.cs b/src/StackExchange.Redis/PhysicalBridge.cs index 21047efa2..de9edb7de 100644 --- a/src/StackExchange.Redis/PhysicalBridge.cs +++ b/src/StackExchange.Redis/PhysicalBridge.cs @@ -133,7 +133,7 @@ public void Dispose() { isDisposed = true; // If there's anything in the backlog and we're being torn down - exfil it immediately (e.g. so all awaitables complete) - AbandonPendingBacklog(new ObjectDisposedException("Connection is being disposed")); + AbandonPendingBacklog(new ObjectDisposedException("Connection is being disposed"), null); try { _backlogAutoReset?.Set(); @@ -152,7 +152,7 @@ public void Dispose() isDisposed = true; // make damn sure we don't true to resurrect // If there's anything in the backlog and we're being torn down - exfil it immediately (e.g. so all awaitables complete) - AbandonPendingBacklog(new ObjectDisposedException("Connection is being finalized")); + AbandonPendingBacklog(new ObjectDisposedException("Connection is being finalized"), null); // shouldn't *really* touch managed objects // in a finalizer, but we need to kill that socket, @@ -192,7 +192,7 @@ private WriteResult QueueOrFailMessage(Message message) // Anything else goes in the bin - we're just not ready for you yet message.Cancel(); Multiplexer.OnMessageFaulted(message, null); - message.Complete(); + message.Complete(null); return WriteResult.NoConnectionAvailable; } @@ -200,7 +200,7 @@ private WriteResult FailDueToNoConnection(Message message) { message.Cancel(); Multiplexer.OnMessageFaulted(message, null); - message.Complete(); + message.Complete(null); return WriteResult.NoConnectionAvailable; } @@ -462,7 +462,7 @@ internal void OnConnectionFailed(PhysicalConnection connection, ConnectionFailur // If we're configured to, fail all pending backlogged messages if (Multiplexer.RawConfig.BacklogPolicy?.AbortPendingOnConnectionFailure == true) { - AbandonPendingBacklog(innerException); + AbandonPendingBacklog(innerException, connection); } if (reportNextFailure) @@ -511,12 +511,12 @@ internal void OnDisconnected(ConnectionFailureType failureType, PhysicalConnecti } } - private void AbandonPendingBacklog(Exception ex) + private void AbandonPendingBacklog(Exception ex, PhysicalConnection? connection) { while (BacklogTryDequeue(out Message? next)) { Multiplexer.OnMessageFaulted(next, ex); - next.SetExceptionAndComplete(ex, this); + next.SetExceptionAndComplete(ex, connection); } } @@ -776,7 +776,7 @@ private WriteResult WriteMessageInsideLock(PhysicalConnection physical, Message // killed the underlying connection Trace("Unable to write to server"); message.Fail(ConnectionFailureType.ProtocolFailure, null, "failure before write: " + result.ToString(), Multiplexer); - message.Complete(); + message.Complete(physical); return result; } // The parent message (next) may be returned from GetMessages @@ -996,7 +996,7 @@ private void CheckBacklogForTimeouts() // Tell the message it has failed // Note: Attempting to *avoid* reentrancy/deadlock issues by not holding the lock while completing messages. var ex = Multiplexer.GetException(WriteResult.TimeoutBeforeWrite, message, ServerEndPoint, this); - message.SetExceptionAndComplete(ex, this); + message.SetExceptionAndComplete(ex, null); } } } @@ -1072,7 +1072,7 @@ private void ProcessBacklog() } var ex = ExceptionFactory.Timeout(Multiplexer, "The message was in the backlog when connection was disposed", message, ServerEndPoint, WriteResult.TimeoutBeforeWrite, this); - message.SetExceptionAndComplete(ex, this); + message.SetExceptionAndComplete(ex, null); } } } @@ -1222,7 +1222,7 @@ private WriteResult TimedOutBeforeWrite(Message message) { message.Cancel(); Multiplexer.OnMessageFaulted(message, null); - message.Complete(); + message.Complete(null); return WriteResult.TimeoutBeforeWrite; } @@ -1355,7 +1355,7 @@ private async ValueTask CompleteWriteAndReleaseLockAsync( private WriteResult HandleWriteException(PhysicalConnection? physical, Message message, Exception ex) { var inner = new RedisConnectionException(ConnectionFailureType.InternalFailure, "Failed to write", ex); - message.SetExceptionAndComplete(inner, this); + message.SetExceptionAndComplete(inner, physical); // Tear down the physical connection. A write that throws may have left a partial frame on the // wire, and continuing to use the same socket would let the next reply match the wrong message // in the response queue. Forcing a reconnect drains the in-flight queue with failures and @@ -1588,7 +1588,7 @@ private WriteResult WriteMessageToServerInsideWriteLock(PhysicalConnection conne { Trace("Write failed: " + ex.Message); message.Fail(ConnectionFailureType.InternalFailure, ex, null, Multiplexer); - message.Complete(); + message.Complete(connection); // This failed without actually writing; we're OK with that... unless there's a transaction if (connection?.TransactionActive == true) @@ -1603,7 +1603,7 @@ private WriteResult WriteMessageToServerInsideWriteLock(PhysicalConnection conne { Trace("Write failed: " + ex.Message); message.Fail(ConnectionFailureType.InternalFailure, ex, null, Multiplexer); - message.Complete(); + message.Complete(connection); // We're not sure *what* happened here - probably an IOException; kill the connection connection?.RecordConnectionFailed(ConnectionFailureType.InternalFailure, ex); diff --git a/src/StackExchange.Redis/PhysicalConnection.Read.cs b/src/StackExchange.Redis/PhysicalConnection.Read.cs index 0de5487cd..ad6973eca 100644 --- a/src/StackExchange.Redis/PhysicalConnection.Read.cs +++ b/src/StackExchange.Redis/PhysicalConnection.Read.cs @@ -632,7 +632,7 @@ private void MatchNextResult(ReadOnlySpan frame) if (_awaitingToken is not null && (msg = Interlocked.Exchange(ref _awaitingToken, null)) is not null) { _readStatus = ReadStatus.ResponseSequenceCheck; - if (!ProcessHighIntegrityResponseToken(msg, frame, BridgeCouldBeNull)) + if (!ProcessHighIntegrityResponseToken(msg, frame, this)) { RecordConnectionFailed(ConnectionFailureType.ResponseIntegrityFailure, origin: nameof(ReadStatus.ResponseSequenceCheck)); } @@ -678,7 +678,7 @@ static void Throw(ReadOnlySpan frame, ConnectionType connection, RedisProt if (highIntegrityToken is 0) { // can't complete yet if needs checksum - msg.Complete(); + msg.Complete(this); } } else @@ -694,7 +694,7 @@ static void Throw(ReadOnlySpan frame, ConnectionType connection, RedisProt _readStatus = ReadStatus.MatchResultComplete; _activeMessage = null; - static bool ProcessHighIntegrityResponseToken(Message message, ReadOnlySpan frame, PhysicalBridge? bridge) + static bool ProcessHighIntegrityResponseToken(Message message, ReadOnlySpan frame, PhysicalConnection? connection) { bool isValid = false; var reader = new RespReader(frame); @@ -716,12 +716,12 @@ static bool ProcessHighIntegrityResponseToken(Message message, ReadOnlySpan queue, [NotNullWhen(true)] out Messa } } - private void RecordMessageFailed(Message next, Exception? ex, string? origin, PhysicalBridge? bridge) + private void RecordMessageFailed(Message next, Exception? ex, string? origin, PhysicalConnection? connection) { if (next.Command == RedisCommand.QUIT && next.TrySetResult(true)) { // fine, death of a socket is close enough - next.Complete(); + next.Complete(this); } else { - if (bridge != null) + var bridge = connection?.BridgeCouldBeNull; + if (bridge is not null) { bridge.Trace("Failing: " + next); bridge.Multiplexer?.OnMessageFaulted(next, ex, origin); } - next.SetExceptionAndComplete(ex!, bridge); + next.SetExceptionAndComplete(ex!, connection); } } @@ -594,7 +597,7 @@ internal void EnqueueInsideWriteLock(Message next, bool enforceMuxer = true) // we can still process it to avoid making things worse/more complex, // but: we can't reliably assume this works, so: shout now! next.Cancel(); - next.Complete(); + next.Complete(null); } bool wasEmpty; @@ -741,7 +744,7 @@ internal void OnBridgeHeartbeat(out int asyncTimeoutDetected, out int syncTimeou lock (_writtenAwaitingResponse) { - if (_writtenAwaitingResponse.Count != 0 && BridgeCouldBeNull is PhysicalBridge bridge) + if (_writtenAwaitingResponse.Count != 0 && BridgeCouldBeNull is { } bridge) { var server = bridge.ServerEndPoint; var multiplexer = bridge.Multiplexer; @@ -760,7 +763,7 @@ internal void OnBridgeHeartbeat(out int asyncTimeoutDetected, out int syncTimeou : $"Timeout awaiting response ({elapsed}ms elapsed, timeout is {timeout}ms)"; var timeoutEx = ExceptionFactory.Timeout(multiplexer, baseErrorMessage, msg, server); multiplexer.OnMessageFaulted(msg, timeoutEx); - msg.SetExceptionAndComplete(timeoutEx, bridge); // tell the message that it is doomed + msg.SetExceptionAndComplete(timeoutEx, this); // tell the message that it is doomed multiplexer.OnAsyncTimeout(); asyncTimeoutDetected++; } @@ -790,6 +793,14 @@ internal void OnBridgeHeartbeat(out int asyncTimeoutDetected, out int syncTimeou } } } + + // backstop for a tripped circuit-breaker: normally the trip worker actuates the teardown, but + // if it hasn't been scheduled yet (and the connection has gone quiet) the heartbeat does it. + // Must be outside the lock above - RecordConnectionFailed takes that same lock. + if (Volatile.Read(ref _circuitBreakerState) is CircuitBreakerTripped) + { + CheckCircuitBreakerTrip(); + } } internal void OnInternalError(Exception exception, [CallerMemberName] string? origin = null) @@ -1165,6 +1176,53 @@ internal bool HasPendingCallerFacingItems() if (lockTaken) Monitor.Exit(_writtenAwaitingResponse); } } + + public void ObserveMessageResult(Exception? fault) + { + // Hot path - runs per completed message. Let the breaker observe the outcome (ObserveResult + // returns true while healthy); if it trips and we're the *first* to notice, hand off the + // teardown rather than doing it inline: RecordConnectionFailed can fail the whole backlog and + // build a detailed exception, which we don't want to pay for on the completion thread. + if (circuitBreaker is { } cb && !cb.ObserveResult(fault) + && Interlocked.CompareExchange(ref _circuitBreakerState, CircuitBreakerTripped, CircuitBreakerHealthy) is CircuitBreakerHealthy) + { + // hand off to a worker; the heartbeat (see OnBridgeHeartbeat) is a backstop in case the + // pool is slow to schedule us and the connection then goes quiet - either way the actual + // teardown happens exactly once, via CheckCircuitBreakerTrip. A cached static callback with + // the connection as state avoids any per-trip delegate/closure allocation. + ThreadPool.QueueUserWorkItem(s_CheckCircuitBreakerTrip, this); + } + } + + private static readonly WaitCallback s_CheckCircuitBreakerTrip = static state => + { + var connection = (PhysicalConnection)state!; + try + { + connection.CheckCircuitBreakerTrip(); + } + catch (Exception ex) + { + connection.OnInternalError(ex); + } + }; + + // Actuates a pending circuit-breaker teardown at most once. Called from both the trip worker and + // the heartbeat backstop; whoever wins the tripped->actuated transition does the work. Routing via + // RecordConnectionFailed (not a bare Shutdown) is what surfaces the failure as a ConnectionFailed + // event, which is what Active:Active reacts to when re-routing away from this member. + private void CheckCircuitBreakerTrip() + { + if (Interlocked.CompareExchange(ref _circuitBreakerState, CircuitBreakerActuated, CircuitBreakerTripped) is CircuitBreakerTripped) + { + RecordConnectionFailed(ConnectionFailureType.CircuitBreaker); + } + } + + private CircuitBreaker.Accumulator? circuitBreaker; + + private int _circuitBreakerState; // transitions strictly Healthy -> Tripped -> Actuated + private const int CircuitBreakerHealthy = 0, CircuitBreakerTripped = 1, CircuitBreakerActuated = 2; } internal sealed class DummyHighIntegrityMessage : Message diff --git a/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt b/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt index e8acf4e02..b1d16d31f 100644 --- a/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt @@ -1,87 +1,157 @@ #nullable enable -[SER007]abstract StackExchange.Redis.HealthCheck.HealthCheckProbe.CheckHealthAsync(StackExchange.Redis.HealthCheck! healthCheck, StackExchange.Redis.IServer! server) -> System.Threading.Tasks.Task! -[SER007]abstract StackExchange.Redis.HealthCheck.HealthCheckProbePolicy.Evaluate(in StackExchange.Redis.HealthCheck.HealthCheckProbeContext context) -> StackExchange.Redis.HealthCheck.HealthCheckResult -[SER007]abstract StackExchange.Redis.HealthCheck.KeyWriteHealthCheckProbe.CheckHealthAsync(StackExchange.Redis.HealthCheck! healthCheck, StackExchange.Redis.IDatabaseAsync! database, StackExchange.Redis.RedisKey key) -> System.Threading.Tasks.Task! -[SER007]override StackExchange.Redis.HealthCheck.HealthCheckProbeContext.ToString() -> string! -[SER007]override StackExchange.Redis.HealthCheck.KeyWriteHealthCheckProbe.CheckHealthAsync(StackExchange.Redis.HealthCheck! healthCheck, StackExchange.Redis.IServer! server) -> System.Threading.Tasks.Task! -[SER007]StackExchange.Redis.HealthCheck -[SER007]StackExchange.Redis.HealthCheck.CheckHealthAsync(StackExchange.Redis.IConnectionMultiplexer! multiplexer) -> System.Threading.Tasks.Task! -[SER007]StackExchange.Redis.HealthCheck.CheckHealthAsync(StackExchange.Redis.IServer! server) -> System.Threading.Tasks.Task! -[SER007]StackExchange.Redis.HealthCheck.Clone() -> StackExchange.Redis.HealthCheck! -[SER007]StackExchange.Redis.HealthCheck.HealthCheck() -> void -[SER007]StackExchange.Redis.HealthCheck.HealthCheckProbe -[SER007]StackExchange.Redis.HealthCheck.HealthCheckProbe.HealthCheckProbe() -> void -[SER007]StackExchange.Redis.HealthCheck.HealthCheckProbeContext -[SER007]StackExchange.Redis.HealthCheck.HealthCheckProbeContext.Failure.get -> int -[SER007]StackExchange.Redis.HealthCheck.HealthCheckProbeContext.HealthCheckProbeContext() -> void -[SER007]StackExchange.Redis.HealthCheck.HealthCheckProbeContext.HealthCheckProbeContext(StackExchange.Redis.HealthCheck.HealthCheckResult result, int success, int failure, int remaining, System.TimeSpan probeInterval) -> void -[SER007]StackExchange.Redis.HealthCheck.HealthCheckProbeContext.ProbeInterval.get -> System.TimeSpan -[SER007]StackExchange.Redis.HealthCheck.HealthCheckProbeContext.Remaining.get -> int -[SER007]StackExchange.Redis.HealthCheck.HealthCheckProbeContext.Result.get -> StackExchange.Redis.HealthCheck.HealthCheckResult -[SER007]StackExchange.Redis.HealthCheck.HealthCheckProbeContext.Success.get -> int -[SER007]StackExchange.Redis.HealthCheck.HealthCheckProbePolicy -[SER007]StackExchange.Redis.HealthCheck.HealthCheckProbePolicy.HealthCheckProbePolicy() -> void -[SER007]StackExchange.Redis.HealthCheck.HealthCheckResult -[SER007]StackExchange.Redis.HealthCheck.HealthCheckResult.Healthy = 1 -> StackExchange.Redis.HealthCheck.HealthCheckResult -[SER007]StackExchange.Redis.HealthCheck.HealthCheckResult.Inconclusive = 0 -> StackExchange.Redis.HealthCheck.HealthCheckResult -[SER007]StackExchange.Redis.HealthCheck.HealthCheckResult.Unhealthy = 2 -> StackExchange.Redis.HealthCheck.HealthCheckResult -[SER007]StackExchange.Redis.HealthCheck.Interval.get -> System.TimeSpan -[SER007]StackExchange.Redis.HealthCheck.Interval.set -> void -[SER007]StackExchange.Redis.HealthCheck.KeyWriteHealthCheckProbe -[SER007]StackExchange.Redis.HealthCheck.KeyWriteHealthCheckProbe.KeyWriteHealthCheckProbe() -> void -[SER007]StackExchange.Redis.HealthCheck.Probe.get -> StackExchange.Redis.HealthCheck.HealthCheckProbe! -[SER007]StackExchange.Redis.HealthCheck.Probe.set -> void -[SER007]StackExchange.Redis.HealthCheck.ProbeCount.get -> int -[SER007]StackExchange.Redis.HealthCheck.ProbeCount.set -> void -[SER007]StackExchange.Redis.HealthCheck.ProbeInterval.get -> System.TimeSpan -[SER007]StackExchange.Redis.HealthCheck.ProbeInterval.set -> void -[SER007]StackExchange.Redis.HealthCheck.ProbePolicy.get -> StackExchange.Redis.HealthCheck.HealthCheckProbePolicy! -[SER007]StackExchange.Redis.HealthCheck.ProbePolicy.set -> void -[SER007]StackExchange.Redis.HealthCheck.ProbeTimeout.get -> System.TimeSpan -[SER007]StackExchange.Redis.HealthCheck.ProbeTimeout.set -> void +abstract StackExchange.Redis.Configuration.LoggingTunnel.Log(System.IO.Stream! stream, System.Net.EndPoint! endpoint, StackExchange.Redis.ConnectionType connectionType) -> System.IO.Stream! +override StackExchange.Redis.Configuration.LoggingTunnel.BeforeAuthenticateAsync(System.Net.EndPoint! endpoint, StackExchange.Redis.ConnectionType connectionType, System.Net.Sockets.Socket? socket, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.ValueTask +override StackExchange.Redis.Configuration.LoggingTunnel.BeforeSocketConnectAsync(System.Net.EndPoint! endPoint, StackExchange.Redis.ConnectionType connectionType, System.Net.Sockets.Socket? socket, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.ValueTask +override StackExchange.Redis.Configuration.LoggingTunnel.GetSocketConnectEndpointAsync(System.Net.EndPoint! endpoint, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.ValueTask +override StackExchange.Redis.Configuration.LoggingTunnel.LoggingDuplexStream.BeginRead(byte[]! buffer, int offset, int count, System.AsyncCallback? callback, object? state) -> System.IAsyncResult! +override StackExchange.Redis.Configuration.LoggingTunnel.LoggingDuplexStream.BeginWrite(byte[]! buffer, int offset, int count, System.AsyncCallback? callback, object? state) -> System.IAsyncResult! +override StackExchange.Redis.Configuration.LoggingTunnel.LoggingDuplexStream.CanRead.get -> bool +override StackExchange.Redis.Configuration.LoggingTunnel.LoggingDuplexStream.CanSeek.get -> bool +override StackExchange.Redis.Configuration.LoggingTunnel.LoggingDuplexStream.CanTimeout.get -> bool +override StackExchange.Redis.Configuration.LoggingTunnel.LoggingDuplexStream.CanWrite.get -> bool +override StackExchange.Redis.Configuration.LoggingTunnel.LoggingDuplexStream.Close() -> void +override StackExchange.Redis.Configuration.LoggingTunnel.LoggingDuplexStream.EndRead(System.IAsyncResult! asyncResult) -> int +override StackExchange.Redis.Configuration.LoggingTunnel.LoggingDuplexStream.EndWrite(System.IAsyncResult! asyncResult) -> void +override StackExchange.Redis.Configuration.LoggingTunnel.LoggingDuplexStream.Flush() -> void +override StackExchange.Redis.Configuration.LoggingTunnel.LoggingDuplexStream.FlushAsync(System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task! +override StackExchange.Redis.Configuration.LoggingTunnel.LoggingDuplexStream.Length.get -> long +override StackExchange.Redis.Configuration.LoggingTunnel.LoggingDuplexStream.Position.get -> long +override StackExchange.Redis.Configuration.LoggingTunnel.LoggingDuplexStream.Position.set -> void +override StackExchange.Redis.Configuration.LoggingTunnel.LoggingDuplexStream.Read(byte[]! buffer, int offset, int count) -> int +override StackExchange.Redis.Configuration.LoggingTunnel.LoggingDuplexStream.ReadAsync(byte[]! buffer, int offset, int count, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task! +override StackExchange.Redis.Configuration.LoggingTunnel.LoggingDuplexStream.ReadByte() -> int +override StackExchange.Redis.Configuration.LoggingTunnel.LoggingDuplexStream.ReadTimeout.get -> int +override StackExchange.Redis.Configuration.LoggingTunnel.LoggingDuplexStream.ReadTimeout.set -> void +override StackExchange.Redis.Configuration.LoggingTunnel.LoggingDuplexStream.Seek(long offset, System.IO.SeekOrigin origin) -> long +override StackExchange.Redis.Configuration.LoggingTunnel.LoggingDuplexStream.SetLength(long value) -> void +override StackExchange.Redis.Configuration.LoggingTunnel.LoggingDuplexStream.Write(byte[]! buffer, int offset, int count) -> void +override StackExchange.Redis.Configuration.LoggingTunnel.LoggingDuplexStream.WriteAsync(byte[]! buffer, int offset, int count, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task! +override StackExchange.Redis.Configuration.LoggingTunnel.LoggingDuplexStream.WriteByte(byte value) -> void +override StackExchange.Redis.Configuration.LoggingTunnel.LoggingDuplexStream.WriteTimeout.get -> int +override StackExchange.Redis.Configuration.LoggingTunnel.LoggingDuplexStream.WriteTimeout.set -> void +StackExchange.Redis.Configuration.LoggingTunnel +StackExchange.Redis.Configuration.LoggingTunnel.LoggingDuplexStream +StackExchange.Redis.Configuration.LoggingTunnel.LoggingTunnel(StackExchange.Redis.ConfigurationOptions? options = null, StackExchange.Redis.Configuration.Tunnel? tail = null) -> void StackExchange.Redis.IServer.InventKey(StackExchange.Redis.RedisKey prefix = default(StackExchange.Redis.RedisKey)) -> StackExchange.Redis.RedisKey -[SER007]static StackExchange.Redis.HealthCheck.Default.get -> StackExchange.Redis.HealthCheck! -[SER007]static StackExchange.Redis.HealthCheck.HealthCheckProbe.HealthyTask.get -> System.Threading.Tasks.Task! -[SER007]static StackExchange.Redis.HealthCheck.HealthCheckProbe.InconclusiveTask.get -> System.Threading.Tasks.Task! -[SER007]static StackExchange.Redis.HealthCheck.HealthCheckProbe.IsConnected.get -> StackExchange.Redis.HealthCheck.HealthCheckProbe! -[SER007]static StackExchange.Redis.HealthCheck.HealthCheckProbe.Ping.get -> StackExchange.Redis.HealthCheck.HealthCheckProbe! -[SER007]static StackExchange.Redis.HealthCheck.HealthCheckProbe.StringSet.get -> StackExchange.Redis.HealthCheck.HealthCheckProbe! -[SER007]static StackExchange.Redis.HealthCheck.HealthCheckProbe.UnhealthyTask.get -> System.Threading.Tasks.Task! -[SER007]static StackExchange.Redis.HealthCheck.HealthCheckProbePolicy.AllSuccess.get -> StackExchange.Redis.HealthCheck.HealthCheckProbePolicy! -[SER007]static StackExchange.Redis.HealthCheck.HealthCheckProbePolicy.AnySuccess.get -> StackExchange.Redis.HealthCheck.HealthCheckProbePolicy! -[SER007]static StackExchange.Redis.HealthCheck.HealthCheckProbePolicy.MajoritySuccess.get -> StackExchange.Redis.HealthCheck.HealthCheckProbePolicy! -[SER007]override StackExchange.Redis.ConnectionGroupMember.ToString() -> string! -[SER007]StackExchange.Redis.ConnectionGroupMember -[SER007]StackExchange.Redis.ConnectionGroupMember.ConnectionGroupMember(StackExchange.Redis.ConfigurationOptions! configuration, string! name = "") -> void -[SER007]StackExchange.Redis.ConnectionGroupMember.ConnectionGroupMember(string! configuration, string! name = "") -> void -[SER007]StackExchange.Redis.ConnectionGroupMember.IsConnected.get -> bool -[SER007]StackExchange.Redis.ConnectionGroupMember.Latency.get -> System.TimeSpan -[SER007]StackExchange.Redis.ConnectionGroupMember.Name.get -> string! -[SER007]StackExchange.Redis.ConnectionGroupMember.Weight.get -> double -[SER007]StackExchange.Redis.ConnectionGroupMember.Weight.set -> void -[SER007]StackExchange.Redis.GroupConnectionChangedEventArgs -[SER007]StackExchange.Redis.GroupConnectionChangedEventArgs.ChangeType -[SER007]StackExchange.Redis.GroupConnectionChangedEventArgs.ChangeType.ActiveChanged = 5 -> StackExchange.Redis.GroupConnectionChangedEventArgs.ChangeType -[SER007]StackExchange.Redis.GroupConnectionChangedEventArgs.ChangeType.Added = 1 -> StackExchange.Redis.GroupConnectionChangedEventArgs.ChangeType -[SER007]StackExchange.Redis.GroupConnectionChangedEventArgs.ChangeType.Disconnected = 3 -> StackExchange.Redis.GroupConnectionChangedEventArgs.ChangeType -[SER007]StackExchange.Redis.GroupConnectionChangedEventArgs.ChangeType.Reconnected = 4 -> StackExchange.Redis.GroupConnectionChangedEventArgs.ChangeType -[SER007]StackExchange.Redis.GroupConnectionChangedEventArgs.ChangeType.Removed = 2 -> StackExchange.Redis.GroupConnectionChangedEventArgs.ChangeType -[SER007]StackExchange.Redis.GroupConnectionChangedEventArgs.ChangeType.Unknown = 0 -> StackExchange.Redis.GroupConnectionChangedEventArgs.ChangeType -[SER007]StackExchange.Redis.GroupConnectionChangedEventArgs.Group.get -> StackExchange.Redis.ConnectionGroupMember! -[SER007]StackExchange.Redis.GroupConnectionChangedEventArgs.GroupConnectionChangedEventArgs(StackExchange.Redis.GroupConnectionChangedEventArgs.ChangeType type, StackExchange.Redis.ConnectionGroupMember! group, StackExchange.Redis.ConnectionGroupMember? previousGroup = null) -> void -[SER007]StackExchange.Redis.GroupConnectionChangedEventArgs.PreviousGroup.get -> StackExchange.Redis.ConnectionGroupMember? -[SER007]StackExchange.Redis.GroupConnectionChangedEventArgs.Type.get -> StackExchange.Redis.GroupConnectionChangedEventArgs.ChangeType -[SER007]StackExchange.Redis.IConnectionGroup -[SER007]StackExchange.Redis.IConnectionGroup.ActiveMember.get -> StackExchange.Redis.ConnectionGroupMember? -[SER007]StackExchange.Redis.IConnectionGroup.AddAsync(StackExchange.Redis.ConnectionGroupMember! member, System.IO.TextWriter? log = null) -> System.Threading.Tasks.Task! -[SER007]StackExchange.Redis.IConnectionGroup.ConnectionChanged -> System.EventHandler? -[SER007]StackExchange.Redis.IConnectionGroup.GetMembers() -> System.ReadOnlySpan -[SER007]StackExchange.Redis.IConnectionGroup.Remove(StackExchange.Redis.ConnectionGroupMember! member) -> bool -[SER007]StackExchange.Redis.IConnectionGroup.TryFailoverTo(StackExchange.Redis.ConnectionGroupMember? member) -> bool -[SER007]StackExchange.Redis.MultiGroupOptions -[SER007]StackExchange.Redis.MultiGroupOptions.HealthCheck.get -> StackExchange.Redis.HealthCheck! -[SER007]StackExchange.Redis.MultiGroupOptions.HealthCheck.set -> void -[SER007]StackExchange.Redis.MultiGroupOptions.MultiGroupOptions() -> void -[SER007]static StackExchange.Redis.ConnectionMultiplexer.ConnectGroupAsync(StackExchange.Redis.ConnectionGroupMember! member0, StackExchange.Redis.ConnectionGroupMember! member1, StackExchange.Redis.MultiGroupOptions? options = null, System.IO.TextWriter? log = null) -> System.Threading.Tasks.Task! -[SER007]static StackExchange.Redis.ConnectionMultiplexer.ConnectGroupAsync(StackExchange.Redis.ConnectionGroupMember![]! members, StackExchange.Redis.MultiGroupOptions? options = null, System.IO.TextWriter? log = null) -> System.Threading.Tasks.Task! -[SER007]static StackExchange.Redis.MultiGroupOptions.Default.get -> StackExchange.Redis.MultiGroupOptions! +static StackExchange.Redis.Configuration.LoggingTunnel.DefaultFormatCommand(StackExchange.Redis.RedisResult! value) -> string! +static StackExchange.Redis.Configuration.LoggingTunnel.DefaultFormatResponse(StackExchange.Redis.RedisResult! value) -> string! +static StackExchange.Redis.Configuration.LoggingTunnel.LogToDirectory(StackExchange.Redis.ConfigurationOptions! options, string! path) -> void +static StackExchange.Redis.Configuration.LoggingTunnel.ReplayAsync(string! path, System.Action! pair) -> System.Threading.Tasks.Task! +static StackExchange.Redis.Configuration.LoggingTunnel.ReplayAsync(System.IO.Stream! out, System.IO.Stream! in, System.Action! pair) -> System.Threading.Tasks.Task! +static StackExchange.Redis.Configuration.LoggingTunnel.ValidateAsync(string! path) -> System.Threading.Tasks.Task! +static StackExchange.Redis.Configuration.LoggingTunnel.ValidateAsync(System.IO.Stream! stream) -> System.Threading.Tasks.Task! +[SER007]abstract StackExchange.Redis.Availability.CircuitBreaker.Accumulator.IsHealthy() -> bool +[SER007]abstract StackExchange.Redis.Availability.CircuitBreaker.Accumulator.ObserveResult(in StackExchange.Redis.Availability.CircuitBreaker.CircuitBreakerContext context) -> bool +[SER007]abstract StackExchange.Redis.Availability.CircuitBreaker.Accumulator.Reset() -> void +[SER007]abstract StackExchange.Redis.Availability.CircuitBreaker.CreateAccumulator() -> StackExchange.Redis.Availability.CircuitBreaker.Accumulator! +[SER007]StackExchange.Redis.Availability.CircuitBreaker +[SER007]StackExchange.Redis.Availability.CircuitBreaker.Accumulator +[SER007]StackExchange.Redis.Availability.CircuitBreaker.Accumulator.Accumulator() -> void +[SER007]StackExchange.Redis.Availability.CircuitBreaker.Builder +[SER007]StackExchange.Redis.Availability.CircuitBreaker.Builder.Builder() -> void +[SER007]StackExchange.Redis.Availability.CircuitBreaker.Builder.Create() -> StackExchange.Redis.Availability.CircuitBreaker! +[SER007]StackExchange.Redis.Availability.CircuitBreaker.Builder.FailureRateThreshold.get -> double +[SER007]StackExchange.Redis.Availability.CircuitBreaker.Builder.FailureRateThreshold.set -> void +[SER007]StackExchange.Redis.Availability.CircuitBreaker.Builder.MetricsWindowSize.get -> System.TimeSpan +[SER007]StackExchange.Redis.Availability.CircuitBreaker.Builder.MetricsWindowSize.set -> void +[SER007]StackExchange.Redis.Availability.CircuitBreaker.Builder.MinimumNumberOfFailures.get -> int +[SER007]StackExchange.Redis.Availability.CircuitBreaker.Builder.MinimumNumberOfFailures.set -> void +[SER007]StackExchange.Redis.Availability.CircuitBreaker.Builder.TrackedExceptions.get -> System.Type![]? +[SER007]StackExchange.Redis.Availability.CircuitBreaker.Builder.TrackedExceptions.set -> void +[SER007]StackExchange.Redis.Availability.CircuitBreaker.CircuitBreaker() -> void +[SER007]StackExchange.Redis.Availability.CircuitBreaker.CircuitBreakerContext +[SER007]StackExchange.Redis.Availability.CircuitBreaker.CircuitBreakerContext.CircuitBreakerContext() -> void +[SER007]StackExchange.Redis.Availability.CircuitBreaker.CircuitBreakerContext.CircuitBreakerContext(System.Exception? fault, bool evaluate = true) -> void +[SER007]StackExchange.Redis.Availability.CircuitBreaker.CircuitBreakerContext.Fault.get -> System.Exception? +[SER007]StackExchange.Redis.Availability.CircuitBreaker.CircuitBreakerContext.Success.get -> bool +[SER007]static StackExchange.Redis.Availability.CircuitBreaker.Builder.implicit operator StackExchange.Redis.Availability.CircuitBreaker!(StackExchange.Redis.Availability.CircuitBreaker.Builder! builder) -> StackExchange.Redis.Availability.CircuitBreaker! +[SER007]StackExchange.Redis.Availability.ConnectionGroupMember +[SER007]StackExchange.Redis.Availability.ConnectionGroupMember.ConnectionGroupMember(StackExchange.Redis.ConfigurationOptions! configuration, string! name = "") -> void +[SER007]StackExchange.Redis.Availability.ConnectionGroupMember.ConnectionGroupMember(string! configuration, string! name = "") -> void +[SER007]StackExchange.Redis.Availability.ConnectionGroupMember.IsConnected.get -> bool +[SER007]StackExchange.Redis.Availability.ConnectionGroupMember.Latency.get -> System.TimeSpan +[SER007]StackExchange.Redis.Availability.ConnectionGroupMember.Name.get -> string! +[SER007]StackExchange.Redis.Availability.ConnectionGroupMember.Weight.get -> double +[SER007]StackExchange.Redis.Availability.ConnectionGroupMember.Weight.set -> void +[SER007]StackExchange.Redis.Availability.GroupConnectionChangedEventArgs +[SER007]StackExchange.Redis.Availability.GroupConnectionChangedEventArgs.ChangeType +[SER007]StackExchange.Redis.Availability.GroupConnectionChangedEventArgs.ChangeType.ActiveChanged = 5 -> StackExchange.Redis.Availability.GroupConnectionChangedEventArgs.ChangeType +[SER007]StackExchange.Redis.Availability.GroupConnectionChangedEventArgs.ChangeType.Added = 1 -> StackExchange.Redis.Availability.GroupConnectionChangedEventArgs.ChangeType +[SER007]StackExchange.Redis.Availability.GroupConnectionChangedEventArgs.ChangeType.Disconnected = 3 -> StackExchange.Redis.Availability.GroupConnectionChangedEventArgs.ChangeType +[SER007]StackExchange.Redis.Availability.GroupConnectionChangedEventArgs.ChangeType.Reconnected = 4 -> StackExchange.Redis.Availability.GroupConnectionChangedEventArgs.ChangeType +[SER007]StackExchange.Redis.Availability.GroupConnectionChangedEventArgs.ChangeType.Removed = 2 -> StackExchange.Redis.Availability.GroupConnectionChangedEventArgs.ChangeType +[SER007]StackExchange.Redis.Availability.GroupConnectionChangedEventArgs.ChangeType.Unknown = 0 -> StackExchange.Redis.Availability.GroupConnectionChangedEventArgs.ChangeType +[SER007]StackExchange.Redis.Availability.GroupConnectionChangedEventArgs.Group.get -> StackExchange.Redis.Availability.ConnectionGroupMember! +[SER007]StackExchange.Redis.Availability.GroupConnectionChangedEventArgs.GroupConnectionChangedEventArgs(StackExchange.Redis.Availability.GroupConnectionChangedEventArgs.ChangeType type, StackExchange.Redis.Availability.ConnectionGroupMember! group, StackExchange.Redis.Availability.ConnectionGroupMember? previousGroup = null) -> void +[SER007]StackExchange.Redis.Availability.GroupConnectionChangedEventArgs.PreviousGroup.get -> StackExchange.Redis.Availability.ConnectionGroupMember? +[SER007]StackExchange.Redis.Availability.GroupConnectionChangedEventArgs.Type.get -> StackExchange.Redis.Availability.GroupConnectionChangedEventArgs.ChangeType +[SER007]StackExchange.Redis.Availability.HealthCheck +[SER007]StackExchange.Redis.Availability.HealthCheck.CheckHealthAsync(StackExchange.Redis.IConnectionMultiplexer! multiplexer) -> System.Threading.Tasks.Task! +[SER007]StackExchange.Redis.Availability.HealthCheck.CheckHealthAsync(StackExchange.Redis.IServer! server) -> System.Threading.Tasks.Task! +[SER007]StackExchange.Redis.Availability.HealthCheck.Clone() -> StackExchange.Redis.Availability.HealthCheck! +[SER007]StackExchange.Redis.Availability.HealthCheck.HealthCheck() -> void +[SER007]StackExchange.Redis.Availability.HealthCheck.HealthCheckProbe +[SER007]StackExchange.Redis.Availability.HealthCheck.HealthCheckProbe.HealthCheckProbe() -> void +[SER007]StackExchange.Redis.Availability.HealthCheck.HealthCheckProbeContext +[SER007]StackExchange.Redis.Availability.HealthCheck.HealthCheckProbeContext.Failure.get -> int +[SER007]StackExchange.Redis.Availability.HealthCheck.HealthCheckProbeContext.HealthCheckProbeContext() -> void +[SER007]StackExchange.Redis.Availability.HealthCheck.HealthCheckProbeContext.HealthCheckProbeContext(StackExchange.Redis.Availability.HealthCheck.HealthCheckResult result, int success, int failure, int remaining, System.TimeSpan probeInterval) -> void +[SER007]StackExchange.Redis.Availability.HealthCheck.HealthCheckProbeContext.ProbeInterval.get -> System.TimeSpan +[SER007]StackExchange.Redis.Availability.HealthCheck.HealthCheckProbeContext.Remaining.get -> int +[SER007]StackExchange.Redis.Availability.HealthCheck.HealthCheckProbeContext.Result.get -> StackExchange.Redis.Availability.HealthCheck.HealthCheckResult +[SER007]StackExchange.Redis.Availability.HealthCheck.HealthCheckProbeContext.Success.get -> int +[SER007]StackExchange.Redis.Availability.HealthCheck.HealthCheckProbePolicy +[SER007]StackExchange.Redis.Availability.HealthCheck.HealthCheckProbePolicy.HealthCheckProbePolicy() -> void +[SER007]StackExchange.Redis.Availability.HealthCheck.HealthCheckResult +[SER007]StackExchange.Redis.Availability.HealthCheck.HealthCheckResult.Healthy = 1 -> StackExchange.Redis.Availability.HealthCheck.HealthCheckResult +[SER007]StackExchange.Redis.Availability.HealthCheck.HealthCheckResult.Inconclusive = 0 -> StackExchange.Redis.Availability.HealthCheck.HealthCheckResult +[SER007]StackExchange.Redis.Availability.HealthCheck.HealthCheckResult.Unhealthy = 2 -> StackExchange.Redis.Availability.HealthCheck.HealthCheckResult +[SER007]StackExchange.Redis.Availability.HealthCheck.Interval.get -> System.TimeSpan +[SER007]StackExchange.Redis.Availability.HealthCheck.Interval.set -> void +[SER007]StackExchange.Redis.Availability.HealthCheck.KeyWriteHealthCheckProbe +[SER007]StackExchange.Redis.Availability.HealthCheck.KeyWriteHealthCheckProbe.KeyWriteHealthCheckProbe() -> void +[SER007]StackExchange.Redis.Availability.HealthCheck.Probe.get -> StackExchange.Redis.Availability.HealthCheck.HealthCheckProbe! +[SER007]StackExchange.Redis.Availability.HealthCheck.Probe.set -> void +[SER007]StackExchange.Redis.Availability.HealthCheck.ProbeCount.get -> int +[SER007]StackExchange.Redis.Availability.HealthCheck.ProbeCount.set -> void +[SER007]StackExchange.Redis.Availability.HealthCheck.ProbeInterval.get -> System.TimeSpan +[SER007]StackExchange.Redis.Availability.HealthCheck.ProbeInterval.set -> void +[SER007]StackExchange.Redis.Availability.HealthCheck.ProbePolicy.get -> StackExchange.Redis.Availability.HealthCheck.HealthCheckProbePolicy! +[SER007]StackExchange.Redis.Availability.HealthCheck.ProbePolicy.set -> void +[SER007]StackExchange.Redis.Availability.HealthCheck.ProbeTimeout.get -> System.TimeSpan +[SER007]StackExchange.Redis.Availability.HealthCheck.ProbeTimeout.set -> void +[SER007]StackExchange.Redis.Availability.IConnectionGroup +[SER007]StackExchange.Redis.Availability.IConnectionGroup.ActiveMember.get -> StackExchange.Redis.Availability.ConnectionGroupMember? +[SER007]StackExchange.Redis.Availability.IConnectionGroup.AddAsync(StackExchange.Redis.Availability.ConnectionGroupMember! member, System.IO.TextWriter? log = null) -> System.Threading.Tasks.Task! +[SER007]StackExchange.Redis.Availability.IConnectionGroup.ConnectionChanged -> System.EventHandler? +[SER007]StackExchange.Redis.Availability.IConnectionGroup.GetMembers() -> System.ReadOnlySpan +[SER007]StackExchange.Redis.Availability.IConnectionGroup.Remove(StackExchange.Redis.Availability.ConnectionGroupMember! member) -> bool +[SER007]StackExchange.Redis.Availability.IConnectionGroup.TryFailoverTo(StackExchange.Redis.Availability.ConnectionGroupMember? member) -> bool +[SER007]StackExchange.Redis.Availability.MultiGroupOptions +[SER007]StackExchange.Redis.Availability.MultiGroupOptions.CircuitBreaker.get -> StackExchange.Redis.Availability.CircuitBreaker! +[SER007]StackExchange.Redis.Availability.MultiGroupOptions.CircuitBreaker.set -> void +[SER007]StackExchange.Redis.Availability.MultiGroupOptions.HealthCheck.get -> StackExchange.Redis.Availability.HealthCheck! +[SER007]StackExchange.Redis.Availability.MultiGroupOptions.HealthCheck.set -> void +[SER007]StackExchange.Redis.Availability.MultiGroupOptions.MultiGroupOptions() -> void +[SER007]static StackExchange.Redis.ConnectionMultiplexer.ConnectGroupAsync(StackExchange.Redis.Availability.ConnectionGroupMember! member0, StackExchange.Redis.Availability.ConnectionGroupMember! member1, StackExchange.Redis.Availability.MultiGroupOptions? options = null, System.IO.TextWriter? log = null) -> System.Threading.Tasks.Task! +[SER007]static StackExchange.Redis.ConnectionMultiplexer.ConnectGroupAsync(StackExchange.Redis.Availability.ConnectionGroupMember![]! members, StackExchange.Redis.Availability.MultiGroupOptions? options = null, System.IO.TextWriter? log = null) -> System.Threading.Tasks.Task! +[SER007]static StackExchange.Redis.Availability.CircuitBreaker.Default.get -> StackExchange.Redis.Availability.CircuitBreaker! +[SER007]static StackExchange.Redis.Availability.CircuitBreaker.None.get -> StackExchange.Redis.Availability.CircuitBreaker! +[SER007]abstract StackExchange.Redis.Availability.HealthCheck.HealthCheckProbe.CheckHealthAsync(StackExchange.Redis.Availability.HealthCheck! healthCheck, StackExchange.Redis.IServer! server) -> System.Threading.Tasks.Task! +[SER007]abstract StackExchange.Redis.Availability.HealthCheck.HealthCheckProbePolicy.Evaluate(in StackExchange.Redis.Availability.HealthCheck.HealthCheckProbeContext context) -> StackExchange.Redis.Availability.HealthCheck.HealthCheckResult +[SER007]abstract StackExchange.Redis.Availability.HealthCheck.KeyWriteHealthCheckProbe.CheckHealthAsync(StackExchange.Redis.Availability.HealthCheck! healthCheck, StackExchange.Redis.IDatabaseAsync! database, StackExchange.Redis.RedisKey key) -> System.Threading.Tasks.Task! +[SER007]override StackExchange.Redis.Availability.ConnectionGroupMember.ToString() -> string! +[SER007]override StackExchange.Redis.Availability.HealthCheck.HealthCheckProbeContext.ToString() -> string! +[SER007]override StackExchange.Redis.Availability.HealthCheck.KeyWriteHealthCheckProbe.CheckHealthAsync(StackExchange.Redis.Availability.HealthCheck! healthCheck, StackExchange.Redis.IServer! server) -> System.Threading.Tasks.Task! +[SER007]static StackExchange.Redis.Availability.HealthCheck.Default.get -> StackExchange.Redis.Availability.HealthCheck! +[SER007]static StackExchange.Redis.Availability.HealthCheck.HealthCheckProbe.HealthyTask.get -> System.Threading.Tasks.Task! +[SER007]static StackExchange.Redis.Availability.HealthCheck.HealthCheckProbe.InconclusiveTask.get -> System.Threading.Tasks.Task! +[SER007]static StackExchange.Redis.Availability.HealthCheck.HealthCheckProbe.IsConnected.get -> StackExchange.Redis.Availability.HealthCheck.HealthCheckProbe! +[SER007]static StackExchange.Redis.Availability.HealthCheck.HealthCheckProbe.Ping.get -> StackExchange.Redis.Availability.HealthCheck.HealthCheckProbe! +[SER007]static StackExchange.Redis.Availability.HealthCheck.HealthCheckProbe.StringSet.get -> StackExchange.Redis.Availability.HealthCheck.HealthCheckProbe! +[SER007]static StackExchange.Redis.Availability.HealthCheck.HealthCheckProbe.UnhealthyTask.get -> System.Threading.Tasks.Task! +[SER007]static StackExchange.Redis.Availability.HealthCheck.HealthCheckProbePolicy.AllSuccess.get -> StackExchange.Redis.Availability.HealthCheck.HealthCheckProbePolicy! +[SER007]static StackExchange.Redis.Availability.HealthCheck.HealthCheckProbePolicy.AnySuccess.get -> StackExchange.Redis.Availability.HealthCheck.HealthCheckProbePolicy! +[SER007]static StackExchange.Redis.Availability.HealthCheck.HealthCheckProbePolicy.MajoritySuccess.get -> StackExchange.Redis.Availability.HealthCheck.HealthCheckProbePolicy! +[SER007]static StackExchange.Redis.Availability.MultiGroupOptions.Default.get -> StackExchange.Redis.Availability.MultiGroupOptions! +[SER007]StackExchange.Redis.ConnectionFailureType.CircuitBreaker = 11 -> StackExchange.Redis.ConnectionFailureType \ No newline at end of file diff --git a/src/StackExchange.Redis/PublicAPI/net6.0/PublicAPI.Unshipped.txt b/src/StackExchange.Redis/PublicAPI/net6.0/PublicAPI.Unshipped.txt new file mode 100644 index 000000000..85928bcda --- /dev/null +++ b/src/StackExchange.Redis/PublicAPI/net6.0/PublicAPI.Unshipped.txt @@ -0,0 +1,6 @@ +#nullable enable +override StackExchange.Redis.Configuration.LoggingTunnel.LoggingDuplexStream.DisposeAsync() -> System.Threading.Tasks.ValueTask +override StackExchange.Redis.Configuration.LoggingTunnel.LoggingDuplexStream.Read(System.Span buffer) -> int +override StackExchange.Redis.Configuration.LoggingTunnel.LoggingDuplexStream.ReadAsync(System.Memory buffer, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.ValueTask +override StackExchange.Redis.Configuration.LoggingTunnel.LoggingDuplexStream.Write(System.ReadOnlySpan buffer) -> void +override StackExchange.Redis.Configuration.LoggingTunnel.LoggingDuplexStream.WriteAsync(System.ReadOnlyMemory buffer, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.ValueTask \ No newline at end of file diff --git a/src/StackExchange.Redis/RedisBatch.cs b/src/StackExchange.Redis/RedisBatch.cs index beda224af..c827ba1dc 100644 --- a/src/StackExchange.Redis/RedisBatch.cs +++ b/src/StackExchange.Redis/RedisBatch.cs @@ -116,13 +116,13 @@ internal override Task ExecuteAsync(Message? message, ResultProcessor? internal override T ExecuteSync(Message? message, ResultProcessor? processor, ServerEndPoint? server = null, T? defaultValue = default) where T : default => throw new NotSupportedException("ExecuteSync cannot be used inside a batch"); - private static void FailNoServer(ConnectionMultiplexer muxer, List messages) + private static void FailNoServer(ConnectionMultiplexer muxer, List? messages) { - if (messages == null) return; + if (messages is null) return; foreach (var msg in messages) { msg.Fail(ConnectionFailureType.UnableToResolvePhysicalConnection, null, "unable to write batch", muxer); - msg.Complete(); + msg.Complete(null); } } } diff --git a/src/StackExchange.Redis/RedisTransaction.cs b/src/StackExchange.Redis/RedisTransaction.cs index 0c369e988..5d5430671 100644 --- a/src/StackExchange.Redis/RedisTransaction.cs +++ b/src/StackExchange.Redis/RedisTransaction.cs @@ -237,17 +237,18 @@ public TransactionMessage(int db, CommandFlags flags, List? con this.conditions = (conditions?.Count > 0) ? conditions.ToArray() : Array.Empty(); } - internal override void SetExceptionAndComplete(Exception exception, PhysicalBridge? bridge) + internal override void SetExceptionAndComplete(Exception exception, PhysicalConnection? connection) { var inner = InnerOperations; - if (inner != null) + // ReSharper disable once ConditionIsAlwaysTrueOrFalseAccordingToNullableAPIContract + if (inner is not null) { for (int i = 0; i < inner.Length; i++) { - inner[i]?.Wrapped?.SetExceptionAndComplete(exception, bridge); + inner[i]?.Wrapped?.SetExceptionAndComplete(exception, connection); } } - base.SetExceptionAndComplete(exception, bridge); + base.SetExceptionAndComplete(exception, connection); } public bool IsAborted => command != RedisCommand.EXEC; @@ -432,7 +433,7 @@ public IEnumerable GetMessages(PhysicalConnection connection) { var inner = op.Wrapped; inner.Cancel(); - inner.Complete(); + inner.Complete(connection); } } connection.Trace("End of transaction: " + Command); @@ -484,7 +485,7 @@ public override bool SetResult(PhysicalConnection connection, Message message, r { var inner = op.Wrapped; ServerFail(inner, error); - inner.Complete(); + inner.Complete(connection); } } return base.SetResult(connection, message, ref copy); @@ -508,7 +509,7 @@ protected override bool SetResultCore(PhysicalConnection connection, Message mes { var inner = op.Wrapped; inner.Cancel(); - inner.Complete(); + inner.Complete(connection); } } SetResult(message, false); @@ -539,7 +540,7 @@ protected override bool SetResultCore(PhysicalConnection connection, Message mes muxer?.OnTransactionLog($"> got {iter.Value.GetOverview()} for {inner.CommandAndKey}"); if (inner.ComputeResult(connection, ref iter.Value)) { - inner.Complete(); + inner.Complete(connection); } } @@ -553,10 +554,10 @@ protected override bool SetResultCore(PhysicalConnection connection, Message mes // the pending tasks foreach (var op in wrapped) { - if (op?.Wrapped is Message inner) + if (op?.Wrapped is { } inner) { inner.Fail(ConnectionFailureType.ProtocolFailure, null, "Transaction failure", muxer); - inner.Complete(); + inner.Complete(connection); } } } diff --git a/src/StackExchange.Redis/ResultBox.cs b/src/StackExchange.Redis/ResultBox.cs index 20b76ba15..160493cc1 100644 --- a/src/StackExchange.Redis/ResultBox.cs +++ b/src/StackExchange.Redis/ResultBox.cs @@ -6,6 +6,7 @@ namespace StackExchange.Redis { internal interface IResultBox { + Exception? Fault { get; } bool IsAsync { get; } bool IsFaulted { get; } void SetException(Exception ex); @@ -24,6 +25,7 @@ internal abstract class SimpleResultBox : IResultBox bool IResultBox.IsAsync => false; bool IResultBox.IsFaulted => _exception != null; + Exception? IResultBox.Fault => _exception; void IResultBox.SetException(Exception exception) => _exception = exception ?? CancelledException; void IResultBox.Cancel() => _exception = CancelledException; @@ -95,6 +97,7 @@ private TaskResultBox(object? asyncState, TaskCreationOptions creationOptions) : bool IResultBox.IsAsync => true; bool IResultBox.IsFaulted => _exception != null; + Exception? IResultBox.Fault => _exception; void IResultBox.Cancel() => _exception = SimpleResultBox.CancelledException; diff --git a/src/StackExchange.Redis/ServerEndPoint.cs b/src/StackExchange.Redis/ServerEndPoint.cs index c6abbd733..eefe22ed5 100644 --- a/src/StackExchange.Redis/ServerEndPoint.cs +++ b/src/StackExchange.Redis/ServerEndPoint.cs @@ -10,6 +10,7 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; +using StackExchange.Redis.Availability; using static StackExchange.Redis.PhysicalBridge; namespace StackExchange.Redis diff --git a/src/StackExchange.Redis/StackExchange.Redis.csproj b/src/StackExchange.Redis/StackExchange.Redis.csproj index addda59b7..5681b41dc 100644 --- a/src/StackExchange.Redis/StackExchange.Redis.csproj +++ b/src/StackExchange.Redis/StackExchange.Redis.csproj @@ -60,11 +60,14 @@ - + MultiGroupDatabase.cs - + HealthCheck.cs + + MultiGroupSubscriber.cs + \ No newline at end of file diff --git a/tests/StackExchange.Redis.Tests/ActiveActiveIntegrationTests.cs b/tests/StackExchange.Redis.Tests/ActiveActiveIntegrationTests.cs index fedf41bd3..406999a27 100644 --- a/tests/StackExchange.Redis.Tests/ActiveActiveIntegrationTests.cs +++ b/tests/StackExchange.Redis.Tests/ActiveActiveIntegrationTests.cs @@ -1,5 +1,6 @@ using System; using System.Threading.Tasks; +using StackExchange.Redis.Availability; using StackExchange.Redis.Tests.Helpers; using Xunit; diff --git a/tests/StackExchange.Redis.Tests/CircuitBreakerServerTests.cs b/tests/StackExchange.Redis.Tests/CircuitBreakerServerTests.cs new file mode 100644 index 000000000..074dafcd9 --- /dev/null +++ b/tests/StackExchange.Redis.Tests/CircuitBreakerServerTests.cs @@ -0,0 +1,83 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using StackExchange.Redis.Availability; +using Xunit; + +namespace StackExchange.Redis.Tests; + +public class CircuitBreakerServerTests(ITestOutputHelper output) : TestBase(output) +{ + [Fact] + public async Task CircuitBreakerObservesMessageResults() + { + using var server = new InProcessTestServer(); + + // take the template options from the server, and slot in our test breaker *before* connecting, + // so every physical connection yanks an accumulator from it during init + var config = server.GetClientConfig(); + var breaker = new CountingCircuitBreaker(); + config.CircuitBreaker = breaker; + + using var client = await ConnectionMultiplexer.ConnectAsync(config); + var db = client.GetDatabase(); + + // some successful operations (these, plus handshake traffic, count as non-fault observations) + RedisKey key = Me(); + await db.StringSetAsync(key, "abc"); + Assert.Equal("abc", await db.StringGetAsync(key)); + await db.StringGetAsync(key); + + var successesAfterGetSets = breaker.Successes; + + // and something the server doesn't support -> server replies with an error -> observed as a fault + var fault = await Assert.ThrowsAnyAsync(() => db.ExecuteAsync("nonesuch")); + Output.WriteLine($"execute fault: {fault.GetType().Name}: {fault.Message}"); + + Output.WriteLine($"observed successes={breaker.Successes}, failures={breaker.Failures}, lastFault={breaker.LastFault?.GetType().Name}"); + + // the get/sets were observed as successes + Assert.True(successesAfterGetSets > 0, "expected the successful operations to be observed"); + + // exactly one fault (the unsupported command), captured as the clean server error. A healthy + // breaker must NOT tear the connection down: a regression there shows up here as a + // RedisConnectionException (and extra faults) rather than this RedisServerException. + Assert.Equal(1, breaker.Failures); + Assert.IsType(breaker.LastFault); + } + + // a minimal breaker for tests: shares counters across all accumulators it creates, so we can + // observe traffic across every physical connection; it never trips (always reports healthy) + private sealed class CountingCircuitBreaker : CircuitBreaker + { + private int _successes, _failures; + + public int Successes => Volatile.Read(ref _successes); + public int Failures => Volatile.Read(ref _failures); + public Exception? LastFault { get; private set; } + + public override Accumulator CreateAccumulator() => new CountingAccumulator(this); + + private sealed class CountingAccumulator(CountingCircuitBreaker owner) : Accumulator + { + public override bool ObserveResult(in CircuitBreakerContext context) + { + if (context.Success) + { + Interlocked.Increment(ref owner._successes); + } + else + { + Interlocked.Increment(ref owner._failures); + owner.LastFault = context.Fault; + } + + return true; // stay healthy; we're only here to observe, not to trip the connection + } + + public override bool IsHealthy() => true; + + public override void Reset() { } + } + } +} diff --git a/tests/StackExchange.Redis.Tests/CircuitBreakerUnitTests.cs b/tests/StackExchange.Redis.Tests/CircuitBreakerUnitTests.cs new file mode 100644 index 000000000..a7c25a488 --- /dev/null +++ b/tests/StackExchange.Redis.Tests/CircuitBreakerUnitTests.cs @@ -0,0 +1,239 @@ +using System; +using StackExchange.Redis.Availability; +using Xunit; +#if NET8_0_OR_GREATER +using System.Threading; +#endif + +namespace StackExchange.Redis.Tests; + +public class CircuitBreakerUnitTests +{ + [Fact] + public void Builder_AllDefaults_ReturnsSharedDefaultInstance() + { + // a builder that hasn't been touched should collapse onto the shared default instance... + var a = new CircuitBreaker.Builder().Create(); + var b = new CircuitBreaker.Builder().Create(); + + Assert.Same(a, b); + Assert.Same(CircuitBreaker.Default, a); + } + + [Fact] + public void Builder_NonDefaults_ReturnsDistinctValidInstances() + { + // ...but as soon as any knob is changed, we get a fresh, distinct instance per Create() + CircuitBreaker.Builder Configured() => new() { FailureRateThreshold = 42 }; + + var a = Configured().Create(); + var b = Configured().Create(); + + Assert.NotNull(a); + Assert.NotNull(b); + Assert.NotSame(a, b); + Assert.NotSame(CircuitBreaker.Default, a); + } + + [Fact] + public void None_IsDistinctFromDefault_ButStable() + { + Assert.NotNull(CircuitBreaker.None); + Assert.Same(CircuitBreaker.None, CircuitBreaker.None); + Assert.NotSame(CircuitBreaker.Default, CircuitBreaker.None); + } + + [Fact] + public void None_IsAlwaysHealthy() + { + var acc = CircuitBreaker.None.CreateAccumulator(); + // even a solid wall of tracked failures never trips the no-op breaker + Assert.True(Record(acc, 10_000, new RedisTimeoutException("boom", CommandStatus.Unknown))); + } + + [Fact] + public void NonEvaluatingObservation_IsNeverTreatedAsUnhealthy() + { + // a deliberately naive breaker whose accumulator *always* reports unhealthy - e.g. one that + // returns default(false) whether or not it was actually asked to evaluate. A success is + // observed without evaluation, so that bogus verdict must be ignored (not read as a trip); + // only a genuine evaluation (a fault) is allowed to report unhealthy. + var acc = new AlwaysUnhealthyBreaker().CreateAccumulator(); + + Assert.True(acc.ObserveResult((Exception?)null)); // success -> not evaluating -> verdict ignored + Assert.False(acc.ObserveResult(Timeout())); // fault -> evaluating -> verdict honoured + } + + // never reports healthy, even when not evaluating; stands in for a buggy/naive custom breaker + private sealed class AlwaysUnhealthyBreaker : CircuitBreaker + { + public override Accumulator CreateAccumulator() => new Acc(); + + private sealed class Acc : Accumulator + { + public override bool ObserveResult(in CircuitBreakerContext context) => false; + public override bool IsHealthy() => false; + public override void Reset() { } + } + } + +#if NET8_0_OR_GREATER + // the time-windowed logic needs a controllable clock; TimeProvider is only available on net8.0+, + // and we don't want to pull the BCL shim in just for down-level test coverage. + [Fact] + public void BelowMinimumFailures_StaysHealthy() + { + var time = new ManualTimeProvider(); + // threshold is trivially low (1%), so the *only* thing keeping us healthy is the minimum-count gate + var acc = Build(time, failureRateThreshold: 1, minimumNumberOfFailures: 10).CreateAccumulator(); + + // nine tracked failures: one short of the minimum, so we withhold judgement + Assert.True(Record(acc, 9, Timeout())); + } + + [Fact] + public void AboveThreshold_Trips() + { + var time = new ManualTimeProvider(); + var acc = Build(time, failureRateThreshold: 50, minimumNumberOfFailures: 10).CreateAccumulator(); + + // 20 tracked failures, 0 successes -> 100% failure rate, well past both gates + Assert.False(Record(acc, 20, Timeout())); + } + + [Fact] + public void BelowThreshold_StaysHealthy() + { + var time = new ManualTimeProvider(); + var acc = Build(time, failureRateThreshold: 50, minimumNumberOfFailures: 10).CreateAccumulator(); + + Record(acc, 10, Timeout()); // enough failures to clear the minimum-count gate + Record(acc, 190); // but drowned out by successes -> 5% failure rate + + // a pure health read confirms we're comfortably under the 50% threshold + Assert.True(acc.IsHealthy()); + } + + [Fact] + public void UntrackedExceptions_CountAsSuccess() + { + var time = new ManualTimeProvider(); + // default tracking set (null) == RedisConnectionException + RedisTimeoutException only + var acc = Build(time, failureRateThreshold: 1, minimumNumberOfFailures: 1).CreateAccumulator(); + + // a flood of *untracked* failures must not trip the breaker... + Assert.True(Record(acc, 100, new InvalidOperationException("not tracked"))); + + // ...whereas the same volume of tracked failures does + Assert.False(Record(acc, 100, Timeout())); + } + + [Fact] + public void CustomTrackedExceptions_AreHonoured() + { + var time = new ManualTimeProvider(); + var acc = Build( + time, + failureRateThreshold: 1, + minimumNumberOfFailures: 1, + trackedExceptions: [typeof(InvalidOperationException)]).CreateAccumulator(); + + // the default Redis faults are now *un*tracked, so they read as success + Assert.True(Record(acc, 100, Timeout())); + + // whereas our nominated type trips it + Assert.False(Record(acc, 100, new InvalidOperationException("tracked"))); + } + + [Fact] + public void OldFailures_AgeOutOfTheWindow() + { + var time = new ManualTimeProvider(); + var acc = Build(time, failureRateThreshold: 50, minimumNumberOfFailures: 1).CreateAccumulator(); + + // saturate the window with failures -> tripped + Assert.False(Record(acc, 100, Timeout())); + + // step past the whole window; the earlier failures should no longer count + time.Advance(TimeSpan.FromSeconds(11)); + + // the window is now empty of in-range failures -> healthy again + Assert.True(acc.IsHealthy()); + } + + [Fact] + public void IsHealthy_ReflectsStateWithoutObserving() + { + var time = new ManualTimeProvider(); + var acc = Build(time, failureRateThreshold: 50, minimumNumberOfFailures: 1).CreateAccumulator(); + + Assert.True(acc.IsHealthy()); // nothing observed yet + + Assert.False(Record(acc, 100, Timeout())); // trip it via observations + + // the context-free overload reports the same verdict, purely by reading the window + Assert.False(acc.IsHealthy()); + } + + [Fact] + public void Reset_DiscardsHistory() + { + var time = new ManualTimeProvider(); + var acc = Build(time, failureRateThreshold: 50, minimumNumberOfFailures: 1).CreateAccumulator(); + + // trip it wide open... + Assert.False(Record(acc, 100, Timeout())); + + // ...then wipe the slate; the prior failures are forgotten + acc.Reset(); + + // an empty window reads as healthy again + Assert.True(acc.IsHealthy()); + } + + private static CircuitBreaker Build( + TimeProvider time, + double failureRateThreshold, + int minimumNumberOfFailures, + Type[]? trackedExceptions = null) + => new CircuitBreaker.Builder + { + FailureRateThreshold = failureRateThreshold, + MinimumNumberOfFailures = minimumNumberOfFailures, + MetricsWindowSize = TimeSpan.FromSeconds(10), + TrackedExceptions = trackedExceptions, + TimeProvider = time, + }.Create(); + + /// + /// A hand-cranked whose clock only moves when we tell it to, + /// so the bucketed metrics window is fully deterministic. + /// + private sealed class ManualTimeProvider : TimeProvider + { + private long _timestamp; + + // one tick == 100ns, matching TimeSpan; keeps Advance(TimeSpan) a straight addition + public override long TimestampFrequency => TimeSpan.TicksPerSecond; + + public override long GetTimestamp() => Interlocked.Read(ref _timestamp); + + public void Advance(TimeSpan by) => Interlocked.Add(ref _timestamp, by.Ticks); + } +#endif + + private static RedisTimeoutException Timeout() => new("timeout", CommandStatus.Unknown); + + private static bool Record(CircuitBreaker.Accumulator accumulator, int count, Exception? fault = null) + { + // success/failure is derived from the presence of a fault; pass a fault for a failure, none for a success + var context = new CircuitBreaker.CircuitBreakerContext(fault); + bool healthy = true; + for (int i = 0; i < count; i++) + { + healthy = accumulator.ObserveResult(in context); + } + + return healthy; + } +} diff --git a/tests/StackExchange.Redis.Tests/ConfigTests.cs b/tests/StackExchange.Redis.Tests/ConfigTests.cs index 89ffd851e..93f0fbc1e 100644 --- a/tests/StackExchange.Redis.Tests/ConfigTests.cs +++ b/tests/StackExchange.Redis.Tests/ConfigTests.cs @@ -69,6 +69,7 @@ orderby name "CertificateSelection", "CertificateValidation", "ChannelPrefix", + "CircuitBreaker", "ClientName", "commandMap", "configChannel", diff --git a/tests/StackExchange.Redis.Tests/HealthCheckPolicyUnitTests.cs b/tests/StackExchange.Redis.Tests/HealthCheckPolicyUnitTests.cs index b2dd97b71..1bbcebb4e 100644 --- a/tests/StackExchange.Redis.Tests/HealthCheckPolicyUnitTests.cs +++ b/tests/StackExchange.Redis.Tests/HealthCheckPolicyUnitTests.cs @@ -1,6 +1,6 @@ using System; using Xunit; -using static StackExchange.Redis.HealthCheck; +using static StackExchange.Redis.Availability.HealthCheck; namespace StackExchange.Redis.Tests; diff --git a/tests/StackExchange.Redis.Tests/MultiGroupTests/BasicMultiGroupTests.cs b/tests/StackExchange.Redis.Tests/MultiGroupTests/BasicMultiGroupTests.cs index 707250859..a99fcd724 100644 --- a/tests/StackExchange.Redis.Tests/MultiGroupTests/BasicMultiGroupTests.cs +++ b/tests/StackExchange.Redis.Tests/MultiGroupTests/BasicMultiGroupTests.cs @@ -3,6 +3,7 @@ using System.IO; using System.Net; using System.Threading.Tasks; +using StackExchange.Redis.Availability; using StackExchange.Redis.Tests.Helpers; using Xunit; diff --git a/tests/StackExchange.Redis.Tests/MultiGroupTests/CircuitBreakerRerouteTests.cs b/tests/StackExchange.Redis.Tests/MultiGroupTests/CircuitBreakerRerouteTests.cs new file mode 100644 index 000000000..18436db46 --- /dev/null +++ b/tests/StackExchange.Redis.Tests/MultiGroupTests/CircuitBreakerRerouteTests.cs @@ -0,0 +1,132 @@ +using System; +using System.Net; +using System.Threading; +using System.Threading.Tasks; +using StackExchange.Redis.Availability; +using Xunit; + +namespace StackExchange.Redis.Tests.MultiGroupTests; + +[RunPerProtocol] +public class CircuitBreakerRerouteTests(ITestOutputHelper log) +{ + // A circuit-breaker trip must steer the group away from the affected member *promptly* - i.e. via + // the shim's ConnectionFailed(CircuitBreaker) fast-path, not by waiting for the next health-check + // poll. To isolate that mechanism we make the poll interval enormous, so the *only* thing that can + // reroute inside the test window is the fast-path; and we hold the tripped member unhealthy via a + // controllable probe, so the reroute is deterministic even though the physical connection reconnects + // immediately after being torn down. + [Fact] + public async Task CircuitBreakerTrip_ReroutesAwayFromMember() + { + EndPoint alpha = new DnsEndPoint("alpha", 6379); + EndPoint bravo = new DnsEndPoint("bravo", 6379); + EndPoint charlie = new DnsEndPoint("charlie", 6379); + + using var serverA = new InProcessTestServer(log, endpoint: alpha); + using var serverB = new InProcessTestServer(log, endpoint: bravo); + using var serverC = new InProcessTestServer(log, endpoint: charlie); + + var breaker = new FlipBreaker(); + var probe = new ControllableProbe(); + + // only member A carries the trippable breaker; B and C are left with the default (which requires + // an implausible number of failures to trip, so they never interfere) + var configA = serverA.GetClientConfig(); + configA.CircuitBreaker = breaker; + + ConnectionGroupMember[] members = + [ + new(configA, "A") { Weight = 9 }, // highest weight -> initially active + new(serverB.GetClientConfig(), "B") { Weight = 3 }, // preferred failover target + new(serverC.GetClientConfig(), "C") { Weight = 1 }, + ]; + + var options = new MultiGroupOptions + { + HealthCheck = new HealthCheck + { + Probe = probe, + // enormous, so the poll loop cannot be what reroutes us during the test + Interval = TimeSpan.FromMinutes(30), + ProbeCount = 1, + ProbeTimeout = TimeSpan.FromSeconds(5), + }, + }; + + await using var conn = await ConnectionMultiplexer.ConnectGroupAsync(members, options); + var typed = Assert.IsType(conn); + + int circuitBreakerEvents = 0; + typed.ConnectionFailed += (_, e) => + { + log.WriteLine($"ConnectionFailed: {e.FailureType} @ {e.EndPoint}"); + if (e.FailureType == ConnectionFailureType.CircuitBreaker) + { + Interlocked.Increment(ref circuitBreakerEvents); + } + }; + + // sanity: A (highest weight) is the active member to begin with + Assert.True(conn.IsConnected); + Assert.Same(members[0], conn.ActiveMember); + + // arm the breaker and hold A unhealthy, then drive a *faulting* command to the active member (A): + // the fault is what the breaker evaluates, and a tripped breaker tears the connection down + probe.MarkDown(alpha); + breaker.Trip(); + var db = conn.GetDatabase(); + var fault = await Assert.ThrowsAnyAsync(() => db.ExecuteAsync("nonesuch")); + log.WriteLine($"observed fault: {fault.GetType().Name}: {fault.Message}"); + + // wait (briefly) for the fast-path to react; nothing else can move us within this window + var moved = await WaitForActiveAsync(conn, notMember: members[0], timeout: TimeSpan.FromSeconds(5)); + + Assert.True(moved, "expected the circuit-breaker trip to reroute away from member A"); + Assert.Same(members[1], conn.ActiveMember); // B is the next-highest weight + Assert.True(circuitBreakerEvents > 0, "expected a ConnectionFailed event with FailureType == CircuitBreaker"); + } + + private static async Task WaitForActiveAsync( + IConnectionGroup conn, ConnectionGroupMember notMember, TimeSpan timeout) + { + var deadline = DateTime.UtcNow + timeout; + while (DateTime.UtcNow < deadline) + { + if (!ReferenceEquals(conn.ActiveMember, notMember) && conn.ActiveMember is not null) + { + return true; + } + await Task.Delay(50, TestContext.Current.CancellationToken); + } + return false; + } + + // trips on demand: healthy until Trip() is called; the trip only actuates on a *fault* observation + // (successes are never evaluated, by design), which the test provides via a bad command + private sealed class FlipBreaker : CircuitBreaker + { + private volatile bool _tripped; + public void Trip() => _tripped = true; + + public override Accumulator CreateAccumulator() => new Acc(this); + + private sealed class Acc(FlipBreaker owner) : Accumulator + { + public override bool ObserveResult(in CircuitBreakerContext context) => !owner._tripped; + public override bool IsHealthy() => !owner._tripped; + public override void Reset() { } + } + } + + // reports the nominated endpoints unhealthy on demand; everything else is healthy. This keeps the + // tripped member deselected even after its physical connection reconnects. + private sealed class ControllableProbe : HealthCheck.HealthCheckProbe + { + private volatile EndPoint? _down; + public void MarkDown(EndPoint endpoint) => _down = endpoint; + + public override Task CheckHealthAsync(HealthCheck healthCheck, IServer server) + => Equals(server.EndPoint, _down) ? UnhealthyTask : HealthyTask; + } +} diff --git a/tests/StackExchange.Redis.Tests/ResultBoxTests.cs b/tests/StackExchange.Redis.Tests/ResultBoxTests.cs index adb1b309f..99c8358e8 100644 --- a/tests/StackExchange.Redis.Tests/ResultBoxTests.cs +++ b/tests/StackExchange.Redis.Tests/ResultBoxTests.cs @@ -46,7 +46,7 @@ public void SyncResultBox() Assert.Equal(0, Volatile.Read(ref activated)); // check that complete signals continuation - msg.Complete(); + msg.Complete(null); Thread.Sleep(100); Assert.Equal(1, Volatile.Read(ref activated)); @@ -81,7 +81,7 @@ public void TaskResultBox() Assert.False(tcs.Task.IsCompleted); // check that complete signals continuation - msg.Complete(); + msg.Complete(null); Thread.Sleep(100); Assert.True(tcs.Task.IsCompleted);