From b45fbd24f5a494b99ffef6c5c0b8572c909cca7f Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Mon, 13 Jul 2026 16:07:10 +0100 Subject: [PATCH 01/23] start dedup core --- .../AutoDatabaseGenerator.cs | 135 ++++++++++++++++++ eng/StackExchange.Redis.Build/BasicArray.cs | 4 +- .../Availability/RetryDatabase.cs | 15 ++ 3 files changed, 152 insertions(+), 2 deletions(-) create mode 100644 eng/StackExchange.Redis.Build/AutoDatabaseGenerator.cs create mode 100644 src/StackExchange.Redis/Availability/RetryDatabase.cs diff --git a/eng/StackExchange.Redis.Build/AutoDatabaseGenerator.cs b/eng/StackExchange.Redis.Build/AutoDatabaseGenerator.cs new file mode 100644 index 000000000..53511db1a --- /dev/null +++ b/eng/StackExchange.Redis.Build/AutoDatabaseGenerator.cs @@ -0,0 +1,135 @@ +using System.Collections.Immutable; +using System.Text; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp.Syntax; + +// readable names for the info we gather; kept as tuples (+ BasicArray) so they have +// value equality and play nicely with incremental generator caching. +using ParamInfo = (string Name, string Type, Microsoft.CodeAnalysis.RefKind RefKind, bool IsParams, bool IsOptional, bool HasDefault, string? Default); +using MethodInfo = (string Name, string ReturnType, StackExchange.Redis.Build.BasicArray<(string Name, string Type, Microsoft.CodeAnalysis.RefKind RefKind, bool IsParams, bool IsOptional, bool HasDefault, string? Default)> Parameters); +using InterfaceInfo = (string Name, string Namespace, StackExchange.Redis.Build.BasicArray<(string Name, string ReturnType, StackExchange.Redis.Build.BasicArray<(string Name, string Type, Microsoft.CodeAnalysis.RefKind RefKind, bool IsParams, bool IsOptional, bool HasDefault, string? Default)> Parameters)> Methods); + +namespace StackExchange.Redis.Build; + +[Generator(LanguageNames.CSharp)] +public class AutoDatabaseGenerator : IIncrementalGenerator +{ + public void Initialize(IncrementalGeneratorInitializationContext ctx) + { + var interfaces = ctx.SyntaxProvider + .CreateSyntaxProvider( + static (node, _) => node is InterfaceDeclarationSyntax decl && FastFilter(decl), ExtractInterfaceMethods) + .Where(pair => pair.Name is { Length: > 0 }) + .Collect(); + + ctx.RegisterSourceOutput(interfaces, static (ctx, content) => Generate(ctx, content)); + } + + private static bool FastFilter(InterfaceDeclarationSyntax decl) // limit to IDatabase, IDatabaseAsync + => decl.Identifier.ValueText is "IDatabase" or "IDatabaseAsync"; + + private static InterfaceInfo ExtractInterfaceMethods(GeneratorSyntaxContext context, CancellationToken cancel) + { + // note: we deliberately do NOT interpret anything here - just capture the raw shape of every + // method (name, return type, and per-parameter name/type/modifiers/optionality/default) so that + // later passes have everything they might need. + if (context.SemanticModel.GetDeclaredSymbol(context.Node, cancel) is not INamedTypeSymbol + { + TypeKind: TypeKind.Interface, + Name: "IDatabase" or "IDatabaseAsync", + ContainingType: null, + ContainingNamespace: + { + Name: "Redis", + ContainingNamespace: + { + Name: "StackExchange", + ContainingNamespace.IsGlobalNamespace: true + } + } + } iface) + { + return default; + } + + var methods = ImmutableArray.CreateBuilder(); + foreach (var member in iface.GetMembers()) + { + cancel.ThrowIfCancellationRequested(); + if (member is not IMethodSymbol { MethodKind: MethodKind.Ordinary } method) continue; + + var parameters = new BasicArray.Builder(method.Parameters.Length); + foreach (var p in method.Parameters) + { + parameters.Add(( + Name: p.Name, + Type: p.Type.ToDisplayString(SymbolDisplayFormat.CSharpErrorMessageFormat), + RefKind: p.RefKind, + IsParams: p.IsParams, + IsOptional: p.IsOptional, + HasDefault: p.HasExplicitDefaultValue, + Default: p.HasExplicitDefaultValue ? FormatDefault(p.ExplicitDefaultValue) : null)); + } + + methods.Add(( + Name: method.Name, + ReturnType: method.ReturnType.ToDisplayString(SymbolDisplayFormat.CSharpErrorMessageFormat), + Parameters: parameters.Build())); + } + + var ns = iface.ContainingNamespace.ToDisplayString(SymbolDisplayFormat.CSharpErrorMessageFormat); + var methodArray = new BasicArray.Builder(methods.Count); + foreach (var m in methods) methodArray.Add(m); + return (iface.Name, ns, methodArray.Build()); + } + + private static string FormatDefault(object? value) => value switch + { + null => "null", + string s => $"\"{s}\"", + bool b => b ? "true" : "false", + _ => value.ToString() ?? "null", + }; + + private static void Generate(SourceProductionContext ctx, ImmutableArray interfaces) + { + if (interfaces.IsDefaultOrEmpty) return; // nothing to do + + // each interface is declared across several partial files, so we get one (identical) entry per + // partial declaration; structural equality lets us collapse those down to one per interface. + var distinct = interfaces.Distinct(); + + var sb = new StringBuilder(); + sb.AppendLine("// "); + sb.AppendLine("// AutoDatabaseGenerator diagnostic dump - what ExtractInterfaceMethods found."); + sb.AppendLine("// This file is informational only (everything is in comments); no real code is emitted yet."); + sb.AppendLine(); + + foreach (var iface in distinct) + { + sb.Append("// ==== ").Append(iface.Namespace).Append('.').Append(iface.Name) + .Append(" (").Append(iface.Methods.Length).AppendLine(" methods) ===="); + foreach (var method in iface.Methods.Span) + { + sb.Append("// ").Append(method.ReturnType).Append(' ').Append(method.Name).AppendLine("("); + for (int i = 0; i < method.Parameters.Length; i++) + { + ref readonly var p = ref method.Parameters[i]; + sb.Append("// [").Append(i).Append("] "); + if (p.RefKind != RefKind.None) sb.Append(p.RefKind.ToString().ToLowerInvariant()).Append(' '); + if (p.IsParams) sb.Append("params "); + sb.Append(p.Type).Append(' ').Append(p.Name); + if (p.IsOptional) sb.Append(" [optional]"); + if (p.HasDefault) sb.Append(" = ").Append(p.Default); + sb.AppendLine(); + } + + sb.AppendLine("// )"); + } + + sb.AppendLine("//"); + } + + ctx.AddSource("AutoDatabase.Diagnostics.g.cs", sb.ToString()); + } +} diff --git a/eng/StackExchange.Redis.Build/BasicArray.cs b/eng/StackExchange.Redis.Build/BasicArray.cs index dc7984c75..b73e0f148 100644 --- a/eng/StackExchange.Redis.Build/BasicArray.cs +++ b/eng/StackExchange.Redis.Build/BasicArray.cs @@ -38,7 +38,7 @@ public bool Equals(BasicArray other) int i = 0; foreach (ref readonly T el in this.Span) { - if (!_comparer.Equals(el, y[i])) return false; + if (!_comparer.Equals(el, y[i++])) return false; } return true; @@ -58,7 +58,7 @@ public override int GetHashCode() var hash = Length; foreach (ref readonly T el in this.Span) { - _ = (hash * -37) + _comparer.GetHashCode(el); + hash = (hash * -37) + _comparer.GetHashCode(el); } return hash; diff --git a/src/StackExchange.Redis/Availability/RetryDatabase.cs b/src/StackExchange.Redis/Availability/RetryDatabase.cs new file mode 100644 index 000000000..c603d0d91 --- /dev/null +++ b/src/StackExchange.Redis/Availability/RetryDatabase.cs @@ -0,0 +1,15 @@ +using System; +using System.Diagnostics; + +namespace StackExchange.Redis.Availability; + +[Conditional("DEBUG")] +[AttributeUsage(AttributeTargets.Class | AttributeTargets.Interface, AllowMultiple = false)] +internal sealed class AutoDatabaseAttribute : Attribute +{ +} + +// [AutoDatabase] +// internal partial class RetryDatabase : IDatabase, IDatabaseAsync +// { +// } From a6a7f19d8f738de658d6b555c962ffe90520d69b Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Mon, 13 Jul 2026 16:48:43 +0100 Subject: [PATCH 02/23] find classes --- .../AutoDatabaseGenerator.cs | 114 ++++++++++++++---- .../Availability/RetryDatabase.cs | 8 +- 2 files changed, 97 insertions(+), 25 deletions(-) diff --git a/eng/StackExchange.Redis.Build/AutoDatabaseGenerator.cs b/eng/StackExchange.Redis.Build/AutoDatabaseGenerator.cs index 53511db1a..543548f26 100644 --- a/eng/StackExchange.Redis.Build/AutoDatabaseGenerator.cs +++ b/eng/StackExchange.Redis.Build/AutoDatabaseGenerator.cs @@ -8,6 +8,7 @@ using ParamInfo = (string Name, string Type, Microsoft.CodeAnalysis.RefKind RefKind, bool IsParams, bool IsOptional, bool HasDefault, string? Default); using MethodInfo = (string Name, string ReturnType, StackExchange.Redis.Build.BasicArray<(string Name, string Type, Microsoft.CodeAnalysis.RefKind RefKind, bool IsParams, bool IsOptional, bool HasDefault, string? Default)> Parameters); using InterfaceInfo = (string Name, string Namespace, StackExchange.Redis.Build.BasicArray<(string Name, string ReturnType, StackExchange.Redis.Build.BasicArray<(string Name, string Type, Microsoft.CodeAnalysis.RefKind RefKind, bool IsParams, bool IsOptional, bool HasDefault, string? Default)> Parameters)> Methods); +using ClassInfo = (string Name, string Namespace, bool IDatabase, bool IDatabaseAsync); namespace StackExchange.Redis.Build; @@ -18,36 +19,49 @@ public void Initialize(IncrementalGeneratorInitializationContext ctx) { var interfaces = ctx.SyntaxProvider .CreateSyntaxProvider( - static (node, _) => node is InterfaceDeclarationSyntax decl && FastFilter(decl), ExtractInterfaceMethods) + static (node, _) => node is InterfaceDeclarationSyntax decl && FastIndexFilter(decl), ExtractInterfaceMethods) .Where(pair => pair.Name is { Length: > 0 }) .Collect(); - ctx.RegisterSourceOutput(interfaces, static (ctx, content) => Generate(ctx, content)); + var classes = ctx.SyntaxProvider + .CreateSyntaxProvider( + static (node, _) => node is ClassDeclarationSyntax decl && FastClassFilter(decl), ExtractClasses) + .Where(pair => pair.Name is { Length: > 0 }) + .Collect(); + + ctx.RegisterSourceOutput(interfaces.Combine(classes), static (ctx, content) => Generate(ctx, content.Left, content.Right)); } - private static bool FastFilter(InterfaceDeclarationSyntax decl) // limit to IDatabase, IDatabaseAsync + private static bool FastIndexFilter(InterfaceDeclarationSyntax decl) // limit to IDatabase, IDatabaseAsync => decl.Identifier.ValueText is "IDatabase" or "IDatabaseAsync"; + private static bool FastClassFilter(ClassDeclarationSyntax decl) // limit to IDatabase, IDatabaseAsync + => decl.AttributeLists.Any(x => x.Attributes.Any(x => x.Name.ToString() is "AutoDatabase" or "AutoDatabaseAttribute")); + + private static bool IsOurInterface(INamedTypeSymbol symbol) => + symbol is + { + TypeKind: TypeKind.Interface, + Name: "IDatabase" or "IDatabaseAsync", + ContainingType: null, + ContainingNamespace: + { + Name: "Redis", + ContainingNamespace: + { + Name: "StackExchange", + ContainingNamespace.IsGlobalNamespace: true + } + } + }; + private static InterfaceInfo ExtractInterfaceMethods(GeneratorSyntaxContext context, CancellationToken cancel) { // note: we deliberately do NOT interpret anything here - just capture the raw shape of every // method (name, return type, and per-parameter name/type/modifiers/optionality/default) so that // later passes have everything they might need. - if (context.SemanticModel.GetDeclaredSymbol(context.Node, cancel) is not INamedTypeSymbol - { - TypeKind: TypeKind.Interface, - Name: "IDatabase" or "IDatabaseAsync", - ContainingType: null, - ContainingNamespace: - { - Name: "Redis", - ContainingNamespace: - { - Name: "StackExchange", - ContainingNamespace.IsGlobalNamespace: true - } - } - } iface) + if (context.SemanticModel.GetDeclaredSymbol(context.Node, cancel) is not INamedTypeSymbol iface + || !IsOurInterface(iface)) { return default; } @@ -83,6 +97,55 @@ private static InterfaceInfo ExtractInterfaceMethods(GeneratorSyntaxContext cont return (iface.Name, ns, methodArray.Build()); } + private ClassInfo ExtractClasses(GeneratorSyntaxContext context, CancellationToken cancel) + { + // note: we deliberately do NOT interpret anything here - just capture the raw shape of every + // method (name, return type, and per-parameter name/type/modifiers/optionality/default) so that + // later passes have everything they might need. + if (context.SemanticModel.GetDeclaredSymbol(context.Node, cancel) is not INamedTypeSymbol cls + || !HasAutoDatabaseAttrib(cls)) + { + return default; + } + + static bool HasAutoDatabaseAttrib(INamedTypeSymbol symbol) + { + var attribs = symbol.GetAttributes(); + foreach (var attrib in attribs) + { + if (attrib.AttributeClass is + { + Name: "AutoDatabaseAttribute" + }) + { + return true; + } + } + + return false; + } + + bool db = false, dba = false; + foreach (var iFace in cls.Interfaces) + { + if (IsOurInterface(iFace)) + { + switch (iFace.Name) + { + case "IDatabase": + db = true; + break; + case "IDatabaseAsync": + dba = true; + break; + } + } + } + + var ns = cls.ContainingNamespace.ToDisplayString(SymbolDisplayFormat.CSharpErrorMessageFormat); + return (cls.Name, ns, db, dba); + } + private static string FormatDefault(object? value) => value switch { null => "null", @@ -91,13 +154,17 @@ private static InterfaceInfo ExtractInterfaceMethods(GeneratorSyntaxContext cont _ => value.ToString() ?? "null", }; - private static void Generate(SourceProductionContext ctx, ImmutableArray interfaces) + private static void Generate(SourceProductionContext ctx, ImmutableArray interfaces, ImmutableArray classes) { if (interfaces.IsDefaultOrEmpty) return; // nothing to do // each interface is declared across several partial files, so we get one (identical) entry per // partial declaration; structural equality lets us collapse those down to one per interface. - var distinct = interfaces.Distinct(); + var iKeyed = new Dictionary(StringComparer.Ordinal); + foreach (var t in interfaces) + { + if (!iKeyed.ContainsKey(t.Name)) iKeyed.Add(t.Name, t); + } var sb = new StringBuilder(); sb.AppendLine("// "); @@ -105,7 +172,12 @@ private static void Generate(SourceProductionContext ctx, ImmutableArray Date: Mon, 13 Jul 2026 17:05:23 +0100 Subject: [PATCH 03/23] start stubbing class --- .../AutoDatabaseGenerator.cs | 30 ++++++++++++++----- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/eng/StackExchange.Redis.Build/AutoDatabaseGenerator.cs b/eng/StackExchange.Redis.Build/AutoDatabaseGenerator.cs index 543548f26..2c50b0413 100644 --- a/eng/StackExchange.Redis.Build/AutoDatabaseGenerator.cs +++ b/eng/StackExchange.Redis.Build/AutoDatabaseGenerator.cs @@ -156,7 +156,7 @@ static bool HasAutoDatabaseAttrib(INamedTypeSymbol symbol) private static void Generate(SourceProductionContext ctx, ImmutableArray interfaces, ImmutableArray classes) { - if (interfaces.IsDefaultOrEmpty) return; // nothing to do + if (interfaces.IsDefaultOrEmpty | classes.IsDefaultOrEmpty) return; // nothing to do // each interface is declared across several partial files, so we get one (identical) entry per // partial declaration; structural equality lets us collapse those down to one per interface. @@ -167,15 +167,29 @@ private static void Generate(SourceProductionContext ctx, ImmutableArray"); - sb.AppendLine("// AutoDatabaseGenerator diagnostic dump - what ExtractInterfaceMethods found."); - sb.AppendLine("// This file is informational only (everything is in comments); no real code is emitted yet."); - sb.AppendLine(); - + var writer = new CodeWriter(sb); + writer.NewLine().Append("// "); + writer.NewLine().Append("// AutoDatabaseGenerator diagnostic dump - what ExtractInterfaceMethods found."); + writer.NewLine().Append("// This file is informational only (everything is in comments); no real code is emitted yet."); + writer.NewLine(); foreach (var cls in classes.Distinct()) { - sb.Append("// ==== ").Append(cls.Name).Append(" db: ").Append(cls.IDatabase).Append(" dba: ").Append(cls.IDatabaseAsync).AppendLine(); + if (!string.IsNullOrWhiteSpace(cls.Namespace)) + { + writer.NewLine().Append("namespace ").Append(cls.Namespace).NewLine().Append("{").Indent(); + } + + writer.NewLine().Append("partial class ").Append(cls.Name).NewLine().Append("{").Indent(); + writer.NewLine().Append($"// IDatabase: {cls.IDatabase}, IDatabaseAsync: {cls.IDatabaseAsync}"); + writer.Outdent().NewLine().Append("}"); + + if (!string.IsNullOrWhiteSpace(cls.Namespace)) + { + writer.Outdent().NewLine().Append("}"); + } + writer.NewLine(); } + writer.NewLine(); foreach (var iface in iKeyed.Values) { @@ -202,6 +216,6 @@ private static void Generate(SourceProductionContext ctx, ImmutableArray Date: Tue, 14 Jul 2026 10:24:52 +0100 Subject: [PATCH 04/23] successfully stub out all methods (it compiles!) --- .../AutoDatabaseGenerator.cs | 159 ++++++++++++------ eng/StackExchange.Redis.Build/BasicArray.cs | 21 +++ .../Availability/RetryDatabase.cs | 2 + 3 files changed, 132 insertions(+), 50 deletions(-) diff --git a/eng/StackExchange.Redis.Build/AutoDatabaseGenerator.cs b/eng/StackExchange.Redis.Build/AutoDatabaseGenerator.cs index 2c50b0413..975b2a9a4 100644 --- a/eng/StackExchange.Redis.Build/AutoDatabaseGenerator.cs +++ b/eng/StackExchange.Redis.Build/AutoDatabaseGenerator.cs @@ -6,9 +6,9 @@ // readable names for the info we gather; kept as tuples (+ BasicArray) so they have // value equality and play nicely with incremental generator caching. using ParamInfo = (string Name, string Type, Microsoft.CodeAnalysis.RefKind RefKind, bool IsParams, bool IsOptional, bool HasDefault, string? Default); -using MethodInfo = (string Name, string ReturnType, StackExchange.Redis.Build.BasicArray<(string Name, string Type, Microsoft.CodeAnalysis.RefKind RefKind, bool IsParams, bool IsOptional, bool HasDefault, string? Default)> Parameters); -using InterfaceInfo = (string Name, string Namespace, StackExchange.Redis.Build.BasicArray<(string Name, string ReturnType, StackExchange.Redis.Build.BasicArray<(string Name, string Type, Microsoft.CodeAnalysis.RefKind RefKind, bool IsParams, bool IsOptional, bool HasDefault, string? Default)> Parameters)> Methods); -using ClassInfo = (string Name, string Namespace, bool IDatabase, bool IDatabaseAsync); +using MethodInfo = (string Name, string ReturnType, StackExchange.Redis.Build.BasicArray<(string Name, string Type, Microsoft.CodeAnalysis.RefKind RefKind, bool IsParams, bool IsOptional, bool HasDefault, string? Default)> Parameters, StackExchange.Redis.Build.BasicArray TypeArgs); +using InterfaceInfo = (string Name, string Namespace, StackExchange.Redis.Build.AutoDatabaseGenerator.KnownInterfaces KnownType, StackExchange.Redis.Build.BasicArray<(string Name, string ReturnType, StackExchange.Redis.Build.BasicArray<(string Name, string Type, Microsoft.CodeAnalysis.RefKind RefKind, bool IsParams, bool IsOptional, bool HasDefault, string? Default)> Parameters, StackExchange.Redis.Build.BasicArray TypeArgs)> Methods); +using ClassInfo = (string Name, string Namespace, StackExchange.Redis.Build.AutoDatabaseGenerator.KnownInterfaces Interfaces); namespace StackExchange.Redis.Build; @@ -32,8 +32,17 @@ public void Initialize(IncrementalGeneratorInitializationContext ctx) ctx.RegisterSourceOutput(interfaces.Combine(classes), static (ctx, content) => Generate(ctx, content.Left, content.Right)); } + static KnownInterfaces Identify(string type) => type switch + { + "IDatabase" => KnownInterfaces.IDatabase, + "IDatabaseAsync" => KnownInterfaces.IDatabaseAsync, + "IRedis" => KnownInterfaces.IRedis, + "IRedisAsync" => KnownInterfaces.IRedisAsync, + _ => KnownInterfaces.None, + }; + private static bool FastIndexFilter(InterfaceDeclarationSyntax decl) // limit to IDatabase, IDatabaseAsync - => decl.Identifier.ValueText is "IDatabase" or "IDatabaseAsync"; + => Identify(decl.Identifier.ValueText) is not 0; private static bool FastClassFilter(ClassDeclarationSyntax decl) // limit to IDatabase, IDatabaseAsync => decl.AttributeLists.Any(x => x.Attributes.Any(x => x.Name.ToString() is "AutoDatabase" or "AutoDatabaseAttribute")); @@ -42,7 +51,7 @@ private static bool IsOurInterface(INamedTypeSymbol symbol) => symbol is { TypeKind: TypeKind.Interface, - Name: "IDatabase" or "IDatabaseAsync", + Name: "IDatabase" or "IDatabaseAsync" or "IRedis" or "IRedisAsync", ContainingType: null, ContainingNamespace: { @@ -66,35 +75,38 @@ private static InterfaceInfo ExtractInterfaceMethods(GeneratorSyntaxContext cont return default; } - var methods = ImmutableArray.CreateBuilder(); + var knownType = Identify(iface.Name); + if (knownType is KnownInterfaces.None) return default; + + var methods = new List(); foreach (var member in iface.GetMembers()) { cancel.ThrowIfCancellationRequested(); if (member is not IMethodSymbol { MethodKind: MethodKind.Ordinary } method) continue; - var parameters = new BasicArray.Builder(method.Parameters.Length); - foreach (var p in method.Parameters) + var parameters = BasicArray.From(method.Parameters, static p => ( + Name: p.Name, + Type: p.Type.ToDisplayString(SymbolDisplayFormat.CSharpErrorMessageFormat), + RefKind: p.RefKind, + IsParams: p.IsParams, + IsOptional: p.IsOptional, + HasDefault: p.HasExplicitDefaultValue, + Default: p.HasExplicitDefaultValue ? FormatDefault(p.ExplicitDefaultValue) : null)); + + BasicArray typeArgs = default; + if (method.IsGenericMethod) { - parameters.Add(( - Name: p.Name, - Type: p.Type.ToDisplayString(SymbolDisplayFormat.CSharpErrorMessageFormat), - RefKind: p.RefKind, - IsParams: p.IsParams, - IsOptional: p.IsOptional, - HasDefault: p.HasExplicitDefaultValue, - Default: p.HasExplicitDefaultValue ? FormatDefault(p.ExplicitDefaultValue) : null)); + typeArgs = BasicArray.From(method.TypeParameters, p => p.Name); } - methods.Add(( Name: method.Name, ReturnType: method.ReturnType.ToDisplayString(SymbolDisplayFormat.CSharpErrorMessageFormat), - Parameters: parameters.Build())); + Parameters: parameters, + TypeArgs: typeArgs)); } var ns = iface.ContainingNamespace.ToDisplayString(SymbolDisplayFormat.CSharpErrorMessageFormat); - var methodArray = new BasicArray.Builder(methods.Count); - foreach (var m in methods) methodArray.Add(m); - return (iface.Name, ns, methodArray.Build()); + return (iface.Name, ns, knownType, BasicArray.From(methods)); } private ClassInfo ExtractClasses(GeneratorSyntaxContext context, CancellationToken cancel) @@ -125,7 +137,7 @@ static bool HasAutoDatabaseAttrib(INamedTypeSymbol symbol) return false; } - bool db = false, dba = false; + KnownInterfaces known = 0; foreach (var iFace in cls.Interfaces) { if (IsOurInterface(iFace)) @@ -133,17 +145,36 @@ static bool HasAutoDatabaseAttrib(INamedTypeSymbol symbol) switch (iFace.Name) { case "IDatabase": - db = true; + known |= KnownInterfaces.IDatabase | KnownInterfaces.IDatabaseAsync | + KnownInterfaces.IRedis | KnownInterfaces.IRedisAsync; break; case "IDatabaseAsync": - dba = true; + known |= KnownInterfaces.IDatabaseAsync | KnownInterfaces.IRedisAsync; + break; + case "IRedis": + known |= KnownInterfaces.IRedis | KnownInterfaces.IRedisAsync; + break; + case "IRedisAsync": + known |= KnownInterfaces.IRedisAsync; break; } } } + if (known is KnownInterfaces.None) return default; // nothing to do! + var ns = cls.ContainingNamespace.ToDisplayString(SymbolDisplayFormat.CSharpErrorMessageFormat); - return (cls.Name, ns, db, dba); + return (cls.Name, ns, known); + } + + [Flags] + internal enum KnownInterfaces + { + None = 0, + IDatabase = 1, + IDatabaseAsync = 2, + IRedis = 4, + IRedisAsync = 8, } private static string FormatDefault(object? value) => value switch @@ -160,10 +191,11 @@ private static void Generate(SourceProductionContext ctx, ImmutableArray(StringComparer.Ordinal); + var iKeyed = new Dictionary(); foreach (var t in interfaces) { - if (!iKeyed.ContainsKey(t.Name)) iKeyed.Add(t.Name, t); + if (t.KnownType is KnownInterfaces.None) continue; + if (!iKeyed.ContainsKey(t.KnownType)) iKeyed.Add(t.KnownType, t); } var sb = new StringBuilder(); @@ -171,6 +203,7 @@ private static void Generate(SourceProductionContext ctx, ImmutableArray"); writer.NewLine().Append("// AutoDatabaseGenerator diagnostic dump - what ExtractInterfaceMethods found."); writer.NewLine().Append("// This file is informational only (everything is in comments); no real code is emitted yet."); + writer.NewLine().Append("#nullable enable"); // this needs to be explicit for code-gen writer.NewLine(); foreach (var cls in classes.Distinct()) { @@ -179,8 +212,18 @@ private static void Generate(SourceProductionContext ctx, ImmutableArray"); + } + writer.Append("("); + bool firstParam = true; + foreach (var p in method.Parameters.Span) + { + if (firstParam) firstParam = false; + else writer.Append(", "); + writer.Append(p.Type).Append(" ").Append(p.Name); + } + writer.Append(')').Indent().NewLine().Append("=> throw new global::System.NotImplementedException();") + .Outdent().NewLine(); } - - sb.AppendLine("// )"); } - - sb.AppendLine("//"); } + writer.NewLine(); ctx.AddSource("AutoDatabase.generated.cs", sb.ToString()); } diff --git a/eng/StackExchange.Redis.Build/BasicArray.cs b/eng/StackExchange.Redis.Build/BasicArray.cs index b73e0f148..0c2df6f92 100644 --- a/eng/StackExchange.Redis.Build/BasicArray.cs +++ b/eng/StackExchange.Redis.Build/BasicArray.cs @@ -82,4 +82,25 @@ public void Add(in T value) public BasicArray Build() => new(elements, Count); } + + public static BasicArray From(ICollection collection) + { + if (collection.Count is 0) return default; + var arr = new T[collection.Count]; + collection.CopyTo(arr, 0); + return new(arr, arr.Length); + } + + public static BasicArray From(ICollection collection, Func selector) + { + if (collection.Count is 0) return default; + var arr = new T[collection.Count]; + int i = 0; + foreach (var item in collection) + { + arr[i++] = selector(item); + } + + return new(arr, i); + } } diff --git a/src/StackExchange.Redis/Availability/RetryDatabase.cs b/src/StackExchange.Redis/Availability/RetryDatabase.cs index 33a64902d..70bc66c99 100644 --- a/src/StackExchange.Redis/Availability/RetryDatabase.cs +++ b/src/StackExchange.Redis/Availability/RetryDatabase.cs @@ -12,4 +12,6 @@ internal sealed class AutoDatabaseAttribute : Attribute [AutoDatabase] internal partial class RetryDatabase : IDatabase { + public int Database => throw new NotImplementedException(); + public IConnectionMultiplexer Multiplexer => throw new NotImplementedException(); } From 3fc9e53d0f30a88ae577db38d6cf22d0d75051b4 Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Tue, 14 Jul 2026 11:10:26 +0100 Subject: [PATCH 05/23] stub out execute and tuples --- .../AutoDatabaseGenerator.cs | 84 ++++++++++++++++++- .../Availability/RetryDatabase.cs | 32 ++++++- 2 files changed, 110 insertions(+), 6 deletions(-) diff --git a/eng/StackExchange.Redis.Build/AutoDatabaseGenerator.cs b/eng/StackExchange.Redis.Build/AutoDatabaseGenerator.cs index 975b2a9a4..47c85c6d0 100644 --- a/eng/StackExchange.Redis.Build/AutoDatabaseGenerator.cs +++ b/eng/StackExchange.Redis.Build/AutoDatabaseGenerator.cs @@ -201,8 +201,9 @@ private static void Generate(SourceProductionContext ctx, ImmutableArray"); - writer.NewLine().Append("// AutoDatabaseGenerator diagnostic dump - what ExtractInterfaceMethods found."); - writer.NewLine().Append("// This file is informational only (everything is in comments); no real code is emitted yet."); + writer.NewLine().Append("// AutoDatabaseGenerator - explicit interface implementations that funnel every"); + writer.NewLine().Append("// call through Execute(state, projection), capturing the arguments in a generated"); + writer.NewLine().Append("// state struct (one per unique parameter-type signature) to avoid per-call closures."); writer.NewLine().Append("#nullable enable"); // this needs to be explicit for code-gen writer.NewLine(); foreach (var cls in classes.Distinct()) @@ -212,6 +213,12 @@ private static void Generate(SourceProductionContext ctx, ImmutableArray(); + var tupleDefs = new List>(); + bool isFirst = true; writer.NewLine().Append("partial class ").Append(cls.Name); AppendInterfaceDeclaration(KnownInterfaces.IDatabase); @@ -223,6 +230,7 @@ private static void Generate(SourceProductionContext ctx, ImmutableArray throw new global::System.NotImplementedException();") - .Outdent().NewLine(); + writer.Append(')').Indent().NewLine(); + + // generic methods can't be captured into a concrete state struct - leave them unimplemented for now + if (!method.TypeArgs.IsEmpty) + { + writer.Append("=> throw new global::System.NotImplementedException();").Outdent().NewLine(); + continue; + } + + // Task / Task methods route to ExecuteAsync (its own retry policy); everything + // else (including IAsyncEnumerable, which returns synchronously) uses Execute. The + // state struct is shared regardless of return type - keyed on parameter types only - + // so e.g. ArraySet and ArraySetAsync land on the same _tupleN. + bool isAsync = method.ReturnType.StartsWith("System.Threading.Tasks.Task", StringComparison.Ordinal); + int tuple = GetTupleIndex(method.Parameters); + writer.Append(isAsync ? "=> ExecuteAsync(new _tuple" : "=> Execute(new _tuple").Append(tuple).Append("("); + firstParam = true; + foreach (var p in method.Parameters.Span) + { + if (firstParam) firstParam = false; + else writer.Append(", "); + writer.Append(p.Name); + } + writer.Append("), static (state, inner) => inner.").Append(method.Name).Append("("); + for (int i = 0; i < method.Parameters.Length; i++) + { + if (i != 0) writer.Append(", "); + writer.Append("state.Arg").Append(i); + } + writer.Append("));").Outdent().NewLine(); + } + } + + int GetTupleIndex(BasicArray parameters) + { + var keyBuilder = new StringBuilder(); + foreach (var p in parameters.Span) + { + keyBuilder.Append(p.Type).Append('|'); // types-only key; no ref/out on this surface, so type is sufficient + } + + var key = keyBuilder.ToString(); + if (!tupleIndex.TryGetValue(key, out var index)) + { + index = tupleDefs.Count; + tupleIndex.Add(key, index); + tupleDefs.Add(parameters); + } + + return index; + } + + void AppendTupleTypes() + { + for (int i = 0; i < tupleDefs.Count; i++) + { + var parameters = tupleDefs[i].Span; + writer.NewLine().NewLine().Append("private readonly struct _tuple").Append(i).Append("("); + for (int p = 0; p < parameters.Length; p++) + { + if (p != 0) writer.Append(", "); + writer.Append(parameters[p].Type).Append(" arg").Append(p); + } + writer.Append(")").NewLine().Append("{").Indent(); + for (int p = 0; p < parameters.Length; p++) + { + writer.NewLine().Append("public readonly ").Append(parameters[p].Type) + .Append(" Arg").Append(p).Append(" = arg").Append(p).Append(";"); + } + writer.Outdent().NewLine().Append("}"); } } } diff --git a/src/StackExchange.Redis/Availability/RetryDatabase.cs b/src/StackExchange.Redis/Availability/RetryDatabase.cs index 70bc66c99..d12784486 100644 --- a/src/StackExchange.Redis/Availability/RetryDatabase.cs +++ b/src/StackExchange.Redis/Availability/RetryDatabase.cs @@ -1,5 +1,6 @@ using System; using System.Diagnostics; +using System.Threading.Tasks; namespace StackExchange.Redis.Availability; @@ -9,9 +10,36 @@ internal sealed class AutoDatabaseAttribute : Attribute { } +internal interface IRedisArgs +{ + void ApplyKeys(Func selector); + CommandFlags Flags { get; set; } +} + [AutoDatabase] internal partial class RetryDatabase : IDatabase { - public int Database => throw new NotImplementedException(); - public IConnectionMultiplexer Multiplexer => throw new NotImplementedException(); + private readonly IDatabase _inner; + + public RetryDatabase(IDatabase inner) => _inner = inner; + + public int Database => _inner.Database; + public IConnectionMultiplexer Multiplexer => _inner.Multiplexer; + + // the generated explicit interface implementations funnel every call through these two + // overloads: the arguments are captured in a generated state struct and replayed against + // the inner database via a cacheable static projection (no per-call closure). Retry/failover + // policy will live here in due course; for now it is a straight pass-through. + private TResult Execute(TState state, Func operation) + => operation(state, _inner); + + private void Execute(TState state, Action operation) + => operation(state, _inner); + + // async counterparts (Task / Task); these get their own retry/failover policy in due course. + private Task ExecuteAsync(TState state, Func> operation) + => operation(state, _inner); + + private Task ExecuteAsync(TState state, Func operation) + => operation(state, _inner); } From af04503593c56525c6e15cb5d3ffeb17a1916efc Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Tue, 14 Jul 2026 13:31:06 +0100 Subject: [PATCH 06/23] generalize mutator API to include channels --- .../AutoDatabaseGenerator.cs | 119 ++++++++++++---- src/StackExchange.Redis/AutoDatabase.cs | 129 ++++++++++++++++++ .../Availability/RetryDatabase.cs | 39 ++++-- 3 files changed, 247 insertions(+), 40 deletions(-) create mode 100644 src/StackExchange.Redis/AutoDatabase.cs diff --git a/eng/StackExchange.Redis.Build/AutoDatabaseGenerator.cs b/eng/StackExchange.Redis.Build/AutoDatabaseGenerator.cs index 47c85c6d0..6400334e6 100644 --- a/eng/StackExchange.Redis.Build/AutoDatabaseGenerator.cs +++ b/eng/StackExchange.Redis.Build/AutoDatabaseGenerator.cs @@ -127,7 +127,17 @@ static bool HasAutoDatabaseAttrib(INamedTypeSymbol symbol) { if (attrib.AttributeClass is { - Name: "AutoDatabaseAttribute" + Name: "AutoDatabaseAttribute", + ContainingType: null, + ContainingNamespace: + { + Name: "Redis", + ContainingNamespace: + { + Name: "StackExchange", + ContainingNamespace.IsGlobalNamespace: true + } + } }) { return true; @@ -177,6 +187,16 @@ internal enum KnownInterfaces IRedisAsync = 8, } + // methods that don't fit the capture-and-replay shape are left for the caller to implement manually: + // - generic methods (open type args can't be captured into a concrete state struct) + // - the Wait family (synchronization over caller-supplied Tasks, not server calls) + // - streaming returns (IEnumerable / IAsyncEnumerable) whose execution is deferred + private static bool SkipMethod(MethodInfo method) + => !method.TypeArgs.IsEmpty + || method.Name.Contains("Wait") + || method.ReturnType.StartsWith("System.Collections.Generic.IEnumerable<", StringComparison.Ordinal) + || method.ReturnType.StartsWith("System.Collections.Generic.IAsyncEnumerable<", StringComparison.Ordinal); + private static string FormatDefault(object? value) => value switch { null => "null", @@ -254,21 +274,10 @@ void AppendInterfaceMethods(KnownInterfaces knownType) if ((cls.Interfaces & knownType) is 0 | !iKeyed.TryGetValue(knownType, out var iType)) return; foreach (var method in iType.Methods) { + if (SkipMethod(method)) continue; // wonky by nature - left for the caller to implement manually + writer.NewLine().Append(method.ReturnType).Append(" global::") - .Append(iType.Namespace).Append('.').Append(iType.Name).Append('.').Append(method.Name); - if (!method.TypeArgs.IsEmpty) - { - bool firstT = true; - writer.Append("<"); - foreach (var t in method.TypeArgs) - { - if (firstT) firstT = false; - else writer.Append(", "); - writer.Append(t); - } - writer.Append(">"); - } - writer.Append("("); + .Append(iType.Namespace).Append('.').Append(iType.Name).Append('.').Append(method.Name).Append("("); bool firstParam = true; foreach (var p in method.Parameters.Span) { @@ -278,17 +287,9 @@ void AppendInterfaceMethods(KnownInterfaces knownType) } writer.Append(')').Indent().NewLine(); - // generic methods can't be captured into a concrete state struct - leave them unimplemented for now - if (!method.TypeArgs.IsEmpty) - { - writer.Append("=> throw new global::System.NotImplementedException();").Outdent().NewLine(); - continue; - } - // Task / Task methods route to ExecuteAsync (its own retry policy); everything - // else (including IAsyncEnumerable, which returns synchronously) uses Execute. The - // state struct is shared regardless of return type - keyed on parameter types only - - // so e.g. ArraySet and ArraySetAsync land on the same _tupleN. + // else uses Execute. The state struct is shared regardless of return type - keyed on + // parameter types only - so e.g. ArraySet and ArraySetAsync land on the same _tupleN. bool isAsync = method.ReturnType.StartsWith("System.Threading.Tasks.Task", StringComparison.Ordinal); int tuple = GetTupleIndex(method.Parameters); writer.Append(isAsync ? "=> ExecuteAsync(new _tuple" : "=> Execute(new _tuple").Append(tuple).Append("("); @@ -330,21 +331,83 @@ int GetTupleIndex(BasicArray parameters) void AppendTupleTypes() { + const string CommandFlagsType = "StackExchange.Redis.CommandFlags"; + const string RedisKeyType = "StackExchange.Redis.RedisKey"; + const string RedisChannelType = "StackExchange.Redis.RedisChannel"; + // types that carry key(s) internally without "RedisKey" in their name, so the + // substring check below won't catch them; they rely on a Map extension method + const string StreamPositionType = "StackExchange.Redis.StreamPosition"; + // the Execute/ExecuteAsync escape hatch boxes keys/channels inside a loosely-typed + // arg list; these route through a Map extension that unboxes and rewrites matches + const string ScriptArgArrayType = "object[]"; + const string ScriptArgCollectionType = "System.Collections.Generic.ICollection"; for (int i = 0; i < tupleDefs.Count; i++) { var parameters = tupleDefs[i].Span; - writer.NewLine().NewLine().Append("private readonly struct _tuple").Append(i).Append("("); + + // locate the (single) CommandFlags argument, if any, to back IRedisArgs.Flags + int flagsArg = -1; + for (int p = 0; p < parameters.Length; p++) + { + if (parameters[p].Type == CommandFlagsType) + { + flagsArg = p; + break; + } + } + + // fields are mutable: Map rewrites key/channel fields and Flags has a setter + writer.NewLine().NewLine().Append("private struct _tuple").Append(i).Append("("); for (int p = 0; p < parameters.Length; p++) { if (p != 0) writer.Append(", "); writer.Append(parameters[p].Type).Append(" arg").Append(p); } - writer.Append(")").NewLine().Append("{").Indent(); + writer.Append(") : global::StackExchange.Redis.IRedisArgs").NewLine().Append("{").Indent(); for (int p = 0; p < parameters.Length; p++) { - writer.NewLine().Append("public readonly ").Append(parameters[p].Type) + writer.NewLine().Append("public ").Append(parameters[p].Type) .Append(" Arg").Append(p).Append(" = arg").Append(p).Append(";"); } + + // Flags maps onto the CommandFlags field when present, else a synthesized default + writer.NewLine().Append("public global::StackExchange.Redis.CommandFlags Flags"); + if (flagsArg >= 0) + { + writer.Append(" { readonly get => Arg").Append(flagsArg).Append("; set => Arg").Append(flagsArg).Append(" = value; }"); + } + else + { + writer.Append(" { get; set; }"); + } + + // Map rewrites each scalar key/channel field directly, and defers container or + // loosely-typed fields to a matching Map extension method + writer.NewLine().Append("public void Map(global::StackExchange.Redis.IRedisArgsMutator mutator)") + .NewLine().Append("{").Indent(); + for (int p = 0; p < parameters.Length; p++) + { + if (parameters[p].Type == RedisKeyType) + { + writer.NewLine().Append("Arg").Append(p).Append(" = mutator.MapKey(Arg").Append(p).Append(");"); + } + else if (parameters[p].Type == RedisChannelType) + { + writer.NewLine().Append("Arg").Append(p).Append(" = mutator.MapChannel(Arg").Append(p).Append(");"); + } + else if (parameters[p].Type.IndexOf(RedisKeyType, StringComparison.Ordinal) >= 0 + || parameters[p].Type.IndexOf(StreamPositionType, StringComparison.Ordinal) >= 0 + || parameters[p].Type == ScriptArgArrayType + || parameters[p].Type == ScriptArgCollectionType + || parameters[p].Type == ScriptArgCollectionType + "?") + { + // key/channel-bearing container (or the loosely-typed script arg list); it + // is the library's job to ensure a suitable Map extension method exists + writer.NewLine().Append("Arg").Append(p).Append(" = mutator.Map(Arg").Append(p).Append(");"); + } + } + writer.Outdent().NewLine().Append("}"); + writer.Outdent().NewLine().Append("}"); } } diff --git a/src/StackExchange.Redis/AutoDatabase.cs b/src/StackExchange.Redis/AutoDatabase.cs new file mode 100644 index 000000000..fe242b0aa --- /dev/null +++ b/src/StackExchange.Redis/AutoDatabase.cs @@ -0,0 +1,129 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; + +namespace StackExchange.Redis; + +[Conditional("DEBUG")] +[AttributeUsage(AttributeTargets.Class | AttributeTargets.Interface, AllowMultiple = false)] +internal sealed class AutoDatabaseAttribute : Attribute +{ +} + +internal interface IRedisArgs +{ + void Map(IRedisArgsMutator mutator); + CommandFlags Flags { get; set; } +} + +internal interface IRedisArgsMutator +{ + RedisKey MapKey(RedisKey key); + RedisChannel MapChannel(RedisChannel channel); +} + +internal static class RedisArgsMutatorExtensions +{ + // these are used by the generated tuple-types via auto-database: each maps the key-bearing parts + // of an argument through the supplied mutator. They hang off IRedisArgsMutator (rather than the + // argument) so a call site reads consistently with the interface's own MapKey/MapChannel. + public static KeyValuePair Map( + this IRedisArgsMutator mutator, + KeyValuePair value) => + new(mutator.MapKey(value.Key), value.Value); + + [return: NotNullIfNotNull("keys")] + public static RedisKey[]? Map(this IRedisArgsMutator mutator, RedisKey[]? keys) + { + if (keys is null || keys.Length is 0) return keys; + var arr = new RedisKey[keys.Length]; + for (int i = 0; i < arr.Length; i++) + { + arr[i] = mutator.MapKey(keys[i]); + } + return arr; + } + + [return: NotNullIfNotNull("pairs")] + public static KeyValuePair[]? Map( + this IRedisArgsMutator mutator, + KeyValuePair[]? pairs) + { + if (pairs is null || pairs.Length is 0) return pairs; + var arr = new KeyValuePair[pairs.Length]; + for (int i = 0; i < arr.Length; i++) + { + ref readonly KeyValuePair pair = ref pairs[i]; + arr[i] = new(mutator.MapKey(pair.Key), pair.Value); + } + return arr; + } + + public static StreamPosition Map(this IRedisArgsMutator mutator, StreamPosition value) => + new(mutator.MapKey(value.Key), value.Position); + + [return: NotNullIfNotNull("positions")] + public static StreamPosition[]? Map(this IRedisArgsMutator mutator, StreamPosition[]? positions) + { + if (positions is null || positions.Length is 0) return positions; + var arr = new StreamPosition[positions.Length]; + for (int i = 0; i < arr.Length; i++) + { + ref readonly StreamPosition position = ref positions[i]; + arr[i] = new(mutator.MapKey(position.Key), position.Position); + } + return arr; + } + + // the Execute/ExecuteAsync escape hatch takes a loosely-typed arg list in which any element may + // be a boxed RedisKey or RedisChannel; these rewrite just those entries (mirroring + // KeyPrefixed.ToInner), copying only when there is something to rewrite so the common call allocates nothing. + [return: NotNullIfNotNull("args")] + public static object[]? Map(this IRedisArgsMutator mutator, object[]? args) + { + if (args is null || args.Length is 0) return args; + object[]? copy = null; + for (int i = 0; i < args.Length; i++) + { + if (args[i] is RedisKey key) + { + (copy ??= (object[])args.Clone())[i] = mutator.MapKey(key); + } + else if (args[i] is RedisChannel channel) + { + (copy ??= (object[])args.Clone())[i] = mutator.MapChannel(channel); + } + } + return copy ?? args; + } + + [return: NotNullIfNotNull("args")] + public static ICollection? Map(this IRedisArgsMutator mutator, ICollection? args) + { + if (args is null || args.Count is 0) return args; + bool any = false; + foreach (var arg in args) + { + if (arg is RedisKey or RedisChannel) + { + any = true; + break; + } + } + if (!any) return args; + + var copy = new object[args.Count]; + int i = 0; + foreach (var arg in args) + { + copy[i++] = arg switch + { + RedisKey key => mutator.MapKey(key), + RedisChannel channel => mutator.MapChannel(channel), + _ => arg, + }; + } + return copy; + } +} diff --git a/src/StackExchange.Redis/Availability/RetryDatabase.cs b/src/StackExchange.Redis/Availability/RetryDatabase.cs index d12784486..fac31829c 100644 --- a/src/StackExchange.Redis/Availability/RetryDatabase.cs +++ b/src/StackExchange.Redis/Availability/RetryDatabase.cs @@ -4,18 +4,6 @@ namespace StackExchange.Redis.Availability; -[Conditional("DEBUG")] -[AttributeUsage(AttributeTargets.Class | AttributeTargets.Interface, AllowMultiple = false)] -internal sealed class AutoDatabaseAttribute : Attribute -{ -} - -internal interface IRedisArgs -{ - void ApplyKeys(Func selector); - CommandFlags Flags { get; set; } -} - [AutoDatabase] internal partial class RetryDatabase : IDatabase { @@ -31,15 +19,42 @@ internal partial class RetryDatabase : IDatabase // the inner database via a cacheable static projection (no per-call closure). Retry/failover // policy will live here in due course; for now it is a straight pass-through. private TResult Execute(TState state, Func operation) + where TState : struct, IRedisArgs => operation(state, _inner); private void Execute(TState state, Action operation) + where TState : struct, IRedisArgs => operation(state, _inner); // async counterparts (Task / Task); these get their own retry/failover policy in due course. private Task ExecuteAsync(TState state, Func> operation) + where TState : struct, IRedisArgs => operation(state, _inner); private Task ExecuteAsync(TState state, Func operation) + where TState : struct, IRedisArgs => operation(state, _inner); + + void IRedisAsync.Wait(Task task) => _inner.Wait(task); + T IRedisAsync.Wait(Task task) => _inner.Wait(task); + void IRedisAsync.WaitAll(Task[] tasks) => _inner.WaitAll(tasks); + bool IRedisAsync.TryWait(Task task) => _inner.TryWait(task); + + // Methods the generator deliberately skips (see AutoDatabaseGenerator.SkipMethod): the Wait + // family and the streaming IEnumerable/IAsyncEnumerable scans don't fit the capture-and-replay + // shape, so they are implemented by hand. + System.Collections.Generic.IEnumerable IDatabase.HashScan(RedisKey key, RedisValue pattern, int pageSize, CommandFlags flags) => throw new NotImplementedException(); + System.Collections.Generic.IEnumerable IDatabase.HashScan(RedisKey key, RedisValue pattern, int pageSize, long cursor, int pageOffset, CommandFlags flags) => throw new NotImplementedException(); + System.Collections.Generic.IEnumerable IDatabase.HashScanNoValues(RedisKey key, RedisValue pattern, int pageSize, long cursor, int pageOffset, CommandFlags flags) => throw new NotImplementedException(); + System.Collections.Generic.IEnumerable IDatabase.SetScan(RedisKey key, RedisValue pattern, int pageSize, CommandFlags flags) => throw new NotImplementedException(); + System.Collections.Generic.IEnumerable IDatabase.SetScan(RedisKey key, RedisValue pattern, int pageSize, long cursor, int pageOffset, CommandFlags flags) => throw new NotImplementedException(); + System.Collections.Generic.IEnumerable IDatabase.VectorSetRangeEnumerate(RedisKey key, RedisValue start, RedisValue end, long count, Exclude exclude, CommandFlags flags) => throw new NotImplementedException(); + System.Collections.Generic.IEnumerable IDatabase.SortedSetScan(RedisKey key, RedisValue pattern, int pageSize, CommandFlags flags) => throw new NotImplementedException(); + System.Collections.Generic.IEnumerable IDatabase.SortedSetScan(RedisKey key, RedisValue pattern, int pageSize, long cursor, int pageOffset, CommandFlags flags) => throw new NotImplementedException(); + + System.Collections.Generic.IAsyncEnumerable IDatabaseAsync.HashScanAsync(RedisKey key, RedisValue pattern, int pageSize, long cursor, int pageOffset, CommandFlags flags) => throw new NotImplementedException(); + System.Collections.Generic.IAsyncEnumerable IDatabaseAsync.HashScanNoValuesAsync(RedisKey key, RedisValue pattern, int pageSize, long cursor, int pageOffset, CommandFlags flags) => throw new NotImplementedException(); + System.Collections.Generic.IAsyncEnumerable IDatabaseAsync.SetScanAsync(RedisKey key, RedisValue pattern, int pageSize, long cursor, int pageOffset, CommandFlags flags) => throw new NotImplementedException(); + System.Collections.Generic.IAsyncEnumerable IDatabaseAsync.VectorSetRangeEnumerateAsync(RedisKey key, RedisValue start, RedisValue end, long count, Exclude exclude, CommandFlags flags) => throw new NotImplementedException(); + System.Collections.Generic.IAsyncEnumerable IDatabaseAsync.SortedSetScanAsync(RedisKey key, RedisValue pattern, int pageSize, long cursor, int pageOffset, CommandFlags flags) => throw new NotImplementedException(); } From d40e8a6bfc7187780c0b115786ef0a332f36e8a4 Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Tue, 14 Jul 2026 15:00:09 +0100 Subject: [PATCH 07/23] wip --- .../AutoDatabaseGenerator.cs | 114 +++++++++++++----- src/StackExchange.Redis/AutoDatabase.cs | 42 +++++-- .../Availability/RetryDatabase.cs | 37 +++++- 3 files changed, 147 insertions(+), 46 deletions(-) diff --git a/eng/StackExchange.Redis.Build/AutoDatabaseGenerator.cs b/eng/StackExchange.Redis.Build/AutoDatabaseGenerator.cs index 6400334e6..b75e056e5 100644 --- a/eng/StackExchange.Redis.Build/AutoDatabaseGenerator.cs +++ b/eng/StackExchange.Redis.Build/AutoDatabaseGenerator.cs @@ -1,4 +1,5 @@ using System.Collections.Immutable; +using System.Reflection.Metadata.Ecma335; using System.Text; using Microsoft.CodeAnalysis; using Microsoft.CodeAnalysis.CSharp.Syntax; @@ -236,7 +237,7 @@ private static void Generate(SourceProductionContext ctx, ImmutableArray(); + var tupleIndex = new Dictionary? ReturnTypes)>(); var tupleDefs = new List>(); bool isFirst = true; @@ -291,7 +292,7 @@ void AppendInterfaceMethods(KnownInterfaces knownType) // else uses Execute. The state struct is shared regardless of return type - keyed on // parameter types only - so e.g. ArraySet and ArraySetAsync land on the same _tupleN. bool isAsync = method.ReturnType.StartsWith("System.Threading.Tasks.Task", StringComparison.Ordinal); - int tuple = GetTupleIndex(method.Parameters); + int tuple = GetTupleIndex(method.Parameters, method.ReturnType); writer.Append(isAsync ? "=> ExecuteAsync(new _tuple" : "=> Execute(new _tuple").Append(tuple).Append("("); firstParam = true; foreach (var p in method.Parameters.Span) @@ -310,7 +311,7 @@ void AppendInterfaceMethods(KnownInterfaces knownType) } } - int GetTupleIndex(BasicArray parameters) + static string GetTupleKey(BasicArray parameters) { var keyBuilder = new StringBuilder(); foreach (var p in parameters.Span) @@ -318,32 +319,70 @@ int GetTupleIndex(BasicArray parameters) keyBuilder.Append(p.Type).Append('|'); // types-only key; no ref/out on this surface, so type is sufficient } - var key = keyBuilder.ToString(); - if (!tupleIndex.TryGetValue(key, out var index)) + return keyBuilder.ToString(); + } + + HashSet? GetTupleMapReturns(BasicArray parameters) => + tupleIndex.TryGetValue(GetTupleKey(parameters), out var found) + ? found.ReturnTypes : null; + + int GetTupleIndex(BasicArray parameters, string returnType) + { + var key = GetTupleKey(parameters); + bool map = NeedsMap(returnType); + int index; + if (tupleIndex.TryGetValue(key, out var found)) + { + index = found.Index; + if (map) + { + if (found.ReturnTypes is null) + { + found.ReturnTypes = []; + tupleIndex[key] = found; + } + found.ReturnTypes.Add(returnType); + } + } + else { index = tupleDefs.Count; - tupleIndex.Add(key, index); + tupleIndex.Add(key, (index, map ? [ returnType ] : null)); tupleDefs.Add(parameters); } return index; } + const string CommandFlagsType = "StackExchange.Redis.CommandFlags"; + const string RedisKeyType = "StackExchange.Redis.RedisKey"; + const string RedisChannelType = "StackExchange.Redis.RedisChannel"; + // types that carry key(s) internally without "RedisKey" in their name, so the + // substring check below won't catch them; they rely on a Map extension method + const string StreamPositionType = "StackExchange.Redis.StreamPosition"; + // the Execute/ExecuteAsync escape hatch boxes keys/channels inside a loosely-typed + // arg list; these route through a Map extension that unboxes and rewrites matches + const string ScriptArgArrayType = "object[]"; + const string ScriptArgCollectionType = "System.Collections.Generic.ICollection"; + + static bool NeedsMap(string name) => + name.IndexOf(RedisKeyType, StringComparison.Ordinal) >= 0 + || name.IndexOf(RedisChannelType, StringComparison.Ordinal) >= 0 + || name.IndexOf(StreamPositionType, StringComparison.Ordinal) >= 0 + || name == ScriptArgArrayType + || name == ScriptArgCollectionType + || name.IndexOf("ListPopResult", StringComparison.Ordinal) >= 0 + || name.IndexOf("SortedSetPopResult", StringComparison.Ordinal) >= 0 + || name == ScriptArgCollectionType + "?"; + void AppendTupleTypes() { - const string CommandFlagsType = "StackExchange.Redis.CommandFlags"; - const string RedisKeyType = "StackExchange.Redis.RedisKey"; - const string RedisChannelType = "StackExchange.Redis.RedisChannel"; - // types that carry key(s) internally without "RedisKey" in their name, so the - // substring check below won't catch them; they rely on a Map extension method - const string StreamPositionType = "StackExchange.Redis.StreamPosition"; - // the Execute/ExecuteAsync escape hatch boxes keys/channels inside a loosely-typed - // arg list; these route through a Map extension that unboxes and rewrites matches - const string ScriptArgArrayType = "object[]"; - const string ScriptArgCollectionType = "System.Collections.Generic.ICollection"; + for (int i = 0; i < tupleDefs.Count; i++) { - var parameters = tupleDefs[i].Span; + var raw = tupleDefs[i]; + var returns = GetTupleMapReturns(raw); + var parameters = raw.Span; // locate the (single) CommandFlags argument, if any, to back IRedisArgs.Flags int flagsArg = -1; @@ -363,7 +402,19 @@ void AppendTupleTypes() if (p != 0) writer.Append(", "); writer.Append(parameters[p].Type).Append(" arg").Append(p); } - writer.Append(") : global::StackExchange.Redis.IRedisArgs").NewLine().Append("{").Indent(); + + writer.Append(") : global::StackExchange.Redis.IRedisArgs"); + if (returns is not null) + { + writer.Indent(); + foreach (var retType in returns) + { + writer.Append(',').NewLine().Append("global::StackExchange.Redis.IRedisArgsResult<") + .Append(retType).Append(">"); + } + writer.Outdent(); + } + writer.NewLine().Append("{").Indent(); for (int p = 0; p < parameters.Length; p++) { writer.NewLine().Append("public ").Append(parameters[p].Type) @@ -387,19 +438,7 @@ void AppendTupleTypes() .NewLine().Append("{").Indent(); for (int p = 0; p < parameters.Length; p++) { - if (parameters[p].Type == RedisKeyType) - { - writer.NewLine().Append("Arg").Append(p).Append(" = mutator.MapKey(Arg").Append(p).Append(");"); - } - else if (parameters[p].Type == RedisChannelType) - { - writer.NewLine().Append("Arg").Append(p).Append(" = mutator.MapChannel(Arg").Append(p).Append(");"); - } - else if (parameters[p].Type.IndexOf(RedisKeyType, StringComparison.Ordinal) >= 0 - || parameters[p].Type.IndexOf(StreamPositionType, StringComparison.Ordinal) >= 0 - || parameters[p].Type == ScriptArgArrayType - || parameters[p].Type == ScriptArgCollectionType - || parameters[p].Type == ScriptArgCollectionType + "?") + if (NeedsMap(parameters[p].Type)) { // key/channel-bearing container (or the loosely-typed script arg list); it // is the library's job to ensure a suitable Map extension method exists @@ -408,6 +447,19 @@ void AppendTupleTypes() } writer.Outdent().NewLine().Append("}"); + if (returns is not null) + { + foreach (var retType in returns) + { + writer.NewLine().NewLine().Append(retType).Append(' ') + .Append("global::StackExchange.Redis.IRedisArgsResult<") + .Append(retType) + .Append(">.UnMap(global::StackExchange.Redis.IRedisArgsMutator mutator, ") + .Append(retType).Append(" value)") + .Indent().NewLine().Append("=> mutator.UnMap(value);").Outdent(); + } + } + writer.Outdent().NewLine().Append("}"); } } diff --git a/src/StackExchange.Redis/AutoDatabase.cs b/src/StackExchange.Redis/AutoDatabase.cs index fe242b0aa..eab9d245e 100644 --- a/src/StackExchange.Redis/AutoDatabase.cs +++ b/src/StackExchange.Redis/AutoDatabase.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; +using System.Runtime.CompilerServices; namespace StackExchange.Redis; @@ -19,8 +20,16 @@ internal interface IRedisArgs internal interface IRedisArgsMutator { - RedisKey MapKey(RedisKey key); - RedisChannel MapChannel(RedisChannel channel); + RedisKey Map(RedisKey key); + RedisChannel Map(RedisChannel channel); + + RedisKey UnMap(RedisKey key); + RedisChannel UnMap(RedisChannel channel); +} + +internal interface IRedisArgsResult +{ + T UnMap(IRedisArgsMutator mutator, T value); } internal static class RedisArgsMutatorExtensions @@ -31,7 +40,12 @@ internal static class RedisArgsMutatorExtensions public static KeyValuePair Map( this IRedisArgsMutator mutator, KeyValuePair value) => - new(mutator.MapKey(value.Key), value.Value); + new(mutator.Map(value.Key), value.Value); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static TResult UnMap(this IRedisArgsMutator mutator, in TState state, TResult result) + where TState : struct, IRedisArgs => + state is IRedisArgsResult unmap ? unmap.UnMap(mutator, result) : result; [return: NotNullIfNotNull("keys")] public static RedisKey[]? Map(this IRedisArgsMutator mutator, RedisKey[]? keys) @@ -40,7 +54,7 @@ public static KeyValuePair Map( var arr = new RedisKey[keys.Length]; for (int i = 0; i < arr.Length; i++) { - arr[i] = mutator.MapKey(keys[i]); + arr[i] = mutator.Map(keys[i]); } return arr; } @@ -55,13 +69,13 @@ public static KeyValuePair Map( for (int i = 0; i < arr.Length; i++) { ref readonly KeyValuePair pair = ref pairs[i]; - arr[i] = new(mutator.MapKey(pair.Key), pair.Value); + arr[i] = new(mutator.Map(pair.Key), pair.Value); } return arr; } public static StreamPosition Map(this IRedisArgsMutator mutator, StreamPosition value) => - new(mutator.MapKey(value.Key), value.Position); + new(mutator.Map(value.Key), value.Position); [return: NotNullIfNotNull("positions")] public static StreamPosition[]? Map(this IRedisArgsMutator mutator, StreamPosition[]? positions) @@ -71,7 +85,7 @@ public static StreamPosition Map(this IRedisArgsMutator mutator, StreamPosition for (int i = 0; i < arr.Length; i++) { ref readonly StreamPosition position = ref positions[i]; - arr[i] = new(mutator.MapKey(position.Key), position.Position); + arr[i] = new(mutator.Map(position.Key), position.Position); } return arr; } @@ -88,11 +102,11 @@ public static StreamPosition Map(this IRedisArgsMutator mutator, StreamPosition { if (args[i] is RedisKey key) { - (copy ??= (object[])args.Clone())[i] = mutator.MapKey(key); + (copy ??= (object[])args.Clone())[i] = mutator.Map(key); } else if (args[i] is RedisChannel channel) { - (copy ??= (object[])args.Clone())[i] = mutator.MapChannel(channel); + (copy ??= (object[])args.Clone())[i] = mutator.Map(channel); } } return copy ?? args; @@ -119,11 +133,17 @@ public static StreamPosition Map(this IRedisArgsMutator mutator, StreamPosition { copy[i++] = arg switch { - RedisKey key => mutator.MapKey(key), - RedisChannel channel => mutator.MapChannel(channel), + RedisKey key => mutator.Map(key), + RedisChannel channel => mutator.Map(channel), _ => arg, }; } return copy; } + + public static SortedSetPopResult UnMap(this IRedisArgsMutator mutator, SortedSetPopResult value) => + value.IsNull ? SortedSetPopResult.Null : new(mutator.Map(value.Key), value.Entries); + + public static ListPopResult UnMap(this IRedisArgsMutator mutator, ListPopResult value) => + value.IsNull ? ListPopResult.Null : new(mutator.Map(value.Key), value.Values); } diff --git a/src/StackExchange.Redis/Availability/RetryDatabase.cs b/src/StackExchange.Redis/Availability/RetryDatabase.cs index fac31829c..d4e9abc4f 100644 --- a/src/StackExchange.Redis/Availability/RetryDatabase.cs +++ b/src/StackExchange.Redis/Availability/RetryDatabase.cs @@ -5,11 +5,15 @@ namespace StackExchange.Redis.Availability; [AutoDatabase] -internal partial class RetryDatabase : IDatabase +internal partial class RetryDatabase : IDatabase, IRedisArgsMutator { private readonly IDatabase _inner; - - public RetryDatabase(IDatabase inner) => _inner = inner; + private readonly int _maxRetryCount; + public RetryDatabase(IDatabase inner, int maxRetryCount) + { + _inner = inner; + _maxRetryCount = maxRetryCount; + } public int Database => _inner.Database; public IConnectionMultiplexer Multiplexer => _inner.Multiplexer; @@ -20,7 +24,25 @@ internal partial class RetryDatabase : IDatabase // policy will live here in due course; for now it is a straight pass-through. private TResult Execute(TState state, Func operation) where TState : struct, IRedisArgs - => operation(state, _inner); + { + state.Map(this); + int i = 0; + TResult result; + while (true) + { + try + { + result = operation(state, _inner); + break; + } + catch (Exception ex) when (++i < _maxRetryCount) + { + // we can give it another attempt + Debug.WriteLine(ex.Message); + } + } + return this.UnMap(state, result); + } private void Execute(TState state, Action operation) where TState : struct, IRedisArgs @@ -57,4 +79,11 @@ private Task ExecuteAsync(TState state, Func op System.Collections.Generic.IAsyncEnumerable IDatabaseAsync.SetScanAsync(RedisKey key, RedisValue pattern, int pageSize, long cursor, int pageOffset, CommandFlags flags) => throw new NotImplementedException(); System.Collections.Generic.IAsyncEnumerable IDatabaseAsync.VectorSetRangeEnumerateAsync(RedisKey key, RedisValue start, RedisValue end, long count, Exclude exclude, CommandFlags flags) => throw new NotImplementedException(); System.Collections.Generic.IAsyncEnumerable IDatabaseAsync.SortedSetScanAsync(RedisKey key, RedisValue pattern, int pageSize, long cursor, int pageOffset, CommandFlags flags) => throw new NotImplementedException(); + + RedisKey IRedisArgsMutator.Map(RedisKey key) => key; + + RedisChannel IRedisArgsMutator.Map(RedisChannel channel) => channel; + + RedisKey IRedisArgsMutator.UnMap(RedisKey key) => key; + RedisChannel IRedisArgsMutator.UnMap(RedisChannel channel) => channel; } From 7a926cb3b70d24d925488263c35f2b330c8f3cdd Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Tue, 14 Jul 2026 17:05:39 +0100 Subject: [PATCH 08/23] fundamentals: compile (untested) --- .../AutoDatabaseGenerator.cs | 114 +++++++++++------- src/StackExchange.Redis/AutoDatabase.cs | 24 +++- .../Availability/RetryDatabase.cs | 64 +++++++++- 3 files changed, 153 insertions(+), 49 deletions(-) diff --git a/eng/StackExchange.Redis.Build/AutoDatabaseGenerator.cs b/eng/StackExchange.Redis.Build/AutoDatabaseGenerator.cs index b75e056e5..f81dea6e4 100644 --- a/eng/StackExchange.Redis.Build/AutoDatabaseGenerator.cs +++ b/eng/StackExchange.Redis.Build/AutoDatabaseGenerator.cs @@ -198,6 +198,22 @@ private static bool SkipMethod(MethodInfo method) || method.ReturnType.StartsWith("System.Collections.Generic.IEnumerable<", StringComparison.Ordinal) || method.ReturnType.StartsWith("System.Collections.Generic.IAsyncEnumerable<", StringComparison.Ordinal); + private const string TaskType = "System.Threading.Tasks.Task"; + + // Task / Task returns route through ExecuteAsync (its own retry policy); everything else + // uses Execute. + private static bool IsAsync(string returnType) + => returnType.StartsWith(TaskType, StringComparison.Ordinal); + + // the retry machinery unwraps the Task and only ever sees the inner result, so key/channel + // (un)mapping must be decided on T, not Task; this strips the Task<...> wrapper from an async + // return type. A bare (non-generic) Task has no result and is returned unchanged (as is any + // non-async return type), which is harmless since neither matches NeedsMap. + private static string StripTask(string returnType) + => IsAsync(returnType) && returnType.StartsWith(TaskType + "<", StringComparison.Ordinal) + ? returnType.Substring(TaskType.Length + 1, returnType.Length - TaskType.Length - 2) + : returnType; + private static string FormatDefault(object? value) => value switch { null => "null", @@ -237,9 +253,14 @@ private static void Generate(SourceProductionContext ctx, ImmutableArray? ReturnTypes)>(); + var tupleIndex = new Dictionary(); var tupleDefs = new List>(); + // every unique key/channel-bearing result type across all shapes; a single shared + // singleton (emitted after the tuples) implements IRedisArgsResult for each, so + // tuples expose unmap dispatch via UnMapper without ever being cast to an interface + var allReturns = new HashSet(); + bool isFirst = true; writer.NewLine().Append("partial class ").Append(cls.Name); AppendInterfaceDeclaration(KnownInterfaces.IDatabase); @@ -288,11 +309,16 @@ void AppendInterfaceMethods(KnownInterfaces knownType) } writer.Append(')').Indent().NewLine(); - // Task / Task methods route to ExecuteAsync (its own retry policy); everything - // else uses Execute. The state struct is shared regardless of return type - keyed on - // parameter types only - so e.g. ArraySet and ArraySetAsync land on the same _tupleN. - bool isAsync = method.ReturnType.StartsWith("System.Threading.Tasks.Task", StringComparison.Ordinal); - int tuple = GetTupleIndex(method.Parameters, method.ReturnType); + // async methods route to ExecuteAsync (its own retry policy); everything else uses + // Execute. The state struct is shared regardless of return type - keyed on parameter + // types only - so e.g. ArraySet and ArraySetAsync land on the same _tupleN. The return + // type is Task-stripped so unmapping is keyed on the inner result the retry machinery + // actually sees, not the Task wrapper. + bool isAsync = IsAsync(method.ReturnType); + var simpleResult = StripTask(method.ReturnType); + bool needsMap = NeedsMap(simpleResult); + if (needsMap) allReturns.Add(simpleResult); + int tuple = GetTupleIndex(method.Parameters, needsMap); writer.Append(isAsync ? "=> ExecuteAsync(new _tuple" : "=> Execute(new _tuple").Append(tuple).Append("("); firstParam = true; foreach (var p in method.Parameters.Span) @@ -322,32 +348,26 @@ static string GetTupleKey(BasicArray parameters) return keyBuilder.ToString(); } - HashSet? GetTupleMapReturns(BasicArray parameters) => - tupleIndex.TryGetValue(GetTupleKey(parameters), out var found) - ? found.ReturnTypes : null; + bool TupleNeedsMap(BasicArray parameters) => + tupleIndex.TryGetValue(GetTupleKey(parameters), out var found) && found.NeedsMap; - int GetTupleIndex(BasicArray parameters, string returnType) + int GetTupleIndex(BasicArray parameters, bool needsMap) { var key = GetTupleKey(parameters); - bool map = NeedsMap(returnType); int index; if (tupleIndex.TryGetValue(key, out var found)) { index = found.Index; - if (map) + if (needsMap && !found.NeedsMap) { - if (found.ReturnTypes is null) - { - found.ReturnTypes = []; - tupleIndex[key] = found; - } - found.ReturnTypes.Add(returnType); + found.NeedsMap = true; + tupleIndex[key] = found; } } else { index = tupleDefs.Count; - tupleIndex.Add(key, (index, map ? [ returnType ] : null)); + tupleIndex.Add(key, (index, needsMap)); tupleDefs.Add(parameters); } @@ -377,11 +397,10 @@ static bool NeedsMap(string name) => void AppendTupleTypes() { - for (int i = 0; i < tupleDefs.Count; i++) { var raw = tupleDefs[i]; - var returns = GetTupleMapReturns(raw); + bool needsMap = TupleNeedsMap(raw); var parameters = raw.Span; // locate the (single) CommandFlags argument, if any, to back IRedisArgs.Flags @@ -404,16 +423,6 @@ void AppendTupleTypes() } writer.Append(") : global::StackExchange.Redis.IRedisArgs"); - if (returns is not null) - { - writer.Indent(); - foreach (var retType in returns) - { - writer.Append(',').NewLine().Append("global::StackExchange.Redis.IRedisArgsResult<") - .Append(retType).Append(">"); - } - writer.Outdent(); - } writer.NewLine().Append("{").Indent(); for (int p = 0; p < parameters.Length; p++) { @@ -447,19 +456,40 @@ void AppendTupleTypes() } writer.Outdent().NewLine().Append("}"); - if (returns is not null) + // tuples with key/channel-bearing results point UnMapper at the shared singleton + // (which knows how to unmap every such result type); the rest return null so the + // Execute helper skips unmapping entirely - and neither path boxes the struct + writer.NewLine().Append("public readonly object? UnMapper => ") + .Append(needsMap ? "_UnMapper.Instance;" : "null;"); + + writer.Outdent().NewLine().Append("}"); + } + + if (allReturns.Count is not 0) + { + // sorted for deterministic (cache-stable) output + var ordered = new List(allReturns); + ordered.Sort(StringComparer.Ordinal); + + writer.NewLine().NewLine().Append("private sealed class _UnMapper").Indent(); + bool firstIface = true; + foreach (var retType in ordered) { - foreach (var retType in returns) - { - writer.NewLine().NewLine().Append(retType).Append(' ') - .Append("global::StackExchange.Redis.IRedisArgsResult<") - .Append(retType) - .Append(">.UnMap(global::StackExchange.Redis.IRedisArgsMutator mutator, ") - .Append(retType).Append(" value)") - .Indent().NewLine().Append("=> mutator.UnMap(value);").Outdent(); - } + writer.NewLine().Append(firstIface ? ": " : ", ") + .Append("global::StackExchange.Redis.IRedisArgsResult<").Append(retType).Append(">"); + firstIface = false; + } + writer.Outdent().NewLine().Append("{").Indent(); + writer.NewLine().Append("public static readonly _UnMapper Instance = new();"); + foreach (var retType in ordered) + { + writer.NewLine().NewLine().Append(retType).Append(' ') + .Append("global::StackExchange.Redis.IRedisArgsResult<") + .Append(retType) + .Append(">.UnMap(global::StackExchange.Redis.IRedisArgsMutator mutator, ") + .Append(retType).Append(" value)") + .Indent().NewLine().Append("=> mutator.UnMap(value);").Outdent(); } - writer.Outdent().NewLine().Append("}"); } } diff --git a/src/StackExchange.Redis/AutoDatabase.cs b/src/StackExchange.Redis/AutoDatabase.cs index eab9d245e..229c63b60 100644 --- a/src/StackExchange.Redis/AutoDatabase.cs +++ b/src/StackExchange.Redis/AutoDatabase.cs @@ -16,6 +16,7 @@ internal interface IRedisArgs { void Map(IRedisArgsMutator mutator); CommandFlags Flags { get; set; } + object? UnMapper { get; } } internal interface IRedisArgsMutator @@ -44,8 +45,27 @@ public static KeyValuePair Map( [MethodImpl(MethodImplOptions.AggressiveInlining)] public static TResult UnMap(this IRedisArgsMutator mutator, in TState state, TResult result) - where TState : struct, IRedisArgs => - state is IRedisArgsResult unmap ? unmap.UnMap(mutator, result) : result; + where TState : struct, IRedisArgs + { + if (typeof(TResult) == typeof(RedisKey)) + { + var tmp = mutator.UnMap(Unsafe.As(ref result)); + return Unsafe.As(ref tmp); + } + else if (typeof(TResult) == typeof(RedisChannel)) + { + var tmp = mutator.UnMap(Unsafe.As(ref result)); + return Unsafe.As(ref tmp); + } + else if (typeof(TResult) == typeof(RedisValue)) + { + return result; // never mapped + } + else + { + return state.UnMapper is IRedisArgsResult unmap ? unmap.UnMap(mutator, result) : result; + } + } [return: NotNullIfNotNull("keys")] public static RedisKey[]? Map(this IRedisArgsMutator mutator, RedisKey[]? keys) diff --git a/src/StackExchange.Redis/Availability/RetryDatabase.cs b/src/StackExchange.Redis/Availability/RetryDatabase.cs index d4e9abc4f..7c569dcd6 100644 --- a/src/StackExchange.Redis/Availability/RetryDatabase.cs +++ b/src/StackExchange.Redis/Availability/RetryDatabase.cs @@ -41,21 +41,75 @@ private TResult Execute(TState state, Func(TState state, Action operation) where TState : struct, IRedisArgs - => operation(state, _inner); + { + state.Map(this); + int i = 0; + while (true) + { + try + { + operation(state, _inner); + return; + } + catch (Exception ex) when (++i < _maxRetryCount) + { + // we can give it another attempt + Debug.WriteLine(ex.Message); + } + } + } // async counterparts (Task / Task); these get their own retry/failover policy in due course. - private Task ExecuteAsync(TState state, Func> operation) + private async Task ExecuteAsync(TState state, Func> operation) where TState : struct, IRedisArgs - => operation(state, _inner); + { + state.Map(this); + + int i = 0; + TResult result; + while (true) + { + try + { + result = await operation(state, _inner).ConfigureAwait(false); + break; + } + catch (Exception ex) when (++i < _maxRetryCount) + { + // we can give it another attempt + Debug.WriteLine(ex.Message); + } + } + // post-process results outside the loop + return this.UnMap(state, result); + } - private Task ExecuteAsync(TState state, Func operation) + private async Task ExecuteAsync(TState state, Func operation) where TState : struct, IRedisArgs - => operation(state, _inner); + { + state.Map(this); + + int i = 0; + while (true) + { + try + { + await operation(state, _inner).ConfigureAwait(false); + return; + } + catch (Exception ex) when (++i < _maxRetryCount) + { + // we can give it another attempt + Debug.WriteLine(ex.Message); + } + } + } void IRedisAsync.Wait(Task task) => _inner.Wait(task); T IRedisAsync.Wait(Task task) => _inner.Wait(task); From d925df0061249ba7261eaa5f46315c7a9642c6dd Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Wed, 15 Jul 2026 16:35:43 +0100 Subject: [PATCH 09/23] I *think* this is the core retry logic! --- .../AutoDatabaseGenerator.cs | 4 + .../Availability/CircuitBreaker.cs | 12 +- .../Availability/MultiGroupDatabase.cs | 18 ++- .../Availability/MultiGroupMultiplexer.cs | 56 ++++++-- .../Availability/RetryDatabase.cs | 134 +++++++++++------- .../Availability/RetryPolicy.cs | 65 +++++++++ .../ConnectionMultiplexer.cs | 2 + .../Interfaces/IConnectionMultiplexer.cs | 3 + .../Interfaces/IInternalDatabaseAsync.cs | 64 +++++++++ .../KeyspaceIsolation/DatabaseExtension.cs | 10 ++ .../KeyspaceIsolation/KeyPrefixed.cs | 17 ++- .../KeyspaceIsolation/KeyPrefixedBatch.cs | 12 +- .../KeyPrefixedTransaction.cs | 9 +- .../PublicAPI/PublicAPI.Unshipped.txt | 31 +--- .../PublicAPI/net6.0/PublicAPI.Unshipped.txt | 9 ++ src/StackExchange.Redis/RedisBatch.cs | 10 +- src/StackExchange.Redis/RedisDatabase.cs | 18 ++- src/StackExchange.Redis/RedisTransaction.cs | 5 + .../ServerSelectionStrategy.cs | 4 +- .../Helpers/SharedConnectionFixture.cs | 2 + 20 files changed, 387 insertions(+), 98 deletions(-) create mode 100644 src/StackExchange.Redis/Availability/RetryPolicy.cs create mode 100644 src/StackExchange.Redis/Interfaces/IInternalDatabaseAsync.cs diff --git a/eng/StackExchange.Redis.Build/AutoDatabaseGenerator.cs b/eng/StackExchange.Redis.Build/AutoDatabaseGenerator.cs index f81dea6e4..87a78869a 100644 --- a/eng/StackExchange.Redis.Build/AutoDatabaseGenerator.cs +++ b/eng/StackExchange.Redis.Build/AutoDatabaseGenerator.cs @@ -191,10 +191,14 @@ internal enum KnownInterfaces // methods that don't fit the capture-and-replay shape are left for the caller to implement manually: // - generic methods (open type args can't be captured into a concrete state struct) // - the Wait family (synchronization over caller-supplied Tasks, not server calls) + // - IsConnected: a synchronous status probe that IDatabaseAsync carries; it would route through the + // sync Execute funnel, which an async-only database (e.g. RetryDatabase) doesn't provide - and a + // connection check shouldn't be retried in any case // - streaming returns (IEnumerable / IAsyncEnumerable) whose execution is deferred private static bool SkipMethod(MethodInfo method) => !method.TypeArgs.IsEmpty || method.Name.Contains("Wait") + || method.Name == "IsConnected" || method.ReturnType.StartsWith("System.Collections.Generic.IEnumerable<", StringComparison.Ordinal) || method.ReturnType.StartsWith("System.Collections.Generic.IAsyncEnumerable<", StringComparison.Ordinal); diff --git a/src/StackExchange.Redis/Availability/CircuitBreaker.cs b/src/StackExchange.Redis/Availability/CircuitBreaker.cs index feaa65399..4b73beaf4 100644 --- a/src/StackExchange.Redis/Availability/CircuitBreaker.cs +++ b/src/StackExchange.Redis/Availability/CircuitBreaker.cs @@ -25,6 +25,12 @@ public abstract class CircuitBreaker /// public static CircuitBreaker None => NulCircuitBreaker.Instance; + /// + /// Indicates a fault that should be handled by the circuit-breaker or related retry logic. + /// + public virtual bool IsConnectionFault(Exception? fault) + => fault is RedisTimeoutException or RedisConnectionException; + /// /// Allows configuration of the default implementation. /// @@ -187,6 +193,8 @@ private NulAccumulator() { } public override bool IsHealthy() => true; public override void Reset() { } } + + // note we leave IsConnectionFault alone - that would impact RetryDatabase, where this is the key } private sealed class DefaultCircuitBreaker : CircuitBreaker { @@ -305,7 +313,7 @@ public void Reset() public override bool ObserveResult(in CircuitBreakerContext context) { // not-tracked failures (based on exception type) count as "success" for the purposes of circuit breaking - bool countAsSuccess = context.Success || !breaker.IsTracked(context.Fault); + bool countAsSuccess = context.Success || !breaker.IsConnectionFault(context.Fault); // which time-slice are we in, and where does it live in the ring? long epoch = breaker.GetEpoch(); @@ -415,7 +423,7 @@ static bool Contains(Type[] array, Type type) return tracked; } - private bool IsTracked(Exception? fault) + public override bool IsConnectionFault(Exception? fault) { if (fault is null) return false; switch (_trackingStrategy) diff --git a/src/StackExchange.Redis/Availability/MultiGroupDatabase.cs b/src/StackExchange.Redis/Availability/MultiGroupDatabase.cs index 77e0fe4d5..7bfc18033 100644 --- a/src/StackExchange.Redis/Availability/MultiGroupDatabase.cs +++ b/src/StackExchange.Redis/Availability/MultiGroupDatabase.cs @@ -1,15 +1,31 @@ using System; +using System.Threading; +using StackExchange.Redis.Interfaces; namespace StackExchange.Redis.Availability; internal sealed partial class MultiGroupDatabase(MultiGroupMultiplexer parent, int database, object? asyncState) - : IDatabase + : IDatabase, IInternalDatabaseAsync { public object? AsyncState => asyncState; public int Database => database < 0 ? GetActiveDatabase().Database : database; public IConnectionMultiplexer Multiplexer => parent; + CancellationToken IInternalDatabaseAsync.GetNextFailover() => parent.GetNextFailover(); + + DatabaseFeatureFlags IInternalDatabaseAsync.GetFeatures(out string name) + { + // fold in the currently-active member's features (when there is one) and label with its name + var active = TryGetActiveDatabase(); + var features = active is null ? DatabaseFeatureFlags.None : active.GetFeatures(out _); + name = ((IConnectionGroup)parent).ActiveMember?.Name ?? ""; + return features | DatabaseFeatureFlags.ConnectionGroup | DatabaseFeatureFlags.Failover; + } + + // uses the active member's name (via GetFeatures) when a member is active + public override string ToString() => this.BuildString(); + // for high DB numbers this might allocate even for null async-state scenarios; unavoidable for now private IDatabase GetActiveDatabase() => parent.Active.GetDatabase(database, asyncState); diff --git a/src/StackExchange.Redis/Availability/MultiGroupMultiplexer.cs b/src/StackExchange.Redis/Availability/MultiGroupMultiplexer.cs index 378489cd4..4ff4eb872 100644 --- a/src/StackExchange.Redis/Availability/MultiGroupMultiplexer.cs +++ b/src/StackExchange.Redis/Availability/MultiGroupMultiplexer.cs @@ -320,12 +320,46 @@ internal void UpdateLatency() internal sealed partial class MultiGroupMultiplexer : IConnectionGroup { - private ConnectionMultiplexer? _active; + private ActiveStub _activeStub = new(null); + + private void SetActive(ConnectionMultiplexer? active) + { + ActiveStub? newObj = null; + while (true) + { + var oldObj = Volatile.Read(ref _activeStub); + + // is it already the same? + if (ReferenceEquals(oldObj.Active, active)) + { + newObj?.Dispose(); // never actually released to the world + return; // nothing to do! + } + + newObj ??= new(active); + if (ReferenceEquals(Interlocked.CompareExchange(ref _activeStub, newObj, oldObj), oldObj)) + { + // successful swap; flag the old one as failed-over + oldObj.Cancel(false); + return; + } + + // race? redo from start, but we can keep our newObj if we created one + } + } + + public CancellationToken GetNextFailover() => _activeStub.Token; + + private sealed class ActiveStub(ConnectionMultiplexer? active) : CancellationTokenSource + { + public ConnectionMultiplexer? Active => active; + } + private ConnectionGroupMember[] _members; public override string ToString() { - var muxer = _active; + var muxer = _activeStub.Active; ConnectionGroupMember? member = null; if (muxer is not null) { @@ -348,7 +382,7 @@ internal ConnectionMultiplexer Active { get { - return _active ?? Throw(); + return _activeStub.Active ?? Throw(); [DoesNotReturn] static ConnectionMultiplexer Throw() => @@ -357,12 +391,12 @@ static ConnectionMultiplexer Throw() => } // non-throwing twin of Active, for callers that have a trivial answer when the group is fully down - internal ConnectionMultiplexer? TryGetActive() => _active; + internal ConnectionMultiplexer? TryGetActive() => _activeStub.Active; // a completed "no endpoint" result, shared by the database/subscriber facades when the group is fully down internal static readonly Task NoEndpoint = Task.FromResult(null); - private ConnectionGroupMember? GetActiveMember() => GetMember(_active); + private ConnectionGroupMember? GetActiveMember() => GetMember(_activeStub.Active); private ConnectionGroupMember? GetMember(ConnectionMultiplexer? muxer) { @@ -433,7 +467,7 @@ private MultiGroupMultiplexer(ConnectionGroupMember[] members, MultiGroupOptions { _options = options; _members = members; - _active = null; + SetActive(null); _connectionFailedWithCircuitBreaker = OnMemberConnectionFailed; // multiplexers should already be attached (ConnectAsync sets them before constructing us) @@ -575,7 +609,7 @@ private long GetFailbackFailureCutoff() internal void SelectPreferredGroup() { if (_disposed) return; - var existingActive = _active; + var existingActive = _activeStub.Active; ConnectionGroupMember? preferredMember = null, previousMember = null; var members = _members; foreach (var member in members) @@ -594,7 +628,7 @@ internal void SelectPreferredGroup() } } - _active = preferredMember?.Multiplexer; + SetActive(preferredMember?.Multiplexer); if (preferredMember is not null && !ReferenceEquals(preferredMember, previousMember)) { @@ -610,7 +644,7 @@ internal void SelectPreferredGroup() private List DropAll() { _pollCancellation.Cancel(); // stop the polling loop promptly (idempotent) - _active = null; + SetActive(null); var members = Interlocked.Exchange(ref _members, []); if (members.Length is 0) return []; var muxers = new List(members.Length); @@ -671,8 +705,8 @@ public bool PreserveAsyncOrder // Unlike most members, these intentionally do *not* go via Active (which throws when no member is // available); callers routinely use IsConnected/IsConnecting as a pre-flight check and expect a // 'false' result - not an exception - when the entire group is down. - public bool IsConnected => _active?.IsConnected ?? false; - public bool IsConnecting => _active?.IsConnecting ?? false; + public bool IsConnected => _activeStub.Active?.IsConnected ?? false; + public bool IsConnecting => _activeStub.Active?.IsConnecting ?? false; [Obsolete] public bool IncludeDetailInExceptions diff --git a/src/StackExchange.Redis/Availability/RetryDatabase.cs b/src/StackExchange.Redis/Availability/RetryDatabase.cs index 7c569dcd6..ffe7ebda4 100644 --- a/src/StackExchange.Redis/Availability/RetryDatabase.cs +++ b/src/StackExchange.Redis/Availability/RetryDatabase.cs @@ -1,112 +1,148 @@ using System; using System.Diagnostics; +using System.Threading; using System.Threading.Tasks; +using StackExchange.Redis.Interfaces; namespace StackExchange.Redis.Availability; [AutoDatabase] -internal partial class RetryDatabase : IDatabase, IRedisArgsMutator +internal partial class RetryDatabase : IDatabaseAsync, IRedisArgsMutator, IInternalDatabaseAsync { - private readonly IDatabase _inner; - private readonly int _maxRetryCount; - public RetryDatabase(IDatabase inner, int maxRetryCount) + // Note: we very deliberately do not include synchronous support for retry; it is inherently delay-ish + + // TODO: use message category; retrying a GET is very different to SET, SETNX, INCR, etc + + // Note that only connection faults (as defined by the circuit-breaker, or the default circuit-breaker if + // not supplied) result in retries; we don't retry caller error. + DatabaseFeatureFlags IInternalDatabaseAsync.GetFeatures(out string name) + => _inner.GetFeatures(out name) | DatabaseFeatureFlags.Retry; + + /// + public override string ToString() => this.BuildString(); + + private readonly IDatabaseAsync _inner; + private readonly CircuitBreaker _circuitBreaker; + private readonly int _maxBeforeFailover, _maxAttempts, _delayMillis, _jitterMillis, _failoverMillis; + + public CancellationToken GetNextFailover() => _inner.GetNextFailover(); + + public RetryDatabase(IDatabaseAsync inner, RetryPolicy policy) { + // cannot nest retry, and cannot issue retries *inside* a batch/transaction + var features = inner.RejectFlags(DatabaseFeatureFlags.Batch | DatabaseFeatureFlags.Transaction | DatabaseFeatureFlags.Retry); + + // capture config locally rather than constant cross-object lookups (plus: mutability) + _maxBeforeFailover = (features & DatabaseFeatureFlags.Failover) == 0 ? int.MaxValue : policy.MaxAttemptsBeforeFailover; + _maxAttempts = policy.MaxAttempts; + if (_maxBeforeFailover == _maxAttempts) _maxBeforeFailover = int.MaxValue; // then we'll never look + if (_maxAttempts < 1) throw new ArgumentOutOfRangeException(nameof(policy.MaxAttempts)); + _delayMillis = policy.DelayMilliseconds; + _failoverMillis = policy.FailoverMilliseconds; + _jitterMillis = policy.JitterMilliseconds; + if (_delayMillis < 0) throw new ArgumentOutOfRangeException(nameof(policy.RetryDelay)); + if (_jitterMillis < 0) throw new ArgumentOutOfRangeException(nameof(policy.JitterMax)); + if (_failoverMillis < 0) throw new ArgumentOutOfRangeException(nameof(policy.FailoverDelay)); _inner = inner; - _maxRetryCount = maxRetryCount; + _circuitBreaker = policy.CircuitBreaker ?? (inner.Multiplexer as IInternalConnectionMultiplexer)?.CircuitBreaker ?? CircuitBreaker.Default; } - public int Database => _inner.Database; + public int Database => _inner is IDatabase db ? db.Database : -1; + public IConnectionMultiplexer Multiplexer => _inner.Multiplexer; // the generated explicit interface implementations funnel every call through these two // overloads: the arguments are captured in a generated state struct and replayed against // the inner database via a cacheable static projection (no per-call closure). Retry/failover // policy will live here in due course; for now it is a straight pass-through. - private TResult Execute(TState state, Func operation) + + // async counterparts (Task / Task); these get their own retry/failover policy in due course. + private async Task ExecuteAsync(TState state, Func> operation) where TState : struct, IRedisArgs { state.Map(this); + int i = 0; TResult result; + CancellationToken ct = CancellationToken.None; while (true) { try { - result = operation(state, _inner); + // note we need to capture this *before* the attempt - otherwise the failover could happen + // between the failed attempt and fetching this, and we'd miss it + if (++i == _maxBeforeFailover) ct = GetNextFailover(); + result = await operation(state, _inner).ConfigureAwait(false); break; } - catch (Exception ex) when (++i < _maxRetryCount) + catch (Exception ex) when (_circuitBreaker.IsConnectionFault(ex) && i < _maxAttempts) { // we can give it another attempt Debug.WriteLine(ex.Message); + await FailoverOrDelayAsync(ct).ConfigureAwait(false); + ct = CancellationToken.None; // we only apply failover one time } } // post-process results outside the loop return this.UnMap(state, result); } - private void Execute(TState state, Action operation) - where TState : struct, IRedisArgs + private Task FailoverOrDelayAsync(CancellationToken failover) { - state.Map(this); - int i = 0; - while (true) + if (failover.CanBeCanceled) { - try - { - operation(state, _inner); - return; - } - catch (Exception ex) when (++i < _maxRetryCount) - { - // we can give it another attempt - Debug.WriteLine(ex.Message); - } + // we're in the failover slot; this gets exciting + return AwaitFailover(failover); + } + else + { + // this is just a routine wait between operations; await delay+jitter + return Task.Delay(_delayMillis + ServerSelectionStrategy.SharedRandom.Next(_jitterMillis), CancellationToken.None); } } - - // async counterparts (Task / Task); these get their own retry/failover policy in due course. - private async Task ExecuteAsync(TState state, Func> operation) - where TState : struct, IRedisArgs + private async Task AwaitFailover(CancellationToken failover) { - state.Map(this); - - int i = 0; - TResult result; - while (true) + if (!failover.IsCancellationRequested) { + // failover hasn't happened yet; allow up to "delay" time for that try { - result = await operation(state, _inner).ConfigureAwait(false); - break; + await Task.Delay(_failoverMillis, failover).ConfigureAwait(false); } - catch (Exception ex) when (++i < _maxRetryCount) + catch (OperationCanceledException) when (failover.IsCancellationRequested) { - // we can give it another attempt - Debug.WriteLine(ex.Message); + // we observed a failover, nice! } } - // post-process results outside the loop - return this.UnMap(state, result); + + // either way, we need to add jitter onto that; we can't add in the original delay, because if the failover + // happened before the timeout+jitter, all the awaiters would stampede + await Task.Delay(ServerSelectionStrategy.SharedRandom.Next(_jitterMillis), CancellationToken.None).ConfigureAwait(false); } - private async Task ExecuteAsync(TState state, Func operation) + private async Task ExecuteAsync(TState state, Func operation) where TState : struct, IRedisArgs { state.Map(this); int i = 0; + CancellationToken ct = CancellationToken.None; while (true) { try { + // note we need to capture this *before* the attempt - otherwise the failover could happen + // between the failed attempt and fetching this, and we'd miss it + if (++i == _maxBeforeFailover) ct = GetNextFailover(); await operation(state, _inner).ConfigureAwait(false); return; } - catch (Exception ex) when (++i < _maxRetryCount) + catch (Exception ex) when (_circuitBreaker.IsConnectionFault(ex) && i < _maxAttempts) { // we can give it another attempt Debug.WriteLine(ex.Message); + await FailoverOrDelayAsync(ct).ConfigureAwait(false); + ct = CancellationToken.None; // we only apply failover one time } } } @@ -117,16 +153,10 @@ private async Task ExecuteAsync(TState state, Func _inner.TryWait(task); // Methods the generator deliberately skips (see AutoDatabaseGenerator.SkipMethod): the Wait - // family and the streaming IEnumerable/IAsyncEnumerable scans don't fit the capture-and-replay - // shape, so they are implemented by hand. - System.Collections.Generic.IEnumerable IDatabase.HashScan(RedisKey key, RedisValue pattern, int pageSize, CommandFlags flags) => throw new NotImplementedException(); - System.Collections.Generic.IEnumerable IDatabase.HashScan(RedisKey key, RedisValue pattern, int pageSize, long cursor, int pageOffset, CommandFlags flags) => throw new NotImplementedException(); - System.Collections.Generic.IEnumerable IDatabase.HashScanNoValues(RedisKey key, RedisValue pattern, int pageSize, long cursor, int pageOffset, CommandFlags flags) => throw new NotImplementedException(); - System.Collections.Generic.IEnumerable IDatabase.SetScan(RedisKey key, RedisValue pattern, int pageSize, CommandFlags flags) => throw new NotImplementedException(); - System.Collections.Generic.IEnumerable IDatabase.SetScan(RedisKey key, RedisValue pattern, int pageSize, long cursor, int pageOffset, CommandFlags flags) => throw new NotImplementedException(); - System.Collections.Generic.IEnumerable IDatabase.VectorSetRangeEnumerate(RedisKey key, RedisValue start, RedisValue end, long count, Exclude exclude, CommandFlags flags) => throw new NotImplementedException(); - System.Collections.Generic.IEnumerable IDatabase.SortedSetScan(RedisKey key, RedisValue pattern, int pageSize, CommandFlags flags) => throw new NotImplementedException(); - System.Collections.Generic.IEnumerable IDatabase.SortedSetScan(RedisKey key, RedisValue pattern, int pageSize, long cursor, int pageOffset, CommandFlags flags) => throw new NotImplementedException(); + // family, the synchronous IsConnected probe, and the streaming IEnumerable/IAsyncEnumerable scans + // don't fit the capture-and-replay shape, so they are implemented by hand. + // IsConnected is a straight pass-through: it is a cheap status check, not a server round-trip to retry. + bool IDatabaseAsync.IsConnected(RedisKey key, CommandFlags flags) => _inner.IsConnected(key, flags); System.Collections.Generic.IAsyncEnumerable IDatabaseAsync.HashScanAsync(RedisKey key, RedisValue pattern, int pageSize, long cursor, int pageOffset, CommandFlags flags) => throw new NotImplementedException(); System.Collections.Generic.IAsyncEnumerable IDatabaseAsync.HashScanNoValuesAsync(RedisKey key, RedisValue pattern, int pageSize, long cursor, int pageOffset, CommandFlags flags) => throw new NotImplementedException(); diff --git a/src/StackExchange.Redis/Availability/RetryPolicy.cs b/src/StackExchange.Redis/Availability/RetryPolicy.cs new file mode 100644 index 000000000..680379da7 --- /dev/null +++ b/src/StackExchange.Redis/Availability/RetryPolicy.cs @@ -0,0 +1,65 @@ +using System; +using System.Threading; + +namespace StackExchange.Redis.Availability; + +/// +/// Configures how messages can be retried due to connection / transient faults. Other faults (such as invalid +/// usage) are not retried. +/// +public class RetryPolicy +{ + /// + /// Controls (via ) which faults can be retried; if not supplied, + /// the connection's configured will be used. + /// + public CircuitBreaker? CircuitBreaker { get; set; } + + /// + /// The maximum number of times an operation can be attempted. Defaults to 3. + /// + public int MaxAttempts { get; set; } = 3; + + /// + /// The maximum number of times to retry an operation before waiting for failover; this only currently + /// applies to multi-group connections created via ConnectionMultiplexer.ConnectGroupAsync. + /// Defaults to 1. + /// + public int MaxAttemptsBeforeFailover { get; set; } = 1; + + private int _delayMillis = 1000, _jitterMillis = 500, _failoverMillis = 5000; + + /// + /// Gets the time to wait between retries that are *not* dependent on a failover happening. Defaults to 1 second. + /// + public TimeSpan RetryDelay + { + get => TimeSpan.FromMilliseconds(_delayMillis); + set => _delayMillis = checked((int)value.TotalMilliseconds); + } + + /// + /// Gets the time to wait for a failover, after attempts. Only one + /// failover attempt is awaited. When this applies, is ignored, + /// but is still respected. Defaults to 5 seconds. + /// + public TimeSpan FailoverDelay + { + get => TimeSpan.FromMilliseconds(_failoverMillis); + set => _failoverMillis = checked((int)value.TotalMilliseconds); + } + + /// + /// Gets or sets the upper bound for jitter - additional random delay between retries to prevent stampedes. + /// Defaults to 0.5 seconds, meaning between 0 and 0.5 *additional* seconds on top of . + /// + public TimeSpan JitterMax + { + get => TimeSpan.FromMilliseconds(_jitterMillis); + set => _jitterMillis = checked((int)value.TotalMilliseconds); + } + + internal int DelayMilliseconds => _delayMillis; + internal int JitterMilliseconds => _jitterMillis; + internal int FailoverMilliseconds => _failoverMillis; +} diff --git a/src/StackExchange.Redis/ConnectionMultiplexer.cs b/src/StackExchange.Redis/ConnectionMultiplexer.cs index 769a64b50..8253988cc 100644 --- a/src/StackExchange.Redis/ConnectionMultiplexer.cs +++ b/src/StackExchange.Redis/ConnectionMultiplexer.cs @@ -23,6 +23,8 @@ namespace StackExchange.Redis /// public sealed partial class ConnectionMultiplexer : IInternalConnectionMultiplexer // implies : IConnectionMultiplexer and : IDisposable { + Availability.CircuitBreaker? IInternalConnectionMultiplexer.CircuitBreaker => RawConfig.CircuitBreaker; + // This gets accessed for every received event; let's make sure we can process it "raw" internal readonly byte[]? ConfigurationChangedChannel; // Unique identifier used when tracing diff --git a/src/StackExchange.Redis/Interfaces/IConnectionMultiplexer.cs b/src/StackExchange.Redis/Interfaces/IConnectionMultiplexer.cs index 96b4ce8f6..2fac75bec 100644 --- a/src/StackExchange.Redis/Interfaces/IConnectionMultiplexer.cs +++ b/src/StackExchange.Redis/Interfaces/IConnectionMultiplexer.cs @@ -4,6 +4,7 @@ using System.IO; using System.Net; using System.Threading.Tasks; +using StackExchange.Redis.Availability; using StackExchange.Redis.Maintenance; using StackExchange.Redis.Profiling; using static StackExchange.Redis.ConnectionMultiplexer; @@ -16,6 +17,8 @@ internal interface IInternalConnectionMultiplexer : IConnectionMultiplexer bool IgnoreConnect { get; set; } + CircuitBreaker? CircuitBreaker { get; } + ReadOnlySpan GetServerSnapshot(); ServerEndPoint GetServerEndPoint(EndPoint endpoint); diff --git a/src/StackExchange.Redis/Interfaces/IInternalDatabaseAsync.cs b/src/StackExchange.Redis/Interfaces/IInternalDatabaseAsync.cs new file mode 100644 index 000000000..7ac83afda --- /dev/null +++ b/src/StackExchange.Redis/Interfaces/IInternalDatabaseAsync.cs @@ -0,0 +1,64 @@ +using System; +using System.Threading; + +namespace StackExchange.Redis.Interfaces; + +[Flags] +internal enum DatabaseFeatureFlags +{ + None = 0, + Cluster = 1 << 0, + ConnectionGroup = 1 << 1, + Batch = 1 << 2, + Transaction = 1 << 3, + KeyPrefix = 1 << 4, + Retry = 1 << 5, + Unknown = 1 << 6, + Failover = 1 << 7, +} + +internal interface IInternalDatabaseAsync : IDatabaseAsync +{ + DatabaseFeatureFlags GetFeatures(out string name); + CancellationToken GetNextFailover(); +} + +internal static class InternalDatabaseExtension +{ + internal static DatabaseFeatureFlags GetFeatures(this IDatabaseAsync database, out string name) + { + if (database is IInternalDatabaseAsync idb) + { + return idb.GetFeatures(out name); + } + + name = ""; + return DatabaseFeatureFlags.Unknown; + } + + internal static string BuildString(this IDatabaseAsync database) + { + var features = database.GetFeatures(out string name); + return string.IsNullOrEmpty(name) ? features.ToString() : $"{name}: {features}"; + } + + internal static DatabaseFeatureFlags RejectFlags(this IDatabaseAsync database, DatabaseFeatureFlags incompatible) + { + // note: returns *all* the features of the database provided + var features = database.GetFeatures(out _); + var overlap = features & incompatible; + if (overlap is not 0) Throw(overlap); + return features; + + static void Throw(DatabaseFeatureFlags overlap) => throw new InvalidOperationException( + $"This operation is not compatible with feature(s): {overlap}"); + } + + internal static CancellationToken GetNextFailover(this IDatabaseAsync database) + { + // get a CT that represents the next failover; you might be asking "shouldn't that be a Task getter?" - no, + // because Task *does* have ContinueWith, but it doesn't have any mechanism to *undo* that; conversely, + // CancellationToken is expressly designed with that intent, with Register(..) being scoped. + return database is IInternalDatabaseAsync ida ? ida.GetNextFailover() : CancellationToken.None; + } +} diff --git a/src/StackExchange.Redis/KeyspaceIsolation/DatabaseExtension.cs b/src/StackExchange.Redis/KeyspaceIsolation/DatabaseExtension.cs index 742bc06eb..ae362d834 100644 --- a/src/StackExchange.Redis/KeyspaceIsolation/DatabaseExtension.cs +++ b/src/StackExchange.Redis/KeyspaceIsolation/DatabaseExtension.cs @@ -1,4 +1,6 @@ using System; +using StackExchange.Redis.Availability; +using StackExchange.Redis.Interfaces; namespace StackExchange.Redis.KeyspaceIsolation { @@ -63,5 +65,13 @@ public static IDatabase WithKeyPrefix(this IDatabase database, RedisKey keyPrefi return new KeyPrefixedDatabase(database, keyPrefix.AsPrefix()!); } + + /// + /// Automatically retry operations when connection failure occurs. This has deep integration with + /// SE.Redis concepts, so can respond to server failover events, apply circuit-breaker rules, and + /// respect command effect categorization. + /// + public static IDatabaseAsync WithRetry(this IDatabaseAsync database, RetryPolicy retryPolicy) + => new RetryDatabase(database, retryPolicy); } } diff --git a/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixed.cs b/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixed.cs index 952fe0ed3..df0a3a833 100644 --- a/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixed.cs +++ b/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixed.cs @@ -3,11 +3,13 @@ using System.Diagnostics.CodeAnalysis; using System.Linq; using System.Net; +using System.Threading; using System.Threading.Tasks; +using StackExchange.Redis.Interfaces; namespace StackExchange.Redis.KeyspaceIsolation { - internal partial class KeyPrefixed : IDatabaseAsync where TInner : IDatabaseAsync + internal partial class KeyPrefixed : IDatabaseAsync, IInternalDatabaseAsync where TInner : IDatabaseAsync { internal KeyPrefixed(TInner inner, byte[] keyPrefix) { @@ -21,6 +23,19 @@ internal KeyPrefixed(TInner inner, byte[] keyPrefix) internal byte[] Prefix { get; } + DatabaseFeatureFlags IInternalDatabaseAsync.GetFeatures(out string name) + => Inner.GetFeatures(out name) | GetDatabaseFeatures(); + + CancellationToken IInternalDatabaseAsync.GetNextFailover() => Inner.GetNextFailover(); + + // the flags contributed by this wrapper itself (on top of the inner database); the batch and + // transaction subclasses override to fold in their own flag, mirroring RedisDatabase/RedisBatch/ + // RedisTransaction rather than relying on the inner instance to carry it. + private protected virtual DatabaseFeatureFlags GetDatabaseFeatures() + => DatabaseFeatureFlags.KeyPrefix; + + public override string ToString() => this.BuildString(); + public Task DebugObjectAsync(RedisKey key, CommandFlags flags = CommandFlags.None) => Inner.DebugObjectAsync(ToInner(key), flags); diff --git a/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixedBatch.cs b/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixedBatch.cs index 6f5679a66..a5e07a011 100644 --- a/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixedBatch.cs +++ b/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixedBatch.cs @@ -1,8 +1,16 @@ -namespace StackExchange.Redis.KeyspaceIsolation +using StackExchange.Redis.Interfaces; + +namespace StackExchange.Redis.KeyspaceIsolation { internal sealed class KeyPrefixedBatch : KeyPrefixed, IBatch { - public KeyPrefixedBatch(IBatch inner, byte[] prefix) : base(inner, prefix) { } + public KeyPrefixedBatch(IBatch inner, byte[] prefix) : base(inner, prefix) + { + inner.RejectFlags(DatabaseFeatureFlags.Batch | DatabaseFeatureFlags.Transaction); + } + + private protected override DatabaseFeatureFlags GetDatabaseFeatures() + => base.GetDatabaseFeatures() | DatabaseFeatureFlags.Batch; public void Execute() => Inner.Execute(); } diff --git a/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixedTransaction.cs b/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixedTransaction.cs index 89703ba6a..7b7c6f0ce 100644 --- a/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixedTransaction.cs +++ b/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixedTransaction.cs @@ -1,10 +1,17 @@ using System.Threading.Tasks; +using StackExchange.Redis.Interfaces; namespace StackExchange.Redis.KeyspaceIsolation { internal sealed class KeyPrefixedTransaction : KeyPrefixed, ITransaction { - public KeyPrefixedTransaction(ITransaction inner, byte[] prefix) : base(inner, prefix) { } + public KeyPrefixedTransaction(ITransaction inner, byte[] prefix) : base(inner, prefix) + { + inner.RejectFlags(DatabaseFeatureFlags.Batch | DatabaseFeatureFlags.Transaction); + } + + private protected override DatabaseFeatureFlags GetDatabaseFeatures() + => base.GetDatabaseFeatures() | DatabaseFeatureFlags.Transaction; public ConditionResult AddCondition(Condition condition) => Inner.AddCondition(condition.MapKeys(GetMapFunction())); diff --git a/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt b/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt index 9f2483eb9..6db22118e 100644 --- a/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt @@ -1,9 +1,12 @@ #nullable enable +StackExchange.Redis.Availability.RetryPolicy +StackExchange.Redis.Availability.RetryPolicy.CircuitBreaker.get -> StackExchange.Redis.Availability.CircuitBreaker? +StackExchange.Redis.Availability.RetryPolicy.CircuitBreaker.set -> void +StackExchange.Redis.Availability.RetryPolicy.MaxAttempts.get -> int +StackExchange.Redis.Availability.RetryPolicy.MaxAttempts.set -> void +StackExchange.Redis.Availability.RetryPolicy.RetryPolicy() -> void StackExchange.Redis.IServer.InventKey(StackExchange.Redis.RedisKey prefix = default(StackExchange.Redis.RedisKey)) -> StackExchange.Redis.RedisKey -[SER007]abstract StackExchange.Redis.Availability.CircuitBreaker.Accumulator.IsHealthy() -> bool -[SER007]abstract StackExchange.Redis.Availability.CircuitBreaker.Accumulator.ObserveResult(in StackExchange.Redis.Availability.CircuitBreaker.CircuitBreakerContext context) -> bool -[SER007]abstract StackExchange.Redis.Availability.CircuitBreaker.Accumulator.Reset() -> void -[SER007]abstract StackExchange.Redis.Availability.CircuitBreaker.CreateAccumulator() -> StackExchange.Redis.Availability.CircuitBreaker.Accumulator! +static StackExchange.Redis.KeyspaceIsolation.DatabaseExtensions.WithRetry(this StackExchange.Redis.IDatabaseAsync! database, StackExchange.Redis.Availability.RetryPolicy! retryPolicy) -> StackExchange.Redis.IDatabaseAsync! [SER007]StackExchange.Redis.Availability.CircuitBreaker [SER007]StackExchange.Redis.Availability.CircuitBreaker.Accumulator [SER007]StackExchange.Redis.Availability.CircuitBreaker.Accumulator.Accumulator() -> void @@ -24,7 +27,6 @@ StackExchange.Redis.IServer.InventKey(StackExchange.Redis.RedisKey prefix = defa [SER007]StackExchange.Redis.Availability.CircuitBreaker.CircuitBreakerContext.CircuitBreakerContext(System.Exception? fault, bool evaluate = true) -> void [SER007]StackExchange.Redis.Availability.CircuitBreaker.CircuitBreakerContext.Fault.get -> System.Exception? [SER007]StackExchange.Redis.Availability.CircuitBreaker.CircuitBreakerContext.Success.get -> bool -[SER007]static StackExchange.Redis.Availability.CircuitBreaker.Builder.implicit operator StackExchange.Redis.Availability.CircuitBreaker!(StackExchange.Redis.Availability.CircuitBreaker.Builder! builder) -> StackExchange.Redis.Availability.CircuitBreaker! [SER007]StackExchange.Redis.Availability.ConnectionGroupMember [SER007]StackExchange.Redis.Availability.ConnectionGroupMember.ConnectionGroupMember(StackExchange.Redis.ConfigurationOptions! configuration, string! name = "") -> void [SER007]StackExchange.Redis.Availability.ConnectionGroupMember.ConnectionGroupMember(string! configuration, string! name = "") -> void @@ -97,25 +99,6 @@ StackExchange.Redis.IServer.InventKey(StackExchange.Redis.RedisKey prefix = defa [SER007]StackExchange.Redis.Availability.MultiGroupOptions.MultiGroupOptions() -> void [SER007]static StackExchange.Redis.ConnectionMultiplexer.ConnectGroupAsync(StackExchange.Redis.Availability.ConnectionGroupMember! member0, StackExchange.Redis.Availability.ConnectionGroupMember! member1, StackExchange.Redis.Availability.MultiGroupOptions? options = null, System.IO.TextWriter? log = null) -> System.Threading.Tasks.Task! [SER007]static StackExchange.Redis.ConnectionMultiplexer.ConnectGroupAsync(StackExchange.Redis.Availability.ConnectionGroupMember![]! members, StackExchange.Redis.Availability.MultiGroupOptions? options = null, System.IO.TextWriter? log = null) -> System.Threading.Tasks.Task! -[SER007]static StackExchange.Redis.Availability.CircuitBreaker.Default.get -> StackExchange.Redis.Availability.CircuitBreaker! -[SER007]static StackExchange.Redis.Availability.CircuitBreaker.None.get -> StackExchange.Redis.Availability.CircuitBreaker! -[SER007]abstract StackExchange.Redis.Availability.HealthCheck.HealthCheckProbe.CheckHealthAsync(StackExchange.Redis.Availability.HealthCheck! healthCheck, StackExchange.Redis.IServer! server) -> System.Threading.Tasks.Task! -[SER007]abstract StackExchange.Redis.Availability.HealthCheck.HealthCheckProbePolicy.Evaluate(in StackExchange.Redis.Availability.HealthCheck.HealthCheckProbeContext context) -> StackExchange.Redis.Availability.HealthCheck.HealthCheckResult -[SER007]abstract StackExchange.Redis.Availability.HealthCheck.KeyWriteHealthCheckProbe.CheckHealthAsync(StackExchange.Redis.Availability.HealthCheck! healthCheck, StackExchange.Redis.IDatabaseAsync! database, StackExchange.Redis.RedisKey key) -> System.Threading.Tasks.Task! -[SER007]override StackExchange.Redis.Availability.ConnectionGroupMember.ToString() -> string! -[SER007]override StackExchange.Redis.Availability.HealthCheck.HealthCheckProbeContext.ToString() -> string! -[SER007]override StackExchange.Redis.Availability.HealthCheck.KeyWriteHealthCheckProbe.CheckHealthAsync(StackExchange.Redis.Availability.HealthCheck! healthCheck, StackExchange.Redis.IServer! server) -> System.Threading.Tasks.Task! -[SER007]static StackExchange.Redis.Availability.HealthCheck.Default.get -> StackExchange.Redis.Availability.HealthCheck! -[SER007]static StackExchange.Redis.Availability.HealthCheck.HealthCheckProbe.HealthyTask.get -> System.Threading.Tasks.Task! -[SER007]static StackExchange.Redis.Availability.HealthCheck.HealthCheckProbe.InconclusiveTask.get -> System.Threading.Tasks.Task! -[SER007]static StackExchange.Redis.Availability.HealthCheck.HealthCheckProbe.IsConnected.get -> StackExchange.Redis.Availability.HealthCheck.HealthCheckProbe! -[SER007]static StackExchange.Redis.Availability.HealthCheck.HealthCheckProbe.Ping.get -> StackExchange.Redis.Availability.HealthCheck.HealthCheckProbe! -[SER007]static StackExchange.Redis.Availability.HealthCheck.HealthCheckProbe.StringSet.get -> StackExchange.Redis.Availability.HealthCheck.HealthCheckProbe! -[SER007]static StackExchange.Redis.Availability.HealthCheck.HealthCheckProbe.UnhealthyTask.get -> System.Threading.Tasks.Task! -[SER007]static StackExchange.Redis.Availability.HealthCheck.HealthCheckProbePolicy.AllSuccess.get -> StackExchange.Redis.Availability.HealthCheck.HealthCheckProbePolicy! -[SER007]static StackExchange.Redis.Availability.HealthCheck.HealthCheckProbePolicy.AnySuccess.get -> StackExchange.Redis.Availability.HealthCheck.HealthCheckProbePolicy! -[SER007]static StackExchange.Redis.Availability.HealthCheck.HealthCheckProbePolicy.MajoritySuccess.get -> StackExchange.Redis.Availability.HealthCheck.HealthCheckProbePolicy! -[SER007]static StackExchange.Redis.Availability.MultiGroupOptions.Default.get -> StackExchange.Redis.Availability.MultiGroupOptions! [SER007]StackExchange.Redis.ConfigurationOptions.CircuitBreaker.get -> StackExchange.Redis.Availability.CircuitBreaker? [SER007]StackExchange.Redis.ConfigurationOptions.CircuitBreaker.set -> void [SER007]StackExchange.Redis.ConfigurationOptions.HealthCheck.get -> StackExchange.Redis.Availability.HealthCheck? diff --git a/src/StackExchange.Redis/PublicAPI/net6.0/PublicAPI.Unshipped.txt b/src/StackExchange.Redis/PublicAPI/net6.0/PublicAPI.Unshipped.txt index ab058de62..7c88a99f9 100644 --- a/src/StackExchange.Redis/PublicAPI/net6.0/PublicAPI.Unshipped.txt +++ b/src/StackExchange.Redis/PublicAPI/net6.0/PublicAPI.Unshipped.txt @@ -1 +1,10 @@ #nullable enable +StackExchange.Redis.Availability.RetryPolicy.FailoverDelay.get -> System.TimeSpan +StackExchange.Redis.Availability.RetryPolicy.FailoverDelay.set -> void +StackExchange.Redis.Availability.RetryPolicy.JitterMax.get -> System.TimeSpan +StackExchange.Redis.Availability.RetryPolicy.JitterMax.set -> void +StackExchange.Redis.Availability.RetryPolicy.MaxAttemptsBeforeFailover.get -> int +StackExchange.Redis.Availability.RetryPolicy.MaxAttemptsBeforeFailover.set -> void +StackExchange.Redis.Availability.RetryPolicy.RetryDelay.get -> System.TimeSpan +StackExchange.Redis.Availability.RetryPolicy.RetryDelay.set -> void +[SER007]override StackExchange.Redis.Availability.ConnectionGroupMember.ToString() -> string! diff --git a/src/StackExchange.Redis/RedisBatch.cs b/src/StackExchange.Redis/RedisBatch.cs index c827ba1dc..645265890 100644 --- a/src/StackExchange.Redis/RedisBatch.cs +++ b/src/StackExchange.Redis/RedisBatch.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Threading.Tasks; +using StackExchange.Redis.Interfaces; namespace StackExchange.Redis { @@ -8,7 +9,14 @@ internal sealed class RedisBatch : RedisDatabase, IBatch { private List? pending; - public RedisBatch(RedisDatabase wrapped, object? asyncState) : base(wrapped.multiplexer, wrapped.Database, asyncState ?? wrapped.AsyncState) { } + public RedisBatch(RedisDatabase wrapped, object? asyncState) : base(wrapped.multiplexer, wrapped.Database, + asyncState ?? wrapped.AsyncState) + { + wrapped.RejectFlags(DatabaseFeatureFlags.Batch | DatabaseFeatureFlags.Transaction); + } + + private protected override DatabaseFeatureFlags GetDatabaseFeatures() + => base.GetDatabaseFeatures() | DatabaseFeatureFlags.Batch; public void Execute() { diff --git a/src/StackExchange.Redis/RedisDatabase.cs b/src/StackExchange.Redis/RedisDatabase.cs index 21518eaa7..aadaf5abe 100644 --- a/src/StackExchange.Redis/RedisDatabase.cs +++ b/src/StackExchange.Redis/RedisDatabase.cs @@ -4,12 +4,14 @@ using System.Diagnostics; using System.Net; using System.Runtime.CompilerServices; +using System.Threading; using System.Threading.Tasks; using RESPite.Messages; +using StackExchange.Redis.Interfaces; namespace StackExchange.Redis { - internal partial class RedisDatabase : RedisBase, IDatabase + internal partial class RedisDatabase : RedisBase, IDatabase, IInternalDatabaseAsync { internal RedisDatabase(ConnectionMultiplexer multiplexer, int db, object? asyncState) : base(multiplexer, asyncState) @@ -21,6 +23,20 @@ internal RedisDatabase(ConnectionMultiplexer multiplexer, int db, object? asyncS public int Database { get; } + DatabaseFeatureFlags IInternalDatabaseAsync.GetFeatures(out string name) + { + name = multiplexer.ClientName; + return GetDatabaseFeatures(); + } + + CancellationToken IInternalDatabaseAsync.GetNextFailover() => CancellationToken.None; + + // the base feature set for this database; wrapping databases (batch, transaction, ...) override + // to fold in their own flag. Cluster is intrinsic to the underlying connection, so it lives here. + private protected virtual DatabaseFeatureFlags GetDatabaseFeatures() + => multiplexer.ServerSelectionStrategy.ServerType == ServerType.Cluster + ? DatabaseFeatureFlags.Cluster : DatabaseFeatureFlags.None; + public IBatch CreateBatch(object? asyncState) { if (this is IBatch) throw new NotSupportedException("Nested batches are not supported"); diff --git a/src/StackExchange.Redis/RedisTransaction.cs b/src/StackExchange.Redis/RedisTransaction.cs index 5d5430671..7ae9ca085 100644 --- a/src/StackExchange.Redis/RedisTransaction.cs +++ b/src/StackExchange.Redis/RedisTransaction.cs @@ -5,6 +5,7 @@ using System.Threading; using System.Threading.Tasks; using RESPite.Messages; +using StackExchange.Redis.Interfaces; namespace StackExchange.Redis { @@ -14,8 +15,12 @@ internal sealed class RedisTransaction : RedisDatabase, ITransaction private List? _pending; private object SyncLock => this; + private protected override DatabaseFeatureFlags GetDatabaseFeatures() + => base.GetDatabaseFeatures() | DatabaseFeatureFlags.Transaction; + public RedisTransaction(RedisDatabase wrapped, object? asyncState) : base(wrapped.multiplexer, wrapped.Database, asyncState ?? wrapped.AsyncState) { + wrapped.RejectFlags(DatabaseFeatureFlags.Batch | DatabaseFeatureFlags.Transaction); // need to check we can reliably do this... var commandMap = multiplexer.CommandMap; commandMap.AssertAvailable(RedisCommand.MULTI); diff --git a/src/StackExchange.Redis/ServerSelectionStrategy.cs b/src/StackExchange.Redis/ServerSelectionStrategy.cs index 27358e8c7..80c9b9efe 100644 --- a/src/StackExchange.Redis/ServerSelectionStrategy.cs +++ b/src/StackExchange.Redis/ServerSelectionStrategy.cs @@ -52,9 +52,9 @@ internal sealed partial class ServerSelectionStrategy private int anyStartOffset = SharedRandom.Next(); // initialize to a random value so routing isn't uniform #if NET - private static Random SharedRandom => Random.Shared; + internal static Random SharedRandom => Random.Shared; #else - private static Random SharedRandom { get; } = new(); + internal static Random SharedRandom { get; } = new(); #endif private ServerEndPoint[]? map; diff --git a/tests/StackExchange.Redis.Tests/Helpers/SharedConnectionFixture.cs b/tests/StackExchange.Redis.Tests/Helpers/SharedConnectionFixture.cs index 8da84bce6..9f894d907 100644 --- a/tests/StackExchange.Redis.Tests/Helpers/SharedConnectionFixture.cs +++ b/tests/StackExchange.Redis.Tests/Helpers/SharedConnectionFixture.cs @@ -58,6 +58,8 @@ internal sealed class NonDisposingConnection(IInternalConnectionMultiplexer inne { public IInternalConnectionMultiplexer UnderlyingConnection => _inner; + Availability.CircuitBreaker? IInternalConnectionMultiplexer.CircuitBreaker => _inner.CircuitBreaker; + public bool AllowConnect { get => _inner.AllowConnect; From 30ff35f5a9395bcdc66991f70f3cf2022ea43d1e Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Wed, 15 Jul 2026 16:41:14 +0100 Subject: [PATCH 10/23] nit --- .../Availability/RetryDatabase.cs | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/StackExchange.Redis/Availability/RetryDatabase.cs b/src/StackExchange.Redis/Availability/RetryDatabase.cs index ffe7ebda4..ed5508819 100644 --- a/src/StackExchange.Redis/Availability/RetryDatabase.cs +++ b/src/StackExchange.Redis/Availability/RetryDatabase.cs @@ -37,6 +37,9 @@ public RetryDatabase(IDatabaseAsync inner, RetryPolicy policy) _maxAttempts = policy.MaxAttempts; if (_maxBeforeFailover == _maxAttempts) _maxBeforeFailover = int.MaxValue; // then we'll never look if (_maxAttempts < 1) throw new ArgumentOutOfRangeException(nameof(policy.MaxAttempts)); + // guard the failover threshold: values < 1 can never be hit by the loop counter (which starts at 1), + // so they would *silently* disable failover rather than erroring; validate the raw policy value + if (policy.MaxAttemptsBeforeFailover < 1) throw new ArgumentOutOfRangeException(nameof(policy.MaxAttemptsBeforeFailover)); _delayMillis = policy.DelayMilliseconds; _failoverMillis = policy.FailoverMilliseconds; _jitterMillis = policy.JitterMilliseconds; @@ -67,11 +70,11 @@ private async Task ExecuteAsync(TState state, Func(TState state, Func Date: Wed, 15 Jul 2026 16:56:58 +0100 Subject: [PATCH 11/23] fix build --- .../MultiGroupSubscriber.Tracking.cs | 4 +-- .../PublicAPI/PublicAPI.Unshipped.txt | 33 ++++++++++++++++++ .../PublicAPI/net6.0/PublicAPI.Unshipped.txt | 9 ----- src/StackExchange.Redis/RedisBatch.cs | 4 +-- .../RetryTests/BasicRetryTests.cs | 34 +++++++++++++++++++ 5 files changed, 71 insertions(+), 13 deletions(-) create mode 100644 tests/StackExchange.Redis.Tests/RetryTests/BasicRetryTests.cs diff --git a/src/StackExchange.Redis/Availability/MultiGroupSubscriber.Tracking.cs b/src/StackExchange.Redis/Availability/MultiGroupSubscriber.Tracking.cs index eb560004d..a029eef56 100644 --- a/src/StackExchange.Redis/Availability/MultiGroupSubscriber.Tracking.cs +++ b/src/StackExchange.Redis/Availability/MultiGroupSubscriber.Tracking.cs @@ -60,7 +60,7 @@ private static Action FilteredHandler(MultiGroupMultip // invoke a handler only if the active member is the one we expect return (channel, value) => { - if (ReferenceEquals(parent._active, active.Multiplexer)) + if (ReferenceEquals(parent._activeStub.Active, active.Multiplexer)) { handler(channel, value); } @@ -89,7 +89,7 @@ static async Task ForwardFilteredMessagesAsync( { while (readFrom.TryRead(out var message)) { - if (ReferenceEquals(parent._active, active.Multiplexer)) + if (ReferenceEquals(parent._activeStub.Active, active.Multiplexer)) { // Because of the switchover being imperfect, we can't guarantee exactly one writer, so // we need to be synchronized; in reality, it will *almost never* be contended, so diff --git a/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt b/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt index 6db22118e..9d22f90a7 100644 --- a/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt @@ -2,8 +2,16 @@ StackExchange.Redis.Availability.RetryPolicy StackExchange.Redis.Availability.RetryPolicy.CircuitBreaker.get -> StackExchange.Redis.Availability.CircuitBreaker? StackExchange.Redis.Availability.RetryPolicy.CircuitBreaker.set -> void +StackExchange.Redis.Availability.RetryPolicy.FailoverDelay.get -> System.TimeSpan +StackExchange.Redis.Availability.RetryPolicy.FailoverDelay.set -> void +StackExchange.Redis.Availability.RetryPolicy.JitterMax.get -> System.TimeSpan +StackExchange.Redis.Availability.RetryPolicy.JitterMax.set -> void StackExchange.Redis.Availability.RetryPolicy.MaxAttempts.get -> int StackExchange.Redis.Availability.RetryPolicy.MaxAttempts.set -> void +StackExchange.Redis.Availability.RetryPolicy.MaxAttemptsBeforeFailover.get -> int +StackExchange.Redis.Availability.RetryPolicy.MaxAttemptsBeforeFailover.set -> void +StackExchange.Redis.Availability.RetryPolicy.RetryDelay.get -> System.TimeSpan +StackExchange.Redis.Availability.RetryPolicy.RetryDelay.set -> void StackExchange.Redis.Availability.RetryPolicy.RetryPolicy() -> void StackExchange.Redis.IServer.InventKey(StackExchange.Redis.RedisKey prefix = default(StackExchange.Redis.RedisKey)) -> StackExchange.Redis.RedisKey static StackExchange.Redis.KeyspaceIsolation.DatabaseExtensions.WithRetry(this StackExchange.Redis.IDatabaseAsync! database, StackExchange.Redis.Availability.RetryPolicy! retryPolicy) -> StackExchange.Redis.IDatabaseAsync! @@ -108,3 +116,28 @@ static StackExchange.Redis.KeyspaceIsolation.DatabaseExtensions.WithRetry(this S [SER007]StackExchange.Redis.Availability.ConnectionGroupMember.ResetIsUnhealthy() -> void [SER007]StackExchange.Redis.Availability.MultiGroupOptions.FailbackDelay.get -> System.TimeSpan [SER007]StackExchange.Redis.Availability.MultiGroupOptions.FailbackDelay.set -> void +[SER007]abstract StackExchange.Redis.Availability.CircuitBreaker.Accumulator.IsHealthy() -> bool +[SER007]abstract StackExchange.Redis.Availability.CircuitBreaker.Accumulator.ObserveResult(in StackExchange.Redis.Availability.CircuitBreaker.CircuitBreakerContext context) -> bool +[SER007]abstract StackExchange.Redis.Availability.CircuitBreaker.Accumulator.Reset() -> void +[SER007]abstract StackExchange.Redis.Availability.CircuitBreaker.CreateAccumulator() -> StackExchange.Redis.Availability.CircuitBreaker.Accumulator! +[SER007]virtual StackExchange.Redis.Availability.CircuitBreaker.IsConnectionFault(System.Exception? fault) -> bool +[SER007]static StackExchange.Redis.Availability.CircuitBreaker.Default.get -> StackExchange.Redis.Availability.CircuitBreaker! +[SER007]static StackExchange.Redis.Availability.CircuitBreaker.None.get -> StackExchange.Redis.Availability.CircuitBreaker! +[SER007]static StackExchange.Redis.Availability.CircuitBreaker.Builder.implicit operator StackExchange.Redis.Availability.CircuitBreaker!(StackExchange.Redis.Availability.CircuitBreaker.Builder! builder) -> StackExchange.Redis.Availability.CircuitBreaker! +[SER007]abstract StackExchange.Redis.Availability.HealthCheck.HealthCheckProbe.CheckHealthAsync(StackExchange.Redis.Availability.HealthCheck! healthCheck, StackExchange.Redis.IServer! server) -> System.Threading.Tasks.Task! +[SER007]static StackExchange.Redis.Availability.HealthCheck.HealthCheckProbe.HealthyTask.get -> System.Threading.Tasks.Task! +[SER007]static StackExchange.Redis.Availability.HealthCheck.HealthCheckProbe.InconclusiveTask.get -> System.Threading.Tasks.Task! +[SER007]static StackExchange.Redis.Availability.HealthCheck.HealthCheckProbe.UnhealthyTask.get -> System.Threading.Tasks.Task! +[SER007]static StackExchange.Redis.Availability.HealthCheck.HealthCheckProbe.IsConnected.get -> StackExchange.Redis.Availability.HealthCheck.HealthCheckProbe! +[SER007]static StackExchange.Redis.Availability.HealthCheck.HealthCheckProbe.Ping.get -> StackExchange.Redis.Availability.HealthCheck.HealthCheckProbe! +[SER007]static StackExchange.Redis.Availability.HealthCheck.HealthCheckProbe.StringSet.get -> StackExchange.Redis.Availability.HealthCheck.HealthCheckProbe! +[SER007]abstract StackExchange.Redis.Availability.HealthCheck.HealthCheckProbePolicy.Evaluate(in StackExchange.Redis.Availability.HealthCheck.HealthCheckProbeContext context) -> StackExchange.Redis.Availability.HealthCheck.HealthCheckResult +[SER007]static StackExchange.Redis.Availability.HealthCheck.HealthCheckProbePolicy.AllSuccess.get -> StackExchange.Redis.Availability.HealthCheck.HealthCheckProbePolicy! +[SER007]static StackExchange.Redis.Availability.HealthCheck.HealthCheckProbePolicy.AnySuccess.get -> StackExchange.Redis.Availability.HealthCheck.HealthCheckProbePolicy! +[SER007]static StackExchange.Redis.Availability.HealthCheck.HealthCheckProbePolicy.MajoritySuccess.get -> StackExchange.Redis.Availability.HealthCheck.HealthCheckProbePolicy! +[SER007]abstract StackExchange.Redis.Availability.HealthCheck.KeyWriteHealthCheckProbe.CheckHealthAsync(StackExchange.Redis.Availability.HealthCheck! healthCheck, StackExchange.Redis.IDatabaseAsync! database, StackExchange.Redis.RedisKey key) -> System.Threading.Tasks.Task! +[SER007]override StackExchange.Redis.Availability.HealthCheck.KeyWriteHealthCheckProbe.CheckHealthAsync(StackExchange.Redis.Availability.HealthCheck! healthCheck, StackExchange.Redis.IServer! server) -> System.Threading.Tasks.Task! +[SER007]override StackExchange.Redis.Availability.HealthCheck.HealthCheckProbeContext.ToString() -> string! +[SER007]static StackExchange.Redis.Availability.HealthCheck.Default.get -> StackExchange.Redis.Availability.HealthCheck! +[SER007]static StackExchange.Redis.Availability.MultiGroupOptions.Default.get -> StackExchange.Redis.Availability.MultiGroupOptions! +[SER007]override StackExchange.Redis.Availability.ConnectionGroupMember.ToString() -> string! diff --git a/src/StackExchange.Redis/PublicAPI/net6.0/PublicAPI.Unshipped.txt b/src/StackExchange.Redis/PublicAPI/net6.0/PublicAPI.Unshipped.txt index 7c88a99f9..ab058de62 100644 --- a/src/StackExchange.Redis/PublicAPI/net6.0/PublicAPI.Unshipped.txt +++ b/src/StackExchange.Redis/PublicAPI/net6.0/PublicAPI.Unshipped.txt @@ -1,10 +1 @@ #nullable enable -StackExchange.Redis.Availability.RetryPolicy.FailoverDelay.get -> System.TimeSpan -StackExchange.Redis.Availability.RetryPolicy.FailoverDelay.set -> void -StackExchange.Redis.Availability.RetryPolicy.JitterMax.get -> System.TimeSpan -StackExchange.Redis.Availability.RetryPolicy.JitterMax.set -> void -StackExchange.Redis.Availability.RetryPolicy.MaxAttemptsBeforeFailover.get -> int -StackExchange.Redis.Availability.RetryPolicy.MaxAttemptsBeforeFailover.set -> void -StackExchange.Redis.Availability.RetryPolicy.RetryDelay.get -> System.TimeSpan -StackExchange.Redis.Availability.RetryPolicy.RetryDelay.set -> void -[SER007]override StackExchange.Redis.Availability.ConnectionGroupMember.ToString() -> string! diff --git a/src/StackExchange.Redis/RedisBatch.cs b/src/StackExchange.Redis/RedisBatch.cs index 645265890..5b9e527ea 100644 --- a/src/StackExchange.Redis/RedisBatch.cs +++ b/src/StackExchange.Redis/RedisBatch.cs @@ -9,8 +9,8 @@ internal sealed class RedisBatch : RedisDatabase, IBatch { private List? pending; - public RedisBatch(RedisDatabase wrapped, object? asyncState) : base(wrapped.multiplexer, wrapped.Database, - asyncState ?? wrapped.AsyncState) + public RedisBatch(RedisDatabase wrapped, object? asyncState) + : base(wrapped.multiplexer, wrapped.Database, asyncState ?? wrapped.AsyncState) { wrapped.RejectFlags(DatabaseFeatureFlags.Batch | DatabaseFeatureFlags.Transaction); } diff --git a/tests/StackExchange.Redis.Tests/RetryTests/BasicRetryTests.cs b/tests/StackExchange.Redis.Tests/RetryTests/BasicRetryTests.cs new file mode 100644 index 000000000..0c8203872 --- /dev/null +++ b/tests/StackExchange.Redis.Tests/RetryTests/BasicRetryTests.cs @@ -0,0 +1,34 @@ +using System.IO; +using System.Threading.Tasks; +using StackExchange.Redis.Tests.Helpers; +using Xunit; + +namespace StackExchange.Redis.Tests.RetryTests; + +[RunPerProtocol] +public class BasicRetryTests(ITestOutputHelper log) +{ + protected TextWriter Log { get; } = new TextWriterOutputHelper(log); + + // Baseline: connect to a single in-proc server and exercise the *regular* (non-retry) database. + // This is the control case the retry suite will build on - no RetryDatabase wrapper yet. + [Fact] + public async Task ConnectAndGet() + { + using var server = new InProcessTestServer(log); + await using var conn = await server.ConnectAsync(log: Log); + Assert.True(conn.IsConnected); + + var db = conn.GetDatabase(); + + RedisKey key = "retry:basic"; + Assert.True(await db.StringSetAsync(key, "hello")); + + var value = await db.StringGetAsync(key); + Assert.Equal("hello", value); + + // a couple more gets, including a miss + Assert.Equal("hello", await db.StringGetAsync(key)); + Assert.Equal(RedisValue.Null, await db.StringGetAsync("retry:missing")); + } +} From 03eb654b3626edcac934fdbc9a018439f8264995 Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Thu, 16 Jul 2026 14:07:35 +0100 Subject: [PATCH 12/23] iterating on RetryPolicy --- .../Availability/CircuitBreaker.cs | 224 ++++------------ .../Availability/FaultContext.cs | 46 ++++ .../Availability/RedisErrorKind.cs | 248 ++++++++++++++++++ .../Availability/RetryDatabase.cs | 88 ++++--- .../Availability/RetryPolicy.cs | 114 +++++++- .../ConnectionMultiplexer.cs | 2 - src/StackExchange.Redis/Enums/CommandFlags.cs | 29 ++ src/StackExchange.Redis/Exceptions.cs | 5 + .../Interfaces/IConnectionMultiplexer.cs | 2 - src/StackExchange.Redis/PhysicalConnection.cs | 2 +- .../PublicAPI/PublicAPI.Unshipped.txt | 50 +++- src/StackExchange.Redis/RedisTransaction.cs | 4 +- .../ResultProcessor.Literals.cs | 10 - src/StackExchange.Redis/ResultProcessor.cs | 38 +-- .../CircuitBreakerServerTests.cs | 2 +- .../CircuitBreakerUnitTests.cs | 2 +- .../Helpers/SharedConnectionFixture.cs | 2 - .../CircuitBreakerRerouteTests.cs | 2 +- .../RetryTests/BasicRetryTests.cs | 28 ++ .../StackExchange.Redis.Server/RedisServer.cs | 10 + 20 files changed, 643 insertions(+), 265 deletions(-) create mode 100644 src/StackExchange.Redis/Availability/FaultContext.cs create mode 100644 src/StackExchange.Redis/Availability/RedisErrorKind.cs diff --git a/src/StackExchange.Redis/Availability/CircuitBreaker.cs b/src/StackExchange.Redis/Availability/CircuitBreaker.cs index 4b73beaf4..4f9b08b65 100644 --- a/src/StackExchange.Redis/Availability/CircuitBreaker.cs +++ b/src/StackExchange.Redis/Availability/CircuitBreaker.cs @@ -25,12 +25,6 @@ public abstract class CircuitBreaker /// public static CircuitBreaker None => NulCircuitBreaker.Instance; - /// - /// Indicates a fault that should be handled by the circuit-breaker or related retry logic. - /// - public virtual bool IsConnectionFault(Exception? fault) - => fault is RedisTimeoutException or RedisConnectionException; - /// /// Allows configuration of the default implementation. /// @@ -40,14 +34,15 @@ public class Builder private const int DefaultMinimumNumberOfFailures = 1000; private static readonly TimeSpan DefaultMetricsWindowSize = TimeSpan.FromSeconds(2); +#pragma warning disable SA1114 // Parameter list should follow declaration - false positive: the #if directive splits the argument list internal static CircuitBreaker DefaultInstance = new DefaultCircuitBreaker( - DefaultFailureRateThreshold, - DefaultMinimumNumberOfFailures, - DefaultMetricsWindowSize, #if NET8_0_OR_GREATER - timeProvider: null, + null, #endif - trackedExceptions: null); + DefaultFailureRateThreshold, + DefaultMinimumNumberOfFailures, + DefaultMetricsWindowSize); +#pragma warning restore SA1114 /// /// Percentage of failures to trigger circuit breaker. @@ -79,34 +74,28 @@ public class Builder public CircuitBreaker Create() { if ((FailureRateThreshold is DefaultFailureRateThreshold - & MinimumNumberOfFailures is DefaultMinimumNumberOfFailures #if NET8_0_OR_GREATER & TimeProvider is null #endif - & TrackedExceptions is null) + & MinimumNumberOfFailures is DefaultMinimumNumberOfFailures) && MetricsWindowSize == DefaultMetricsWindowSize) return DefaultInstance; +#pragma warning disable SA1114 // Parameter list should follow declaration - false positive: the #if directive splits the argument list return new DefaultCircuitBreaker( - FailureRateThreshold, - MinimumNumberOfFailures, - MetricsWindowSize, #if NET8_0_OR_GREATER TimeProvider, #endif - TrackedExceptions); + FailureRateThreshold, + MinimumNumberOfFailures, + MetricsWindowSize); +#pragma warning restore SA1114 } /// /// Create a new circuit-breaker instance. /// public static implicit operator CircuitBreaker(Builder builder) => builder.Create(); - - /// - /// Exceptions that count as failures. When null, - /// and are assumed. - /// - public Type[]? TrackedExceptions { get; set; } } /// @@ -114,30 +103,40 @@ public CircuitBreaker Create() /// public abstract Accumulator CreateAccumulator(); + internal static bool DefaultIsFailure(in FaultContext fault) + { + if (fault.ConnectionFailureType is not ConnectionFailureType.None) return true; + switch (fault.ErrorKind) + { + // what things *don't* trip the breaker? + case RedisErrorKind.None: // not even flagged + case RedisErrorKind.UnknownCommand: // application failure + case RedisErrorKind.ExecAbort: // transient to one command + case RedisErrorKind.WrongType: // application failure + case RedisErrorKind.NoPermission: // using the wrong keys? + case RedisErrorKind.UnknownError: // not sure what it is, but it starts ERR + case RedisErrorKind.Unknown: // pretty much anything we don't recognize; should we assume this is BAD? + return false; + default: + return true; + } + } + /// /// Collates observations for a connection. /// - public abstract class Accumulator + public abstract class Accumulator() { /// - /// Record a message outcome, and indicate whether the connection is considered healthy. + /// Record a message outcome. /// - /// True if the connection is still considered healthy. /// struct arg here is in case we want to add more things later - public abstract bool ObserveResult(in CircuitBreakerContext context); + public abstract void ObserveResult(in FaultContext fault); - internal bool ObserveResult(Exception? fault) - { - // only evaluate state upon failure; don't pay that overhead for success, just increment the counters - bool evaluate = fault is not null; - var ctx = new CircuitBreakerContext(fault, evaluate: evaluate); - bool healthy = ObserveResult(in ctx); - - // when we didn't ask for an evaluation, the returned verdict is meaningless: a custom - // implementation might return default(false) without having actually computed anything. - // never treat a non-evaluating observation as unhealthy - only a genuine evaluation may trip. - return !evaluate || healthy; - } + /// + /// Indicate whether a given fault should be considered a failure for the . + /// + protected virtual bool IsFailure(in FaultContext fault) => DefaultIsFailure(in fault); /// /// Indicate whether the connection is currently considered healthy, without recording an observation. @@ -149,34 +148,21 @@ internal bool ObserveResult(Exception? fault) /// Discard all accumulated observations, returning to a clean state. /// public abstract void Reset(); - } - - /// - /// Provides information about a circuit-breaker test. - /// - public readonly struct CircuitBreakerContext(Exception? fault, bool evaluate = true) - { - /// - /// Was the operation a success. - /// - [MemberNotNullWhen(false, nameof(Fault))] - public bool Success => fault is null; - - /// - /// The fault associated with the operation. - /// - public Exception? Fault => fault; - internal bool Evaluate => evaluate; - } - - private enum ExceptionStrategy - { - Default, - Any, - None, - CustomOpen, - CustomSealed, + internal bool Trip(Exception? fault) + { + if (fault is not null) + { + var ctx = new FaultContext(fault); + if (IsFailure(ctx)) + { + ObserveResult(ctx); + return !IsHealthy(); + } + } + ObserveResult(in FaultContext.Success); + return false; // never trip through success + } } private sealed class NulCircuitBreaker : CircuitBreaker @@ -189,7 +175,7 @@ private sealed class NulAccumulator : Accumulator { public static readonly NulAccumulator AccumulatorInstance = new(); private NulAccumulator() { } - public override bool ObserveResult(in CircuitBreakerContext context) => true; + public override void ObserveResult(in FaultContext context) { } public override bool IsHealthy() => true; public override void Reset() { } } @@ -198,8 +184,6 @@ public override void Reset() { } } private sealed class DefaultCircuitBreaker : CircuitBreaker { - private readonly ExceptionStrategy _trackingStrategy; - private readonly Type[] _trackedExceptions; private readonly double _failureRateThreshold; private readonly int _minimumNumberOfFailures; @@ -225,15 +209,13 @@ private long GetEpoch() => / _bucketTicks; public DefaultCircuitBreaker( - double failureRateThreshold, - int minimumNumberOfFailures, - TimeSpan metricsWindowSize, #if NET8_0_OR_GREATER TimeProvider? timeProvider, #endif - Type[]? trackedExceptions) + double failureRateThreshold, + int minimumNumberOfFailures, + TimeSpan metricsWindowSize) { - _trackedExceptions = CheckExceptions(trackedExceptions, out _trackingStrategy); _failureRateThreshold = failureRateThreshold; _minimumNumberOfFailures = minimumNumberOfFailures; @@ -310,11 +292,8 @@ public void Reset() } } - public override bool ObserveResult(in CircuitBreakerContext context) + public override void ObserveResult(in FaultContext result) { - // not-tracked failures (based on exception type) count as "success" for the purposes of circuit breaking - bool countAsSuccess = context.Success || !breaker.IsConnectionFault(context.Fault); - // which time-slice are we in, and where does it live in the ring? long epoch = breaker.GetEpoch(); int index = (int)(epoch % BucketCount); @@ -326,9 +305,7 @@ public override bool ObserveResult(in CircuitBreakerContext context) // only getting results one at a time, so we shouldn't be over-stomping much *anyway* Span buckets = _buckets; // *not* a payload copy; this is in-place over the data ref Bucket bucket = ref buckets[index]; - bucket.Count(epoch, countAsSuccess); - - return !context.Evaluate || Evaluate(epoch); + bucket.Count(epoch, success: !result.HasValue); } public override bool IsHealthy() => Evaluate(breaker.GetEpoch()); @@ -370,90 +347,5 @@ public override void Reset() } } } - - private static Type[] CheckExceptions(Type[]? tracked, out ExceptionStrategy strategy) - { - if (tracked is null) - { - strategy = ExceptionStrategy.Default; - return []; - } - if (tracked.Length is 0) - { - strategy = ExceptionStrategy.None; - return []; - } - strategy = ExceptionStrategy.CustomOpen; - - static bool Contains(Type[] array, Type type) - { - for (int i = 0; i < array.Length; i++) - { - if (array[i] == type) return true; - } - - return false; - } - - // if we have Exception anywhere: we'll track everything - if (Contains(tracked, typeof(Exception))) - { - strategy = ExceptionStrategy.Any; - return []; - } - - if (tracked.Length is 2 - && Contains(tracked, typeof(RedisConnectionException)) - && Contains(tracked, typeof(RedisTimeoutException))) - { - strategy = ExceptionStrategy.Default; - return []; - } - - // finally, see if they're all sealed types - strategy = ExceptionStrategy.CustomSealed; - foreach (var exception in tracked) - { - if (!exception.IsSealed) - { - strategy = ExceptionStrategy.CustomOpen; - break; - } - } - return tracked; - } - - public override bool IsConnectionFault(Exception? fault) - { - if (fault is null) return false; - switch (_trackingStrategy) - { - case ExceptionStrategy.Any: - return true; - case ExceptionStrategy.Default: - return fault is RedisTimeoutException or RedisConnectionException; - case ExceptionStrategy.None: - return false; - } - - var span = _trackedExceptions.AsSpan(); - var actualType = fault.GetType(); - // check for exact matches - foreach (var testType in span) - { - if (ReferenceEquals(testType, actualType)) return true; - } - - if (_trackingStrategy is ExceptionStrategy.CustomOpen) - { - // we need to check for subclasses (more expensive) - foreach (var testType in span) - { - if (!testType.IsSealed && testType.IsAssignableFrom(actualType)) return true; - } - } - - return false; - } } } diff --git a/src/StackExchange.Redis/Availability/FaultContext.cs b/src/StackExchange.Redis/Availability/FaultContext.cs new file mode 100644 index 000000000..41232557a --- /dev/null +++ b/src/StackExchange.Redis/Availability/FaultContext.cs @@ -0,0 +1,46 @@ +using System; +using System.Diagnostics.CodeAnalysis; + +namespace StackExchange.Redis.Availability; + +/// +/// Provides information about a circuit-breaker test. +/// +public readonly struct FaultContext +{ + private readonly Exception _fault; + private readonly RedisErrorKind _errorKind; + private readonly ConnectionFailureType _connectionFailureType; + + internal static readonly FaultContext Success = default; + + /// + /// Create a new . + /// + /// The fault associated with the operation, or null on success. + public FaultContext(Exception fault) + { + _fault = fault; + _errorKind = fault.GetErrorKind(out _connectionFailureType); + } + + /// + /// Indicates whether a fault is present. + /// + public bool HasValue => _fault is not null; // excludes: default(FaultContext) + + /// + /// The fault associated with the operation. + /// + public Exception Fault => _fault; + + /// + /// The classified server error condition associated with the fault, if any. + /// + public RedisErrorKind ErrorKind => _errorKind; + + /// + /// The connection failure type associated with the fault, if any. + /// + public ConnectionFailureType ConnectionFailureType => _connectionFailureType; +} diff --git a/src/StackExchange.Redis/Availability/RedisErrorKind.cs b/src/StackExchange.Redis/Availability/RedisErrorKind.cs new file mode 100644 index 000000000..f61256422 --- /dev/null +++ b/src/StackExchange.Redis/Availability/RedisErrorKind.cs @@ -0,0 +1,248 @@ +using System; +using System.Diagnostics.CodeAnalysis; +using RESPite; +using RESPite.Messages; + +namespace StackExchange.Redis.Availability; + +/// +/// Well-known server error conditions, identified from the error-reply prefix (and in some cases the +/// message text). Used to classify faults consistently - in particular to decide whether a fault is +/// transient (worth retrying / awaiting failover) or permanent (retrying will never help). +/// +[Experimental(Experiments.ActiveActive, UrlFormat = Experiments.UrlFormat)] +public enum RedisErrorKind +{ + /// + /// No error condition; the reply was not an error. + /// + [AsciiHash("")] + None = 0, + + /// + /// The error was not recognized as one of the well-known conditions and did not start with ERR. + /// + [AsciiHash("")] + Unknown, + + /// + /// The error was not recognized as one of the well-known conditions, but started with ERR. + /// + [AsciiHash("")] + UnknownError, + + /// + /// The error was due to a connection fault, categorized separately by . + /// + [AsciiHash("")] + ConnectionFault, + + /// + /// The operation timed out; this is a client-side condition and does not correspond to a server reply. + /// + [AsciiHash("")] + Timeout, + + // --- availability / typically transient (retry or failover may recover) --- + + /// + /// LOADING - the server is still loading its dataset into memory and is not yet ready. + /// + Loading, + + /// + /// CLUSTERDOWN - the cluster is down; the hash slot is not currently being served. + /// + ClusterDown, + + /// + /// MASTERDOWN - the link with the primary is down and replica-serve-stale-data is no. + /// + MasterDown, + + /// + /// TRYAGAIN - a multi-key operation spans slots that are being migrated; the client should retry. + /// + TryAgain, + + /// + /// NOREPLICAS - not enough healthy replicas to satisfy the configured min-replicas-to-write. + /// + NoReplicas, + + /// + /// MISCONF - RDB/AOF persistence is misconfigured and writes are currently refused. + /// + [AsciiHash("MISCONF")] + Misconfigured, + + /// + /// OOM - the command was refused because used memory is above the maxmemory limit. + /// + [AsciiHash("OOM")] + OutOfMemory, + + /// + /// BUSY - a script or function is running and blocking the server (needs SCRIPT KILL etc.). + /// + Busy, + + /// + /// MAXCLIENTS - the configured maximum number of client connections has been reached. + /// + MaxClients, + + // --- cluster slot routing --- + + /// + /// MOVED - the hash slot has permanently moved to another endpoint. + /// + Moved, + + /// + /// ASK - the hash slot is temporarily served by another endpoint during migration. + /// + Ask, + + /// + /// CROSSSLOT - the keys in a multi-key operation do not all hash to the same slot. + /// + CrossSlot, + + // --- authentication / authorization (will not recover without a credential or ACL change) --- + + /// + /// NOAUTH - authentication is required before commands can be issued. + /// + NoAuth, + + /// + /// WRONGPASS - the supplied username/password pair was rejected. + /// + WrongPass, + + /// + /// NOPERM - the authenticated ACL user is not permitted to run this command/key/channel. + /// + [AsciiHash("NOPERM")] + NoPermission, + + /// + /// ERR operation not permitted - the operation is not permitted in the current context. + /// + [AsciiHash("")] // matched via the "ERR " branch, not the first token + NotPermitted, + + // --- client / usage errors (permanent - retry and failover will never help) --- + + /// + /// ERR unknown command - the command is not known to the server (typo, disabled, or unsupported version). + /// + [AsciiHash("")] // matched via the "ERR " branch, not the first token + UnknownCommand, + + /// + /// WRONGTYPE - the operation was applied against a key holding an incompatible value type. + /// + WrongType, + + /// + /// EXECABORT - the transaction was discarded because of an earlier error while queuing commands. + /// + ExecAbort, + + /// + /// READONLY - a write was attempted against a read-only replica. + /// + ReadOnly, + + /// + /// NOSCRIPT - no matching script for EVALSHA; the script must be re-loaded. + /// + NoScript, + + /// + /// ERR DB index is out of range / ERR invalid DB index - the database index passed to + /// SELECT is out of range or not a valid integer. + /// + [AsciiHash("")] // matched via the "ERR " branch, not the first token + InvalidDatabaseIndex, + + /// + /// ERR SELECT is not allowed in cluster mode - selecting a non-default database is not supported + /// on this server/topology (e.g. cluster mode, or a server without multi-database support). + /// + [AsciiHash("")] // matched via the "ERR " branch, not the first token + DatabaseSelectDisabled, +} + +internal static partial class RedisErrorKindMetadata +{ + [AsciiHash(CaseSensitive = false)] + private static partial bool TryParseFirstTokenCI(ReadOnlySpan value, out RedisErrorKind kind); + + /// + /// Classifies the error currently held by . The caller is assumed to have + /// already established that this is an error, so an unrecognized reply yields + /// (never ). + /// + internal static unsafe RedisErrorKind Classify(in RespReader reader) + => reader.TryParseScalar(&TryParse, out RedisErrorKind kind) ? kind : RedisErrorKind.Unknown; + + internal static bool TryParse(ReadOnlySpan value, out RedisErrorKind kind) + { + var space = value.IndexOf((byte)' '); + // Deal with the exact matches on the first token first - many errors may or may not + // have descriptive text after the leading token. + var firstToken = space > 0 ? value.Slice(0, space) : value; + if (TryParseFirstTokenCI(firstToken, out kind)) return true; + + // check for "ERR ..." scenarios + if (space > 0 && Err.IsCI(firstToken, AsciiHash.HashUC(firstToken))) + { + value = value.Slice(space + 1); // message text after "ERR " + + // most ERR conditions are fixed messages matched in full; the exception is + // "unknown command '', with args ...", which always carries a trailing + // command name and so is matched on its leading text (the final guarded arm) + var valueHash = AsciiHash.HashUC(value); + kind = value.Length switch + { + DbIndexOutOfRange.Length when DbIndexOutOfRange.IsCI(value, valueHash) => RedisErrorKind + .InvalidDatabaseIndex, + InvalidDbIndex.Length when InvalidDbIndex.IsCI(value, valueHash) => RedisErrorKind + .InvalidDatabaseIndex, + OperationNotPermitted.Length when OperationNotPermitted.IsCI(value, valueHash) => RedisErrorKind + .NotPermitted, + SelectNotAllowedInClusterMode.Length when SelectNotAllowedInClusterMode.IsCI(value, valueHash) => RedisErrorKind + .DatabaseSelectDisabled, + _ when value.Length >= UnknownCommand.Length + && AsciiHash.SequenceEqualsCI(value.Slice(0, UnknownCommand.Length), UnknownCommand.U8) => RedisErrorKind + .UnknownCommand, + _ => RedisErrorKind.UnknownError, + }; + return true; + } + + kind = value.IsEmpty ? RedisErrorKind.None : RedisErrorKind.Unknown; + return true; + } + + [AsciiHash("ERR")] + private static partial class Err { } + + [AsciiHash("DB index is out of range")] + private static partial class DbIndexOutOfRange { } + + [AsciiHash("invalid DB index")] + private static partial class InvalidDbIndex { } + + [AsciiHash("operation not permitted")] + private static partial class OperationNotPermitted { } + + [AsciiHash("unknown command")] + private static partial class UnknownCommand { } + + [AsciiHash("SELECT is not allowed in cluster mode")] + private static partial class SelectNotAllowedInClusterMode { } +} diff --git a/src/StackExchange.Redis/Availability/RetryDatabase.cs b/src/StackExchange.Redis/Availability/RetryDatabase.cs index ed5508819..9ed028705 100644 --- a/src/StackExchange.Redis/Availability/RetryDatabase.cs +++ b/src/StackExchange.Redis/Availability/RetryDatabase.cs @@ -1,5 +1,6 @@ using System; using System.Diagnostics; +using System.IO.Pipes; using System.Threading; using System.Threading.Tasks; using StackExchange.Redis.Interfaces; @@ -22,21 +23,24 @@ DatabaseFeatureFlags IInternalDatabaseAsync.GetFeatures(out string name) public override string ToString() => this.BuildString(); private readonly IDatabaseAsync _inner; - private readonly CircuitBreaker _circuitBreaker; private readonly int _maxBeforeFailover, _maxAttempts, _delayMillis, _jitterMillis, _failoverMillis; + private readonly RetryPolicy _policy; - public CancellationToken GetNextFailover() => _inner.GetNextFailover(); + public CancellationToken GetNextFailover() + => _maxAttempts > 1 & _maxBeforeFailover < _maxAttempts ? _inner.GetNextFailover() : CancellationToken.None; public RetryDatabase(IDatabaseAsync inner, RetryPolicy policy) { // cannot nest retry, and cannot issue retries *inside* a batch/transaction var features = inner.RejectFlags(DatabaseFeatureFlags.Batch | DatabaseFeatureFlags.Transaction | DatabaseFeatureFlags.Retry); + _policy = policy; + // capture config locally rather than constant cross-object lookups (plus: mutability) _maxBeforeFailover = (features & DatabaseFeatureFlags.Failover) == 0 ? int.MaxValue : policy.MaxAttemptsBeforeFailover; _maxAttempts = policy.MaxAttempts; if (_maxBeforeFailover == _maxAttempts) _maxBeforeFailover = int.MaxValue; // then we'll never look - if (_maxAttempts < 1) throw new ArgumentOutOfRangeException(nameof(policy.MaxAttempts)); + // guard the failover threshold: values < 1 can never be hit by the loop counter (which starts at 1), // so they would *silently* disable failover rather than erroring; validate the raw policy value if (policy.MaxAttemptsBeforeFailover < 1) throw new ArgumentOutOfRangeException(nameof(policy.MaxAttemptsBeforeFailover)); @@ -47,7 +51,6 @@ public RetryDatabase(IDatabaseAsync inner, RetryPolicy policy) if (_jitterMillis < 0) throw new ArgumentOutOfRangeException(nameof(policy.JitterMax)); if (_failoverMillis < 0) throw new ArgumentOutOfRangeException(nameof(policy.FailoverDelay)); _inner = inner; - _circuitBreaker = policy.CircuitBreaker ?? (inner.Multiplexer as IInternalConnectionMultiplexer)?.CircuitBreaker ?? CircuitBreaker.Default; } public int Database => _inner is IDatabase db ? db.Database : -1; @@ -67,35 +70,61 @@ private async Task ExecuteAsync(TState state, Func(TState state, Func operation) + where TState : struct, IRedisArgs { - if (failover.CanBeCanceled) + state.Map(this); + + int i = 0; + // note we need to capture this *before* the attempt - otherwise the failover could happen + // between the failed attempt and fetching this, and we'd miss it + CancellationToken ct = GetNextFailover(); + while (true) { - // we're in the failover slot; this gets exciting - return AwaitFailover(failover); + ++i; + try + { + await operation(state, _inner).ConfigureAwait(false); + break; + } + catch (Exception ex) when (i < _maxAttempts && _policy.CanRetry(ex, state.Flags, out var switchToNewServer)) + { + // we can give it another attempt + await FailoverOrDelayAsync(i, switchToNewServer, ref ct).ConfigureAwait(false); + } + } + // (nothing to post-process) + } + + private Task FailoverOrDelayAsync(int attempt, bool switchToNewServer, ref CancellationToken failover) + { + if (failover.CanBeCanceled && (attempt == _maxBeforeFailover | switchToNewServer)) + { + // we're in the failover slot - possibly by count, possibly by the retry-policy + var snapshot = failover; + failover = CancellationToken.None; // only retry at most once + return AwaitFailover(snapshot); } else { @@ -123,33 +152,6 @@ private async Task AwaitFailover(CancellationToken failover) await Task.Delay(ServerSelectionStrategy.SharedRandom.Next(_jitterMillis), CancellationToken.None).ConfigureAwait(false); } - private async Task ExecuteAsync(TState state, Func operation) - where TState : struct, IRedisArgs - { - state.Map(this); - - int i = 0; - CancellationToken ct = CancellationToken.None; - while (true) - { - // note we need to capture this *before* the attempt - otherwise the failover could happen - // between the failed attempt and fetching this, and we'd miss it - if (++i == _maxBeforeFailover) ct = GetNextFailover(); - try - { - await operation(state, _inner).ConfigureAwait(false); - return; - } - catch (Exception ex) when (_circuitBreaker.IsConnectionFault(ex) && i < _maxAttempts) - { - // we can give it another attempt - Debug.WriteLine(ex.Message); - await FailoverOrDelayAsync(ct).ConfigureAwait(false); - ct = CancellationToken.None; // we only apply failover one time - } - } - } - void IRedisAsync.Wait(Task task) => _inner.Wait(task); T IRedisAsync.Wait(Task task) => _inner.Wait(task); void IRedisAsync.WaitAll(Task[] tasks) => _inner.WaitAll(tasks); diff --git a/src/StackExchange.Redis/Availability/RetryPolicy.cs b/src/StackExchange.Redis/Availability/RetryPolicy.cs index 680379da7..f971f9ed4 100644 --- a/src/StackExchange.Redis/Availability/RetryPolicy.cs +++ b/src/StackExchange.Redis/Availability/RetryPolicy.cs @@ -1,5 +1,5 @@ using System; -using System.Threading; +using System.Diagnostics; namespace StackExchange.Redis.Availability; @@ -9,12 +9,6 @@ namespace StackExchange.Redis.Availability; /// public class RetryPolicy { - /// - /// Controls (via ) which faults can be retried; if not supplied, - /// the connection's configured will be used. - /// - public CircuitBreaker? CircuitBreaker { get; set; } - /// /// The maximum number of times an operation can be attempted. Defaults to 3. /// @@ -62,4 +56,110 @@ public TimeSpan JitterMax internal int DelayMilliseconds => _delayMillis; internal int JitterMilliseconds => _jitterMillis; internal int FailoverMilliseconds => _failoverMillis; + + internal bool CanRetry(Exception ex, CommandFlags flags, out bool onNewServer) + { + FaultContext ctx = new(ex); + onNewServer = false; + return CanRetry(ctx, flags, ref onNewServer); + } + + /// + /// Controls which operations can be repeated, optionally indicating that this should progress to + /// a new server. + /// + public virtual bool CanRetry(in FaultContext fault, CommandFlags flags, ref bool onNewServer) + => CircuitBreaker.DefaultIsFailure(in fault); // use the same default logic that governs CircuitBreaker +} + +internal static class RetryPolicyExtensions +{ + private const string ServerKey = "redis-server"; + private const string ConnectionKey = "redis-conn"; + + internal static Exception Add(Exception target, string key, object boxed) + { + try // best effort + { + target.Data[key] = boxed; + } + catch (Exception fault) + { + Debug.WriteLine(fault.Message); + } + return target; + } + + // we optimize for our expected exception kinds, but fallback to using .Data + internal static Exception With(this Exception target, RedisErrorKind kind) => Add(target, ServerKey, kind); + + internal static Exception With(this RedisServerException target, RedisErrorKind kind) + { + target.Kind = kind; + return target; + } + + internal static Exception With(this Exception target, ConnectionFailureType kind) => Add(target, ConnectionKey, kind); + + internal static Exception With(this RedisConnectionException target, ConnectionFailureType kind) + { + target.Kind = kind; + return target; + } + + internal static RedisErrorKind GetErrorKind(this Exception? target, out ConnectionFailureType connectionFailure) + { + RedisErrorKind kind = RedisErrorKind.None; + connectionFailure = ConnectionFailureType.None; + switch (target) + { + case null: + break; + case RedisServerException server: + kind = server.Kind; + break; + case RedisConnectionException connection: + connectionFailure = connection.Kind; + kind = RedisErrorKind.ConnectionFault; + break; + case TimeoutException: // includes RedisTimeoutException + kind = RedisErrorKind.Timeout; + break; + default: + try + { + if (target.Data[ServerKey] is RedisErrorKind server) + { + kind = server; + } + if (target.Data[ConnectionKey] is ConnectionFailureType conn) + { + connectionFailure = conn; + if (kind is RedisErrorKind.None) kind = RedisErrorKind.ConnectionFault; + } + } + catch (Exception fault) + { + Debug.WriteLine(fault.Message); + } + + break; + } + + if (kind is not RedisErrorKind.None & connectionFailure is ConnectionFailureType.None) + { + // fill in some blanks + switch (kind) + { + case RedisErrorKind.Loading: + connectionFailure = ConnectionFailureType.Loading; + break; + case RedisErrorKind.NoAuth: + case RedisErrorKind.WrongPass: + connectionFailure = ConnectionFailureType.AuthenticationFailure; + break; + } + } + return kind; + } } diff --git a/src/StackExchange.Redis/ConnectionMultiplexer.cs b/src/StackExchange.Redis/ConnectionMultiplexer.cs index 8253988cc..769a64b50 100644 --- a/src/StackExchange.Redis/ConnectionMultiplexer.cs +++ b/src/StackExchange.Redis/ConnectionMultiplexer.cs @@ -23,8 +23,6 @@ namespace StackExchange.Redis /// public sealed partial class ConnectionMultiplexer : IInternalConnectionMultiplexer // implies : IConnectionMultiplexer and : IDisposable { - Availability.CircuitBreaker? IInternalConnectionMultiplexer.CircuitBreaker => RawConfig.CircuitBreaker; - // This gets accessed for every received event; let's make sure we can process it "raw" internal readonly byte[]? ConfigurationChangedChannel; // Unique identifier used when tracing diff --git a/src/StackExchange.Redis/Enums/CommandFlags.cs b/src/StackExchange.Redis/Enums/CommandFlags.cs index ff69fb750..c8e380c6b 100644 --- a/src/StackExchange.Redis/Enums/CommandFlags.cs +++ b/src/StackExchange.Redis/Enums/CommandFlags.cs @@ -111,5 +111,34 @@ public enum CommandFlags // 2048: Use subscription connection type; never user-specified, so not visible on the public API // 4096: Identifies handshake completion messages; never user-specified, so not visible on the public API + + /* + Command side-effect/retry logic: + Reserve a chunk of bits for command retry logic/categorization; + by default, nothing in this chunk will be passed and the library will + supply a value based on the command being issued. The caller can override, + though - ultimately it is their data/server! They will need to supply a + suitable value for Execute[Async] etc, otherwise we'll assume the worst. + + The math here is intended so that we can specify a numeric "max" that we'll + retry, and can filter with <=, so the higher numbers have more side-effects. + As such, the main chunk are not flags per-se, and we are intentionally leaving gaps + to slide other things in later. + + the numbers here are before << into their respective slot + CommandRetryAlways = 1, (distinct from 0, which is implicit "not specified") + + CommandRetryConnection = 4 - things like CLIENT SETNAME or safe metadata (COMMAND, CONFIG GET); note that CLIENT KILL would also have CommandServerSpecific + CommandRetryReadOnly = 8 - things like GET + CommandRetryWriteChecked = 12 - things like SETNX or SET IFEQ + CommandRetryWriteLastWins = 16 - things like SET + CommandRetryWriteAccumulating = 20 - things like INCR, LPOP + CommandRetryServerAdmin = 24 - things like REPLICAOF, CONFIG SET; note these will commonly also include CommandServerSpecific + CommandRetryNever = (all the bits for the above region - I guess this means it is 31?) + + CommandServerSpecific = (a single bit, means "never retry on a different endpoint" - think cursors) + + + */ } } diff --git a/src/StackExchange.Redis/Exceptions.cs b/src/StackExchange.Redis/Exceptions.cs index 1f1c973ce..806804219 100644 --- a/src/StackExchange.Redis/Exceptions.cs +++ b/src/StackExchange.Redis/Exceptions.cs @@ -1,6 +1,7 @@ using System; using System.ComponentModel; using System.Runtime.Serialization; +using StackExchange.Redis.Availability; namespace StackExchange.Redis { @@ -120,6 +121,8 @@ public RedisConnectionException(ConnectionFailureType failureType, string messag /// public CommandStatus CommandStatus { get; } + internal ConnectionFailureType Kind { get; set; } + #if NET8_0_OR_GREATER [Obsolete(Obsoletions.LegacyFormatterImplMessage, DiagnosticId = Obsoletions.LegacyFormatterImplDiagId)] [EditorBrowsable(EditorBrowsableState.Never)] @@ -195,5 +198,7 @@ public RedisServerException(string message) : base(message) { } [EditorBrowsable(EditorBrowsableState.Never)] #endif private RedisServerException(SerializationInfo info, StreamingContext ctx) : base(info, ctx) { } + + internal RedisErrorKind Kind { get; set; } } } diff --git a/src/StackExchange.Redis/Interfaces/IConnectionMultiplexer.cs b/src/StackExchange.Redis/Interfaces/IConnectionMultiplexer.cs index 2fac75bec..f9a4238d2 100644 --- a/src/StackExchange.Redis/Interfaces/IConnectionMultiplexer.cs +++ b/src/StackExchange.Redis/Interfaces/IConnectionMultiplexer.cs @@ -17,8 +17,6 @@ internal interface IInternalConnectionMultiplexer : IConnectionMultiplexer bool IgnoreConnect { get; set; } - CircuitBreaker? CircuitBreaker { get; } - ReadOnlySpan GetServerSnapshot(); ServerEndPoint GetServerEndPoint(EndPoint endpoint); diff --git a/src/StackExchange.Redis/PhysicalConnection.cs b/src/StackExchange.Redis/PhysicalConnection.cs index c4cba0312..e85e00454 100644 --- a/src/StackExchange.Redis/PhysicalConnection.cs +++ b/src/StackExchange.Redis/PhysicalConnection.cs @@ -1183,7 +1183,7 @@ public void ObserveMessageResult(Exception? fault) // returns true while healthy); if it trips and we're the *first* to notice, hand off the // teardown rather than doing it inline: RecordConnectionFailed can fail the whole backlog and // build a detailed exception, which we don't want to pay for on the completion thread. - if (circuitBreaker is { } cb && !cb.ObserveResult(fault) + if (circuitBreaker is { } cb && cb.Trip(fault) && Interlocked.CompareExchange(ref _circuitBreakerState, CircuitBreakerTripped, CircuitBreakerHealthy) is CircuitBreakerHealthy) { // hand off to a worker; the heartbeat (see OnBridgeHeartbeat) is a backstop in case the diff --git a/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt b/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt index 9d22f90a7..cc51c5f10 100644 --- a/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt @@ -1,7 +1,5 @@ #nullable enable StackExchange.Redis.Availability.RetryPolicy -StackExchange.Redis.Availability.RetryPolicy.CircuitBreaker.get -> StackExchange.Redis.Availability.CircuitBreaker? -StackExchange.Redis.Availability.RetryPolicy.CircuitBreaker.set -> void StackExchange.Redis.Availability.RetryPolicy.FailoverDelay.get -> System.TimeSpan StackExchange.Redis.Availability.RetryPolicy.FailoverDelay.set -> void StackExchange.Redis.Availability.RetryPolicy.JitterMax.get -> System.TimeSpan @@ -27,14 +25,7 @@ static StackExchange.Redis.KeyspaceIsolation.DatabaseExtensions.WithRetry(this S [SER007]StackExchange.Redis.Availability.CircuitBreaker.Builder.MetricsWindowSize.set -> void [SER007]StackExchange.Redis.Availability.CircuitBreaker.Builder.MinimumNumberOfFailures.get -> int [SER007]StackExchange.Redis.Availability.CircuitBreaker.Builder.MinimumNumberOfFailures.set -> void -[SER007]StackExchange.Redis.Availability.CircuitBreaker.Builder.TrackedExceptions.get -> System.Type![]? -[SER007]StackExchange.Redis.Availability.CircuitBreaker.Builder.TrackedExceptions.set -> void [SER007]StackExchange.Redis.Availability.CircuitBreaker.CircuitBreaker() -> void -[SER007]StackExchange.Redis.Availability.CircuitBreaker.CircuitBreakerContext -[SER007]StackExchange.Redis.Availability.CircuitBreaker.CircuitBreakerContext.CircuitBreakerContext() -> void -[SER007]StackExchange.Redis.Availability.CircuitBreaker.CircuitBreakerContext.CircuitBreakerContext(System.Exception? fault, bool evaluate = true) -> void -[SER007]StackExchange.Redis.Availability.CircuitBreaker.CircuitBreakerContext.Fault.get -> System.Exception? -[SER007]StackExchange.Redis.Availability.CircuitBreaker.CircuitBreakerContext.Success.get -> bool [SER007]StackExchange.Redis.Availability.ConnectionGroupMember [SER007]StackExchange.Redis.Availability.ConnectionGroupMember.ConnectionGroupMember(StackExchange.Redis.ConfigurationOptions! configuration, string! name = "") -> void [SER007]StackExchange.Redis.Availability.ConnectionGroupMember.ConnectionGroupMember(string! configuration, string! name = "") -> void @@ -117,10 +108,8 @@ static StackExchange.Redis.KeyspaceIsolation.DatabaseExtensions.WithRetry(this S [SER007]StackExchange.Redis.Availability.MultiGroupOptions.FailbackDelay.get -> System.TimeSpan [SER007]StackExchange.Redis.Availability.MultiGroupOptions.FailbackDelay.set -> void [SER007]abstract StackExchange.Redis.Availability.CircuitBreaker.Accumulator.IsHealthy() -> bool -[SER007]abstract StackExchange.Redis.Availability.CircuitBreaker.Accumulator.ObserveResult(in StackExchange.Redis.Availability.CircuitBreaker.CircuitBreakerContext context) -> bool [SER007]abstract StackExchange.Redis.Availability.CircuitBreaker.Accumulator.Reset() -> void [SER007]abstract StackExchange.Redis.Availability.CircuitBreaker.CreateAccumulator() -> StackExchange.Redis.Availability.CircuitBreaker.Accumulator! -[SER007]virtual StackExchange.Redis.Availability.CircuitBreaker.IsConnectionFault(System.Exception? fault) -> bool [SER007]static StackExchange.Redis.Availability.CircuitBreaker.Default.get -> StackExchange.Redis.Availability.CircuitBreaker! [SER007]static StackExchange.Redis.Availability.CircuitBreaker.None.get -> StackExchange.Redis.Availability.CircuitBreaker! [SER007]static StackExchange.Redis.Availability.CircuitBreaker.Builder.implicit operator StackExchange.Redis.Availability.CircuitBreaker!(StackExchange.Redis.Availability.CircuitBreaker.Builder! builder) -> StackExchange.Redis.Availability.CircuitBreaker! @@ -141,3 +130,42 @@ static StackExchange.Redis.KeyspaceIsolation.DatabaseExtensions.WithRetry(this S [SER007]static StackExchange.Redis.Availability.HealthCheck.Default.get -> StackExchange.Redis.Availability.HealthCheck! [SER007]static StackExchange.Redis.Availability.MultiGroupOptions.Default.get -> StackExchange.Redis.Availability.MultiGroupOptions! [SER007]override StackExchange.Redis.Availability.ConnectionGroupMember.ToString() -> string! +[SER007]StackExchange.Redis.Availability.RedisErrorKind +[SER007]StackExchange.Redis.Availability.RedisErrorKind.None = 0 -> StackExchange.Redis.Availability.RedisErrorKind +[SER007]StackExchange.Redis.Availability.RedisErrorKind.Unknown = 1 -> StackExchange.Redis.Availability.RedisErrorKind +StackExchange.Redis.Availability.FaultContext +StackExchange.Redis.Availability.FaultContext.ConnectionFailureType.get -> StackExchange.Redis.ConnectionFailureType +StackExchange.Redis.Availability.FaultContext.ErrorKind.get -> StackExchange.Redis.Availability.RedisErrorKind +StackExchange.Redis.Availability.FaultContext.Fault.get -> System.Exception! +StackExchange.Redis.Availability.FaultContext.FaultContext() -> void +StackExchange.Redis.Availability.FaultContext.FaultContext(System.Exception! fault) -> void +StackExchange.Redis.Availability.FaultContext.HasValue.get -> bool +[SER007]StackExchange.Redis.Availability.RedisErrorKind.Ask = 15 -> StackExchange.Redis.Availability.RedisErrorKind +[SER007]StackExchange.Redis.Availability.RedisErrorKind.Busy = 12 -> StackExchange.Redis.Availability.RedisErrorKind +[SER007]StackExchange.Redis.Availability.RedisErrorKind.ClusterDown = 6 -> StackExchange.Redis.Availability.RedisErrorKind +[SER007]StackExchange.Redis.Availability.RedisErrorKind.ConnectionFault = 3 -> StackExchange.Redis.Availability.RedisErrorKind +[SER007]StackExchange.Redis.Availability.RedisErrorKind.CrossSlot = 16 -> StackExchange.Redis.Availability.RedisErrorKind +[SER007]StackExchange.Redis.Availability.RedisErrorKind.DatabaseSelectDisabled = 27 -> StackExchange.Redis.Availability.RedisErrorKind +[SER007]StackExchange.Redis.Availability.RedisErrorKind.ExecAbort = 23 -> StackExchange.Redis.Availability.RedisErrorKind +[SER007]StackExchange.Redis.Availability.RedisErrorKind.InvalidDatabaseIndex = 26 -> StackExchange.Redis.Availability.RedisErrorKind +[SER007]StackExchange.Redis.Availability.RedisErrorKind.Loading = 5 -> StackExchange.Redis.Availability.RedisErrorKind +[SER007]StackExchange.Redis.Availability.RedisErrorKind.MasterDown = 7 -> StackExchange.Redis.Availability.RedisErrorKind +[SER007]StackExchange.Redis.Availability.RedisErrorKind.MaxClients = 13 -> StackExchange.Redis.Availability.RedisErrorKind +[SER007]StackExchange.Redis.Availability.RedisErrorKind.Misconfigured = 10 -> StackExchange.Redis.Availability.RedisErrorKind +[SER007]StackExchange.Redis.Availability.RedisErrorKind.Moved = 14 -> StackExchange.Redis.Availability.RedisErrorKind +[SER007]StackExchange.Redis.Availability.RedisErrorKind.NoAuth = 17 -> StackExchange.Redis.Availability.RedisErrorKind +[SER007]StackExchange.Redis.Availability.RedisErrorKind.NoPermission = 19 -> StackExchange.Redis.Availability.RedisErrorKind +[SER007]StackExchange.Redis.Availability.RedisErrorKind.NoReplicas = 9 -> StackExchange.Redis.Availability.RedisErrorKind +[SER007]StackExchange.Redis.Availability.RedisErrorKind.NoScript = 25 -> StackExchange.Redis.Availability.RedisErrorKind +[SER007]StackExchange.Redis.Availability.RedisErrorKind.NotPermitted = 20 -> StackExchange.Redis.Availability.RedisErrorKind +[SER007]StackExchange.Redis.Availability.RedisErrorKind.OutOfMemory = 11 -> StackExchange.Redis.Availability.RedisErrorKind +[SER007]StackExchange.Redis.Availability.RedisErrorKind.ReadOnly = 24 -> StackExchange.Redis.Availability.RedisErrorKind +[SER007]StackExchange.Redis.Availability.RedisErrorKind.Timeout = 4 -> StackExchange.Redis.Availability.RedisErrorKind +[SER007]StackExchange.Redis.Availability.RedisErrorKind.TryAgain = 8 -> StackExchange.Redis.Availability.RedisErrorKind +[SER007]StackExchange.Redis.Availability.RedisErrorKind.UnknownCommand = 21 -> StackExchange.Redis.Availability.RedisErrorKind +[SER007]StackExchange.Redis.Availability.RedisErrorKind.UnknownError = 2 -> StackExchange.Redis.Availability.RedisErrorKind +[SER007]StackExchange.Redis.Availability.RedisErrorKind.WrongPass = 18 -> StackExchange.Redis.Availability.RedisErrorKind +[SER007]StackExchange.Redis.Availability.RedisErrorKind.WrongType = 22 -> StackExchange.Redis.Availability.RedisErrorKind +[SER007]abstract StackExchange.Redis.Availability.CircuitBreaker.Accumulator.ObserveResult(in StackExchange.Redis.Availability.FaultContext fault) -> void +[SER007]virtual StackExchange.Redis.Availability.CircuitBreaker.Accumulator.IsFailure(in StackExchange.Redis.Availability.FaultContext fault) -> bool +virtual StackExchange.Redis.Availability.RetryPolicy.CanRetry(in StackExchange.Redis.Availability.FaultContext fault, StackExchange.Redis.CommandFlags flags, ref bool onNewServer) -> bool diff --git a/src/StackExchange.Redis/RedisTransaction.cs b/src/StackExchange.Redis/RedisTransaction.cs index 7ae9ca085..d76974b3f 100644 --- a/src/StackExchange.Redis/RedisTransaction.cs +++ b/src/StackExchange.Redis/RedisTransaction.cs @@ -5,6 +5,7 @@ using System.Threading; using System.Threading.Tasks; using RESPite.Messages; +using StackExchange.Redis.Availability; using StackExchange.Redis.Interfaces; namespace StackExchange.Redis @@ -485,11 +486,12 @@ public override bool SetResult(PhysicalConnection connection, Message message, r reader.MovePastBof(); if (reader.IsError && message is TransactionMessage tran) { + var errorKind = RedisErrorKindMetadata.Classify(reader); string error = reader.ReadString()!; foreach (var op in tran.InnerOperations) { var inner = op.Wrapped; - ServerFail(inner, error); + ServerFail(inner, errorKind, error); inner.Complete(connection); } } diff --git a/src/StackExchange.Redis/ResultProcessor.Literals.cs b/src/StackExchange.Redis/ResultProcessor.Literals.cs index 7d50cc09e..3deaa7027 100644 --- a/src/StackExchange.Redis/ResultProcessor.Literals.cs +++ b/src/StackExchange.Redis/ResultProcessor.Literals.cs @@ -8,16 +8,6 @@ internal partial class Literals { #pragma warning disable CS8981, SA1300, SA1134 // forgive naming etc // ReSharper disable InconsistentNaming - [AsciiHash] internal static partial class NOAUTH { } - [AsciiHash] internal static partial class WRONGPASS { } - [AsciiHash] internal static partial class NOSCRIPT { } - [AsciiHash] internal static partial class MOVED { } - [AsciiHash] internal static partial class ASK { } - [AsciiHash] internal static partial class READONLY { } - [AsciiHash] internal static partial class LOADING { } - [AsciiHash("ERR operation not permitted")] - internal static partial class ERR_not_permitted { } - // Result processor literals [AsciiHash] internal static partial class OK diff --git a/src/StackExchange.Redis/ResultProcessor.cs b/src/StackExchange.Redis/ResultProcessor.cs index c4ad5e801..18403ba65 100644 --- a/src/StackExchange.Redis/ResultProcessor.cs +++ b/src/StackExchange.Redis/ResultProcessor.cs @@ -11,6 +11,7 @@ using Microsoft.Extensions.Logging; using RESPite; using RESPite.Messages; +using StackExchange.Redis.Availability; namespace StackExchange.Redis { @@ -211,14 +212,14 @@ public void ConnectionFail(Message message, ConnectionFailureType fail, Exceptio sb.Append(annotation); } var ex = new RedisConnectionException(fail, sb.ToString(), innerException); - SetException(message, ex); + SetException(message, ex.With(fail)); } public static void ConnectionFail(Message message, ConnectionFailureType fail, string errorMessage) => - SetException(message, new RedisConnectionException(fail, errorMessage)); + SetException(message, new RedisConnectionException(fail, errorMessage).With(fail)); - public static void ServerFail(Message message, string errorMessage) => - SetException(message, new RedisServerException(errorMessage)); + public static void ServerFail(Message message, RedisErrorKind kind, string errorMessage) => + SetException(message, new RedisServerException(errorMessage).With(kind)); public static void SetException(Message? message, Exception ex) { @@ -263,22 +264,24 @@ private bool HandleCommonError(Message message, RespReader reader, PhysicalConne { connection.OnDetailLog($"applying common error-handling: {reader.GetOverview()}"); var bridge = connection.BridgeCouldBeNull; - if (reader.StartsWith(Literals.NOAUTH.U8)) + var errorKind = RedisErrorKindMetadata.Classify(reader); + switch (errorKind) { - bridge?.Multiplexer.SetAuthSuspect(new RedisServerException("NOAUTH Returned - connection has not yet authenticated")); - } - else if (reader.StartsWith(Literals.WRONGPASS.U8)) - { - bridge?.Multiplexer.SetAuthSuspect(new RedisServerException(reader.GetOverview())); + case RedisErrorKind.NoAuth: + bridge?.Multiplexer.SetAuthSuspect(new RedisServerException("NOAUTH Returned - connection has not yet authenticated").With(errorKind)); + break; + case RedisErrorKind.WrongPass: + bridge?.Multiplexer.SetAuthSuspect(new RedisServerException(reader.GetOverview()).With(errorKind)); + break; } var server = bridge?.ServerEndPoint; bool log = !message.IsInternalCall; - bool isMoved = reader.StartsWith(Literals.MOVED.U8); + bool isMoved = errorKind == RedisErrorKind.Moved; bool wasNoRedirect = (message.Flags & CommandFlags.NoRedirect) != 0; string? err = string.Empty; bool unableToConnectError = false; - if (isMoved || reader.StartsWith(Literals.ASK.U8)) + if (isMoved || errorKind == RedisErrorKind.Ask) { connection.OnDetailLog($"redirect via {(isMoved ? "MOVED" : "ASK")} to '{reader.ReadString()}'"); message.SetResponseReceived(); @@ -361,7 +364,7 @@ private bool HandleCommonError(Message message, RespReader reader, PhysicalConne } else { - ServerFail(message, err); + ServerFail(message, errorKind, err); } return true; @@ -852,7 +855,7 @@ public override bool SetResult(PhysicalConnection connection, Message message, r { var copy = reader; reader.MovePastBof(); - if (reader.IsError && reader.StartsWith(Literals.READONLY.U8)) + if (reader.IsError && RedisErrorKindMetadata.Classify(reader) == RedisErrorKind.ReadOnly) { var bridge = connection.BridgeCouldBeNull; if (bridge != null) @@ -2249,7 +2252,7 @@ public override bool SetResult(PhysicalConnection connection, Message message, r { var copy = reader; reader.MovePastBof(); - if (reader.IsError && reader.StartsWith(Literals.NOSCRIPT.U8)) + if (reader.IsError && RedisErrorKindMetadata.Classify(reader) == RedisErrorKind.NoScript) { // scripts are not flushed individually, so assume the entire script cache is toast ("SCRIPT FLUSH") connection.BridgeCouldBeNull?.ServerEndPoint?.FlushScriptCache(); message.SetScriptUnavailable(); @@ -3167,11 +3170,12 @@ public override bool SetResult(PhysicalConnection connection, Message message, r if (isError) { reader = copy; // rewind and re-parse - if (reader.StartsWith(Literals.ERR_not_permitted.U8) || reader.StartsWith(Literals.NOAUTH.U8)) + var errorKind = RedisErrorKindMetadata.Classify(reader); + if (errorKind is RedisErrorKind.NotPermitted or RedisErrorKind.NoAuth) { connection.RecordConnectionFailed(ConnectionFailureType.AuthenticationFailure, new Exception(reader.GetOverview() + " Verify if the Redis password provided is correct. Attempted command: " + message.Command)); } - else if (reader.StartsWith(Literals.LOADING.U8)) + else if (errorKind == RedisErrorKind.Loading) { connection.RecordConnectionFailed(ConnectionFailureType.Loading); } diff --git a/tests/StackExchange.Redis.Tests/CircuitBreakerServerTests.cs b/tests/StackExchange.Redis.Tests/CircuitBreakerServerTests.cs index 074dafcd9..d34afe8ee 100644 --- a/tests/StackExchange.Redis.Tests/CircuitBreakerServerTests.cs +++ b/tests/StackExchange.Redis.Tests/CircuitBreakerServerTests.cs @@ -60,7 +60,7 @@ private sealed class CountingCircuitBreaker : CircuitBreaker private sealed class CountingAccumulator(CountingCircuitBreaker owner) : Accumulator { - public override bool ObserveResult(in CircuitBreakerContext context) + public override bool ObserveResult(in FaultContext context) { if (context.Success) { diff --git a/tests/StackExchange.Redis.Tests/CircuitBreakerUnitTests.cs b/tests/StackExchange.Redis.Tests/CircuitBreakerUnitTests.cs index dad3dd015..89f930218 100644 --- a/tests/StackExchange.Redis.Tests/CircuitBreakerUnitTests.cs +++ b/tests/StackExchange.Redis.Tests/CircuitBreakerUnitTests.cs @@ -71,7 +71,7 @@ private sealed class AlwaysUnhealthyBreaker : CircuitBreaker private sealed class Acc : Accumulator { - public override bool ObserveResult(in CircuitBreakerContext context) => false; + public override bool ObserveResult(in FaultContext context) => false; public override bool IsHealthy() => false; public override void Reset() { } } diff --git a/tests/StackExchange.Redis.Tests/Helpers/SharedConnectionFixture.cs b/tests/StackExchange.Redis.Tests/Helpers/SharedConnectionFixture.cs index 9f894d907..8da84bce6 100644 --- a/tests/StackExchange.Redis.Tests/Helpers/SharedConnectionFixture.cs +++ b/tests/StackExchange.Redis.Tests/Helpers/SharedConnectionFixture.cs @@ -58,8 +58,6 @@ internal sealed class NonDisposingConnection(IInternalConnectionMultiplexer inne { public IInternalConnectionMultiplexer UnderlyingConnection => _inner; - Availability.CircuitBreaker? IInternalConnectionMultiplexer.CircuitBreaker => _inner.CircuitBreaker; - public bool AllowConnect { get => _inner.AllowConnect; diff --git a/tests/StackExchange.Redis.Tests/MultiGroupTests/CircuitBreakerRerouteTests.cs b/tests/StackExchange.Redis.Tests/MultiGroupTests/CircuitBreakerRerouteTests.cs index 18436db46..39fef72b3 100644 --- a/tests/StackExchange.Redis.Tests/MultiGroupTests/CircuitBreakerRerouteTests.cs +++ b/tests/StackExchange.Redis.Tests/MultiGroupTests/CircuitBreakerRerouteTests.cs @@ -113,7 +113,7 @@ private sealed class FlipBreaker : CircuitBreaker private sealed class Acc(FlipBreaker owner) : Accumulator { - public override bool ObserveResult(in CircuitBreakerContext context) => !owner._tripped; + public override bool ObserveResult(in FaultContext context) => !owner._tripped; public override bool IsHealthy() => !owner._tripped; public override void Reset() { } } diff --git a/tests/StackExchange.Redis.Tests/RetryTests/BasicRetryTests.cs b/tests/StackExchange.Redis.Tests/RetryTests/BasicRetryTests.cs index 0c8203872..400dac5e4 100644 --- a/tests/StackExchange.Redis.Tests/RetryTests/BasicRetryTests.cs +++ b/tests/StackExchange.Redis.Tests/RetryTests/BasicRetryTests.cs @@ -1,3 +1,4 @@ +using System; using System.IO; using System.Threading.Tasks; using StackExchange.Redis.Tests.Helpers; @@ -31,4 +32,31 @@ public async Task ConnectAndGet() Assert.Equal("hello", await db.StringGetAsync(key)); Assert.Equal(RedisValue.Null, await db.StringGetAsync("retry:missing")); } + + // Exploratory (no retry wrapper yet): flip the server into a LOADING state *after* connecting, + // then issue a GET and observe what exception type SE.Redis surfaces. No hard assertion on the + // type yet - we just want to see it in the log so we can decide how the circuit-breaker should + // classify it. + [Fact] + public async Task LoadingSurfacesAs() + { + using var server = new InProcessTestServer(log); + await using var conn = await server.ConnectAsync(log: Log); + Assert.True(conn.IsConnected); + + var db = conn.GetDatabase(); + Assert.True(await db.StringSetAsync("retry:loading", "before")); // works before loading + + server.IsLoading = true; + try + { + var value = await db.StringGetAsync("retry:loading"); + Log.WriteLine($"No exception; got value: {value}"); + } + catch (Exception ex) + { + Log.WriteLine($"Exception type: {ex.GetType().FullName}"); + Log.WriteLine($"Message: {ex.Message}"); + } + } } diff --git a/toys/StackExchange.Redis.Server/RedisServer.cs b/toys/StackExchange.Redis.Server/RedisServer.cs index 26f1a2fc3..77ad4f1ed 100644 --- a/toys/StackExchange.Redis.Server/RedisServer.cs +++ b/toys/StackExchange.Redis.Server/RedisServer.cs @@ -160,8 +160,18 @@ protected override void AppendStats(StringBuilder sb) public string Password { get; set; } = ""; + /// + /// When set, every command is rejected with a LOADING error, mimicking a server that is + /// still loading its dataset into memory. + /// + public bool IsLoading { get; set; } + public override TypedRedisValue Execute(RedisClient client, in RedisRequest request) { + if (IsLoading) + { + return TypedRedisValue.Error("LOADING"); + } var pw = Password; if (!string.IsNullOrEmpty(pw) & !client.IsAuthenticated) { From cbeb121cf71bd5185abc4771f4c7611c179f1db9 Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Thu, 16 Jul 2026 14:32:36 +0100 Subject: [PATCH 13/23] nits --- .../Availability/CircuitBreaker.cs | 10 ++--- .../Availability/FaultContext.cs | 14 +++---- .../PublicAPI/PublicAPI.Unshipped.txt | 6 +-- .../CircuitBreakerServerTests.cs | 12 +++--- .../CircuitBreakerUnitTests.cs | 40 +++---------------- .../CircuitBreakerRerouteTests.cs | 2 +- 6 files changed, 26 insertions(+), 58 deletions(-) diff --git a/src/StackExchange.Redis/Availability/CircuitBreaker.cs b/src/StackExchange.Redis/Availability/CircuitBreaker.cs index 4f9b08b65..67c075bd6 100644 --- a/src/StackExchange.Redis/Availability/CircuitBreaker.cs +++ b/src/StackExchange.Redis/Availability/CircuitBreaker.cs @@ -34,15 +34,15 @@ public class Builder private const int DefaultMinimumNumberOfFailures = 1000; private static readonly TimeSpan DefaultMetricsWindowSize = TimeSpan.FromSeconds(2); -#pragma warning disable SA1114 // Parameter list should follow declaration - false positive: the #if directive splits the argument list internal static CircuitBreaker DefaultInstance = new DefaultCircuitBreaker( +#pragma warning disable SA1114 // Parameter list should follow declaration - false positive: the #if directive splits the argument list #if NET8_0_OR_GREATER null, #endif +#pragma warning restore SA1114 DefaultFailureRateThreshold, DefaultMinimumNumberOfFailures, DefaultMetricsWindowSize); -#pragma warning restore SA1114 /// /// Percentage of failures to trigger circuit breaker. @@ -81,15 +81,15 @@ public CircuitBreaker Create() && MetricsWindowSize == DefaultMetricsWindowSize) return DefaultInstance; -#pragma warning disable SA1114 // Parameter list should follow declaration - false positive: the #if directive splits the argument list return new DefaultCircuitBreaker( +#pragma warning disable SA1114 // Parameter list should follow declaration - false positive: the #if directive splits the argument list #if NET8_0_OR_GREATER TimeProvider, #endif +#pragma warning restore SA1114 FailureRateThreshold, MinimumNumberOfFailures, MetricsWindowSize); -#pragma warning restore SA1114 } /// @@ -305,7 +305,7 @@ public override void ObserveResult(in FaultContext result) // only getting results one at a time, so we shouldn't be over-stomping much *anyway* Span buckets = _buckets; // *not* a payload copy; this is in-place over the data ref Bucket bucket = ref buckets[index]; - bucket.Count(epoch, success: !result.HasValue); + bucket.Count(epoch, success: !result.IsFault); } public override bool IsHealthy() => Evaluate(breaker.GetEpoch()); diff --git a/src/StackExchange.Redis/Availability/FaultContext.cs b/src/StackExchange.Redis/Availability/FaultContext.cs index 41232557a..f7901f9c1 100644 --- a/src/StackExchange.Redis/Availability/FaultContext.cs +++ b/src/StackExchange.Redis/Availability/FaultContext.cs @@ -8,8 +8,7 @@ namespace StackExchange.Redis.Availability; /// public readonly struct FaultContext { - private readonly Exception _fault; - private readonly RedisErrorKind _errorKind; + private readonly Exception? _fault; private readonly ConnectionFailureType _connectionFailureType; internal static readonly FaultContext Success = default; @@ -18,26 +17,27 @@ public readonly struct FaultContext /// Create a new . /// /// The fault associated with the operation, or null on success. - public FaultContext(Exception fault) + public FaultContext(Exception? fault) { _fault = fault; - _errorKind = fault.GetErrorKind(out _connectionFailureType); + ErrorKind = fault.GetErrorKind(out _connectionFailureType); } /// /// Indicates whether a fault is present. /// - public bool HasValue => _fault is not null; // excludes: default(FaultContext) + [MemberNotNullWhen(true, nameof(Fault))] + public bool IsFault => _fault is not null; // excludes: default(FaultContext) /// /// The fault associated with the operation. /// - public Exception Fault => _fault; + public Exception? Fault => _fault; /// /// The classified server error condition associated with the fault, if any. /// - public RedisErrorKind ErrorKind => _errorKind; + public RedisErrorKind ErrorKind { get; } /// /// The connection failure type associated with the fault, if any. diff --git a/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt b/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt index cc51c5f10..5c3ecb8f4 100644 --- a/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt @@ -136,10 +136,10 @@ static StackExchange.Redis.KeyspaceIsolation.DatabaseExtensions.WithRetry(this S StackExchange.Redis.Availability.FaultContext StackExchange.Redis.Availability.FaultContext.ConnectionFailureType.get -> StackExchange.Redis.ConnectionFailureType StackExchange.Redis.Availability.FaultContext.ErrorKind.get -> StackExchange.Redis.Availability.RedisErrorKind -StackExchange.Redis.Availability.FaultContext.Fault.get -> System.Exception! +StackExchange.Redis.Availability.FaultContext.Fault.get -> System.Exception? StackExchange.Redis.Availability.FaultContext.FaultContext() -> void -StackExchange.Redis.Availability.FaultContext.FaultContext(System.Exception! fault) -> void -StackExchange.Redis.Availability.FaultContext.HasValue.get -> bool +StackExchange.Redis.Availability.FaultContext.FaultContext(System.Exception? fault) -> void +StackExchange.Redis.Availability.FaultContext.IsFault.get -> bool [SER007]StackExchange.Redis.Availability.RedisErrorKind.Ask = 15 -> StackExchange.Redis.Availability.RedisErrorKind [SER007]StackExchange.Redis.Availability.RedisErrorKind.Busy = 12 -> StackExchange.Redis.Availability.RedisErrorKind [SER007]StackExchange.Redis.Availability.RedisErrorKind.ClusterDown = 6 -> StackExchange.Redis.Availability.RedisErrorKind diff --git a/tests/StackExchange.Redis.Tests/CircuitBreakerServerTests.cs b/tests/StackExchange.Redis.Tests/CircuitBreakerServerTests.cs index d34afe8ee..54783b8b0 100644 --- a/tests/StackExchange.Redis.Tests/CircuitBreakerServerTests.cs +++ b/tests/StackExchange.Redis.Tests/CircuitBreakerServerTests.cs @@ -60,19 +60,17 @@ private sealed class CountingCircuitBreaker : CircuitBreaker private sealed class CountingAccumulator(CountingCircuitBreaker owner) : Accumulator { - public override bool ObserveResult(in FaultContext context) + public override void ObserveResult(in FaultContext context) { - if (context.Success) + if (context.IsFault) { - Interlocked.Increment(ref owner._successes); + Interlocked.Increment(ref owner._failures); + owner.LastFault = context.Fault; } else { - Interlocked.Increment(ref owner._failures); - owner.LastFault = context.Fault; + Interlocked.Increment(ref owner._successes); } - - return true; // stay healthy; we're only here to observe, not to trip the connection } public override bool IsHealthy() => true; diff --git a/tests/StackExchange.Redis.Tests/CircuitBreakerUnitTests.cs b/tests/StackExchange.Redis.Tests/CircuitBreakerUnitTests.cs index 89f930218..7b1ad9c04 100644 --- a/tests/StackExchange.Redis.Tests/CircuitBreakerUnitTests.cs +++ b/tests/StackExchange.Redis.Tests/CircuitBreakerUnitTests.cs @@ -51,32 +51,6 @@ public void None_IsAlwaysHealthy() Assert.True(Record(acc, 10_000, new RedisTimeoutException("boom", CommandStatus.Unknown))); } - [Fact] - public void NonEvaluatingObservation_IsNeverTreatedAsUnhealthy() - { - // a deliberately naive breaker whose accumulator *always* reports unhealthy - e.g. one that - // returns default(false) whether or not it was actually asked to evaluate. A success is - // observed without evaluation, so that bogus verdict must be ignored (not read as a trip); - // only a genuine evaluation (a fault) is allowed to report unhealthy. - var acc = new AlwaysUnhealthyBreaker().CreateAccumulator(); - - Assert.True(acc.ObserveResult((Exception?)null)); // success -> not evaluating -> verdict ignored - Assert.False(acc.ObserveResult(Timeout())); // fault -> evaluating -> verdict honoured - } - - // never reports healthy, even when not evaluating; stands in for a buggy/naive custom breaker - private sealed class AlwaysUnhealthyBreaker : CircuitBreaker - { - public override Accumulator CreateAccumulator() => new Acc(); - - private sealed class Acc : Accumulator - { - public override bool ObserveResult(in FaultContext context) => false; - public override bool IsHealthy() => false; - public override void Reset() { } - } - } - #if NET8_0_OR_GREATER // the time-windowed logic needs a controllable clock; TimeProvider is only available on net8.0+, // and we don't want to pull the BCL shim in just for down-level test coverage. @@ -135,8 +109,7 @@ public void CustomTrackedExceptions_AreHonoured() var acc = Build( time, failureRateThreshold: 1, - minimumNumberOfFailures: 1, - trackedExceptions: [typeof(InvalidOperationException)]).CreateAccumulator(); + minimumNumberOfFailures: 1).CreateAccumulator(); // the default Redis faults are now *un*tracked, so they read as success Assert.True(Record(acc, 100, Timeout())); @@ -194,14 +167,12 @@ public void Reset_DiscardsHistory() private static CircuitBreaker Build( TimeProvider time, double failureRateThreshold, - int minimumNumberOfFailures, - Type[]? trackedExceptions = null) + int minimumNumberOfFailures) => new CircuitBreaker.Builder { FailureRateThreshold = failureRateThreshold, MinimumNumberOfFailures = minimumNumberOfFailures, MetricsWindowSize = TimeSpan.FromSeconds(10), - TrackedExceptions = trackedExceptions, TimeProvider = time, }.Create(); @@ -227,13 +198,12 @@ private sealed class ManualTimeProvider : TimeProvider private static bool Record(CircuitBreaker.Accumulator accumulator, int count, Exception? fault = null) { // success/failure is derived from the presence of a fault; pass a fault for a failure, none for a success - var context = new CircuitBreaker.CircuitBreakerContext(fault); - bool healthy = true; + var context = new FaultContext(fault); for (int i = 0; i < count; i++) { - healthy = accumulator.ObserveResult(in context); + accumulator.ObserveResult(in context); } - return healthy; + return accumulator.IsHealthy(); } } diff --git a/tests/StackExchange.Redis.Tests/MultiGroupTests/CircuitBreakerRerouteTests.cs b/tests/StackExchange.Redis.Tests/MultiGroupTests/CircuitBreakerRerouteTests.cs index 39fef72b3..546075fa8 100644 --- a/tests/StackExchange.Redis.Tests/MultiGroupTests/CircuitBreakerRerouteTests.cs +++ b/tests/StackExchange.Redis.Tests/MultiGroupTests/CircuitBreakerRerouteTests.cs @@ -113,7 +113,7 @@ private sealed class FlipBreaker : CircuitBreaker private sealed class Acc(FlipBreaker owner) : Accumulator { - public override bool ObserveResult(in FaultContext context) => !owner._tripped; + public override void ObserveResult(in FaultContext context) { } public override bool IsHealthy() => !owner._tripped; public override void Reset() { } } From be0b43918e5c8e2efc0b30272a59c377ec3990e0 Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Thu, 16 Jul 2026 16:21:39 +0100 Subject: [PATCH 14/23] start command categorization --- .../Availability/RetryDatabase.cs | 83 +++++++++++++------ .../Availability/RetryPolicy.cs | 70 ++++++++++++++-- .../Enums/CommandFlags.Category.cs | 49 +++++++++++ src/StackExchange.Redis/Enums/CommandFlags.cs | 72 ++++++++++++---- src/StackExchange.Redis/Message.cs | 13 +++ .../PublicAPI/PublicAPI.Unshipped.txt | 17 +++- 6 files changed, 255 insertions(+), 49 deletions(-) create mode 100644 src/StackExchange.Redis/Enums/CommandFlags.Category.cs diff --git a/src/StackExchange.Redis/Availability/RetryDatabase.cs b/src/StackExchange.Redis/Availability/RetryDatabase.cs index 9ed028705..ca8f6561f 100644 --- a/src/StackExchange.Redis/Availability/RetryDatabase.cs +++ b/src/StackExchange.Redis/Availability/RetryDatabase.cs @@ -12,10 +12,8 @@ internal partial class RetryDatabase : IDatabaseAsync, IRedisArgsMutator, IInter { // Note: we very deliberately do not include synchronous support for retry; it is inherently delay-ish - // TODO: use message category; retrying a GET is very different to SET, SETNX, INCR, etc - - // Note that only connection faults (as defined by the circuit-breaker, or the default circuit-breaker if - // not supplied) result in retries; we don't retry caller error. + // Note that only transient faults result in retries; this is defined by the RetryPolicy, along with + // understanding the category. The default RetryPolicy works the same as the default CircuitBreaker. DatabaseFeatureFlags IInternalDatabaseAsync.GetFeatures(out string name) => _inner.GetFeatures(out name) | DatabaseFeatureFlags.Retry; @@ -68,23 +66,21 @@ private async Task ExecuteAsync(TState state, Func(TState state, Func= _maxAttempts) + { + // all used up + return false; + } + + // ask the retry policy for advice, and mask off the bits we know about + FaultContext ctx = new(fault); + var policy = _policy.CanRetry(ctx, flags) & + (RetryPolicy.RetryPolicyResult.FailoverServer | RetryPolicy.RetryPolicyResult.SameServer); + if (policy is 0) { - // we're in the failover slot - possibly by count, possibly by the retry-policy - var snapshot = failover; - failover = CancellationToken.None; // only retry at most once - return AwaitFailover(snapshot); + // retry policy says: nope + return false; } - else + + if (policy is RetryPolicy.RetryPolicyResult.FailoverServer) + { + // we can *only* retry on a different server; is failover available? + delay = failover; + failover = CancellationToken.None; // only failover once + return delay.CanBeCanceled; + } + + if (attempt == _maxBeforeFailover) { - // this is just a routine wait between operations; await delay+jitter - return Task.Delay(_delayMillis + ServerSelectionStrategy.SharedRandom.Next(_jitterMillis), CancellationToken.None); + // by count, we should really switch over to the failover now; is failover available *and* are we allowed? + delay = failover; + failover = CancellationToken.None; // only failover once + return delay.CanBeCanceled & (policy & RetryPolicy.RetryPolicyResult.FailoverServer) != 0; } + + // can we pause and retry on the same server? + return (policy & RetryPolicy.RetryPolicyResult.SameServer) != 0; + } + + private Task FailoverOrDelayAsync(CancellationToken delay) + { + if (delay.CanBeCanceled) + { + return AwaitFailover(delay); + } + + // this is just a routine wait between operations; await delay+jitter + return Task.Delay(_delayMillis + ServerSelectionStrategy.SharedRandom.Next(_jitterMillis), CancellationToken.None); } private async Task AwaitFailover(CancellationToken failover) { diff --git a/src/StackExchange.Redis/Availability/RetryPolicy.cs b/src/StackExchange.Redis/Availability/RetryPolicy.cs index f971f9ed4..bfff8efef 100644 --- a/src/StackExchange.Redis/Availability/RetryPolicy.cs +++ b/src/StackExchange.Redis/Availability/RetryPolicy.cs @@ -57,19 +57,77 @@ public TimeSpan JitterMax internal int JitterMilliseconds => _jitterMillis; internal int FailoverMilliseconds => _failoverMillis; - internal bool CanRetry(Exception ex, CommandFlags flags, out bool onNewServer) + /// + /// Gets or sets the max side-effect category that will be retried; defaults to . + /// + public CommandFlags MaxCommandRetryCategory { - FaultContext ctx = new(ex); - onNewServer = false; - return CanRetry(ctx, flags, ref onNewServer); + get => _maxCommandRetryCategory; + set + { + if ((value & Message.MaskRetryCategory) is 0 | (value & ~Message.MaskRetryCategory) is not 0) + throw new InvalidOperationException("Valid CommandRetry* flags should be specified"); + _maxCommandRetryCategory = value; + } } + private CommandFlags _maxCommandRetryCategory = CommandFlags.CommandRetryWriteLastWins; + /// /// Controls which operations can be repeated, optionally indicating that this should progress to /// a new server. /// - public virtual bool CanRetry(in FaultContext fault, CommandFlags flags, ref bool onNewServer) - => CircuitBreaker.DefaultIsFailure(in fault); // use the same default logic that governs CircuitBreaker + public virtual RetryPolicyResult CanRetry(in FaultContext fault, CommandFlags flags) + { + var actual = flags & Message.MaskRetryCategory; + if (actual is 0) actual = CommandFlags.CommandRetryWriteAccumulating; // if not set, assume similar to INCR + + if (actual is CommandFlags.CommandRetryNever) + { + // explicitly disabled at command level + return RetryPolicyResult.None; + } + + if (actual < MaxCommandRetryCategory) // note this also covers CommandRetryAlways + { + // side-effects are beyond what the policy allows + return RetryPolicyResult.None; + } + + if (CircuitBreaker.DefaultIsFailure(in fault)) + { + // assume we can send it everywhere + var result = RetryPolicyResult.SameServer | RetryPolicyResult.FailoverServer; + if ((flags & CommandFlags.CommandServerSpecific) != 0) + result &= ~RetryPolicyResult.FailoverServer; + return result; + } + + // do not retry + return RetryPolicyResult.None; + } + + /// + /// Indicates the result of a query. + /// + [Flags] + public enum RetryPolicyResult + { + /// + /// None; the operation should not be retried. + /// + None = 0, + + /// + /// The operation can be retried on the same server. + /// + SameServer = 1, + + /// + /// The operation can be retried on a different server after a failover operation. + /// + FailoverServer = 2, + } } internal static class RetryPolicyExtensions diff --git a/src/StackExchange.Redis/Enums/CommandFlags.Category.cs b/src/StackExchange.Redis/Enums/CommandFlags.Category.cs new file mode 100644 index 000000000..c602d48b3 --- /dev/null +++ b/src/StackExchange.Redis/Enums/CommandFlags.Category.cs @@ -0,0 +1,49 @@ +namespace StackExchange.Redis; + +internal static class CommandFlagsExtensions +{ + public static CommandFlags WithDefaultCategory(this CommandFlags flags, RedisCommand command) + { + if ((flags & Message.MaskRetryCategory) is 0) + { + // Get the suggested flags; note that the user might have included CommandServerSpecific, + // but we *also* suggest that below - we'll live with it, additively. + // Note also that some commands may have *conditionally* included their category based on + // rules specific to the parameters, for example SCAN 0 is not server specific, + // but SCAN 12341234 *is*. + flags |= DefaultCategory(command); + } + + return flags; + + static CommandFlags DefaultCategory(RedisCommand command) + { + // This is *not* using switch expressions very deliberately, because there are a *lot* of + // options in each; let's keep things vertical rather than horizontal. + switch (command) + { + case RedisCommand.INFO: + // etc + return CommandFlags.CommandRetryConnection; + case RedisCommand.GET: + // etc + return CommandFlags.CommandRetryReadOnly; + case RedisCommand.SETNX: + // etc + return CommandFlags.CommandRetryWriteChecked; + case RedisCommand.SET: + // etc + return CommandFlags.CommandRetryWriteLastWins; + case RedisCommand.INCR: + // etc + return CommandFlags.CommandRetryWriteAccumulating; + case RedisCommand.REPLICAOF: + // etc + return CommandFlags.CommandRetryServerAdmin | CommandFlags.CommandServerSpecific; + default: + // fail safe + return CommandFlags.CommandRetryNever; + } + } + } +} diff --git a/src/StackExchange.Redis/Enums/CommandFlags.cs b/src/StackExchange.Redis/Enums/CommandFlags.cs index c8e380c6b..557e0e67d 100644 --- a/src/StackExchange.Redis/Enums/CommandFlags.cs +++ b/src/StackExchange.Redis/Enums/CommandFlags.cs @@ -114,31 +114,69 @@ public enum CommandFlags /* Command side-effect/retry logic: - Reserve a chunk of bits for command retry logic/categorization; - by default, nothing in this chunk will be passed and the library will - supply a value based on the command being issued. The caller can override, - though - ultimately it is their data/server! They will need to supply a + The values below reserve a chunk of bits (bits 13-17, i.e. << 13) for command + retry logic/categorization; by default, nothing in this chunk will be passed and + the library will supply a value based on the command being issued. The caller can + override, though - ultimately it is their data/server! They will need to supply a suitable value for Execute[Async] etc, otherwise we'll assume the worst. The math here is intended so that we can specify a numeric "max" that we'll retry, and can filter with <=, so the higher numbers have more side-effects. - As such, the main chunk are not flags per-se, and we are intentionally leaving gaps - to slide other things in later. + As such, this region is not flags per-se, and we are intentionally leaving gaps + to slide other things in later. Note that 0 is the implicit "not specified" value + (distinct from CommandRetryAlways), and must be resolved to a concrete category + downstream before any <= comparison. CommandServerSpecific (bit 18) is a genuine + single-bit flag, orthogonal to the severity ladder. + */ - the numbers here are before << into their respective slot - CommandRetryAlways = 1, (distinct from 0, which is implicit "not specified") + /// + /// The command is always safe to retry, regardless of connection or server state. + /// + CommandRetryAlways = 1 << 13, // pre-shift value 1 + + /// + /// The command relates to the connection or to safe metadata (for example CLIENT SETNAME, + /// COMMAND, or CONFIG GET) and can be retried at the connection level. + /// + CommandRetryConnection = 4 << 13, // pre-shift value 4 + + /// + /// The command only reads data (for example GET) and can be safely retried. + /// + CommandRetryReadOnly = 8 << 13, // pre-shift value 8 + + /// + /// The command writes data conditionally (for example SETNX or SET ... IFEQ), so a retry + /// is checked against server state. + /// + CommandRetryWriteChecked = 12 << 13, // pre-shift value 12 - CommandRetryConnection = 4 - things like CLIENT SETNAME or safe metadata (COMMAND, CONFIG GET); note that CLIENT KILL would also have CommandServerSpecific - CommandRetryReadOnly = 8 - things like GET - CommandRetryWriteChecked = 12 - things like SETNX or SET IFEQ - CommandRetryWriteLastWins = 16 - things like SET - CommandRetryWriteAccumulating = 20 - things like INCR, LPOP - CommandRetryServerAdmin = 24 - things like REPLICAOF, CONFIG SET; note these will commonly also include CommandServerSpecific - CommandRetryNever = (all the bits for the above region - I guess this means it is 31?) + /// + /// The command writes data such that a retry simply overwrites (last-writer-wins, for example SET). + /// + CommandRetryWriteLastWins = 16 << 13, // pre-shift value 16 - CommandServerSpecific = (a single bit, means "never retry on a different endpoint" - think cursors) + /// + /// The command writes data cumulatively (for example INCR or LPOP), so a retry can + /// double-apply and change the result. + /// + CommandRetryWriteAccumulating = 20 << 13, // pre-shift value 20 + /// + /// The command performs server administration (for example REPLICAOF or CONFIG SET); these + /// will commonly also carry . + /// + CommandRetryServerAdmin = 24 << 13, // pre-shift value 24 - */ + /// + /// The command should never be retried. + /// + CommandRetryNever = 31 << 13, // pre-shift value 31 (the full retry-category region) + + /// + /// The command is tied to a specific endpoint and must never be retried on a different endpoint + /// (for example cursor-based operations); orthogonal to the retry-category region. + /// + CommandServerSpecific = 1 << 18, } } diff --git a/src/StackExchange.Redis/Message.cs b/src/StackExchange.Redis/Message.cs index b37995b78..5ad0719b0 100644 --- a/src/StackExchange.Redis/Message.cs +++ b/src/StackExchange.Redis/Message.cs @@ -72,6 +72,11 @@ internal const CommandFlags | CommandFlags.PreferMaster | CommandFlags.PreferReplica; + // the 5-bit retry-category severity region (bits 13-17); numerically equal to CommandRetryNever. + // deliberately excludes CommandServerSpecific (bit 18), which is an orthogonal flag, not part of + // the <=-comparable severity ladder. + internal const CommandFlags MaskRetryCategory = CommandFlags.CommandRetryNever; + private const CommandFlags UserSelectableFlags = CommandFlags.None | CommandFlags.DemandMaster | CommandFlags.DemandReplica @@ -83,6 +88,8 @@ internal const CommandFlags | CommandFlags.FireAndForget | CommandFlags.NoRedirect | CommandFlags.NoScriptCache + | MaskRetryCategory // caller may override the retry category... + | CommandFlags.CommandServerSpecific // ...and the server-specific flag | NoFlushFlag; // we'll allow this one even though not advertised private IResultBox? resultBox; @@ -599,6 +606,12 @@ internal static CommandFlags GetPrimaryReplicaFlags(CommandFlags flags) return flags & MaskPrimaryServerPreference; } + internal static CommandFlags GetRetryCategory(CommandFlags flags) + { + // isolate the retry-category region; 0 here means "not specified" (resolved downstream) + return flags & MaskRetryCategory; + } + internal static bool RequiresDatabase(RedisCommand command) { switch (command) diff --git a/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt b/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt index 5c3ecb8f4..615d889a8 100644 --- a/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt @@ -8,6 +8,8 @@ StackExchange.Redis.Availability.RetryPolicy.MaxAttempts.get -> int StackExchange.Redis.Availability.RetryPolicy.MaxAttempts.set -> void StackExchange.Redis.Availability.RetryPolicy.MaxAttemptsBeforeFailover.get -> int StackExchange.Redis.Availability.RetryPolicy.MaxAttemptsBeforeFailover.set -> void +StackExchange.Redis.Availability.RetryPolicy.MaxCommandRetryCategory.get -> StackExchange.Redis.CommandFlags +StackExchange.Redis.Availability.RetryPolicy.MaxCommandRetryCategory.set -> void StackExchange.Redis.Availability.RetryPolicy.RetryDelay.get -> System.TimeSpan StackExchange.Redis.Availability.RetryPolicy.RetryDelay.set -> void StackExchange.Redis.Availability.RetryPolicy.RetryPolicy() -> void @@ -168,4 +170,17 @@ StackExchange.Redis.Availability.FaultContext.IsFault.get -> bool [SER007]StackExchange.Redis.Availability.RedisErrorKind.WrongType = 22 -> StackExchange.Redis.Availability.RedisErrorKind [SER007]abstract StackExchange.Redis.Availability.CircuitBreaker.Accumulator.ObserveResult(in StackExchange.Redis.Availability.FaultContext fault) -> void [SER007]virtual StackExchange.Redis.Availability.CircuitBreaker.Accumulator.IsFailure(in StackExchange.Redis.Availability.FaultContext fault) -> bool -virtual StackExchange.Redis.Availability.RetryPolicy.CanRetry(in StackExchange.Redis.Availability.FaultContext fault, StackExchange.Redis.CommandFlags flags, ref bool onNewServer) -> bool +virtual StackExchange.Redis.Availability.RetryPolicy.CanRetry(in StackExchange.Redis.Availability.FaultContext fault, StackExchange.Redis.CommandFlags flags) -> StackExchange.Redis.Availability.RetryPolicy.RetryPolicyResult +StackExchange.Redis.Availability.RetryPolicy.RetryPolicyResult +StackExchange.Redis.Availability.RetryPolicy.RetryPolicyResult.None = 0 -> StackExchange.Redis.Availability.RetryPolicy.RetryPolicyResult +StackExchange.Redis.Availability.RetryPolicy.RetryPolicyResult.SameServer = 1 -> StackExchange.Redis.Availability.RetryPolicy.RetryPolicyResult +StackExchange.Redis.Availability.RetryPolicy.RetryPolicyResult.FailoverServer = 2 -> StackExchange.Redis.Availability.RetryPolicy.RetryPolicyResult +StackExchange.Redis.CommandFlags.CommandRetryAlways = 8192 -> StackExchange.Redis.CommandFlags +StackExchange.Redis.CommandFlags.CommandRetryConnection = 32768 -> StackExchange.Redis.CommandFlags +StackExchange.Redis.CommandFlags.CommandRetryReadOnly = 65536 -> StackExchange.Redis.CommandFlags +StackExchange.Redis.CommandFlags.CommandRetryWriteChecked = StackExchange.Redis.CommandFlags.CommandRetryConnection | StackExchange.Redis.CommandFlags.CommandRetryReadOnly -> StackExchange.Redis.CommandFlags +StackExchange.Redis.CommandFlags.CommandRetryWriteLastWins = 131072 -> StackExchange.Redis.CommandFlags +StackExchange.Redis.CommandFlags.CommandRetryWriteAccumulating = StackExchange.Redis.CommandFlags.CommandRetryConnection | StackExchange.Redis.CommandFlags.CommandRetryWriteLastWins -> StackExchange.Redis.CommandFlags +StackExchange.Redis.CommandFlags.CommandRetryServerAdmin = StackExchange.Redis.CommandFlags.CommandRetryReadOnly | StackExchange.Redis.CommandFlags.CommandRetryWriteLastWins -> StackExchange.Redis.CommandFlags +StackExchange.Redis.CommandFlags.CommandRetryNever = 253952 -> StackExchange.Redis.CommandFlags +StackExchange.Redis.CommandFlags.CommandServerSpecific = 262144 -> StackExchange.Redis.CommandFlags From 341064fce4b534021df70fb4f1574d977075ba72 Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Fri, 17 Jul 2026 10:21:44 +0100 Subject: [PATCH 15/23] cleanup API --- src/RESPite/Messages/RespReader.cs | 23 ++ .../Availability/CircuitBreaker.cs | 10 +- .../Availability/FaultContext.cs | 26 +- .../Availability/RetryDatabase.cs | 4 +- .../Availability/RetryPolicy.cs | 23 +- .../Enums/CommandFlags.Category.cs | 334 +++++++++++++++++- src/StackExchange.Redis/Enums/CommandFlags.cs | 11 + src/StackExchange.Redis/Exceptions.cs | 19 +- .../KeyspaceIsolation/DatabaseExtension.cs | 3 + src/StackExchange.Redis/Message.cs | 4 +- src/StackExchange.Redis/PhysicalConnection.cs | 4 +- .../PublicAPI/PublicAPI.Unshipped.txt | 132 +++---- .../{Availability => }/RedisErrorKind.cs | 85 ++--- src/StackExchange.Redis/RedisResult.cs | 51 +-- src/StackExchange.Redis/RedisTransaction.cs | 1 - src/StackExchange.Redis/ResultProcessor.cs | 13 +- .../CircuitBreakerUnitTests.cs | 2 +- 17 files changed, 562 insertions(+), 183 deletions(-) rename src/StackExchange.Redis/{Availability => }/RedisErrorKind.cs (73%) diff --git a/src/RESPite/Messages/RespReader.cs b/src/RESPite/Messages/RespReader.cs index 413d0174d..2137946d8 100644 --- a/src/RESPite/Messages/RespReader.cs +++ b/src/RESPite/Messages/RespReader.cs @@ -722,6 +722,29 @@ public readonly unsafe bool TryParseScalar( return TryGetSpan(out var span) ? parser(span, out value) : TryParseSlow(parser, out value); } + // same thing as ^^^, but as a utility method for text values, paying UTF8 encode + internal static unsafe bool TryParseScalar( + ReadOnlySpan source, + delegate* managed, out T, bool> parser, + out T value) + { + const int MAX_STACK = 128; + byte[]? lease = null; + int maxLen = RespConstants.UTF8.GetMaxByteCount(source.Length); + Span buffer = maxLen <= MAX_STACK + ? stackalloc byte[MAX_STACK] // prefer fixed size for perf + : (lease = ArrayPool.Shared.Rent(maxLen)); + try + { + var len = RespConstants.UTF8.GetBytes(source, buffer); + return parser(buffer.Slice(0, len), out value); + } + finally + { + if (lease is not null) ArrayPool.Shared.Return(lease); + } + } + [MethodImpl(MethodImplOptions.NoInlining)] private readonly unsafe bool TryParseSlow( delegate* managed, out T, bool> parser, diff --git a/src/StackExchange.Redis/Availability/CircuitBreaker.cs b/src/StackExchange.Redis/Availability/CircuitBreaker.cs index 67c075bd6..a503849a7 100644 --- a/src/StackExchange.Redis/Availability/CircuitBreaker.cs +++ b/src/StackExchange.Redis/Availability/CircuitBreaker.cs @@ -149,18 +149,22 @@ public abstract class Accumulator() /// public abstract void Reset(); - internal bool Trip(Exception? fault) + internal bool Trip(Exception? fault, CommandFlags flags) { + FaultContext ctx; if (fault is not null) { - var ctx = new FaultContext(fault); + ctx = new FaultContext(fault, flags); if (IsFailure(ctx)) { ObserveResult(ctx); return !IsHealthy(); } + // otherwise, treat as success for the purposes of counting } - ObserveResult(in FaultContext.Success); + + ctx = new(flags); + ObserveResult(ctx); return false; // never trip through success } } diff --git a/src/StackExchange.Redis/Availability/FaultContext.cs b/src/StackExchange.Redis/Availability/FaultContext.cs index f7901f9c1..d1e706b08 100644 --- a/src/StackExchange.Redis/Availability/FaultContext.cs +++ b/src/StackExchange.Redis/Availability/FaultContext.cs @@ -1,28 +1,43 @@ using System; using System.Diagnostics.CodeAnalysis; +using RESPite; namespace StackExchange.Redis.Availability; /// /// Provides information about a circuit-breaker test. /// +[Experimental(Experiments.ActiveActive, UrlFormat = Experiments.UrlFormat)] public readonly struct FaultContext { private readonly Exception? _fault; private readonly ConnectionFailureType _connectionFailureType; - - internal static readonly FaultContext Success = default; + private readonly CommandFlags _flags; /// /// Create a new . /// /// The fault associated with the operation, or null on success. - public FaultContext(Exception? fault) + /// The command-flags associated with the operation. + public FaultContext(Exception fault, CommandFlags flags) { _fault = fault; + _flags = flags & Message.UserSelectableFlags; // just the user-visible ones ErrorKind = fault.GetErrorKind(out _connectionFailureType); } + /// + /// Create a new . + /// + /// The command-flags associated with the operation. + public FaultContext(CommandFlags flags) + { + _fault = null; + _flags = flags & Message.UserSelectableFlags; // just the user-visible ones + ErrorKind = RedisErrorKind.None; + _connectionFailureType = ConnectionFailureType.None; + } + /// /// Indicates whether a fault is present. /// @@ -34,6 +49,11 @@ public FaultContext(Exception? fault) /// public Exception? Fault => _fault; + /// + /// Get any command-flags associated with the operation. + /// + public CommandFlags Flags => _flags; + /// /// The classified server error condition associated with the fault, if any. /// diff --git a/src/StackExchange.Redis/Availability/RetryDatabase.cs b/src/StackExchange.Redis/Availability/RetryDatabase.cs index ca8f6561f..1068c8e61 100644 --- a/src/StackExchange.Redis/Availability/RetryDatabase.cs +++ b/src/StackExchange.Redis/Availability/RetryDatabase.cs @@ -126,8 +126,8 @@ private bool CanRetry( } // ask the retry policy for advice, and mask off the bits we know about - FaultContext ctx = new(fault); - var policy = _policy.CanRetry(ctx, flags) & + FaultContext ctx = new(fault, flags); + var policy = _policy.CanRetry(ctx) & (RetryPolicy.RetryPolicyResult.FailoverServer | RetryPolicy.RetryPolicyResult.SameServer); if (policy is 0) { diff --git a/src/StackExchange.Redis/Availability/RetryPolicy.cs b/src/StackExchange.Redis/Availability/RetryPolicy.cs index bfff8efef..243967f75 100644 --- a/src/StackExchange.Redis/Availability/RetryPolicy.cs +++ b/src/StackExchange.Redis/Availability/RetryPolicy.cs @@ -1,5 +1,7 @@ using System; using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using RESPite; namespace StackExchange.Redis.Availability; @@ -7,6 +9,7 @@ namespace StackExchange.Redis.Availability; /// Configures how messages can be retried due to connection / transient faults. Other faults (such as invalid /// usage) are not retried. /// +[Experimental(Experiments.ActiveActive, UrlFormat = Experiments.UrlFormat)] public class RetryPolicy { /// @@ -77,9 +80,9 @@ public CommandFlags MaxCommandRetryCategory /// Controls which operations can be repeated, optionally indicating that this should progress to /// a new server. /// - public virtual RetryPolicyResult CanRetry(in FaultContext fault, CommandFlags flags) + public virtual RetryPolicyResult CanRetry(in FaultContext fault) { - var actual = flags & Message.MaskRetryCategory; + var actual = fault.Flags & Message.MaskRetryCategory; if (actual is 0) actual = CommandFlags.CommandRetryWriteAccumulating; // if not set, assume similar to INCR if (actual is CommandFlags.CommandRetryNever) @@ -98,7 +101,7 @@ public virtual RetryPolicyResult CanRetry(in FaultContext fault, CommandFlags fl { // assume we can send it everywhere var result = RetryPolicyResult.SameServer | RetryPolicyResult.FailoverServer; - if ((flags & CommandFlags.CommandServerSpecific) != 0) + if ((fault.Flags & CommandFlags.CommandServerSpecific) != 0) result &= ~RetryPolicyResult.FailoverServer; return result; } @@ -151,19 +154,15 @@ internal static Exception Add(Exception target, string key, object boxed) // we optimize for our expected exception kinds, but fallback to using .Data internal static Exception With(this Exception target, RedisErrorKind kind) => Add(target, ServerKey, kind); + [Obsolete("Prefer .ctor", true)] internal static Exception With(this RedisServerException target, RedisErrorKind kind) - { - target.Kind = kind; - return target; - } + => throw new NotSupportedException(); internal static Exception With(this Exception target, ConnectionFailureType kind) => Add(target, ConnectionKey, kind); + [Obsolete("Prefer .ctor", true)] internal static Exception With(this RedisConnectionException target, ConnectionFailureType kind) - { - target.Kind = kind; - return target; - } + => throw new NotSupportedException(); internal static RedisErrorKind GetErrorKind(this Exception? target, out ConnectionFailureType connectionFailure) { @@ -177,7 +176,7 @@ internal static RedisErrorKind GetErrorKind(this Exception? target, out Connecti kind = server.Kind; break; case RedisConnectionException connection: - connectionFailure = connection.Kind; + connectionFailure = connection.FailureType; kind = RedisErrorKind.ConnectionFault; break; case TimeoutException: // includes RedisTimeoutException diff --git a/src/StackExchange.Redis/Enums/CommandFlags.Category.cs b/src/StackExchange.Redis/Enums/CommandFlags.Category.cs index c602d48b3..e07fd94cb 100644 --- a/src/StackExchange.Redis/Enums/CommandFlags.Category.cs +++ b/src/StackExchange.Redis/Enums/CommandFlags.Category.cs @@ -22,26 +22,346 @@ static CommandFlags DefaultCategory(RedisCommand command) // options in each; let's keep things vertical rather than horizontal. switch (command) { + // ========================================================================== + // CONNECTION / SESSION — node-agnostic, no keyspace side effects, safe to + // replay on a fresh connection. + // ========================================================================== + case RedisCommand.PING: + case RedisCommand.ECHO: + case RedisCommand.AUTH: + case RedisCommand.HELLO: + case RedisCommand.SELECT: + case RedisCommand.QUIT: + case RedisCommand.SUBSCRIBE: + case RedisCommand.UNSUBSCRIBE: + case RedisCommand.PSUBSCRIBE: + case RedisCommand.PUNSUBSCRIBE: + case RedisCommand.SSUBSCRIBE: + case RedisCommand.SUNSUBSCRIBE: case RedisCommand.INFO: - // etc + case RedisCommand.TIME: + case RedisCommand.DBSIZE: + case RedisCommand.LASTSAVE: + case RedisCommand.COMMAND: return CommandFlags.CommandRetryConnection; + + // CLIENT etc often use server-specific IDs + case RedisCommand.CLIENT: // note some can be considered admin, overridden locally + return CommandFlags.CommandRetryConnection | CommandFlags.CommandServerSpecific; + + // ========================================================================== + // READ-ONLY — no mutation, always safe to retry. + // ========================================================================== case RedisCommand.GET: - // etc + case RedisCommand.MGET: + case RedisCommand.STRLEN: + case RedisCommand.GETRANGE: + case RedisCommand.EXISTS: + case RedisCommand.TYPE: + case RedisCommand.TTL: + case RedisCommand.PTTL: + case RedisCommand.EXPIRETIME: + case RedisCommand.PEXPIRETIME: + case RedisCommand.KEYS: + case RedisCommand.RANDOMKEY: + case RedisCommand.DUMP: + case RedisCommand.TOUCH: // technically bumps LRU/LFU state, but that's not a "real" side effect worth blocking retries over + case RedisCommand.OBJECT: + case RedisCommand.MEMORY: + case RedisCommand.SORT_RO: + case RedisCommand.SORT: // ignoring the STORE variant + case RedisCommand.LCS: + case RedisCommand.GETEX: // ignoring the TTL-mutating option variants + case RedisCommand.HGET: + case RedisCommand.HMGET: + case RedisCommand.HGETALL: + case RedisCommand.HKEYS: + case RedisCommand.HVALS: + case RedisCommand.HLEN: + case RedisCommand.HEXISTS: + case RedisCommand.HSTRLEN: + case RedisCommand.HRANDFIELD: + case RedisCommand.HPTTL: + case RedisCommand.HEXPIRETIME: + case RedisCommand.HPEXPIRETIME: + case RedisCommand.HGETEX: // ignoring TTL-mutating option variants + case RedisCommand.LLEN: + case RedisCommand.LRANGE: + case RedisCommand.LINDEX: + case RedisCommand.LPOS: + case RedisCommand.SMEMBERS: + case RedisCommand.SCARD: + case RedisCommand.SISMEMBER: + case RedisCommand.SMISMEMBER: + case RedisCommand.SRANDMEMBER: + case RedisCommand.SDIFF: + case RedisCommand.SINTER: + case RedisCommand.SINTERCARD: + case RedisCommand.SUNION: + case RedisCommand.ZCARD: + case RedisCommand.ZSCORE: + case RedisCommand.ZMSCORE: + case RedisCommand.ZRANK: + case RedisCommand.ZREVRANK: + case RedisCommand.ZCOUNT: + case RedisCommand.ZLEXCOUNT: + case RedisCommand.ZRANGE: + case RedisCommand.ZREVRANGE: + case RedisCommand.ZRANGEBYSCORE: + case RedisCommand.ZREVRANGEBYSCORE: + case RedisCommand.ZRANGEBYLEX: + case RedisCommand.ZREVRANGEBYLEX: + case RedisCommand.ZRANDMEMBER: + case RedisCommand.ZDIFF: + case RedisCommand.ZINTER: + case RedisCommand.ZUNION: + case RedisCommand.ZINTERCARD: + case RedisCommand.BITCOUNT: + case RedisCommand.BITPOS: + case RedisCommand.GETBIT: + case RedisCommand.PFCOUNT: + case RedisCommand.GEOPOS: + case RedisCommand.GEODIST: + case RedisCommand.GEOHASH: + case RedisCommand.GEOSEARCH: + case RedisCommand.XLEN: + case RedisCommand.XRANGE: + case RedisCommand.XREVRANGE: + case RedisCommand.XREAD: // group-less read only; XREADGROUP is handled separately below + case RedisCommand.XPENDING: + case RedisCommand.XINFO: + case RedisCommand.EVAL_RO: + case RedisCommand.EVALSHA_RO: return CommandFlags.CommandRetryReadOnly; + + // PUBSUB/SCAN/etc are *basically* read-only, but make limited sense between nodes + case RedisCommand.PUBSUB: + case RedisCommand.SCAN: + case RedisCommand.ZSCAN: + case RedisCommand.SSCAN: + case RedisCommand.HSCAN: + return CommandFlags.CommandRetryReadOnly | CommandFlags.CommandServerSpecific; + + // ========================================================================== + // WRITE - CHECKED — inherently conditional/idempotent; a retry either + // no-ops or fails in a way that leaves the end-state identical. + // ========================================================================== case RedisCommand.SETNX: - // etc + case RedisCommand.MSETNX: + case RedisCommand.HSETNX: + case RedisCommand.DEL: + case RedisCommand.UNLINK: + case RedisCommand.PERSIST: + case RedisCommand.RENAMENX: + case RedisCommand.COPY: // ignoring REPLACE — default behavior fails if dest exists + case RedisCommand.MOVE: + case RedisCommand.RESTORE: // ignoring REPLACE + case RedisCommand.GETDEL: + case RedisCommand.HGETDEL: + case RedisCommand.HPERSIST: + case RedisCommand.LTRIM: + case RedisCommand.SADD: + case RedisCommand.SREM: + case RedisCommand.SMOVE: + case RedisCommand.ZREM: + case RedisCommand.ZREMRANGEBYRANK: + case RedisCommand.ZREMRANGEBYSCORE: + case RedisCommand.ZREMRANGEBYLEX: + case RedisCommand.HDEL: + case RedisCommand.PFADD: + case RedisCommand.XDEL: + case RedisCommand.XTRIM: + case RedisCommand.XACK: + case RedisCommand.XGROUP: + case RedisCommand.XNACK: return CommandFlags.CommandRetryWriteChecked; + + // ========================================================================== + // WRITE - LAST WINS — unconditional overwrite of a specific value/state; + // repeating with the same args always converges to the same final value. + // ========================================================================== case RedisCommand.SET: - // etc + case RedisCommand.GETSET: + case RedisCommand.MSET: + case RedisCommand.SETEX: + case RedisCommand.PSETEX: + case RedisCommand.SETRANGE: + case RedisCommand.SETBIT: + case RedisCommand.BITOP: + case RedisCommand.RENAME: + case RedisCommand.EXPIRE: + case RedisCommand.PEXPIRE: + case RedisCommand.EXPIREAT: + case RedisCommand.PEXPIREAT: + case RedisCommand.HSET: + case RedisCommand.HMSET: + case RedisCommand.HEXPIRE: + case RedisCommand.HPEXPIRE: + case RedisCommand.HEXPIREAT: + case RedisCommand.HPEXPIREAT: + case RedisCommand.LSET: + case RedisCommand.ZADD: // ignoring INCR option + case RedisCommand.ZRANGESTORE: + case RedisCommand.ZUNIONSTORE: + case RedisCommand.ZINTERSTORE: + case RedisCommand.ZDIFFSTORE: + case RedisCommand.SDIFFSTORE: + case RedisCommand.SINTERSTORE: + case RedisCommand.SUNIONSTORE: + case RedisCommand.GEOADD: + case RedisCommand.GEOSEARCHSTORE: + case RedisCommand.GEORADIUS: // because of store scenarios + case RedisCommand.GEORADIUSBYMEMBER: + case RedisCommand.XCLAIM: + case RedisCommand.XAUTOCLAIM: return CommandFlags.CommandRetryWriteLastWins; + + // ========================================================================== + // WRITE - ACCUMULATING — effect compounds with every additional call. + // ========================================================================== case RedisCommand.INCR: - // etc + case RedisCommand.DECR: + case RedisCommand.INCRBY: + case RedisCommand.DECRBY: + case RedisCommand.INCRBYFLOAT: + case RedisCommand.APPEND: + case RedisCommand.HINCRBY: + case RedisCommand.HINCRBYFLOAT: + case RedisCommand.ZINCRBY: + case RedisCommand.LPUSH: + case RedisCommand.RPUSH: + case RedisCommand.LPUSHX: + case RedisCommand.RPUSHX: + case RedisCommand.LINSERT: + case RedisCommand.LREM: + case RedisCommand.LPOP: + case RedisCommand.RPOP: + case RedisCommand.RPOPLPUSH: + case RedisCommand.LMOVE: + case RedisCommand.LMPOP: + case RedisCommand.SPOP: + case RedisCommand.ZPOPMIN: + case RedisCommand.ZPOPMAX: + case RedisCommand.ZMPOP: + case RedisCommand.XADD: + case RedisCommand.PFMERGE: // destination accumulates union each call return CommandFlags.CommandRetryWriteAccumulating; + + // ========================================================================== + // SERVER ADMIN / NODE-SPECIFIC + // ========================================================================== case RedisCommand.REPLICAOF: - // etc + case RedisCommand.SLAVEOF: + case RedisCommand.BGSAVE: + case RedisCommand.BGREWRITEAOF: + case RedisCommand.SAVE: + case RedisCommand.SHUTDOWN: + case RedisCommand.FLUSHALL: + case RedisCommand.FLUSHDB: + case RedisCommand.SWAPDB: + case RedisCommand.MIGRATE: + case RedisCommand.DEBUG: + case RedisCommand.MONITOR: + case RedisCommand.CONFIG: // note CONFIG GET con be considered more safe + case RedisCommand.SLOWLOG: + case RedisCommand.LATENCY: + case RedisCommand.SCRIPT: + case RedisCommand.CLUSTER: // note: some like MYID can be considered more safe + return CommandFlags.CommandRetryServerAdmin | CommandFlags.CommandServerSpecific; + + // ========================================================================== + // NEVER — transactions, arbitrary scripts, and blocking/destructive or + // fire-and-forget commands where a lost ack makes blind retry dangerous. + // ========================================================================== + case RedisCommand.MULTI: + case RedisCommand.EXEC: + case RedisCommand.DISCARD: + case RedisCommand.WATCH: + case RedisCommand.UNWATCH: + case RedisCommand.PUBLISH: + case RedisCommand.SPUBLISH: + case RedisCommand.XREADGROUP: + case RedisCommand.BLPOP: + case RedisCommand.BRPOP: + case RedisCommand.BRPOPLPUSH: + return CommandFlags.CommandRetryNever; + + // ========================================================================== + // scripts / modules / functions; we're going to assume nothing too weird, + // at worst similar to INCR; but it is *hoped* that callers will supply hints. + // ========================================================================== + case RedisCommand.EVAL: + case RedisCommand.EVALSHA: + return CommandFlags.CommandRetryWriteAccumulating; + + // ---- CONNECTION / SESSION ---- + case RedisCommand.ASKING: // cluster ASK redirection flag - per-connection state + case RedisCommand.READONLY: // cluster client read-routing flag - per-connection state + case RedisCommand.READWRITE: // cluster client read-routing flag - per-connection state + case RedisCommand.ROLE: // replication role report, like INFO - no keyspace effect + return CommandFlags.CommandRetryConnection; + + // ---- READ-ONLY ---- + case RedisCommand.ARCOUNT: + case RedisCommand.ARINFO: // structure metadata, like INFO + case RedisCommand.ARGET: + case RedisCommand.ARGETRANGE: + case RedisCommand.ARGREP: + case RedisCommand.ARLASTITEMS: + case RedisCommand.ARLEN: + case RedisCommand.ARMGET: + case RedisCommand.ARSCAN: + case RedisCommand.AROP: + case RedisCommand.DIGEST: // computes a hash of existing data, doesn't mutate it + case RedisCommand.VCARD: + case RedisCommand.VDIM: + case RedisCommand.VEMB: + case RedisCommand.VGETATTR: + case RedisCommand.VINFO: + case RedisCommand.VISMEMBER: + case RedisCommand.VLINKS: + case RedisCommand.VRANDMEMBER: + case RedisCommand.VRANGE: + case RedisCommand.VSIM: + return CommandFlags.CommandRetryReadOnly; + + // ---- WRITE - CHECKED ---- + case RedisCommand.DELEX: // conditional/expiry-aware delete - converges same as DEL + case RedisCommand.VADD: // idempotent add-member, like SADD + case RedisCommand.VREM: // idempotent remove-member, like SREM/ZREM + case RedisCommand.XACKDEL: // ack+delete, converges like XACK/XDEL combined + case RedisCommand.XDELEX: + return CommandFlags.CommandRetryWriteChecked; + + // ---- WRITE - LAST WINS ---- + case RedisCommand.ARDEL: + case RedisCommand.ARDELRANGE: + case RedisCommand.ARMSET: + case RedisCommand.ARSEEK: // repositions a cursor to an explicit point - overwrite semantics + case RedisCommand.ARSET: + case RedisCommand.HSETEX: // HSET + expiry, unconditional overwrite + case RedisCommand.MSETEX: + case RedisCommand.VSETATTR: // unconditional attribute overwrite + case RedisCommand.XCFGSET: + return CommandFlags.CommandRetryWriteLastWins; + + // ---- WRITE - ACCUMULATING ---- + case RedisCommand.ARRING: // presumed create/configure-ring, unconditional define + case RedisCommand.ARINSERT: // ring-buffer insert, compounds like a push + case RedisCommand.ARNEXT: // advances a cursor/position with each call + case RedisCommand.INCREX: // counter semantics + expiry, still accumulating + return CommandFlags.CommandRetryWriteAccumulating; + + // ---- SERVER ADMIN / NODE-SPECIFIC ---- + case RedisCommand.HOTKEYS: // diagnostic/introspection, node-local + case RedisCommand.SENTINEL: + case RedisCommand.SYNC: // replication stream handshake return CommandFlags.CommandRetryServerAdmin | CommandFlags.CommandServerSpecific; + + // if we don't recognize it: default to the most pessimistic + case RedisCommand.NONE: + case RedisCommand.UNKNOWN: default: - // fail safe return CommandFlags.CommandRetryNever; } } diff --git a/src/StackExchange.Redis/Enums/CommandFlags.cs b/src/StackExchange.Redis/Enums/CommandFlags.cs index 557e0e67d..dd460eafc 100644 --- a/src/StackExchange.Redis/Enums/CommandFlags.cs +++ b/src/StackExchange.Redis/Enums/CommandFlags.cs @@ -1,5 +1,7 @@ using System; using System.ComponentModel; +using System.Diagnostics.CodeAnalysis; +using RESPite; namespace StackExchange.Redis { @@ -132,51 +134,60 @@ to slide other things in later. Note that 0 is the implicit "not specified" valu /// /// The command is always safe to retry, regardless of connection or server state. /// + [Experimental(Experiments.ActiveActive, UrlFormat = Experiments.UrlFormat)] CommandRetryAlways = 1 << 13, // pre-shift value 1 /// /// The command relates to the connection or to safe metadata (for example CLIENT SETNAME, /// COMMAND, or CONFIG GET) and can be retried at the connection level. /// + [Experimental(Experiments.ActiveActive, UrlFormat = Experiments.UrlFormat)] CommandRetryConnection = 4 << 13, // pre-shift value 4 /// /// The command only reads data (for example GET) and can be safely retried. /// + [Experimental(Experiments.ActiveActive, UrlFormat = Experiments.UrlFormat)] CommandRetryReadOnly = 8 << 13, // pre-shift value 8 /// /// The command writes data conditionally (for example SETNX or SET ... IFEQ), so a retry /// is checked against server state. /// + [Experimental(Experiments.ActiveActive, UrlFormat = Experiments.UrlFormat)] CommandRetryWriteChecked = 12 << 13, // pre-shift value 12 /// /// The command writes data such that a retry simply overwrites (last-writer-wins, for example SET). /// + [Experimental(Experiments.ActiveActive, UrlFormat = Experiments.UrlFormat)] CommandRetryWriteLastWins = 16 << 13, // pre-shift value 16 /// /// The command writes data cumulatively (for example INCR or LPOP), so a retry can /// double-apply and change the result. /// + [Experimental(Experiments.ActiveActive, UrlFormat = Experiments.UrlFormat)] CommandRetryWriteAccumulating = 20 << 13, // pre-shift value 20 /// /// The command performs server administration (for example REPLICAOF or CONFIG SET); these /// will commonly also carry . /// + [Experimental(Experiments.ActiveActive, UrlFormat = Experiments.UrlFormat)] CommandRetryServerAdmin = 24 << 13, // pre-shift value 24 /// /// The command should never be retried. /// + [Experimental(Experiments.ActiveActive, UrlFormat = Experiments.UrlFormat)] CommandRetryNever = 31 << 13, // pre-shift value 31 (the full retry-category region) /// /// The command is tied to a specific endpoint and must never be retried on a different endpoint /// (for example cursor-based operations); orthogonal to the retry-category region. /// + [Experimental(Experiments.ActiveActive, UrlFormat = Experiments.UrlFormat)] CommandServerSpecific = 1 << 18, } } diff --git a/src/StackExchange.Redis/Exceptions.cs b/src/StackExchange.Redis/Exceptions.cs index 806804219..770c957df 100644 --- a/src/StackExchange.Redis/Exceptions.cs +++ b/src/StackExchange.Redis/Exceptions.cs @@ -1,7 +1,6 @@ using System; using System.ComponentModel; using System.Runtime.Serialization; -using StackExchange.Redis.Availability; namespace StackExchange.Redis { @@ -121,8 +120,6 @@ public RedisConnectionException(ConnectionFailureType failureType, string messag /// public CommandStatus CommandStatus { get; } - internal ConnectionFailureType Kind { get; set; } - #if NET8_0_OR_GREATER [Obsolete(Obsoletions.LegacyFormatterImplMessage, DiagnosticId = Obsoletions.LegacyFormatterImplDiagId)] [EditorBrowsable(EditorBrowsableState.Never)] @@ -191,7 +188,16 @@ public sealed partial class RedisServerException : RedisException /// Creates a new . /// /// The message for the exception. - public RedisServerException(string message) : base(message) { } + [Obsolete("Specify Kind when possible")] + public RedisServerException(string message) : this(RedisErrorKind.Unknown, message) { } + + /// + /// Creates a new . + /// + /// The categorized meaning of the error. + /// The message for the exception. + public RedisServerException(RedisErrorKind kind, string message) : base(message) + => Kind = kind; #if NET8_0_OR_GREATER [Obsolete(Obsoletions.LegacyFormatterImplMessage, DiagnosticId = Obsoletions.LegacyFormatterImplDiagId)] @@ -199,6 +205,9 @@ public RedisServerException(string message) : base(message) { } #endif private RedisServerException(SerializationInfo info, StreamingContext ctx) : base(info, ctx) { } - internal RedisErrorKind Kind { get; set; } + /// + /// Identifies the kind of error received. + /// + public RedisErrorKind Kind { get; } } } diff --git a/src/StackExchange.Redis/KeyspaceIsolation/DatabaseExtension.cs b/src/StackExchange.Redis/KeyspaceIsolation/DatabaseExtension.cs index ae362d834..a65be46d0 100644 --- a/src/StackExchange.Redis/KeyspaceIsolation/DatabaseExtension.cs +++ b/src/StackExchange.Redis/KeyspaceIsolation/DatabaseExtension.cs @@ -1,4 +1,6 @@ using System; +using System.Diagnostics.CodeAnalysis; +using RESPite; using StackExchange.Redis.Availability; using StackExchange.Redis.Interfaces; @@ -71,6 +73,7 @@ public static IDatabase WithKeyPrefix(this IDatabase database, RedisKey keyPrefi /// SE.Redis concepts, so can respond to server failover events, apply circuit-breaker rules, and /// respect command effect categorization. /// + [Experimental(Experiments.ActiveActive, UrlFormat = Experiments.UrlFormat)] public static IDatabaseAsync WithRetry(this IDatabaseAsync database, RetryPolicy retryPolicy) => new RetryDatabase(database, retryPolicy); } diff --git a/src/StackExchange.Redis/Message.cs b/src/StackExchange.Redis/Message.cs index 5ad0719b0..bf1726ef1 100644 --- a/src/StackExchange.Redis/Message.cs +++ b/src/StackExchange.Redis/Message.cs @@ -77,7 +77,7 @@ internal const CommandFlags // the <=-comparable severity ladder. internal const CommandFlags MaskRetryCategory = CommandFlags.CommandRetryNever; - private const CommandFlags UserSelectableFlags = CommandFlags.None + internal const CommandFlags UserSelectableFlags = CommandFlags.None | CommandFlags.DemandMaster | CommandFlags.DemandReplica | CommandFlags.PreferMaster @@ -503,7 +503,7 @@ public void Complete(PhysicalConnection? connection) performance?.SetCompleted(); if (currBox is not null) { - connection?.ObserveMessageResult(currBox.Fault); + connection?.ObserveMessageResult(currBox.Fault, Flags); } currBox?.ActivateContinuations(); } diff --git a/src/StackExchange.Redis/PhysicalConnection.cs b/src/StackExchange.Redis/PhysicalConnection.cs index e85e00454..fbd400a20 100644 --- a/src/StackExchange.Redis/PhysicalConnection.cs +++ b/src/StackExchange.Redis/PhysicalConnection.cs @@ -1177,13 +1177,13 @@ internal bool HasPendingCallerFacingItems() } } - public void ObserveMessageResult(Exception? fault) + public void ObserveMessageResult(Exception? fault, CommandFlags flags) { // Hot path - runs per completed message. Let the breaker observe the outcome (ObserveResult // returns true while healthy); if it trips and we're the *first* to notice, hand off the // teardown rather than doing it inline: RecordConnectionFailed can fail the whole backlog and // build a detailed exception, which we don't want to pay for on the completion thread. - if (circuitBreaker is { } cb && cb.Trip(fault) + if (circuitBreaker is { } cb && cb.Trip(fault, flags) && Interlocked.CompareExchange(ref _circuitBreakerState, CircuitBreakerTripped, CircuitBreakerHealthy) is CircuitBreakerHealthy) { // hand off to a worker; the heartbeat (see OnBridgeHeartbeat) is a backstop in case the diff --git a/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt b/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt index 615d889a8..9297cd113 100644 --- a/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt @@ -1,20 +1,20 @@ #nullable enable -StackExchange.Redis.Availability.RetryPolicy -StackExchange.Redis.Availability.RetryPolicy.FailoverDelay.get -> System.TimeSpan -StackExchange.Redis.Availability.RetryPolicy.FailoverDelay.set -> void -StackExchange.Redis.Availability.RetryPolicy.JitterMax.get -> System.TimeSpan -StackExchange.Redis.Availability.RetryPolicy.JitterMax.set -> void -StackExchange.Redis.Availability.RetryPolicy.MaxAttempts.get -> int -StackExchange.Redis.Availability.RetryPolicy.MaxAttempts.set -> void -StackExchange.Redis.Availability.RetryPolicy.MaxAttemptsBeforeFailover.get -> int -StackExchange.Redis.Availability.RetryPolicy.MaxAttemptsBeforeFailover.set -> void -StackExchange.Redis.Availability.RetryPolicy.MaxCommandRetryCategory.get -> StackExchange.Redis.CommandFlags -StackExchange.Redis.Availability.RetryPolicy.MaxCommandRetryCategory.set -> void -StackExchange.Redis.Availability.RetryPolicy.RetryDelay.get -> System.TimeSpan -StackExchange.Redis.Availability.RetryPolicy.RetryDelay.set -> void -StackExchange.Redis.Availability.RetryPolicy.RetryPolicy() -> void +[SER007]StackExchange.Redis.Availability.RetryPolicy +[SER007]StackExchange.Redis.Availability.RetryPolicy.FailoverDelay.get -> System.TimeSpan +[SER007]StackExchange.Redis.Availability.RetryPolicy.FailoverDelay.set -> void +[SER007]StackExchange.Redis.Availability.RetryPolicy.JitterMax.get -> System.TimeSpan +[SER007]StackExchange.Redis.Availability.RetryPolicy.JitterMax.set -> void +[SER007]StackExchange.Redis.Availability.RetryPolicy.MaxAttempts.get -> int +[SER007]StackExchange.Redis.Availability.RetryPolicy.MaxAttempts.set -> void +[SER007]StackExchange.Redis.Availability.RetryPolicy.MaxAttemptsBeforeFailover.get -> int +[SER007]StackExchange.Redis.Availability.RetryPolicy.MaxAttemptsBeforeFailover.set -> void +[SER007]StackExchange.Redis.Availability.RetryPolicy.MaxCommandRetryCategory.get -> StackExchange.Redis.CommandFlags +[SER007]StackExchange.Redis.Availability.RetryPolicy.MaxCommandRetryCategory.set -> void +[SER007]StackExchange.Redis.Availability.RetryPolicy.RetryDelay.get -> System.TimeSpan +[SER007]StackExchange.Redis.Availability.RetryPolicy.RetryDelay.set -> void +[SER007]StackExchange.Redis.Availability.RetryPolicy.RetryPolicy() -> void StackExchange.Redis.IServer.InventKey(StackExchange.Redis.RedisKey prefix = default(StackExchange.Redis.RedisKey)) -> StackExchange.Redis.RedisKey -static StackExchange.Redis.KeyspaceIsolation.DatabaseExtensions.WithRetry(this StackExchange.Redis.IDatabaseAsync! database, StackExchange.Redis.Availability.RetryPolicy! retryPolicy) -> StackExchange.Redis.IDatabaseAsync! +[SER007]static StackExchange.Redis.KeyspaceIsolation.DatabaseExtensions.WithRetry(this StackExchange.Redis.IDatabaseAsync! database, StackExchange.Redis.Availability.RetryPolicy! retryPolicy) -> StackExchange.Redis.IDatabaseAsync! [SER007]StackExchange.Redis.Availability.CircuitBreaker [SER007]StackExchange.Redis.Availability.CircuitBreaker.Accumulator [SER007]StackExchange.Redis.Availability.CircuitBreaker.Accumulator.Accumulator() -> void @@ -132,55 +132,57 @@ static StackExchange.Redis.KeyspaceIsolation.DatabaseExtensions.WithRetry(this S [SER007]static StackExchange.Redis.Availability.HealthCheck.Default.get -> StackExchange.Redis.Availability.HealthCheck! [SER007]static StackExchange.Redis.Availability.MultiGroupOptions.Default.get -> StackExchange.Redis.Availability.MultiGroupOptions! [SER007]override StackExchange.Redis.Availability.ConnectionGroupMember.ToString() -> string! -[SER007]StackExchange.Redis.Availability.RedisErrorKind -[SER007]StackExchange.Redis.Availability.RedisErrorKind.None = 0 -> StackExchange.Redis.Availability.RedisErrorKind -[SER007]StackExchange.Redis.Availability.RedisErrorKind.Unknown = 1 -> StackExchange.Redis.Availability.RedisErrorKind -StackExchange.Redis.Availability.FaultContext -StackExchange.Redis.Availability.FaultContext.ConnectionFailureType.get -> StackExchange.Redis.ConnectionFailureType -StackExchange.Redis.Availability.FaultContext.ErrorKind.get -> StackExchange.Redis.Availability.RedisErrorKind -StackExchange.Redis.Availability.FaultContext.Fault.get -> System.Exception? -StackExchange.Redis.Availability.FaultContext.FaultContext() -> void -StackExchange.Redis.Availability.FaultContext.FaultContext(System.Exception? fault) -> void -StackExchange.Redis.Availability.FaultContext.IsFault.get -> bool -[SER007]StackExchange.Redis.Availability.RedisErrorKind.Ask = 15 -> StackExchange.Redis.Availability.RedisErrorKind -[SER007]StackExchange.Redis.Availability.RedisErrorKind.Busy = 12 -> StackExchange.Redis.Availability.RedisErrorKind -[SER007]StackExchange.Redis.Availability.RedisErrorKind.ClusterDown = 6 -> StackExchange.Redis.Availability.RedisErrorKind -[SER007]StackExchange.Redis.Availability.RedisErrorKind.ConnectionFault = 3 -> StackExchange.Redis.Availability.RedisErrorKind -[SER007]StackExchange.Redis.Availability.RedisErrorKind.CrossSlot = 16 -> StackExchange.Redis.Availability.RedisErrorKind -[SER007]StackExchange.Redis.Availability.RedisErrorKind.DatabaseSelectDisabled = 27 -> StackExchange.Redis.Availability.RedisErrorKind -[SER007]StackExchange.Redis.Availability.RedisErrorKind.ExecAbort = 23 -> StackExchange.Redis.Availability.RedisErrorKind -[SER007]StackExchange.Redis.Availability.RedisErrorKind.InvalidDatabaseIndex = 26 -> StackExchange.Redis.Availability.RedisErrorKind -[SER007]StackExchange.Redis.Availability.RedisErrorKind.Loading = 5 -> StackExchange.Redis.Availability.RedisErrorKind -[SER007]StackExchange.Redis.Availability.RedisErrorKind.MasterDown = 7 -> StackExchange.Redis.Availability.RedisErrorKind -[SER007]StackExchange.Redis.Availability.RedisErrorKind.MaxClients = 13 -> StackExchange.Redis.Availability.RedisErrorKind -[SER007]StackExchange.Redis.Availability.RedisErrorKind.Misconfigured = 10 -> StackExchange.Redis.Availability.RedisErrorKind -[SER007]StackExchange.Redis.Availability.RedisErrorKind.Moved = 14 -> StackExchange.Redis.Availability.RedisErrorKind -[SER007]StackExchange.Redis.Availability.RedisErrorKind.NoAuth = 17 -> StackExchange.Redis.Availability.RedisErrorKind -[SER007]StackExchange.Redis.Availability.RedisErrorKind.NoPermission = 19 -> StackExchange.Redis.Availability.RedisErrorKind -[SER007]StackExchange.Redis.Availability.RedisErrorKind.NoReplicas = 9 -> StackExchange.Redis.Availability.RedisErrorKind -[SER007]StackExchange.Redis.Availability.RedisErrorKind.NoScript = 25 -> StackExchange.Redis.Availability.RedisErrorKind -[SER007]StackExchange.Redis.Availability.RedisErrorKind.NotPermitted = 20 -> StackExchange.Redis.Availability.RedisErrorKind -[SER007]StackExchange.Redis.Availability.RedisErrorKind.OutOfMemory = 11 -> StackExchange.Redis.Availability.RedisErrorKind -[SER007]StackExchange.Redis.Availability.RedisErrorKind.ReadOnly = 24 -> StackExchange.Redis.Availability.RedisErrorKind -[SER007]StackExchange.Redis.Availability.RedisErrorKind.Timeout = 4 -> StackExchange.Redis.Availability.RedisErrorKind -[SER007]StackExchange.Redis.Availability.RedisErrorKind.TryAgain = 8 -> StackExchange.Redis.Availability.RedisErrorKind -[SER007]StackExchange.Redis.Availability.RedisErrorKind.UnknownCommand = 21 -> StackExchange.Redis.Availability.RedisErrorKind -[SER007]StackExchange.Redis.Availability.RedisErrorKind.UnknownError = 2 -> StackExchange.Redis.Availability.RedisErrorKind -[SER007]StackExchange.Redis.Availability.RedisErrorKind.WrongPass = 18 -> StackExchange.Redis.Availability.RedisErrorKind -[SER007]StackExchange.Redis.Availability.RedisErrorKind.WrongType = 22 -> StackExchange.Redis.Availability.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.None = 0 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.Unknown = 1 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.Availability.FaultContext +[SER007]StackExchange.Redis.Availability.FaultContext.ConnectionFailureType.get -> StackExchange.Redis.ConnectionFailureType +[SER007]StackExchange.Redis.Availability.FaultContext.ErrorKind.get -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.Availability.FaultContext.Fault.get -> System.Exception? +[SER007]StackExchange.Redis.Availability.FaultContext.FaultContext() -> void +[SER007]StackExchange.Redis.Availability.FaultContext.FaultContext(StackExchange.Redis.CommandFlags flags) -> void +[SER007]StackExchange.Redis.Availability.FaultContext.FaultContext(System.Exception! fault, StackExchange.Redis.CommandFlags flags) -> void +[SER007]StackExchange.Redis.Availability.FaultContext.Flags.get -> StackExchange.Redis.CommandFlags +[SER007]StackExchange.Redis.Availability.FaultContext.IsFault.get -> bool +[SER007]StackExchange.Redis.RedisErrorKind.Ask = 15 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.Busy = 12 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.ClusterDown = 6 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.ConnectionFault = 3 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.CrossSlot = 16 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.ExecAbort = 23 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.Loading = 5 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.MasterDown = 7 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.MaxClients = 13 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.Misconfigured = 10 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.Moved = 14 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.NoAuth = 17 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.NoPermission = 19 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.NoReplicas = 9 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.NoScript = 25 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.NotPermitted = 20 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.OutOfMemory = 11 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.ReadOnly = 24 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.Timeout = 4 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.TryAgain = 8 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.UnknownCommand = 21 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.UnknownError = 2 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.WrongPass = 18 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.WrongType = 22 -> StackExchange.Redis.RedisErrorKind [SER007]abstract StackExchange.Redis.Availability.CircuitBreaker.Accumulator.ObserveResult(in StackExchange.Redis.Availability.FaultContext fault) -> void [SER007]virtual StackExchange.Redis.Availability.CircuitBreaker.Accumulator.IsFailure(in StackExchange.Redis.Availability.FaultContext fault) -> bool -virtual StackExchange.Redis.Availability.RetryPolicy.CanRetry(in StackExchange.Redis.Availability.FaultContext fault, StackExchange.Redis.CommandFlags flags) -> StackExchange.Redis.Availability.RetryPolicy.RetryPolicyResult -StackExchange.Redis.Availability.RetryPolicy.RetryPolicyResult -StackExchange.Redis.Availability.RetryPolicy.RetryPolicyResult.None = 0 -> StackExchange.Redis.Availability.RetryPolicy.RetryPolicyResult -StackExchange.Redis.Availability.RetryPolicy.RetryPolicyResult.SameServer = 1 -> StackExchange.Redis.Availability.RetryPolicy.RetryPolicyResult -StackExchange.Redis.Availability.RetryPolicy.RetryPolicyResult.FailoverServer = 2 -> StackExchange.Redis.Availability.RetryPolicy.RetryPolicyResult -StackExchange.Redis.CommandFlags.CommandRetryAlways = 8192 -> StackExchange.Redis.CommandFlags -StackExchange.Redis.CommandFlags.CommandRetryConnection = 32768 -> StackExchange.Redis.CommandFlags -StackExchange.Redis.CommandFlags.CommandRetryReadOnly = 65536 -> StackExchange.Redis.CommandFlags -StackExchange.Redis.CommandFlags.CommandRetryWriteChecked = StackExchange.Redis.CommandFlags.CommandRetryConnection | StackExchange.Redis.CommandFlags.CommandRetryReadOnly -> StackExchange.Redis.CommandFlags -StackExchange.Redis.CommandFlags.CommandRetryWriteLastWins = 131072 -> StackExchange.Redis.CommandFlags -StackExchange.Redis.CommandFlags.CommandRetryWriteAccumulating = StackExchange.Redis.CommandFlags.CommandRetryConnection | StackExchange.Redis.CommandFlags.CommandRetryWriteLastWins -> StackExchange.Redis.CommandFlags -StackExchange.Redis.CommandFlags.CommandRetryServerAdmin = StackExchange.Redis.CommandFlags.CommandRetryReadOnly | StackExchange.Redis.CommandFlags.CommandRetryWriteLastWins -> StackExchange.Redis.CommandFlags -StackExchange.Redis.CommandFlags.CommandRetryNever = 253952 -> StackExchange.Redis.CommandFlags -StackExchange.Redis.CommandFlags.CommandServerSpecific = 262144 -> StackExchange.Redis.CommandFlags +[SER007]virtual StackExchange.Redis.Availability.RetryPolicy.CanRetry(in StackExchange.Redis.Availability.FaultContext fault) -> StackExchange.Redis.Availability.RetryPolicy.RetryPolicyResult +[SER007]StackExchange.Redis.Availability.RetryPolicy.RetryPolicyResult +[SER007]StackExchange.Redis.Availability.RetryPolicy.RetryPolicyResult.None = 0 -> StackExchange.Redis.Availability.RetryPolicy.RetryPolicyResult +[SER007]StackExchange.Redis.Availability.RetryPolicy.RetryPolicyResult.SameServer = 1 -> StackExchange.Redis.Availability.RetryPolicy.RetryPolicyResult +[SER007]StackExchange.Redis.Availability.RetryPolicy.RetryPolicyResult.FailoverServer = 2 -> StackExchange.Redis.Availability.RetryPolicy.RetryPolicyResult +[SER007]StackExchange.Redis.CommandFlags.CommandRetryAlways = 8192 -> StackExchange.Redis.CommandFlags +[SER007]StackExchange.Redis.CommandFlags.CommandRetryConnection = 32768 -> StackExchange.Redis.CommandFlags +[SER007]StackExchange.Redis.CommandFlags.CommandRetryReadOnly = 65536 -> StackExchange.Redis.CommandFlags +[SER007]StackExchange.Redis.CommandFlags.CommandRetryWriteChecked = StackExchange.Redis.CommandFlags.CommandRetryConnection | StackExchange.Redis.CommandFlags.CommandRetryReadOnly -> StackExchange.Redis.CommandFlags +[SER007]StackExchange.Redis.CommandFlags.CommandRetryWriteLastWins = 131072 -> StackExchange.Redis.CommandFlags +[SER007]StackExchange.Redis.CommandFlags.CommandRetryWriteAccumulating = StackExchange.Redis.CommandFlags.CommandRetryConnection | StackExchange.Redis.CommandFlags.CommandRetryWriteLastWins -> StackExchange.Redis.CommandFlags +[SER007]StackExchange.Redis.CommandFlags.CommandRetryServerAdmin = StackExchange.Redis.CommandFlags.CommandRetryReadOnly | StackExchange.Redis.CommandFlags.CommandRetryWriteLastWins -> StackExchange.Redis.CommandFlags +[SER007]StackExchange.Redis.CommandFlags.CommandRetryNever = 253952 -> StackExchange.Redis.CommandFlags +[SER007]StackExchange.Redis.CommandFlags.CommandServerSpecific = 262144 -> StackExchange.Redis.CommandFlags +StackExchange.Redis.RedisServerException.Kind.get -> StackExchange.Redis.RedisErrorKind +StackExchange.Redis.RedisServerException.RedisServerException(StackExchange.Redis.RedisErrorKind kind, string! message) -> void diff --git a/src/StackExchange.Redis/Availability/RedisErrorKind.cs b/src/StackExchange.Redis/RedisErrorKind.cs similarity index 73% rename from src/StackExchange.Redis/Availability/RedisErrorKind.cs rename to src/StackExchange.Redis/RedisErrorKind.cs index f61256422..4fdebf33f 100644 --- a/src/StackExchange.Redis/Availability/RedisErrorKind.cs +++ b/src/StackExchange.Redis/RedisErrorKind.cs @@ -1,9 +1,11 @@ using System; +using System.Buffers; using System.Diagnostics.CodeAnalysis; +using System.Text; using RESPite; using RESPite.Messages; -namespace StackExchange.Redis.Availability; +namespace StackExchange.Redis; /// /// Well-known server error conditions, identified from the error-reply prefix (and in some cases the @@ -138,7 +140,7 @@ public enum RedisErrorKind /// /// ERR unknown command - the command is not known to the server (typo, disabled, or unsupported version). /// - [AsciiHash("")] // matched via the "ERR " branch, not the first token + [AsciiHash("ERR")] // matched via the "ERR " branch, not the first token UnknownCommand, /// @@ -160,20 +162,6 @@ public enum RedisErrorKind /// NOSCRIPT - no matching script for EVALSHA; the script must be re-loaded. /// NoScript, - - /// - /// ERR DB index is out of range / ERR invalid DB index - the database index passed to - /// SELECT is out of range or not a valid integer. - /// - [AsciiHash("")] // matched via the "ERR " branch, not the first token - InvalidDatabaseIndex, - - /// - /// ERR SELECT is not allowed in cluster mode - selecting a non-default database is not supported - /// on this server/topology (e.g. cluster mode, or a server without multi-database support). - /// - [AsciiHash("")] // matched via the "ERR " branch, not the first token - DatabaseSelectDisabled, } internal static partial class RedisErrorKindMetadata @@ -189,38 +177,36 @@ internal static partial class RedisErrorKindMetadata internal static unsafe RedisErrorKind Classify(in RespReader reader) => reader.TryParseScalar(&TryParse, out RedisErrorKind kind) ? kind : RedisErrorKind.Unknown; + internal static unsafe RedisErrorKind Classify(string message) + => RespReader.TryParseScalar(message.AsSpan(), &TryParse, out RedisErrorKind kind) ? kind : RedisErrorKind.Unknown; + internal static bool TryParse(ReadOnlySpan value, out RedisErrorKind kind) { var space = value.IndexOf((byte)' '); // Deal with the exact matches on the first token first - many errors may or may not // have descriptive text after the leading token. var firstToken = space > 0 ? value.Slice(0, space) : value; - if (TryParseFirstTokenCI(firstToken, out kind)) return true; - - // check for "ERR ..." scenarios - if (space > 0 && Err.IsCI(firstToken, AsciiHash.HashUC(firstToken))) + if (TryParseFirstTokenCI(firstToken, out kind)) { - value = value.Slice(space + 1); // message text after "ERR " - - // most ERR conditions are fixed messages matched in full; the exception is - // "unknown command '', with args ...", which always carries a trailing - // command name and so is matched on its leading text (the final guarded arm) - var valueHash = AsciiHash.HashUC(value); - kind = value.Length switch + // check for more specific "ERR ..." scenarios + if (kind is RedisErrorKind.UnknownError & space > 0) { - DbIndexOutOfRange.Length when DbIndexOutOfRange.IsCI(value, valueHash) => RedisErrorKind - .InvalidDatabaseIndex, - InvalidDbIndex.Length when InvalidDbIndex.IsCI(value, valueHash) => RedisErrorKind - .InvalidDatabaseIndex, - OperationNotPermitted.Length when OperationNotPermitted.IsCI(value, valueHash) => RedisErrorKind - .NotPermitted, - SelectNotAllowedInClusterMode.Length when SelectNotAllowedInClusterMode.IsCI(value, valueHash) => RedisErrorKind - .DatabaseSelectDisabled, - _ when value.Length >= UnknownCommand.Length - && AsciiHash.SequenceEqualsCI(value.Slice(0, UnknownCommand.Length), UnknownCommand.U8) => RedisErrorKind - .UnknownCommand, - _ => RedisErrorKind.UnknownError, - }; + // get the message text after "ERR " + value = value.Slice(space + 1); + + // some ERR conditions can be identified further, noting that the text may or + // may not have some tokens - sometimes we need partial match. + var valueHash = AsciiHash.HashUC(value); + if (value.Length is OperationNotPermitted.Length && OperationNotPermitted.IsCI(value, valueHash)) + { + kind = RedisErrorKind.NotPermitted; + } + else if (value.Length >= UnknownCommand.Length && + AsciiHash.SequenceEqualsCI(value.Slice(0, UnknownCommand.Length), UnknownCommand.U8)) + { + kind = RedisErrorKind.UnknownCommand; + } + } return true; } @@ -228,8 +214,14 @@ SelectNotAllowedInClusterMode.Length when SelectNotAllowedInClusterMode.IsCI(val return true; } - [AsciiHash("ERR")] - private static partial class Err { } + [AsciiHash("operation not permitted")] + private static partial class OperationNotPermitted { } + + [AsciiHash("unknown command")] + private static partial class UnknownCommand { } + + /* while these are recognizable, we issue SELECT *on behalf* of the user + and react internally, so reporting them seems... unnecessary. [AsciiHash("DB index is out of range")] private static partial class DbIndexOutOfRange { } @@ -237,12 +229,7 @@ private static partial class DbIndexOutOfRange { } [AsciiHash("invalid DB index")] private static partial class InvalidDbIndex { } - [AsciiHash("operation not permitted")] - private static partial class OperationNotPermitted { } - - [AsciiHash("unknown command")] - private static partial class UnknownCommand { } - [AsciiHash("SELECT is not allowed in cluster mode")] - private static partial class SelectNotAllowedInClusterMode { } + private static partial class SelectNotAllowedInClusterMode { } + */ } diff --git a/src/StackExchange.Redis/RedisResult.cs b/src/StackExchange.Redis/RedisResult.cs index 78ed649bc..589ab69a7 100644 --- a/src/StackExchange.Redis/RedisResult.cs +++ b/src/StackExchange.Redis/RedisResult.cs @@ -558,7 +558,10 @@ internal override RedisValue AsRedisValue() private sealed class ErrorRedisResult : RedisResult { private readonly string value; + private RedisErrorKind kind; // lazily parsed from the value + private RedisErrorKind Kind => kind is RedisErrorKind.None ? GetKind() : kind; + private RedisErrorKind GetKind() => kind = RedisErrorKindMetadata.Classify(value); public ErrorRedisResult(string? value, ResultType type) : base(type) { this.value = value ?? throw new ArgumentNullException(nameof(value)); @@ -570,30 +573,30 @@ public ErrorRedisResult(string? value, ResultType type) : base(type) type = null; return value; } - internal override bool AsBoolean() => throw new RedisServerException(value); - internal override bool[] AsBooleanArray() => throw new RedisServerException(value); - internal override byte[] AsByteArray() => throw new RedisServerException(value); - internal override byte[][] AsByteArrayArray() => throw new RedisServerException(value); - internal override double AsDouble() => throw new RedisServerException(value); - internal override double[] AsDoubleArray() => throw new RedisServerException(value); - internal override int AsInt32() => throw new RedisServerException(value); - internal override int[] AsInt32Array() => throw new RedisServerException(value); - internal override long AsInt64() => throw new RedisServerException(value); - internal override ulong AsUInt64() => throw new RedisServerException(value); - internal override long[] AsInt64Array() => throw new RedisServerException(value); - internal override ulong[] AsUInt64Array() => throw new RedisServerException(value); - internal override bool? AsNullableBoolean() => throw new RedisServerException(value); - internal override double? AsNullableDouble() => throw new RedisServerException(value); - internal override int? AsNullableInt32() => throw new RedisServerException(value); - internal override long? AsNullableInt64() => throw new RedisServerException(value); - internal override ulong? AsNullableUInt64() => throw new RedisServerException(value); - internal override RedisKey AsRedisKey() => throw new RedisServerException(value); - internal override RedisKey[] AsRedisKeyArray() => throw new RedisServerException(value); - internal override RedisResult[] AsRedisResultArray() => throw new RedisServerException(value); - internal override RedisValue AsRedisValue() => throw new RedisServerException(value); - internal override RedisValue[] AsRedisValueArray() => throw new RedisServerException(value); - internal override string? AsString() => throw new RedisServerException(value); - internal override string?[]? AsStringArray() => throw new RedisServerException(value); + internal override bool AsBoolean() => throw new RedisServerException(Kind, value); + internal override bool[] AsBooleanArray() => throw new RedisServerException(Kind, value); + internal override byte[] AsByteArray() => throw new RedisServerException(Kind, value); + internal override byte[][] AsByteArrayArray() => throw new RedisServerException(Kind, value); + internal override double AsDouble() => throw new RedisServerException(Kind, value); + internal override double[] AsDoubleArray() => throw new RedisServerException(Kind, value); + internal override int AsInt32() => throw new RedisServerException(Kind, value); + internal override int[] AsInt32Array() => throw new RedisServerException(Kind, value); + internal override long AsInt64() => throw new RedisServerException(Kind, value); + internal override ulong AsUInt64() => throw new RedisServerException(Kind, value); + internal override long[] AsInt64Array() => throw new RedisServerException(Kind, value); + internal override ulong[] AsUInt64Array() => throw new RedisServerException(Kind, value); + internal override bool? AsNullableBoolean() => throw new RedisServerException(Kind, value); + internal override double? AsNullableDouble() => throw new RedisServerException(Kind, value); + internal override int? AsNullableInt32() => throw new RedisServerException(Kind, value); + internal override long? AsNullableInt64() => throw new RedisServerException(Kind, value); + internal override ulong? AsNullableUInt64() => throw new RedisServerException(Kind, value); + internal override RedisKey AsRedisKey() => throw new RedisServerException(Kind, value); + internal override RedisKey[] AsRedisKeyArray() => throw new RedisServerException(Kind, value); + internal override RedisResult[] AsRedisResultArray() => throw new RedisServerException(Kind, value); + internal override RedisValue AsRedisValue() => throw new RedisServerException(Kind, value); + internal override RedisValue[] AsRedisValueArray() => throw new RedisServerException(Kind, value); + internal override string? AsString() => throw new RedisServerException(Kind, value); + internal override string?[]? AsStringArray() => throw new RedisServerException(Kind, value); } private sealed class SingleRedisResult : RedisResult, IConvertible diff --git a/src/StackExchange.Redis/RedisTransaction.cs b/src/StackExchange.Redis/RedisTransaction.cs index d76974b3f..c1c39b3c1 100644 --- a/src/StackExchange.Redis/RedisTransaction.cs +++ b/src/StackExchange.Redis/RedisTransaction.cs @@ -5,7 +5,6 @@ using System.Threading; using System.Threading.Tasks; using RESPite.Messages; -using StackExchange.Redis.Availability; using StackExchange.Redis.Interfaces; namespace StackExchange.Redis diff --git a/src/StackExchange.Redis/ResultProcessor.cs b/src/StackExchange.Redis/ResultProcessor.cs index 18403ba65..8605f9899 100644 --- a/src/StackExchange.Redis/ResultProcessor.cs +++ b/src/StackExchange.Redis/ResultProcessor.cs @@ -11,7 +11,6 @@ using Microsoft.Extensions.Logging; using RESPite; using RESPite.Messages; -using StackExchange.Redis.Availability; namespace StackExchange.Redis { @@ -212,14 +211,14 @@ public void ConnectionFail(Message message, ConnectionFailureType fail, Exceptio sb.Append(annotation); } var ex = new RedisConnectionException(fail, sb.ToString(), innerException); - SetException(message, ex.With(fail)); + SetException(message, ex); } public static void ConnectionFail(Message message, ConnectionFailureType fail, string errorMessage) => - SetException(message, new RedisConnectionException(fail, errorMessage).With(fail)); + SetException(message, new RedisConnectionException(fail, errorMessage)); public static void ServerFail(Message message, RedisErrorKind kind, string errorMessage) => - SetException(message, new RedisServerException(errorMessage).With(kind)); + SetException(message, new RedisServerException(kind, errorMessage)); public static void SetException(Message? message, Exception ex) { @@ -268,10 +267,10 @@ private bool HandleCommonError(Message message, RespReader reader, PhysicalConne switch (errorKind) { case RedisErrorKind.NoAuth: - bridge?.Multiplexer.SetAuthSuspect(new RedisServerException("NOAUTH Returned - connection has not yet authenticated").With(errorKind)); + bridge?.Multiplexer.SetAuthSuspect(new RedisServerException(errorKind, "NOAUTH Returned - connection has not yet authenticated")); break; case RedisErrorKind.WrongPass: - bridge?.Multiplexer.SetAuthSuspect(new RedisServerException(reader.GetOverview()).With(errorKind)); + bridge?.Multiplexer.SetAuthSuspect(new RedisServerException(errorKind, reader.GetOverview())); break; } @@ -3181,7 +3180,7 @@ public override bool SetResult(PhysicalConnection connection, Message message, r } else { - connection.RecordConnectionFailed(ConnectionFailureType.ProtocolFailure, new RedisServerException(reader.GetOverview())); + connection.RecordConnectionFailed(ConnectionFailureType.ProtocolFailure, new RedisServerException(RedisErrorKind.ConnectionFault, reader.GetOverview())); } } diff --git a/tests/StackExchange.Redis.Tests/CircuitBreakerUnitTests.cs b/tests/StackExchange.Redis.Tests/CircuitBreakerUnitTests.cs index 7b1ad9c04..abc6eaae5 100644 --- a/tests/StackExchange.Redis.Tests/CircuitBreakerUnitTests.cs +++ b/tests/StackExchange.Redis.Tests/CircuitBreakerUnitTests.cs @@ -198,7 +198,7 @@ private sealed class ManualTimeProvider : TimeProvider private static bool Record(CircuitBreaker.Accumulator accumulator, int count, Exception? fault = null) { // success/failure is derived from the presence of a fault; pass a fault for a failure, none for a success - var context = new FaultContext(fault); + FaultContext context = fault is null ? new(CommandFlags.None) : new(fault, CommandFlags.None); for (int i = 0; i < count; i++) { accumulator.ObserveResult(in context); From 1e4a32f6ad74bd595133d690776bbb698a7f259c Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Fri, 17 Jul 2026 10:31:51 +0100 Subject: [PATCH 16/23] duplicate .Database into IDatabaseAsync (from IDatabase) --- src/StackExchange.Redis/Availability/RetryDatabase.cs | 2 +- src/StackExchange.Redis/Interfaces/IDatabase.cs | 2 +- src/StackExchange.Redis/Interfaces/IDatabaseAsync.cs | 3 +++ src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixed.cs | 2 ++ .../KeyspaceIsolation/KeyPrefixedDatabase.cs | 2 -- src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt | 1 + 6 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/StackExchange.Redis/Availability/RetryDatabase.cs b/src/StackExchange.Redis/Availability/RetryDatabase.cs index 1068c8e61..acf9ee4d4 100644 --- a/src/StackExchange.Redis/Availability/RetryDatabase.cs +++ b/src/StackExchange.Redis/Availability/RetryDatabase.cs @@ -51,7 +51,7 @@ public RetryDatabase(IDatabaseAsync inner, RetryPolicy policy) _inner = inner; } - public int Database => _inner is IDatabase db ? db.Database : -1; + public int Database => _inner.Database; public IConnectionMultiplexer Multiplexer => _inner.Multiplexer; diff --git a/src/StackExchange.Redis/Interfaces/IDatabase.cs b/src/StackExchange.Redis/Interfaces/IDatabase.cs index c0761b9a7..dbaa70d42 100644 --- a/src/StackExchange.Redis/Interfaces/IDatabase.cs +++ b/src/StackExchange.Redis/Interfaces/IDatabase.cs @@ -16,7 +16,7 @@ public partial interface IDatabase : IRedis, IDatabaseAsync /// /// The numeric identifier of this database. /// - int Database { get; } + new int Database { get; } /// /// Allows creation of a group of operations that will be sent to the server as a single unit, diff --git a/src/StackExchange.Redis/Interfaces/IDatabaseAsync.cs b/src/StackExchange.Redis/Interfaces/IDatabaseAsync.cs index 40db55cfd..b7f43848b 100644 --- a/src/StackExchange.Redis/Interfaces/IDatabaseAsync.cs +++ b/src/StackExchange.Redis/Interfaces/IDatabaseAsync.cs @@ -14,6 +14,9 @@ namespace StackExchange.Redis /// public partial interface IDatabaseAsync : IRedisAsync { + /// + int Database { get; } + /// /// Indicates whether the instance can communicate with the server (resolved using the supplied key and optional flags). /// diff --git a/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixed.cs b/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixed.cs index df0a3a833..877ffb0b5 100644 --- a/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixed.cs +++ b/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixed.cs @@ -19,6 +19,8 @@ internal KeyPrefixed(TInner inner, byte[] keyPrefix) public IConnectionMultiplexer Multiplexer => Inner.Multiplexer; + public int Database => Inner.Database; + internal TInner Inner { get; } internal byte[] Prefix { get; } diff --git a/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixedDatabase.cs b/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixedDatabase.cs index 2ca7ca384..15deebb89 100644 --- a/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixedDatabase.cs +++ b/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixedDatabase.cs @@ -16,8 +16,6 @@ public IBatch CreateBatch(object? asyncState = null) => public ITransaction CreateTransaction(object? asyncState = null) => new KeyPrefixedTransaction(Inner.CreateTransaction(asyncState), Prefix); - public int Database => Inner.Database; - public RedisValue DebugObject(RedisKey key, CommandFlags flags = CommandFlags.None) => Inner.DebugObject(ToInner(key), flags); diff --git a/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt b/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt index 9297cd113..d0812f232 100644 --- a/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt @@ -1,4 +1,5 @@ #nullable enable +StackExchange.Redis.IDatabaseAsync.Database.get -> int [SER007]StackExchange.Redis.Availability.RetryPolicy [SER007]StackExchange.Redis.Availability.RetryPolicy.FailoverDelay.get -> System.TimeSpan [SER007]StackExchange.Redis.Availability.RetryPolicy.FailoverDelay.set -> void From 4bc9c1983cf6e333dd3811ca397101848a733ba8 Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Fri, 17 Jul 2026 11:31:59 +0100 Subject: [PATCH 17/23] unit tests --- .../Availability/RetryDatabase.cs | 12 +- .../Availability/RetryPolicy.cs | 4 +- .../Enums/CommandFlags.Category.cs | 8 +- src/StackExchange.Redis/Enums/CommandFlags.cs | 12 +- src/StackExchange.Redis/Message.cs | 7 +- .../PublicAPI/PublicAPI.Unshipped.txt | 1 - ...ts.cs => ReconnectRetryPolicyUnitTests.cs} | 2 +- .../RetryTests/CommandRetryPolicyUnitTests.cs | 191 ++++++++++++++++++ ...sicRetryTests.cs => RetryEndToEndTests.cs} | 2 +- 9 files changed, 218 insertions(+), 21 deletions(-) rename tests/StackExchange.Redis.Tests/{RetryPolicyUnitTests.cs => ReconnectRetryPolicyUnitTests.cs} (99%) create mode 100644 tests/StackExchange.Redis.Tests/RetryTests/CommandRetryPolicyUnitTests.cs rename tests/StackExchange.Redis.Tests/RetryTests/{BasicRetryTests.cs => RetryEndToEndTests.cs} (97%) diff --git a/src/StackExchange.Redis/Availability/RetryDatabase.cs b/src/StackExchange.Redis/Availability/RetryDatabase.cs index acf9ee4d4..9b2c007ce 100644 --- a/src/StackExchange.Redis/Availability/RetryDatabase.cs +++ b/src/StackExchange.Redis/Availability/RetryDatabase.cs @@ -28,10 +28,16 @@ public CancellationToken GetNextFailover() => _maxAttempts > 1 & _maxBeforeFailover < _maxAttempts ? _inner.GetNextFailover() : CancellationToken.None; public RetryDatabase(IDatabaseAsync inner, RetryPolicy policy) - { // cannot nest retry, and cannot issue retries *inside* a batch/transaction - var features = inner.RejectFlags(DatabaseFeatureFlags.Batch | DatabaseFeatureFlags.Transaction | DatabaseFeatureFlags.Retry); + : this(inner, policy, inner.RejectFlags(DatabaseFeatureFlags.Batch | DatabaseFeatureFlags.Transaction | DatabaseFeatureFlags.Retry)) + { + } + // test-only: supply the inner database's feature set directly (in particular whether failover is + // available), instead of probing a live inner - so that failover behaviour can be exercised over a + // null inner without a full IDatabaseAsync double. + internal RetryDatabase(IDatabaseAsync inner, RetryPolicy policy, DatabaseFeatureFlags features) + { _policy = policy; // capture config locally rather than constant cross-object lookups (plus: mutability) @@ -111,7 +117,7 @@ private async Task ExecuteAsync(TState state, Func MaxCommandRetryCategory) // note this also covers CommandRetryAlways { // side-effects are beyond what the policy allows return RetryPolicyResult.None; @@ -101,7 +101,7 @@ public virtual RetryPolicyResult CanRetry(in FaultContext fault) { // assume we can send it everywhere var result = RetryPolicyResult.SameServer | RetryPolicyResult.FailoverServer; - if ((fault.Flags & CommandFlags.CommandServerSpecific) != 0) + if ((fault.Flags & Message.CommandServerSpecific) != 0) result &= ~RetryPolicyResult.FailoverServer; return result; } diff --git a/src/StackExchange.Redis/Enums/CommandFlags.Category.cs b/src/StackExchange.Redis/Enums/CommandFlags.Category.cs index e07fd94cb..e74bd19d2 100644 --- a/src/StackExchange.Redis/Enums/CommandFlags.Category.cs +++ b/src/StackExchange.Redis/Enums/CommandFlags.Category.cs @@ -47,7 +47,7 @@ static CommandFlags DefaultCategory(RedisCommand command) // CLIENT etc often use server-specific IDs case RedisCommand.CLIENT: // note some can be considered admin, overridden locally - return CommandFlags.CommandRetryConnection | CommandFlags.CommandServerSpecific; + return CommandFlags.CommandRetryConnection | Message.CommandServerSpecific; // ========================================================================== // READ-ONLY — no mutation, always safe to retry. @@ -140,7 +140,7 @@ static CommandFlags DefaultCategory(RedisCommand command) case RedisCommand.ZSCAN: case RedisCommand.SSCAN: case RedisCommand.HSCAN: - return CommandFlags.CommandRetryReadOnly | CommandFlags.CommandServerSpecific; + return CommandFlags.CommandRetryReadOnly | Message.CommandServerSpecific; // ========================================================================== // WRITE - CHECKED — inherently conditional/idempotent; a retry either @@ -267,7 +267,7 @@ static CommandFlags DefaultCategory(RedisCommand command) case RedisCommand.LATENCY: case RedisCommand.SCRIPT: case RedisCommand.CLUSTER: // note: some like MYID can be considered more safe - return CommandFlags.CommandRetryServerAdmin | CommandFlags.CommandServerSpecific; + return CommandFlags.CommandRetryServerAdmin | Message.CommandServerSpecific; // ========================================================================== // NEVER — transactions, arbitrary scripts, and blocking/destructive or @@ -356,7 +356,7 @@ static CommandFlags DefaultCategory(RedisCommand command) case RedisCommand.HOTKEYS: // diagnostic/introspection, node-local case RedisCommand.SENTINEL: case RedisCommand.SYNC: // replication stream handshake - return CommandFlags.CommandRetryServerAdmin | CommandFlags.CommandServerSpecific; + return CommandFlags.CommandRetryServerAdmin | Message.CommandServerSpecific; // if we don't recognize it: default to the most pessimistic case RedisCommand.NONE: diff --git a/src/StackExchange.Redis/Enums/CommandFlags.cs b/src/StackExchange.Redis/Enums/CommandFlags.cs index dd460eafc..510510c6b 100644 --- a/src/StackExchange.Redis/Enums/CommandFlags.cs +++ b/src/StackExchange.Redis/Enums/CommandFlags.cs @@ -172,7 +172,7 @@ to slide other things in later. Note that 0 is the implicit "not specified" valu /// /// The command performs server administration (for example REPLICAOF or CONFIG SET); these - /// will commonly also carry . + /// are commonly also endpoint-specific (the internal server-specific flag). /// [Experimental(Experiments.ActiveActive, UrlFormat = Experiments.UrlFormat)] CommandRetryServerAdmin = 24 << 13, // pre-shift value 24 @@ -183,11 +183,9 @@ to slide other things in later. Note that 0 is the implicit "not specified" valu [Experimental(Experiments.ActiveActive, UrlFormat = Experiments.UrlFormat)] CommandRetryNever = 31 << 13, // pre-shift value 31 (the full retry-category region) - /// - /// The command is tied to a specific endpoint and must never be retried on a different endpoint - /// (for example cursor-based operations); orthogonal to the retry-category region. - /// - [Experimental(Experiments.ActiveActive, UrlFormat = Experiments.UrlFormat)] - CommandServerSpecific = 1 << 18, + // 262144 (bit 18): "server specific" - the command is tied to a specific endpoint and must never be + // retried on a different endpoint (for example cursor-based operations); orthogonal to the retry-category + // region. Internal-only (see Message.CommandServerSpecific): the wrapper database cannot yet express + // endpoint-stickiness over a *range* of operations, so this is not (currently) on the public API. } } diff --git a/src/StackExchange.Redis/Message.cs b/src/StackExchange.Redis/Message.cs index bf1726ef1..b0ba56c26 100644 --- a/src/StackExchange.Redis/Message.cs +++ b/src/StackExchange.Redis/Message.cs @@ -58,7 +58,10 @@ internal abstract partial class Message : ICompletable internal const CommandFlags InternalCallFlag = (CommandFlags)128, - NoFlushFlag = (CommandFlags)1024; + NoFlushFlag = (CommandFlags)1024, + // "server specific" (bit 18): tied to a specific endpoint, never retry elsewhere. Not (yet) a + // public CommandFlags member - see the note on the hidden bit-18 value in CommandFlags.cs. + CommandServerSpecific = (CommandFlags)(1 << 18); protected RedisCommand command; @@ -89,7 +92,7 @@ internal const CommandFlags | CommandFlags.NoRedirect | CommandFlags.NoScriptCache | MaskRetryCategory // caller may override the retry category... - | CommandFlags.CommandServerSpecific // ...and the server-specific flag + | CommandServerSpecific // ...and the server-specific flag | NoFlushFlag; // we'll allow this one even though not advertised private IResultBox? resultBox; diff --git a/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt b/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt index d0812f232..a0ae83d19 100644 --- a/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt @@ -184,6 +184,5 @@ StackExchange.Redis.IServer.InventKey(StackExchange.Redis.RedisKey prefix = defa [SER007]StackExchange.Redis.CommandFlags.CommandRetryWriteAccumulating = StackExchange.Redis.CommandFlags.CommandRetryConnection | StackExchange.Redis.CommandFlags.CommandRetryWriteLastWins -> StackExchange.Redis.CommandFlags [SER007]StackExchange.Redis.CommandFlags.CommandRetryServerAdmin = StackExchange.Redis.CommandFlags.CommandRetryReadOnly | StackExchange.Redis.CommandFlags.CommandRetryWriteLastWins -> StackExchange.Redis.CommandFlags [SER007]StackExchange.Redis.CommandFlags.CommandRetryNever = 253952 -> StackExchange.Redis.CommandFlags -[SER007]StackExchange.Redis.CommandFlags.CommandServerSpecific = 262144 -> StackExchange.Redis.CommandFlags StackExchange.Redis.RedisServerException.Kind.get -> StackExchange.Redis.RedisErrorKind StackExchange.Redis.RedisServerException.RedisServerException(StackExchange.Redis.RedisErrorKind kind, string! message) -> void diff --git a/tests/StackExchange.Redis.Tests/RetryPolicyUnitTests.cs b/tests/StackExchange.Redis.Tests/ReconnectRetryPolicyUnitTests.cs similarity index 99% rename from tests/StackExchange.Redis.Tests/RetryPolicyUnitTests.cs rename to tests/StackExchange.Redis.Tests/ReconnectRetryPolicyUnitTests.cs index 78036e635..b3ea620f5 100644 --- a/tests/StackExchange.Redis.Tests/RetryPolicyUnitTests.cs +++ b/tests/StackExchange.Redis.Tests/ReconnectRetryPolicyUnitTests.cs @@ -11,7 +11,7 @@ namespace StackExchange.Redis.Tests; -public class RetryPolicyUnitTests(ITestOutputHelper log) +public class ReconnectRetryPolicyUnitTests(ITestOutputHelper log) { [Theory] [InlineData(FailureMode.Success)] diff --git a/tests/StackExchange.Redis.Tests/RetryTests/CommandRetryPolicyUnitTests.cs b/tests/StackExchange.Redis.Tests/RetryTests/CommandRetryPolicyUnitTests.cs new file mode 100644 index 000000000..b6451b5d7 --- /dev/null +++ b/tests/StackExchange.Redis.Tests/RetryTests/CommandRetryPolicyUnitTests.cs @@ -0,0 +1,191 @@ +using System.Threading; +using StackExchange.Redis.Availability; +using StackExchange.Redis.Interfaces; +using Xunit; + +namespace StackExchange.Redis.Tests.RetryTests; + +public class CommandRetryPolicyUnitTests +{ + // --- RetryPolicy.CanRetry: spoofed fault scenarios ------------------------------- + + // Builds a FaultContext for a spoofed server error of the given kind, carrying the given + // command-flags, and asks the policy whether it may be retried. + private static RetryPolicy.RetryPolicyResult CanRetry(RedisErrorKind kind, CommandFlags flags, RetryPolicy? policy = null) + { + // a RedisServerException carries its Kind through to FaultContext.ErrorKind + var fault = new FaultContext(new RedisServerException(kind, kind.ToString()), flags); + return (policy ?? new RetryPolicy()).CanRetry(in fault); + } + + // The command's retry-category is checked against the policy's max category: the default max is + // CommandRetryWriteLastWins, so anything at-or-below that is in-range, and anything with more + // side-effects is not. Using a transient LOADING fault so the error-kind check permits a retry + // whenever the category is in-range - isolating the category logic. + [Theory] + [InlineData(CommandFlags.CommandRetryAlways, true)] + [InlineData(CommandFlags.CommandRetryConnection, true)] + [InlineData(CommandFlags.CommandRetryReadOnly, true)] + [InlineData(CommandFlags.CommandRetryWriteChecked, true)] + [InlineData(CommandFlags.CommandRetryWriteLastWins, true)] // == default max + [InlineData(CommandFlags.CommandRetryWriteAccumulating, false)] // beyond default max + [InlineData(CommandFlags.CommandRetryServerAdmin, false)] + [InlineData(CommandFlags.CommandRetryNever, false)] + [InlineData(CommandFlags.None, false)] // unspecified => assume worst (accumulating) => beyond default max + public void CanRetry_CategoryVersusDefaultMax(CommandFlags category, bool expectRetry) + { + var result = CanRetry(RedisErrorKind.Loading, category); + Assert.Equal(expectRetry, result != RetryPolicy.RetryPolicyResult.None); + } + + // With an in-range category (== default max), the outcome is decided purely by whether the error + // is transient: LOADING is worth retrying, WRONGTYPE is an application error that will not fix itself. + [Theory] + [InlineData(RedisErrorKind.Loading, true)] // still loading the dataset - transient + [InlineData(RedisErrorKind.ClusterDown, true)] // slot temporarily unserved - transient + [InlineData(RedisErrorKind.WrongType, false)] // wrong data type - application error + [InlineData(RedisErrorKind.NoPermission, false)] // ACL - application error + public void CanRetry_ErrorKindGatesRetry_WhenInRange(RedisErrorKind kind, bool expectRetry) + { + var result = CanRetry(kind, CommandFlags.CommandRetryWriteLastWins); + Assert.Equal(expectRetry, result != RetryPolicy.RetryPolicyResult.None); + } + + // "never" and "always" adjust only the category range - they do not override the error-kind check: + // an "always" command still won't retry an application error, and a "never" command won't retry even + // a transient one. + [Theory] + [InlineData(CommandFlags.CommandRetryAlways, RedisErrorKind.Loading, true)] + [InlineData(CommandFlags.CommandRetryAlways, RedisErrorKind.WrongType, false)] + [InlineData(CommandFlags.CommandRetryNever, RedisErrorKind.Loading, false)] + [InlineData(CommandFlags.CommandRetryNever, RedisErrorKind.WrongType, false)] + public void CanRetry_NeverAndAlwaysAffectRangeNotErrorKind(CommandFlags category, RedisErrorKind kind, bool expectRetry) + { + var result = CanRetry(kind, category); + Assert.Equal(expectRetry, result != RetryPolicy.RetryPolicyResult.None); + } + + // When a retry is permitted, it normally offers both the same server and a failover server; but a + // "server specific" (sticky) command must not move endpoints, so only the same-server option remains. + [Theory] + [InlineData(CommandFlags.None, RetryPolicy.RetryPolicyResult.SameServer | RetryPolicy.RetryPolicyResult.FailoverServer)] + [InlineData(Message.CommandServerSpecific, RetryPolicy.RetryPolicyResult.SameServer)] + public void CanRetry_ServerSpecificRestrictsToSameServer(CommandFlags extra, RetryPolicy.RetryPolicyResult expected) + { + // in-range category (== default max) + transient error => a retry is offered; the sticky flag + // only changes *where* the retry may go, not *whether* it happens. + var result = CanRetry(RedisErrorKind.Loading, CommandFlags.CommandRetryWriteLastWins | extra); + Assert.Equal(expected, result); + } + + // The sticky (server-specific) flag lives outside the retry-category region, so it is masked off + // before the category-vs-max comparison and must not change the range verdict (retry-at-all vs none) + // - it only affects the same/failover choice, covered above. + [Theory] + [InlineData(CommandFlags.CommandRetryReadOnly, true)] // in range + [InlineData(CommandFlags.CommandRetryServerAdmin, false)] // beyond default max + public void CanRetry_ServerSpecificDoesNotAffectRange(CommandFlags category, bool expectRetry) + { + var withoutFlag = CanRetry(RedisErrorKind.Loading, category); + var withFlag = CanRetry(RedisErrorKind.Loading, category | Message.CommandServerSpecific); + + Assert.Equal(expectRetry, withoutFlag != RetryPolicy.RetryPolicyResult.None); + Assert.Equal(expectRetry, withFlag != RetryPolicy.RetryPolicyResult.None); + } + + // --- RetryDatabase.CanRetry: attempt accounting ---------------------------------- + + // With max-attempts-before-failover pinned equal to max-attempts, the failover path is disabled, so + // this exercises pure same-server attempt counting. A transient LOADING fault on an in-range command + // means the policy would allow a retry; the only gate is the attempt counter: with MaxAttempts=3, + // attempts 1 and 2 may retry, attempt 3 is exhausted. Because we never fail over, the out "delay" is + // never cancellable (that is how "don't wait for failover" is expressed) and the ref "failover" is + // left untouched. + [Theory] + [InlineData(1, true)] + [InlineData(2, true)] + [InlineData(3, false)] + public void RetryDatabase_CanRetry_MaxAttempts_NoFailover(int attempt, bool expected) + { + var policy = new RetryPolicy { MaxAttempts = 3, MaxAttemptsBeforeFailover = 3 }; + var db = new RetryDatabase(null!, policy); // CanRetry never touches the inner database + + using var cts = new CancellationTokenSource(); + var token = cts.Token; + var failover = token; + + var fault = new RedisServerException(RedisErrorKind.Loading, "LOADING"); + var result = db.CanRetry(attempt, fault, CommandFlags.CommandRetryWriteLastWins, ref failover, out var delay); + + Assert.Equal(expected, result); + Assert.False(delay.CanBeCanceled); // never waiting for a failover + Assert.Equal(token, failover); // ref failover untouched + } + + // MaxAttempts=4 with failover enabled after 2 attempts. A single "failover" token is threaded through + // the sequence to observe the state machine: attempts 1..3 return true, attempt 4 is exhausted. The + // interesting step is attempt 2 (== MaxAttemptsBeforeFailover): it still returns true, but now hands the + // failover token back as "delay" and clears the ref (we fail over only once); attempt 3 therefore sees + // no failover token and drops back to a plain same-server retry. + [Fact] + public void RetryDatabase_CanRetry_FailoverAtThreshold() + { + var policy = new RetryPolicy { MaxAttempts = 4, MaxAttemptsBeforeFailover = 2 }; + // failover is only armed when the inner database advertises the feature; supply it explicitly + var db = new RetryDatabase(null!, policy, DatabaseFeatureFlags.Failover); + + using var cts = new CancellationTokenSource(); + var token = cts.Token; + var failover = token; + var fault = new RedisServerException(RedisErrorKind.Loading, "LOADING"); + + // attempt 1: plain same-server retry; failover token not yet consumed + Assert.True(db.CanRetry(1, fault, CommandFlags.CommandRetryWriteLastWins, ref failover, out var delay)); + Assert.False(delay.CanBeCanceled); + Assert.Equal(token, failover); + + // attempt 2 (== MaxAttemptsBeforeFailover): still a retry, but now fail over - "delay" becomes the + // failover token and the ref is cleared to None so it only fires once + Assert.True(db.CanRetry(2, fault, CommandFlags.CommandRetryWriteLastWins, ref failover, out delay)); + Assert.Equal(token, delay); + Assert.True(delay.CanBeCanceled); + Assert.Equal(CancellationToken.None, failover); + + // attempt 3: failover already spent (ref is None) -> back to a same-server retry + Assert.True(db.CanRetry(3, fault, CommandFlags.CommandRetryWriteLastWins, ref failover, out delay)); + Assert.False(delay.CanBeCanceled); + Assert.Equal(CancellationToken.None, failover); + + // attempt 4: no retries left + Assert.False(db.CanRetry(4, fault, CommandFlags.CommandRetryWriteLastWins, ref failover, out delay)); + Assert.False(delay.CanBeCanceled); + } + + // As above, but with the sticky (server-specific) flag set: the policy now permits only same-server + // retries, so there is no failover option at the threshold. Current behaviour: CanRetry returns *false* + // at attempt 2 - the command gives up rather than continuing on the same server - because the threshold + // branch (attempt == MaxAttemptsBeforeFailover) requires FailoverServer permission and does not fall + // back to a same-server retry. It also consumes the failover token as a side-effect of that branch. + [Fact] + public void RetryDatabase_CanRetry_ServerSpecific_CannotFailover() + { + var policy = new RetryPolicy { MaxAttempts = 4, MaxAttemptsBeforeFailover = 2 }; + var db = new RetryDatabase(null!, policy, DatabaseFeatureFlags.Failover); + + using var cts = new CancellationTokenSource(); + var token = cts.Token; + var failover = token; + var fault = new RedisServerException(RedisErrorKind.Loading, "LOADING"); + const CommandFlags flags = CommandFlags.CommandRetryWriteLastWins | Message.CommandServerSpecific; + + // attempt 1: same-server retry; failover token untouched + Assert.True(db.CanRetry(1, fault, flags, ref failover, out var delay)); + Assert.False(delay.CanBeCanceled); + Assert.Equal(token, failover); + + // attempt 2 (== MaxAttemptsBeforeFailover): sticky forbids failover -> gives up (false), even though + // attempts remain; the failover token is still consumed to None as a side-effect of the branch + Assert.False(db.CanRetry(2, fault, flags, ref failover, out delay)); + Assert.Equal(CancellationToken.None, failover); + } +} diff --git a/tests/StackExchange.Redis.Tests/RetryTests/BasicRetryTests.cs b/tests/StackExchange.Redis.Tests/RetryTests/RetryEndToEndTests.cs similarity index 97% rename from tests/StackExchange.Redis.Tests/RetryTests/BasicRetryTests.cs rename to tests/StackExchange.Redis.Tests/RetryTests/RetryEndToEndTests.cs index 400dac5e4..451258b7b 100644 --- a/tests/StackExchange.Redis.Tests/RetryTests/BasicRetryTests.cs +++ b/tests/StackExchange.Redis.Tests/RetryTests/RetryEndToEndTests.cs @@ -7,7 +7,7 @@ namespace StackExchange.Redis.Tests.RetryTests; [RunPerProtocol] -public class BasicRetryTests(ITestOutputHelper log) +public class RetryEndToEndTests(ITestOutputHelper log) { protected TextWriter Log { get; } = new TextWriterOutputHelper(log); From e6b05872c39cffd20c9c05917eb58397aa239a56 Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Fri, 17 Jul 2026 11:55:03 +0100 Subject: [PATCH 18/23] end to end test, but it highlights a design fault --- src/StackExchange.Redis/Message.cs | 4 +- .../RetryTests/RetryEndToEndTests.cs | 78 +++++++++++-------- 2 files changed, 50 insertions(+), 32 deletions(-) diff --git a/src/StackExchange.Redis/Message.cs b/src/StackExchange.Redis/Message.cs index b0ba56c26..4043a64dd 100644 --- a/src/StackExchange.Redis/Message.cs +++ b/src/StackExchange.Redis/Message.cs @@ -129,7 +129,9 @@ protected Message(int db, CommandFlags flags, RedisCommand command) bool primaryOnly = command.IsPrimaryOnly(); Db = db; this.command = command; - Flags = flags & UserSelectableFlags; + // apply the user-selectable flags, then fill in the default retry-category for this command + // (WithDefaultCategory is a no-op if the caller already specified a CommandRetry* category) + Flags = (flags & UserSelectableFlags).WithDefaultCategory(command); if (primaryOnly) SetPrimaryOnly(); CreatedDateTime = DateTime.UtcNow; diff --git a/tests/StackExchange.Redis.Tests/RetryTests/RetryEndToEndTests.cs b/tests/StackExchange.Redis.Tests/RetryTests/RetryEndToEndTests.cs index 451258b7b..671eb10ca 100644 --- a/tests/StackExchange.Redis.Tests/RetryTests/RetryEndToEndTests.cs +++ b/tests/StackExchange.Redis.Tests/RetryTests/RetryEndToEndTests.cs @@ -1,6 +1,9 @@ using System; using System.IO; using System.Threading.Tasks; +using StackExchange.Redis.Availability; +using StackExchange.Redis.KeyspaceIsolation; +using StackExchange.Redis.Server; using StackExchange.Redis.Tests.Helpers; using Xunit; @@ -11,52 +14,65 @@ public class RetryEndToEndTests(ITestOutputHelper log) { protected TextWriter Log { get; } = new TextWriterOutputHelper(log); - // Baseline: connect to a single in-proc server and exercise the *regular* (non-retry) database. - // This is the control case the retry suite will build on - no RetryDatabase wrapper yet. + // End-to-end: a server that answers the first couple of GETs with a transient LOADING error, then + // serves normally. Wrapping the database with .WithRetry should transparently ride through the LOADING + // responses; we can then observe (via the server's counter) that it really did take three GETs before + // one succeeded. [Fact] - public async Task ConnectAndGet() + public async Task WithRetry_RidesOutTransientLoading() { - using var server = new InProcessTestServer(log); + using var server = new LoadingServer(log); await using var conn = await server.ConnectAsync(log: Log); Assert.True(conn.IsConnected); var db = conn.GetDatabase(); - RedisKey key = "retry:basic"; - Assert.True(await db.StringSetAsync(key, "hello")); + RedisKey key = "retry:loading"; + Assert.True(await db.StringSetAsync(key, "hello")); // seed the value before we start failing GETs - var value = await db.StringGetAsync(key); - Assert.Equal("hello", value); + // queue up two LOADING responses; the third GET should succeed + server.LoadingOps = 2; - // a couple more gets, including a miss - Assert.Equal("hello", await db.StringGetAsync(key)); - Assert.Equal(RedisValue.Null, await db.StringGetAsync("retry:missing")); + // zero delay/jitter so the test isn't paying the default ~1s retry backoff between attempts + var policy = new RetryPolicy + { + MaxAttempts = 3, + RetryDelay = TimeSpan.Zero, + JitterMax = TimeSpan.Zero, + }; + var retryDb = db.WithRetry(policy); + + // NOTE: explicit category, pending the wrapper picking this up from command categorization + var value = await retryDb.StringGetAsync(key, CommandFlags.CommandRetryReadOnly); + + Assert.Equal("hello", value); // retries rode out the LOADING responses + Assert.Equal(0, server.LoadingOps); // both LOADING responses were consumed + Assert.Equal(3, server.GetOpsReceived); // 2 x LOADING + 1 x success } - // Exploratory (no retry wrapper yet): flip the server into a LOADING state *after* connecting, - // then issue a GET and observe what exception type SE.Redis surfaces. No hard assertion on the - // type yet - we just want to see it in the log so we can decide how the circuit-breaker should - // classify it. - [Fact] - public async Task LoadingSurfacesAs() + // An in-proc server that fails the first LoadingOps GET operations with a transient LOADING error + // (decrementing the counter each time), then serves normally. Every GET bumps GetOpsReceived so the + // test can confirm how many attempts actually reached the server. + private sealed class LoadingServer(ITestOutputHelper? log) : InProcessTestServer(log) { - using var server = new InProcessTestServer(log); - await using var conn = await server.ConnectAsync(log: Log); - Assert.True(conn.IsConnected); + // the server core processes operations under a lock (single-threaded, like Redis), so plain fields + // are fine here + public int GetOpsReceived { get; private set; } - var db = conn.GetDatabase(); - Assert.True(await db.StringSetAsync("retry:loading", "before")); // works before loading + public int LoadingOps { get; set; } - server.IsLoading = true; - try - { - var value = await db.StringGetAsync("retry:loading"); - Log.WriteLine($"No exception; got value: {value}"); - } - catch (Exception ex) + protected override TypedRedisValue Get(RedisClient client, in RedisRequest request) { - Log.WriteLine($"Exception type: {ex.GetType().FullName}"); - Log.WriteLine($"Message: {ex.Message}"); + GetOpsReceived++; + + // while LOADING ops remain, consume one and reply with a transient LOADING error + if (LoadingOps > 0) + { + LoadingOps--; + return TypedRedisValue.Error("LOADING Redis is loading the dataset in memory"); + } + + return base.Get(client, in request); } } } From 3f9c3abffa9b33079779b34ad892ad80ae7a4c72 Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Fri, 17 Jul 2026 13:56:31 +0100 Subject: [PATCH 19/23] retry logic --- src/StackExchange.Redis/AutoDatabase.cs | 1 - .../Availability/CircuitBreaker.cs | 8 +- .../Availability/FaultContext.cs | 65 +++++++++++--- .../Availability/RetryDatabase.cs | 7 +- .../Availability/RetryPolicy.cs | 88 ------------------- .../ConnectionMultiplexer.Sentinel.cs | 5 +- .../ConnectionMultiplexer.cs | 2 +- src/StackExchange.Redis/ExceptionFactory.cs | 13 +-- src/StackExchange.Redis/Exceptions.cs | 79 +++++++++++++---- src/StackExchange.Redis/Message.cs | 2 +- src/StackExchange.Redis/PhysicalBridge.cs | 2 +- src/StackExchange.Redis/PhysicalConnection.cs | 8 +- .../PublicAPI/PublicAPI.Unshipped.txt | 11 ++- src/StackExchange.Redis/RedisResult.cs | 48 +++++----- src/StackExchange.Redis/ResultProcessor.cs | 12 +-- .../CircuitBreakerUnitTests.cs | 26 ++---- .../RetryTests/CommandRetryPolicyUnitTests.cs | 24 ++--- .../RetryTests/RetryEndToEndTests.cs | 3 +- 18 files changed, 195 insertions(+), 209 deletions(-) diff --git a/src/StackExchange.Redis/AutoDatabase.cs b/src/StackExchange.Redis/AutoDatabase.cs index 229c63b60..395c5f70c 100644 --- a/src/StackExchange.Redis/AutoDatabase.cs +++ b/src/StackExchange.Redis/AutoDatabase.cs @@ -15,7 +15,6 @@ internal sealed class AutoDatabaseAttribute : Attribute internal interface IRedisArgs { void Map(IRedisArgsMutator mutator); - CommandFlags Flags { get; set; } object? UnMapper { get; } } diff --git a/src/StackExchange.Redis/Availability/CircuitBreaker.cs b/src/StackExchange.Redis/Availability/CircuitBreaker.cs index a503849a7..07e72d3b5 100644 --- a/src/StackExchange.Redis/Availability/CircuitBreaker.cs +++ b/src/StackExchange.Redis/Availability/CircuitBreaker.cs @@ -149,12 +149,11 @@ public abstract class Accumulator() /// public abstract void Reset(); - internal bool Trip(Exception? fault, CommandFlags flags) + internal bool Trip(Exception? fault) { - FaultContext ctx; if (fault is not null) { - ctx = new FaultContext(fault, flags); + var ctx = new FaultContext(fault); if (IsFailure(ctx)) { ObserveResult(ctx); @@ -163,8 +162,7 @@ internal bool Trip(Exception? fault, CommandFlags flags) // otherwise, treat as success for the purposes of counting } - ctx = new(flags); - ObserveResult(ctx); + ObserveResult(in FaultContext.Success); return false; // never trip through success } } diff --git a/src/StackExchange.Redis/Availability/FaultContext.cs b/src/StackExchange.Redis/Availability/FaultContext.cs index d1e706b08..3323abb66 100644 --- a/src/StackExchange.Redis/Availability/FaultContext.cs +++ b/src/StackExchange.Redis/Availability/FaultContext.cs @@ -14,28 +14,65 @@ public readonly struct FaultContext private readonly ConnectionFailureType _connectionFailureType; private readonly CommandFlags _flags; + internal static readonly FaultContext Success = default; + /// /// Create a new . /// /// The fault associated with the operation, or null on success. - /// The command-flags associated with the operation. - public FaultContext(Exception fault, CommandFlags flags) + public FaultContext(Exception fault) { _fault = fault; - _flags = flags & Message.UserSelectableFlags; // just the user-visible ones - ErrorKind = fault.GetErrorKind(out _connectionFailureType); - } - /// - /// Create a new . - /// - /// The command-flags associated with the operation. - public FaultContext(CommandFlags flags) - { - _fault = null; - _flags = flags & Message.UserSelectableFlags; // just the user-visible ones - ErrorKind = RedisErrorKind.None; + var kind = RedisErrorKind.None; _connectionFailureType = ConnectionFailureType.None; + var flags = CommandFlags.None; + switch (fault) + { + case RedisServerException server: + kind = server.Kind; + flags = server.Flags; + break; + case RedisConnectionException connection: + _connectionFailureType = connection.FailureType; + kind = RedisErrorKind.ConnectionFault; + flags = connection.Flags; + break; + case RedisTimeoutException timeout: + kind = RedisErrorKind.Timeout; + flags = timeout.Flags; + break; + case TimeoutException: + kind = RedisErrorKind.Timeout; + break; + case RedisException redis: + flags = redis.Flags; + break; + } + + if (kind is not RedisErrorKind.None & _connectionFailureType is ConnectionFailureType.None) + { + // fill in some blanks + switch (kind) + { + case RedisErrorKind.Loading: + _connectionFailureType = ConnectionFailureType.Loading; + break; + case RedisErrorKind.NoAuth: + case RedisErrorKind.WrongPass: + _connectionFailureType = ConnectionFailureType.AuthenticationFailure; + break; + } + } + + flags &= Message.UserSelectableFlags; + if ((flags & Message.MaskRetryCategory) is 0) + { + // if no retry category found: assume the worst + flags |= CommandFlags.CommandRetryNever; + } + _flags = flags; + ErrorKind = kind; } /// diff --git a/src/StackExchange.Redis/Availability/RetryDatabase.cs b/src/StackExchange.Redis/Availability/RetryDatabase.cs index 9b2c007ce..8dfa132a1 100644 --- a/src/StackExchange.Redis/Availability/RetryDatabase.cs +++ b/src/StackExchange.Redis/Availability/RetryDatabase.cs @@ -84,7 +84,7 @@ private async Task ExecuteAsync(TState state, Func(TState state, Func(TState state, Func Add(target, ServerKey, kind); - - [Obsolete("Prefer .ctor", true)] - internal static Exception With(this RedisServerException target, RedisErrorKind kind) - => throw new NotSupportedException(); - - internal static Exception With(this Exception target, ConnectionFailureType kind) => Add(target, ConnectionKey, kind); - - [Obsolete("Prefer .ctor", true)] - internal static Exception With(this RedisConnectionException target, ConnectionFailureType kind) - => throw new NotSupportedException(); - - internal static RedisErrorKind GetErrorKind(this Exception? target, out ConnectionFailureType connectionFailure) - { - RedisErrorKind kind = RedisErrorKind.None; - connectionFailure = ConnectionFailureType.None; - switch (target) - { - case null: - break; - case RedisServerException server: - kind = server.Kind; - break; - case RedisConnectionException connection: - connectionFailure = connection.FailureType; - kind = RedisErrorKind.ConnectionFault; - break; - case TimeoutException: // includes RedisTimeoutException - kind = RedisErrorKind.Timeout; - break; - default: - try - { - if (target.Data[ServerKey] is RedisErrorKind server) - { - kind = server; - } - if (target.Data[ConnectionKey] is ConnectionFailureType conn) - { - connectionFailure = conn; - if (kind is RedisErrorKind.None) kind = RedisErrorKind.ConnectionFault; - } - } - catch (Exception fault) - { - Debug.WriteLine(fault.Message); - } - - break; - } - - if (kind is not RedisErrorKind.None & connectionFailure is ConnectionFailureType.None) - { - // fill in some blanks - switch (kind) - { - case RedisErrorKind.Loading: - connectionFailure = ConnectionFailureType.Loading; - break; - case RedisErrorKind.NoAuth: - case RedisErrorKind.WrongPass: - connectionFailure = ConnectionFailureType.AuthenticationFailure; - break; - } - } - return kind; - } -} diff --git a/src/StackExchange.Redis/ConnectionMultiplexer.Sentinel.cs b/src/StackExchange.Redis/ConnectionMultiplexer.Sentinel.cs index 6f11fa2f3..658f6920c 100644 --- a/src/StackExchange.Redis/ConnectionMultiplexer.Sentinel.cs +++ b/src/StackExchange.Redis/ConnectionMultiplexer.Sentinel.cs @@ -169,6 +169,7 @@ public ConnectionMultiplexer GetSentinelMasterConnection(ConfigurationOptions co { throw new RedisConnectionException( ConnectionFailureType.UnableToConnect, + CommandFlags.None, "Sentinel: The ConnectionMultiplexer is not a Sentinel connection. Detected as: " + ServerSelectionStrategy.ServerType); } @@ -204,6 +205,7 @@ public ConnectionMultiplexer GetSentinelMasterConnection(ConfigurationOptions co { throw new RedisConnectionException( ConnectionFailureType.UnableToConnect, + CommandFlags.None, $"Sentinel: Failed connecting to configured primary for service: {config.ServiceName}"); } @@ -271,6 +273,7 @@ public ConnectionMultiplexer GetSentinelMasterConnection(ConfigurationOptions co { throw new RedisConnectionException( ConnectionFailureType.UnableToConnect, + CommandFlags.None, $"Sentinel: Failed connecting to configured primary for service: {config.ServiceName}"); } @@ -418,7 +421,7 @@ internal void SwitchPrimary(EndPoint? switchBlame, ConnectionMultiplexer connect // Get new primary - try twice EndPoint newPrimaryEndPoint = GetConfiguredPrimaryForService(serviceName) ?? GetConfiguredPrimaryForService(serviceName) - ?? throw new RedisConnectionException(ConnectionFailureType.UnableToConnect, $"Sentinel: Failed connecting to switch primary for service: {serviceName}"); + ?? throw new RedisConnectionException(ConnectionFailureType.UnableToConnect, CommandFlags.None, $"Sentinel: Failed connecting to switch primary for service: {serviceName}"); connection.currentSentinelPrimaryEndPoint = newPrimaryEndPoint; diff --git a/src/StackExchange.Redis/ConnectionMultiplexer.cs b/src/StackExchange.Redis/ConnectionMultiplexer.cs index 769a64b50..86fd7783d 100644 --- a/src/StackExchange.Redis/ConnectionMultiplexer.cs +++ b/src/StackExchange.Redis/ConnectionMultiplexer.cs @@ -2036,7 +2036,7 @@ private WriteResult TryPushMessageToBridgeSync(Message message, ResultProcess WriteResult.Success => throw new ArgumentOutOfRangeException(nameof(result), "Be sure to check result isn't successful before calling GetException."), WriteResult.NoConnectionAvailable => ExceptionFactory.NoConnectionAvailable(this, message, server), WriteResult.TimeoutBeforeWrite => ExceptionFactory.Timeout(this, null, message, server, result, bridge), - _ => ExceptionFactory.ConnectionFailure(RawConfig.IncludeDetailInExceptions, ConnectionFailureType.ProtocolFailure, "An unknown error occurred when writing the message", server), + _ => ExceptionFactory.ConnectionFailure(RawConfig.IncludeDetailInExceptions, ConnectionFailureType.ProtocolFailure, message.Flags, "An unknown error occurred when writing the message", server), }; [System.Diagnostics.CodeAnalysis.SuppressMessage("Usage", "CA1816:Dispose methods should call SuppressFinalize", Justification = "Intentional observation")] diff --git a/src/StackExchange.Redis/ExceptionFactory.cs b/src/StackExchange.Redis/ExceptionFactory.cs index 380ac0c0f..8428d41af 100644 --- a/src/StackExchange.Redis/ExceptionFactory.cs +++ b/src/StackExchange.Redis/ExceptionFactory.cs @@ -33,9 +33,9 @@ internal static Exception TooManyArgs(string command, int argCount) internal static Exception CommandHasWhitespace(string command) => new RedisCommandException($"The command '{command}' contains whitespace and would be sent as a single unknown token; pass each word as a separate argument, for example Execute(\"ACL\", \"SETUSER\", \"x\") rather than Execute(\"ACL SETUSER x\")."); - internal static Exception ConnectionFailure(bool includeDetail, ConnectionFailureType failureType, string message, ServerEndPoint? server) + internal static Exception ConnectionFailure(bool includeDetail, ConnectionFailureType failureType, CommandFlags flags, string message, ServerEndPoint? server) { - var ex = new RedisConnectionException(failureType, message); + var ex = new RedisConnectionException(failureType, flags, message); if (includeDetail) AddExceptionDetail(ex, null, server, null); return ex; } @@ -154,7 +154,7 @@ internal static Exception NoConnectionAvailable( data = new List>(); AddCommonDetail(data, sb, message, multiplexer, server); } - var ex = new RedisConnectionException(ConnectionFailureType.UnableToResolvePhysicalConnection, sb.ToString(), innerException, message?.Status ?? CommandStatus.Unknown); + var ex = new RedisConnectionException(ConnectionFailureType.UnableToResolvePhysicalConnection, message?.Flags ?? CommandFlags.None, sb.ToString(), innerException, message?.Status ?? CommandStatus.Unknown); if (multiplexer.RawConfig.IncludeDetailInExceptions) { CopyDataToException(data, ex); @@ -283,12 +283,13 @@ internal static Exception Timeout(ConnectionMultiplexer multiplexer, string? bas // If we're from a backlog timeout scenario, we log a more intuitive connection exception for the timeout...because the timeout was a symptom // and we have a more direct cause: we had no connection to send it on. + var msgFlags = message?.Flags ?? CommandFlags.CommandRetryNever; Exception ex = logConnectionException && lastConnectionException is not null - ? new RedisConnectionException(lastConnectionException.FailureType, sb.ToString(), lastConnectionException, message?.Status ?? CommandStatus.Unknown) + ? new RedisConnectionException(lastConnectionException.FailureType, msgFlags, sb.ToString(), lastConnectionException, message?.Status ?? CommandStatus.Unknown) { HelpLink = TimeoutHelpLink, } - : new RedisTimeoutException(sb.ToString(), message?.Status ?? CommandStatus.Unknown) + : new RedisTimeoutException(msgFlags, sb.ToString(), message?.Status ?? CommandStatus.Unknown) { HelpLink = TimeoutHelpLink, }; @@ -448,7 +449,7 @@ internal static Exception UnableToConnect(ConnectionMultiplexer muxer, string? f sb.Append(' ').Append(failureMessage.Trim()); } - return new RedisConnectionException(failureType, sb.ToString(), inner); + return new RedisConnectionException(failureType, CommandFlags.None, sb.ToString(), inner); } } } diff --git a/src/StackExchange.Redis/Exceptions.cs b/src/StackExchange.Redis/Exceptions.cs index 770c957df..8d6419819 100644 --- a/src/StackExchange.Redis/Exceptions.cs +++ b/src/StackExchange.Redis/Exceptions.cs @@ -25,7 +25,6 @@ public RedisCommandException(string message, Exception innerException) : base(me #if NET8_0_OR_GREATER [Obsolete(Obsoletions.LegacyFormatterImplMessage, DiagnosticId = Obsoletions.LegacyFormatterImplDiagId)] - [EditorBrowsable(EditorBrowsableState.Never)] #endif private RedisCommandException(SerializationInfo info, StreamingContext ctx) : base(info, ctx) { } } @@ -41,8 +40,19 @@ public sealed partial class RedisTimeoutException : TimeoutException /// /// The message for the exception. /// The command status, as of when the timeout happened. - public RedisTimeoutException(string message, CommandStatus commandStatus) : base(message) + [Obsolete("Prefer the overload that specifies CommandFlags")] + [Browsable(false), EditorBrowsable(EditorBrowsableState.Never)] + public RedisTimeoutException(string message, CommandStatus commandStatus) : this(CommandFlags.CommandRetryNever, message, commandStatus) { } + + /// + /// Creates a new . + /// + /// The command-flags associated with the faulting operation. + /// The message for the exception. + /// The command status, as of when the timeout happened. + public RedisTimeoutException(CommandFlags flags, string message, CommandStatus commandStatus) : base(message) { + Flags = flags; Commandstatus = commandStatus; } @@ -51,9 +61,13 @@ public RedisTimeoutException(string message, CommandStatus commandStatus) : base /// public CommandStatus Commandstatus { get; } + /// + /// The command-flags associated with the faulting operation (including its retry category). + /// + public CommandFlags Flags { get; } + #if NET8_0_OR_GREATER [Obsolete(Obsoletions.LegacyFormatterImplMessage, DiagnosticId = Obsoletions.LegacyFormatterImplDiagId)] - [EditorBrowsable(EditorBrowsableState.Never)] #endif private RedisTimeoutException(SerializationInfo info, StreamingContext ctx) : base(info, ctx) { @@ -67,7 +81,7 @@ private RedisTimeoutException(SerializationInfo info, StreamingContext ctx) : ba /// Serialization context. #if NET8_0_OR_GREATER [Obsolete(Obsoletions.LegacyFormatterImplMessage, DiagnosticId = Obsoletions.LegacyFormatterImplDiagId)] - [EditorBrowsable(EditorBrowsableState.Never)] + [Browsable(false), EditorBrowsable(EditorBrowsableState.Never)] #endif public override void GetObjectData(SerializationInfo info, StreamingContext context) { @@ -87,7 +101,19 @@ public sealed partial class RedisConnectionException : RedisException /// /// The type of connection failure. /// The message for the exception. - public RedisConnectionException(ConnectionFailureType failureType, string message) : this(failureType, message, null, CommandStatus.Unknown) { } + [Obsolete("Prefer the overload that specifies CommandFlags")] + [Browsable(false), EditorBrowsable(EditorBrowsableState.Never)] + public RedisConnectionException(ConnectionFailureType failureType, string message) : this(failureType, CommandFlags.CommandRetryNever, message, null, CommandStatus.Unknown) { } + + /// + /// Creates a new . + /// + /// The type of connection failure. + /// The message for the exception. + /// The inner exception. + [Obsolete("Prefer the overload that specifies CommandFlags")] + [Browsable(false), EditorBrowsable(EditorBrowsableState.Never)] + public RedisConnectionException(ConnectionFailureType failureType, string message, Exception? innerException) : this(failureType, CommandFlags.CommandRetryNever, message, innerException, CommandStatus.Unknown) { } /// /// Creates a new . @@ -95,18 +121,23 @@ public RedisConnectionException(ConnectionFailureType failureType, string messag /// The type of connection failure. /// The message for the exception. /// The inner exception. - public RedisConnectionException(ConnectionFailureType failureType, string message, Exception? innerException) : this(failureType, message, innerException, CommandStatus.Unknown) { } + /// The status of the command. + [Obsolete("Prefer the overload that specifies CommandFlags")] + [Browsable(false), EditorBrowsable(EditorBrowsableState.Never)] + public RedisConnectionException(ConnectionFailureType failureType, string message, Exception? innerException, CommandStatus commandStatus) : this(failureType, CommandFlags.CommandRetryNever, message, innerException, commandStatus) { } /// /// Creates a new . /// /// The type of connection failure. + /// The command-flags associated with the faulting operation. /// The message for the exception. /// The inner exception. /// The status of the command. - public RedisConnectionException(ConnectionFailureType failureType, string message, Exception? innerException, CommandStatus commandStatus) : base(message, innerException) + public RedisConnectionException(ConnectionFailureType failureType, CommandFlags flags, string message, Exception? innerException = null, CommandStatus commandStatus = CommandStatus.Unknown) : base(message, innerException) { FailureType = failureType; + Flags = flags; CommandStatus = commandStatus; } @@ -115,6 +146,11 @@ public RedisConnectionException(ConnectionFailureType failureType, string messag /// public ConnectionFailureType FailureType { get; } + /// + /// The command-flags associated with the faulting operation (including its retry category). + /// + public override CommandFlags Flags { get; } + /// /// Status of the command while communicating with Redis. /// @@ -122,7 +158,6 @@ public RedisConnectionException(ConnectionFailureType failureType, string messag #if NET8_0_OR_GREATER [Obsolete(Obsoletions.LegacyFormatterImplMessage, DiagnosticId = Obsoletions.LegacyFormatterImplDiagId)] - [EditorBrowsable(EditorBrowsableState.Never)] #endif private RedisConnectionException(SerializationInfo info, StreamingContext ctx) : base(info, ctx) { @@ -137,7 +172,7 @@ private RedisConnectionException(SerializationInfo info, StreamingContext ctx) : /// Serialization context. #if NET8_0_OR_GREATER [Obsolete(Obsoletions.LegacyFormatterImplMessage, DiagnosticId = Obsoletions.LegacyFormatterImplDiagId)] - [EditorBrowsable(EditorBrowsableState.Never)] + [Browsable(false), EditorBrowsable(EditorBrowsableState.Never)] #endif public override void GetObjectData(SerializationInfo info, StreamingContext context) { @@ -173,9 +208,14 @@ public RedisException(string message, Exception? innerException) : base(message, /// Serialization context. #if NET8_0_OR_GREATER [Obsolete(Obsoletions.LegacyFormatterImplMessage, DiagnosticId = Obsoletions.LegacyFormatterImplDiagId)] - [EditorBrowsable(EditorBrowsableState.Never)] #endif protected RedisException(SerializationInfo info, StreamingContext ctx) : base(info, ctx) { } + + /// + /// The command-flags associated with the faulting operation (including its retry category), or + /// when it is not known. + /// + public virtual CommandFlags Flags => CommandFlags.None; } /// @@ -188,20 +228,24 @@ public sealed partial class RedisServerException : RedisException /// Creates a new . /// /// The message for the exception. - [Obsolete("Specify Kind when possible")] - public RedisServerException(string message) : this(RedisErrorKind.Unknown, message) { } + [Obsolete("Specify Kind and CommandFlags when possible")] + [Browsable(false), EditorBrowsable(EditorBrowsableState.Never)] + public RedisServerException(string message) : this(RedisErrorKind.Unknown, CommandFlags.CommandRetryNever, message) { } /// /// Creates a new . /// /// The categorized meaning of the error. + /// The command-flags associated with the faulting operation. /// The message for the exception. - public RedisServerException(RedisErrorKind kind, string message) : base(message) - => Kind = kind; + public RedisServerException(RedisErrorKind kind, CommandFlags flags, string message) : base(message) + { + Kind = kind; + Flags = flags; + } #if NET8_0_OR_GREATER [Obsolete(Obsoletions.LegacyFormatterImplMessage, DiagnosticId = Obsoletions.LegacyFormatterImplDiagId)] - [EditorBrowsable(EditorBrowsableState.Never)] #endif private RedisServerException(SerializationInfo info, StreamingContext ctx) : base(info, ctx) { } @@ -209,5 +253,10 @@ private RedisServerException(SerializationInfo info, StreamingContext ctx) : bas /// Identifies the kind of error received. /// public RedisErrorKind Kind { get; } + + /// + /// The command-flags associated with the faulting operation (including its retry category). + /// + public override CommandFlags Flags { get; } } } diff --git a/src/StackExchange.Redis/Message.cs b/src/StackExchange.Redis/Message.cs index 4043a64dd..b36f3e082 100644 --- a/src/StackExchange.Redis/Message.cs +++ b/src/StackExchange.Redis/Message.cs @@ -508,7 +508,7 @@ public void Complete(PhysicalConnection? connection) performance?.SetCompleted(); if (currBox is not null) { - connection?.ObserveMessageResult(currBox.Fault, Flags); + connection?.ObserveMessageResult(currBox.Fault); } currBox?.ActivateContinuations(); } diff --git a/src/StackExchange.Redis/PhysicalBridge.cs b/src/StackExchange.Redis/PhysicalBridge.cs index ca70965bb..a1ac6e2cf 100644 --- a/src/StackExchange.Redis/PhysicalBridge.cs +++ b/src/StackExchange.Redis/PhysicalBridge.cs @@ -1354,7 +1354,7 @@ private async ValueTask CompleteWriteAndReleaseLockAsync( private WriteResult HandleWriteException(PhysicalConnection? physical, Message message, Exception ex) { - var inner = new RedisConnectionException(ConnectionFailureType.InternalFailure, "Failed to write", ex); + var inner = new RedisConnectionException(ConnectionFailureType.InternalFailure, message.Flags, "Failed to write", ex); message.SetExceptionAndComplete(inner, physical); // Tear down the physical connection. A write that throws may have left a partial frame on the // wire, and continuing to use the same socket would let the next reply match the wrong message diff --git a/src/StackExchange.Redis/PhysicalConnection.cs b/src/StackExchange.Redis/PhysicalConnection.cs index fbd400a20..ab32d8ae2 100644 --- a/src/StackExchange.Redis/PhysicalConnection.cs +++ b/src/StackExchange.Redis/PhysicalConnection.cs @@ -485,7 +485,7 @@ void AddData(string? lk, string? sk, string? v) AddData("Version", "v", Utils.GetLibVersion()); - outerException = new RedisConnectionException(failureType, exMessage.ToString(), innerException); + outerException = new RedisConnectionException(failureType, CommandFlags.None, exMessage.ToString(), innerException); foreach (var kv in data) { @@ -1108,7 +1108,7 @@ private void OnDebugAbort() var bridge = BridgeCouldBeNull; if (bridge == null || !bridge.Multiplexer.AllowConnect) { - throw new RedisConnectionException(ConnectionFailureType.InternalFailure, "Aborting (AllowConnect: False)"); + throw new RedisConnectionException(ConnectionFailureType.InternalFailure, CommandFlags.None, "Aborting (AllowConnect: False)"); } } @@ -1177,13 +1177,13 @@ internal bool HasPendingCallerFacingItems() } } - public void ObserveMessageResult(Exception? fault, CommandFlags flags) + public void ObserveMessageResult(Exception? fault) { // Hot path - runs per completed message. Let the breaker observe the outcome (ObserveResult // returns true while healthy); if it trips and we're the *first* to notice, hand off the // teardown rather than doing it inline: RecordConnectionFailed can fail the whole backlog and // build a detailed exception, which we don't want to pay for on the completion thread. - if (circuitBreaker is { } cb && cb.Trip(fault, flags) + if (circuitBreaker is { } cb && cb.Trip(fault) && Interlocked.CompareExchange(ref _circuitBreakerState, CircuitBreakerTripped, CircuitBreakerHealthy) is CircuitBreakerHealthy) { // hand off to a worker; the heartbeat (see OnBridgeHeartbeat) is a backstop in case the diff --git a/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt b/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt index a0ae83d19..84bbeddc8 100644 --- a/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt @@ -141,8 +141,7 @@ StackExchange.Redis.IServer.InventKey(StackExchange.Redis.RedisKey prefix = defa [SER007]StackExchange.Redis.Availability.FaultContext.ErrorKind.get -> StackExchange.Redis.RedisErrorKind [SER007]StackExchange.Redis.Availability.FaultContext.Fault.get -> System.Exception? [SER007]StackExchange.Redis.Availability.FaultContext.FaultContext() -> void -[SER007]StackExchange.Redis.Availability.FaultContext.FaultContext(StackExchange.Redis.CommandFlags flags) -> void -[SER007]StackExchange.Redis.Availability.FaultContext.FaultContext(System.Exception! fault, StackExchange.Redis.CommandFlags flags) -> void +[SER007]StackExchange.Redis.Availability.FaultContext.FaultContext(System.Exception! fault) -> void [SER007]StackExchange.Redis.Availability.FaultContext.Flags.get -> StackExchange.Redis.CommandFlags [SER007]StackExchange.Redis.Availability.FaultContext.IsFault.get -> bool [SER007]StackExchange.Redis.RedisErrorKind.Ask = 15 -> StackExchange.Redis.RedisErrorKind @@ -184,5 +183,11 @@ StackExchange.Redis.IServer.InventKey(StackExchange.Redis.RedisKey prefix = defa [SER007]StackExchange.Redis.CommandFlags.CommandRetryWriteAccumulating = StackExchange.Redis.CommandFlags.CommandRetryConnection | StackExchange.Redis.CommandFlags.CommandRetryWriteLastWins -> StackExchange.Redis.CommandFlags [SER007]StackExchange.Redis.CommandFlags.CommandRetryServerAdmin = StackExchange.Redis.CommandFlags.CommandRetryReadOnly | StackExchange.Redis.CommandFlags.CommandRetryWriteLastWins -> StackExchange.Redis.CommandFlags [SER007]StackExchange.Redis.CommandFlags.CommandRetryNever = 253952 -> StackExchange.Redis.CommandFlags +override StackExchange.Redis.RedisConnectionException.Flags.get -> StackExchange.Redis.CommandFlags +virtual StackExchange.Redis.RedisException.Flags.get -> StackExchange.Redis.CommandFlags +StackExchange.Redis.RedisConnectionException.RedisConnectionException(StackExchange.Redis.ConnectionFailureType failureType, StackExchange.Redis.CommandFlags flags, string! message, System.Exception? innerException = null, StackExchange.Redis.CommandStatus commandStatus = StackExchange.Redis.CommandStatus.Unknown) -> void +override StackExchange.Redis.RedisServerException.Flags.get -> StackExchange.Redis.CommandFlags StackExchange.Redis.RedisServerException.Kind.get -> StackExchange.Redis.RedisErrorKind -StackExchange.Redis.RedisServerException.RedisServerException(StackExchange.Redis.RedisErrorKind kind, string! message) -> void +StackExchange.Redis.RedisServerException.RedisServerException(StackExchange.Redis.RedisErrorKind kind, StackExchange.Redis.CommandFlags flags, string! message) -> void +StackExchange.Redis.RedisTimeoutException.Flags.get -> StackExchange.Redis.CommandFlags +StackExchange.Redis.RedisTimeoutException.RedisTimeoutException(StackExchange.Redis.CommandFlags flags, string! message, StackExchange.Redis.CommandStatus commandStatus) -> void diff --git a/src/StackExchange.Redis/RedisResult.cs b/src/StackExchange.Redis/RedisResult.cs index 589ab69a7..1d496468f 100644 --- a/src/StackExchange.Redis/RedisResult.cs +++ b/src/StackExchange.Redis/RedisResult.cs @@ -573,30 +573,30 @@ public ErrorRedisResult(string? value, ResultType type) : base(type) type = null; return value; } - internal override bool AsBoolean() => throw new RedisServerException(Kind, value); - internal override bool[] AsBooleanArray() => throw new RedisServerException(Kind, value); - internal override byte[] AsByteArray() => throw new RedisServerException(Kind, value); - internal override byte[][] AsByteArrayArray() => throw new RedisServerException(Kind, value); - internal override double AsDouble() => throw new RedisServerException(Kind, value); - internal override double[] AsDoubleArray() => throw new RedisServerException(Kind, value); - internal override int AsInt32() => throw new RedisServerException(Kind, value); - internal override int[] AsInt32Array() => throw new RedisServerException(Kind, value); - internal override long AsInt64() => throw new RedisServerException(Kind, value); - internal override ulong AsUInt64() => throw new RedisServerException(Kind, value); - internal override long[] AsInt64Array() => throw new RedisServerException(Kind, value); - internal override ulong[] AsUInt64Array() => throw new RedisServerException(Kind, value); - internal override bool? AsNullableBoolean() => throw new RedisServerException(Kind, value); - internal override double? AsNullableDouble() => throw new RedisServerException(Kind, value); - internal override int? AsNullableInt32() => throw new RedisServerException(Kind, value); - internal override long? AsNullableInt64() => throw new RedisServerException(Kind, value); - internal override ulong? AsNullableUInt64() => throw new RedisServerException(Kind, value); - internal override RedisKey AsRedisKey() => throw new RedisServerException(Kind, value); - internal override RedisKey[] AsRedisKeyArray() => throw new RedisServerException(Kind, value); - internal override RedisResult[] AsRedisResultArray() => throw new RedisServerException(Kind, value); - internal override RedisValue AsRedisValue() => throw new RedisServerException(Kind, value); - internal override RedisValue[] AsRedisValueArray() => throw new RedisServerException(Kind, value); - internal override string? AsString() => throw new RedisServerException(Kind, value); - internal override string?[]? AsStringArray() => throw new RedisServerException(Kind, value); + internal override bool AsBoolean() => throw new RedisServerException(Kind, CommandFlags.CommandRetryNever, value); + internal override bool[] AsBooleanArray() => throw new RedisServerException(Kind, CommandFlags.CommandRetryNever, value); + internal override byte[] AsByteArray() => throw new RedisServerException(Kind, CommandFlags.CommandRetryNever, value); + internal override byte[][] AsByteArrayArray() => throw new RedisServerException(Kind, CommandFlags.CommandRetryNever, value); + internal override double AsDouble() => throw new RedisServerException(Kind, CommandFlags.CommandRetryNever, value); + internal override double[] AsDoubleArray() => throw new RedisServerException(Kind, CommandFlags.CommandRetryNever, value); + internal override int AsInt32() => throw new RedisServerException(Kind, CommandFlags.CommandRetryNever, value); + internal override int[] AsInt32Array() => throw new RedisServerException(Kind, CommandFlags.CommandRetryNever, value); + internal override long AsInt64() => throw new RedisServerException(Kind, CommandFlags.CommandRetryNever, value); + internal override ulong AsUInt64() => throw new RedisServerException(Kind, CommandFlags.CommandRetryNever, value); + internal override long[] AsInt64Array() => throw new RedisServerException(Kind, CommandFlags.CommandRetryNever, value); + internal override ulong[] AsUInt64Array() => throw new RedisServerException(Kind, CommandFlags.CommandRetryNever, value); + internal override bool? AsNullableBoolean() => throw new RedisServerException(Kind, CommandFlags.CommandRetryNever, value); + internal override double? AsNullableDouble() => throw new RedisServerException(Kind, CommandFlags.CommandRetryNever, value); + internal override int? AsNullableInt32() => throw new RedisServerException(Kind, CommandFlags.CommandRetryNever, value); + internal override long? AsNullableInt64() => throw new RedisServerException(Kind, CommandFlags.CommandRetryNever, value); + internal override ulong? AsNullableUInt64() => throw new RedisServerException(Kind, CommandFlags.CommandRetryNever, value); + internal override RedisKey AsRedisKey() => throw new RedisServerException(Kind, CommandFlags.CommandRetryNever, value); + internal override RedisKey[] AsRedisKeyArray() => throw new RedisServerException(Kind, CommandFlags.CommandRetryNever, value); + internal override RedisResult[] AsRedisResultArray() => throw new RedisServerException(Kind, CommandFlags.CommandRetryNever, value); + internal override RedisValue AsRedisValue() => throw new RedisServerException(Kind, CommandFlags.CommandRetryNever, value); + internal override RedisValue[] AsRedisValueArray() => throw new RedisServerException(Kind, CommandFlags.CommandRetryNever, value); + internal override string? AsString() => throw new RedisServerException(Kind, CommandFlags.CommandRetryNever, value); + internal override string?[]? AsStringArray() => throw new RedisServerException(Kind, CommandFlags.CommandRetryNever, value); } private sealed class SingleRedisResult : RedisResult, IConvertible diff --git a/src/StackExchange.Redis/ResultProcessor.cs b/src/StackExchange.Redis/ResultProcessor.cs index 8605f9899..45a09cfbc 100644 --- a/src/StackExchange.Redis/ResultProcessor.cs +++ b/src/StackExchange.Redis/ResultProcessor.cs @@ -210,15 +210,15 @@ public void ConnectionFail(Message message, ConnectionFailureType fail, Exceptio sb.Append(", "); sb.Append(annotation); } - var ex = new RedisConnectionException(fail, sb.ToString(), innerException); + var ex = new RedisConnectionException(fail, message?.Flags ?? CommandFlags.None, sb.ToString(), innerException); SetException(message, ex); } public static void ConnectionFail(Message message, ConnectionFailureType fail, string errorMessage) => - SetException(message, new RedisConnectionException(fail, errorMessage)); + SetException(message, new RedisConnectionException(fail, message.Flags, errorMessage)); public static void ServerFail(Message message, RedisErrorKind kind, string errorMessage) => - SetException(message, new RedisServerException(kind, errorMessage)); + SetException(message, new RedisServerException(kind, message.Flags, errorMessage)); public static void SetException(Message? message, Exception ex) { @@ -267,10 +267,10 @@ private bool HandleCommonError(Message message, RespReader reader, PhysicalConne switch (errorKind) { case RedisErrorKind.NoAuth: - bridge?.Multiplexer.SetAuthSuspect(new RedisServerException(errorKind, "NOAUTH Returned - connection has not yet authenticated")); + bridge?.Multiplexer.SetAuthSuspect(new RedisServerException(errorKind, message.Flags, "NOAUTH Returned - connection has not yet authenticated")); break; case RedisErrorKind.WrongPass: - bridge?.Multiplexer.SetAuthSuspect(new RedisServerException(errorKind, reader.GetOverview())); + bridge?.Multiplexer.SetAuthSuspect(new RedisServerException(errorKind, message.Flags, reader.GetOverview())); break; } @@ -3180,7 +3180,7 @@ public override bool SetResult(PhysicalConnection connection, Message message, r } else { - connection.RecordConnectionFailed(ConnectionFailureType.ProtocolFailure, new RedisServerException(RedisErrorKind.ConnectionFault, reader.GetOverview())); + connection.RecordConnectionFailed(ConnectionFailureType.ProtocolFailure, new RedisServerException(RedisErrorKind.ConnectionFault, message.Flags, reader.GetOverview())); } } diff --git a/tests/StackExchange.Redis.Tests/CircuitBreakerUnitTests.cs b/tests/StackExchange.Redis.Tests/CircuitBreakerUnitTests.cs index abc6eaae5..ab0b15545 100644 --- a/tests/StackExchange.Redis.Tests/CircuitBreakerUnitTests.cs +++ b/tests/StackExchange.Redis.Tests/CircuitBreakerUnitTests.cs @@ -48,7 +48,7 @@ public void None_IsAlwaysHealthy() { var acc = CircuitBreaker.None.CreateAccumulator(); // even a solid wall of tracked failures never trips the no-op breaker - Assert.True(Record(acc, 10_000, new RedisTimeoutException("boom", CommandStatus.Unknown))); + Assert.True(Record(acc, 10_000, new RedisTimeoutException(CommandFlags.None, "boom", CommandStatus.Unknown))); } #if NET8_0_OR_GREATER @@ -102,22 +102,6 @@ public void UntrackedExceptions_CountAsSuccess() Assert.False(Record(acc, 100, Timeout())); } - [Fact] - public void CustomTrackedExceptions_AreHonoured() - { - var time = new ManualTimeProvider(); - var acc = Build( - time, - failureRateThreshold: 1, - minimumNumberOfFailures: 1).CreateAccumulator(); - - // the default Redis faults are now *un*tracked, so they read as success - Assert.True(Record(acc, 100, Timeout())); - - // whereas our nominated type trips it - Assert.False(Record(acc, 100, new InvalidOperationException("tracked"))); - } - [Fact] public void OldFailures_AgeOutOfTheWindow() { @@ -193,15 +177,15 @@ private sealed class ManualTimeProvider : TimeProvider } #endif - private static RedisTimeoutException Timeout() => new("timeout", CommandStatus.Unknown); + private static RedisTimeoutException Timeout() => new(CommandFlags.None, "timeout", CommandStatus.Unknown); private static bool Record(CircuitBreaker.Accumulator accumulator, int count, Exception? fault = null) { - // success/failure is derived from the presence of a fault; pass a fault for a failure, none for a success - FaultContext context = fault is null ? new(CommandFlags.None) : new(fault, CommandFlags.None); + // Trip applies the IsFailure gate (a fault the breaker doesn't consider a failure is counted as a + // success), then records via ObserveResult; call it rather than ObserveResult directly so the gate runs for (int i = 0; i < count; i++) { - accumulator.ObserveResult(in context); + accumulator.Trip(fault); } return accumulator.IsHealthy(); diff --git a/tests/StackExchange.Redis.Tests/RetryTests/CommandRetryPolicyUnitTests.cs b/tests/StackExchange.Redis.Tests/RetryTests/CommandRetryPolicyUnitTests.cs index b6451b5d7..b8e630da1 100644 --- a/tests/StackExchange.Redis.Tests/RetryTests/CommandRetryPolicyUnitTests.cs +++ b/tests/StackExchange.Redis.Tests/RetryTests/CommandRetryPolicyUnitTests.cs @@ -13,8 +13,8 @@ public class CommandRetryPolicyUnitTests // command-flags, and asks the policy whether it may be retried. private static RetryPolicy.RetryPolicyResult CanRetry(RedisErrorKind kind, CommandFlags flags, RetryPolicy? policy = null) { - // a RedisServerException carries its Kind through to FaultContext.ErrorKind - var fault = new FaultContext(new RedisServerException(kind, kind.ToString()), flags); + // the exception carries both the Kind and the command-flags; FaultContext reads them back + var fault = new FaultContext(new RedisServerException(kind, flags, kind.ToString())); return (policy ?? new RetryPolicy()).CanRetry(in fault); } @@ -114,8 +114,8 @@ public void RetryDatabase_CanRetry_MaxAttempts_NoFailover(int attempt, bool expe var token = cts.Token; var failover = token; - var fault = new RedisServerException(RedisErrorKind.Loading, "LOADING"); - var result = db.CanRetry(attempt, fault, CommandFlags.CommandRetryWriteLastWins, ref failover, out var delay); + var fault = new RedisServerException(RedisErrorKind.Loading, CommandFlags.CommandRetryWriteLastWins, "LOADING"); + var result = db.CanRetry(attempt, fault, ref failover, out var delay); Assert.Equal(expected, result); Assert.False(delay.CanBeCanceled); // never waiting for a failover @@ -137,27 +137,27 @@ public void RetryDatabase_CanRetry_FailoverAtThreshold() using var cts = new CancellationTokenSource(); var token = cts.Token; var failover = token; - var fault = new RedisServerException(RedisErrorKind.Loading, "LOADING"); + var fault = new RedisServerException(RedisErrorKind.Loading, CommandFlags.CommandRetryWriteLastWins, "LOADING"); // attempt 1: plain same-server retry; failover token not yet consumed - Assert.True(db.CanRetry(1, fault, CommandFlags.CommandRetryWriteLastWins, ref failover, out var delay)); + Assert.True(db.CanRetry(1, fault, ref failover, out var delay)); Assert.False(delay.CanBeCanceled); Assert.Equal(token, failover); // attempt 2 (== MaxAttemptsBeforeFailover): still a retry, but now fail over - "delay" becomes the // failover token and the ref is cleared to None so it only fires once - Assert.True(db.CanRetry(2, fault, CommandFlags.CommandRetryWriteLastWins, ref failover, out delay)); + Assert.True(db.CanRetry(2, fault, ref failover, out delay)); Assert.Equal(token, delay); Assert.True(delay.CanBeCanceled); Assert.Equal(CancellationToken.None, failover); // attempt 3: failover already spent (ref is None) -> back to a same-server retry - Assert.True(db.CanRetry(3, fault, CommandFlags.CommandRetryWriteLastWins, ref failover, out delay)); + Assert.True(db.CanRetry(3, fault, ref failover, out delay)); Assert.False(delay.CanBeCanceled); Assert.Equal(CancellationToken.None, failover); // attempt 4: no retries left - Assert.False(db.CanRetry(4, fault, CommandFlags.CommandRetryWriteLastWins, ref failover, out delay)); + Assert.False(db.CanRetry(4, fault, ref failover, out delay)); Assert.False(delay.CanBeCanceled); } @@ -175,17 +175,17 @@ public void RetryDatabase_CanRetry_ServerSpecific_CannotFailover() using var cts = new CancellationTokenSource(); var token = cts.Token; var failover = token; - var fault = new RedisServerException(RedisErrorKind.Loading, "LOADING"); const CommandFlags flags = CommandFlags.CommandRetryWriteLastWins | Message.CommandServerSpecific; + var fault = new RedisServerException(RedisErrorKind.Loading, flags, "LOADING"); // attempt 1: same-server retry; failover token untouched - Assert.True(db.CanRetry(1, fault, flags, ref failover, out var delay)); + Assert.True(db.CanRetry(1, fault, ref failover, out var delay)); Assert.False(delay.CanBeCanceled); Assert.Equal(token, failover); // attempt 2 (== MaxAttemptsBeforeFailover): sticky forbids failover -> gives up (false), even though // attempts remain; the failover token is still consumed to None as a side-effect of the branch - Assert.False(db.CanRetry(2, fault, flags, ref failover, out delay)); + Assert.False(db.CanRetry(2, fault, ref failover, out delay)); Assert.Equal(CancellationToken.None, failover); } } diff --git a/tests/StackExchange.Redis.Tests/RetryTests/RetryEndToEndTests.cs b/tests/StackExchange.Redis.Tests/RetryTests/RetryEndToEndTests.cs index 671eb10ca..3a4602d69 100644 --- a/tests/StackExchange.Redis.Tests/RetryTests/RetryEndToEndTests.cs +++ b/tests/StackExchange.Redis.Tests/RetryTests/RetryEndToEndTests.cs @@ -42,8 +42,7 @@ public async Task WithRetry_RidesOutTransientLoading() }; var retryDb = db.WithRetry(policy); - // NOTE: explicit category, pending the wrapper picking this up from command categorization - var value = await retryDb.StringGetAsync(key, CommandFlags.CommandRetryReadOnly); + var value = await retryDb.StringGetAsync(key); Assert.Equal("hello", value); // retries rode out the LOADING responses Assert.Equal(0, server.LoadingOps); // both LOADING responses were consumed From b0cd92fac259e35d48a3a35998c5e16866ba1bc7 Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Fri, 17 Jul 2026 14:16:12 +0100 Subject: [PATCH 20/23] nits --- .../AutoDatabaseGenerator.cs | 26 ++----------------- eng/StackExchange.Redis.Build/CodeWriter.cs | 18 ++++++++++++- .../Availability/FaultContext.cs | 3 --- .../Enums/CommandFlags.Category.cs | 4 +++ src/StackExchange.Redis/Exceptions.cs | 10 ++----- .../PublicAPI/PublicAPI.Unshipped.txt | 5 ++-- 6 files changed, 27 insertions(+), 39 deletions(-) diff --git a/eng/StackExchange.Redis.Build/AutoDatabaseGenerator.cs b/eng/StackExchange.Redis.Build/AutoDatabaseGenerator.cs index 87a78869a..356e72619 100644 --- a/eng/StackExchange.Redis.Build/AutoDatabaseGenerator.cs +++ b/eng/StackExchange.Redis.Build/AutoDatabaseGenerator.cs @@ -378,7 +378,7 @@ int GetTupleIndex(BasicArray parameters, bool needsMap) return index; } - const string CommandFlagsType = "StackExchange.Redis.CommandFlags"; + // const string CommandFlagsType = "StackExchange.Redis.CommandFlags"; const string RedisKeyType = "StackExchange.Redis.RedisKey"; const string RedisChannelType = "StackExchange.Redis.RedisChannel"; // types that carry key(s) internally without "RedisKey" in their name, so the @@ -407,17 +407,6 @@ void AppendTupleTypes() bool needsMap = TupleNeedsMap(raw); var parameters = raw.Span; - // locate the (single) CommandFlags argument, if any, to back IRedisArgs.Flags - int flagsArg = -1; - for (int p = 0; p < parameters.Length; p++) - { - if (parameters[p].Type == CommandFlagsType) - { - flagsArg = p; - break; - } - } - // fields are mutable: Map rewrites key/channel fields and Flags has a setter writer.NewLine().NewLine().Append("private struct _tuple").Append(i).Append("("); for (int p = 0; p < parameters.Length; p++) @@ -430,21 +419,10 @@ void AppendTupleTypes() writer.NewLine().Append("{").Indent(); for (int p = 0; p < parameters.Length; p++) { - writer.NewLine().Append("public ").Append(parameters[p].Type) + writer.NewLine().Append("public ").Append(NeedsMap(parameters[p].Type) ? "" : "readonly ").Append(parameters[p].Type) .Append(" Arg").Append(p).Append(" = arg").Append(p).Append(";"); } - // Flags maps onto the CommandFlags field when present, else a synthesized default - writer.NewLine().Append("public global::StackExchange.Redis.CommandFlags Flags"); - if (flagsArg >= 0) - { - writer.Append(" { readonly get => Arg").Append(flagsArg).Append("; set => Arg").Append(flagsArg).Append(" = value; }"); - } - else - { - writer.Append(" { get; set; }"); - } - // Map rewrites each scalar key/channel field directly, and defers container or // loosely-typed fields to a matching Map extension method writer.NewLine().Append("public void Map(global::StackExchange.Redis.IRedisArgsMutator mutator)") diff --git a/eng/StackExchange.Redis.Build/CodeWriter.cs b/eng/StackExchange.Redis.Build/CodeWriter.cs index 94d30ef47..8d3091322 100644 --- a/eng/StackExchange.Redis.Build/CodeWriter.cs +++ b/eng/StackExchange.Redis.Build/CodeWriter.cs @@ -14,7 +14,8 @@ internal sealed class CodeWriter(StringBuilder buffer) /// Starts a new line, applying the current indent. public CodeWriter NewLine() { - buffer.AppendLine().Append(' ', _indent * 4); + buffer.AppendLine(); + _lineHasContent = false; return this; } @@ -32,26 +33,41 @@ public CodeWriter Outdent() return this; } + private bool _lineHasContent; + + private void IndentIfNeeded() + { + if (!_lineHasContent) + { + buffer.Append(' ', _indent * 4); + _lineHasContent = true; + } + } + public CodeWriter Append(string? value) { + IndentIfNeeded(); buffer.Append(value); return this; } public CodeWriter Append(char value) { + IndentIfNeeded(); buffer.Append(value); return this; } public CodeWriter Append(int value) { + IndentIfNeeded(); buffer.Append(value); return this; } public CodeWriter Append(long value) { + IndentIfNeeded(); buffer.Append(value); return this; } diff --git a/src/StackExchange.Redis/Availability/FaultContext.cs b/src/StackExchange.Redis/Availability/FaultContext.cs index 3323abb66..4321630af 100644 --- a/src/StackExchange.Redis/Availability/FaultContext.cs +++ b/src/StackExchange.Redis/Availability/FaultContext.cs @@ -45,9 +45,6 @@ public FaultContext(Exception fault) case TimeoutException: kind = RedisErrorKind.Timeout; break; - case RedisException redis: - flags = redis.Flags; - break; } if (kind is not RedisErrorKind.None & _connectionFailureType is ConnectionFailureType.None) diff --git a/src/StackExchange.Redis/Enums/CommandFlags.Category.cs b/src/StackExchange.Redis/Enums/CommandFlags.Category.cs index e74bd19d2..02abc3b8b 100644 --- a/src/StackExchange.Redis/Enums/CommandFlags.Category.cs +++ b/src/StackExchange.Redis/Enums/CommandFlags.Category.cs @@ -2,6 +2,10 @@ namespace StackExchange.Redis; internal static class CommandFlagsExtensions { + public static CommandFlags WithCategory(this CommandFlags flags, CommandFlags category) + // if the user hasn't already specified a category: use the category supplied + => ((flags & Message.MaskRetryCategory) is 0) ? flags | (category & Message.MaskRetryCategory) : flags; + public static CommandFlags WithDefaultCategory(this CommandFlags flags, RedisCommand command) { if ((flags & Message.MaskRetryCategory) is 0) diff --git a/src/StackExchange.Redis/Exceptions.cs b/src/StackExchange.Redis/Exceptions.cs index 8d6419819..405a0ea4e 100644 --- a/src/StackExchange.Redis/Exceptions.cs +++ b/src/StackExchange.Redis/Exceptions.cs @@ -149,7 +149,7 @@ public RedisConnectionException(ConnectionFailureType failureType, CommandFlags /// /// The command-flags associated with the faulting operation (including its retry category). /// - public override CommandFlags Flags { get; } + public CommandFlags Flags { get; } /// /// Status of the command while communicating with Redis. @@ -210,12 +210,6 @@ public RedisException(string message, Exception? innerException) : base(message, [Obsolete(Obsoletions.LegacyFormatterImplMessage, DiagnosticId = Obsoletions.LegacyFormatterImplDiagId)] #endif protected RedisException(SerializationInfo info, StreamingContext ctx) : base(info, ctx) { } - - /// - /// The command-flags associated with the faulting operation (including its retry category), or - /// when it is not known. - /// - public virtual CommandFlags Flags => CommandFlags.None; } /// @@ -257,6 +251,6 @@ private RedisServerException(SerializationInfo info, StreamingContext ctx) : bas /// /// The command-flags associated with the faulting operation (including its retry category). /// - public override CommandFlags Flags { get; } + public CommandFlags Flags { get; } } } diff --git a/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt b/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt index 84bbeddc8..599eb6faf 100644 --- a/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt @@ -183,10 +183,9 @@ StackExchange.Redis.IServer.InventKey(StackExchange.Redis.RedisKey prefix = defa [SER007]StackExchange.Redis.CommandFlags.CommandRetryWriteAccumulating = StackExchange.Redis.CommandFlags.CommandRetryConnection | StackExchange.Redis.CommandFlags.CommandRetryWriteLastWins -> StackExchange.Redis.CommandFlags [SER007]StackExchange.Redis.CommandFlags.CommandRetryServerAdmin = StackExchange.Redis.CommandFlags.CommandRetryReadOnly | StackExchange.Redis.CommandFlags.CommandRetryWriteLastWins -> StackExchange.Redis.CommandFlags [SER007]StackExchange.Redis.CommandFlags.CommandRetryNever = 253952 -> StackExchange.Redis.CommandFlags -override StackExchange.Redis.RedisConnectionException.Flags.get -> StackExchange.Redis.CommandFlags -virtual StackExchange.Redis.RedisException.Flags.get -> StackExchange.Redis.CommandFlags +StackExchange.Redis.RedisConnectionException.Flags.get -> StackExchange.Redis.CommandFlags StackExchange.Redis.RedisConnectionException.RedisConnectionException(StackExchange.Redis.ConnectionFailureType failureType, StackExchange.Redis.CommandFlags flags, string! message, System.Exception? innerException = null, StackExchange.Redis.CommandStatus commandStatus = StackExchange.Redis.CommandStatus.Unknown) -> void -override StackExchange.Redis.RedisServerException.Flags.get -> StackExchange.Redis.CommandFlags +StackExchange.Redis.RedisServerException.Flags.get -> StackExchange.Redis.CommandFlags StackExchange.Redis.RedisServerException.Kind.get -> StackExchange.Redis.RedisErrorKind StackExchange.Redis.RedisServerException.RedisServerException(StackExchange.Redis.RedisErrorKind kind, StackExchange.Redis.CommandFlags flags, string! message) -> void StackExchange.Redis.RedisTimeoutException.Flags.get -> StackExchange.Redis.CommandFlags From bc74b0f6c9e2c62d515fdbb75c07f72d52b41fc4 Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Fri, 17 Jul 2026 14:44:58 +0100 Subject: [PATCH 21/23] end-to-end tests --- .../ControllableProbe.cs | 18 ++++ .../CircuitBreakerRerouteTests.cs | 27 ++---- .../RetryTests/RetryEndToEndTests.cs | 91 +++++++++++++++++-- 3 files changed, 112 insertions(+), 24 deletions(-) create mode 100644 tests/StackExchange.Redis.Tests/ControllableProbe.cs diff --git a/tests/StackExchange.Redis.Tests/ControllableProbe.cs b/tests/StackExchange.Redis.Tests/ControllableProbe.cs new file mode 100644 index 000000000..4d8e47dc7 --- /dev/null +++ b/tests/StackExchange.Redis.Tests/ControllableProbe.cs @@ -0,0 +1,18 @@ +using System.Net; +using System.Threading.Tasks; +using StackExchange.Redis.Availability; + +namespace StackExchange.Redis.Tests; + +// A health-check probe whose verdict is driven by the test: nominated endpoints report unhealthy on +// demand, everything else is healthy. This keeps a specific member deselected deterministically, even +// after its physical connection reconnects underneath us. +internal sealed class ControllableProbe : HealthCheck.HealthCheckProbe +{ + private volatile EndPoint? _down; + + public void MarkDown(EndPoint endpoint) => _down = endpoint; + + public override Task CheckHealthAsync(HealthCheck healthCheck, IServer server) + => Equals(server.EndPoint, _down) ? UnhealthyTask : HealthyTask; +} diff --git a/tests/StackExchange.Redis.Tests/MultiGroupTests/CircuitBreakerRerouteTests.cs b/tests/StackExchange.Redis.Tests/MultiGroupTests/CircuitBreakerRerouteTests.cs index 546075fa8..da739ae8f 100644 --- a/tests/StackExchange.Redis.Tests/MultiGroupTests/CircuitBreakerRerouteTests.cs +++ b/tests/StackExchange.Redis.Tests/MultiGroupTests/CircuitBreakerRerouteTests.cs @@ -8,7 +8,7 @@ namespace StackExchange.Redis.Tests.MultiGroupTests; [RunPerProtocol] -public class CircuitBreakerRerouteTests(ITestOutputHelper log) +public class CircuitBreakerRerouteTests(ITestOutputHelper log) : TestBase(log) { // A circuit-breaker trip must steer the group away from the affected member *promptly* - i.e. via // the shim's ConnectionFailed(CircuitBreaker) fast-path, not by waiting for the next health-check @@ -23,9 +23,9 @@ public async Task CircuitBreakerTrip_ReroutesAwayFromMember() EndPoint bravo = new DnsEndPoint("bravo", 6379); EndPoint charlie = new DnsEndPoint("charlie", 6379); - using var serverA = new InProcessTestServer(log, endpoint: alpha); - using var serverB = new InProcessTestServer(log, endpoint: bravo); - using var serverC = new InProcessTestServer(log, endpoint: charlie); + using var serverA = new InProcessTestServer(Output, endpoint: alpha); + using var serverB = new InProcessTestServer(Output, endpoint: bravo); + using var serverC = new InProcessTestServer(Output, endpoint: charlie); var breaker = new FlipBreaker(); var probe = new ControllableProbe(); @@ -60,7 +60,7 @@ public async Task CircuitBreakerTrip_ReroutesAwayFromMember() int circuitBreakerEvents = 0; typed.ConnectionFailed += (_, e) => { - log.WriteLine($"ConnectionFailed: {e.FailureType} @ {e.EndPoint}"); + Log($"ConnectionFailed: {e.FailureType} @ {e.EndPoint}"); if (e.FailureType == ConnectionFailureType.CircuitBreaker) { Interlocked.Increment(ref circuitBreakerEvents); @@ -77,7 +77,7 @@ public async Task CircuitBreakerTrip_ReroutesAwayFromMember() breaker.Trip(); var db = conn.GetDatabase(); var fault = await Assert.ThrowsAnyAsync(() => db.ExecuteAsync("nonesuch")); - log.WriteLine($"observed fault: {fault.GetType().Name}: {fault.Message}"); + Log($"observed fault: {fault.GetType().Name}: {fault.Message}"); // wait (briefly) for the fast-path to react; nothing else can move us within this window var moved = await WaitForActiveAsync(conn, notMember: members[0], timeout: TimeSpan.FromSeconds(5)); @@ -113,20 +113,13 @@ private sealed class FlipBreaker : CircuitBreaker private sealed class Acc(FlipBreaker owner) : Accumulator { + // this breaker trips on demand regardless of *what* faulted, so treat every observed fault as a + // failure - otherwise Trip's IsFailure gate would filter out non-failure faults (e.g. an unknown + // command) before the tripped state is ever consulted + protected override bool IsFailure(in FaultContext fault) => true; public override void ObserveResult(in FaultContext context) { } public override bool IsHealthy() => !owner._tripped; public override void Reset() { } } } - - // reports the nominated endpoints unhealthy on demand; everything else is healthy. This keeps the - // tripped member deselected even after its physical connection reconnects. - private sealed class ControllableProbe : HealthCheck.HealthCheckProbe - { - private volatile EndPoint? _down; - public void MarkDown(EndPoint endpoint) => _down = endpoint; - - public override Task CheckHealthAsync(HealthCheck healthCheck, IServer server) - => Equals(server.EndPoint, _down) ? UnhealthyTask : HealthyTask; - } } diff --git a/tests/StackExchange.Redis.Tests/RetryTests/RetryEndToEndTests.cs b/tests/StackExchange.Redis.Tests/RetryTests/RetryEndToEndTests.cs index 3a4602d69..393f8ff2c 100644 --- a/tests/StackExchange.Redis.Tests/RetryTests/RetryEndToEndTests.cs +++ b/tests/StackExchange.Redis.Tests/RetryTests/RetryEndToEndTests.cs @@ -1,19 +1,16 @@ using System; -using System.IO; +using System.Net; using System.Threading.Tasks; using StackExchange.Redis.Availability; using StackExchange.Redis.KeyspaceIsolation; using StackExchange.Redis.Server; -using StackExchange.Redis.Tests.Helpers; using Xunit; namespace StackExchange.Redis.Tests.RetryTests; [RunPerProtocol] -public class RetryEndToEndTests(ITestOutputHelper log) +public class RetryEndToEndTests(ITestOutputHelper log) : TestBase(log) { - protected TextWriter Log { get; } = new TextWriterOutputHelper(log); - // End-to-end: a server that answers the first couple of GETs with a transient LOADING error, then // serves normally. Wrapping the database with .WithRetry should transparently ride through the LOADING // responses; we can then observe (via the server's counter) that it really did take three GETs before @@ -21,8 +18,8 @@ public class RetryEndToEndTests(ITestOutputHelper log) [Fact] public async Task WithRetry_RidesOutTransientLoading() { - using var server = new LoadingServer(log); - await using var conn = await server.ConnectAsync(log: Log); + using var server = new LoadingServer(Output); + await using var conn = await server.ConnectAsync(log: Writer); Assert.True(conn.IsConnected); var db = conn.GetDatabase(); @@ -49,6 +46,86 @@ public async Task WithRetry_RidesOutTransientLoading() Assert.Equal(3, server.GetOpsReceived); // 2 x LOADING + 1 x success } + // Multi-group + WithRetry: two backends hold *different* values for the same key. The group is weighted + // towards A, so a normal read returns A's value. When A drops into LOADING, the faults trip A's + // (deliberately hair-trigger) circuit-breaker; the group reroutes to B, and the retry wrapper rides that + // failover transparently - the very same StringGet now returns B's value, with no caller intervention and + // no configuration change. + [Fact] + public async Task WithRetry_FailsOverBetweenGroupsOnLoading() + { + EndPoint alpha = new DnsEndPoint("alpha", 6379); + EndPoint bravo = new DnsEndPoint("bravo", 6379); + using var serverA = new InProcessTestServer(Output, endpoint: alpha); + using var serverB = new InProcessTestServer(Output, endpoint: bravo); + + RedisKey key = "retry:multigroup"; + + // seed each backend with its own distinct value (direct connections, so each cache gets its own) + await using (var seedA = await serverA.ConnectAsync()) + { + Assert.True(await seedA.GetDatabase().StringSetAsync(key, "from-A")); + } + await using (var seedB = await serverB.ConnectAsync()) + { + Assert.True(await seedB.GetDatabase().StringSetAsync(key, "from-B")); + } + + var probe = new ControllableProbe(); + + // A carries a hair-trigger breaker (trips on the first fault); B keeps the default (never trips here) + var configA = serverA.GetClientConfig(); + configA.CircuitBreaker = new CircuitBreaker.Builder + { + MinimumNumberOfFailures = 1, + FailureRateThreshold = 1, + }; + + ConnectionGroupMember[] members = + [ + new(configA, "A") { Weight = 9 }, // highest weight -> initially active + new(serverB.GetClientConfig(), "B") { Weight = 1 }, // failover target + ]; + + var options = new MultiGroupOptions + { + HealthCheck = new HealthCheck + { + Probe = probe, + Interval = TimeSpan.FromMinutes(30), // huge: the breaker fast-path is what reroutes us + ProbeCount = 1, + ProbeTimeout = TimeSpan.FromSeconds(5), + }, + }; + + await using var conn = await ConnectionMultiplexer.ConnectGroupAsync(members, options); + Assert.True(conn.IsConnected); + Assert.Same(members[0], conn.ActiveMember); // A is active (highest weight) + + // failover enabled, plenty of attempts, no artificial delay between them + var policy = new RetryPolicy + { + MaxAttempts = 20, + MaxAttemptsBeforeFailover = 1, + RetryDelay = TimeSpan.Zero, + JitterMax = TimeSpan.Zero, + }; + var db = conn.GetDatabase().WithRetry(policy); + + // normal read: routed to the active (weighted) member, A + string? before = await db.StringGetAsync(key); + Assert.Equal("from-A", before); + + // knock A into LOADING and hold it down; nothing else about the setup changes + serverA.IsLoading = true; + probe.MarkDown(alpha); + + // the *same* call now transparently rides the circuit-break failover across to B + string? after = await db.StringGetAsync(key); + Assert.Equal("from-B", after); + Assert.Same(members[1], conn.ActiveMember); // we really did move to B + } + // An in-proc server that fails the first LoadingOps GET operations with a transient LOADING error // (decrementing the counter each time), then serves normally. Every GET bumps GetOpsReceived so the // test can confirm how many attempts actually reached the server. From a182c1007a01457892344811382cd59c14671eeb Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Fri, 17 Jul 2026 15:18:35 +0100 Subject: [PATCH 22/23] docs --- docs/ActiveActive.md | 365 ++++++++++++------ .../Availability/DatabaseExtensions.cs | 20 + .../KeyspaceIsolation/DatabaseExtension.cs | 13 - .../PublicAPI/PublicAPI.Unshipped.txt | 3 +- .../RetryTests/RetryEndToEndTests.cs | 1 - 5 files changed, 262 insertions(+), 140 deletions(-) create mode 100644 src/StackExchange.Redis/Availability/DatabaseExtensions.cs diff --git a/docs/ActiveActive.md b/docs/ActiveActive.md index be926a44f..f9341dcc3 100644 --- a/docs/ActiveActive.md +++ b/docs/ActiveActive.md @@ -2,8 +2,14 @@ ## Overview -The Active:Active feature provides automatic failover and intelligent routing across multiple Redis deployments. The library -automatically selects the best available endpoint based on: +The Active:Active feature provides automatic failover and intelligent routing across multiple Redis deployments. It is built from several cooperating pieces: + +1. **Connecting to multiple servers or regions at once**, giving a deployment redundant endpoints to fall back on. +2. **Health checks** that *actively* probe each endpoint on a timer to monitor its availability. +3. **Circuit breakers** that *passively* monitor availability from the observed success and failure of the traffic already flowing. +4. **Automatic retries** that let straightforward operations ride out a possibly-unstable connection — including silently transitioning to another endpoint during a failover, with no change to your calling code. + +The library automatically selects the best available endpoint based on: 1. **Availability** - Connected endpoints are always preferred over disconnected ones 2. **Weight** - User-defined preference values (higher is better) @@ -354,93 +360,9 @@ var healthCheck = new HealthCheck ProbeCount = 3, ProbePolicy = HealthCheckProbePolicy.MajoritySuccess }; -// Healthy if 2 or more of probes succeed +// Healthy if 2 or more of 3 probes succeed ``` -### Custom Health Check Probes - -You can implement custom health check logic by extending `HealthCheckProbe`. Note that care must be used -if the probe involves talking to data via a `RedisKey`, as on "cluster" configurations, it must be ensured that the -key used resolves to the correct server; for this purpose, the `server.InventKey` method can be used: - -```csharp -public abstract class CustomProbe : HealthCheckProbe -{ - public override Task CheckHealthAsync(HealthCheck healthCheck, IServer server) - { - // create a random key that routes to the correct server, using - // the specified prefix - RedisKey key = server.InventKey("health-check/"); - // ... - } -} -```` - -Or more conveniently, the key-specific `KeyWriteHealthCheckProbe` encapsulates this logic: - -```csharp -public class CustomWriteProbe : KeyWriteHealthCheckProbe -{ - public override async Task CheckHealthAsync( - HealthCheck healthCheck, - IDatabaseAsync database, - RedisKey key) - { - try - { - var value = Guid.NewGuid().ToString(); - await database.StringSetAsync(key, value, expiry: healthCheck.ProbeTimeout); - bool isMatch = value == await database.StringGetAsync(key); - - return isMatch ? HealthCheckResult.Healthy : HealthCheckResult.Unhealthy; - } - catch - { - return HealthCheckResult.Unhealthy; - } - } -} -``` - -### Custom Probe Policies - -In addition to the inbuilt policies, custom policies can be implemented by extending `HealthCheckProbePolicy`. -By checking the properties of the `HealthCheckProbeContext` parameter, your policy can make a determination -about the health of the server - returning `HealthCheckResult.Healthy` or `HealthCheckResult.Unhealthy` as -appropriate. If you return `HealthCheckResult.Inconclusive`, the health check will continue with additional probes. - -#### Example: Require at Least N Successes - -This example demonstrates a policy that requires at least a specified number of successful probes before declaring the endpoint healthy: - -```csharp -public class AtLeastPolicy(int requiredSuccesses) : HealthCheckProbePolicy -{ - public override HealthCheckResult Evaluate(in HealthCheckProbeContext context) - { - // Success if we have at least the required number of successful probes - if (context.Success >= requiredSuccesses) return HealthCheckResult.Healthy; - - // If no more probes remaining, we haven't met our threshold; otherwise: keep trying - return context.Remaining == 0 ? HealthCheckResult.Unhealthy : HealthCheckResult.Inconclusive; - } -} - -// Use the custom policy requiring at least 2 successes -var healthCheck = new HealthCheck -{ - ProbeCount = 5, // Need enough probes to allow for the required successes - ProbePolicy = new AtLeastPolicy(2) -}; - -var options = new MultiGroupOptions -{ - HealthCheck = healthCheck -}; -``` - -This policy ensures that transient successes don't immediately mark an endpoint as healthy. It requires at least the specified number of successful probes, which provides better confidence in the endpoint's stability while still being more lenient than `AllSuccess`. - ### Health Check Behavior When a health check fails for a member: @@ -509,7 +431,6 @@ var options = new MultiGroupOptions FailureRateThreshold = 25, // trip above 25% failures MinimumNumberOfFailures = 100, // ...but only after 100 failures in the window MetricsWindowSize = TimeSpan.FromSeconds(5), // rolling window to measure over - TrackedExceptions = [typeof(RedisTimeoutException)], // which faults count as failures } }; ``` @@ -521,9 +442,8 @@ A `Builder` converts implicitly to a `CircuitBreaker`, so it can be assigned dir | `FailureRateThreshold` | 10 | Percentage of failures within the window that trips the breaker | | `MinimumNumberOfFailures` | 1000 | Minimum tracked failures in the window before the breaker can trip (avoids acting on tiny samples) | | `MetricsWindowSize` | 2 seconds | Rolling window over which successes and failures are counted | -| `TrackedExceptions` | `null` | Exception types that count as failures; when `null`, `RedisConnectionException` and `RedisTimeoutException` are tracked | -`TrackedExceptions` controls which faults count against the breaker; anything not tracked is treated as a success for circuit-breaking purposes (for example, a `RedisServerException` for a bad command is not a *connection* problem). Passing an array containing `typeof(Exception)` tracks everything; passing an empty array tracks nothing. +Which faults count against the breaker is decided by *classification*, not by exception type: transient and connection-level errors (and timeouts) count as failures, while application-level errors — for example a `RedisServerException` for a bad command, or a `WRONGTYPE` — are treated as a success for circuit-breaking purposes, since they are not a sign of an unhealthy *connection*. This is derived from the fault's `RedisErrorKind`; to change what counts, override `IsFailure(in FaultContext)` on a custom accumulator (see *Custom circuit breakers* below). ### Disabling the Circuit Breaker @@ -536,54 +456,87 @@ var options = new MultiGroupOptions }; ``` -### Custom Circuit Breakers (Advanced) - -> This is an advanced extension point; most applications should use `CircuitBreaker.Default` (optionally tuned via `CircuitBreaker.Builder`) or `CircuitBreaker.None`. +## Automatic Retries -You can implement your own policy by extending `CircuitBreaker` and its `Accumulator`. `CreateAccumulator()` is called once per underlying connection, so each accumulator holds the state for a single connection. -For every completed message the connection calls `ObserveResult`, passing a `CircuitBreakerContext` that exposes whether the operation succeeded (`Success`) and the associated `Fault` (if any). -Return `true` from `IsHealthy()` while the connection should be considered **healthy**, or `false` to **trip** the breaker and tear the connection down: +Health checks and circuit breakers keep the *group* pointed at a healthy member; **automatic retries** deal with the *individual operation* that was in flight when something went wrong. Wrapping a database with `WithRetry(...)` returns a database that transparently re-issues failed operations according to a `RetryPolicy` — riding out transient faults, and (in an Active:Active group) following a failover across to another member, without the caller having to catch-and-retry by hand. ```csharp -public sealed class ConsecutiveFailureBreaker(int limit) : CircuitBreaker -{ - public override Accumulator CreateAccumulator() => new Acc(limit); +await using var conn = await ConnectionMultiplexer.ConnectGroupAsync(members); - private sealed class Acc(int limit) : Accumulator - { - private int _consecutiveFailures; +// wrap the database once; reuse the wrapper like any other IDatabaseAsync +IDatabaseAsync db = conn.GetDatabase().WithRetry(new RetryPolicy()); - public override bool ObserveResult(in CircuitBreakerContext context) - { - if (context.Success) - { - _consecutiveFailures = 0; - } - else - { - _consecutiveFailures++; - } +// a transient fault (e.g. the active member briefly returning LOADING) is retried +// automatically; if the group fails over in the meantime, the retry lands on the new member +var value = await db.StringGetAsync("mykey"); +``` - // if Evaluate is specified, we should also compute IsHealthy(), - // often this can reuse local state needed by ObserveResult - return context.Evaluate ? IsHealthy() : true; - } +> You can call `WithRetry` on any database (`IDatabase` or `IDatabaseAsync`), but the wrapper it returns exposes only the **async** API — there is no synchronous form, since retrying may inherently have delays. It cannot wrap a batch or transaction, nor an already-retrying database. - // healthy until we hit the configured run of consecutive failures - public override bool IsHealthy() => _consecutiveFailures < limit; +### RetryPolicy settings - // called if the connection wants to discard accumulated history - public override void Reset() => _consecutiveFailures = 0; - } -} +`RetryPolicy` controls how many times, how often, and how far an operation is retried: -var options = new MultiGroupOptions +| Property | Default | Description | +|----------|---------|-------------| +| `MaxAttempts` | 3 | Total attempts (including the first) before giving up | +| `MaxAttemptsBeforeFailover` | 1 | Attempts against the current member before a retry is allowed to wait for / move to a failover member (only meaningful for multi-group connections) | +| `RetryDelay` | 1 second | Delay between same-server retries | +| `JitterMax` | 0.5 seconds | Upper bound of the additional random delay added to each retry, to avoid stampedes | +| `FailoverDelay` | 5 seconds | Maximum time to wait for a failover, when a retry is gated on one happening | +| `MaxCommandRetryCategory` | `CommandRetryWriteLastWins` | The most side-effecting command category that will be retried (see below) | + +```csharp +var policy = new RetryPolicy { - CircuitBreaker = new ConsecutiveFailureBreaker(limit: 5) + MaxAttempts = 5, + RetryDelay = TimeSpan.FromMilliseconds(200), + JitterMax = TimeSpan.FromMilliseconds(100), }; +IDatabaseAsync db = conn.GetDatabase().WithRetry(policy); ``` -Keep `ObserveResult` cheap and thread-safe: it runs on the hot path for every completed message, and may be called concurrently. +Only *transient* faults are retried — the same `RedisErrorKind`-based classification the default circuit breaker uses. An application-level error such as `WRONGTYPE`, or an unknown command, is not transient and is never retried, no matter what the policy says. + +### Which operations are safe to retry + +Retrying is not free of consequence: replaying `INCR` after an ambiguous failure could double-count, whereas replaying `GET` is harmless; `SET` is "last wins", so: *usally* fine. Every command therefore carries a **retry category** describing its side-effects, and a policy only retries commands at or below its `MaxCommandRetryCategory`. + +For the built-in typed methods (`StringGet`, `StringSet`, `HashSet`, ...) the library assigns the appropriate category automatically, so retries "just work" within the default policy. + +### Custom commands: `Execute` and `ScriptEvaluate` + +The library cannot infer the side-effects of a command it doesn't recognise — and that includes arbitrary commands issued via `Execute`/`ExecuteAsync`, and Lua run via `ScriptEvaluate`/`ScriptEvaluateAsync` (whose effect depends entirely on the script). Such commands are therefore treated **pessimistically**: an uncategorised command defaults to `CommandRetryNever` and is *not* retried. + +The categories, from safest to most dangerous, are: + +| `CommandFlags` value | Meaning | +|----------------------|---------| +| `CommandRetryAlways` | Always safe to retry, regardless of connection/server state | +| `CommandRetryConnection` | Connection-level or safe metadata (e.g. `CLIENT SETNAME`, `CONFIG GET`) | +| `CommandRetryReadOnly` | Pure reads (e.g. `GET`) | +| `CommandRetryWriteChecked` | Conditional writes (e.g. `SETNX`, `SET ... IFEQ`) | +| `CommandRetryWriteLastWins` | Unconditional overwrite — last-writer-wins (e.g. `SET`) | +| `CommandRetryWriteAccumulating` | Cumulative writes where a retry can double-apply (e.g. `INCR`, `LPUSH`) | +| `CommandRetryServerAdmin` | Server administration (e.g. `CONFIG SET`) | +| `CommandRetryNever` | Never retry | + + +When possible when using ad-hoc commands or script, callers should supply the most appropriate `CommandRetry*` category in the command's `CommandFlags`: + +```csharp +// an arbitrary read-only command: safe to retry +var result = await db.ExecuteAsync("LOLWUT", args: [], flags: CommandFlags.CommandRetryReadOnly); + +// a Lua script that only reads: opt into retries +var value = await db.ScriptEvaluateAsync( + "return redis.call('GET', KEYS[1])", + keys: [key], + flags: CommandFlags.CommandRetryReadOnly); +``` + +Choose the category honestly — it describes what a *replay* would do. If a retry could double-apply a side-effect, use `CommandRetryWriteAccumulating` (or leave it uncategorised) rather than claiming it's a read. +Conversely, if you want more-side-effecting operations retried across the board, raise the policy's `MaxCommandRetryCategory` instead of tagging each call. ## Unhealthy State and Failback @@ -825,3 +778,165 @@ if (newDC.IsConnected) } } ``` + +## Advanced Customization + +The building blocks above cover the common cases. The extension points below let you replace the default health-check, retry, and circuit-breaker behavior with your own logic; most applications will not need them. + +### Custom Health Check Probes + +You can implement custom health check logic by extending `HealthCheckProbe`. Note that care must be used +if the probe involves talking to data via a `RedisKey`, as on "cluster" configurations, it must be ensured that the +key used resolves to the correct server; for this purpose, the `server.InventKey` method can be used: + +```csharp +public abstract class CustomProbe : HealthCheckProbe +{ + public override Task CheckHealthAsync(HealthCheck healthCheck, IServer server) + { + // create a random key that routes to the correct server, using + // the specified prefix + RedisKey key = server.InventKey("health-check/"); + // ... + } +} +``` + +Or more conveniently, the key-specific `KeyWriteHealthCheckProbe` encapsulates this logic: + +```csharp +public class CustomWriteProbe : KeyWriteHealthCheckProbe +{ + public override async Task CheckHealthAsync( + HealthCheck healthCheck, + IDatabaseAsync database, + RedisKey key) + { + try + { + var value = Guid.NewGuid().ToString(); + await database.StringSetAsync(key, value, expiry: healthCheck.ProbeTimeout); + bool isMatch = value == await database.StringGetAsync(key); + + return isMatch ? HealthCheckResult.Healthy : HealthCheckResult.Unhealthy; + } + catch + { + return HealthCheckResult.Unhealthy; + } + } +} +``` + +### Custom Probe Policies + +In addition to the inbuilt policies, custom policies can be implemented by extending `HealthCheckProbePolicy`. +By checking the properties of the `HealthCheckProbeContext` parameter, your policy can make a determination +about the health of the server - returning `HealthCheckResult.Healthy` or `HealthCheckResult.Unhealthy` as +appropriate. If you return `HealthCheckResult.Inconclusive`, the health check will continue with additional probes. + +#### Example: Require at Least N Successes + +This example demonstrates a policy that requires at least a specified number of successful probes before declaring the endpoint healthy: + +```csharp +public class AtLeastPolicy(int requiredSuccesses) : HealthCheckProbePolicy +{ + public override HealthCheckResult Evaluate(in HealthCheckProbeContext context) + { + // Success if we have at least the required number of successful probes + if (context.Success >= requiredSuccesses) return HealthCheckResult.Healthy; + + // If no more probes remaining, we haven't met our threshold; otherwise: keep trying + return context.Remaining == 0 ? HealthCheckResult.Unhealthy : HealthCheckResult.Inconclusive; + } +} + +// Use the custom policy requiring at least 2 successes +var healthCheck = new HealthCheck +{ + ProbeCount = 5, // Need enough probes to allow for the required successes + ProbePolicy = new AtLeastPolicy(2) +}; + +var options = new MultiGroupOptions +{ + HealthCheck = healthCheck +}; +``` + +This policy ensures that transient successes don't immediately mark an endpoint as healthy. It requires at least the specified number of successful probes, which provides better confidence in the endpoint's stability while still being more lenient than `AllSuccess`. + +### Custom Circuit Breakers + +> This is an advanced extension point; most applications should use `CircuitBreaker.Default` (optionally tuned via `CircuitBreaker.Builder`) or `CircuitBreaker.None`. + +You can implement your own policy by extending `CircuitBreaker` and its `Accumulator`. `CreateAccumulator()` is called once per underlying connection, so each accumulator holds the state for a single connection. +For every completed message the connection calls `ObserveResult`, passing a `FaultContext` that describes the outcome: `IsFault` indicates whether a fault occurred, with the associated `Fault`, `ErrorKind` and `ConnectionFailureType` available for inspection. `IsHealthy()` is consulted separately: return `true` while the connection should be considered **healthy**, or `false` to **trip** the breaker and tear the connection down. + +By default only faults that `IsFailure` regards as genuine failures reach `ObserveResult` *as* failures (transient/connection errors, timeouts, and similar - classified from the fault's `ErrorKind`); everything else - including application-level errors such as a bad command - is passed as a success. Override `IsFailure(in FaultContext)` if you need different rules for what counts: + +```csharp +public sealed class ConsecutiveFailureBreaker(int limit) : CircuitBreaker +{ + public override Accumulator CreateAccumulator() => new Acc(limit); + + private sealed class Acc(int limit) : Accumulator + { + private int _consecutiveFailures; + + // a fault is present iff context.IsFault; a success resets the run + public override void ObserveResult(in FaultContext context) + { + if (context.IsFault) + { + _consecutiveFailures++; + } + else + { + _consecutiveFailures = 0; + } + } + + // healthy until we hit the configured run of consecutive failures + public override bool IsHealthy() => _consecutiveFailures < limit; + + // called if the connection wants to discard accumulated history + public override void Reset() => _consecutiveFailures = 0; + + // (optional) override to change what counts as a failure; the default classifies from ErrorKind + // protected override bool IsFailure(in FaultContext fault) => fault.IsFault; + } +} + +var options = new MultiGroupOptions +{ + CircuitBreaker = new ConsecutiveFailureBreaker(limit: 5) +}; +``` + +Keep `ObserveResult` cheap and thread-safe: it runs on the hot path for every completed message, and may be called concurrently. + +### Custom Retry Policies + +`RetryPolicy` is itself extensible: override `CanRetry(in FaultContext fault)` to make the retry decision yourself. It returns a `RetryPolicyResult` — `None` to give up, or a combination of `SameServer` and `FailoverServer` to indicate where a retry may be attempted. +The `FaultContext` gives you the classified `ErrorKind`, the `ConnectionFailureType`, and the command `Flags` (including its retry category) to base the decision on: + +```csharp +public sealed class ReadOnlyOnlyRetryPolicy : RetryPolicy +{ + public override RetryPolicyResult CanRetry(in FaultContext fault) + { + // only ever retry pure reads, and only on the same server + if (fault.ErrorKind == RedisErrorKind.Loading + && (fault.Flags & CommandFlags.CommandRetryReadOnly) != 0) + { + return RetryPolicyResult.SameServer; + } + // note: base.CanRetry(fault) would apply the default logic + return RetryPolicyResult.None; + } +} + +IDatabaseAsync db = conn.GetDatabase().WithRetry(new ReadOnlyOnlyRetryPolicy()); +``` diff --git a/src/StackExchange.Redis/Availability/DatabaseExtensions.cs b/src/StackExchange.Redis/Availability/DatabaseExtensions.cs new file mode 100644 index 000000000..9733bf485 --- /dev/null +++ b/src/StackExchange.Redis/Availability/DatabaseExtensions.cs @@ -0,0 +1,20 @@ +using System.Diagnostics.CodeAnalysis; +using RESPite; + +namespace StackExchange.Redis.Availability +{ + /// + /// Provides availability-related extension methods (such as ) to database instances. + /// + public static class DatabaseExtensions + { + /// + /// Automatically retry operations when connection failure occurs. This has deep integration with + /// SE.Redis concepts, so can respond to server failover events, apply circuit-breaker rules, and + /// respect command effect categorization. + /// + [Experimental(Experiments.ActiveActive, UrlFormat = Experiments.UrlFormat)] + public static IDatabaseAsync WithRetry(this IDatabaseAsync database, RetryPolicy retryPolicy) + => new RetryDatabase(database, retryPolicy); + } +} diff --git a/src/StackExchange.Redis/KeyspaceIsolation/DatabaseExtension.cs b/src/StackExchange.Redis/KeyspaceIsolation/DatabaseExtension.cs index a65be46d0..742bc06eb 100644 --- a/src/StackExchange.Redis/KeyspaceIsolation/DatabaseExtension.cs +++ b/src/StackExchange.Redis/KeyspaceIsolation/DatabaseExtension.cs @@ -1,8 +1,4 @@ using System; -using System.Diagnostics.CodeAnalysis; -using RESPite; -using StackExchange.Redis.Availability; -using StackExchange.Redis.Interfaces; namespace StackExchange.Redis.KeyspaceIsolation { @@ -67,14 +63,5 @@ public static IDatabase WithKeyPrefix(this IDatabase database, RedisKey keyPrefi return new KeyPrefixedDatabase(database, keyPrefix.AsPrefix()!); } - - /// - /// Automatically retry operations when connection failure occurs. This has deep integration with - /// SE.Redis concepts, so can respond to server failover events, apply circuit-breaker rules, and - /// respect command effect categorization. - /// - [Experimental(Experiments.ActiveActive, UrlFormat = Experiments.UrlFormat)] - public static IDatabaseAsync WithRetry(this IDatabaseAsync database, RetryPolicy retryPolicy) - => new RetryDatabase(database, retryPolicy); } } diff --git a/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt b/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt index 599eb6faf..47cf16bcf 100644 --- a/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt @@ -15,7 +15,8 @@ StackExchange.Redis.IDatabaseAsync.Database.get -> int [SER007]StackExchange.Redis.Availability.RetryPolicy.RetryDelay.set -> void [SER007]StackExchange.Redis.Availability.RetryPolicy.RetryPolicy() -> void StackExchange.Redis.IServer.InventKey(StackExchange.Redis.RedisKey prefix = default(StackExchange.Redis.RedisKey)) -> StackExchange.Redis.RedisKey -[SER007]static StackExchange.Redis.KeyspaceIsolation.DatabaseExtensions.WithRetry(this StackExchange.Redis.IDatabaseAsync! database, StackExchange.Redis.Availability.RetryPolicy! retryPolicy) -> StackExchange.Redis.IDatabaseAsync! +StackExchange.Redis.Availability.DatabaseExtensions +[SER007]static StackExchange.Redis.Availability.DatabaseExtensions.WithRetry(this StackExchange.Redis.IDatabaseAsync! database, StackExchange.Redis.Availability.RetryPolicy! retryPolicy) -> StackExchange.Redis.IDatabaseAsync! [SER007]StackExchange.Redis.Availability.CircuitBreaker [SER007]StackExchange.Redis.Availability.CircuitBreaker.Accumulator [SER007]StackExchange.Redis.Availability.CircuitBreaker.Accumulator.Accumulator() -> void diff --git a/tests/StackExchange.Redis.Tests/RetryTests/RetryEndToEndTests.cs b/tests/StackExchange.Redis.Tests/RetryTests/RetryEndToEndTests.cs index 393f8ff2c..e2b9583f7 100644 --- a/tests/StackExchange.Redis.Tests/RetryTests/RetryEndToEndTests.cs +++ b/tests/StackExchange.Redis.Tests/RetryTests/RetryEndToEndTests.cs @@ -2,7 +2,6 @@ using System.Net; using System.Threading.Tasks; using StackExchange.Redis.Availability; -using StackExchange.Redis.KeyspaceIsolation; using StackExchange.Redis.Server; using Xunit; From b5c0449de64c3b3267d09f2277243b26abbeeefb Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Fri, 17 Jul 2026 15:19:44 +0100 Subject: [PATCH 23/23] namespace --- docs/ActiveActive.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/ActiveActive.md b/docs/ActiveActive.md index f9341dcc3..daa49f826 100644 --- a/docs/ActiveActive.md +++ b/docs/ActiveActive.md @@ -9,6 +9,11 @@ The Active:Active feature provides automatic failover and intelligent routing ac 3. **Circuit breakers** that *passively* monitor availability from the observed success and failure of the traffic already flowing. 4. **Automatic retries** that let straightforward operations ride out a possibly-unstable connection — including silently transitioning to another endpoint during a failover, with no change to your calling code. +The features for Active:Active are available in the `Availability` sub-namespace: + +``` csharp +using StackExchange.Redis.Availability; +``` The library automatically selects the best available endpoint based on: 1. **Availability** - Connected endpoints are always preferred over disconnected ones