| | | 1 | | namespace AsyncResponse; |
| | | 2 | | |
| | | 3 | | internal static class AsyncResponseRetry |
| | | 4 | | { |
| | | 5 | | /// <summary>Runs this background operation until cancellation is requested.</summary> |
| | | 6 | | public static async Task<T> ExecuteAsync<T>( |
| | | 7 | | Func<CancellationToken, Task<T>> action, |
| | | 8 | | Func<Exception, bool> isTransient, |
| | | 9 | | int maxAttempts, |
| | | 10 | | TimeSpan baseDelay, |
| | | 11 | | TimeSpan maxDelay, |
| | | 12 | | CancellationToken cancellationToken, |
| | | 13 | | TimeProvider? timeProvider = null) |
| | | 14 | | { |
| | 6331 | 15 | | ArgumentNullException.ThrowIfNull(action); |
| | 6331 | 16 | | ArgumentNullException.ThrowIfNull(isTransient); |
| | 6331 | 17 | | ArgumentOutOfRangeException.ThrowIfNegativeOrZero(maxAttempts); |
| | | 18 | | |
| | 6331 | 19 | | var attempt = 0; |
| | | 20 | | while (true) |
| | | 21 | | { |
| | 6441 | 22 | | cancellationToken.ThrowIfCancellationRequested(); |
| | 6441 | 23 | | attempt++; |
| | | 24 | | |
| | | 25 | | try |
| | | 26 | | { |
| | 6441 | 27 | | return await action(cancellationToken).ConfigureAwait(false); |
| | | 28 | | } |
| | | 29 | | // The filter deliberately tests only the cheap, throw-free attempt count. isTransient |
| | | 30 | | // is caller-supplied and runs in the BODY below: the CLR swallows an exception thrown |
| | | 31 | | // inside an exception filter and evaluates the filter as false, so a predicate that |
| | | 32 | | // faults (a null-deref on ex.InnerException, say) would silently reclassify every |
| | | 33 | | // retryable fault as permanent and burn the whole retry budget with its own bug |
| | | 34 | | // invisible in every log and trace. |
| | 191 | 35 | | catch (Exception ex) when (attempt < maxAttempts) |
| | | 36 | | { |
| | 165 | 37 | | if (!IsTransientOrThrow(isTransient, ex)) |
| | 53 | 38 | | throw; |
| | | 39 | | |
| | 110 | 40 | | await Task.Delay(Backoff(attempt, baseDelay, maxDelay), timeProvider ?? TimeProvider.System, cancellatio |
| | 110 | 41 | | } |
| | | 42 | | } |
| | 6250 | 43 | | } |
| | | 44 | | |
| | | 45 | | /// <summary> |
| | | 46 | | /// Evaluates the caller's transience predicate OUTSIDE an exception filter, so a predicate |
| | | 47 | | /// that throws surfaces instead of being silently read as "not transient". Both failures are |
| | | 48 | | /// raised together: the predicate's own fault (the bug to fix) and the exception it was |
| | | 49 | | /// judging (the reason the retry ran at all), so neither is lost. Only reachable when the |
| | | 50 | | /// predicate itself is broken; a well-behaved one returns and this is a plain call. |
| | | 51 | | /// </summary> |
| | | 52 | | private static bool IsTransientOrThrow(Func<Exception, bool> isTransient, Exception ex) |
| | | 53 | | { |
| | | 54 | | try |
| | | 55 | | { |
| | 165 | 56 | | return isTransient(ex); |
| | | 57 | | } |
| | 2 | 58 | | catch (Exception predicateFailure) |
| | | 59 | | { |
| | 2 | 60 | | throw new AggregateException( |
| | 2 | 61 | | "The retry policy's isTransient predicate threw while classifying a fault. The predicate's own failure a |
| | 2 | 62 | | "exception it was judging are both attached; fix the predicate — until then no fault can be classified a |
| | 2 | 63 | | ex, |
| | 2 | 64 | | predicateFailure); |
| | | 65 | | } |
| | 163 | 66 | | } |
| | | 67 | | |
| | | 68 | | /// <summary>Computes the retry backoff delay: exponential with half-jitter.</summary> |
| | | 69 | | public static TimeSpan Backoff(int completedAttempts, TimeSpan baseDelay, TimeSpan maxDelay) |
| | | 70 | | { |
| | 412 | 71 | | var multiplier = 1 << Math.Min(Math.Max(completedAttempts, 1) - 1, 10); |
| | 412 | 72 | | var milliseconds = Math.Min(maxDelay.TotalMilliseconds, baseDelay.TotalMilliseconds * multiplier); |
| | | 73 | | |
| | | 74 | | // Half-jitter: keep at least half the exponential step so backoff still backs off, and |
| | | 75 | | // randomize the rest — a broker blip fails many waiters across many replicas at once, and |
| | | 76 | | // un-jittered exponential delays would send them all reconnecting in lockstep waves. |
| | 412 | 77 | | milliseconds = milliseconds / 2 + Random.Shared.NextDouble() * (milliseconds / 2); |
| | 412 | 78 | | return TimeSpan.FromMilliseconds(Math.Max(1, milliseconds)); |
| | | 79 | | } |
| | | 80 | | } |