| | | 1 | | using Microsoft.Extensions.Logging; |
| | | 2 | | using System.Text.Json; |
| | | 3 | | using System.Threading.Channels; |
| | | 4 | | |
| | | 5 | | namespace AsyncResponse.Transports; |
| | | 6 | | |
| | | 7 | | // Shared source for the database-backed worker transports (PostgreSQL, SQL Server, MongoDB), |
| | | 8 | | // mirroring src/Channels/Shared/DbChannelShared.cs: each transport csproj pulls this file in via |
| | | 9 | | // <Compile Include="..\Shared\DbTransportShared.cs" />, so the base class compiles INTO each |
| | | 10 | | // provider assembly against that provider's concrete seam types. The seam is bound per project |
| | | 11 | | // with global using aliases (declared at the top of the provider's MessageDispatcher file): |
| | | 12 | | // |
| | | 13 | | // DbTransportOptions -> the provider's transport options (e.g. PostgreSqlAsyncResponseTransportOptions) |
| | | 14 | | // DbSubscriberOptions -> the provider's subscriber options (e.g. PostgreSqlSubscriberOptions) |
| | | 15 | | // DbTransportDelivery -> the provider's claimed-delivery type (e.g. PostgreSqlTransportDelivery) |
| | | 16 | | // DbSubscriberRole -> the provider's subscriber-role enum |
| | | 17 | | // DbAckMode -> the provider's ack-mode enum |
| | | 18 | | // DbTransportOptionsValidator -> the provider's static options validator |
| | | 19 | | // DbBackgroundFailureContext -> the provider's OnBackgroundFailure context type |
| | | 20 | | // |
| | | 21 | | // Because the aliases resolve to concrete sealed types at compile time, delivery calls stay |
| | | 22 | | // direct — no interface dispatch on the per-message path. The only provider-specific inputs are |
| | | 23 | | // three display strings supplied by the derived constructor: the provider name rendered into log |
| | | 24 | | // messages, the queue-item noun ("row"/"document"), and the lowercase telemetry tag. Rendered log |
| | | 25 | | // output and activity tags are byte-identical to the pre-extraction per-provider sources. |
| | | 26 | | |
| | | 27 | | /// <summary> |
| | | 28 | | /// Applies acknowledgement, redelivery, and dead-letter policy to database transport deliveries: |
| | | 29 | | /// ack-after-handler with fenced lease renewal, opt-in early ACK behind a bounded in-process |
| | | 30 | | /// queue with drain-on-dispose, attempt-capped dead-lettering, and the consumer receive span. |
| | | 31 | | /// Derived dispatchers supply only the provider display name, queue-item noun, and telemetry tag. |
| | | 32 | | /// </summary> |
| | | 33 | | internal abstract class DbMessageDispatcherBase : IAsyncDisposable |
| | | 34 | | { |
| | | 35 | | private readonly Func<DbTransportDelivery, CancellationToken, Task> _handler; |
| | | 36 | | private readonly DbTransportOptions _options; |
| | | 37 | | private readonly DbSubscriberOptions _subscriberOptions; |
| | | 38 | | private readonly ILogger _logger; |
| | | 39 | | private readonly DbSubscriberRole _role; |
| | | 40 | | private readonly string _providerName; |
| | | 41 | | private readonly string _unitNoun; |
| | | 42 | | private readonly string _receiveActivityName; |
| | | 43 | | private readonly string _transportTag; |
| | | 44 | | private readonly string _roleTagName; |
| | | 45 | | private readonly string _ackModeTagName; |
| | | 46 | | private readonly TimeProvider _timeProvider; |
| | | 47 | | |
| | | 48 | | private readonly Channel<DbTransportDelivery>? _backgroundQueue; |
| | | 49 | | private readonly Task[]? _backgroundWorkers; |
| | | 50 | | private readonly CancellationTokenSource? _backgroundCts; |
| | | 51 | | |
| | 480 | 52 | | protected DbMessageDispatcherBase( |
| | 480 | 53 | | Func<DbTransportDelivery, CancellationToken, Task> handler, |
| | 480 | 54 | | DbTransportOptions options, |
| | 480 | 55 | | DbSubscriberOptions subscriberOptions, |
| | 480 | 56 | | ILogger logger, |
| | 480 | 57 | | DbSubscriberRole role, |
| | 480 | 58 | | string providerName, |
| | 480 | 59 | | string unitNoun, |
| | 480 | 60 | | string telemetryName, |
| | 480 | 61 | | TimeProvider? timeProvider = null) |
| | | 62 | | { |
| | 480 | 63 | | DbTransportOptionsValidator.ValidateSubscriber(options, subscriberOptions, role.ToString()); |
| | | 64 | | |
| | 478 | 65 | | _handler = handler; |
| | 478 | 66 | | _options = options; |
| | 478 | 67 | | _subscriberOptions = subscriberOptions; |
| | 478 | 68 | | _logger = logger; |
| | 478 | 69 | | _role = role; |
| | 478 | 70 | | _providerName = providerName; |
| | 478 | 71 | | _unitNoun = unitNoun; |
| | 478 | 72 | | _receiveActivityName = $"asyncresponse.{telemetryName}.receive"; |
| | 478 | 73 | | _transportTag = telemetryName; |
| | 478 | 74 | | _roleTagName = $"asyncresponse.{telemetryName}.role"; |
| | 478 | 75 | | _ackModeTagName = $"asyncresponse.{telemetryName}.ack_mode"; |
| | | 76 | | |
| | | 77 | | // Clocks the lease-renewal beat only (a test seam; the system clock when omitted). |
| | 478 | 78 | | _timeProvider = timeProvider ?? TimeProvider.System; |
| | | 79 | | |
| | 478 | 80 | | if (subscriberOptions.AckMode is DbAckMode.AckAfterEnqueue) |
| | | 81 | | { |
| | 40 | 82 | | _backgroundQueue = Channel.CreateBounded<DbTransportDelivery>(new BoundedChannelOptions(subscriberOptions.Ba |
| | 40 | 83 | | { |
| | 40 | 84 | | SingleReader = false, |
| | 40 | 85 | | SingleWriter = true, |
| | 40 | 86 | | FullMode = BoundedChannelFullMode.Wait |
| | 40 | 87 | | }); |
| | 40 | 88 | | _backgroundCts = new CancellationTokenSource(); |
| | 40 | 89 | | _backgroundWorkers = new Task[subscriberOptions.BackgroundWorkerCount]; |
| | 172 | 90 | | for (var i = 0; i < _backgroundWorkers.Length; i++) |
| | 92 | 91 | | _backgroundWorkers[i] = Task.Run(() => BackgroundWorkerLoopAsync(_backgroundCts.Token)); |
| | | 92 | | } |
| | 478 | 93 | | } |
| | | 94 | | |
| | | 95 | | /// <summary>Handles one claimed queue item.</summary> |
| | | 96 | | public async Task HandleAsync(DbTransportDelivery delivery, CancellationToken cancellationToken) |
| | | 97 | | { |
| | | 98 | | // Pre-execution cap, BEFORE either ack mode. HandleFailureAsync below is the only other |
| | | 99 | | // place the cap is consulted, and it runs only when the handler THREW — so a delivery that |
| | | 100 | | // ends any other way (the process dies mid-handler, the host is killed, the lease lapses |
| | | 101 | | // while the DB is unreachable at settlement) never reaches it. The claim already stamped |
| | | 102 | | // attempts+1, so the row comes back at attempts cap+1, cap+2, ... and would be executed |
| | | 103 | | // again every time: redelivered forever, killing each replica in turn, and never |
| | | 104 | | // dead-lettered — the opposite of what MaxDeliveryAttempts documents. Settlement uses |
| | | 105 | | // CancellationToken.None for the usual reason: burying a poison row must not be abandoned |
| | | 106 | | // half-done by a shutdown. Mirrors the Redis dispatcher's AlreadyExceededDeliveryAttempts. |
| | 528 | 107 | | var cap = _subscriberOptions.MaxDeliveryAttempts; |
| | 528 | 108 | | if (cap > 0 && delivery.Attempt > cap) |
| | | 109 | | { |
| | 8 | 110 | | _logger.LogError( |
| | 8 | 111 | | "{Provider} message on queue {Queue} ({Role}) arrived on attempt {Attempt} with a cap of {MaxDeliveryAtt |
| | 8 | 112 | | _providerName, |
| | 8 | 113 | | delivery.Queue, |
| | 8 | 114 | | _role, |
| | 8 | 115 | | delivery.Attempt, |
| | 8 | 116 | | cap); |
| | | 117 | | |
| | 8 | 118 | | var buried = await DeadLetterSwallowingFailureAsync( |
| | 8 | 119 | | delivery, |
| | 8 | 120 | | new InvalidOperationException( |
| | 8 | 121 | | $"Message exceeded {cap} delivery attempts without settling (attempt {delivery.Attempt})."), |
| | 8 | 122 | | deleteOriginal: true) |
| | 8 | 123 | | .ConfigureAwait(false); |
| | | 124 | | |
| | 8 | 125 | | if (!buried) |
| | | 126 | | { |
| | 4 | 127 | | _logger.LogWarning( |
| | 4 | 128 | | "{Provider} dead-letter publish failed for over-cap message on queue {Queue} ({Role}); releasing for |
| | 4 | 129 | | _providerName, |
| | 4 | 130 | | delivery.Queue, |
| | 4 | 131 | | _role); |
| | 4 | 132 | | await NakSwallowingFailureAsync(delivery).ConfigureAwait(false); |
| | | 133 | | } |
| | | 134 | | |
| | 8 | 135 | | return; |
| | | 136 | | } |
| | | 137 | | |
| | 520 | 138 | | if (_subscriberOptions.AckMode is DbAckMode.AckAfterEnqueue) |
| | | 139 | | { |
| | 66 | 140 | | await HandleEarlyAckAsync(delivery, cancellationToken).ConfigureAwait(false); |
| | 66 | 141 | | return; |
| | | 142 | | } |
| | | 143 | | |
| | | 144 | | try |
| | | 145 | | { |
| | | 146 | | // While the handler runs, a fenced heartbeat keeps extending the claim's lease at |
| | | 147 | | // LockTimeout/3 cadence so a slow handler does not let the lock lapse and a competing |
| | | 148 | | // subscriber re-claim (and duplicate-process) the queue item. The heartbeat MUST be |
| | | 149 | | // armed before any user code runs: a handler can burn its lease entirely |
| | | 150 | | // synchronously (CPU work or blocking I/O before its first await), and only an |
| | | 151 | | // already-armed beat — firing on a timer thread — renews under a blocked handler |
| | | 152 | | // thread. Teardown is exception-free (SuppressThrowing beat), so the always-armed |
| | | 153 | | // loop costs allocations per delivery, not a thrown TaskCanceledException. |
| | 454 | 154 | | using var renewalCancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); |
| | 454 | 155 | | var renewalTask = RenewLeaseLoopAsync(delivery, renewalCancellation.Token); |
| | | 156 | | try |
| | | 157 | | { |
| | 454 | 158 | | await ExecuteHandlerAsync(delivery, cancellationToken).ConfigureAwait(false); |
| | 430 | 159 | | } |
| | | 160 | | finally |
| | | 161 | | { |
| | 454 | 162 | | renewalCancellation.Cancel(); |
| | 454 | 163 | | ObserveRenewal(renewalTask); |
| | | 164 | | } |
| | 430 | 165 | | } |
| | 2 | 166 | | catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) |
| | | 167 | | { |
| | | 168 | | // Host shutdown, not a handler failure: NAK would burn an attempt and dead-letter |
| | | 169 | | // would bury healthy work once the cap is reached. Leave the claim unsettled — the |
| | | 170 | | // lease lapses on its own and at-least-once redelivery applies after restart. |
| | 2 | 171 | | throw; |
| | | 172 | | } |
| | 22 | 173 | | catch (Exception ex) |
| | | 174 | | { |
| | 22 | 175 | | await HandleFailureAsync(delivery, ex).ConfigureAwait(false); |
| | 22 | 176 | | return; |
| | | 177 | | } |
| | | 178 | | |
| | | 179 | | // The ack runs outside the handler's try/catch: a transient ack failure after a |
| | | 180 | | // successful handler must not be misread as a handler failure — NAK/dead-letter here |
| | | 181 | | // would redeliver (or bury) work whose side effects already completed. Swallow and log |
| | | 182 | | // instead; the claim's lease lapses on its own and at-least-once redelivery applies. |
| | | 183 | | try |
| | | 184 | | { |
| | 430 | 185 | | await delivery.AckAsync().ConfigureAwait(false); |
| | 430 | 186 | | } |
| | 0 | 187 | | catch (Exception ex) |
| | | 188 | | { |
| | 0 | 189 | | _logger.LogWarning( |
| | 0 | 190 | | ex, |
| | 0 | 191 | | "Failed to ACK {Provider} message {MessageId} on queue {Queue} ({Role}) after a successful handler; the |
| | 0 | 192 | | _providerName, |
| | 0 | 193 | | delivery.Id, |
| | 0 | 194 | | delivery.Queue, |
| | 0 | 195 | | _role, |
| | 0 | 196 | | _unitNoun); |
| | 0 | 197 | | } |
| | 526 | 198 | | } |
| | | 199 | | |
| | | 200 | | private async Task RenewLeaseLoopAsync(DbTransportDelivery delivery, CancellationToken cancellationToken) |
| | | 201 | | { |
| | | 202 | | // A third of the lease, and a FAILED beat retries on a short backoff instead of waiting out |
| | | 203 | | // another full beat. At LockTimeout/2 with the retry one more beat away, the retry landed |
| | | 204 | | // at claim + LockTimeout — after locked_until, every time: ONE transient renew failure (a |
| | | 205 | | // command timeout, a broken pooled connection, a SQL Server 1205 deadlock victim) |
| | | 206 | | // guaranteed the lease lapsed, a peer claimed the row within its EmptyPollDelay, and a |
| | | 207 | | // healthy long handler ran twice concurrently. Now a failed beat leaves two thirds of the |
| | | 208 | | // lease for retries a second (or LockTimeout/10) apart. Both waits are floored at a |
| | | 209 | | // millisecond: Task.Delay truncates to whole milliseconds, and a zero wait would spin. |
| | 463 | 210 | | var interval = TimeSpan.FromTicks(Math.Max(TimeSpan.TicksPerMillisecond, _options.LockTimeout.Ticks / 3)); |
| | 463 | 211 | | var retryInterval = TimeSpan.FromTicks(Math.Max(TimeSpan.TicksPerMillisecond, Math.Min(TimeSpan.TicksPerSecond, |
| | 463 | 212 | | var wait = interval; |
| | 463 | 213 | | var failing = false; |
| | | 214 | | try |
| | | 215 | | { |
| | 14 | 216 | | while (true) |
| | | 217 | | { |
| | | 218 | | // Exception-free beat: the loop is cancelled once per delivery when the handler |
| | | 219 | | // finishes, and a thrown-and-caught TaskCanceledException per message dominated |
| | | 220 | | // the dispatch cost. SuppressThrowing observes the cancelled delay without |
| | | 221 | | // throwing; cancellation still disarms the underlying timer immediately. |
| | 491 | 222 | | await Task.Delay(wait, _timeProvider, cancellationToken).ConfigureAwait(ConfigureAwaitOptions.SuppressTh |
| | 491 | 223 | | if (cancellationToken.IsCancellationRequested) |
| | 453 | 224 | | return; // The handler finished or the subscriber is stopping. |
| | | 225 | | |
| | | 226 | | bool renewed; |
| | | 227 | | try |
| | | 228 | | { |
| | 38 | 229 | | renewed = await delivery.RenewAsync().ConfigureAwait(false); |
| | 24 | 230 | | } |
| | 14 | 231 | | catch (Exception ex) |
| | | 232 | | { |
| | 14 | 233 | | if (cancellationToken.IsCancellationRequested) |
| | 0 | 234 | | return; |
| | | 235 | | |
| | | 236 | | // Keep retrying past locked_until too: the renew is fenced on lock_id alone, |
| | | 237 | | // so until a peer actually re-claims the row a late renew still re-establishes |
| | | 238 | | // the lease. Only the first failure of a streak is a warning — at this |
| | | 239 | | // cadence a database outage would otherwise log one per second per in-flight |
| | | 240 | | // delivery. |
| | 14 | 241 | | _logger.Log( |
| | 14 | 242 | | failing ? LogLevel.Debug : LogLevel.Warning, |
| | 14 | 243 | | ex, |
| | 14 | 244 | | "Failed to renew the lease of {Provider} message {MessageId} on queue {Queue} ({Role}); retrying |
| | 14 | 245 | | _providerName, |
| | 14 | 246 | | delivery.Id, |
| | 14 | 247 | | delivery.Queue, |
| | 14 | 248 | | _role, |
| | 14 | 249 | | retryInterval); |
| | 14 | 250 | | failing = true; |
| | 14 | 251 | | wait = retryInterval; |
| | 14 | 252 | | continue; |
| | | 253 | | } |
| | | 254 | | |
| | | 255 | | // The beat is not joined before settlement (see ObserveRenewal), so a renew that |
| | | 256 | | // was in flight when the handler finished can land AFTER the fenced ack/NAK cleared |
| | | 257 | | // the row's lock_id. That "no match" is the settlement's own doing, not a lost lease. |
| | 24 | 258 | | if (cancellationToken.IsCancellationRequested) |
| | 6 | 259 | | return; |
| | | 260 | | |
| | 18 | 261 | | if (!renewed) |
| | | 262 | | { |
| | | 263 | | // The lock_id fence no longer matches: the lease expired and another subscriber |
| | | 264 | | // claimed the queue item. Stop renewing; the fenced ack/NAK will no-op for this |
| | | 265 | | // claim. |
| | 4 | 266 | | _logger.LogWarning( |
| | 4 | 267 | | "Lease of {Provider} message {MessageId} on queue {Queue} ({Role}) was lost; another subscriber |
| | 4 | 268 | | _providerName, |
| | 4 | 269 | | delivery.Id, |
| | 4 | 270 | | delivery.Queue, |
| | 4 | 271 | | _role); |
| | 4 | 272 | | return; |
| | | 273 | | } |
| | | 274 | | |
| | 14 | 275 | | failing = false; |
| | 14 | 276 | | wait = interval; |
| | | 277 | | } |
| | | 278 | | } |
| | 0 | 279 | | catch (OperationCanceledException) |
| | | 280 | | { |
| | | 281 | | // A cancellation surfacing through RenewAsync while the token fires; the beat wait |
| | | 282 | | // itself never throws. |
| | 0 | 283 | | } |
| | 463 | 284 | | } |
| | | 285 | | |
| | | 286 | | // Single choke point for handler execution so both ACK modes emit the consumer receive span. |
| | | 287 | | private async Task ExecuteHandlerAsync(DbTransportDelivery delivery, CancellationToken cancellationToken) |
| | | 288 | | { |
| | 506 | 289 | | using var activity = AsyncResponseDiagnostics.StartActivity( |
| | 506 | 290 | | _receiveActivityName, |
| | 506 | 291 | | System.Diagnostics.ActivityKind.Consumer); |
| | 506 | 292 | | activity?.SetTag("asyncresponse.transport", _transportTag); |
| | 506 | 293 | | activity?.SetTag(_roleTagName, _role.ToString()); |
| | 506 | 294 | | activity?.SetTag(_ackModeTagName, _subscriberOptions.AckMode.ToString()); |
| | 506 | 295 | | activity?.SetTag("messaging.system", _transportTag); |
| | 506 | 296 | | activity?.SetTag("messaging.destination.name", delivery.Queue); |
| | 506 | 297 | | activity?.SetTag("messaging.message.id", delivery.Id.ToString()); |
| | 506 | 298 | | activity?.SetTag("messaging.message.delivery_attempt", delivery.Attempt); |
| | | 299 | | |
| | 506 | 300 | | if (delivery.Headers.TryGetValue(_options.CorrelationIdHeader, out var correlationId)) |
| | 102 | 301 | | AsyncResponseDiagnostics.SetCorrelationId(activity, correlationId); |
| | | 302 | | |
| | | 303 | | try |
| | | 304 | | { |
| | 506 | 305 | | await _handler(delivery, cancellationToken).ConfigureAwait(false); |
| | 466 | 306 | | } |
| | 40 | 307 | | catch (Exception ex) |
| | | 308 | | { |
| | 40 | 309 | | AsyncResponseDiagnostics.SetError(activity, ex); |
| | 40 | 310 | | throw; |
| | | 311 | | } |
| | 466 | 312 | | } |
| | | 313 | | |
| | | 314 | | /// <summary> |
| | | 315 | | /// The cancelled lease-renewal heartbeat is NOT joined before settlement. Every settlement (ack, |
| | | 316 | | /// NAK, dead-letter) is fenced by <c>lock_id</c> in all three stores, so a beat still in flight |
| | | 317 | | /// is a no-op against it — while the in-flight renew pins <see cref="CancellationToken.None"/> |
| | | 318 | | /// for its connect and command, so a join held the ack behind a slow renew: a handler that had |
| | | 319 | | /// already SUCCEEDED waited on a degraded database until the lease it was trying to extend had |
| | | 320 | | /// lapsed, and the row was claimed and run again before its ack went out. (While the subscriber |
| | | 321 | | /// is stopping the wait was also up to <c>LockTimeout</c> of the host's stop budget, a term no |
| | | 322 | | /// shutdown validator sums.) The loop swallows its own faults; observe defensively and let the |
| | | 323 | | /// beat finish on its own. |
| | | 324 | | /// </summary> |
| | | 325 | | private static void ObserveRenewal(Task renewalTask) |
| | 463 | 326 | | => _ = renewalTask.ContinueWith( |
| | 0 | 327 | | static task => _ = task.Exception, |
| | 463 | 328 | | CancellationToken.None, |
| | 463 | 329 | | TaskContinuationOptions.OnlyOnFaulted | TaskContinuationOptions.ExecuteSynchronously, |
| | 463 | 330 | | TaskScheduler.Default); |
| | | 331 | | |
| | | 332 | | private async Task HandleEarlyAckAsync(DbTransportDelivery delivery, CancellationToken cancellationToken) |
| | | 333 | | { |
| | 66 | 334 | | if (!_backgroundQueue!.Writer.TryWrite(delivery)) |
| | | 335 | | { |
| | | 336 | | // Saturated: wait for a worker to free a slot instead of NAKing. The subscriber loop |
| | | 337 | | // treats every claimed row as progress and re-claims immediately, so NAK-on-full spins |
| | | 338 | | // at full database rate — one claim plus one NAK round trip per queued row, each NAK |
| | | 339 | | // burning an attempt (and on PostgreSQL notifying the whole fleet to come do the same). |
| | | 340 | | // Parking here pauses the claim loop, which is the actual backpressure (mirrors the |
| | | 341 | | // RabbitMQ/Kafka/NATS pause); the queue is built with FullMode.Wait for exactly this. |
| | 9 | 342 | | _logger.LogDebug("Background queue full for {Provider} {Role}; pausing the claim loop until capacity frees." |
| | | 343 | | |
| | | 344 | | // The park is unbounded by design, but the claim's lease is not — and in early-ACK |
| | | 345 | | // mode the inline path's heartbeat never runs, so nothing renews it. A park longer |
| | | 346 | | // than LockTimeout would let the lock lapse, a competing subscriber re-claim and run |
| | | 347 | | // the row, and this subscriber enqueue its own copy once the park completes: one job, |
| | | 348 | | // two concurrent executions, with the second ack's lock_id fence failing silently. |
| | | 349 | | // Arm the same fenced heartbeat as the inline path for exactly the park's duration. |
| | 9 | 350 | | using var renewalCancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); |
| | 9 | 351 | | var renewalTask = RenewLeaseLoopAsync(delivery, renewalCancellation.Token); |
| | | 352 | | try |
| | | 353 | | { |
| | 9 | 354 | | await _backgroundQueue.Writer.WriteAsync(delivery, cancellationToken).ConfigureAwait(false); |
| | 5 | 355 | | } |
| | 4 | 356 | | catch (Exception ex) when (ex is OperationCanceledException or ChannelClosedException) |
| | | 357 | | { |
| | | 358 | | // Subscriber stopping or dispatcher draining while parked: the delivery was never |
| | | 359 | | // enqueued, so release it promptly; if the NAK itself fails the lease lapses to |
| | | 360 | | // the same effect. |
| | | 361 | | try |
| | | 362 | | { |
| | 4 | 363 | | await delivery.NakAsync(_subscriberOptions.RedeliveryDelay).ConfigureAwait(false); |
| | 4 | 364 | | } |
| | 0 | 365 | | catch (Exception nakException) |
| | | 366 | | { |
| | 0 | 367 | | _logger.LogWarning( |
| | 0 | 368 | | nakException, |
| | 0 | 369 | | "Failed to NAK {Provider} message {MessageId} on queue {Queue} ({Role}) while stopping; the leas |
| | 0 | 370 | | _providerName, |
| | 0 | 371 | | delivery.Id, |
| | 0 | 372 | | delivery.Queue, |
| | 0 | 373 | | _role, |
| | 0 | 374 | | _unitNoun); |
| | 0 | 375 | | } |
| | | 376 | | |
| | 4 | 377 | | return; |
| | | 378 | | } |
| | | 379 | | finally |
| | | 380 | | { |
| | 9 | 381 | | renewalCancellation.Cancel(); |
| | 9 | 382 | | ObserveRenewal(renewalTask); |
| | | 383 | | } |
| | 5 | 384 | | } |
| | | 385 | | |
| | | 386 | | // Same rule as the post-handler ACK above: the delivery is already owned by a background |
| | | 387 | | // worker, so an ACK failure must not escape and tear down the subscriber — that would |
| | | 388 | | // drain the workers (running the handler) while the un-ACKed row is re-claimed and run |
| | | 389 | | // again. Swallow and log; the lease lapses and at-least-once redelivery applies. |
| | | 390 | | try |
| | | 391 | | { |
| | 62 | 392 | | await delivery.AckAsync().ConfigureAwait(false); |
| | 60 | 393 | | } |
| | 2 | 394 | | catch (Exception ex) |
| | | 395 | | { |
| | 2 | 396 | | _logger.LogWarning( |
| | 2 | 397 | | ex, |
| | 2 | 398 | | "Failed to ACK {Provider} message {MessageId} on queue {Queue} ({Role}) after enqueueing it for backgrou |
| | 2 | 399 | | _providerName, |
| | 2 | 400 | | delivery.Id, |
| | 2 | 401 | | delivery.Queue, |
| | 2 | 402 | | _role, |
| | 2 | 403 | | _unitNoun); |
| | 2 | 404 | | } |
| | 66 | 405 | | } |
| | | 406 | | |
| | | 407 | | private async Task BackgroundWorkerLoopAsync(CancellationToken cancellationToken) |
| | | 408 | | { |
| | | 409 | | // Token-less ReadAllAsync: on shutdown the queue is completed and fully drained, so every |
| | | 410 | | // already-ACKed queue item is accounted for instead of being silently dropped; each |
| | | 411 | | // failure is dead-lettered and surfaced below. |
| | 216 | 412 | | await foreach (var delivery in _backgroundQueue!.Reader.ReadAllAsync().ConfigureAwait(false)) |
| | | 413 | | { |
| | | 414 | | // Once the drain budget has lapsed, STOP executing (Redis/Pub-Sub parity). The token |
| | | 415 | | // below cannot stop the real handler — it is `_ingress.HandleWorkerMessageAsync(payload)`, |
| | | 416 | | // whose target takes no CancellationToken — so past the budget the loop kept starting |
| | | 417 | | // fresh work beyond the HostShutdownTimeout the options size, and every entry still |
| | | 418 | | // queued at process exit vanished with no record (its queue row was deleted by the |
| | | 419 | | // early ACK, so nothing redelivers it). Route the rest through the same |
| | | 420 | | // dead-letter/OnBackgroundFailure path instead of losing them silently. |
| | 62 | 421 | | if (cancellationToken.IsCancellationRequested) |
| | | 422 | | { |
| | 10 | 423 | | var lapsed = new OperationCanceledException( |
| | 10 | 424 | | "The ACK-after-enqueue drain budget lapsed before this already-ACKed message was handled."); |
| | | 425 | | |
| | 10 | 426 | | _logger.LogWarning( |
| | 10 | 427 | | "{Provider} background handler for already-ACKed message {MessageId} on queue {Queue} ({Role}) was n |
| | 10 | 428 | | _providerName, |
| | 10 | 429 | | delivery.Id, |
| | 10 | 430 | | delivery.Queue, |
| | 10 | 431 | | _role); |
| | | 432 | | |
| | 10 | 433 | | if (!await DeadLetterSwallowingFailureAsync(delivery, lapsed, deleteOriginal: false).ConfigureAwait(fals |
| | | 434 | | { |
| | 0 | 435 | | _logger.LogError( |
| | 0 | 436 | | "Failed to dead-letter undrained {Provider} message {MessageId} on queue {Queue} ({Role}); the l |
| | 0 | 437 | | _providerName, |
| | 0 | 438 | | delivery.Id, |
| | 0 | 439 | | delivery.Queue, |
| | 0 | 440 | | _role); |
| | | 441 | | } |
| | | 442 | | |
| | 10 | 443 | | await InvokeBackgroundFailureAsync(delivery, lapsed).ConfigureAwait(false); |
| | 10 | 444 | | continue; |
| | | 445 | | } |
| | | 446 | | |
| | | 447 | | try |
| | | 448 | | { |
| | 52 | 449 | | await ExecuteHandlerAsync(delivery, cancellationToken).ConfigureAwait(false); |
| | 36 | 450 | | } |
| | 16 | 451 | | catch (Exception ex) |
| | | 452 | | { |
| | 16 | 453 | | _logger.LogError(ex, "{Provider} background handler failed for {Role} on queue {Queue} after early ACK." |
| | 16 | 454 | | if (!await DeadLetterSwallowingFailureAsync(delivery, ex, deleteOriginal: false).ConfigureAwait(false)) |
| | | 455 | | { |
| | 4 | 456 | | _logger.LogError( |
| | 4 | 457 | | "Failed to dead-letter already-ACKed {Provider} message {MessageId} on queue {Queue} ({Role}); t |
| | 4 | 458 | | _providerName, |
| | 4 | 459 | | delivery.Id, |
| | 4 | 460 | | delivery.Queue, |
| | 4 | 461 | | _role); |
| | | 462 | | } |
| | | 463 | | |
| | 16 | 464 | | await InvokeBackgroundFailureAsync(delivery, ex).ConfigureAwait(false); |
| | 16 | 465 | | } |
| | 52 | 466 | | } |
| | 46 | 467 | | } |
| | | 468 | | |
| | | 469 | | private async Task HandleFailureAsync(DbTransportDelivery delivery, Exception exception) |
| | | 470 | | { |
| | 22 | 471 | | var maxAttempts = _subscriberOptions.MaxDeliveryAttempts; |
| | 22 | 472 | | if (maxAttempts > 0 && delivery.Attempt >= maxAttempts) |
| | | 473 | | { |
| | 11 | 474 | | _logger.LogError( |
| | 11 | 475 | | exception, |
| | 11 | 476 | | "{Provider} message on queue {Queue} ({Role}) failed after {Attempts} attempts; dead-lettering.", |
| | 11 | 477 | | _providerName, |
| | 11 | 478 | | delivery.Queue, |
| | 11 | 479 | | _role, |
| | 11 | 480 | | delivery.Attempt); |
| | | 481 | | |
| | | 482 | | // CancellationToken.None like every other settlement in this file: burying a poison |
| | | 483 | | // row must not be abandoned half-done by a shutdown — with the stopping token, a |
| | | 484 | | // handler failing on its LAST attempt during a stop had the burial aborted (the |
| | | 485 | | // store's connection/transaction calls throw on the cancelled token) and the row was |
| | | 486 | | // NAKed back instead of dead-lettered. |
| | 11 | 487 | | var deadLettered = await DeadLetterSwallowingFailureAsync(delivery, exception, deleteOriginal: true).Configu |
| | 11 | 488 | | if (!deadLettered) |
| | | 489 | | { |
| | 6 | 490 | | _logger.LogWarning(exception, "{Provider} dead-letter publish failed for queue {Queue} ({Role}); releasi |
| | 6 | 491 | | await NakSwallowingFailureAsync(delivery).ConfigureAwait(false); |
| | | 492 | | } |
| | | 493 | | } |
| | | 494 | | else |
| | | 495 | | { |
| | 11 | 496 | | _logger.LogWarning( |
| | 11 | 497 | | exception, |
| | 11 | 498 | | "{Provider} message on queue {Queue} ({Role}) failed on attempt {Attempt}; releasing for redelivery.", |
| | 11 | 499 | | _providerName, |
| | 11 | 500 | | delivery.Queue, |
| | 11 | 501 | | _role, |
| | 11 | 502 | | delivery.Attempt); |
| | 11 | 503 | | await NakSwallowingFailureAsync(delivery).ConfigureAwait(false); |
| | | 504 | | } |
| | 22 | 505 | | } |
| | | 506 | | |
| | | 507 | | // Burial with the same containment rule as NakSwallowingFailureAsync below: the delivery |
| | | 508 | | // contract says DeadLetterAsync returns false rather than throwing, but the stores' |
| | | 509 | | // DeadLetterEnabled = false branch runs its ack OUTSIDE their guarded region, so a transient |
| | | 510 | | // DB failure there escaped as a throw — out of HandleAsync, tearing the subscriber down (and, |
| | | 511 | | // from the drain loop, killing the background worker). A burial that throws is a burial that |
| | | 512 | | // failed: report false and let the caller's NAK / lease-lapse path apply. |
| | | 513 | | private async Task<bool> DeadLetterSwallowingFailureAsync(DbTransportDelivery delivery, Exception exception, bool de |
| | | 514 | | { |
| | | 515 | | try |
| | | 516 | | { |
| | 45 | 517 | | return await delivery.DeadLetterAsync(exception, deleteOriginal, CancellationToken.None).ConfigureAwait(fals |
| | | 518 | | } |
| | 4 | 519 | | catch (Exception ex) |
| | | 520 | | { |
| | 4 | 521 | | _logger.LogWarning( |
| | 4 | 522 | | ex, |
| | 4 | 523 | | "Failed to dead-letter {Provider} message {MessageId} on queue {Queue} ({Role}); treating the burial as |
| | 4 | 524 | | _providerName, |
| | 4 | 525 | | delivery.Id, |
| | 4 | 526 | | delivery.Queue, |
| | 4 | 527 | | _role); |
| | 4 | 528 | | return false; |
| | | 529 | | } |
| | 45 | 530 | | } |
| | | 531 | | |
| | | 532 | | // Same rule as the post-handler ACK above: the handler's outcome is already decided, so a |
| | | 533 | | // transient NAK failure must not escape HandleAsync and tear down the subscriber — that would |
| | | 534 | | // dispose the dispatcher mid-flight and dead-letter unrelated already-ACKed background work on |
| | | 535 | | // the way down. Swallow and log; the claim's lease lapses on its own and at-least-once |
| | | 536 | | // redelivery applies either way. |
| | | 537 | | private async Task NakSwallowingFailureAsync(DbTransportDelivery delivery) |
| | | 538 | | { |
| | | 539 | | try |
| | | 540 | | { |
| | 21 | 541 | | await delivery.NakAsync(_subscriberOptions.RedeliveryDelay).ConfigureAwait(false); |
| | 17 | 542 | | } |
| | 4 | 543 | | catch (Exception ex) |
| | | 544 | | { |
| | 4 | 545 | | _logger.LogWarning( |
| | 4 | 546 | | ex, |
| | 4 | 547 | | "Failed to NAK {Provider} message {MessageId} on queue {Queue} ({Role}) after a failed handler; the leas |
| | 4 | 548 | | _providerName, |
| | 4 | 549 | | delivery.Id, |
| | 4 | 550 | | delivery.Queue, |
| | 4 | 551 | | _role, |
| | 4 | 552 | | _unitNoun); |
| | 4 | 553 | | } |
| | 21 | 554 | | } |
| | | 555 | | |
| | | 556 | | private async Task InvokeBackgroundFailureAsync(DbTransportDelivery delivery, Exception exception) |
| | | 557 | | { |
| | 26 | 558 | | if (_subscriberOptions.OnBackgroundFailure is null) |
| | 6 | 559 | | return; |
| | | 560 | | |
| | | 561 | | try |
| | | 562 | | { |
| | 20 | 563 | | delivery.Headers.TryGetValue(_options.CorrelationIdHeader, out var correlationId); |
| | 20 | 564 | | var context = new DbBackgroundFailureContext(delivery.Queue, _role.ToString(), delivery.Attempt, correlation |
| | 20 | 565 | | await _subscriberOptions.OnBackgroundFailure(context).ConfigureAwait(false); |
| | 18 | 566 | | } |
| | 2 | 567 | | catch (Exception ex) |
| | | 568 | | { |
| | 2 | 569 | | _logger.LogError(ex, "{Provider} OnBackgroundFailure callback threw for {Role}.", _providerName, _role); |
| | 2 | 570 | | } |
| | 26 | 571 | | } |
| | | 572 | | |
| | | 573 | | /// <inheritdoc /> |
| | | 574 | | public async ValueTask DisposeAsync() |
| | | 575 | | { |
| | 458 | 576 | | if (_backgroundQueue is null) |
| | 418 | 577 | | return; |
| | | 578 | | |
| | 40 | 579 | | _backgroundQueue.Writer.TryComplete(); |
| | | 580 | | |
| | | 581 | | // BackgroundDrainTimeout is the whole spend the shutdown-budget validator sums for this |
| | | 582 | | // dispatcher, so it is split rather than exceeded: most of it lets queued and running |
| | | 583 | | // handlers finish, and the rest is RESERVED for the post-lapse routing the worker loop |
| | | 584 | | // performs once cancelled (dead-letter + OnBackgroundFailure for every entry still |
| | | 585 | | // queued). That routing used to be fire-and-forget with no budget at all: DisposeAsync |
| | | 586 | | // returned, the subscriber and then the host finished stopping, and the already-ACKed |
| | | 587 | | // entries the workers were only starting to bury vanished with no record — the very |
| | | 588 | | // loss docs/transport-semantics.md promises this path prevents. |
| | 40 | 589 | | var routingReserve = TimeSpan.FromTicks(_subscriberOptions.BackgroundDrainTimeout.Ticks / 4); |
| | 40 | 590 | | var drainBudget = _subscriberOptions.BackgroundDrainTimeout - routingReserve; |
| | | 591 | | try |
| | | 592 | | { |
| | 40 | 593 | | await Task.WhenAll(_backgroundWorkers!).WaitAsync(drainBudget).ConfigureAwait(false); |
| | 30 | 594 | | _backgroundCts!.Dispose(); |
| | 30 | 595 | | } |
| | | 596 | | catch (TimeoutException) |
| | | 597 | | { |
| | 10 | 598 | | _logger.LogWarning("{Provider} background handlers for {Role} did not drain within {Timeout}; dead-lettering |
| | 10 | 599 | | await _backgroundCts!.CancelAsync().ConfigureAwait(false); |
| | | 600 | | |
| | | 601 | | try |
| | | 602 | | { |
| | 10 | 603 | | await Task.WhenAll(_backgroundWorkers!).WaitAsync(routingReserve).ConfigureAwait(false); |
| | 6 | 604 | | } |
| | 4 | 605 | | catch (TimeoutException) |
| | | 606 | | { |
| | 4 | 607 | | _logger.LogError( |
| | 4 | 608 | | "{Provider} background workers for {Role} did not finish dead-lettering the undrained entries within |
| | 4 | 609 | | _providerName, |
| | 4 | 610 | | _role, |
| | 4 | 611 | | routingReserve); |
| | | 612 | | |
| | | 613 | | // The workers are still running and observe _backgroundCts.Token inside ReadAllAsync, so disposing |
| | | 614 | | // it now would throw ObjectDisposedException inside them. Dispose once they actually finish, off |
| | | 615 | | // the shutdown path, so the source is not leaked either. |
| | 4 | 616 | | _ = Task.WhenAll(_backgroundWorkers!).ContinueWith( |
| | 4 | 617 | | _ => _backgroundCts.Dispose(), |
| | 4 | 618 | | CancellationToken.None, |
| | 4 | 619 | | TaskContinuationOptions.ExecuteSynchronously, |
| | 4 | 620 | | TaskScheduler.Default); |
| | 4 | 621 | | return; |
| | | 622 | | } |
| | 0 | 623 | | catch (Exception ex) |
| | | 624 | | { |
| | 0 | 625 | | _logger.LogDebug(ex, "{Provider} background worker drain for {Role} ended with an error.", _providerName |
| | 0 | 626 | | } |
| | | 627 | | |
| | | 628 | | // WhenAll completed one way or the other, so every worker has finished. |
| | 6 | 629 | | _backgroundCts.Dispose(); |
| | | 630 | | } |
| | 0 | 631 | | catch (Exception ex) |
| | | 632 | | { |
| | | 633 | | // WhenAll only completes once every worker has finished, so the source is safe to dispose here. |
| | 0 | 634 | | _logger.LogDebug(ex, "{Provider} background worker drain for {Role} ended with an error.", _providerName, _r |
| | 0 | 635 | | _backgroundCts!.Dispose(); |
| | 0 | 636 | | } |
| | 458 | 637 | | } |
| | | 638 | | } |
| | | 639 | | |
| | | 640 | | /// <summary> |
| | | 641 | | /// Extracts the AsyncResponse correlation id from the queue item's metadata first, then from the |
| | | 642 | | /// JSON response body via configured paths (walked by the shared <see cref="CorrelationIdJsonPaths"/>, |
| | | 643 | | /// same as the broker transports). Shared verbatim by the three database transports — the header |
| | | 644 | | /// name and JSON paths both come from the aliased options type. |
| | | 645 | | /// </summary> |
| | | 646 | | internal static class DbCorrelationIdExtractor |
| | | 647 | | { |
| | | 648 | | public static string? Extract( |
| | | 649 | | IReadOnlyDictionary<string, string>? headers, |
| | | 650 | | string messageJson, |
| | | 651 | | DbTransportOptions options) |
| | | 652 | | { |
| | | 653 | | var headerName = DbTransportOptionsValidator.Required(options.CorrelationIdHeader, nameof(options.CorrelationIdH |
| | | 654 | | if (headers is not null && headers.TryGetValue(headerName, out var headerValue) && !string.IsNullOrWhiteSpace(he |
| | | 655 | | return headerValue; |
| | | 656 | | |
| | | 657 | | return CorrelationIdJsonPaths.Extract(messageJson, options.CorrelationIdJsonPaths); |
| | | 658 | | } |
| | | 659 | | } |
| | | 660 | | |
| | | 661 | | /// <summary> |
| | | 662 | | /// Materializes a claimed queue item's <c>headers_json</c> without rejecting ANY content the |
| | | 663 | | /// column can legally hold. This runs after the claim already committed <c>attempts+1</c>/<c>lock_id</c> |
| | | 664 | | /// and before any delivery object exists, so a throw here (a wrong-typed value, a non-object root, |
| | | 665 | | /// malformed text in an unchecked column) could never reach the failure handler or dead-letter: |
| | | 666 | | /// an unkillable poison row that tears down the subscriber on every re-claim. Instead, string |
| | | 667 | | /// values are taken as-is, scalars keep their raw JSON text (culture-free by construction), |
| | | 668 | | /// object/array values keep their raw JSON so correlation extraction still sees a usable string, |
| | | 669 | | /// nulls are skipped — as is a header whose name or string value cannot be transcoded (an escaped |
| | | 670 | | /// lone surrogate) — and anything unusable degrades to no headers — a genuinely poison message |
| | | 671 | | /// then fails in the handler and flows through the NORMAL dead-letter path. Keys differing only |
| | | 672 | | /// in case (legal JSON from foreign producers) are last-wins, matching the ASB/SQS receive |
| | | 673 | | /// adapters. |
| | | 674 | | /// </summary> |
| | | 675 | | internal static class DbTransportHeaders |
| | | 676 | | { |
| | | 677 | | public static IReadOnlyDictionary<string, string> Materialize(string json) |
| | | 678 | | { |
| | | 679 | | JsonDocument document; |
| | | 680 | | try |
| | | 681 | | { |
| | | 682 | | document = JsonDocument.Parse(json); |
| | | 683 | | } |
| | | 684 | | catch (JsonException) |
| | | 685 | | { |
| | | 686 | | return Empty; |
| | | 687 | | } |
| | | 688 | | |
| | | 689 | | using (document) |
| | | 690 | | { |
| | | 691 | | if (document.RootElement.ValueKind is not JsonValueKind.Object) |
| | | 692 | | return Empty; |
| | | 693 | | |
| | | 694 | | var headers = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase); |
| | | 695 | | foreach (var property in document.RootElement.EnumerateObject()) |
| | | 696 | | { |
| | | 697 | | try |
| | | 698 | | { |
| | | 699 | | var value = property.Value.ValueKind switch |
| | | 700 | | { |
| | | 701 | | JsonValueKind.String => property.Value.GetString(), |
| | | 702 | | JsonValueKind.Null or JsonValueKind.Undefined => null, |
| | | 703 | | _ => property.Value.GetRawText() |
| | | 704 | | }; |
| | | 705 | | if (value is not null) |
| | | 706 | | headers[property.Name] = value; |
| | | 707 | | } |
| | | 708 | | catch (InvalidOperationException) |
| | | 709 | | { |
| | | 710 | | // An ESCAPED lone surrogate ("\ud800") parses — it is well-formed JSON — but has |
| | | 711 | | // no UTF-16 string form, so GetString/Name throw InvalidOperationException, not |
| | | 712 | | // the JsonException guarded above: the same after-the-claim, before-any-delivery |
| | | 713 | | // throw this type exists to prevent. The header is unusable; skip it and keep |
| | | 714 | | // the rest. |
| | | 715 | | } |
| | | 716 | | } |
| | | 717 | | |
| | | 718 | | return headers; |
| | | 719 | | } |
| | | 720 | | } |
| | | 721 | | |
| | | 722 | | private static readonly IReadOnlyDictionary<string, string> Empty = |
| | | 723 | | new Dictionary<string, string>(0, StringComparer.OrdinalIgnoreCase); |
| | | 724 | | } |