| | | 1 | | using Microsoft.Extensions.Diagnostics.HealthChecks; |
| | | 2 | | using System.Text.Json.Serialization; |
| | | 3 | | |
| | | 4 | | namespace AsyncResponse; |
| | | 5 | | |
| | | 6 | | /// <summary> |
| | | 7 | | /// Aggregate counts the recovery health check reports under the <c>stats</c> key of its |
| | | 8 | | /// <see cref="HealthCheckResult.Data"/>. A named type (with pinned JSON property names) rather |
| | | 9 | | /// than an anonymous one so trimmed/Native AOT apps can serialize health reports: register it — |
| | | 10 | | /// plus <see cref="AsyncResponseStaleRecoveryEntry"/> — in the app's |
| | | 11 | | /// <see cref="JsonSerializerContext"/> if the app writes health data as JSON. |
| | | 12 | | /// </summary> |
| | | 13 | | public sealed record AsyncResponseRecoveryStats( |
| | | 14 | | [property: JsonPropertyName("outstandingRegistrations")] int OutstandingRegistrations, |
| | | 15 | | [property: JsonPropertyName("withLiveWaiter")] int WithLiveWaiter, |
| | | 16 | | [property: JsonPropertyName("stale")] int Stale, |
| | | 17 | | [property: JsonPropertyName("unknownAge")] int UnknownAge); |
| | | 18 | | |
| | | 19 | | /// <summary> |
| | | 20 | | /// One stale recovery registration listed under the <c>staleEntries</c> key of the recovery |
| | | 21 | | /// health check's <see cref="HealthCheckResult.Data"/> (JSON names pinned; see |
| | | 22 | | /// <see cref="AsyncResponseRecoveryStats"/> for the AOT registration note). |
| | | 23 | | /// </summary> |
| | 3 | 24 | | public sealed record AsyncResponseStaleRecoveryEntry( |
| | 3 | 25 | | [property: JsonPropertyName("correlationId")] string? CorrelationId, |
| | 3 | 26 | | [property: JsonPropertyName("payloadType")] string? PayloadType, |
| | 3 | 27 | | [property: JsonPropertyName("registeredAtUtc")] DateTime? RegisteredAtUtc); |
| | | 28 | | |
| | | 29 | | /// <summary> |
| | | 30 | | /// Surfaces the async-response watchdog findings on the health endpoints (e.g. <c>/readyz</c>). |
| | | 31 | | /// <para> |
| | | 32 | | /// Reads the snapshot cached by <see cref="AsyncResponseWatchdogState"/> — probes never touch the |
| | | 33 | | /// recovery store. The check reports at most <see cref="HealthStatus.Degraded"/>: stale recovery |
| | | 34 | | /// state means business flows are likely stuck and need operator attention, but the process itself |
| | | 35 | | /// is fully able to serve traffic, so this check should never flip readiness to 503 and pull |
| | | 36 | | /// instances out of rotation (map <see cref="HealthStatus.Degraded"/> to HTTP 200 in your |
| | | 37 | | /// health-endpoint options, which is the ASP.NET Core default). |
| | | 38 | | /// </para> |
| | | 39 | | /// <list type="bullet"> |
| | | 40 | | /// <item><description><b>Healthy</b> — last scan found no stale entries (or no scan has run yet: |
| | | 41 | | /// the watchdog starts with a delay, and readiness must not block on it).</description></item> |
| | | 42 | | /// <item><description><b>Degraded</b> — stale entries exist, the last scan failed, the last scan |
| | | 43 | | /// was truncated at the buffer cap (its verdict covers a subset only), or the watchdog stopped |
| | | 44 | | /// publishing (snapshot older than twice the scan interval).</description></item> |
| | | 45 | | /// </list> |
| | | 46 | | /// </summary> |
| | | 47 | | public sealed class AsyncResponseRecoveryHealthCheck(AsyncResponseWatchdogState _state) : IHealthCheck |
| | | 48 | | { |
| | | 49 | | /// <summary>Caps the number of stale entries listed in the health payload.</summary> |
| | | 50 | | private const int MaxReportedStaleEntries = 10; |
| | | 51 | | |
| | | 52 | | /// <summary>Runs the CheckHealthAsync operation.</summary> |
| | | 53 | | public Task<HealthCheckResult> CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = de |
| | | 54 | | => Task.FromResult(Evaluate(_state.Latest, DateTime.UtcNow)); |
| | | 55 | | |
| | | 56 | | internal static HealthCheckResult Evaluate(AsyncResponseWatchdogSnapshot? snapshot, DateTime utcNow) |
| | | 57 | | { |
| | | 58 | | if (snapshot is null) |
| | | 59 | | { |
| | | 60 | | return HealthCheckResult.Healthy( |
| | | 61 | | "Async-response watchdog has not completed a scan yet.", |
| | | 62 | | new Dictionary<string, object> { ["scanned"] = false }); |
| | | 63 | | } |
| | | 64 | | |
| | | 65 | | var snapshotAge = utcNow - snapshot.ScanCompletedUtc; |
| | | 66 | | if (snapshotAge > snapshot.ScanInterval * 2) |
| | | 67 | | { |
| | | 68 | | return HealthCheckResult.Degraded( |
| | | 69 | | $"Async-response watchdog stopped reporting: last scan {snapshot.ScanCompletedUtc:u} is older than twice |
| | | 70 | | data: BuildData(snapshot)); |
| | | 71 | | } |
| | | 72 | | |
| | | 73 | | if (snapshot.Error is not null) |
| | | 74 | | { |
| | | 75 | | return HealthCheckResult.Degraded( |
| | | 76 | | $"Async-response watchdog scan failed: {snapshot.Error}", |
| | | 77 | | data: BuildData(snapshot)); |
| | | 78 | | } |
| | | 79 | | |
| | | 80 | | var report = snapshot.Report!; |
| | | 81 | | if (report.StaleEntries.Count > 0) |
| | | 82 | | { |
| | | 83 | | return HealthCheckResult.Degraded( |
| | | 84 | | $"{report.StaleEntries.Count} async-response flow(s) look stuck: persisted recovery state with no live w |
| | | 85 | | data: BuildData(snapshot)); |
| | | 86 | | } |
| | | 87 | | |
| | | 88 | | if (report.Truncated) |
| | | 89 | | { |
| | | 90 | | // Zero stale entries in a truncated scan is not a verdict — arbitrarily many stale |
| | | 91 | | // entries can sit past the buffer cap. "The scan was incomplete" is a health fact. |
| | | 92 | | return HealthCheckResult.Degraded( |
| | | 93 | | "Async-response watchdog scan was truncated at the MaxScanEntries buffer cap; staleness was assessed for |
| | | 94 | | data: BuildData(snapshot)); |
| | | 95 | | } |
| | | 96 | | |
| | | 97 | | return HealthCheckResult.Healthy( |
| | | 98 | | "No stale async-response recovery state.", |
| | | 99 | | BuildData(snapshot)); |
| | | 100 | | } |
| | | 101 | | |
| | | 102 | | private static Dictionary<string, object> BuildData(AsyncResponseWatchdogSnapshot snapshot) |
| | | 103 | | { |
| | | 104 | | var data = new Dictionary<string, object> |
| | | 105 | | { |
| | | 106 | | ["lastScanUtc"] = snapshot.ScanCompletedUtc, |
| | | 107 | | ["scanIntervalMinutes"] = (int)snapshot.ScanInterval.TotalMinutes |
| | | 108 | | }; |
| | | 109 | | |
| | | 110 | | if (snapshot.Report is not { } report) |
| | | 111 | | return data; |
| | | 112 | | |
| | | 113 | | data["truncated"] = report.Truncated; |
| | | 114 | | |
| | | 115 | | data["stats"] = new AsyncResponseRecoveryStats( |
| | | 116 | | report.TotalEntries, |
| | | 117 | | report.EntriesWithActiveWaiter, |
| | | 118 | | report.StaleEntries.Count, |
| | | 119 | | report.UnknownAgeEntries); |
| | | 120 | | |
| | | 121 | | if (report.StaleEntries.Count > 0) |
| | | 122 | | { |
| | | 123 | | data["staleEntries"] = report.StaleEntries |
| | | 124 | | .Take(MaxReportedStaleEntries) |
| | | 125 | | .Select(e => new AsyncResponseStaleRecoveryEntry( |
| | | 126 | | e.CorrelationId, |
| | | 127 | | e.PayloadTypeFullName, |
| | | 128 | | e.RegisteredAtUtc)) |
| | | 129 | | .ToList(); |
| | | 130 | | |
| | | 131 | | if (report.StaleEntries.Count > MaxReportedStaleEntries) |
| | | 132 | | data["staleEntriesTruncated"] = report.StaleEntries.Count - MaxReportedStaleEntries; |
| | | 133 | | } |
| | | 134 | | |
| | | 135 | | return data; |
| | | 136 | | } |
| | | 137 | | } |