| | | 1 | | using Microsoft.Extensions.Diagnostics.HealthChecks; |
| | | 2 | | using System.Text.Json.Serialization; |
| | | 3 | | |
| | | 4 | | namespace AsyncResponse; |
| | | 5 | | |
| | | 6 | | /// <summary> |
| | | 7 | | /// Aggregate counts the recovery health check reports under the <c>stats</c> key of its |
| | | 8 | | /// <see cref="HealthCheckResult.Data"/>. A named type (with pinned JSON property names) rather |
| | | 9 | | /// than an anonymous one so trimmed/Native AOT apps can serialize health reports: register it — |
| | | 10 | | /// plus <see cref="AsyncResponseStaleRecoveryEntry"/> — in the app's |
| | | 11 | | /// <see cref="JsonSerializerContext"/> if the app writes health data as JSON. |
| | | 12 | | /// </summary> |
| | | 13 | | public sealed record AsyncResponseRecoveryStats( |
| | | 14 | | [property: JsonPropertyName("outstandingRegistrations")] int OutstandingRegistrations, |
| | | 15 | | [property: JsonPropertyName("withLiveWaiter")] int WithLiveWaiter, |
| | | 16 | | [property: JsonPropertyName("stale")] int Stale, |
| | | 17 | | [property: JsonPropertyName("unknownAge")] int UnknownAge, |
| | | 18 | | [property: JsonPropertyName("unprobeable")] int Unprobeable = 0); |
| | | 19 | | |
| | | 20 | | /// <summary> |
| | | 21 | | /// One stale recovery registration listed under the <c>staleEntries</c> key of the recovery |
| | | 22 | | /// health check's <see cref="HealthCheckResult.Data"/> (JSON names pinned; see |
| | | 23 | | /// <see cref="AsyncResponseRecoveryStats"/> for the AOT registration note). |
| | | 24 | | /// </summary> |
| | | 25 | | public sealed record AsyncResponseStaleRecoveryEntry( |
| | | 26 | | [property: JsonPropertyName("correlationId")] string? CorrelationId, |
| | | 27 | | [property: JsonPropertyName("payloadType")] string? PayloadType, |
| | | 28 | | [property: JsonPropertyName("registeredAtUtc")] DateTime? RegisteredAtUtc); |
| | | 29 | | |
| | | 30 | | /// <summary> |
| | | 31 | | /// Surfaces the async-response watchdog findings on the health endpoints (e.g. <c>/readyz</c>). |
| | | 32 | | /// <para> |
| | | 33 | | /// Reads the snapshot cached by <see cref="AsyncResponseWatchdogState"/> — probes never touch the |
| | | 34 | | /// recovery store. The check reports at most <see cref="HealthStatus.Degraded"/>: stale recovery |
| | | 35 | | /// state means business flows are likely stuck and need operator attention, but the process itself |
| | | 36 | | /// is fully able to serve traffic, so this check should never flip readiness to 503 and pull |
| | | 37 | | /// instances out of rotation (map <see cref="HealthStatus.Degraded"/> to HTTP 200 in your |
| | | 38 | | /// health-endpoint options, which is the ASP.NET Core default). |
| | | 39 | | /// </para> |
| | | 40 | | /// <list type="bullet"> |
| | | 41 | | /// <item><description><b>Healthy</b> — last scan found no stale entries; no scan has run yet but |
| | | 42 | | /// the watchdog is inside its first-scan budget (it starts with a delay, and readiness must not |
| | | 43 | | /// block on it); or this host's watchdog is deliberately idle (disabled, or the channel has no |
| | | 44 | | /// scanner — the multi-host pattern), reported as explicit data rather than an alert.</description></item> |
| | | 45 | | /// <item><description><b>Degraded</b> — stale entries exist, the last scan failed, the last scan |
| | | 46 | | /// was truncated at the buffer cap (its verdict covers a subset only), waiter liveness could not |
| | | 47 | | /// be probed for some entries (their staleness is unknown), or the watchdog stopped publishing |
| | | 48 | | /// (snapshot older than twice the scan interval, or no first snapshot after the startup delay |
| | | 49 | | /// plus twice the interval).</description></item> |
| | | 50 | | /// </list> |
| | | 51 | | /// </summary> |
| | 35 | 52 | | public sealed class AsyncResponseRecoveryHealthCheck(AsyncResponseWatchdogState _state, TimeProvider? _timeProvider = nu |
| | | 53 | | { |
| | | 54 | | /// <summary>Caps the number of stale entries listed in the health payload.</summary> |
| | | 55 | | private const int MaxReportedStaleEntries = 10; |
| | | 56 | | |
| | | 57 | | /// <summary>Runs the CheckHealthAsync operation.</summary> |
| | | 58 | | public Task<HealthCheckResult> CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = de |
| | 37 | 59 | | => Task.FromResult(Evaluate(_state.Latest, _state.Activation, (_timeProvider ?? TimeProvider.System).GetUtcNow() |
| | | 60 | | |
| | | 61 | | internal static HealthCheckResult Evaluate( |
| | | 62 | | AsyncResponseWatchdogSnapshot? snapshot, |
| | | 63 | | AsyncResponseWatchdogState.WatchdogActivation? activation, |
| | | 64 | | DateTime utcNow) |
| | | 65 | | { |
| | 41 | 66 | | if (snapshot is null) |
| | 17 | 67 | | return EvaluateBeforeFirstScan(activation, utcNow); |
| | | 68 | | |
| | 24 | 69 | | var snapshotAge = utcNow - snapshot.ScanCompletedUtc; |
| | 24 | 70 | | if (snapshotAge > snapshot.ScanInterval * 2) |
| | | 71 | | { |
| | 2 | 72 | | return HealthCheckResult.Degraded( |
| | 2 | 73 | | $"Async-response watchdog stopped reporting: last scan {snapshot.ScanCompletedUtc:u} is older than twice |
| | 2 | 74 | | data: BuildData(snapshot)); |
| | | 75 | | } |
| | | 76 | | |
| | 22 | 77 | | if (snapshot.Error is not null) |
| | | 78 | | { |
| | 4 | 79 | | return HealthCheckResult.Degraded( |
| | 4 | 80 | | $"Async-response watchdog scan failed: {snapshot.Error}", |
| | 4 | 81 | | data: BuildData(snapshot)); |
| | | 82 | | } |
| | | 83 | | |
| | 18 | 84 | | var report = snapshot.Report!; |
| | 18 | 85 | | if (report.StaleEntries.Count > 0) |
| | | 86 | | { |
| | 8 | 87 | | return HealthCheckResult.Degraded( |
| | 8 | 88 | | $"{report.StaleEntries.Count} async-response flow(s) look stuck: persisted recovery state with no live w |
| | 8 | 89 | | data: BuildData(snapshot)); |
| | | 90 | | } |
| | | 91 | | |
| | 10 | 92 | | if (report.Truncated) |
| | | 93 | | { |
| | | 94 | | // Zero stale entries in a truncated scan is not a verdict — arbitrarily many stale |
| | | 95 | | // entries can sit past the buffer cap. "The scan was incomplete" is a health fact. |
| | 2 | 96 | | return HealthCheckResult.Degraded( |
| | 2 | 97 | | "Async-response watchdog scan was truncated at the MaxScanEntries buffer cap; staleness was assessed for |
| | 2 | 98 | | data: BuildData(snapshot)); |
| | | 99 | | } |
| | | 100 | | |
| | 8 | 101 | | if (report.UnprobeableEntries > 0) |
| | | 102 | | { |
| | | 103 | | // Unknown liveness is never flagged stale (no false alarms), so without this a probe |
| | | 104 | | // outage would zero every counter and read as a clean pass the scan never computed. |
| | 4 | 105 | | return HealthCheckResult.Degraded( |
| | 4 | 106 | | $"Async-response watchdog could not probe waiter liveness for {report.UnprobeableEntries} of {report.Tot |
| | 4 | 107 | | data: BuildData(snapshot)); |
| | | 108 | | } |
| | | 109 | | |
| | 4 | 110 | | return HealthCheckResult.Healthy( |
| | 4 | 111 | | "No stale async-response recovery state.", |
| | 4 | 112 | | BuildData(snapshot)); |
| | | 113 | | } |
| | | 114 | | |
| | | 115 | | /// <summary> |
| | | 116 | | /// Attestation before any snapshot exists. Three honest answers instead of a blanket Healthy: |
| | | 117 | | /// a deliberately idle watchdog (disabled, or no scanner — the documented multi-host pattern) |
| | | 118 | | /// stays alert-quiet but says so in data; an armed scan loop is Healthy only inside its |
| | | 119 | | /// first-scan budget (startup delay plus two intervals) — past it the loop is as dead as one |
| | | 120 | | /// that stopped publishing, and "no scan yet" must not mask that forever; with no activation |
| | | 121 | | /// marker (watchdog not started, or none registered) there is no deadline to hold it to. |
| | | 122 | | /// </summary> |
| | | 123 | | private static HealthCheckResult EvaluateBeforeFirstScan( |
| | | 124 | | AsyncResponseWatchdogState.WatchdogActivation? activation, |
| | | 125 | | DateTime utcNow) |
| | | 126 | | { |
| | 17 | 127 | | if (activation is { Scanning: false }) |
| | | 128 | | { |
| | 4 | 129 | | return HealthCheckResult.Healthy( |
| | 4 | 130 | | $"Async-response watchdog on this host does not scan ({activation.IdleReason}); recovery staleness is at |
| | 4 | 131 | | new Dictionary<string, object> |
| | 4 | 132 | | { |
| | 4 | 133 | | ["scanning"] = false, |
| | 4 | 134 | | ["reason"] = activation.IdleReason ?? "unknown" |
| | 4 | 135 | | }); |
| | | 136 | | } |
| | | 137 | | |
| | 13 | 138 | | if (activation is { Scanning: true, StartedUtc: { } startedUtc }) |
| | | 139 | | { |
| | 9 | 140 | | var firstScanDueByUtc = startedUtc + activation.StartupDelay + 2 * activation.Interval; |
| | 9 | 141 | | var data = new Dictionary<string, object> |
| | 9 | 142 | | { |
| | 9 | 143 | | ["scanned"] = false, |
| | 9 | 144 | | ["scanning"] = true, |
| | 9 | 145 | | ["firstScanDueByUtc"] = firstScanDueByUtc |
| | 9 | 146 | | }; |
| | | 147 | | |
| | 9 | 148 | | if (utcNow > firstScanDueByUtc) |
| | | 149 | | { |
| | 4 | 150 | | return HealthCheckResult.Degraded( |
| | 4 | 151 | | $"Async-response watchdog never completed its first scan: it started {startedUtc:u} and published no |
| | 4 | 152 | | data: data); |
| | | 153 | | } |
| | | 154 | | |
| | 5 | 155 | | return HealthCheckResult.Healthy("Async-response watchdog has not completed a scan yet.", data); |
| | | 156 | | } |
| | | 157 | | |
| | 4 | 158 | | return HealthCheckResult.Healthy( |
| | 4 | 159 | | "Async-response watchdog has not completed a scan yet.", |
| | 4 | 160 | | new Dictionary<string, object> { ["scanned"] = false }); |
| | | 161 | | } |
| | | 162 | | |
| | | 163 | | private static Dictionary<string, object> BuildData(AsyncResponseWatchdogSnapshot snapshot) |
| | | 164 | | { |
| | 24 | 165 | | var data = new Dictionary<string, object> |
| | 24 | 166 | | { |
| | 24 | 167 | | ["lastScanUtc"] = snapshot.ScanCompletedUtc, |
| | 24 | 168 | | // Human-readable text plus a lossless numeric — alert math derived from a |
| | 24 | 169 | | // whole-minutes value reads 0 for any sub-minute interval. |
| | 24 | 170 | | ["scanInterval"] = snapshot.ScanInterval.ToString(), |
| | 24 | 171 | | ["scanIntervalSeconds"] = snapshot.ScanInterval.TotalSeconds |
| | 24 | 172 | | }; |
| | | 173 | | |
| | 24 | 174 | | if (snapshot.Report is not { } report) |
| | 4 | 175 | | return data; |
| | | 176 | | |
| | 20 | 177 | | data["truncated"] = report.Truncated; |
| | | 178 | | |
| | 20 | 179 | | data["stats"] = new AsyncResponseRecoveryStats( |
| | 20 | 180 | | report.TotalEntries, |
| | 20 | 181 | | report.EntriesWithActiveWaiter, |
| | 20 | 182 | | report.StaleEntries.Count, |
| | 20 | 183 | | report.UnknownAgeEntries, |
| | 20 | 184 | | report.UnprobeableEntries); |
| | | 185 | | |
| | 20 | 186 | | if (report.StaleEntries.Count > 0) |
| | | 187 | | { |
| | 8 | 188 | | data["staleEntries"] = report.StaleEntries |
| | 8 | 189 | | .Take(MaxReportedStaleEntries) |
| | 26 | 190 | | .Select(e => new AsyncResponseStaleRecoveryEntry( |
| | 26 | 191 | | e.CorrelationId, |
| | 26 | 192 | | e.PayloadTypeFullName, |
| | 26 | 193 | | e.RegisteredAtUtc)) |
| | 8 | 194 | | .ToList(); |
| | | 195 | | |
| | 8 | 196 | | if (report.StaleEntries.Count > MaxReportedStaleEntries) |
| | 2 | 197 | | data["staleEntriesTruncated"] = report.StaleEntries.Count - MaxReportedStaleEntries; |
| | | 198 | | } |
| | | 199 | | |
| | 20 | 200 | | return data; |
| | | 201 | | } |
| | | 202 | | } |