| | | 1 | | using System.Diagnostics; |
| | | 2 | | using System.Diagnostics.Metrics; |
| | | 3 | | using System.Runtime.CompilerServices; |
| | | 4 | | |
| | | 5 | | namespace AsyncResponse; |
| | | 6 | | |
| | | 7 | | /// <summary> |
| | | 8 | | /// Provides diagnostic identifiers and sources for the AsyncResponse library. |
| | | 9 | | /// </summary> |
| | | 10 | | public static class AsyncResponseDiagnostics |
| | | 11 | | { |
| | | 12 | | /// <summary> |
| | | 13 | | /// The name of the OpenTelemetry <see cref="ActivitySource"/> used by AsyncResponse. |
| | | 14 | | /// Consumers can subscribe to this name to capture distributed traces. |
| | | 15 | | /// </summary> |
| | | 16 | | public const string ActivitySourceName = "AsyncResponse"; |
| | | 17 | | |
| | | 18 | | /// <summary> |
| | | 19 | | /// The shared OpenTelemetry <see cref="ActivitySource"/> instance for the AsyncResponse library. |
| | | 20 | | /// </summary> |
| | 17 | 21 | | internal static readonly ActivitySource ActivitySource = new(ActivitySourceName); |
| | | 22 | | |
| | | 23 | | /// <summary> |
| | | 24 | | /// The name of the <see cref="System.Diagnostics.Metrics.Meter"/> used by AsyncResponse. |
| | | 25 | | /// Consumers subscribe to this name (e.g. via OpenTelemetry) to collect the library's metrics. |
| | | 26 | | /// </summary> |
| | | 27 | | public const string MeterName = "AsyncResponse"; |
| | | 28 | | |
| | | 29 | | /// <summary>The shared <see cref="System.Diagnostics.Metrics.Meter"/> for the AsyncResponse library.</summary> |
| | 17 | 30 | | internal static readonly Meter Meter = new(MeterName); |
| | | 31 | | |
| | | 32 | | // The core SLO instrument: every time a response or exception is published with no live waiter, |
| | | 33 | | // the recovery path is entered. Tags break it down by kind (response/exception), route |
| | | 34 | | // (resume/failure/unclassified), and whether a recovery callback actually ran — so an operator |
| | | 35 | | // can graph "how often does recovery fire" and the resume-vs-fail split that this library exists |
| | | 36 | | // to provide. |
| | 17 | 37 | | private static readonly Counter<long> LostSubscriberDispatches = |
| | 17 | 38 | | Meter.CreateCounter<long>("asyncresponse.lost_subscriber.dispatches", unit: "{dispatch}", |
| | 17 | 39 | | description: "Responses/exceptions published with no live subscriber (the recovery path was entered)."); |
| | | 40 | | |
| | 17 | 41 | | private static readonly Counter<long> WaiterTimeoutsCounter = |
| | 17 | 42 | | Meter.CreateCounter<long>("asyncresponse.waiter.timeouts", unit: "{timeout}", |
| | 17 | 43 | | description: "Waiters that faulted because no response arrived before the timeout."); |
| | | 44 | | |
| | 17 | 45 | | private static readonly Counter<long> WorkerJobsCounter = |
| | 17 | 46 | | Meter.CreateCounter<long>("asyncresponse.worker.jobs", unit: "{job}", |
| | 17 | 47 | | description: "Worker jobs processed, tagged by outcome (executed/failed/rejected)."); |
| | | 48 | | |
| | 17 | 49 | | private static readonly Counter<long> TypeResolutionFailures = |
| | 17 | 50 | | Meter.CreateCounter<long>("asyncresponse.type_resolution.unresolved", unit: "{failure}", |
| | 17 | 51 | | description: "Persisted service/payload type names that could not be resolved; the callback may silently fai |
| | | 52 | | |
| | 17 | 53 | | private static readonly Counter<long> UnroutableResponsesCounter = |
| | 17 | 54 | | Meter.CreateCounter<long>("asyncresponse.ingress.unroutable_responses", unit: "{message}", |
| | 17 | 55 | | description: "Inbound response messages acknowledged without routing because they carry no correlation id (d |
| | | 56 | | |
| | 17 | 57 | | private static readonly Counter<long> OversizedInboundCounter = |
| | 17 | 58 | | Meter.CreateCounter<long>("asyncresponse.ingress.oversized_messages", unit: "{message}", |
| | 17 | 59 | | description: "Inbound messages acknowledged without processing because they exceed AsyncResponseOptions.MaxI |
| | | 60 | | |
| | 17 | 61 | | private static readonly Counter<long> FlowStatePrunedRows = |
| | 17 | 62 | | Meter.CreateCounter<long>("asyncresponse.flow_state.pruned_rows", unit: "{row}", |
| | 17 | 63 | | description: "Expired durable-flow ledger rows deleted by the relational stores' opportunistic prune, tagged |
| | | 64 | | |
| | 17 | 65 | | private static readonly Counter<long> FlowStatePruneFailures = |
| | 17 | 66 | | Meter.CreateCounter<long>("asyncresponse.flow_state.prune_failures", unit: "{failure}", |
| | 17 | 67 | | description: "Opportunistic durable-flow prunes that failed (the flow creation they rode on still succeeded; |
| | | 68 | | |
| | 17 | 69 | | private static readonly Counter<long> FlowStatePruneBudgetExhausted = |
| | 17 | 70 | | Meter.CreateCounter<long>("asyncresponse.flow_state.prune_budget_exhausted", unit: "{prune}", |
| | 17 | 71 | | description: "Opportunistic durable-flow prunes that stopped at PruneBudget with expired rows still remainin |
| | | 72 | | |
| | 17 | 73 | | private static readonly Counter<long> OverloadedWaitsCounter = |
| | 17 | 74 | | Meter.CreateCounter<long>("asyncresponse.channel.overloaded_waits", unit: "{wait}", |
| | 17 | 75 | | description: "Waits faulted as indeterminate because responses for their correlation id arrived faster than |
| | | 76 | | |
| | 17 | 77 | | private static readonly Counter<long> InMemoryOverflowRejections = |
| | 17 | 78 | | Meter.CreateCounter<long>("asyncresponse.worker.inmemory_overflow_rejections", unit: "{job}", |
| | 17 | 79 | | description: "Follow-up jobs the in-memory worker transport refused because its queue was full and the in-jo |
| | | 80 | | |
| | 17 | 81 | | private static readonly Counter<long> InMemoryDelayedRejections = |
| | 17 | 82 | | Meter.CreateCounter<long>("asyncresponse.worker.inmemory_delayed_rejections", unit: "{job}", |
| | 17 | 83 | | description: "Delayed jobs published from inside a running job that the in-memory worker transport refused b |
| | | 84 | | |
| | 17 | 85 | | private static readonly Counter<long> FlowOwnJobRedeliveries = |
| | 17 | 86 | | Meter.CreateCounter<long>("asyncresponse.flow.own_job_redeliveries", unit: "{delivery}", |
| | 17 | 87 | | description: "Durable-flow wake-ups that found the execution lease held by a live execution of their OWN job |
| | | 88 | | |
| | | 89 | | // Every live in-memory transport in the process, for the overflow-depth gauge: the meter is |
| | | 90 | | // static and a process may host several transports (test harnesses, host-per-tenant workers), |
| | | 91 | | // so the gauge sums them and drops the ones that have been collected. Weak references keep a |
| | | 92 | | // disposed host's transport from being pinned for the process lifetime by its own telemetry. |
| | 17 | 93 | | private static readonly List<WeakReference<InMemoryWorkerTransport>> _inMemoryTransports = []; |
| | | 94 | | private static int _inMemoryOverflowGaugeRegistered; |
| | | 95 | | |
| | | 96 | | private static int _watchdogGaugesRegistered; |
| | | 97 | | private static AsyncResponseWatchdogState? _watchdogState; |
| | | 98 | | |
| | | 99 | | /// <summary>Records one follow-up publish the in-memory transport rejected at its in-job overflow capacity.</summar |
| | | 100 | | internal static void RecordInMemoryOverflowRejection() |
| | | 101 | | { |
| | 8 | 102 | | if (InMemoryOverflowRejections.Enabled) |
| | 2 | 103 | | InMemoryOverflowRejections.Add(1); |
| | 8 | 104 | | } |
| | | 105 | | |
| | | 106 | | /// <summary> |
| | | 107 | | /// Records one durable-flow wake-up recognised as the lease holder's own job redelivered |
| | | 108 | | /// (<paramref name="resolution"/>: <c>redelayed</c> or <c>waiting</c>). |
| | | 109 | | /// </summary> |
| | | 110 | | internal static void RecordFlowOwnJobRedelivery(string resolution) |
| | | 111 | | { |
| | 10 | 112 | | if (FlowOwnJobRedeliveries.Enabled) |
| | 0 | 113 | | FlowOwnJobRedeliveries.Add(1, new KeyValuePair<string, object?>("resolution", resolution)); |
| | 10 | 114 | | } |
| | | 115 | | |
| | | 116 | | /// <summary>Records one in-job delayed publish the in-memory transport rejected at its delayed-job capacity.</summa |
| | | 117 | | internal static void RecordInMemoryDelayedRejection() |
| | | 118 | | { |
| | 2 | 119 | | if (InMemoryDelayedRejections.Enabled) |
| | 2 | 120 | | InMemoryDelayedRejections.Add(1); |
| | 2 | 121 | | } |
| | | 122 | | |
| | | 123 | | /// <summary> |
| | | 124 | | /// Registers a transport with the <c>asyncresponse.worker.inmemory_overflow_depth</c> and |
| | | 125 | | /// <c>asyncresponse.worker.inmemory_delayed_jobs</c> gauges (created once, process-wide, on |
| | | 126 | | /// first use). |
| | | 127 | | /// </summary> |
| | | 128 | | internal static void TrackInMemoryOverflow(InMemoryWorkerTransport transport) |
| | | 129 | | { |
| | 1142 | 130 | | lock (_inMemoryTransports) |
| | | 131 | | { |
| | 63290 | 132 | | _inMemoryTransports.RemoveAll(static reference => !reference.TryGetTarget(out _)); |
| | 1142 | 133 | | _inMemoryTransports.Add(new WeakReference<InMemoryWorkerTransport>(transport)); |
| | 1142 | 134 | | } |
| | | 135 | | |
| | 1142 | 136 | | if (Interlocked.Exchange(ref _inMemoryOverflowGaugeRegistered, 1) != 0) |
| | 1136 | 137 | | return; |
| | | 138 | | |
| | 6 | 139 | | Meter.CreateObservableGauge("asyncresponse.worker.inmemory_overflow_depth", |
| | 135 | 140 | | static () => SumOverInMemoryTransports(static transport => transport.OverflowDepth), unit: "{job}", |
| | 6 | 141 | | description: "Follow-up jobs the in-memory worker transport currently holds past QueueCapacity (summed over |
| | 6 | 142 | | Meter.CreateObservableGauge("asyncresponse.worker.inmemory_delayed_jobs", |
| | 135 | 143 | | static () => SumOverInMemoryTransports(static transport => transport.DelayedJobsHeld), unit: "{job}", |
| | 6 | 144 | | description: "Delayed jobs the in-memory worker transport currently holds — waiting on their due time, or fi |
| | 6 | 145 | | } |
| | | 146 | | |
| | | 147 | | private static long SumOverInMemoryTransports(Func<InMemoryWorkerTransport, int> measure) |
| | | 148 | | { |
| | 28 | 149 | | long total = 0; |
| | 28 | 150 | | lock (_inMemoryTransports) |
| | | 151 | | { |
| | 270 | 152 | | _inMemoryTransports.RemoveAll(static reference => !reference.TryGetTarget(out _)); |
| | 540 | 153 | | foreach (var reference in _inMemoryTransports) |
| | | 154 | | { |
| | 242 | 155 | | if (reference.TryGetTarget(out var transport)) |
| | 242 | 156 | | total += measure(transport); |
| | | 157 | | } |
| | | 158 | | } |
| | | 159 | | |
| | 28 | 160 | | return total; |
| | | 161 | | } |
| | | 162 | | |
| | | 163 | | internal static Activity? StartActivity( |
| | | 164 | | string name, |
| | | 165 | | ActivityKind kind = ActivityKind.Internal, |
| | | 166 | | string? correlationId = null) |
| | | 167 | | { |
| | 73743 | 168 | | if (!ActivitySource.HasListeners()) |
| | 66243 | 169 | | return null; |
| | | 170 | | |
| | 7500 | 171 | | var activity = ActivitySource.StartActivity(name, kind); |
| | 7500 | 172 | | if (correlationId is not null) |
| | 1273 | 173 | | SetCorrelationId(activity, correlationId); |
| | | 174 | | |
| | 7500 | 175 | | return activity; |
| | | 176 | | } |
| | | 177 | | |
| | | 178 | | [MethodImpl(MethodImplOptions.AggressiveInlining)] |
| | | 179 | | internal static void SetCorrelationId(Activity? activity, string? correlationId) |
| | | 180 | | { |
| | 13147 | 181 | | if (activity is null || string.IsNullOrWhiteSpace(correlationId)) |
| | 10634 | 182 | | return; |
| | | 183 | | |
| | 2513 | 184 | | activity.SetTag("asyncresponse.correlation_id", correlationId); |
| | 2513 | 185 | | } |
| | | 186 | | |
| | | 187 | | [MethodImpl(MethodImplOptions.AggressiveInlining)] |
| | | 188 | | internal static void SetPayloadType(Activity? activity, Type? payloadType) |
| | | 189 | | { |
| | 12628 | 190 | | if (activity is null || payloadType is null) |
| | 10390 | 191 | | return; |
| | | 192 | | |
| | 2238 | 193 | | activity.SetTag("asyncresponse.payload_type", payloadType.FullName ?? payloadType.Name); |
| | 2238 | 194 | | } |
| | | 195 | | |
| | | 196 | | internal static void SetReplyTarget(Activity? activity, AsyncResponseReplyTarget? replyTarget) |
| | | 197 | | { |
| | 51854 | 198 | | if (replyTarget is null) |
| | 51812 | 199 | | return; |
| | | 200 | | |
| | 42 | 201 | | activity?.SetTag("asyncresponse.reply_target.name", replyTarget.Name); |
| | 42 | 202 | | activity?.SetTag("asyncresponse.reply_target.transport", replyTarget.Transport); |
| | 42 | 203 | | } |
| | | 204 | | |
| | | 205 | | internal static void SetWorker(Activity? activity, ReflectionCallDto? call) |
| | | 206 | | { |
| | 51858 | 207 | | if (call is null) |
| | 4 | 208 | | return; |
| | | 209 | | |
| | 51854 | 210 | | activity?.SetTag("asyncresponse.worker.service", call.ServiceInterfaceFullName); |
| | 51854 | 211 | | activity?.SetTag("asyncresponse.worker.method", call.MethodName); |
| | 51854 | 212 | | } |
| | | 213 | | |
| | | 214 | | internal static void SetLostSubscriberRoute(Activity? activity, RecoveryAction? action, bool mixed = false) |
| | 690 | 215 | | => activity?.SetTag("asyncresponse.lost_subscriber_route", LostSubscriberRouteName(action, mixed)); |
| | | 216 | | |
| | 132 | 217 | | private static string LostSubscriberRouteName(RecoveryAction? action, bool mixed) => mixed |
| | 132 | 218 | | ? "mixed" |
| | 132 | 219 | | : action switch |
| | 132 | 220 | | { |
| | 32 | 221 | | RecoveryAction.Resume => "resume", |
| | 46 | 222 | | RecoveryAction.Fail => "failure", |
| | 9 | 223 | | RecoveryAction.KeepWaiting => "keep_waiting", |
| | 40 | 224 | | _ => "unclassified" |
| | 132 | 225 | | }; |
| | | 226 | | |
| | | 227 | | internal static void SetError(Activity? activity, Exception exception) |
| | | 228 | | { |
| | 1970 | 229 | | activity?.SetTag("error.type", exception.GetType().FullName ?? exception.GetType().Name); |
| | 1970 | 230 | | activity?.SetStatus(ActivityStatusCode.Error, exception.Message); |
| | 1970 | 231 | | } |
| | | 232 | | |
| | | 233 | | internal static void SetError(Activity? activity, string errorType, string? description = null) |
| | | 234 | | { |
| | 228 | 235 | | activity?.SetTag("error.type", errorType); |
| | 228 | 236 | | activity?.SetStatus(ActivityStatusCode.Error, description); |
| | 228 | 237 | | } |
| | | 238 | | |
| | | 239 | | /// <summary>Records one lost-subscriber dispatch (the recovery path was entered for a publish).</summary> |
| | | 240 | | internal static void RecordLostSubscriber(string kind, RecoveryAction? action, bool callbackInvoked, bool mixed = fa |
| | | 241 | | { |
| | 301 | 242 | | if (!LostSubscriberDispatches.Enabled) |
| | 299 | 243 | | return; |
| | | 244 | | |
| | 2 | 245 | | LostSubscriberDispatches.Add( |
| | 2 | 246 | | 1, |
| | 2 | 247 | | new KeyValuePair<string, object?>("kind", kind), |
| | 2 | 248 | | new KeyValuePair<string, object?>("route", LostSubscriberRouteName(action, mixed)), |
| | 2 | 249 | | new KeyValuePair<string, object?>("invoked", callbackInvoked)); |
| | 2 | 250 | | } |
| | | 251 | | |
| | | 252 | | /// <summary> |
| | | 253 | | /// Records one wait faulted as indeterminate because its bounded buffer overflowed: the channel |
| | | 254 | | /// is fire-and-forget, the publisher was never backpressured, and admitting the next response |
| | | 255 | | /// would have meant buffering without bound. Every occurrence is a saturated consumer worth |
| | | 256 | | /// alerting on. |
| | | 257 | | /// </summary> |
| | | 258 | | internal static void RecordWaiterOverload(string channel) |
| | | 259 | | { |
| | 4 | 260 | | if (OverloadedWaitsCounter.Enabled) |
| | 2 | 261 | | OverloadedWaitsCounter.Add(1, new KeyValuePair<string, object?>("channel", channel)); |
| | 4 | 262 | | } |
| | | 263 | | |
| | | 264 | | /// <summary>Records one waiter timeout on the given channel kind.</summary> |
| | | 265 | | internal static void RecordWaiterTimeout(string channel) |
| | | 266 | | { |
| | 47 | 267 | | if (WaiterTimeoutsCounter.Enabled) |
| | 2 | 268 | | WaiterTimeoutsCounter.Add(1, new KeyValuePair<string, object?>("channel", channel)); |
| | 47 | 269 | | } |
| | | 270 | | |
| | | 271 | | /// <summary>Records one worker-job outcome (executed/failed/rejected).</summary> |
| | | 272 | | internal static void RecordWorkerOutcome(string outcome) |
| | | 273 | | { |
| | 6338 | 274 | | if (WorkerJobsCounter.Enabled) |
| | 2 | 275 | | WorkerJobsCounter.Add(1, new KeyValuePair<string, object?>("outcome", outcome)); |
| | 6338 | 276 | | } |
| | | 277 | | |
| | | 278 | | /// <summary> |
| | | 279 | | /// Records that a persisted type name (kind = "service" or "payload") could not be resolved, so |
| | | 280 | | /// operators can correlate a silently-failing recovery callback with a missing/ALC-loaded type. |
| | | 281 | | /// </summary> |
| | | 282 | | internal static void RecordTypeResolutionFailure(string kind) |
| | | 283 | | { |
| | 70 | 284 | | if (TypeResolutionFailures.Enabled) |
| | 2 | 285 | | TypeResolutionFailures.Add(1, new KeyValuePair<string, object?>("kind", kind)); |
| | 70 | 286 | | } |
| | | 287 | | |
| | | 288 | | /// <summary> |
| | | 289 | | /// Records an inbound response acknowledged without routing because it carried no correlation |
| | | 290 | | /// id — every occurrence is a producer-side contract violation worth alerting on. |
| | | 291 | | /// </summary> |
| | | 292 | | internal static void RecordUnroutableResponse() |
| | | 293 | | { |
| | 20 | 294 | | if (UnroutableResponsesCounter.Enabled) |
| | 0 | 295 | | UnroutableResponsesCounter.Add(1); |
| | 20 | 296 | | } |
| | | 297 | | |
| | | 298 | | /// <summary> |
| | | 299 | | /// Records an inbound message dropped for exceeding the configured size budget, tagged by the |
| | | 300 | | /// route it arrived on ("response" or "worker"). Producer-side contract violations, and the |
| | | 301 | | /// only visible trace of a message deliberately not processed — alert on them. |
| | | 302 | | /// </summary> |
| | | 303 | | internal static void RecordOversizedInboundMessage(string route) |
| | | 304 | | { |
| | 2 | 305 | | if (OversizedInboundCounter.Enabled) |
| | 0 | 306 | | OversizedInboundCounter.Add(1, new KeyValuePair<string, object?>("route", route)); |
| | 2 | 307 | | } |
| | | 308 | | |
| | | 309 | | /// <summary>Records the rows one opportunistic durable-flow prune deleted (zero is not recorded).</summary> |
| | | 310 | | internal static void RecordFlowStatePruned(string provider, long rows) |
| | | 311 | | { |
| | 1027 | 312 | | if (rows > 0 && FlowStatePrunedRows.Enabled) |
| | 24 | 313 | | FlowStatePrunedRows.Add(rows, new KeyValuePair<string, object?>("provider", provider)); |
| | 1027 | 314 | | } |
| | | 315 | | |
| | | 316 | | /// <summary>Records one failed opportunistic durable-flow prune (the create it rode on succeeded).</summary> |
| | | 317 | | internal static void RecordFlowStatePruneFailure(string provider) |
| | | 318 | | { |
| | 56 | 319 | | if (FlowStatePruneFailures.Enabled) |
| | 12 | 320 | | FlowStatePruneFailures.Add(1, new KeyValuePair<string, object?>("provider", provider)); |
| | 56 | 321 | | } |
| | | 322 | | |
| | | 323 | | /// <summary>Records one prune that hit its budget with a full last batch — expired rows remain.</summary> |
| | | 324 | | internal static void RecordFlowStatePruneBudgetExhausted(string provider) |
| | | 325 | | { |
| | 12 | 326 | | if (FlowStatePruneBudgetExhausted.Enabled) |
| | 12 | 327 | | FlowStatePruneBudgetExhausted.Add(1, new KeyValuePair<string, object?>("provider", provider)); |
| | 12 | 328 | | } |
| | | 329 | | |
| | | 330 | | /// <summary> |
| | | 331 | | /// Registers observable gauges reporting the latest watchdog scan: outstanding recovery |
| | | 332 | | /// registrations, those with a live waiter, and stale ones (no waiter, past the threshold). |
| | | 333 | | /// Registered once process-wide; the watchdog is a singleton, and the guard keeps repeated test |
| | | 334 | | /// constructions from stacking duplicate gauges on the shared meter. |
| | | 335 | | /// </summary> |
| | | 336 | | [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] |
| | | 337 | | internal static void EnsureWatchdogGauges(AsyncResponseWatchdogState state) |
| | | 338 | | { |
| | | 339 | | // The NEWEST scanning watchdog's state wins: the gauges are registered once process-wide |
| | | 340 | | // on the static meter, so capturing the first state instance in the gauge callbacks would |
| | | 341 | | // pin a disposed host's last snapshot forever in any process that builds more than one |
| | | 342 | | // host (test harnesses, WebApplicationFactory, host-per-tenant workers). The callbacks |
| | | 343 | | // read this holder instead, which every watchdog that begins scanning re-publishes and |
| | | 344 | | // ReleaseWatchdogGauges conditionally clears on stop. |
| | | 345 | | Volatile.Write(ref _watchdogState, state); |
| | | 346 | | |
| | | 347 | | if (Interlocked.Exchange(ref _watchdogGaugesRegistered, 1) != 0) |
| | | 348 | | return; |
| | | 349 | | |
| | | 350 | | Meter.CreateObservableGauge("asyncresponse.recovery.outstanding", |
| | | 351 | | static () => (long)(Volatile.Read(ref _watchdogState)?.Latest?.Report?.TotalEntries ?? 0), unit: "{entry}", |
| | | 352 | | description: "Outstanding recovery registrations at the last watchdog scan."); |
| | | 353 | | Meter.CreateObservableGauge("asyncresponse.recovery.active_waiters", |
| | | 354 | | static () => (long)(Volatile.Read(ref _watchdogState)?.Latest?.Report?.EntriesWithActiveWaiter ?? 0), unit: |
| | | 355 | | description: "Recovery registrations with a live waiter at the last watchdog scan."); |
| | | 356 | | Meter.CreateObservableGauge("asyncresponse.recovery.stale", |
| | | 357 | | static () => (long)(Volatile.Read(ref _watchdogState)?.Latest?.Report?.StaleEntries.Count ?? 0), unit: "{ent |
| | | 358 | | description: "Stale recovery registrations (no live waiter, past the threshold) at the last watchdog scan.") |
| | | 359 | | Meter.CreateObservableGauge("asyncresponse.recovery.scan_truncated", |
| | | 360 | | static () => Volatile.Read(ref _watchdogState)?.Latest?.Report?.Truncated == true ? 1L : 0L, unit: "{scan}", |
| | | 361 | | description: "1 when the last watchdog scan stopped at the MaxScanEntries buffer cap — outstanding/stale the |
| | | 362 | | Meter.CreateObservableGauge("asyncresponse.recovery.unprobeable", |
| | | 363 | | static () => (long)(Volatile.Read(ref _watchdogState)?.Latest?.Report?.UnprobeableEntries ?? 0), unit: "{ent |
| | | 364 | | description: "Recovery registrations whose waiter liveness could not be probed at the last watchdog scan — s |
| | | 365 | | } |
| | | 366 | | |
| | | 367 | | /// <summary> |
| | | 368 | | /// Clears the gauge holder when a stopping watchdog still owns it. Identity-conditional: a |
| | | 369 | | /// host stopping after a successor took over must not clear the successor's state, and once |
| | | 370 | | /// the last host stops the still-registered gauges must read zero instead of pinning the |
| | | 371 | | /// disposed host's final snapshot (and its stale-entry list) for process lifetime. |
| | | 372 | | /// </summary> |
| | | 373 | | internal static void ReleaseWatchdogGauges(AsyncResponseWatchdogState state) |
| | 4886 | 374 | | => Interlocked.CompareExchange(ref _watchdogState, null, state); |
| | | 375 | | } |