| | | 1 | | using Microsoft.Extensions.Logging; |
| | | 2 | | using Microsoft.Extensions.Options; |
| | | 3 | | using System.Runtime.CompilerServices; |
| | | 4 | | using System.Text.Json; |
| | | 5 | | using System.Text.Json.Serialization; |
| | | 6 | | |
| | | 7 | | namespace AsyncResponse.Channels.NATS; |
| | | 8 | | |
| | | 9 | | /// <summary> |
| | | 10 | | /// NATS JetStream Key-Value implementation of <see cref="IRecoveryStateStore"/> and |
| | | 11 | | /// <see cref="IRecoveryStateScanner"/>. |
| | | 12 | | /// <para> |
| | | 13 | | /// NATS KV applies a single <c>MaxAge</c> per bucket rather than a TTL per key, so each entry also |
| | | 14 | | /// carries an absolute <see cref="StoredRecoveryState.ExpiresAtUtc"/>: reads and scans treat an entry |
| | | 15 | | /// past its expiry as absent (and delete it best-effort), giving precise per-correlation expiry while |
| | | 16 | | /// the bucket's <c>MaxAge</c> acts as a garbage-collection ceiling for orphans. |
| | | 17 | | /// </para> |
| | | 18 | | /// </summary> |
| | | 19 | | internal sealed class NatsRecoveryStateStore : IRecoveryStateStore, IRecoveryStateScanner |
| | | 20 | | { |
| | | 21 | | private readonly INatsKvStore _store; |
| | | 22 | | private readonly ILogger<NatsRecoveryStateStore> _logger; |
| | | 23 | | private readonly TimeProvider _timeProvider; |
| | | 24 | | |
| | | 25 | | /// <summary>Creates a NATS JetStream Key-Value recovery state store.</summary> |
| | 3 | 26 | | public NatsRecoveryStateStore( |
| | 3 | 27 | | INatsKvStore store, |
| | 3 | 28 | | IOptions<NatsAsyncResponseChannelOptions> options, |
| | 3 | 29 | | ILogger<NatsRecoveryStateStore> logger, |
| | 3 | 30 | | TimeProvider? timeProvider = null) |
| | | 31 | | { |
| | 3 | 32 | | options.Value.Validate(); |
| | 3 | 33 | | _store = store; |
| | 3 | 34 | | _logger = logger; |
| | 3 | 35 | | _timeProvider = timeProvider ?? TimeProvider.System; |
| | 3 | 36 | | } |
| | | 37 | | |
| | | 38 | | /// <inheritdoc /> |
| | | 39 | | public async Task SaveAsync( |
| | | 40 | | string correlationId, |
| | | 41 | | RecoveryState state, |
| | | 42 | | TimeSpan ttl, |
| | | 43 | | CancellationToken cancellationToken = default) |
| | | 44 | | { |
| | 3 | 45 | | ArgumentException.ThrowIfNullOrWhiteSpace(correlationId); |
| | 3 | 46 | | ArgumentNullException.ThrowIfNull(state); |
| | 3 | 47 | | if (ttl <= TimeSpan.Zero) |
| | 3 | 48 | | throw new ArgumentOutOfRangeException(nameof(ttl), "TTL must be greater than zero."); |
| | 3 | 49 | | if (!string.Equals(state.CorrelationId, correlationId, StringComparison.Ordinal)) |
| | 3 | 50 | | throw new ArgumentException("The recovery-state correlation id must match the store key.", nameof(state)); |
| | 3 | 51 | | if (state.SchemaVersion != RecoveryStateSchema.Current) |
| | 3 | 52 | | throw new ArgumentException("The recovery state must use the current schema version.", nameof(state)); |
| | | 53 | | |
| | 3 | 54 | | cancellationToken.ThrowIfCancellationRequested(); |
| | 3 | 55 | | if (state.RegistrationId == Guid.Empty) |
| | 3 | 56 | | state.RegistrationId = Guid.NewGuid(); |
| | | 57 | | |
| | 3 | 58 | | var key = NatsSubjectSchema.RecoveryKey(correlationId); |
| | | 59 | | |
| | | 60 | | // Revision-conditioned read-modify-write: two waiters registering the same correlation id |
| | | 61 | | // concurrently must both survive. |
| | 3 | 62 | | for (var attempt = 0; attempt < MaxCasAttempts; attempt++) |
| | | 63 | | { |
| | 3 | 64 | | cancellationToken.ThrowIfCancellationRequested(); |
| | | 65 | | |
| | 3 | 66 | | var entry = await _store.GetAsync(key, cancellationToken).ConfigureAwait(false); |
| | 3 | 67 | | var stored = entry is { } existing ? TryDeserialize(existing.Value, key) : null; |
| | 3 | 68 | | var states = stored is not null && !IsExpired(stored) |
| | 3 | 69 | | ? StatesFrom(stored) |
| | 3 | 70 | | : []; |
| | 3 | 71 | | states.RemoveAll(existingState => !IsStateReadable(existingState, key, correlationId)); |
| | 3 | 72 | | states.RemoveAll(existingState => existingState.RegistrationId == state.RegistrationId); |
| | 3 | 73 | | states.Add(state); |
| | 3 | 74 | | var json = SerializeStates(states, _timeProvider.GetUtcNow() + ttl); |
| | | 75 | | |
| | 3 | 76 | | var written = entry is { } current |
| | 3 | 77 | | ? await _store.TryUpdateAsync(key, json, current.Revision, cancellationToken).ConfigureAwait(false) |
| | 3 | 78 | | : await _store.TryCreateAsync(key, json, cancellationToken).ConfigureAwait(false); |
| | 3 | 79 | | if (written) |
| | 3 | 80 | | return; |
| | | 81 | | } |
| | | 82 | | |
| | 3 | 83 | | throw new InvalidOperationException( |
| | 3 | 84 | | $"Recovery-state save for correlationId '{correlationId}' could not commit after {MaxCasAttempts} optimistic |
| | 3 | 85 | | } |
| | | 86 | | |
| | | 87 | | /// <inheritdoc /> |
| | | 88 | | public async Task<IReadOnlyList<RecoveryState>> GetAllAsync(string correlationId, CancellationToken cancellationToke |
| | | 89 | | { |
| | 3 | 90 | | ArgumentException.ThrowIfNullOrWhiteSpace(correlationId); |
| | 3 | 91 | | cancellationToken.ThrowIfCancellationRequested(); |
| | | 92 | | |
| | 3 | 93 | | var key = NatsSubjectSchema.RecoveryKey(correlationId); |
| | 3 | 94 | | var stored = await LoadStoredAsync(key, cancellationToken).ConfigureAwait(false); |
| | 3 | 95 | | if (stored is null) |
| | 3 | 96 | | return []; |
| | | 97 | | |
| | 3 | 98 | | if (IsExpired(stored)) |
| | | 99 | | { |
| | | 100 | | // Past its logical expiry but still physically present (bucket MaxAge has not collected it |
| | | 101 | | // yet): treat as gone and remove it best-effort so it never resurfaces. |
| | 3 | 102 | | await TryDeleteSilentlyAsync(key, cancellationToken).ConfigureAwait(false); |
| | 2 | 103 | | return []; |
| | | 104 | | } |
| | | 105 | | |
| | 3 | 106 | | var states = StatesFrom(stored); |
| | 3 | 107 | | for (var i = states.Count - 1; i >= 0; i--) |
| | | 108 | | { |
| | 3 | 109 | | var state = states[i]; |
| | 3 | 110 | | if (!IsStateReadable(state, key, correlationId)) |
| | | 111 | | { |
| | 3 | 112 | | states.RemoveAt(i); |
| | | 113 | | continue; |
| | | 114 | | } |
| | | 115 | | } |
| | | 116 | | |
| | 3 | 117 | | return states; |
| | 3 | 118 | | } |
| | | 119 | | |
| | | 120 | | /// <inheritdoc /> |
| | | 121 | | public async Task<bool> TryDeleteAsync(string correlationId, Guid registrationId, CancellationToken cancellationToke |
| | | 122 | | { |
| | 3 | 123 | | ArgumentException.ThrowIfNullOrWhiteSpace(correlationId); |
| | 3 | 124 | | if (registrationId == Guid.Empty) |
| | 3 | 125 | | throw new ArgumentException("Registration id cannot be empty.", nameof(registrationId)); |
| | 3 | 126 | | cancellationToken.ThrowIfCancellationRequested(); |
| | | 127 | | |
| | 3 | 128 | | var key = NatsSubjectSchema.RecoveryKey(correlationId); |
| | | 129 | | |
| | | 130 | | // Revision-conditioned removal: deleting one registration must not clobber a registration |
| | | 131 | | // that another writer appended between our read and our write. |
| | 3 | 132 | | for (var attempt = 0; attempt < MaxCasAttempts; attempt++) |
| | | 133 | | { |
| | 3 | 134 | | cancellationToken.ThrowIfCancellationRequested(); |
| | | 135 | | |
| | 3 | 136 | | var entry = await _store.GetAsync(key, cancellationToken).ConfigureAwait(false); |
| | 3 | 137 | | if (entry is not { } existing) |
| | 3 | 138 | | return false; |
| | | 139 | | |
| | 3 | 140 | | var stored = TryDeserialize(existing.Value, key); |
| | 3 | 141 | | if (stored is null) |
| | 3 | 142 | | return false; |
| | | 143 | | |
| | 3 | 144 | | if (IsExpired(stored)) |
| | | 145 | | { |
| | 3 | 146 | | await TryDeleteSilentlyAsync(key, cancellationToken).ConfigureAwait(false); |
| | 3 | 147 | | return false; |
| | | 148 | | } |
| | | 149 | | |
| | 3 | 150 | | var states = StatesFrom(stored); |
| | 3 | 151 | | states.RemoveAll(state => !IsStateReadable(state, key, correlationId)); |
| | 3 | 152 | | var removed = states.RemoveAll(state => state.RegistrationId == registrationId) > 0; |
| | 3 | 153 | | if (!removed) |
| | 3 | 154 | | return false; |
| | | 155 | | |
| | 3 | 156 | | var succeeded = states.Count == 0 |
| | 3 | 157 | | ? await _store.TryDeleteAsync(key, existing.Revision, cancellationToken).ConfigureAwait(false) |
| | 3 | 158 | | : await _store.TryUpdateAsync(key, SerializeStates(states, stored.ExpiresAtUtc), existing.Revision, canc |
| | 3 | 159 | | if (succeeded) |
| | 3 | 160 | | return true; |
| | | 161 | | } |
| | | 162 | | |
| | 3 | 163 | | _logger.LogWarning( |
| | 3 | 164 | | "Recovery-state delete for correlationId {CorrelationId} registration {RegistrationId} exhausted {Attempts} |
| | 3 | 165 | | correlationId, registrationId, MaxCasAttempts); |
| | 2 | 166 | | return false; |
| | 3 | 167 | | } |
| | | 168 | | |
| | | 169 | | /// <inheritdoc /> |
| | | 170 | | public async IAsyncEnumerable<RecoveryState> ScanAsync([EnumeratorCancellation] CancellationToken cancellationToken |
| | | 171 | | { |
| | 3 | 172 | | await foreach (var key in _store.GetKeysAsync(cancellationToken).ConfigureAwait(false)) |
| | | 173 | | { |
| | 3 | 174 | | cancellationToken.ThrowIfCancellationRequested(); |
| | | 175 | | |
| | 3 | 176 | | var stored = await LoadStoredAsync(key, cancellationToken).ConfigureAwait(false); |
| | 3 | 177 | | if (stored is null) |
| | | 178 | | continue; |
| | | 179 | | |
| | 3 | 180 | | if (IsExpired(stored)) |
| | | 181 | | { |
| | 2 | 182 | | await TryDeleteSilentlyAsync(key, cancellationToken).ConfigureAwait(false); |
| | 3 | 183 | | continue; |
| | | 184 | | } |
| | | 185 | | |
| | 3 | 186 | | foreach (var state in StatesFrom(stored)) |
| | | 187 | | { |
| | 3 | 188 | | var correlationId = NatsSubjectSchema.CorrelationIdFromRecoveryKey(key); |
| | 3 | 189 | | if (!IsStateReadable(state, key, correlationId)) |
| | | 190 | | continue; |
| | | 191 | | |
| | 3 | 192 | | yield return state; |
| | | 193 | | } |
| | 3 | 194 | | } |
| | 3 | 195 | | } |
| | | 196 | | |
| | | 197 | | private const int MaxCasAttempts = 4; |
| | | 198 | | |
| | | 199 | | private async Task<StoredRecoveryState?> LoadStoredAsync(string key, CancellationToken cancellationToken) |
| | | 200 | | { |
| | 3 | 201 | | var entry = await _store.GetAsync(key, cancellationToken).ConfigureAwait(false); |
| | 3 | 202 | | return entry is { } existing ? TryDeserialize(existing.Value, key) : null; |
| | 3 | 203 | | } |
| | | 204 | | |
| | | 205 | | private static string SerializeStates(List<RecoveryState> states, DateTimeOffset expiresAtUtc) |
| | 3 | 206 | | => JsonSerializer.Serialize(new StoredRecoveryState |
| | 3 | 207 | | { |
| | 3 | 208 | | States = states, |
| | 3 | 209 | | ExpiresAtUtc = expiresAtUtc |
| | 3 | 210 | | }, NatsChannelJsonContext.Default.StoredRecoveryState); |
| | | 211 | | |
| | 3 | 212 | | private bool IsExpired(StoredRecoveryState stored) => stored.ExpiresAtUtc <= _timeProvider.GetUtcNow(); |
| | | 213 | | |
| | | 214 | | private bool IsStateReadable(RecoveryState? state, string key, string correlationId) |
| | | 215 | | { |
| | 3 | 216 | | if (state is null || state.RegistrationId == Guid.Empty) |
| | | 217 | | { |
| | 3 | 218 | | _logger.LogWarning( |
| | 3 | 219 | | "Recovery state at key {RecoveryKey} has no registration id; rejecting it because it cannot be deleted s |
| | 3 | 220 | | key); |
| | 3 | 221 | | return false; |
| | | 222 | | } |
| | | 223 | | |
| | 3 | 224 | | if (!RecoveryStateSchema.IsReadable(state.SchemaVersion)) |
| | | 225 | | { |
| | 3 | 226 | | _logger.LogWarning( |
| | 3 | 227 | | "Recovery state at key {RecoveryKey} has unsupported schema version {SchemaVersion} (current: {Current}) |
| | 3 | 228 | | key, state.SchemaVersion, RecoveryStateSchema.Current); |
| | 3 | 229 | | return false; |
| | | 230 | | } |
| | | 231 | | |
| | 3 | 232 | | if (string.Equals(state.CorrelationId, correlationId, StringComparison.Ordinal)) |
| | 3 | 233 | | return true; |
| | | 234 | | |
| | 3 | 235 | | _logger.LogWarning( |
| | 3 | 236 | | "Recovery state at key {RecoveryKey} has correlationId {StoredCorrelationId}, expected {CorrelationId}; reje |
| | 3 | 237 | | key, state.CorrelationId, correlationId); |
| | 3 | 238 | | return false; |
| | | 239 | | } |
| | | 240 | | |
| | | 241 | | private StoredRecoveryState? TryDeserialize(string json, string key) |
| | | 242 | | { |
| | | 243 | | try |
| | | 244 | | { |
| | 3 | 245 | | return JsonSerializer.Deserialize(json, NatsChannelJsonContext.Default.StoredRecoveryState); |
| | | 246 | | } |
| | 3 | 247 | | catch (JsonException ex) |
| | | 248 | | { |
| | 3 | 249 | | _logger.LogWarning(ex, "Unreadable recovery state at key {RecoveryKey}; skipping.", key); |
| | 3 | 250 | | return null; |
| | | 251 | | } |
| | 3 | 252 | | } |
| | | 253 | | |
| | | 254 | | private static List<RecoveryState> StatesFrom(StoredRecoveryState stored) |
| | 3 | 255 | | => stored.States is { Count: > 0 } ? [.. stored.States] : []; |
| | | 256 | | |
| | | 257 | | private async Task TryDeleteSilentlyAsync(string key, CancellationToken cancellationToken) |
| | | 258 | | { |
| | | 259 | | try |
| | | 260 | | { |
| | 2 | 261 | | await _store.DeleteAsync(key, cancellationToken).ConfigureAwait(false); |
| | 2 | 262 | | } |
| | 2 | 263 | | catch (Exception ex) |
| | | 264 | | { |
| | 2 | 265 | | _logger.LogDebug(ex, "Best-effort delete of expired recovery state at key {RecoveryKey} failed.", key); |
| | 2 | 266 | | } |
| | 2 | 267 | | } |
| | | 268 | | |
| | | 269 | | /// <summary>The stored envelope: the recovery state plus its absolute expiry, for per-key logical TTL.</summary> |
| | | 270 | | internal sealed class StoredRecoveryState |
| | | 271 | | { |
| | | 272 | | public List<RecoveryState>? States { get; set; } |
| | | 273 | | public DateTimeOffset ExpiresAtUtc { get; set; } |
| | | 274 | | } |
| | | 275 | | } |
| | | 276 | | |
| | | 277 | | /// <summary> |
| | | 278 | | /// Source-generated metadata for the package-local KV envelope (trim/AOT-safe; the wire format is |
| | | 279 | | /// unchanged — Metadata-mode generation with default options matches the previous reflection-based |
| | | 280 | | /// serialization exactly). |
| | | 281 | | /// </summary> |
| | | 282 | | [JsonSourceGenerationOptions(GenerationMode = JsonSourceGenerationMode.Metadata)] |
| | | 283 | | [JsonSerializable(typeof(NatsRecoveryStateStore.StoredRecoveryState))] |
| | | 284 | | internal sealed partial class NatsChannelJsonContext : JsonSerializerContext; |