| | | 1 | | using System.Collections.Concurrent; |
| | | 2 | | |
| | | 3 | | namespace AsyncResponse; |
| | | 4 | | |
| | | 5 | | /// <summary>Atomic process-local flow-state store for development, tests, and single-process apps.</summary> |
| | | 6 | | internal sealed class InMemoryFlowStateStore : IFlowStateStore |
| | | 7 | | { |
| | | 8 | | // Saturating expiry stamp: the ttl parameter arrives from callers as well as options, and the |
| | | 9 | | // external stores deliberately saturate the same arithmetic — a raw Add threw |
| | | 10 | | // ArgumentOutOfRangeException on large ttls where every other store clamped. |
| | | 11 | | private static DateTime Expiry(DateTime now, TimeSpan ttl) |
| | 9442 | 12 | | => ttl >= DateTime.MaxValue - now ? DateTime.MaxValue : now.Add(ttl); |
| | | 13 | | |
| | | 14 | | /// <summary> |
| | | 15 | | /// How often <see cref="TryCreateAsync"/> sweeps expired entries whose ids are never touched |
| | | 16 | | /// again. Expiry used to be enforced only on access to the SAME id (a load or a replacement), |
| | | 17 | | /// which a completed run normally never gets — so a long-lived process minting unique flow |
| | | 18 | | /// ids retained every expired ledger (inputs, memoized results, value bags) for its lifetime; |
| | | 19 | | /// <c>StateExpiry</c> hid them from reads without ever bounding memory. On the engine's clock, |
| | | 20 | | /// like every other stamp here: a virtual clock that never advances never sweeps, and |
| | | 21 | | /// nothing has expired under it either. |
| | | 22 | | /// </summary> |
| | 7 | 23 | | internal static readonly TimeSpan SweepInterval = TimeSpan.FromMinutes(1); |
| | | 24 | | |
| | 914 | 25 | | private readonly ConcurrentDictionary<string, Entry> _entries = new(StringComparer.Ordinal); |
| | | 26 | | private readonly TimeProvider _timeProvider; |
| | | 27 | | private long _nextSweepTicks; |
| | | 28 | | |
| | | 29 | | /// <summary>Creates the store; expiry and lease stamps come from the engine's clock.</summary> |
| | 914 | 30 | | public InMemoryFlowStateStore(TimeProvider? timeProvider = null) |
| | 914 | 31 | | => _timeProvider = timeProvider ?? TimeProvider.System; |
| | | 32 | | |
| | | 33 | | public Task<bool> TryCreateAsync( |
| | | 34 | | string flowId, |
| | | 35 | | FlowState state, |
| | | 36 | | TimeSpan ttl, |
| | | 37 | | CancellationToken cancellationToken = default) |
| | | 38 | | { |
| | 3706 | 39 | | ValidateWrite(flowId, state, ttl); |
| | 3696 | 40 | | cancellationToken.ThrowIfCancellationRequested(); |
| | 3694 | 41 | | if (state.Revision != 0) |
| | 2 | 42 | | throw new ArgumentException("A new flow ledger must start at revision zero.", nameof(state)); |
| | | 43 | | |
| | 3692 | 44 | | var now = _timeProvider.GetUtcNow().UtcDateTime; |
| | 3692 | 45 | | SweepExpired(now); |
| | 3692 | 46 | | var created = CreateEntry(state, Expiry(now, ttl)); |
| | | 47 | | while (true) |
| | | 48 | | { |
| | 3692 | 49 | | if (_entries.TryAdd(flowId, created)) |
| | 3308 | 50 | | return Task.FromResult(true); |
| | | 51 | | |
| | 384 | 52 | | if (!_entries.TryGetValue(flowId, out var existing)) |
| | | 53 | | continue; |
| | | 54 | | |
| | 384 | 55 | | if (existing.ExpiresAtUtc > now) |
| | 382 | 56 | | return Task.FromResult(false); |
| | | 57 | | |
| | 2 | 58 | | if (_entries.TryUpdate(flowId, created, existing)) |
| | 2 | 59 | | return Task.FromResult(true); |
| | | 60 | | } |
| | | 61 | | } |
| | | 62 | | |
| | | 63 | | public Task<FlowState?> LoadAsync(string flowId, CancellationToken cancellationToken = default) |
| | | 64 | | { |
| | 8087 | 65 | | ArgumentException.ThrowIfNullOrWhiteSpace(flowId); |
| | 8085 | 66 | | cancellationToken.ThrowIfCancellationRequested(); |
| | | 67 | | |
| | 8090 | 68 | | while (_entries.TryGetValue(flowId, out var entry)) |
| | | 69 | | { |
| | 7918 | 70 | | if (entry.ExpiresAtUtc <= _timeProvider.GetUtcNow().UtcDateTime) |
| | | 71 | | { |
| | 6 | 72 | | _entries.TryRemove(KeyValuePair.Create(flowId, entry)); |
| | 6 | 73 | | continue; |
| | | 74 | | } |
| | | 75 | | |
| | | 76 | | // Unreadable JSON, an unknown schema version, and an entry whose JSON disagrees with |
| | | 77 | | // its own revision or key all throw out of here rather than masquerading as a deleted |
| | | 78 | | // flow: the entry is present, so acknowledging its wake-up as "gone" would strand the |
| | | 79 | | // run. Same contract as the durable stores — see DurableFlowStoreShared.ReadState. |
| | 7912 | 80 | | var state = FlowStateJson.Deserialize(entry.StateJson, flowId); |
| | 7908 | 81 | | if (state.Revision != entry.Revision) |
| | | 82 | | { |
| | 6 | 83 | | throw new FlowStateUnreadableException( |
| | 6 | 84 | | flowId, |
| | 6 | 85 | | $"its stored revision is {entry.Revision} but the revision inside its JSON is {state.Revision}"); |
| | | 86 | | } |
| | | 87 | | |
| | 7902 | 88 | | if (!string.Equals(state.FlowId, flowId, StringComparison.Ordinal)) |
| | 2 | 89 | | throw new FlowStateUnreadableException(flowId, "the flow id inside its JSON is not the id it is stored u |
| | | 90 | | |
| | 7900 | 91 | | return Task.FromResult<FlowState?>(state); |
| | | 92 | | } |
| | | 93 | | |
| | 172 | 94 | | return Task.FromResult<FlowState?>(null); |
| | | 95 | | } |
| | | 96 | | |
| | | 97 | | public Task<bool> TryUpdateAsync( |
| | | 98 | | string flowId, |
| | | 99 | | FlowState state, |
| | | 100 | | long expectedRevision, |
| | | 101 | | TimeSpan ttl, |
| | | 102 | | string? leaseId = null, |
| | | 103 | | CancellationToken cancellationToken = default) |
| | | 104 | | { |
| | 5212 | 105 | | ValidateWrite(flowId, state, ttl); |
| | 5212 | 106 | | cancellationToken.ThrowIfCancellationRequested(); |
| | 5212 | 107 | | if (expectedRevision < 0) |
| | 4 | 108 | | throw new ArgumentOutOfRangeException(nameof(expectedRevision), "Expected revision cannot be negative."); |
| | 5208 | 109 | | if (state.Revision != checked(expectedRevision + 1)) |
| | 2 | 110 | | throw new ArgumentException("The new flow-state revision must increment the expected revision by one.", name |
| | | 111 | | |
| | 5206 | 112 | | while (_entries.TryGetValue(flowId, out var current)) |
| | | 113 | | { |
| | 5198 | 114 | | var now = _timeProvider.GetUtcNow().UtcDateTime; |
| | 5198 | 115 | | if (current.ExpiresAtUtc <= now || current.Revision != expectedRevision) |
| | 22 | 116 | | return Task.FromResult(false); |
| | 5176 | 117 | | if (leaseId is not null |
| | 5176 | 118 | | && (!string.Equals(current.LeaseId, leaseId, StringComparison.Ordinal) |
| | 5176 | 119 | | || current.LeaseExpiresAtUtc <= now)) |
| | 16 | 120 | | return Task.FromResult(false); |
| | | 121 | | |
| | 5160 | 122 | | var updated = CreateEntry( |
| | 5160 | 123 | | state, |
| | 5160 | 124 | | Expiry(now, ttl), |
| | 5160 | 125 | | current.LeaseId, |
| | 5160 | 126 | | current.LeaseExpiresAtUtc); |
| | 5160 | 127 | | if (_entries.TryUpdate(flowId, updated, current)) |
| | 5160 | 128 | | return Task.FromResult(true); |
| | | 129 | | } |
| | | 130 | | |
| | 8 | 131 | | return Task.FromResult(false); |
| | | 132 | | } |
| | | 133 | | |
| | | 134 | | public Task<bool> TryAcquireLeaseAsync( |
| | | 135 | | string flowId, |
| | | 136 | | string leaseId, |
| | | 137 | | TimeSpan leaseDuration, |
| | | 138 | | CancellationToken cancellationToken = default) |
| | 1278 | 139 | | => TryChangeLeaseAsync(flowId, leaseId, leaseDuration, acquire: true, cancellationToken); |
| | | 140 | | |
| | | 141 | | public Task<bool> TryRenewLeaseAsync( |
| | | 142 | | string flowId, |
| | | 143 | | string leaseId, |
| | | 144 | | TimeSpan leaseDuration, |
| | | 145 | | CancellationToken cancellationToken = default) |
| | 1739 | 146 | | => TryChangeLeaseAsync(flowId, leaseId, leaseDuration, acquire: false, cancellationToken); |
| | | 147 | | |
| | | 148 | | public Task ReleaseLeaseAsync( |
| | | 149 | | string flowId, |
| | | 150 | | string leaseId, |
| | | 151 | | CancellationToken cancellationToken = default) |
| | | 152 | | { |
| | 886 | 153 | | ArgumentException.ThrowIfNullOrWhiteSpace(flowId); |
| | 886 | 154 | | ArgumentException.ThrowIfNullOrWhiteSpace(leaseId); |
| | 886 | 155 | | cancellationToken.ThrowIfCancellationRequested(); |
| | | 156 | | |
| | 884 | 157 | | while (_entries.TryGetValue(flowId, out var current)) |
| | | 158 | | { |
| | 874 | 159 | | if (!string.Equals(current.LeaseId, leaseId, StringComparison.Ordinal)) |
| | | 160 | | break; |
| | | 161 | | |
| | 856 | 162 | | if (_entries.TryUpdate(flowId, current with { LeaseId = null, LeaseExpiresAtUtc = null }, current)) |
| | | 163 | | break; |
| | | 164 | | } |
| | | 165 | | |
| | 884 | 166 | | return Task.CompletedTask; |
| | | 167 | | } |
| | | 168 | | |
| | | 169 | | public Task<FlowLeaseObservation?> ObserveLeaseAsync(string flowId, CancellationToken cancellationToken = default) |
| | | 170 | | { |
| | 295 | 171 | | ArgumentException.ThrowIfNullOrWhiteSpace(flowId); |
| | 293 | 172 | | cancellationToken.ThrowIfCancellationRequested(); |
| | | 173 | | |
| | | 174 | | // Raw, like every other store: an expired lease is reported as persisted. Whether it has |
| | | 175 | | // lapsed is TryAcquireLeaseAsync's call, on this store's clock. |
| | 293 | 176 | | return Task.FromResult<FlowLeaseObservation?>( |
| | 293 | 177 | | _entries.TryGetValue(flowId, out var current) && current.LeaseId is not null |
| | 293 | 178 | | ? new FlowLeaseObservation(current.LeaseId, current.LeaseExpiresAtUtc) |
| | 293 | 179 | | : FlowLeaseObservation.Unheld); |
| | | 180 | | } |
| | | 181 | | |
| | | 182 | | /// <summary> |
| | | 183 | | /// Breaks every held execution lease — the test harness's crash semantics for a simulated |
| | | 184 | | /// restart. A crashed process goes silent and its leases expire; a simulated restart shares |
| | | 185 | | /// the virtual clock with the "crashed" incarnation, whose parked executions would otherwise |
| | | 186 | | /// keep renewing against this shared store forever and the new incarnation could never take |
| | | 187 | | /// their flows over. Breaking the lease makes the zombie's next renewal fail (its lease loop |
| | | 188 | | /// marks itself lost and stops) and lets the new incarnation acquire immediately. |
| | | 189 | | /// </summary> |
| | | 190 | | internal void ExpireAllLeases() |
| | | 191 | | { |
| | 96 | 192 | | foreach (var flowId in _entries.Keys) |
| | | 193 | | { |
| | | 194 | | // CAS loop: a zombie renewal can swap the entry between the read and the update, and |
| | | 195 | | // TryUpdate compares against the snapshot — a silently lost break would recreate the |
| | | 196 | | // exact "executing on another live worker" hang this method exists to eliminate. |
| | | 197 | | // Retry until no lease is observed; once cleared, the zombie's next renewal fails |
| | | 198 | | // (its lease id no longer matches) and its loop stops, so this converges. |
| | 18 | 199 | | while (_entries.TryGetValue(flowId, out var entry) |
| | 18 | 200 | | && entry.LeaseId is not null |
| | 18 | 201 | | && !_entries.TryUpdate(flowId, entry with { LeaseId = null, LeaseExpiresAtUtc = null }, entry)) |
| | | 202 | | { |
| | | 203 | | } |
| | | 204 | | } |
| | 30 | 205 | | } |
| | | 206 | | |
| | | 207 | | public Task<bool> TryDeleteAsync(string flowId, CancellationToken cancellationToken = default) |
| | | 208 | | { |
| | 14 | 209 | | ArgumentException.ThrowIfNullOrWhiteSpace(flowId); |
| | 14 | 210 | | cancellationToken.ThrowIfCancellationRequested(); |
| | 12 | 211 | | return Task.FromResult(_entries.TryRemove(flowId, out _)); |
| | | 212 | | } |
| | | 213 | | |
| | | 214 | | /// <summary> |
| | | 215 | | /// Removes every entry expired at <paramref name="now"/>, at most once per |
| | | 216 | | /// <see cref="SweepInterval"/>; one sweeper at a time (the interval stamp is claimed by |
| | | 217 | | /// compare-exchange). Removal is conditional on the observed entry, so a concurrent |
| | | 218 | | /// update/create that swapped the entry in between keeps its (unexpired) replacement. |
| | | 219 | | /// </summary> |
| | | 220 | | private void SweepExpired(DateTime now) |
| | | 221 | | { |
| | 3692 | 222 | | var due = Interlocked.Read(ref _nextSweepTicks); |
| | 3692 | 223 | | if (now.Ticks < due) |
| | 3102 | 224 | | return; |
| | | 225 | | |
| | 590 | 226 | | var next = Expiry(now, SweepInterval).Ticks; |
| | 590 | 227 | | if (Interlocked.CompareExchange(ref _nextSweepTicks, next, due) != due) |
| | 0 | 228 | | return; |
| | | 229 | | |
| | 5220 | 230 | | foreach (var pair in _entries) |
| | | 231 | | { |
| | 2020 | 232 | | if (pair.Value.ExpiresAtUtc <= now) |
| | 2000 | 233 | | _entries.TryRemove(pair); |
| | | 234 | | } |
| | 590 | 235 | | } |
| | | 236 | | |
| | | 237 | | private Task<bool> TryChangeLeaseAsync( |
| | | 238 | | string flowId, |
| | | 239 | | string leaseId, |
| | | 240 | | TimeSpan leaseDuration, |
| | | 241 | | bool acquire, |
| | | 242 | | CancellationToken cancellationToken) |
| | | 243 | | { |
| | 3017 | 244 | | ArgumentException.ThrowIfNullOrWhiteSpace(flowId); |
| | 3017 | 245 | | ArgumentException.ThrowIfNullOrWhiteSpace(leaseId); |
| | 3017 | 246 | | if (leaseDuration <= TimeSpan.Zero) |
| | 2 | 247 | | throw new ArgumentOutOfRangeException(nameof(leaseDuration), "Lease duration must be greater than zero."); |
| | 3015 | 248 | | cancellationToken.ThrowIfCancellationRequested(); |
| | | 249 | | |
| | 3013 | 250 | | while (_entries.TryGetValue(flowId, out var current)) |
| | | 251 | | { |
| | 3005 | 252 | | var now = _timeProvider.GetUtcNow().UtcDateTime; |
| | 3005 | 253 | | if (current.ExpiresAtUtc <= now) |
| | 4 | 254 | | return Task.FromResult(false); |
| | | 255 | | |
| | 3001 | 256 | | var ownsLease = string.Equals(current.LeaseId, leaseId, StringComparison.Ordinal); |
| | 3001 | 257 | | if (acquire ? current.LeaseId is not null && current.LeaseExpiresAtUtc > now && !ownsLease : !ownsLease || c |
| | 343 | 258 | | return Task.FromResult(false); |
| | | 259 | | |
| | 2658 | 260 | | var updated = current with |
| | 2658 | 261 | | { |
| | 2658 | 262 | | LeaseId = leaseId, |
| | 2658 | 263 | | LeaseExpiresAtUtc = now.Add(leaseDuration) |
| | 2658 | 264 | | }; |
| | 2658 | 265 | | if (_entries.TryUpdate(flowId, updated, current)) |
| | 2658 | 266 | | return Task.FromResult(true); |
| | | 267 | | } |
| | | 268 | | |
| | 8 | 269 | | return Task.FromResult(false); |
| | | 270 | | } |
| | | 271 | | |
| | | 272 | | private static Entry CreateEntry( |
| | | 273 | | FlowState state, |
| | | 274 | | DateTime expiresAtUtc, |
| | | 275 | | string? leaseId = null, |
| | | 276 | | DateTime? leaseExpiresAtUtc = null) |
| | 8852 | 277 | | => new(FlowStateJson.Serialize(state), state.Revision, expiresAtUtc, leaseId, leaseExpiresAtUtc); |
| | | 278 | | |
| | | 279 | | private static void ValidateWrite(string flowId, FlowState state, TimeSpan ttl) |
| | | 280 | | { |
| | 8918 | 281 | | ArgumentException.ThrowIfNullOrWhiteSpace(flowId); |
| | 8918 | 282 | | ArgumentNullException.ThrowIfNull(state); |
| | 8918 | 283 | | if (!string.Equals(state.FlowId, flowId, StringComparison.Ordinal)) |
| | 4 | 284 | | throw new ArgumentException("The flow state id must match the store key.", nameof(state)); |
| | 8914 | 285 | | if (state.SchemaVersion != FlowStateSchema.Current) |
| | 4 | 286 | | throw new ArgumentException("The flow state must use the current schema version.", nameof(state)); |
| | 8910 | 287 | | if (ttl <= TimeSpan.Zero) |
| | 2 | 288 | | throw new ArgumentOutOfRangeException(nameof(ttl), "TTL must be greater than zero."); |
| | 8908 | 289 | | } |
| | | 290 | | |
| | 8860 | 291 | | private sealed record Entry( |
| | 7920 | 292 | | string StateJson, |
| | 13112 | 293 | | long Revision, |
| | 18535 | 294 | | DateTime ExpiresAtUtc, |
| | 18967 | 295 | | string? LeaseId = null, |
| | 24443 | 296 | | DateTime? LeaseExpiresAtUtc = null); |
| | | 297 | | } |