| | | 1 | | using Microsoft.Extensions.DependencyInjection; |
| | | 2 | | using Microsoft.Extensions.Logging; |
| | | 3 | | using System.Diagnostics.CodeAnalysis; |
| | | 4 | | |
| | | 5 | | namespace AsyncResponse; |
| | | 6 | | |
| | | 7 | | /// <inheritdoc cref="IDurableFlows" /> |
| | | 8 | | internal sealed class DurableFlowService : IDurableFlows |
| | | 9 | | { |
| | | 10 | | private readonly IServiceScopeFactory _scopeFactory; |
| | | 11 | | private readonly IAsyncResponseBuilder _builder; |
| | | 12 | | private readonly AsyncResponseContextPropagation _propagation; |
| | | 13 | | private readonly DurableFlowOptions _options; |
| | | 14 | | private readonly ILogger<DurableFlowService> _logger; |
| | | 15 | | private readonly TimeProvider _timeProvider; |
| | | 16 | | |
| | | 17 | | /// <summary>Creates the durable-flows starter.</summary> |
| | 1597 | 18 | | public DurableFlowService( |
| | 1597 | 19 | | IServiceScopeFactory scopeFactory, |
| | 1597 | 20 | | IAsyncResponseBuilder builder, |
| | 1597 | 21 | | AsyncResponseContextPropagation propagation, |
| | 1597 | 22 | | DurableFlowOptions options, |
| | 1597 | 23 | | ILogger<DurableFlowService> logger, |
| | 1597 | 24 | | TimeProvider? timeProvider = null) |
| | | 25 | | { |
| | 1597 | 26 | | _scopeFactory = scopeFactory; |
| | 1597 | 27 | | _builder = builder; |
| | 1597 | 28 | | _propagation = propagation; |
| | 1597 | 29 | | _options = options; |
| | 1597 | 30 | | FlowStateConcurrency.ValidateOptions(_options); |
| | 1597 | 31 | | _options.ValidateInProcessPark(); |
| | 1591 | 32 | | _logger = logger; |
| | 1591 | 33 | | _timeProvider = timeProvider ?? TimeProvider.System; |
| | 1591 | 34 | | } |
| | | 35 | | |
| | | 36 | | /// <inheritdoc /> |
| | | 37 | | public async Task<string> StartAsync<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors | |
| | | 38 | | TInput input, |
| | | 39 | | string? flowId = null, |
| | | 40 | | CancellationToken cancellationToken = default) |
| | | 41 | | where TFlow : class, IDurableFlow<TInput> |
| | | 42 | | { |
| | 1610 | 43 | | ArgumentNullException.ThrowIfNull(input); |
| | 1610 | 44 | | cancellationToken.ThrowIfCancellationRequested(); |
| | 1608 | 45 | | if (flowId is null) |
| | 1470 | 46 | | flowId = $"flow-{AsyncResponseContext.GenerateCorrelationId()}"; |
| | | 47 | | else |
| | 138 | 48 | | ArgumentException.ThrowIfNullOrWhiteSpace(flowId); |
| | | 49 | | |
| | | 50 | | // Every id is validated BEFORE anything is published: the publish below is the start's |
| | | 51 | | // commit point, and a job for an id every store would reject must never leave the process. |
| | 1604 | 52 | | FlowStateConcurrency.EnsurePortableFlowId(flowId); |
| | | 53 | | |
| | 1582 | 54 | | await using var scope = _scopeFactory.CreateAsyncScope(); |
| | 1582 | 55 | | var store = scope.ServiceProvider.GetRequiredService<IFlowStateStore>(); |
| | | 56 | | |
| | 1582 | 57 | | var now = _timeProvider.GetUtcNow().UtcDateTime; |
| | 1582 | 58 | | var inputJson = AsyncResponseJson.Serialize(input); |
| | 1582 | 59 | | var state = new FlowState |
| | 1582 | 60 | | { |
| | 1582 | 61 | | FlowId = flowId, |
| | 1582 | 62 | | FlowTypeName = typeof(TFlow).FullName, |
| | 1582 | 63 | | InputTypeName = typeof(TInput).FullName, |
| | 1582 | 64 | | InputJson = inputJson, |
| | 1582 | 65 | | Status = FlowRunStatus.Running, |
| | 1582 | 66 | | LastMessage = "Flow started.", |
| | 1582 | 67 | | CreatedAtUtc = now, |
| | 1582 | 68 | | UpdatedAtUtc = now, |
| | 1582 | 69 | | Revision = 0, |
| | 1582 | 70 | | Context = _propagation.Capture() |
| | 1582 | 71 | | }; |
| | | 72 | | |
| | | 73 | | // PUBLISH FIRST, then create. The worker job carries the whole initial ledger, and |
| | | 74 | | // IDurableFlowExecutor.CreateAndExecuteAsync creates the ledger itself (insert-if-absent) |
| | | 75 | | // before executing — so the publish is the single durable commit point of a start: |
| | | 76 | | // - a crash before the publish leaves nothing behind (the caller sees a fault and retries); |
| | | 77 | | // - a crash after the publish leaves a job whose execution creates and runs the flow. |
| | | 78 | | // The previous order (create, then publish) had an unrecoverable gap: a process dying |
| | | 79 | | // between the two left a committed Running ledger with Attempts = 0 that nothing would |
| | | 80 | | // ever execute, and IFlowStateStore has no enumeration for a reconciler to go find it. |
| | | 81 | | // The publish still runs the retry ladder the ingress uses, and a publish that fails for |
| | | 82 | | // good surfaces the id (DurableFlowNotDispatchedException) — now with nothing persisted. |
| | 1582 | 83 | | var id = flowId; |
| | 1582 | 84 | | var initialStateJson = FlowStateJson.Serialize(state); |
| | 1582 | 85 | | store.ValidateCreate(flowId, state, _options.StateExpiry); |
| | 1580 | 86 | | await PublishStartAsync( |
| | 1580 | 87 | | executor => executor.CreateAndExecuteAsync(id, initialStateJson), |
| | 1580 | 88 | | id, |
| | 1580 | 89 | | cancellationToken).ConfigureAwait(false); |
| | | 90 | | |
| | | 91 | | // Normally the starter's own create makes state immediately queryable and reports an |
| | | 92 | | // explicit-id conflict to this caller. Losing the race to the executor or an identical |
| | | 93 | | // start is expected. After a transient store fault the published job creates the ledger; |
| | | 94 | | // a query can return null until it does. Deterministic size/argument rejection still |
| | | 95 | | // propagates, including from custom stores whose preflight uses the no-op default. |
| | | 96 | | bool created; |
| | | 97 | | try |
| | | 98 | | { |
| | 1570 | 99 | | created = await FlowStateConcurrency.TryCreateAsync( |
| | 1570 | 100 | | store, |
| | 1570 | 101 | | flowId, |
| | 1570 | 102 | | state, |
| | 1570 | 103 | | _options.StateExpiry, |
| | 1570 | 104 | | cancellationToken).ConfigureAwait(false); |
| | 1570 | 105 | | } |
| | 0 | 106 | | catch (Exception ex) when (ex is not (FlowStateTooLargeException or ArgumentException) |
| | 0 | 107 | | && (ex is not OperationCanceledException || !cancellationToken.IsCancellationRequested)) |
| | | 108 | | { |
| | 0 | 109 | | _logger.LogWarning( |
| | 0 | 110 | | ex, |
| | 0 | 111 | | "Durable flow {FlowId} start job is published but the starter could not write the ledger; the executor c |
| | 0 | 112 | | flowId); |
| | 0 | 113 | | return flowId; |
| | | 114 | | } |
| | | 115 | | |
| | 1570 | 116 | | if (created) |
| | | 117 | | { |
| | 890 | 118 | | _logger.LogInformation("Started durable flow {FlowId} ({FlowType}).", flowId, typeof(TFlow).Name); |
| | 890 | 119 | | return flowId; |
| | | 120 | | } |
| | | 121 | | |
| | 680 | 122 | | var existing = await store.LoadAsync(flowId, cancellationToken).ConfigureAwait(false); |
| | 680 | 123 | | if (existing is null) |
| | | 124 | | { |
| | | 125 | | // Lost the create to a ledger that has since expired: the published job's create wins |
| | | 126 | | // the next time round. Nothing for the caller to do. |
| | 0 | 127 | | _logger.LogWarning("Durable flow {FlowId} start job is published; the existing ledger is expired and the exe |
| | 0 | 128 | | return flowId; |
| | | 129 | | } |
| | | 130 | | |
| | | 131 | | // Throws DurableFlowIdConflictException for different work; the executor drops the |
| | | 132 | | // already-published job on the same test. |
| | 680 | 133 | | EnsureIdempotentStart<TFlow, TInput>(existing, inputJson, flowId); |
| | | 134 | | |
| | | 135 | | // A semantically identical retry: the published job re-enqueues the existing run |
| | | 136 | | // (completed steps skip) instead of creating a duplicate. |
| | 664 | 137 | | _logger.LogInformation("Durable flow {FlowId} already exists; the start job re-enqueues the existing run instead |
| | 664 | 138 | | return flowId; |
| | 1554 | 139 | | } |
| | | 140 | | |
| | | 141 | | /// <summary> |
| | | 142 | | /// Publishes a start job through the ingress's retry ladder. A publish that still fails |
| | | 143 | | /// surfaces as <see cref="DurableFlowNotDispatchedException"/> carrying the id: nothing was |
| | | 144 | | /// persisted, so the caller simply retries the start (idempotent with the same id). |
| | | 145 | | /// </summary> |
| | | 146 | | private async Task PublishStartAsync( |
| | | 147 | | System.Linq.Expressions.Expression<Func<IDurableFlowExecutor, Task>> job, |
| | | 148 | | string flowId, |
| | | 149 | | CancellationToken cancellationToken) |
| | | 150 | | { |
| | | 151 | | try |
| | | 152 | | { |
| | 1580 | 153 | | await AsyncResponseRetry.ExecuteAsync( |
| | 1580 | 154 | | async token => |
| | 1580 | 155 | | { |
| | 1602 | 156 | | await _builder.EnqueueWorkerAsync(job, token).ConfigureAwait(false); |
| | 1570 | 157 | | return true; |
| | 1570 | 158 | | }, |
| | 1580 | 159 | | // Only the CALLER's cancellation ends the ladder. An OperationCanceledException |
| | 1580 | 160 | | // whose token is not the caller's is a transport or SDK timeout — brokers surface |
| | 1580 | 161 | | // those as TaskCanceledException all the time — and that is exactly the transient |
| | 1580 | 162 | | // shape this retry exists for. Excluding the whole exception type meant the most |
| | 1580 | 163 | | // common recoverable publish failure got zero retries. An envelope over the |
| | 1580 | 164 | | // ingress's size budget is deterministic (the same input serializes to the same |
| | 1580 | 165 | | // length): no attempt can succeed, so it is not retried either. |
| | 26 | 166 | | isTransient: ex => ex is not WorkerJobTooLargeException |
| | 26 | 167 | | && (ex is not OperationCanceledException || !cancellationToken.IsCancellationRequested), |
| | 1580 | 168 | | maxAttempts: 4, |
| | 1580 | 169 | | baseDelay: TimeSpan.FromMilliseconds(250), |
| | 1580 | 170 | | maxDelay: TimeSpan.FromSeconds(2), |
| | 1580 | 171 | | cancellationToken, |
| | 1580 | 172 | | _timeProvider).ConfigureAwait(false); |
| | 1570 | 173 | | } |
| | 4 | 174 | | catch (WorkerJobTooLargeException ex) |
| | | 175 | | { |
| | | 176 | | // Not a dispatch failure to retry: the start job carries the initial ledger, and this |
| | | 177 | | // input serializes past what the consuming ingress accepts — it would be acknowledged |
| | | 178 | | // there without ever executing. Surfaced as itself (nothing was persisted) so the |
| | | 179 | | // caller can shrink the input or move it behind a claim check. |
| | 4 | 180 | | _logger.LogError( |
| | 4 | 181 | | ex, |
| | 4 | 182 | | "Durable flow {FlowId} could not be started: its start job ({SerializedLength} UTF-16 code units) exceed |
| | 4 | 183 | | flowId, |
| | 4 | 184 | | ex.SerializedLength, |
| | 4 | 185 | | ex.Limit); |
| | 4 | 186 | | throw; |
| | | 187 | | } |
| | 6 | 188 | | catch (Exception ex) |
| | | 189 | | { |
| | 6 | 190 | | _logger.LogError( |
| | 6 | 191 | | ex, |
| | 6 | 192 | | "Durable flow {FlowId} could not be started: its worker job was not published after retries. Nothing was |
| | 6 | 193 | | flowId); |
| | 6 | 194 | | throw new DurableFlowNotDispatchedException(flowId, ex); |
| | | 195 | | } |
| | 1570 | 196 | | } |
| | | 197 | | |
| | | 198 | | /// <inheritdoc /> |
| | | 199 | | public async Task ResumeAsync(string flowId, CancellationToken cancellationToken = default) |
| | | 200 | | { |
| | 684 | 201 | | ArgumentException.ThrowIfNullOrWhiteSpace(flowId); |
| | | 202 | | |
| | 682 | 203 | | await using var scope = _scopeFactory.CreateAsyncScope(); |
| | 682 | 204 | | var store = scope.ServiceProvider.GetRequiredService<IFlowStateStore>(); |
| | | 205 | | |
| | 682 | 206 | | var state = await store.LoadAsync(flowId, cancellationToken).ConfigureAwait(false) |
| | 682 | 207 | | ?? throw new InvalidOperationException($"No flow state found for '{flowId}' (unknown, expired, or unreadable |
| | | 208 | | |
| | 680 | 209 | | if (state.Status != FlowRunStatus.Running) |
| | | 210 | | { |
| | 666 | 211 | | _logger.LogDebug("Durable flow {FlowId} is already {Status}; ignoring resume.", flowId, state.Status); |
| | 666 | 212 | | return; |
| | | 213 | | } |
| | | 214 | | |
| | 14 | 215 | | var id = flowId; |
| | 14 | 216 | | await _builder.EnqueueWorkerAsync<IDurableFlowExecutor>( |
| | 14 | 217 | | executor => executor.ExecuteAsync(id), |
| | 14 | 218 | | cancellationToken).ConfigureAwait(false); |
| | 680 | 219 | | } |
| | | 220 | | |
| | | 221 | | /// <inheritdoc /> |
| | | 222 | | public async Task<FlowState?> GetStateAsync(string flowId, CancellationToken cancellationToken = default) |
| | | 223 | | { |
| | 4428 | 224 | | ArgumentException.ThrowIfNullOrWhiteSpace(flowId); |
| | | 225 | | |
| | 4428 | 226 | | await using var scope = _scopeFactory.CreateAsyncScope(); |
| | 4428 | 227 | | var store = scope.ServiceProvider.GetRequiredService<IFlowStateStore>(); |
| | 4428 | 228 | | return await store.LoadAsync(flowId, cancellationToken).ConfigureAwait(false); |
| | 4427 | 229 | | } |
| | | 230 | | |
| | | 231 | | private static void EnsureIdempotentStart<TFlow, TInput>( |
| | | 232 | | FlowState existing, |
| | | 233 | | string requestedInputJson, |
| | | 234 | | string flowId) |
| | | 235 | | { |
| | 680 | 236 | | if (FlowStateConcurrency.IsSameStart(existing, typeof(TFlow).FullName, typeof(TInput).FullName, requestedInputJs |
| | 664 | 237 | | return; |
| | | 238 | | |
| | 16 | 239 | | throw new DurableFlowIdConflictException( |
| | 16 | 240 | | $"Durable flow id '{flowId}' is already bound to a different flow type or input. " + |
| | 16 | 241 | | "Idempotent retries must use the same TFlow, TInput, and semantically identical input value."); |
| | | 242 | | } |
| | | 243 | | |
| | | 244 | | } |