diff --git a/.github/workflows/auto-retry.yml b/.github/workflows/auto-retry.yml index 1f00adaa7..aab864e0f 100644 --- a/.github/workflows/auto-retry.yml +++ b/.github/workflows/auto-retry.yml @@ -2,13 +2,24 @@ name: Auto-retry failed jobs # Hosted runners are flaky and this repo runs 24 job legs per push to main. When a run on main # fails, re-run only its failed jobs (and their dependents) — the same button as "Re-run failed -# jobs" in the UI, pressed automatically. The FIRST re-run is unconditional. Later re-runs need -# EVERY failed job's log to match a known infrastructure signature: an unconditional five-attempt -# loop gave an intermittently failing product bug — a race, a lease, a watchdog — five chances to -# land green on main, and publish.yml relies on main's check as the reviewed gate. Any-job -# matching was not enough either: rerun-failed-jobs re-runs ALL failed jobs, so one genuine -# fixture flake re-ran a co-failing real bug alongside it. The run_attempt guard is what stops -# this recursing: each re-run fires `workflow_run: completed` again when it finishes. +# jobs" in the UI, pressed automatically — but ONLY when every failed job is provably an +# infrastructure flake. Two gates, applied from the FIRST re-run on (an unconditional first re-run +# gave an intermittently failing product bug — a lease race, a timing defect, a watchdog — a free +# second chance to land green on main, and publish.yml relies on main's check as the reviewed +# gate): +# 1. every failed job's log must match a known infrastructure signature, and +# 2. no failed job's log may contain an executed-test assertion failure or a build error — a +# fixture-boot flake alongside a genuine assertion failure in the same job is a real +# failure, whatever else the log says. +# Both signature sets live in ONE place, scripts/ci-retryable-failure.sh, which ci.yml's in-job +# integration retry consults too — the in-job retry used to carry a weaker copy (gate 1 only), so a +# mixed log was retried into a green job this workflow never got to see. Any-job matching is not +# enough either: rerun-failed-jobs re-runs ALL failed jobs, so one genuine fixture flake would +# re-run a co-failing real bug alongside it. A log that cannot be fetched counts as unmatched. +# Every automatic re-run leaves a ::warning:: annotation on the run naming the flake evidence, so +# a run that went green on a re-run is never indistinguishable from one that was green on its own. +# The run_attempt guard is what stops this recursing: each re-run fires `workflow_run: completed` +# again when it finishes. on: workflow_run: workflows: [CI, CodeQL] @@ -28,6 +39,10 @@ jobs: github.event.workflow_run.run_attempt < 5 runs-on: ubuntu-latest steps: + # The classifier is repo-tracked (scripts/ci-retryable-failure.sh); a workflow_run job + # checks out the default branch's copy, which is the one ci.yml on main runs with. + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Re-run failed jobs env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -35,29 +50,19 @@ jobs: RUN_ID: ${{ github.event.workflow_run.id }} ATTEMPT: ${{ github.event.workflow_run.run_attempt }} WORKFLOW: ${{ github.event.workflow_run.name }} - # ci.yml's own in-job retry gates on the same fixture-boot flake. "database is locked" - # is SQLite on a slow runner disk during the EF Core storm tests (see the tests' own - # comments); the last two are hosted-runner infrastructure loss. - FLAKE_SIGNATURES: >- - Fixture' threw in InitializeAsync|SQLite Error 5: 'database is locked'|lost communication with the server|The runner has received a shutdown signal run: | set -euo pipefail failed_jobs=$(gh api --paginate "repos/$GH_REPO/actions/runs/$RUN_ID/jobs?per_page=100" \ --jq '.jobs[] | select(.conclusion == "failure") | "\(.id) \(.name)"') echo "Failed jobs of $WORKFLOW run $RUN_ID (attempt $ATTEMPT):"; echo "${failed_jobs:-}" - if [ "$ATTEMPT" -lt 2 ]; then - echo "::notice::Re-running failed jobs of $WORKFLOW run $RUN_ID (attempt $ATTEMPT of 5): the first re-run is unconditional." - gh api --method POST "repos/$GH_REPO/actions/runs/$RUN_ID/rerun-failed-jobs" - exit 0 - fi - - # From the second re-run on, EVERY failed job's log must match a known flake signature: - # the gate is per run but rerun-failed-jobs is per job, so one matching job must not - # carry an unmatched (real) failure along for another attempt. A log that cannot be + # EVERY failed job's log must match a known flake signature and none may carry a real + # failure: the gate is per run but rerun-failed-jobs is per job, so one matching job must + # not carry an unmatched (real) failure along for another attempt. A log that cannot be # fetched counts as unmatched — an unreadable failure cannot be proven a flake. matched="" unmatched="" + real="" fetched=0 while read -r job_id job_name; do [ -n "$job_id" ] || continue @@ -76,20 +81,28 @@ jobs: continue fi fetched=$((fetched + 1)) - if grep -Eq "$FLAKE_SIGNATURES" "job-$job_id.log"; then - matched="${matched:+$matched, }$job_id ($job_name)" - else - unmatched="${unmatched:+$unmatched, }$job_id ($job_name)" - fi + # Exit 0 = flake (retryable), 1 = a real executed-test or build failure, 2 = unmatched, + # 3 = unreadable; the first line names the evidence. + verdict=0 + evidence=$(./scripts/ci-retryable-failure.sh "job-$job_id.log") || verdict=$? + case "$verdict" in + 0) matched="${matched:+$matched, }$job_id ($job_name: $evidence)" ;; + 1) real="${real:+$real, }$job_id ($job_name: $evidence)" ;; + *) unmatched="${unmatched:+$unmatched, }$job_id ($job_name: $evidence)" ;; + esac done <<< "$failed_jobs" if [ "$fetched" -eq 0 ]; then echo "::warning::Not re-running $WORKFLOW run $RUN_ID (attempt $ATTEMPT): could not read any failed job log, so the gate cannot tell a flake from a bug. Fix the log fetch above." exit 0 fi + if [ -n "$real" ]; then + echo "::warning::Not re-running $WORKFLOW run $RUN_ID (attempt $ATTEMPT): failed job(s) $real contain an executed-test or build failure; a correctness failure is never retried into green (flake-matching jobs alongside: ${matched:-none})." + exit 0 + fi if [ -n "$unmatched" ]; then - echo "::warning::Not re-running $WORKFLOW run $RUN_ID (attempt $ATTEMPT): failed job(s) $unmatched match no known flake signature; a failure that survives one re-run is treated as real (flake-matching jobs alongside: ${matched:-none})." + echo "::warning::Not re-running $WORKFLOW run $RUN_ID (attempt $ATTEMPT): failed job(s) $unmatched match no known flake signature; an unexplained failure is treated as real (flake-matching jobs alongside: ${matched:-none})." exit 0 fi - echo "::notice::Re-running failed jobs of $WORKFLOW run $RUN_ID (attempt $ATTEMPT of 5): every failed job matched a known flake signature ($matched)." + echo "::warning::Re-running failed jobs of $WORKFLOW run $RUN_ID (attempt $ATTEMPT of 5): every failed job matched a known infrastructure flake signature and none carried a test or build failure ($matched). A green re-run of this run is a flake, not a clean pass." gh api --method POST "repos/$GH_REPO/actions/runs/$RUN_ID/rerun-failed-jobs" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4cc575085..b6feb571c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,6 +42,13 @@ jobs: - name: Build run: dotnet build --configuration Release --no-restore + # The one retry classifier both integration legs and auto-retry.yml consult + # (scripts/ci-retryable-failure.sh) is plain bash; its fixture logs — including the mixed + # "fixture failed to boot AND an executed test failed" log that must never be retried — are + # checked here so a signature edit cannot silently widen what CI retries into green. + - name: CI retry classifier self-test + run: ./scripts/tests/ci-retryable-failure.test.sh + # Coverage runs through coverlet rather than MTP's `--coverage` flag or `dotnet-coverage`: # pre-instrumenting with `dotnet-coverage instrument` discards condition data, so the published # report measured zero branches, and the only Microsoft mode that keeps it on linux-x64 is the @@ -215,9 +222,13 @@ jobs: run: | # The suite runs in batches, each booting its own Aspire AppHost; on 2-core hosted runners a # boot intermittently times out or a resource fails to start before any test executes. Retry - # once on that exact signature only — genuine test failures never produce it, so they still - # fail the job on the first pass. The signature matches the *BatchFixture types in - # Batches.cs; keep it in sync if a batch fixture is ever renamed. + # once, but only when scripts/ci-retryable-failure.sh — the same classifier auto-retry.yml + # applies to whole runs — says the log is a known infrastructure flake AND carries no + # executed-test or build failure. Keying on the boot signature alone let a log that also + # held an assertion failure into the retry, and a passing second attempt made that + # correctness failure a green job. Each attempt's console log is kept in the results + # directory (itest-console.attempt.log), so a green re-run is never indistinguishable + # from a clean pass. run_integration_tests() { # The cross-product shards are excluded deliberately. Everything in them builds its hosts # in this test process and never touches the sample app, so running them here proves @@ -228,19 +239,21 @@ jobs: # Excluded by trait rather than class name on purpose: the transport contract classes live # in the matrix collections too but are not named Matrix*, and a name-based filter would # leave them behind to boot a shard fleet here. + mkdir -p "${{ github.workspace }}/TestResults/IntegrationAot" dotnet test --project tests/AsyncResponse.IntegrationTests/AsyncResponse.IntegrationTests.csproj \ --configuration Release --no-build \ --filter-not-trait "batch=matrix-*" \ --report-trx \ --results-directory "${{ github.workspace }}/TestResults/IntegrationAot" \ - 2>&1 | tee itest-console.log + 2>&1 | tee "${{ github.workspace }}/TestResults/IntegrationAot/itest-console.attempt$1.log" return "${PIPESTATUS[0]}" } - if ! run_integration_tests; then - if grep -q "BatchFixture' threw in InitializeAsync" itest-console.log; then - echo "::warning::Integration fixture failed to boot (known hosted-runner resource flake) — retrying the suite once." - run_integration_tests + if ! run_integration_tests 1; then + if evidence=$(./scripts/ci-retryable-failure.sh "${{ github.workspace }}/TestResults/IntegrationAot/itest-console.attempt1.log"); then + echo "::warning::Integration run failed on a known hosted-runner infrastructure flake ($evidence) with no executed-test or build failure — retrying the suite once. A green second attempt is a flake, not a clean pass; attempt 1's log is in the results artifact." + run_integration_tests 2 else + echo "::error::Integration run failed and is not retried ($evidence): a correctness failure is never retried into green." exit 1 fi fi @@ -342,27 +355,29 @@ jobs: # in docs/operations.md. "none" boots nothing at all — it is the in-memory suite, the Native # AOT publish gate, and the batch guards. run: | - # Same targeted boot-flake retry as the AOT integration job: rerun once only when the - # failure signature is a batch fixture failing to initialize, never for test failures. - # coverlet returns non-zero when the wrapped command fails, so the retry logic keys off the - # same exit code as before; a second attempt re-instruments and overwrites the same output. + # Same retry gate as the AOT integration job — scripts/ci-retryable-failure.sh, shared with + # auto-retry.yml: rerun once only when the log is a known infrastructure flake AND carries + # no executed-test or build failure (a boot flake next to an assertion failure is a real + # failure). coverlet returns non-zero when the wrapped command fails, so the retry keys off + # the same exit code; a second attempt re-instruments and overwrites the same coverage + # output, while each attempt keeps its own console log in the results directory. run_integration_tests() { + mkdir -p "${{ github.workspace }}/TestResults/Integration" ./scripts/coverage-collect.sh \ "${{ github.workspace }}/TestResults/Integration/integration.${{ matrix.batch }}.cobertura.xml" \ tests/AsyncResponse.IntegrationTests/bin/Release/net10.0 \ --share tests/AsyncResponse.IntegrationTests.AppHost/bin/Release/net10.0 \ --share samples/AsyncResponse.Sample/bin/Release/net10.0 \ -- dotnet test --project tests/AsyncResponse.IntegrationTests/AsyncResponse.IntegrationTests.csproj --configuration Release --no-build --filter-trait batch=${{ matrix.batch }} --report-trx --results-directory ${{ github.workspace }}/TestResults/Integration \ - 2>&1 | tee itest-console.log + 2>&1 | tee "${{ github.workspace }}/TestResults/Integration/itest-console.${{ matrix.batch }}.attempt$1.log" return "${PIPESTATUS[0]}" } - if ! run_integration_tests; then - # Matches both the original *BatchFixture types and the cross-product shards' fixtures - # (MatrixDatabaseLightFixture and friends), which carry no "Batch" infix. - if grep -q "Fixture' threw in InitializeAsync" itest-console.log; then - echo "::warning::Batch '${{ matrix.batch }}' failed to boot (known hosted-runner resource flake) — retrying once." - run_integration_tests + if ! run_integration_tests 1; then + if evidence=$(./scripts/ci-retryable-failure.sh "${{ github.workspace }}/TestResults/Integration/itest-console.${{ matrix.batch }}.attempt1.log"); then + echo "::warning::Batch '${{ matrix.batch }}' failed on a known hosted-runner infrastructure flake ($evidence) with no executed-test or build failure — retrying once. A green second attempt is a flake, not a clean pass; attempt 1's log is in the results artifact." + run_integration_tests 2 else + echo "::error::Batch '${{ matrix.batch }}' failed and is not retried ($evidence): a correctness failure is never retried into green." exit 1 fi fi diff --git a/.github/workflows/loadtest.yml b/.github/workflows/loadtest.yml index 087109a60..41886080d 100644 --- a/.github/workflows/loadtest.yml +++ b/.github/workflows/loadtest.yml @@ -106,7 +106,17 @@ jobs: rate="${LOADTEST_RATE:-20}" if [ "${GITHUB_EVENT_NAME:-}" = "push" ]; then - rate="${LOADTEST_RATE:-5}" + # The broad profile is 60 scenarios, so this is a PER-SCENARIO rate: 5/s offered 300 + # requests per second to 14 SUT apps plus ~10 broker/database containers on one runner, + # and the request/response scenarios each carry ~1 s of simulated remote work. That put + # the fleet ~30x past its knee: even on runs that PASSED, the Redis-channel scenarios + # averaged ~31 s per request against the client's ~35 s ceiling, so the gate was decided + # by how fast that particular runner happened to be — a 5 % slower one pushed the mean + # past the ceiling and the failure rate from 2.5 % to 7.2 % with no code change behind + # it (the sub-second worker scenarios were unchanged at 0.65 s across both runs). + # 3/s keeps every scenario and both gate thresholds while restoring headroom, so a + # failure means the library, not the runner. Override with LOADTEST_RATE to push harder. + rate="${LOADTEST_RATE:-3}" fi args=( diff --git a/CHANGELOG.md b/CHANGELOG.md index c597e6d0f..4585ea660 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,418 @@ work that has landed on `main` but not yet shipped. Security reporters credited ### Changed +- **Round-39 review (2026-09-14): settlement over the whole failure set, storage-side history costs, and honest test claims.** + - *A transient sibling failure keeps the message for redelivery whatever precedes it.* The + lost-subscriber dispatcher settled a shared-correlation fan-out on the FIRST failure it saw: + with a deterministic fault (an unresolvable target) ahead of a transient one (a resume whose + dependency was briefly down) and a sibling that succeeded, the residual was classified as + deterministic, the message was acknowledged, and the transient registration — a valid waiter + — lost the only copy of its payload; the opposite order propagated for redelivery. The verdict + is now taken over every failed registration, on the response and the exception route alike: + deterministic failures are logged and their registrations kept for the watchdog, any transient + failure in the set propagates as `RecoveryCallbackFailedException` so the transport redelivers + to the registrations still armed, and the message is acknowledged only when every failure was + deterministic. With no success at all, a sibling whose failure-callback ladder was exhausted + propagates ahead of an earlier sibling's fault, so the ingress no longer burns its own retry + ladder and escalates through `SetException` into the callback that just gave up. + - *Typed in-memory delivery is body-free.* Each waiter re-materializes the published payload + from its wire bytes through the raw reader; a payload that did not fit the waiter's type failed + inside the payload with the reader's own `JsonException`, whose `Path` named the offending + dictionary key — into the waiter's task and, through `SetError`, the wait activity's status. + The materialization now goes through `JsonSafety`: the waiter faults with the same position-only + `InvalidDataException` every broker channel uses. + - *MySQL: a duplicate create no longer needs a second pooled connection.* The 1062 handling + confirmed "this flow id exists" on a freshly opened connection while the create still held its + own, so on a pool of one a single duplicate start timed out with "All pooled connections are in + use", and concurrent idempotent starts starved any pool waiting on each other. The check runs + on the connection the create holds. + - *Database channels: acknowledged history no longer travels with every sweep.* The sweep + re-reads a subscribed correlation id's retained rows on every tick and every targeted signal, + acknowledged rows included (a fan-out waiter in another process still needs them), and every + row came back with its body only to be dropped by the pre-filter — a long-lived progress + subscription's sweep cost grew with its whole retained history. The PostgreSQL, SQL Server, and + MongoDB page queries now ship `envelope_json` only for rows nobody has acknowledged; + acknowledged rows come back header-only and the sweep hydrates, in one by-id read + (`LoadMessagesByIdAsync`), only the rare acknowledged row a live subscription has not seen, + then admits the page in order as before. Same delivery semantics, same watermark, no schema + change. + - *Cosmos: the size budget is enforced on the document Cosmos receives.* `MaxStateBytes` + measured the ledger JSON, but the document embeds it as a string and escapes it again, so a + 1.2 MB ledger of escaped characters was a 2.4 MB document — accepted by the guard, refused by + the 2 MB item cap on every retry. Creates and checkpoints now measure the complete document + through the registered client's serializer (or the SDK default's shape) and fail with + `FlowStateTooLargeException` naming the document size; the ledger-only check stays as the + cheap pre-check. + - *Testing: a simulated restart refuses to overstate what it proved.* `SimulateRestartAsync` + proceeded past user code that outlived the graceful stop — the "dead" execution kept running + beside the new incarnation and performed its side effect after the restart had returned. The + restart is cooperative (there is no process to kill), so it now fails with + `InvalidOperationException` when user code is still executing after the stop lapsed; the new + `AsyncResponseTestHarnessOptions.AbandonLingeringExecutionsOnRestart` accepts the overlap + explicitly. Engine-owned parks (awaited steps, in-process timers) are unaffected. The + testing guide documents what a restart can and cannot claim and points crash-at-checkpoint + assertions at `CrashBeforeStep`/`CrashAfterStep`. + - *NATS: a stalled in-progress heartbeat cannot strand the subscriber loop.* The heartbeat ran + with `CancellationToken.None` and the batch joined the renewal loop without a bound, so one + heartbeat wedged on a dead socket held the batch after every message in it had settled — no + further batch was fetched, a stop never completed, and the supervisor had nothing to restart. + The heartbeat now carries the batch's cancellation token into the SDK call and the join is + bounded by one heartbeat interval (`AckWait`/3), after which the loop is abandoned with a + warning and unsettled deliveries fall back to the server's `AckWait`. + - *Durable flows: the ledger cost curve is benchmarked.* Every checkpoint rewrites the whole + ledger (about N²/2 step-results over an N-step run — documented under "Supported ledger + budgets"); the new `LedgerGrowthBenchmarks` measures a complete run at 50/200/400 steps so the + curve is checked, not inferred. Incremental checkpoint persistence stays on the roadmap. + - Tests: red-on-old proofs against f92f1e7 for every code change — six orderings × two routes + of the settlement verdict plus the exhausted-sibling precedence, the typed in-memory leak + probe, the escaped-Cosmos-document guard, the header-only sweep hydration (Mongo harness) and + the three providers' direct integration tests, the MySQL pool-of-one duplicate create, the + stalled-heartbeat NATS batch, and the harness refusing a lingering execution — plus new-API + pins. + +- **Round-38 review (2026-09-11): recovery invariants enforced consistently across persistence and settlement.** + - *A parked child can no longer lose its parent to a concurrent checkpoint.* Round 36 made a + descendant's long park extend every ancestor's ledger, but a lost revision race was treated + as success ("a concurrent writer means the ancestor is alive and re-stamping its own + expiry"). The competing write was computed without the park in view — the parent replaying + its child-await from a snapshot taken before the child persisted its sleep, or the executor's + per-attempt save — and stamped the plain `StateExpiry`, so the parent expired under an + hour-long wait with the child's wake-up already published, and every step past the parent's + child-await was lost. The ledger now carries a **retention floor**, `FlowState.RetainUntilUtc` + (additive wire property, omitted when unset): a park stamps it on the run and on every + `Running` ancestor, and every ledger write of a non-terminal run — checkpoint, per-attempt + save, recovery or operator mutation — raises the TTL it stamps to reach the floor, so no + write that knows nothing about a wait can shrink the retention under it. The ancestor + extension re-reads after a lost race: a write that already carries a floor reaching the park + proves the retention and ends the walk step; otherwise the extension is retried against the + new revision, up to four times, and losing every attempt abandons the park with nothing + published (the delivery retries it later) instead of parking on unproven retention. + - *Kafka: a malformed message behind a detached handler is settled in partition order.* A + message that could not be parsed into a delivery was dead-lettered and its offset stored the + moment it was consumed. Consumed behind a detached handler of the same partition (a rebalance + handing the partition back with its pause reset delivers the next record), that stored the + partition **past** the unfinished message, the auto-committer committed it, and a crash + skipped the valid job for good — with only the malformed record's copy in the dead-letter + topic. Such a message is now held behind the partition's detached handler, exactly like a + valid delivery, and buried with its offset stored in its turn once the handler settles. With + nothing detached on the partition it is discarded at once, as before. + - *Kafka: a poll-loop failure no longer waits without limit for unrelated handlers.* When a + consume failed, the fault teardown awaited every detached handler before closing the + consumer, so a transient broker failure disabled the subscriber for as long as a durable-flow + step awaiting a remote response took — and the configured reconnect policy never ran. The new + `KafkaSubscriberOptions.FaultDrainTimeout` (default 5 s; `0` abandons at once) bounds it: + handlers that settle within the budget get their offsets stored and committed by the close; + the rest are abandoned — offsets unstored, messages redelivered on the rebuilt consumer while + the abandoned handler may still be running (handlers are at-least-once), each one's eventual + outcome logged — and the session's cancellation token stops their retry ladders. A graceful + stop is unchanged (the host's shutdown budget bounds it). + - *In-memory transport: delayed jobs are bounded.* Neither `QueueCapacity` nor + `InJobOverflowCapacity` covered delayed jobs: every scheduled publish retained its envelope + and captured execution context against no limit, and when a burst's timers fired each started + a channel write that pended outside the bounded queue. The new + `InMemoryWorkerTransportOptions.DelayedJobCapacity` (default 4096) reserves a slot per delayed + job from acceptance until the fired job has entered the queue: a delayed publish from outside + a job waits for a slot (honoring its cancellation token); one made from inside a running job — + a flow parking on a timer — is rejected with `InvalidOperationException` (the publishing job + fails and is redelivered), never parked. New gauge `asyncresponse.worker.inmemory_delayed_jobs` + and counter `asyncresponse.worker.inmemory_delayed_rejections`. + - *Recovery-state readers are body-free.* The Redis, NATS, PostgreSQL, SQL Server, and MongoDB + recovery-state stores deserialized stored registrations with the raw reader and logged its + `JsonException`, whose `Path` is built from the registration's `Context` keys — tenant and + auth baggage — so a malformed blob copied them into the application log. They now go through + `JsonSafety` like every other reader of a body the library did not write; the logged failure + carries size and position only, and unreadable-state behavior (skip, count, refuse to + overwrite) is unchanged. + - *A ledger inconsistent with itself is unreadable, not absent.* Every built-in store reads the + JSON and the revision from one row or document, so a revision inside the JSON that disagrees + with the stored one, or a flow id inside the JSON that is not the key, is a corrupt or + mis-restored row that is physically present. Loading it as `null` told the executor to + acknowledge the wake-up as belonging to a deleted flow, and the run behind the row lost its + only wake-up. `DurableFlowStoreShared.ReadState` and the in-memory store now throw + `FlowStateUnreadableException` naming both revisions (or the identity mismatch), which rides + the transport's retry and dead-letter path; the `IFlowStateStore.LoadAsync` contract and the + custom-store checklist say so. + - Tests: 9 red-on-old proofs against 94c3ddb (the lost extension race, the inconsistent + in-memory ledger through the store and the executor, five recovery-reader leak probes, the + delayed-job bound, the malformed message behind a detached Kafka handler), plus new-API pins + for the floor's wire shape and arithmetic, the lease and mutation write paths, the floor on + every ancestor, the proven-by-re-read and losing-every-attempt outcomes, the delayed-job + capacity (rejection, counter, gauge, drain, validation), and the Kafka fault teardown (abandon + and reconnect, settle-within-budget, validation). +- **Round-37 review (2026-09-11): where provider behavior meets the orchestration guarantees.** + - *Kafka: a long handler no longer stalls the poll loop.* In `AckAfterHandlerCompletes` mode the + poll thread awaited the whole handler, so a durable-flow step awaiting a remote response or a + timer for longer than `max.poll.interval.ms` (5 minutes by default) got the consumer evicted + from its group, its partitions rebalanced, the same job redelivered to a peer that started it + again, and every other partition assigned to the consumer stalled behind it. A handler still + running after the new `KafkaSubscriberOptions.DetachHandlerAfter` (default 1 s; `0` detaches + at once) is now detached: its partition is paused (Kafka's own ordering primitive — nothing + buffered in-process), the handler and its retry ladder run on the thread pool, and the poll + thread keeps polling. The poll thread — the only thread that touches the consumer — stores the + offset and resumes the partition once the handler settles (within one `BackpressurePollDelay`), + starts a message held behind it in order, and surfaces a burial that failed for good exactly + as the inline path did (`KafkaDeadLetterPublishFailedException`, offset unstored). A stop + waits for detached handlers and commits their offsets before the consumer closes. Detached + handlers for different partitions run concurrently. Startup validation now bounds + `DetachHandlerAfter + PollTimeout` to half of `MaxPollInterval` and no longer bounds the + retry-delay budget, which never overran the poll thread on its own once detached. Verified + against the real broker: a 12-second handler under an 8-second `max.poll.interval.ms` runs + exactly once (`KafkaLongHandlerIntegrationTests`, brokers batch). + - *Cosmos DB lease operations no longer share a mutable query.* Round 36's lease projection was + one static `QueryDefinition` parameterized per call, and `WithParameter` replaces the named + parameter in place — so two flows' lease operations interleaving on one store instance could + execute flow A's query under A's partition key asking for flow B's id, read "no document", and + fail a healthy renewal (abandoning and replaying the run). Only the SQL text is shared now; + every call builds its own definition. + - *An oversized worker job fails at the producer.* The ingress acknowledges a message over + `AsyncResponseOptions.MaxInboundMessageChars` without executing it (redelivering it would + hot-loop), but nothing stopped the producer from publishing one: the transport took it, the + ingress dropped it, and `StartAsync` returned a flow id for a `Running` ledger with + `Attempts = 0` that nothing would execute. Every `EnqueueWorkerAsync` overload and + `IDurableFlows.StartAsync` now measure the serialized envelope — the transports' own + serialization, in the UTF-16 units the ingress compares, escaping included — and throw the new + `WorkerJobTooLargeException` (`SerializedLength`, `Limit`) before publishing. A flow start + surfaces it unwrapped and unretried (nothing persisted). The hot path pays no second + serialization: a cheap upper bound (every string fully escaped, scalars at a fixed allowance) + skips the exact measurement for envelopes that provably fit. + - *Redis channel: a progress flood behind a slow predicate faults the wait instead of growing + memory.* The subscription handler awaited admission to the bounded per-correlation-id + executor, which never backpressured the publisher (Redis pub/sub is fire-and-forget) — it only + parked the SDK's message loop while the SDK's unbounded `ChannelMessageQueue` behind it filled + (20,000 messages held against an executor of 1,024). Admission is non-blocking now; a response + that finds the buffer full faults the wait with the new overload form of + `AsyncResponseIndeterminateDeliveryException` (`BufferedMessages`), tears the subscription + down, and counts the new `asyncresponse.channel.overloaded_waits` counter (`channel` tag) — + never buffered without bound, never silently dropped (a terminal response may be among the + queued or refused ones). Messages queued behind the fault are skipped unprocessed. Durable + flows restart the awaiting step on the fault as they do for the disposal-drain form. + - *CI retry classifier: per-test classification, no `grep | head`.* Two gaps let a real + failure be retried into green: an executed test dying with a non-assertion exception (a + `NullReferenceException`) was invisible to the whole-log assertion scan, so a fixture-boot + flake elsewhere in the same log retried the job; and `grep -o … | head -n 1` under `pipefail` + failed with SIGPIPE on a log of thousands of assertion failures, which skipped the real-failure + branch entirely. `scripts/ci-retryable-failure.sh` now splits the log into failed-test blocks + and judges each — assertion or `XunitException` is real, a flake signature explains it, and a + block with neither is an executed test that failed for an unrecognized reason, which is real + too — and every signature match is a single `grep -m 1` trimmed in bash. A block-less log with + no signature is still `unmatched`. Both reproduced inputs are in the self-test, which the old + script answers `flake` and the new one `real`. + - *Local Redis binds to loopback.* `docker-compose.yml` published `6379:6379` on every host + interface with protected mode off — an unauthenticated write path into recovery descriptors + and response envelopes for anyone on the segment. It is `127.0.0.1:6379:6379` now; + `docs/security.md` says why. + - *Documented ledger budgets.* Every checkpoint rewrites the whole ledger (quadratic cumulative + cost over a run), which the size warning detects but does not reduce. `docs/durable-flows.md` + now states the supported budgets — ledger ≤ `LedgerSizeWarningBytes`, a few hundred retained + step results of a few KiB each, an input that fits the worker envelope — with the measured + curve, the store-large-results-by-reference pattern in code, and child flows as the partition + strategy; incremental persistence stays on the roadmap under the same revision and lease + fences. + - Tests: 9 red-on-old proofs against ba63beb (the Cosmos parameter race with a query mock that + answers what it is asked; the producer-side budget, escaping included, and the flow start; the + Redis flood; the Kafka poll loop under a long handler and the other partitions behind it; both + classifier inputs), plus new-API pins for the exception, the estimator's upper-bound property, + the overload constructor and counter, the detach knob's validation and the dispatcher's detach + path (inline settle, detach and pause, zero budget, held message order, burial fault, disposal + with and without cancellation, revoked-partition pause). 2886 unit tests green on net10.0 and + net8.0; the classifier self-test grew from 13 to 20 logs. +- **Round-36 review (2026-09-10): the contracts between scheduling, delivery, checkpoints, and + retention that recovery depends on.** + - *A scheduled occurrence that fell due during a broker outage is no longer lost.* Round 35 + made a start publish-first (a failed publish persists nothing), but the scheduler's re-drive + still read an absent ledger as "expired or deleted" and settled the entry — so once an outage + outlasted the start's own retry ladder the occurrence was never started, and the startup probe + could not find a run that was never persisted. The re-drive now distinguishes an occurrence + still awaiting its first successful publish (no ledger is the expected shape; start it again) + from one the startup probe queued off an existing ledger (absence really is expiry). An + occurrence whose publish was still failing when the process died is skipped like any other + missed occurrence, as documented. + - *`AwaitChildFlowAsync` returns the same object on the first completion as on every replay.* + The first completion used to return the fully loaded child (ambient `Context`, grandchild + results) while replays returned the reduced memoized snapshot, so a parent could branch + differently after a restart on a step it had already completed. Both paths now return the + snapshot (no `Context`; the `ResultJson` of the child's own child-flow steps elided); the + interface documentation states the shape. + - *Response-channel and ledger readers no longer echo inbound property names.* The Redis, NATS, + and database (PostgreSQL, SQL Server, MongoDB) channels deserialized envelopes with the raw + reader, so a payload that failed to convert faulted the waiter with — and logged — a + `JsonException` whose message quotes dictionary keys read off the wire + (`Path: $.Payload.Values['…']`); the durable-flow ledger reader chained the same raw exception + into `FlowStateUnreadableException`, reachable through a start job's carrier. All of them go + through the body-free `JsonSafety` contract now: a failure the *reader* authored is replaced + by size and position alone (so a malformed body faults the waiter with `InvalidDataException` + instead of `JsonException`), while a violation the *library* authored keeps its message — + "SchemaVersion is required.", "Payload is null or absent on a Success envelope" — because + those name only the wire contract's own properties and never a byte of the body. They stay + plain `JsonException`s, so every classification and `catch` is unchanged. `docs/security.md` + names the covered readers and the distinction. + - *The in-memory transport's in-job overflow is bounded.* A follow-up publish that finds the + queue full spills into an overflow that was unbounded, so a fan-out handler could retain every + envelope and captured execution context until the process ran out of memory with + `QueueCapacity` giving no signal. New `InMemoryWorkerTransportOptions.InJobOverflowCapacity` + (default 4096; `0` allows none; negative rejected at startup): past it a follow-up publish + throws `InvalidOperationException` and the publishing job is redelivered by the retry ladder — + make in-job publishes idempotent. Two new instruments: + `asyncresponse.worker.inmemory_overflow_depth` (gauge) and + `asyncresponse.worker.inmemory_overflow_rejections` (counter). + - *Cosmos DB lease maintenance stops moving the ledger.* Acquire, renew, and release + point-read the whole document (`stateJson` included) and replaced it; an idle execution's + every 20-second heartbeat therefore transferred and re-serialized its full ledger twice. They + now read a projection of the lease fields plus `_etag` with a partition-scoped query and apply + a conditional partial update (`PatchItemAsync` on `leaseId`, `leaseExpiresAtUtc`, `ttl`; + `IfMatchEtag`; no content response). A projection without `_etag` throws rather than reporting + the lease free. Verified against the Cosmos emulator; measure RU before sizing throughput. + - *Ancestor retention is part of a child's park.* A descendant parking longer than `StateExpiry` + extends its ancestors' ledgers; that walk swallowed store failures and stopped silently after + 16 levels, so a child could park "successfully" — wake-up published — while the parent it + would complete into expired mid-wait, abandoning everything past the parent's child-await. A + failed ancestor write now fails the park before any wake-up is published (the delivery is + redelivered and retries the chain), the chain is walked to the root with cycle detection, and + a chain that revisits an id or is nested more than 256 levels deep fails the run terminally. + - *Lease release on disposal is bounded.* The final `ReleaseLeaseAsync` ran unbounded with + `CancellationToken.None`, so a store that never answered kept a finished execution's disposal + — the executor's `await using`, the job's scope, the worker slot, the acknowledgement — + pending indefinitely. It now gets a cancelable token and a 10-second budget; past it the call + is cancelled and abandoned with a warning (its eventual outcome observed), and the server-side + lease expires on its own. + - *CI: one retry classifier.* The integration jobs' in-job retry keyed on the fixture-boot + signature alone, so a log carrying both a boot flake and an assertion failure was retried and + a passing second attempt made the correctness failure a green job — one `auto-retry.yml` + (which already vetoed such logs) never got to see. Both now consult + `scripts/ci-retryable-failure.sh`; each attempt's console log is kept in the results + artifact; the classifier's fixture logs (the mixed one included) run as a self-test in + `build-and-test`. + - Tests: 20 behavior pins proven red against 145aa8c in a worktree (the scheduler with the real + starter; child-snapshot parity; body-free readers on Redis, NATS, the shared database-channel + source, the ledger reader, and the start carrier through the ingress; the in-job overflow; + ancestor outage, depth, and cycle; a hanging release), plus new-API pins for the overflow + option, its metrics, and the depth ceiling; the Cosmos unit tests rewritten for query + patch + and the store contract run against the Cosmos emulator. 2842 unit tests green on net10.0 and + net8.0. +- **Round-35 review (2026-09-09): delivery correctness at the persisted-state / acknowledgement / + execution handoffs.** + - *A durable-flow start can no longer be stranded.* `IDurableFlows.StartAsync` publishes its + worker job **first** and writes the ledger second; the job carries the initial ledger and its + new target, `IDurableFlowExecutor.CreateAndExecuteAsync(flowId, initialStateJson)`, creates + the run (insert-if-absent) before executing it. The publish is therefore the start's single + commit point: a crash before it leaves nothing, a crash after it leaves a job whose execution + creates and runs the flow — the previous order left a committed `Running` ledger with + `Attempts = 0` that nothing would ever execute and no store API could enumerate. + `DurableFlowNotDispatchedException` now means *nothing was persisted* (its `FlowId` is for an + idempotent retry, not for re-driving an orphan); a conflicting reuse of an explicit id is still + reported to the starter (`DurableFlowIdConflictException`) and the already-published job is + dropped by the executor on the same test. The input travels twice (job + ledger) — mind the + 256 KiB SQS/Azure Service Bus message caps for large inputs. Custom + `IAsyncResponseCallbackAuthorizer`s that allow `IDurableFlowExecutor` type-level need no + change; per-method allowlists must add `CreateAndExecuteAsync`. + - *Kafka never commits past a message it could not dead-letter.* In ack-after-handler mode and + the malformed-message discard, an exhausted dead-letter publish faults the subscriber + (`KafkaDeadLetterPublishFailedException`, internal) instead of being swallowed with the offset + left unstored: Kafka commits a partition position, so the next successful settlement on the + partition was committing past the failed message and a restart skipped it with no record. The + supervisor rebuilds the consumer after its backoff and the burial is retried per restart — a + loud, bounded-rate loop that parks the subscriber at the poison message until the dead-letter + topic is fixed. Early-ACK burial failures are unchanged (already committed; surfaced via + `OnBackgroundFailure`). + - *Partial recovery success no longer acknowledges the sibling's payload.* When several + registrations share a correlation id and one callback succeeds while another fails + transiently, the publish throws `RecoveryCallbackFailedException` (the ingress passes it + through, so the transport redelivers) instead of returning normally; the successful + registrations are consumed first, so the redelivery reaches only the failed one. A + deterministic sibling failure keeps the log-and-acknowledge behavior. The exception's message + and doc now cover both paths (`Attempts` is 1 for a partial fan-out). + - *`async void` callback implementations are refused before invocation.* A void-returning + target whose resolved implementation carries the compiler's async state-machine marker throws + `CallbackTargetUnresolvableException` (deterministic) instead of being acknowledged with its + body still running and its DI scope disposed. Checked at plan time for class-typed services + and on first dispatch (cached per implementation type) through interfaces; synchronous `void` + targets are unaffected. + - *Ambiguous callback targets fail at registration.* The expression converter behind + `EnqueueWorkerAsync` and `OnLostSubscriberResume/Failure` runs the dispatcher's binding + validation (unique name + arity, no by-ref or open-generic parameters) and throws in the + caller's stack; an interface with `Run(int)` / `Run(string)` used to accept `svc => svc.Run(1)` + and fail every dispatch as ambiguous after publication. + - *One saturated correlation id no longer stalls database-channel delivery.* The PostgreSQL, + SQL Server, and MongoDB dispatch sweep admits work to a correlation id's serial executor + without waiting (`SerialExecutorRegistry.TryEnqueue`): at capacity the rest of that id's + messages stay unclaimed in the store, in order, and only that id is rescanned after one poll + interval, while every other correlation id keeps delivering. Previously the sweep awaited the + capacity, so a waiter wedged in a slow `Until` predicate under a progress flood blocked every + waiter in the process. + - *Ledger-growth early warning.* New `DurableFlowOptions.LedgerSizeWarningBytes` (default + 512 KiB; `null` disables; validated positive): the executor logs a warning naming the flow when + its estimated ledger size first crosses the threshold and again at each doubling. Every + checkpoint rewrites the whole ledger, so a run of N similar steps serializes about N²/2 + step-results over its lifetime; the docs now state that cost model, the mitigations (small + results, references, child flows), and the DynamoDB caveat. + - *Sample: every test affordance is behind the switch.* `/arm`, `/crash`, `/publish`, + `/lost-subscriber-flow`, `/emit-response`, `/calls`, `GET /durable-flow/{flowId}`, and + `POST /durable-flow/{flowId}/resume` join the round-34 mutation routes behind + `Sample:EnableTestEndpoints` (Development default on; Production 404). An integration test + pins the exact Production route inventory; the Native AOT gate opts its published sample in. + - Tests: 29 new or rewritten cases; 18 behavior pins proven red against 684a3fb in a worktree + (the DB-channel sweep pin runs the shared source through the Mongo mock harness). 2823 unit + tests green on net10.0. + - *CI: two wall-clock races and a saturated load profile.* Follow-up to the round-35 push, whose + red pipelines were both environment-timing, not behavior (the sub-second load-test scenarios + and the `ExpressionToReflectionCall` benchmark were unchanged). The early-ACK drain fact now + reserves 1.5 s for 30 ms of burials instead of 750 ms for 75 ms, and the durable-flow store + end-to-end fact waits 30 s (not 5 s) for a run that finishes in milliseconds — both assert + that something happened, never how fast. The load test's push profile drops to 3 requests per + second per scenario: at 5/s its 60 scenarios offered 300 req/s to a fleet that completes about + 210, so even passing runs averaged ~31 s per request against the client's ~35 s ceiling and + the gate was decided by runner speed. Both gate thresholds are unchanged. + +- **Round-34 review (2026-09-08): recovery correctness, retention, and settlement.** + - *Lease-less checkpoint fencing.* The "won response after the lease lapsed" checkpoint is now + fenced to the attempt that won it, exactly like a recovered payload: it applies only while the + reloaded step is still pending on the same correlation id and the run is Running or + Suspended. A takeover that re-triggered the step under a new id, or failed the run, no longer + has its ledger completed with the stale executor's response (the response is discarded with a + warning). + - *Failure callbacks that keep failing no longer acknowledge the response.* When the + lost-subscriber failure callback fails all four in-process attempts transiently, the publish + throws the new `RecoveryCallbackFailedException` (correlation id, attempts, cause) instead of + returning normally. The broker ingress passes it through untouched — no second retry ladder, + no `SetException` escalation — so the transport redelivers the terminal signal under its own + `MaxDeliveryAttempts`/dead-letter policy; the registration stays armed. Deterministic faults + (unresolvable target, unbound method) are still logged and acknowledged. Direct callers of + `SetResponse`/`SetException` now see the exception too. + - *Worker-argument conversion is body-free.* Converting a worker-job argument or recovery + payload into the callback's parameter type goes through the same scrubbing as the envelope + parse: the exception the ingress logs carries size and reader position only, never the + payload's property names or dictionary keys (`Path: $.`). + - *Memoized child snapshots are depth-independent.* A parent's memo of a completed child elides + the child's own child-flow step results (id, completion, and fault marker stay), so nested + parent → child → grandchild chains no longer grow the ledger exponentially with depth (a + 72-byte leaf reached ~600 KB at depth 15). Load a grandchild's snapshot from its own ledger via + `ChildFlowId` when you need it. + - *In-memory flow store retention.* `WithInMemoryDurableFlows` now sweeps every expired ledger + on flow creation, at most once per minute of the engine clock — expired entries were removed + only when their own id was loaded again, which a completed run's never is. + - *Relational prune budget.* PostgreSQL, SQL Server, MySQL, SQLite, Oracle, and EF Core stores + drain expired rows in 1000-row batches for up to a new `PruneBudget` (default 2 seconds; zero + keeps the historical single batch) instead of one batch per `PruneInterval` (~3 rows/second). + The outcome is no longer silent: `asyncresponse.flow_state.pruned_rows`, `prune_failures`, and + `prune_budget_exhausted` counters (tagged by provider), plus Warning logs through the store's + `ILogger` (a new optional constructor parameter DI supplies). `DurableFlowStoreShared. + PruneQuietlyAsync` changed shape for the shared source. + - *Scheduled occurrences are never abandoned once committed.* An occurrence whose ledger was + committed but whose worker job could not be published is re-driven every + `ScheduledFlowOptions.RedriveInterval` (new, default 30 s) until published, seen executed, or + expired; at startup each schedule probes the last `StartupRedriveWindow` (new, default 1 hour) + of occurrences and re-drives any Running ledger with zero attempts — the crash-between-commit- + and-publish signature. Both options validate at registration. + - *Sample test routes are gated.* The sample's `/seed-recovery`, `/test/recovery/{id}`, and + `/test/reset` are mapped only in Development or with `Sample:EnableTestEndpoints=true` (the + integration AppHost and load-test launcher set it); Production answers 404. + - *CI auto-retry never retries a correctness failure.* The flake-signature gate applies from the + first re-run (the first was unconditional), a job whose log carries an executed-test assertion + or build error is never re-run whatever else it matched, and every automatic re-run leaves a + warning annotation naming the flake evidence. + - Tests: 38 new cases (2 in-process integration tests for the sample gate); the behavior pins + were proven red against 54b9590 in a worktree. Docs: durable-flows, recovery, security, + configuration, timers-and-scheduling, durable-flow-state-stores, observability. - **Durable-flow failure callbacks are correlation-scoped.** The lost-subscriber FAILURE target a flow registers for an awaited step is now `IDurableFlowExecutor.FailAsync(flowId, exception, correlationId)` (new overload): it fails the run only while a step is still pending on that diff --git a/Directory.Packages.props b/Directory.Packages.props index 2cc1f3ccf..5b2754266 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -18,74 +18,75 @@ - - - - - - - + + + + + + + - + - + - - + + - + - - - + + + - + - + - - - - + + + + - - - - - - - + + + + + - - - - + + + + @@ -115,13 +116,13 @@ SQL Server, and that shared source compiles into the multi-targeted unit-test project too. ========================================================================================== --> - - - + + + - - - + + + \ No newline at end of file diff --git a/README.md b/README.md index 4d15696da..f493a7989 100644 --- a/README.md +++ b/README.md @@ -363,7 +363,7 @@ execution leases. They are independent axes — combine any one of each. | Azure Service Bus | peek-lock ACKs; reuses your own `ServiceBusClient` (e.g. Azure Identity) if registered | | Google Pub/Sub | streaming pull; redelivery bounds via the subscription's DeadLetterPolicy | | AWS SQS | long-poll `ReceiveMessage` (up to 10/batch), visibility-timeout redelivery, native dead-letter via redrive policies (provisionable with `CreateQueues`), opt-in FIFO ordering per flow; reuses your own `IAmazonSQS` if registered | -| Kafka | classic consumer groups, manual offset management, in-process bounded retry, `{topic}.deadletter` topics; also covers Redpanda / Amazon MSK / WarpStream / Aiven / Confluent Cloud | +| Kafka | classic consumer groups, manual offset management, in-process bounded retry, `{topic}.deadletter` topics; a handler that outlives `DetachHandlerAfter` runs detached with its partition paused, so long flow steps never overrun `max.poll.interval.ms`; also covers Redpanda / Amazon MSK / WarpStream / Aiven / Confluent Cloud | | NATS | JetStream explicit ACKs, NAK-with-delay redelivery, dead-lettering | | PostgreSQL | queue table claimed with `FOR UPDATE SKIP LOCKED`, idempotent publish, dead-lettering | | SQL Server | queue table claimed with `UPDLOCK, ROWLOCK, READPAST` (the `SKIP LOCKED` equivalent), idempotent publish, dead-lettering | @@ -570,7 +570,7 @@ is reused automatically; otherwise the AWS SDK credential and region chain is us | Existing infrastructure | Typical registration | Important behavior | |---|---|---| -| Kafka / Redpanda / MSK / Confluent | durable channel + `.WithKafkaTransport(...)` + one flow store | Correlation id is the partition key; partition count bounds consumer parallelism and a retry delays that partition. | +| Kafka / Redpanda / MSK / Confluent | durable channel + `.WithKafkaTransport(...)` + one flow store | Correlation id is the partition key; partition count bounds consumer parallelism and a retry delays that partition — only that partition: long handlers are detached from the poll thread. | | PostgreSQL | `.WithPostgreSqlChannel()` + `.WithPostgreSqlTransport(...)` + `.WithPostgreSqlDurableFlows(...)` | `LISTEN/NOTIFY` wakes response readers; workers claim queue rows with `FOR UPDATE SKIP LOCKED`. | | SQL Server | `.WithSqlServerChannel(...)` + `.WithSqlServerTransport(...)` + `.WithSqlServerDurableFlows(...)` | Adaptive response polling; workers claim rows with `UPDLOCK, ROWLOCK, READPAST`. | | AWS | Redis/PostgreSQL channel + `.WithSqsTransport(...)` + `.WithDynamoDbDurableFlows(...)` | Native visibility-timeout redelivery and redrive-policy dead letters; FIFO queues order by correlation id. | @@ -651,7 +651,7 @@ code pushes to `main`; per-commit trends with regression alerting are published ## How it's tested -**8,000+ test executions per CI run** — 5,500+ unit and 2,500+ integration cases against real +**8,300+ test executions per CI run** — 5,800+ unit and 2,500+ integration cases against real servers. The only skips are declared ones: capability-gated conformance facts (the delayed-delivery timing contract skips on the five transports without native scheduling — Redis, NATS, Kafka, RabbitMQ, and Google Pub/Sub) and two explicit opt-out switches (`ASYNCRESPONSE_SKIP_AOT_GATE` for diff --git a/benchmarks/AsyncResponse.Benchmarks/KafkaBenchmarkFakes.cs b/benchmarks/AsyncResponse.Benchmarks/KafkaBenchmarkFakes.cs index 5cc8a9cf6..99dc642d0 100644 --- a/benchmarks/AsyncResponse.Benchmarks/KafkaBenchmarkFakes.cs +++ b/benchmarks/AsyncResponse.Benchmarks/KafkaBenchmarkFakes.cs @@ -25,6 +25,14 @@ public void ResumeAssignment() { } + public void PausePartition(string topic, int partition) + { + } + + public void ResumePartition(string topic, int partition) + { + } + public void Close() { } diff --git a/benchmarks/AsyncResponse.Benchmarks/LedgerGrowthBenchmarks.cs b/benchmarks/AsyncResponse.Benchmarks/LedgerGrowthBenchmarks.cs new file mode 100644 index 000000000..782f3eeb9 --- /dev/null +++ b/benchmarks/AsyncResponse.Benchmarks/LedgerGrowthBenchmarks.cs @@ -0,0 +1,83 @@ +using BenchmarkDotNet.Attributes; +using Microsoft.Extensions.DependencyInjection; + +namespace AsyncResponse.Benchmarks; + +/// +/// The durable-flow ledger's cost curve: every checkpoint rewrites the WHOLE ledger, so a run of +/// N steps with similar result sizes serializes about N²/2 step-results over its lifetime. This +/// measures one full run — N checkpoints through the process-local store, each carrying every +/// earlier result — so the cumulative bytes written and the time per run can be compared +/// against the step count instead of inferred (docs/durable-flows.md, "Supported ledger budgets"). +/// +[MemoryDiagnoser] +public class LedgerGrowthBenchmarks +{ + private ServiceProvider _serviceProvider = null!; + private IFlowStateStore _store = null!; + private string _resultJson = ""; + private int _sequence; + + /// Retained step results per run. + [Params(50, 200, 400)] + public int Steps { get; set; } + + /// Size of each step's result (a JSON string), in bytes. + [Params(1024)] + public int ResultBytes { get; set; } + + [GlobalSetup] + public void Setup() + { + var services = new ServiceCollection(); + services.AddLogging(); + services.AddAsyncResponse() + .WithInMemoryChannel() + .WithInMemoryTransport() + .WithInMemoryDurableFlows(); + _serviceProvider = services.BuildServiceProvider(); + _store = _serviceProvider.GetRequiredService(); + _resultJson = "\"" + new string('r', Math.Max(0, ResultBytes - 2)) + "\""; + } + + [GlobalCleanup] + public async Task Cleanup() + => await _serviceProvider.DisposeAsync(); + + /// + /// One complete run: create, then one checkpoint per step, each rewriting the ledger with + /// every result so far. Returns the bytes the LAST checkpoint serialized, so the growth is + /// visible beside the per-run time and allocations. + /// + [Benchmark] + public async Task RunOfNSteps() + { + var flowId = $"ledger-growth-{Interlocked.Increment(ref _sequence)}"; + var state = new FlowState + { + FlowId = flowId, + FlowTypeName = "Bench.Flow", + InputTypeName = "Bench.Input", + InputJson = "{}", + Status = FlowRunStatus.Running, + CreatedAtUtc = DateTime.UtcNow, + UpdatedAtUtc = DateTime.UtcNow, + Steps = new Dictionary(StringComparer.Ordinal) + }; + await _store.TryCreateAsync(flowId, state, TimeSpan.FromMinutes(30)); + + long lastLedgerChars = 0; + for (var step = 0; step < Steps; step++) + { + state.Steps![$"step-{step}"] = new FlowStepState { Completed = true, ResultJson = _resultJson, CompletedAtUtc = DateTime.UtcNow }; + var expected = state.Revision; + state.Revision = expected + 1; + state.UpdatedAtUtc = DateTime.UtcNow; + await _store.TryUpdateAsync(flowId, state, expected, TimeSpan.FromMinutes(30)); + lastLedgerChars += _resultJson.Length; // the ledger retains every result so far + } + + await _store.TryDeleteAsync(flowId); + return lastLedgerChars; + } +} diff --git a/docker-compose.yml b/docker-compose.yml index ea489a27c..bb3ff6746 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,13 @@ # Local Redis for running the sample and experimenting with the library: # docker compose up -d +# +# Bound to loopback on purpose. Docker publishes an unqualified port on every host interface, and +# the official image runs with protected mode off, so "6379:6379" exposed an unauthenticated Redis +# to the whole network segment — and AsyncResponse trusts what that Redis holds (recovery +# descriptors, response envelopes). Anything that must be reachable from another machine needs +# `requirepass`/ACLs and network isolation, not a wider bind (docs/security.md). services: redis: image: redis:8-alpine ports: - - "6379:6379" + - "127.0.0.1:6379:6379" diff --git a/docs/configuration.md b/docs/configuration.md index d9755536b..e6ee7f852 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -64,7 +64,7 @@ Configured through the `AddAsyncResponse(options => …)` callback. | `Watchdog.ProbeConcurrency` | 8 | Upper bound on liveness probes one scan runs concurrently — each probe is its own round trip to the channel, and a scan issues one per buffered entry, so probing strictly sequentially would serialize up to `MaxScanEntries` round trips. Must be at least 1. | | `Watchdog.StartupDelay` | 5 minutes | Delay before the first scan after host start. | | `Watchdog.IntervalJitter` | 10% of `Interval` | Random extra delay added to the startup delay and to every interval wait, so replicas deployed together do not scan in lockstep and fire their liveness probes at the same instant. Set to `TimeSpan.Zero` for an exactly-periodic scan. Cannot exceed `Interval`. | -| `MaxInboundMessageChars` | 8 Mi | Largest inbound message the ingress will process, in UTF-16 code units. Larger messages are acknowledged without dispatch, with an error log and the `asyncresponse.ingress.oversized_messages` counter — an oversized message never gets smaller, so redelivering it would hot-loop. `null` removes the limit; a non-positive value is rejected at startup. A memory guard, not a business rule: put large payloads behind a claim check rather than raising it. | +| `MaxInboundMessageChars` | 8 Mi | Largest inbound message the ingress will process, in UTF-16 code units. Larger messages are acknowledged without dispatch, with an error log and the `asyncresponse.ingress.oversized_messages` counter — an oversized message never gets smaller, so redelivering it would hot-loop. Also enforced **producer-side**: every `EnqueueWorkerAsync` overload, and `IDurableFlows.StartAsync` (whose start job carries the initial ledger, input included), measures the serialized envelope and throws `WorkerJobTooLargeException` before publishing when it would exceed this budget — so an oversized job fails in the caller's stack instead of being acknowledged unexecuted by a consumer while the caller holds a flow id. The producer's own value stands in for the consumer's: keep it identical across the processes of one deployment. `null` removes the limit; a non-positive value is rejected at startup. A memory guard, not a business rule: put large payloads behind a claim check rather than raising it. | The watchdog values in the [example above](#configuration) are exactly these defaults — shown so you can see which knobs exist, not because they need changing. @@ -94,8 +94,9 @@ setting) is rejected at startup, because the engine would consume only the last | `ExecutionLeaseDuration` | 1 minute | How long one store lease owns a flow execution before another replica may take over after owner loss. | | `ExecutionLeaseRenewInterval` | 20 seconds | Renewal cadence; must be positive and shorter than `ExecutionLeaseDuration`. | | `ProgressPersistenceInterval` | 1 second | Minimum interval between writes caused only by progress reports. Faster updates are coalesced into the next checkpoint/outcome; zero writes every report. | +| `LedgerSizeWarningBytes` | 512 KiB (`null` disables) | Estimated ledger size past which the executor logs a warning naming the flow — once when first crossed, again at each doubling. Every checkpoint rewrites the whole ledger, so persistence cost grows with each completed step until `MaxStateBytes` (or the provider's item cap) fails the run; this is the early signal to keep step results small or partition into child flows. Lower it on DynamoDB (350 KB item cap). Must be positive. | | `TimerInProcessThreshold` | 10 seconds | Timer remainders (`flow.DelayAsync`) at or under this wait in process under the execution lease; longer remainders suspend the run behind a delayed wake-up job when the transport supports native delayed delivery. Zero always prefers suspension; on transports without delayed delivery every timer waits in process regardless. See [timers-and-scheduling.md](timers-and-scheduling.md). | -| `MaxStateBytes` | DynamoDB 350 000 · Cosmos 1 900 000 · MongoDB 15 000 000 · `null` (unlimited) elsewhere | Serialized-ledger size budget checked on every create/checkpoint. An oversized write fails with a diagnosable error (flow id, size, limit) instead of the raw provider error, before the run burns redeliveries — defaults sit under each provider's hard item/document cap. Keep large payloads in your own storage and pass references (see the ledger-size note in [durable-flows.md](durable-flows.md#child-flows)). | +| `MaxStateBytes` | DynamoDB 350 000 · Cosmos 1 900 000 · MongoDB 15 000 000 · `null` (unlimited) elsewhere | Serialized-ledger size budget checked on every create/checkpoint. An oversized write fails with a diagnosable error (flow id, size, limit) instead of the raw provider error, before the run burns redeliveries — defaults sit under each provider's hard item/document cap. On Cosmos the budget is enforced on the **complete document** as it is sent (the ledger JSON is embedded as a string and escaped a second time, so a ledger well under the budget can produce a document over Cosmos's 2 MB item cap); the ledger JSON alone is checked first as the cheap pre-check. Keep large payloads in your own storage and pass references (see the ledger-size note in [durable-flows.md](durable-flows.md#child-flows)). | Configure these on the selected store, for example: @@ -120,20 +121,25 @@ Configure these on the selected store, for example: | Package | Provider-specific options (in addition to the common options above) | |---|---| -| `SqlServer` | `ConnectionString`, `SchemaName`, `TableName`, `AutoCreateSchema`, `PruneInterval` | -| `PostgreSQL` | `ConnectionString` or registered `NpgsqlDataSource`, `SchemaName`, `TableName`, `AutoCreateSchema`, `PruneInterval` | -| `MySql` | `ConnectionString`, `TableName`, `AutoCreateSchema`, `PruneInterval` | -| `Sqlite` | `ConnectionString`, `TableName`, `AutoCreateSchema`, `PruneInterval` | -| `Oracle` | `ConnectionString`, `TableName`, `AutoCreateSchema`, `PruneInterval` | +| `SqlServer` | `ConnectionString`, `SchemaName`, `TableName`, `AutoCreateSchema`, `PruneInterval`, `PruneBudget` | +| `PostgreSQL` | `ConnectionString` or registered `NpgsqlDataSource`, `SchemaName`, `TableName`, `AutoCreateSchema`, `PruneInterval`, `PruneBudget` | +| `MySql` | `ConnectionString`, `TableName`, `AutoCreateSchema`, `PruneInterval`, `PruneBudget` | +| `Sqlite` | `ConnectionString`, `TableName`, `AutoCreateSchema`, `PruneInterval`, `PruneBudget` | +| `Oracle` | `ConnectionString`, `TableName`, `AutoCreateSchema`, `PruneInterval`, `PruneBudget` | | `MongoDB` | `ConnectionString` or registered `IMongoDatabase`/`IMongoClient`, `DatabaseName`, `CollectionName`, `AutoCreateIndexes` | | `Cosmos` | `ConnectionString` or registered `CosmosClient`, `DatabaseName`, `ContainerName`, `PartitionKeyPath`, `AutoCreateContainer`, `Throughput` | | `DynamoDB` | registered/default `IAmazonDynamoDB`, `TableName`, `AutoCreateTable`, `EnableTimeToLive`, `TimeToLiveAttributeName` | -| `EFCore` | application `DbContext` mapping via `ConfigureAsyncResponseDurableFlows(...)`; schema changes are owned by your EF migrations | +| `EFCore` | application `DbContext` mapping via `ConfigureAsyncResponseDurableFlows(...)`, `PruneInterval`, `PruneBudget`; schema changes are owned by your EF migrations | The SQL stores prune expired rows opportunistically on flow creation, throttled by `PruneInterval` -(default 5 minutes; zero or negative prunes on every save) — a prune that fails is skipped -until the next interval, never failing the `StartAsync` it rides on; MongoDB, Cosmos, and -DynamoDB use native TTL instead. All packages register their store as a singleton and reuse a +(default 5 minutes; zero or negative prunes on every save). Each prune deletes in batches of 1000 +rows and keeps going while batches come back full, for at most `PruneBudget` (default 2 seconds; +zero keeps the historical single batch) — the create that triggered it waits, so the budget also +bounds that create's added latency. A prune that fails is skipped until the next interval, never +failing the `StartAsync` it rides on, and the outcome is reported either way: deleted rows, +failures, and a lapsed budget with rows remaining land on the `AsyncResponse` meter +(`asyncresponse.flow_state.pruned_rows` / `prune_failures` / `prune_budget_exhausted`, tagged by +provider) and the store's logger at Warning. MongoDB, Cosmos, and DynamoDB use native TTL instead. All packages register their store as a singleton and reuse a host-registered client when one exists. See [durable-flow-state-stores.md](durable-flow-state-stores.md) for package examples, lifetimes, cleanup mechanics, and schema ownership guidance. @@ -156,7 +162,7 @@ Every channel has a complete registration in [provider-examples.md](provider-exa | `ConnectionString` | SQL Server, MongoDB | — | SQL Server: connection string; must point at an existing database (the package creates schema/tables, never the database). MongoDB: optional — the package prefers a host-registered `IMongoDatabase` (or `IMongoClient` + `DatabaseName`); against a single-node replica set include `directConnection=true`. | | `DatabaseName` | MongoDB | — | Database used when no `IMongoDatabase` is registered. | | `RecoveryStateTable` / `RecoveryStateCollection` | PostgreSQL, SQL Server, MongoDB | `asyncresponse_recovery_state` | Durable lost-subscriber recovery registrations, one row/document per waiter. MongoDB expires them natively via a TTL index. | -| `MessageTable` / `MessageCollection` | PostgreSQL, SQL Server, MongoDB | `asyncresponse_channel_messages` | Stored response envelopes loaded after `LISTEN/NOTIFY` wakeups (PostgreSQL), by the adaptive polling sweep (SQL Server), or after change-stream wakeups (MongoDB). | +| `MessageTable` / `MessageCollection` | PostgreSQL, SQL Server, MongoDB | `asyncresponse_channel_messages` | Stored response envelopes loaded after `LISTEN/NOTIFY` wakeups (PostgreSQL), by the adaptive polling sweep (SQL Server), or after change-stream wakeups (MongoDB). The sweep's page carries the envelope only for rows nobody has acknowledged; acknowledged history comes back header-only and is fetched by id only when a live subscription still has to receive it. | | `SubscriberTable` / `SubscriberCollection` | PostgreSQL, SQL Server, MongoDB | `asyncresponse_channel_subscribers` | Live waiter heartbeat rows/documents used for subscriber counts and delivery confirmation. | | `NotificationChannel` | PostgreSQL | `asyncresponse_channel_notify` | PostgreSQL `LISTEN/NOTIFY` channel; must be a simple identifier. | | `AutoCreateSchema` / `AutoCreateIndexes` | PostgreSQL, SQL Server, MongoDB | `true` | Create schema/tables/indexes (or TTL + lookup indexes on MongoDB) on first use; set `false` when migrations/provisioning own DDL. With `AutoCreateIndexes = false`, MongoDB runs a one-time read-only check instead and **warns** (never throws) if the TTL or correlation-id lookup index is missing — indexes affect retention/performance, not correctness. | @@ -196,8 +202,10 @@ The in-memory transport is configured directly on registration: ```csharp .WithInMemoryTransport(options => { - options.QueueCapacity = 1_024; // default; PublishAsync waits when full - options.WorkerCount = 1; // default; increase for independent parallel jobs + options.QueueCapacity = 1_024; // default; PublishAsync waits when full + options.WorkerCount = 1; // default; increase for independent parallel jobs + options.InJobOverflowCapacity = 4_096; // default; follow-up publishes held past QueueCapacity, then rejected + options.DelayedJobCapacity = 4_096; // default; delayed jobs held at once — external publishers wait, in-job ones are rejected }) ``` @@ -205,8 +213,28 @@ The in-memory transport is configured directly on registration: running job — a durable flow starting a child, or a child waking its parent — never waits for capacity: the workers are the only consumers, so a worker parking on a full queue would be waiting on itself (with the default `WorkerCount = 1`, permanently). Follow-up work is a continuation of a -job the queue already admitted, so it is accepted past the bound and drains as soon as a worker -frees a slot; it still counts toward the shutdown drain, so nothing is lost at exit. +job the queue already admitted, so it is accepted past the bound into an **in-job overflow** and +drains as soon as a worker frees a slot; it still counts toward the shutdown drain, so nothing is +lost at exit. The overflow is bounded by `InJobOverflowCapacity` (default 4096; `0` allows none; +negative is rejected at startup): past it a follow-up publish throws `InvalidOperationException` — +the publishing job fails and is redelivered by the retry ladder below, so make in-job publishes +idempotent. Every held job retains its materialized envelope and captured execution context, and an +unbounded overflow let a runaway fan-out exhaust memory with the configured queue capacity giving +no signal. The current depth is the `asyncresponse.worker.inmemory_overflow_depth` gauge; +rejections count on `asyncresponse.worker.inmemory_overflow_rejections`. + +**Delayed jobs** — `EnqueueWorkerAsync(..., delay)` and the wake-ups behind suspended flow timers +— are neither queued nor overflow while they wait on their due time, so neither bound covered +them: every scheduled publish retained its envelope and captured execution context against no +limit, and a burst's timers firing at once started that many channel writes pending outside the +bounded queue. `DelayedJobCapacity` (default 4096; must be positive) bounds the jobs held at once +— waiting on their timer, or fired and waiting for queue room. At the bound a delayed publish from +outside a job waits (honoring its cancellation token) until a scheduled job enters the queue; one +made from inside a running job — a flow parking on a timer — never waits, for the same reason as +the overflow, and throws `InvalidOperationException` instead: the publishing job fails and is +redelivered, so make it idempotent. Size it above the number of flows you expect to be sleeping +at once on this transport. The current count is the `asyncresponse.worker.inmemory_delayed_jobs` +gauge; in-job rejections count on `asyncresponse.worker.inmemory_delayed_rejections`. Failed jobs retry with backoff (`RetryBaseDelay` 100 ms → `RetryMaxDelay` 5 s) up to `MaxDeliveryAttempts` (default 5; `0` = unlimited). Retries keep running during the shutdown drain, @@ -224,7 +252,10 @@ it still drain. | `WorkerTopic` / `ResponseTopic` + `WorkerConsumerGroup` / `ResponseConsumerGroup` | Kafka | Topics for worker jobs and response ingress (default `{TopicPrefix}.transport.worker` / `.response`) and the consumer group per role. | | `CreateTopics` / `TopicNumPartitions` / `TopicReplicationFactor` | Kafka | Provision missing topics on subscriber startup. Partitions are the unit of consumer parallelism and ordering; `-1` uses broker defaults. | | `OffsetCommitInterval` | Kafka | Auto-commit cadence for offsets stored after each resolved message; a crash inside the window redelivers at-least-once. | -| `MaxPollInterval` | Kafka | Maximum gap between consumer polls before the broker evicts the consumer from its group and rebalances its partitions (the librdkafka `max.poll.interval.ms`); default 5 minutes. The in-process handler-retry delays run on the poll thread, so startup validation requires the worst-case retry delay budget plus `PollTimeout` to fit within half of it (unlimited retries, `MaxDeliveryAttempts = 0`, have no finite budget and are the operator's call). | +| `BackpressurePollDelay` | Kafka | The short poll slice used while the poll thread waits on in-process work: capacity re-checks while consumption is paused under a full `AckAfterEnqueue` queue, and completion checks while `AckAfterHandlerCompletes` handlers run detached (a finished handler's offset is stored and its partition resumed within one slice). Default 50 ms. | +| `MaxPollInterval` | Kafka | Maximum gap between consumer polls before the broker evicts the consumer from its group and rebalances its partitions (the librdkafka `max.poll.interval.ms`); default 5 minutes. The poll thread's longest gap is `DetachHandlerAfter` plus `PollTimeout`, and startup validation requires that sum to fit within half of it. Handler execution time and the in-process retry ladder no longer count: a handler that outlives `DetachHandlerAfter` runs detached while polling continues. | +| `DetachHandlerAfter` | Kafka | In `AckAfterHandlerCompletes` mode, how long the poll thread waits for a message's handler inline before detaching it. Within the budget a fast handler settles as before (offset stored, next message consumed, no pause). Past it the message's partition is paused — its order holds with nothing buffered in-process — the handler and its retry ladder continue on the thread pool, and the poll thread keeps polling: the consumer's other partitions keep flowing, `MaxPollInterval` is honored, rebalance callbacks fire. The poll thread stores the offset and resumes the partition once the handler settles (checked every `BackpressurePollDelay`); a stop waits for detached handlers so their offsets are committed. This is what lets a durable-flow step await a remote response or sleep on a timer for minutes without the consumer being evicted from its group. Default 1 s; `0` detaches every handler at once. | +| `FaultDrainTimeout` | Kafka | In `AckAfterHandlerCompletes` mode, how long a subscriber whose poll loop *failed* (a consume error, a dropped broker connection, a burial that failed for good) waits for its detached handlers before it closes the consumer and the supervisor rebuilds it. Handlers that settle within the budget get their offsets stored and committed by the close, as after a stop; the rest are abandoned — offsets unstored, messages redelivered on the rebuilt consumer (possibly while the abandoned handler still runs: handlers are at-least-once), retry ladders stopped, outcomes logged. Without it the teardown waited for every detached handler with no limit, so a transient broker failure disabled the subscriber for as long as an unrelated long handler took and the reconnect policy (`SubscriberRetryBaseDelay` → `SubscriberRetryMaxDelay`) never ran. A graceful stop is bounded by the host's shutdown budget instead. Default 5 s; `0` abandons at once. | | `DeadLetterTopic` / `DeadLetterTopicSuffix` | Kafka | Explicit dead-letter topic, or the suffix appended per source topic (default `.deadletter` → `{topic}.deadletter`). | | `ConfigureProducer` / `ConfigureConsumer` / `ConfigureAdminClient` | Kafka | Last-chance hooks over the Confluent client configs (security, compression, fetch tuning, …). | | `WorkerQueue` / `ResponseQueue` | Azure Service Bus | Service Bus queues used for worker jobs and response ingress; they must be distinct. | @@ -290,10 +321,11 @@ Kafka is built on classic consumer groups with manual offset management (`enable `enable.auto.offset.store=false`; an offset is stored only once its message is fully resolved). Two consequences to plan for: ordering is per-partition, so consumer parallelism equals the partition count — size `TopicNumPartitions` accordingly — and a slow or retrying message delays its partition -(head-of-line blocking). Keep the worst-case in-process retry budget -(`MaxDeliveryAttempts × HandlerRetryMaxDelay`) well under `MaxPollInterval` (default 5 minutes) or -the broker evicts the consumer mid-retry — startup validation enforces half that margin -automatically (see the option table above). +(head-of-line blocking) and nothing else: a handler still running after `DetachHandlerAfter` +(default 1 s) is detached — its partition paused, the handler and its retry ladder running on, the +poll thread polling — so neither a long handler nor a long retry ladder can overrun `MaxPollInterval` +(default 5 minutes) and get the consumer evicted. Only `DetachHandlerAfter + PollTimeout` must fit +within half of it, which startup validation enforces (see the option table above). In `AckAfterEnqueue`, the offset is stored at enqueue time and partition fetching pauses while the bounded in-process queue is full; later handler failures are retried in-process, then reported via `OnBackgroundFailure` and produced to the dead-letter topic with failure-detail headers. The message diff --git a/docs/durable-flow-state-stores.md b/docs/durable-flow-state-stores.md index 44c324db3..26a4e016b 100644 --- a/docs/durable-flow-state-stores.md +++ b/docs/durable-flow-state-stores.md @@ -135,8 +135,11 @@ The required invariants are: executor therefore cannot checkpoint after another replica takes over. 4. Acquire succeeds only for an unowned or expired lease. Renew succeeds only for the current, unexpired owner. Release never clears another owner's lease. -5. Loads treat expired, malformed, identity-mismatched, revision-mismatched, and unrecognized-schema - records as absent. +5. Loads treat expired records as absent (`null`). A record that is present but cannot be trusted + — malformed JSON, an unrecognized schema version, a revision inside the JSON that disagrees + with the stored one, a flow id inside the JSON that is not the key — is **not** absent: it + throws `FlowStateUnreadableException`, because callers acknowledge a wake-up on `null` and an + acknowledged wake-up strands the run that is still in the table. There is no weaker compatibility path and no process-local fallback for an incomplete custom store. That keeps single-node tests and multi-replica production on the same correctness model. @@ -232,6 +235,11 @@ Without the shared data source, set `options.ConnectionString = connectionString ### MySQL or MariaDB +A duplicate-key failure on create (`1062`) is confirmed as "this flow id exists" on the connection +the create already holds, so an idempotent re-start never needs a second pooled connection — a +pool of one serves it, and concurrent identical starts cannot starve the pool waiting on each +other. + ```csharp var connectionString = builder.Configuration.GetConnectionString("MySql") ?? throw new InvalidOperationException("ConnectionStrings:MySql is required."); @@ -411,6 +419,13 @@ builder.Services.AddAsyncResponse() An application-registered `CosmosClient` is reused automatically; omit `ConnectionString` in that case. Existing containers must already use the configured partition key and have TTL enabled. +`MaxStateBytes` (1.9 MB by default) bounds the **document** Cosmos receives, measured through the +registered client's serializer: the ledger JSON travels inside it as the `stateJson` string, so +every quote and backslash in the ledger is escaped a second time and a 1.2 MB ledger of escaped +characters is a 2.4 MB document — over the 2 MB item cap, and refused by Cosmos on every retry. +An oversized document fails the write with `FlowStateTooLargeException` naming the document size +instead; the ledger JSON alone is checked first, as the cheap pre-check it can only understate. + Every ledger operation — loads, updates, lease acquire/renew/release, and deletes — treats only a `404` with sub-status `0` as a genuinely absent flow. Cosmos also answers `404` for conditions where the ledger still exists — `1002` (`ReadSessionNotAvailable`, routine Session-consistency lag @@ -420,6 +435,18 @@ being acknowledged against a live run, and so an update or lease call does not m lost lease (the same contract point DynamoDB pins with `ConsistentRead` and MongoDB with primary reads). +Lease maintenance — acquire, the renewal heartbeat (every `ExecutionLeaseRenewInterval`, 20 seconds +by default), and release — never moves the ledger body. Each reads a projection of the lease +fields and the document's `_etag` with a partition-scoped point query, then applies a conditional +**partial update** (`PatchItemAsync` on `leaseId`, `leaseExpiresAtUtc`, and `ttl`, fenced by +`IfMatchEtag`, with no content in the response). Earlier versions point-read the whole document +and replaced it, so an idle execution moved and re-serialized its entire ledger twice per +heartbeat, proportional to ledger size. Wire and CPU cost are now O(lease fields); the +request-unit charge still follows the service's accounting for the loaded document, so measure RU +on your own ledger sizes before sizing throughput. A projection that comes back without `_etag` +(a serializer that hides system properties) throws rather than reporting the lease free. +Checkpoints (`TryUpdateAsync`) still replace the document — they carry the new ledger. + ### DynamoDB The package reuses a registered `IAmazonDynamoDB`; otherwise it uses the normal AWS SDK credential @@ -568,9 +595,12 @@ The library does not silently upgrade an incomplete concurrency schema: `ConfigureAsyncResponseDurableFlows()`. Persisted state has two revision copies: the indexed/provider field and the value inside -`state_json`. Loads require them to match. MongoDB, Cosmos DB, and DynamoDB records without a -physical revision are rejected. This prevents a malformed or partially migrated record from -entering execution with a fabricated revision. +`state_json`. Loads require them to match; a record where they disagree, like one whose +`state_json` names a different flow id than its key, is refused as unreadable +(`FlowStateUnreadableException` naming both revisions) — never reported as absent, since the +record is physically there and "absent" acknowledges its wake-up. MongoDB, Cosmos DB, and DynamoDB +records without a physical revision are rejected the same way. This prevents a malformed or +partially migrated record from entering execution with a fabricated revision. ## Expiry and cleanup @@ -581,12 +611,12 @@ state; physical cleanup is separate: | Store | Cleanup | |---|---| -| PostgreSQL, SQL Server, MySQL, SQLite, Oracle | Opportunistic expired-row prune on flow creation, throttled by `PruneInterval` (default 5 minutes); a prune that fails — a deadlock victim, a lock timeout — is skipped until the next interval, never failing the `StartAsync` it rides on (loads filter on expiry, so the cost until then is disk) | -| EF Core | Provider-side expired-row cleanup through the mapped table, pruned opportunistically like the SQL stores (a failing prune is skipped, not surfaced) | +| PostgreSQL, SQL Server, MySQL, SQLite, Oracle | Opportunistic expired-row prune on flow creation, throttled by `PruneInterval` (default 5 minutes), draining 1000-row batches while they come back full for up to `PruneBudget` (default 2 seconds; zero = one batch); a prune that fails — a deadlock victim, a lock timeout — is skipped until the next interval, never failing the `StartAsync` it rides on (loads filter on expiry, so the cost until then is disk). Deleted rows, failures, and a lapsed budget with rows remaining are counted on the `AsyncResponse` meter and logged at Warning through the store's `ILogger` | +| EF Core | Provider-side expired-row cleanup through the mapped table, pruned opportunistically like the SQL stores (same batches, budget, metrics, and logging) | | MongoDB | TTL index on `expires_at_utc`; reads still filter because Mongo's TTL monitor is periodic | | Cosmos DB | Container TTL plus a per-item `ttl` value | | DynamoDB | Native TTL on `TimeToLiveAttributeName`; expiry is rounded up to avoid shortening the requested lifetime | -| In-memory | Expired entries are removed on access or replacement | +| In-memory | Expired entries are removed on access or replacement, and every flow creation sweeps all expired entries at most once per minute of the engine clock — so a long-lived process with unique flow ids does not retain expired ledgers | Keep `StateExpiry` longer than the longest legitimate period without a checkpoint. Deleting a ledger or allowing it to expire while a flow is suspended makes its outcome unknowable. @@ -602,12 +632,15 @@ a custom store in production, test all of these against the real backend: - a lease cannot be renewed or released by another owner; - takeover works after lease expiry; - TTL refresh and expired-record replacement are atomic; -- a wrong `flowId` and a missing/mismatched revision load as `null` — the row does not belong to - this flow, so it is absent; -- **unreadable is not missing:** malformed JSON and an unknown schema version instead throw - `FlowStateUnreadableException`. Returning `null` there says "this flow was deleted", and the - caller acknowledges the wake-up that was a live flow's only one — the failure mode a rolling - deployment hits when an older replica reads a row a newer one wrote; +- **unreadable is not missing:** malformed JSON, an unknown schema version, a revision inside + the JSON that disagrees with the stored one, and a stored `flowId` that is not the key all throw + `FlowStateUnreadableException` — never `null`. Returning `null` there says "this flow was + deleted", and the caller acknowledges the wake-up that was a live flow's only one — the failure + mode a rolling deployment hits when an older replica reads a row a newer one wrote, and the one + a corrupt or mis-restored row hits on any deployment; +- a ledger write honors the run's retention floor: the engine raises the TTL it passes to + `TryUpdateAsync` to reach `FlowState.RetainUntilUtc` for non-terminal runs, so a store needs + no special handling — but it must stamp the TTL it is given, not one of its own; - `flow_id` compares **ordinally**: a case- or accent-insensitive column collation folds distinct runs onto one row. The built-in stores verify the deployed collation at startup and refuse a folding one rather than corrupting state silently; diff --git a/docs/durable-flows.md b/docs/durable-flows.md index e0b72daef..074013913 100644 --- a/docs/durable-flows.md +++ b/docs/durable-flows.md @@ -222,9 +222,19 @@ transport's dead-lettering. If a child gets stuck, that shows up as the child's entry or its stale ledger), not as a silent parent hang — see the failure table below. **Ledger-size note.** The memoized child snapshot excludes the captured ambient `Context` (it is -propagation machinery the parent never needs), but it does embed the child's own step results — -so deeply nested parent → child → grandchild chains grow the parent's ledger with each completed -child. Keep very large payloads in your own storage and pass references through flow state. +propagation machinery the parent never needs) and the child's **own memoized child snapshots**: a +child step whose `ChildFlowId` is set keeps its id, completion, and fault marker in the parent's +memo, but its `ResultJson` is elided. The snapshot is stored as a JSON *string* inside the parent's +ledger, so each ancestor level re-escapes the level below it; carrying grandchild snapshots along +made a parent → child → grandchild chain grow **exponentially** with depth (a 72-byte leaf reached +~600 KB at depth 15 with no business payload — past DynamoDB's item cap). With the elision a +memoized snapshot is depth-independent: a parent sees its direct children's outcomes and local +step results; a grandchild's snapshot lives in the grandchild's own ledger, loadable through +`IDurableFlows.GetStateAsync(step.ChildFlowId)` while that ledger lives. The child's own local +step results are still embedded, so keep very large payloads in your own storage and pass +references through flow state. The `FlowState` that `AwaitChildFlowAsync` returns is this snapshot +on **every** execution — the first completion reads it back from the memo exactly as a replay +does — so parent logic sees one shape before and after a restart. Stores enforce a `MaxStateBytes` budget on every write (defaulted below each provider's hard item/document cap — DynamoDB 400 KB, Cosmos 2 MB, MongoDB 16 MB); an oversized checkpoint fails with an error naming the flow id, size, and limit instead of a raw provider error @@ -273,12 +283,13 @@ property makes every failure mode collapse into "run it again": | Process is down when a **failed** response arrives | `OnRecovery() == Fail` routes to the auto-registered **failure** callback: the run is marked `Failed` — a failure is never resumed as a success | | The **terminal** response itself was the lost message | Its payload is already the step result. The resumed run skips that completed await and continues; it does not wait for a consumed correlation id or re-send the remote request | | The same flow job is delivered to two replicas | Atomic start preserves the first input, and the execution lease lets one worker run. The duplicate delivery returns without entering flow code; if the owner disappears, the lease expires and another worker resumes from the last compare-and-swap checkpoint | -| `StartAsync`'s **publish fails ambiguously** (the job may or may not have been accepted) | With a **caller-supplied `flowId`**, retrying `StartAsync` is safe: the atomic create dedupes and re-enqueues the same run. With a **generated id** (the `flowId: null` default), a retry mints a fresh id — a second independent run is created and, if the first publish had actually been accepted, **both execute**. Supply deterministic ids wherever the caller may retry. If the create succeeded but the publish threw outright, `StartAsync` retries the publish and then throws **`DurableFlowNotDispatchedException`**, whose `FlowId` carries the id out — including a generated one — so the orphan is re-drivable: retry `StartAsync` with that id (the atomic create dedupes and re-enqueues the same run), or call `ResumeAsync(flowId)` | +| The process **dies mid-`StartAsync`** | The publish of the start job is the start's commit point, and the job carries the initial ledger: `IDurableFlowExecutor.CreateAndExecuteAsync` creates the run (insert-if-absent) before executing it. A crash **before** the publish leaves nothing behind; a crash **after** it leaves a job whose execution creates and runs the flow. There is no window in which a committed `Running` ledger exists that nothing will ever execute (the pre-1.0 order — create, then publish — had exactly that window, and `IFlowStateStore` has no enumeration a reconciler could use to find such a run). The starter still writes the ledger itself after the publish, so `GetStateAsync`/`ResumeAsync` right after a start see the run; losing that write to the executor (or to a concurrent identical start) is the expected shape, not an error | +| `StartAsync`'s **publish fails** | After the start's own retry ladder, `StartAsync` throws **`DurableFlowNotDispatchedException`** and **nothing was persisted**. Its `FlowId` carries the id the start would have used — including a generated one — so the retry stays idempotent: if the publish had in fact landed (the ambiguous case is deliberately included), the same id dedupes against the run the job created; a retry with a fresh generated id would start a second, independent run. Supply deterministic ids wherever the caller may retry | | A child flow is running | The parent run is parked as `Running`; the child terminal state re-enqueues the parent, which reloads the child state and continues | | A **child run dead-letters** (a retriable failure exhausts the transport's delivery attempts) | The child stays `Running` and the parent stays suspended — **the child's DLQ entry is the alarm**. Replay the DLQ entry or call `ResumeAsync(childFlowId)`; re-enqueueing the parent (`ResumeAsync(parentFlowId)`) also works — it re-enqueues the child. The parent resumes automatically once the child reaches a terminal state | | You want a dead-lettered run to **wait for you** | A `Running` run can be resurrected at any time by a late response or recovery — by design. To take manual control first, set the run's status to `FlowRunStatus.Suspended` in the flow store: wake-ups, resumes, and failure signals are ignored while suspended (a parent awaiting a suspended child keeps waiting). A recovered **terminal** response is not discarded: it is checkpointed into the suspended run's ledger *without waking it*, so un-parking replays from that preserved result; non-terminal checkpoints keep the recovery registration armed. When ready, set it back to `Running` and call `ResumeAsync(flowId)` to replay from checkpoints. **Park only runs that are not mid-execution**: the store write bumps the ledger revision, so a worker actively executing that flow fails its next checkpoint (logged as a lost execution lease) and everything after its last checkpoint replays on un-park — the normal at-least-once replay, but with side effects that already ran once | | The **child's ledger expired** while the parent was suspended | The parent step fails terminally with `DurableFlowFailedException` (`"has no state (expired or deleted)"`) instead of silently re-running the child's side effects — the child's outcome is unknowable. Size `DurableFlowOptions.StateExpiry` beyond the longest child idle time; the TTL refreshes on every checkpoint | -| The **parent's ledger expired** while suspended | A descendant's long park (a timer sleep, or an awaited step whose wait window exceeds `StateExpiry`) auto-extends every `Running` ancestor up the chain to cover it, so a parent suspended only because a child is parked no longer needs separate sizing for that case. This propagation fires on parking only: a child that keeps running and checkpointing without ever parking longer than its own `StateExpiry` does not extend the parent, so size `StateExpiry` above that child's total wall-clock duration too. Either way, an expired run cannot be resumed: the executor logs a warning and no-ops | +| The **parent's ledger expired** while suspended | A descendant's long park (a timer sleep, or an awaited step whose wait window exceeds `StateExpiry`) extends every `Running` ancestor up to the root to cover it, so a parent suspended only because a child is parked no longer needs separate sizing for that case. The park stamps a **retention floor** — `FlowState.RetainUntilUtc` — on the run and on every ancestor, and every ledger write of a non-terminal run raises the TTL it stamps to reach that floor: a checkpoint, the executor's per-attempt save, or a recovery/operator mutation that knows nothing about the wait cannot shrink the retention under it. The extension is **part of the park**, not insurance around it: if an ancestor's ledger cannot be written (a store outage), the park fails with nothing published — the delivery is redelivered and replays the same step, which retries the chain — instead of the child parking "successfully" on a parent that would expire under it. A lost revision race is retried against the re-read ancestor (a competing write that already carries a floor reaching the park proves the retention); losing every attempt abandons the park the same way. The whole chain is walked with cycle detection; a chain that revisits an id, or is nested more than 256 child flows deep, fails the run terminally (`DurableFlowFailedException`) rather than being truncated in silence. This propagation fires on parking only: a child that keeps running and checkpointing without ever parking longer than its own `StateExpiry` does not extend the parent, so size `StateExpiry` above that child's total wall-clock duration too. Either way, an expired run cannot be resumed: the executor logs a warning and no-ops | | A step keeps failing | The exception propagates; the worker transport redelivers the run with bounded attempts, then **dead-letters it — that's your "run is stuck" alarm** | | The flow decides it's hopeless | Throw `DurableFlowFailedException`: the run is marked `Failed` terminally, with no redelivery | | The **parent fails (or is failed) while a child still runs** | The child is deliberately independent: it keeps running to completion — its side effects happen — and its terminal notification to the already-terminal parent is a no-op. There is no cascade-cancel. If the child's work must not continue, act on the child explicitly: park it (`FlowRunStatus.Suspended`) or fail it (`IDurableFlowExecutor.FailAsync`). Policy modes (cascade cancel/park on parent failure) are on the roadmap | @@ -446,6 +457,60 @@ Supported packages: | DynamoDB | `WithDynamoDbDurableFlows(...)` | | Entity Framework Core (any relational provider) | `WithEFCoreDurableFlows(...)` | +**Ledger growth is the cost model to watch.** Every checkpoint rewrites the *whole* ledger — +input, every completed step's result, values, context — so a run of N steps with similar result +sizes serializes about N²/2 step-results over its lifetime (100 steps of 1 KiB: ~6 MB written for a +115 KB final ledger; 400 steps: ~92 MB for 458 KB). The store's `MaxStateBytes` (or the provider's +item cap) is the hard limit; `DurableFlowOptions.LedgerSizeWarningBytes` (default 512 KiB, `null` +disables) is the early signal — a warning naming the flow when its estimated size first crosses +the threshold and again at each doubling. Keep step results small (persist large data yourself +and pass references — the claim-check seam on the [roadmap](roadmap.md) will do this +transparently), partition a long history into [child flows](#child-flows) (a parent memoizes +only a compact snapshot of each child), and lower the threshold on DynamoDB, whose 350 KB item +cap sits under the default. + +#### Supported ledger budgets + +The full-ledger checkpoint is a deliberate design: one document, one revision check, one lease +fence, readable on any store and byte-identical across providers. Its cost is quadratic in the +number of retained step results, and these are the budgets the library is built and tested for — +outside them the persistence cost arrives well before the size cap does: + +| Budget | Supported | What happens past it | +|---|---|---| +| Ledger size | ≤ `LedgerSizeWarningBytes` (512 KiB by default; ≤ 350 KB on DynamoDB) | The warning fires at the threshold and each doubling; `MaxStateBytes` fails the run. | +| Retained step results per run | a few hundred (≈ 250 steps of 100-byte results ≈ 5 MB written over the run; 1,000 ≈ 87 MB) | Every further checkpoint re-serializes the whole history; latency and transaction-log volume grow with each step. | +| Size of one step result | a few KiB | One large result is paid again on every later checkpoint of the run. | +| Flow input | must fit the worker envelope: `AsyncResponseOptions.MaxInboundMessageChars` (8 Mi characters) — the start job carries the initial ledger | `StartAsync` throws `WorkerJobTooLargeException` before publishing (nothing is persisted). | + +Two patterns keep a long-running or data-heavy flow inside them. **Store large results by +reference**: the step persists its payload where it belongs (blob storage, a table, a cache) and +returns only the key, so the ledger retains a few dozen bytes per step: + +```csharp +// The step's checkpoint holds the key, not the report. +var reportKey = await flow.StepAsync("render-report", async () => +{ + var report = await renderer.RenderAsync(input, ct); + var key = $"reports/{flow.FlowId}/{Guid.NewGuid():N}"; + await blobs.UploadAsync(key, report, ct); + return key; +}); + +// A later step re-reads it by key; a replay after a restart re-reads the same key. +await flow.StepAsync("publish", () => publisher.PublishAsync(blobs.OpenRead(reportKey), ct)); +``` + +**Partition a long history into child flows**: a parent that fans out or loops for hundreds of +steps starts a [child flow](#child-flows) per batch and memoizes only each child's compact +snapshot, so neither ledger grows past a bounded number of steps. Incremental (append-only) +checkpoint persistence for workloads that genuinely need thousands of retained results is on the +[roadmap](roadmap.md) and will keep the same revision and lease fences. The curve itself is +measured, not inferred: `LedgerGrowthBenchmarks` in `benchmarks/AsyncResponse.Benchmarks` runs a +complete N-step run through the process-local store (N = 50, 200, 400 checkpoints of 1 KiB +results) and reports the time and allocations per run, so a change to the checkpoint path — or +to your own step-result sizes — can be checked against the budgets above. + For tests, development, or a deliberately one-process application: ```csharp @@ -537,12 +602,20 @@ The API encodes the *checkpointed-flow pattern*, extracted from years of product loss within one process lifetime and the simulated restarts of [AsyncResponse.Testing](testing.md); durable channels extend the same contract across real restarts). -- Starting a flow enqueues a worker job carrying only the flow id; resume, redelivery, and - operator kicks all re-enqueue that same job. `StartAsync` with a caller-supplied `flowId` is - atomically idempotent for the same flow type and semantically identical input. Conflicting reuse - is rejected; an existing run is never replaced silently. A **generated** id (the default) cannot - survive a retried ambiguous publish — the retry mints a fresh id and a second independent run — - so supply deterministic ids wherever the caller may retry (see the failure table above). +- Starting a flow **publishes first**: the start job carries the initial ledger (flow and input + type names, the input JSON, the captured ambient context — the same wire format the stores + hold), and its target, `IDurableFlowExecutor.CreateAndExecuteAsync`, creates the ledger if the + starter's own write never happened before running the flow. That makes the publish the start's + single commit point (see the failure table above). Resume, redelivery, and operator kicks all + re-enqueue the lighter `ExecuteAsync(flowId)` job. The cost is the input travelling twice — + once in the job, once in the ledger — so mind the transport's payload ceiling for large inputs + (SQS and Azure Service Bus cap a message at 256 KiB). `StartAsync` with a caller-supplied + `flowId` is atomically idempotent for the same flow type and semantically identical input; the + starter reports a conflicting reuse as `DurableFlowIdConflictException` and the executor drops + the job it already published on the same test, so an existing run is never replaced silently. A + **generated** id (the default) cannot survive a retried ambiguous publish — the retry mints a + fresh id and a second independent run — so supply deterministic ids wherever the caller may + retry. - Built-in stores persist a monotonic `FlowState.Revision`. Every execution owns a renewable lease and every checkpoint requires both the expected revision and that lease, so a stale worker cannot overwrite recovery state written by a newer execution. One deliberate exception: a response won @@ -550,7 +623,12 @@ The API encodes the *checkpointed-flow pattern*, extracted from years of product compare-and-swap that recovery uses — on the normal completion path as well as the cancellation branch — because the channel has already acked that payload and it exists nowhere else; the execution then stops as lease-lost and the redelivery replays from that checkpoint instead of - re-attaching to a consumed correlation id and burning the step timeout. + re-attaching to a consumed correlation id and burning the step timeout. That lease-less write + is fenced to the attempt that won the response, exactly as a recovered payload is: it applies + only while the reloaded step is still pending on the **same** correlation id and the run is + still Running (or Suspended). If a takeover already timed the breadcrumb out and re-triggered the + step under a new id, or failed the run, the stale response is discarded with a warning — it + answers a request the newer attempt no longer owns. ## Honest comparison with a dedicated workflow engine diff --git a/docs/observability.md b/docs/observability.md index f1d23b2ec..a993113c1 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -80,7 +80,12 @@ builder.Services.AddOpenTelemetry() |---|---|---|---| | `asyncresponse.lost_subscriber.dispatches` | counter | `kind` = `response`\|`exception`, `route` = `resume`\|`failure`\|`keep_waiting`\|`mixed`\|`unclassified`, `invoked` = bool | The core "how often does recovery fire" SLO — every late response that found nobody listening, classified by how it was routed and whether a callback was actually invoked. `mixed` means shared-correlation registrations legitimately took different routes in one dispatch; each registration's own dispatch span carries its true route. | | `asyncresponse.waiter.timeouts` | counter | `channel` | Waiters that hit their timeout before a terminal response. | +| `asyncresponse.channel.overloaded_waits` | counter | `channel` | Waits faulted as indeterminate because responses for their correlation id arrived faster than the wait could process them and the bounded per-wait buffer (1,024 on Redis) was full. Fire-and-forget channels only. Alert on any sustained rate: a consumer is saturated — speed up the completion predicate, publish fewer progress messages, or move the wait to a database channel. | | `asyncresponse.worker.jobs` | counter | `outcome` = `executed`\|`failed`\|`rejected`\|`dropped` | Worker job dispatch outcomes. `failed` counts individual attempts; `rejected` is an envelope the ingress refused without dispatching — an unusable correlation id, a failed authorization, or a body no build can parse (acknowledged rather than redelivered forever); `dropped` is the in-memory transport's terminal outcome after `MaxDeliveryAttempts` (broker transports dead-letter instead). Alert on `rejected`: every one is a producer-side contract violation. | +| `asyncresponse.worker.inmemory_overflow_depth` | observable gauge | — | Follow-up jobs the in-memory worker transport currently holds past `QueueCapacity` (summed over the process's transports), bounded by `InJobOverflowCapacity`. A depth that stays near the bound means a handler fans out faster than the workers drain. | +| `asyncresponse.worker.inmemory_overflow_rejections` | counter | — | Follow-up publishes the in-memory transport refused at `InJobOverflowCapacity`; the publishing job failed and is redelivered by the in-process retry ladder. Alert on any sustained rate: raise the capacities or add workers. | +| `asyncresponse.worker.inmemory_delayed_jobs` | observable gauge | — | Delayed jobs the in-memory worker transport currently holds — waiting on their due time, or fired and waiting for queue room (summed over the process's transports), bounded by `DelayedJobCapacity`. A count that stays near the bound means more flows are sleeping at once than the capacity was sized for. | +| `asyncresponse.worker.inmemory_delayed_rejections` | counter | — | Delayed publishes made from inside a running job (a flow parking on a timer) that the in-memory transport refused at `DelayedJobCapacity`; the publishing job failed and is redelivered by the in-process retry ladder. Alert on any sustained rate: raise `DelayedJobCapacity`. | | `asyncresponse.ingress.unroutable_responses` | counter | — | Inbound responses acknowledged without routing because they carry no correlation id (deliberate poison guard — redelivery could never route them). Alert on any non-zero rate: each one is a producer-side contract violation. | | `asyncresponse.ingress.oversized_messages` | counter | `route` = `response`\|`worker` | Inbound messages acknowledged without processing because they exceed `AsyncResponseOptions.MaxInboundMessageChars`. Alert on any non-zero rate: the message is gone, and either a producer is sending more than the deployment allows or the cap is set too low. | | `asyncresponse.recovery.outstanding` | observable gauge | — | Persisted recovery-state entries (from the watchdog scan). | @@ -89,6 +94,9 @@ builder.Services.AddOpenTelemetry() | `asyncresponse.recovery.unprobeable` | observable gauge | — | Entries whose waiter liveness could not be probed (a probe outage, or no `IActiveSubscriberProbe` registered) — their staleness is unknown and they are never flagged stale. A non-zero value also degrades the recovery health check. | | `asyncresponse.recovery.scan_truncated` | observable gauge | — | `1` when the last watchdog scan stopped at the `MaxScanEntries` buffer cap: `outstanding`/`stale` then describe the buffered subset only, and the recovery health check reports **Degraded**. Alert on it — a capped scan cannot attest staleness. | | `asyncresponse.type_resolution.unresolved` | counter | `kind` = `service`\|`payload` | Callback/payload type names that could not be resolved (see [security.md](security.md)). | +| `asyncresponse.flow_state.pruned_rows` | counter | `provider` | Expired durable-flow ledger rows deleted by the relational stores' opportunistic prune (PostgreSQL, SQL Server, MySQL, SQLite, Oracle, EF Core). | +| `asyncresponse.flow_state.prune_failures` | counter | `provider` | Opportunistic prunes that failed; the flow creation they rode on still succeeded and the next `PruneInterval` retries. Alert on a sustained rate: expired rows are accumulating. | +| `asyncresponse.flow_state.prune_budget_exhausted` | counter | `provider` | Prunes that stopped at `PruneBudget` with a full last batch — expired rows remain and the backlog is outgrowing the prune. Raise `PruneBudget` or shorten `PruneInterval`. | The lost-subscriber counter is the one to alert on: a nonzero `route=failure` or `route=unclassified` rate means flows are dying mid-wait and being failed on recovery (a diff --git a/docs/operations.md b/docs/operations.md index 1fd7b767b..27d803b6b 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -179,6 +179,23 @@ images, which is what the disk-reclaim step in that job exists to fight. Legs up `coverage-integration-`; the coverage job globs `coverage-*` and merges, so the published number still covers the whole suite. +#### CI retries + +Two mechanisms retry hosted-runner flakes, and both consult the **same classifier**, +`scripts/ci-retryable-failure.sh`: the integration legs retry their own suite once in-job, and +`auto-retry.yml` re-runs the failed jobs of a `main` run. A log qualifies only when it matches a +known infrastructure signature (a batch or matrix fixture that failed to boot, SQLite's "database +is locked" on a slow runner disk, the runner going away) **and** carries no evidence that a test +executed and failed on its merits or that the build broke (an xunit assertion message, +`XunitException`, a `CS`/`MSB`/`NU` error code). A fixture-boot flake next to an assertion failure +in the same log is a real failure and is never retried — the in-job retry used to key on the boot +signature alone, so such a log was retried into a green job the standalone workflow never got to +see. Each attempt's console log is kept in the results artifact +(`itest-console..attempt.log`), and every automatic retry leaves a `::warning::` naming +its evidence, so a green re-run is never indistinguishable from a clean pass. The classifier's +fixture logs — the mixed one included — run as a self-test in `build-and-test` +(`scripts/tests/ci-retryable-failure.test.sh`). + #### The provider cross product Channels, transports, and durable-flow stores are chosen independently, so "it works" has to mean diff --git a/docs/postgresql.md b/docs/postgresql.md index 85488027e..c4addae1b 100644 --- a/docs/postgresql.md +++ b/docs/postgresql.md @@ -173,9 +173,24 @@ Recommended Npgsql connection-string settings: performs one update for the process's current active-registration snapshot. Rows no longer in that snapshot are allowed to expire even if cleanup deletion failed. A failed batch is logged and the next interval retries, so leave enough timeout headroom for multiple attempts. +- The sweep re-reads a subscribed correlation id's retained rows on every tick (and on every + targeted signal): acknowledged rows stay in the result so a fan-out waiter in another process + still receives a response this one already consumed. Their **bodies** do not travel: the page + query ships `envelope_json` only for rows nobody has acknowledged, an acknowledged row comes back + header-only (id, timestamps, `acked_seq`), and the sweep fetches the envelope by id only for the + rare acknowledged row a live subscription has not seen. A long-lived progress subscription's + sweep cost therefore no longer grows with its whole retained history. (Until round 39 every + sweep re-transferred and re-materialized every retained body just to drop it in the pre-filter.) - `PendingMessageBatchSize` is a page-size tuning knob, not a cap per sweep. Smaller pages lower peak materialization; larger pages reduce round trips when one correlation id carries heavy progress traffic. +- Delivery is serialized per correlation id on a bounded (1024-item) executor. The sweep admits + work to it **without waiting**: when one correlation id's executor is full — a waiter wedged in + a slow `Until` predicate under a progress flood — the rest of that id's messages stay unclaimed + in the table, in order, and only that id is rescanned after one poll interval; every other + correlation id keeps delivering. (Until round 35 the sweep awaited the capacity, so one + saturated correlation stalled every waiter in the process.) The same-process fast path still + applies backpressure to the publisher of that one correlation id. - Keep `DeliveryConfirmationTimeout` long enough for the slowest expected live delivery, but short enough that a truly lost subscriber routes to recovery promptly. - Set `DeadLetterRetention` if operators do not inspect dead-letter rows indefinitely. diff --git a/docs/recovery.md b/docs/recovery.md index 22548f8a7..62f915524 100644 --- a/docs/recovery.md +++ b/docs/recovery.md @@ -163,6 +163,18 @@ public sealed class OrderFlow(ILogger _logger, IOrderStore _orders) : > change for in-flight recovery state — deploy renames with care (keep a forwarding method for one > expiry window). +> ⚠️ **Binding contract:** the persisted descriptor is *name + parameter count*, so the target +> must be the only public method on the interface (base interfaces included) with that name and +> arity — an overload set such as `Run(int)` / `Run(string)` cannot be told apart on the wire. +> The expression overloads (`OnLostSubscriberResume(...)`, `EnqueueWorkerAsync(...)`) +> validate this **at registration**, in your stack, and throw for an ambiguous, by-ref, or +> open-generic target; the same check runs at dispatch, so the two never disagree. Targets must +> return `Task`, `ValueTask`, or `void` **synchronously** — an `async void` implementation is +> refused before it is invoked (its body would still be running when the job is acknowledged and +> its DI scope disposed, and any later exception would escape to the thread pool). The refusal +> applies to the *implementation* the interface resolves to, so it is checked on the first +> dispatch and cached per implementation type. + ### Make resume callbacks re-entrant A resume may re-trigger a flow whose step is still running remotely; resume should *re-attach* @@ -180,6 +192,22 @@ distributed claim step in front of the callback — resume must already be re-at extra store round-trip per recovery would buy nothing. Treat both callbacks as idempotent: key side effects on the correlation id, not on the invocation. +**When the failure callback cannot be invoked.** The dispatcher retries a failure callback four +times in-process (250 ms → 2 s backoff) for transient faults. If every attempt fails, the publish +throws `RecoveryCallbackFailedException` (carrying the correlation id and attempt count) instead +of returning normally: the registration stays armed, the broker ingress passes the exception +through untouched — no further retry, no `SetException` escalation (that would only invoke the +same failing callback again) — and the **transport redelivers the message** under its own +`MaxDeliveryAttempts`/dead-letter policy. A terminal signal is therefore never acknowledged into +a log line while the flow stays stuck: it waits in the broker until the callback's dependency +recovers or an operator replays it from the dead-letter destination. On RabbitMQ's default +`MaxDeliveryAttempts = 0` that is the same unlimited requeue any failing handler gets — configure +a cap and a `DeadLetterExchange` there as you would for worker jobs. Deterministic faults — an +unauthorized or unresolvable target, a method that no longer binds — are still logged and +acknowledged (redelivery cannot fix them); the kept registration is what the watchdog surfaces. +A direct caller of `SetResponse`/`SetException` (an HTTP callback endpoint) sees the same +exception; answer the remote system with a retriable status. + The complete multi-step recipe built on these rules — a persisted step ledger, re-attach via the pending correlation id, subset runs, and compensation — is documented in [durable-flows.md](durable-flows.md). @@ -333,5 +361,27 @@ waiters are lost, the recovery store keeps one registration per waiter and a late response/exception dispatches to every stored callback for that correlation id. A waiter that completes normally removes only its own registration, so a still-active sibling remains recoverable. +Each registration keeps its own delivery guarantee through that fan-out. A registration whose +callback succeeds is consumed (deleted) immediately. If a **sibling's** callback then fails +*transiently* (its dependency blipped), the publish throws `RecoveryCallbackFailedException` — the +same type the failure-callback ladder uses — and the broker ingress passes it through untouched: +no second retry ladder, no `SetException` escalation (which would fail flows whose resume merely +blipped), so the transport **redelivers** the terminal signal under its own `MaxDeliveryAttempts` +and dead-letter policy. Because the successful registrations are already gone, the redelivery +reaches only the ones that failed and settles them when the dependency is back (the same holds for +a direct `SetResponse`/`SetException` caller that retries). A sibling failure that is +*deterministic* — an unauthorized or unresolvable target, a method that no longer binds — is +logged, since redelivery cannot fix it; that registration stays for the watchdog to surface. The +verdict is taken over **every** failed registration, not the first one the store returned: the +message is acknowledged only when every failure was deterministic, and one transient failure +anywhere in the set keeps it unacknowledged whatever precedes it. When no callback succeeded at +all, a sibling whose failure-callback ladder was exhausted (`RecoveryCallbackFailedException`) +propagates ahead of any other sibling's fault, so the ingress does not burn its own retry ladder on +a deterministic fault and then escalate through `SetException` into the very callback that just +gave up. (Until round 35 a partial success was swallowed outright, which returned success to the +broker for a payload the failed registration never received; until round 39 the verdict came from +the first failure alone, so a deterministic fault ahead of a transient sibling acknowledged the +message and the transient registration lost its only copy of the payload.) + The watchdog reports shared-correlation recovery state once per correlation id, not once per stored waiter registration. diff --git a/docs/roadmap.md b/docs/roadmap.md index 937f8aed4..ab3022556 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -158,8 +158,12 @@ storage and passes references on the wire, transparently on publish and material market moved here too: Temporal productized exactly this as "External Storage" at Replay 2026, and SQS raised its maximum payload to 1 MB in January 2026 (which helps, and also signals where payloads are heading). This is also the structural answer to flow-ledger size limits: the -`MaxStateBytes` guard (Train 0) tells you when you hit the wall; claim-check is how you stop -hitting it. +`MaxStateBytes` guard (Train 0) tells you when you hit the wall, the `LedgerSizeWarningBytes` +warning (round 35) tells you when you are heading for it; claim-check is how you stop hitting it. +The store-side complement for genuinely long histories — **incremental step persistence** 🟡 +(append completed steps instead of rewriting the whole ledger per checkpoint, under the same +revision and lease fencing) — waits for a workload that measurably needs it; the current +quadratic write cost is documented in [durable-flows.md](durable-flows.md#storage-where-flow-state-lives). ### 4.4 Flow operations API + observability pack 🟠 diff --git a/docs/sample.md b/docs/sample.md index a9b775923..0e08b9b86 100644 --- a/docs/sample.md +++ b/docs/sample.md @@ -159,6 +159,14 @@ curl -X POST 'http://localhost:5000/lost-subscriber-flow?outcome=Failed' curl -X POST 'http://localhost:5000/lost-subscriber-flow?outcome=Exception' # arm + drop this channel + late SetException → fail ``` +> The scenarios marked *Recovery*, the composed `/lost-subscriber-flow`, `/emit-response`, +> `/calls`, and the flow **read/resume** routes (`GET /durable-flow/`, +> `POST /durable-flow//resume`) are test affordances: they inject responses, drop +> subscriptions, or return a run's full ledger without authentication. They are mapped only in the +> Development environment (the `dotnet run` default) or with `Sample:EnableTestEndpoints=true`; a +> Production instance answers 404 for all of them (see +> [security.md](security.md#the-samples-test-only-routes-are-gated)). + For the lost-subscriber flow, copy the `correlationId` returned by `/arm` and replace `` in a `/publish` request. `Completed` exercises the resume callback; `Failed` exercises the failure callback with an `AsyncResponseDomainFailureException`; `exception=...` exercises the technical diff --git a/docs/security.md b/docs/security.md index 3bbd9619f..575638840 100644 --- a/docs/security.md +++ b/docs/security.md @@ -27,6 +27,12 @@ has itself loaded. SQL Server), distinct queues (Azure Service Bus, SQS, RabbitMQ), or topic/consumer-group names (Kafka, Google Pub/Sub). - Enable transport-level TLS and credentials end to end. +- Mind local conveniences too: the repository's `docker-compose.yml` binds its Redis to + `127.0.0.1` on purpose — an unqualified `6379:6379` publishes on every host interface, and the + official image runs with protected mode off, so a development Redis reachable from the network + segment is an unauthenticated write path into recovery descriptors and response envelopes for + any application pointed at it. Anything that must be reachable from another machine needs + `requirepass`/ACLs and network isolation, never a wider bind. The callback authorizer below is a second layer on top of this — not a replacement for it. @@ -71,13 +77,15 @@ reaches the store, only an explicitly allowlisted surface can be driven. ### The durable-flow executor and the allowlist -Durable flows persist `IDurableFlowExecutor` methods (`ExecuteAsync`, `ResumeAsync`, `RecoverAsync`, -`FailAsync`) as every flow's resume/recover/fail targets, so the **allowlist builder admits the -executor by default** — rejecting it would break flow recovery. This is a deliberate, visible -trade-off: an attacker with write access to the recovery store or worker transport can then drive -those four methods, which is bounded to waking/failing flows by id and checkpointing a chosen -payload into a flow's pending step (`RecoverAsync`) — not arbitrary service invocation. If you do -not use durable flows, or want to gate the executor yourself, opt out: +Durable flows persist `IDurableFlowExecutor` methods (`CreateAndExecuteAsync`, `ExecuteAsync`, +`ResumeAsync`, `RecoverAsync`, `FailAsync`) as every flow's start/resume/recover/fail targets, so +the **allowlist builder admits the executor by default** — rejecting it would break flow starts and +recovery. This is a deliberate, visible trade-off: an attacker with write access to the recovery +store or worker transport can then drive those methods, which is bounded to waking/failing flows +by id, checkpointing a chosen payload into a flow's pending step (`RecoverAsync`), and starting a +run of a *registered* flow type with a chosen input (`CreateAndExecuteAsync` — the same thing a +worker-transport writer could already do by publishing any worker job) — not arbitrary service +invocation. If you do not use durable flows, or want to gate the executor yourself, opt out: ```csharp .AuthorizeCallbacks(a => @@ -133,7 +141,33 @@ dictionary keys read straight off the wire, such as a worker envelope's propagat for a malformed literal it quotes several raw body characters. Both the message and the chained inner exception the ingress logs (and, on the response path, republishes to the waiter through `SetException`) are rebuilt from position only: line, byte position, and size. The reader's own -message and path are dropped, not chained. +message and path are dropped, not chained. The same scrubbing covers the **second** reader pass — +converting an already-parsed worker-job argument or recovery payload into the callback's +parameter type, which walks the payload's own property names and dictionary keys — because the +exception that escapes it is logged by the worker ingress too. The same contract covers every +reader that materializes a body the library did not write itself: the Redis, NATS, and database +(PostgreSQL, SQL Server, MongoDB) channels' response readers, whose parse failure is both logged +and handed to the waiter (as `InvalidDataException`), the durable-flow ledger reader, whether +it reads a stored ledger or the initial state a start job carries — the +`FlowStateUnreadableException` it raises chains the rebuilt, position-only failure, never the +reader's own — and the recovery-state readers (Redis, NATS, PostgreSQL, SQL Server, MongoDB), +on the delivery path and the watchdog scan alike: a stored registration's `Context` carries the +same propagated tenant and auth keys a worker envelope does, and the reader's `Path` names them. +The in-memory channel's typed delivery is covered too: every waiter re-materializes the published +payload from its wire bytes, and a payload that does not fit the waiter's type (a string-valued +dictionary published to an int-valued waiter) fails *inside* the payload, where the reader's own +message would name the offending dictionary key — into the waiter's exception and, through the +wait activity's error status, into telemetry. It faults the waiter with the same body-free +`InvalidDataException` the broker channels use. + +**But not our own diagnostics.** The distinction is who wrote the message. `System.Text.Json`'s +messages quote the body, so they are dropped; the envelope reader's own contract violations — +`SchemaVersion is required.`, `Payload is null or absent on a Success envelope`, `Success must be +a boolean.` — name only the wire contract's own property names and are preserved verbatim. They +are the primary operator diagnosis for the commonest malformed-envelope cause in production, a +foreign or mismatched producer writing to the response channel, and scrubbing them to "failed at +line 0, byte position 2" would cost the diagnosis while protecting nothing. Such a failure stays +a plain `JsonException` (the ingress classifies it as permanent, so it is never retried). **Nor a hash of one.** A content digest reads like harmless metadata and is not: it is deterministic, so two log entries showing the same prefix prove the two payloads were identical — @@ -142,6 +176,24 @@ id, a yes/no result) can be confirmed outright by hashing the candidates until o correlation id and the trace id already tie an entry to its conversation, which is what the digest was there for. +### The sample's test-only routes are gated + +The sample application (the integration suite's system under test) exposes unauthenticated routes +that exist for tests and demos, in two groups: the **mutation** routes — `/seed-recovery`, +`DELETE /test/recovery/{correlationId}`, and `POST /test/reset`, which erases every recovery +registration the scanner can see — and the **simulation, injection, and observability** routes — +`/arm`, `/crash` (drops every local subscription on the shared channel; with Redis it calls +`UnsubscribeAll` on the shared multiplexer), `/publish` and `/emit-response` (inject a response or +exception for any correlation id), `/lost-subscriber-flow` (composes all three), `/calls` (recorded +call data), and `GET /durable-flow/{flowId}` / `POST /durable-flow/{flowId}/resume` (a run's full +ledger, input JSON included, and an operator kick). All of them are mapped only in the Development +environment or when `Sample:EnableTestEndpoints=true` is configured (the integration AppHost, the +in-process test factory, the load-test launcher, and the Native AOT gate set it); a Production +instance answers 404 for every one and logs that they are disabled, and an integration test pins +the exact Production route inventory. If you fork the sample into a service, keep them behind that +switch — and put flow reads/resumes behind real authorization and ownership checks — rather than on +a shared backend. + ## Explicit correlation id `IAsyncResponsePublisher.SetResponse`/`SetException` take the correlation id as a **required** diff --git a/docs/sqlserver.md b/docs/sqlserver.md index 87cf7e418..830769449 100644 --- a/docs/sqlserver.md +++ b/docs/sqlserver.md @@ -29,6 +29,12 @@ mode can be added later behind the same options if demand appears): - The sweep advances a stable `created_at, id` keyset cursor until every retained row for that correlation id is considered. `PendingMessageBatchSize` controls page shape; it no longer limits one sweep to the oldest batch, so sustained progress cannot starve a later terminal response. +- Delivery is serialized per correlation id on a bounded (1024-item) executor, and the sweep + admits work to it **without waiting**: a correlation id whose executor is full (a waiter wedged + in a slow `Until` predicate under a progress flood) has the rest of its rows left unclaimed, in + order, and is rescanned alone after one poll interval, while every other correlation id keeps + delivering. Until round 35 the sweep awaited that capacity, so one saturated correlation + stalled every waiter in the process. Active waiters write rows to `asyncresponse_channel_subscribers`; one channel-level loop snapshots the registrations that are still active locally and extends only those rows in bounded SQL batches @@ -243,6 +249,14 @@ Connection-string notes: updates the process's current active-registration snapshot in bounded batches. Rows no longer in that snapshot are allowed to expire even if cleanup deletion failed. A failed batch is logged and the next interval retries, so leave enough timeout headroom for multiple attempts. +- The sweep re-reads a subscribed correlation id's retained rows on every tick (and on every + targeted signal): acknowledged rows stay in the result so a fan-out waiter in another process + still receives a response this one already consumed. Their **bodies** do not travel: the page + query ships `envelope_json` only for rows nobody has acknowledged, an acknowledged row comes back + header-only (id, timestamps, `acked_seq`), and the sweep fetches the envelope by id only for the + rare acknowledged row a live subscription has not seen. A long-lived progress subscription's + sweep cost therefore no longer grows with its whole retained history. (Until round 39 every + sweep re-transferred and re-materialized every retained body just to drop it in the pre-filter.) - `PendingMessageBatchSize` is a page-size tuning knob, not a cap per sweep. Smaller pages lower peak materialization; larger pages reduce round trips under progress-heavy correlations. - Keep `DeliveryConfirmationTimeout` long enough for the slowest expected live delivery (including diff --git a/docs/testing.md b/docs/testing.md index 1bc261a19..2fb0c0dd4 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -174,6 +174,21 @@ anyone still holding them, and disposing one afterwards (a flow disposes its wai cancelled wait; an `await using` caller does the same) is a no-op that leaves the recovery registration in place — exactly what a crashed process leaves behind. +**The restart is cooperative.** It discards everything a crash would lose and breaks the dead +incarnation's execution leases, but there is no process to kill: a step body that outlives the +graceful stop (bounded by `options.RealTimeGuard`) — it ignored its cancellation and is blocked +on something the test controls — keeps running beside the new incarnation and performs its side +effects *after* the restart returned, which is less than a "restart" claims. `SimulateRestartAsync` +therefore refuses with `InvalidOperationException` when user code is still executing after the +stop lapsed (engine-owned parks — an awaited step or an in-process timer holding its worker slot +on the virtual clock — are expected and never trip this). Let the step observe its cancellation +token or finish before restarting; for crash-*at-a-checkpoint* semantics use +`FlowTestHarness.CrashBeforeStep` / `CrashAfterStep`, which fail the attempt at the exact +boundary with nothing left running. A test that deliberately wants the overlap sets +`options.AbandonLingeringExecutionsOnRestart = true` and then owns it: the abandoned execution's +side effects land whenever it unblocks. Nothing in the harness is a subprocess kill; a guarantee +that must hold against abrupt termination needs a real process and a real broker. + This is the recovery tri-state (`Resume` / `Fail` / `KeepWaiting`) — the part of the API teams most need to test and previously could not without a broker. Waiter tasks obtained before the restart never carry a response or a timeout: the restart abandons them exactly as a crash does — diff --git a/docs/timers-and-scheduling.md b/docs/timers-and-scheduling.md index 458ffd107..a341e799e 100644 --- a/docs/timers-and-scheduling.md +++ b/docs/timers-and-scheduling.md @@ -86,7 +86,7 @@ flow, prefer `flow.DelayAsync(...)` followed by a normal enqueue — that works | Transport | Native mechanism | Per-hop cap | Notes | |---|---|---|---| -| In-memory | `TimeProvider` timer wheel | none | Delayed jobs share the process lifetime; dropped (loudly) at shutdown. Virtual-clock aware in tests. | +| In-memory | `TimeProvider` timer wheel | none | Delayed jobs share the process lifetime; dropped (loudly) at shutdown. Bounded by `DelayedJobCapacity` (default 4096): an external publisher waits for a slot, a flow parking from inside a job is rejected and redelivered. Virtual-clock aware in tests. | | Azure Service Bus | scheduled messages (`ScheduledEnqueueTime`) | none | The broker holds the message; survives restarts. | | AWS SQS | `DelaySeconds` | 15 min (chunked) | Standard queues only — SQS rejects per-message delays on FIFO queues, so a FIFO worker queue advertises **no** delay capability (`MaxPublishDelay` = zero): flow timers fall back to the in-process path, and a bare delayed enqueue fails fast at the publish call site. | | PostgreSQL | `available_at` gate on the claim query | none | Due time computed on the **database** clock (`now() + delay`); precision bounded by the subscriber's `EmptyPollDelay`. | @@ -120,6 +120,29 @@ The `input` factory receives the occurrence's scheduled UTC instant and **must b across replicas** (every replica must produce the same value for the same occurrence — don't put `Guid.NewGuid()` in it). +**A due occurrence whose start could not be published is never abandoned while the process +lives.** Starting an occurrence publishes its start job first — the job carries the initial ledger +and creates the run when executed (see +[durable-flows.md](durable-flows.md#what-happens-when-things-die)) — so a publish that fails after +the start's own retry ladder (a broker outage; `DurableFlowNotDispatchedException`) leaves nothing +persisted. The scheduler keeps such an occurrence in an in-process re-drive queue and repeats the +idempotent start every `ScheduledFlowOptions.RedriveInterval` (default 30 seconds) until the job is +published or the run is seen to have executed (another replica started it). An absent ledger is the +*expected* shape of an occurrence still waiting for its first successful publish, and the re-drive +starts it again — an earlier reading treated the absence as "expired" and gave up, which lost every +occurrence that fell due during an outage longer than the start's retry ladder. The queue dies with +its process: an occurrence whose publish was still failing at shutdown persisted nothing, so +nothing can find it after a restart and it is skipped like any occurrence missed while no replica +was up (the run history shows the gap). Separately, each schedule probes the last +`StartupRedriveWindow` (default 1 hour; zero disables it; at most the 64 most recent occurrences) +at startup and re-drives any occurrence whose ledger *exists*, is Running, and has zero attempts — +a run whose wake-up was published and then lost in transit (an early-ACK worker subscriber, a +broker that dropped the job) and that nothing else would find. A run that is merely queued behind +a busy worker looks the same and is re-driven too, harmlessly: the duplicate wake-up is absorbed by +the execution lease. Every re-drive is logged; a queue that exceeds 256 undispatched occurrences +drops the oldest with an error naming its id, which stays startable by hand with the same +occurrence id. + ## Cron syntax Five fields — `minute hour day-of-month month day-of-week` — parsed by `CronSchedule` (public, @@ -153,7 +176,10 @@ are rejected. so editing the code mid-run cannot double- or under-sleep an in-flight run. - **Schedules are at-most-once.** Occurrences that pass while *no* replica is up are skipped on restart, by design — the run history shows the gap. A late timer fire (seconds) still starts - its own occurrence. + its own occurrence. An occurrence the loop did reach but could not publish is re-driven in + process until it is (see above); one whose publish was still failing when the process died is + skipped like any other missed occurrence, because nothing was persisted for it. A published + start whose job was then lost in transit is found by the startup probe. - **Renaming a schedule** changes the ids future occurrences dedup on; in-flight runs are unaffected. - **Suspended-timer wake-ups are broker messages.** Their loss modes are the transport's loss diff --git a/docs/transport-semantics.md b/docs/transport-semantics.md index e9be98719..4ec49a222 100644 --- a/docs/transport-semantics.md +++ b/docs/transport-semantics.md @@ -98,7 +98,7 @@ remainder, which is how capped transports chunk long delays with no transport-sp | Transport | Native delayed delivery | Per-hop cap | Mechanism / caveats | |---|---|---|---| -| **InMemory** | ✅ | — | `TimeProvider` timer wheel (virtual-clock aware in tests); delayed jobs die with the process, logged at shutdown | +| **InMemory** | ✅ | — | `TimeProvider` timer wheel (virtual-clock aware in tests); delayed jobs die with the process, logged at shutdown; at most `DelayedJobCapacity` (4096) held at once — external publishers wait, in-job publishes are rejected | | **AzureServiceBus** | ✅ | — | scheduled messages (`ScheduledEnqueueTime`); broker-held, survives restarts | | **SQS** | ✅ | 15 min (chunked) | `DelaySeconds`; standard queues only — a FIFO worker queue advertises no delay capability (`MaxPublishDelay` = zero), so flow timers fall back in process and a delayed enqueue fails fast at publish | | **PostgreSQL** | ✅ | — | insert with `available_at = now() + delay` (database clock); pickup latency ≤ `EmptyPollDelay` | @@ -172,10 +172,20 @@ Only cells that need more than a phrase. - Kafka offsets cannot NACK a single message, so redelivery is in-process: a failing handler is retried with backoff (`HandlerRetryBaseDelay` 100 ms → `HandlerRetryMaxDelay` 5 s) up to - `MaxDeliveryAttempts`, stalling that partition while it retries (classic consumer-group - semantics — size `TopicNumPartitions` for parallelism). Keep the worst-case retry budget - under the consumer's `max.poll.interval.ms` or the broker evicts the consumer mid-retry - ([troubleshooting](troubleshooting.md#kafka-the-broker-evicts-the-consumer-mid-retry)). + `MaxDeliveryAttempts`, stalling that partition — and only that partition — while it retries + (classic consumer-group semantics — size `TopicNumPartitions` for parallelism). +- **A long handler never stalls the poll loop.** In ack-after-handler mode a handler is awaited + inline for `DetachHandlerAfter` (default 1 s); one still running past that is detached: its + partition is paused (Kafka's own ordering primitive — nothing is fetched for it, nothing is + buffered in-process), the handler and its retry ladder run on the thread pool, and the poll + thread keeps polling — so the consumer's other partitions keep flowing, `max.poll.interval.ms` + is honored, and rebalance callbacks fire. The poll thread (the only thread that touches the + consumer) stores the offset and resumes the partition once the handler settles, within one + `BackpressurePollDelay`; a stop waits for detached handlers and commits their offsets. Detached + handlers for different partitions run concurrently. Before this, the poll thread awaited the + whole handler, and a durable-flow step awaiting a remote response for longer than the interval + got the consumer evicted, its partitions rebalanced, and the same job redelivered to a peer + ([troubleshooting](troubleshooting.md#kafka-rebalances-or-duplicate-runs-while-long-handlers-execute)). - Attempts are counted per process delivery: a consumer restart before the offset commit resets the count. The message that exhausts its attempts is produced to the dead-letter topic with failure-detail headers and its offset committed, so the partition keeps moving. @@ -189,13 +199,46 @@ Only cells that need more than a phrase. - A message that cannot be projected at all (empty payload, unresolvable correlation id) is produced to the dead-letter topic and its offset stored, ignoring the stopping token like every other settlement path — a shutdown landing mid-burial would leave the poison message neither - buried nor committed. A `StoreOffset` that throws because a rebalance revoked the partition is - logged rather than faulting the poll loop; the message simply redelivers. -- Every dead-letter produce runs on the poll thread, so its retry ladder is bounded to a quarter - of `MaxPollInterval`: an undeliverable dead-letter topic (auto-create off, a leaderless - partition, an over-sized payload) would otherwise wait out librdkafka's `message.timeout.ms` - per attempt, overrun `max.poll.interval.ms`, and evict the consumer mid-burial. A burial that - runs out of budget leaves the offset unstored and is retried after the next restart/rebalance. + buried nor committed. **In partition order:** consumed behind a detached handler of the same + partition (a rebalance handing the partition back with its pause reset delivers the next + record), it is held exactly like a valid delivery and buried in its turn once the handler + settles — storing its offset at once committed the partition *past* the unfinished message, and + a crash after that commit skipped the valid job for good with only the malformed record's copy + in the dead-letter topic. A `StoreOffset` that throws because a rebalance revoked the partition + is logged rather than faulting the poll loop; the message simply redelivers. +- **A poll-loop failure tears down within `FaultDrainTimeout`** (default 5 s; `0` abandons at + once). When a consume fails — a dropped broker connection, a burial that failed for good — the + consumer is closed and rebuilt by the supervisor after its backoff; detached handlers that + settle within the budget get their offsets stored and committed by the close, exactly as after + a stop. The rest are abandoned: their offsets stay unstored, their messages redeliver on the + rebuilt consumer — possibly while the abandoned handler is still running, which is the + at-least-once contract every handler on this transport already carries (a durable flow's lease + makes the redelivery a no-op; a plain worker job must be idempotent) — the session's + cancellation token stops their retry ladders, and each one's eventual outcome is logged. Before + the bound, the teardown waited for every detached handler without limit, so a transient broker + failure disabled the whole subscriber for as long as an unrelated long handler took and the + configured reconnect policy never ran. A graceful stop is not bounded here; the host's shutdown + budget bounds it. +- Every dead-letter produce's retry ladder is bounded to a quarter of `MaxPollInterval`: the + malformed-message discard runs it on the poll thread, and an undeliverable dead-letter topic + (auto-create off, a leaderless partition, an over-sized payload) would otherwise wait out + librdkafka's `message.timeout.ms` per attempt, overrun `max.poll.interval.ms`, and evict the + consumer mid-burial; the ack-after-handler burial runs inside the (possibly detached) handler + task and keeps the same bound so a partition is not parked on it either. +- **A burial that fails for good faults the subscriber** (ack-after-handler mode and the + malformed-message discard). Kafka commits a partition *position*, not per-record + acknowledgements, so merely leaving the failed message's offset unstored protects nothing: the + next successful settlement on the same partition stores a higher offset and the auto-committer + commits past the failed message, which a restart then skips with no dead-letter copy anywhere. + Instead the poll loop throws, the consumer closes without ever storing past the message, and the + supervisor rebuilds it after its backoff (`SubscriberRetryBaseDelay` → `SubscriberRetryMaxDelay`); + the restarted consumer re-consumes from the committed position, re-runs the handler up to + `MaxDeliveryAttempts`, and retries the burial — a loud, bounded-rate loop that parks **every** + partition of that subscriber at the poison message until the dead-letter topic is fixed (each + restart logs the failure). That is the at-least-once outcome; the previous swallow was a silent + loss. Messages already committed at enqueue time (early ACK) are outside this rule: their + burial failure is logged and surfaced through `OnBackgroundFailure`, because Kafka will not + redeliver them either way. ### RabbitMQ @@ -228,6 +271,18 @@ Only cells that need more than a phrase. ### Redis +- **The Redis *channel* (pub/sub) is fire-and-forget, and its per-wait buffer is bounded.** A + publisher is never backpressured, and the StackExchange.Redis queue behind a subscription is + unbounded, so the channel buffers at most `1,024` responses per correlation id behind the wait's + serial processing (an `Until` predicate runs one message at a time). A response that finds that + buffer full faults the wait with the overload form of + `AsyncResponseIndeterminateDeliveryException` (a terminal response may be among the queued or the + refused ones), unsubscribes, and counts `asyncresponse.channel.overloaded_waits` — never buffered + without bound, never silently dropped. Durable flows treat the fault like the disposal-drain form + and restart the (idempotent) step. Where a backlog must be lossless, use a database channel + (PostgreSQL, SQL Server, MongoDB): its backlog stays server-side and the dispatch sweep admits it + as capacity frees. + - New entries arrive via `XREADGROUP` at attempt 1. A separate reclaim loop scans the pending-entries list every `PendingClaimInterval` (5 s) and claims entries idle longer than `PendingMessageMinIdleTime` (30 s) with `XCLAIM`, so a crashed consumer's in-flight work is @@ -279,7 +334,13 @@ Only cells that need more than a phrase. unsettled message roughly every `AckWait`/3 (two chances to land a renewal inside every `AckWait` window even when one sweep is delayed), so `AckWait` (30 s) only has to survive one heartbeat round-trip — it no longer needs to exceed the slowest handler, and before batching this was - effectively the slowest handler **× `BatchSize`** (16 by default). + effectively the slowest handler **× `BatchSize`** (16 by default). The heartbeat is advisory, + unlike the settlements (which stay deliberately uncancelable): it carries the batch's + cancellation token into the SDK call, so a heartbeat still in flight when the batch settles is + aborted, and the batch joins the heartbeat loop for at most one heartbeat interval — a heartbeat + wedged on a dead socket is abandoned with a warning rather than holding the loop after every + message in the batch has settled (which left no further batch fetched and a stop never + completing); unsettled deliveries then fall back to the server's own `AckWait`. ### SQS diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index e8d10e3b8..e6d58d328 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -61,14 +61,88 @@ owns the full story — this page is the map, not the territory. handlers), and name the queue `*.fifo` to opt into FIFO publishing. See [transport options](configuration.md#transport-options). -### Kafka: the broker evicts the consumer mid-retry - -- **Symptom:** rebalances and consumer evictions while a failing message is being retried. -- **Cause:** in-process retries happen inside one poll cycle, so the worst-case budget - `MaxDeliveryAttempts × HandlerRetryMaxDelay` can exceed the consumer's `max.poll.interval.ms` - (default 5 minutes) — the broker then considers the consumer dead. -- **Fix:** keep the retry budget well under `max.poll.interval.ms`, or raise the interval via - `ConfigureConsumer`. See [transport options](configuration.md#transport-options). +### Kafka: rebalances or duplicate runs while long handlers execute + +- **Symptom:** `Application maximum poll interval (…ms) exceeded` from librdkafka, rebalances, and + a worker job or flow step that ran twice — once here and once on the peer the partition moved + to — while a handler was still running. +- **Cause:** a consumer that stops polling for `max.poll.interval.ms` (default 5 minutes) is + evicted from its group. Before round 37 the poll thread awaited the whole handler, so a + durable-flow step awaiting a remote response or sleeping on a timer for longer than the + interval — or a long in-process retry ladder — did exactly that. Now a handler still running + after `WorkerSubscriber.DetachHandlerAfter` (default 1 s) is detached: its partition is paused, + the handler runs on, and the poll thread keeps polling; the offset is stored once the handler + settles. The symptom can therefore only remain when the inline budget itself is raised toward + the interval (validation allows up to half of it, minus `PollTimeout`), or when a + `ConfigureConsumer` hook overrides `MaxPollIntervalMs` below what the library configured. +- **Fix:** leave `DetachHandlerAfter` at its default and do not override `MaxPollIntervalMs` in + `ConfigureConsumer`; set `MaxPollInterval` on the subscriber options instead, which validates + the inline budget against it. See [transport options](configuration.md#transport-options) and + [transport semantics](transport-semantics.md#kafka). + +### Kafka: `Abandoning detached Kafka handler …` after a broker failure + +- **Symptom:** a warning naming a message whose handler was still running when the poll loop + failed, then the subscriber reconnects and the same message is handled again — for a durable + flow, a second execution that acknowledges as a duplicate (the lease is held); for a plain + worker job, a second run. +- **Cause:** the consume failed (a dropped connection, a burial that failed for good) and the + fault teardown waited `WorkerSubscriber.FaultDrainTimeout` (default 5 s) for the detached + handlers; this one did not settle in time, so its offset was left unstored and the rebuilt + consumer re-consumed it. The handler's eventual outcome is logged (`Abandoned Kafka handler … + completed/failed/stopped …`). Before the bound, the reconnect waited for every detached handler + with no limit and a transient broker failure parked the subscriber behind one long step. +- **Fix:** nothing, if the handler is idempotent — this is the transport's at-least-once + contract. Raise `FaultDrainTimeout` when handlers reliably settle within a known window and + you would rather delay the reconnect than redeliver; make plain worker jobs idempotent + regardless. See [transport semantics](transport-semantics.md#kafka). + +### `WorkerJobTooLargeException` from `EnqueueWorkerAsync` or `StartAsync` + +- **Symptom:** the publish throws `WorkerJobTooLargeException` naming the envelope's serialized + length and `AsyncResponseOptions.MaxInboundMessageChars`; nothing was published, no ledger exists. +- **Cause:** the serialized worker envelope — arguments, captured context, and for a flow start + the whole initial ledger, input included — exceeds what the consuming ingress accepts. The + ingress acknowledges such a message *without executing it* (an oversized message never gets + smaller, so redelivering it would hot-loop), so before this check the transport took the job, + the ingress dropped it, and the caller held a flow id for a `Running` run nothing would ever + execute. JSON escaping counts: quotes, non-ASCII and control characters serialize to several + times their length. +- **Fix:** put the large argument behind a claim check — persist it yourself and pass a reference + (see the [durable-flows ledger budgets](durable-flows.md#supported-ledger-budgets)) — rather + than raising the limit; if you do raise it, raise it identically on every producer and consumer + of the deployment. + +### Redis: a wait faults with `AsyncResponseIndeterminateDeliveryException` saying responses "arrived faster than the wait could process them" + +- **Symptom:** the waiter faults with the overload form of the exception (`BufferedMessages` = + 1,024), the log carries `Wait for correlationId … is overloaded`, and + `asyncresponse.channel.overloaded_waits` counts up for `channel=redis`. +- **Cause:** responses for one correlation id — typically a progress-message flood — arrived + faster than the wait's serial processing (its `Until` predicate) consumed them, and the bounded + per-wait buffer filled. Redis pub/sub cannot backpressure the publisher, and the SDK queue + behind the subscription is unbounded, so the channel refuses the next response instead of + buffering it without bound. The refused or queued responses may include the terminal one, which + is why the wait is faulted as indeterminate rather than completed or timed out. +- **Fix:** make the predicate cheap (no I/O per progress message), publish fewer progress + messages, or move the wait to a database channel, whose backlog stays server-side and is + admitted as capacity frees. Durable flows restart the awaiting step on this fault automatically; + plain waiters should treat the step as indeterminate and restart it rather than re-attach. + +### Kafka: the subscriber restarts repeatedly, each time naming a message it "could not dead-letter" + +- **Symptom:** `Kafka subscriber failed for topic … retrying in …` on a backoff cadence, each + preceded by `Failed to dead-letter Kafka message …@{offset}`; that subscriber's partitions stop + advancing. +- **Cause:** a message exhausted `MaxDeliveryAttempts` (or could not be parsed) and its produce to + the dead-letter topic keeps failing — the topic does not exist with auto-create off, its + partition is leaderless, the payload exceeds the broker's message cap. The library faults the + subscriber on purpose: leaving the offset unstored and consuming on would let the next + settlement commit past the message, losing it with no record (see + [transport semantics](transport-semantics.md#kafka)). +- **Fix:** fix the dead-letter topic (create it, size `message.max.bytes`, restore its leader). + The next restart buries the message and the partition moves. Do not raise `MaxDeliveryAttempts` + to "get past" it — the handler is re-run per restart regardless. ### RabbitMQ: startup warns about `MaxDeliveryAttempts`, or a poison message loops forever @@ -105,12 +179,26 @@ owns the full story — this page is the map, not the territory. - **Symptom:** `GetStateAsync` reports `Running`, but nothing progresses. - **Cause:** the worker job carrying the flow id dead-lettered (a retriable failure exhausted the transport's delivery attempts), or the owning process died and its execution lease has not - expired yet. + expired yet. A run with `Attempts == 0` was never picked up: its wake-up is queued behind a busy + worker, or was lost in transit (an early-ACK worker subscriber, a broker that dropped it). - **Fix:** check the transport's dead-letter queue first — the DLQ entry is the alarm. Replay it or call `ResumeAsync(flowId)` to re-enqueue the run. After a crash, expect up to `ExecutionLeaseDuration` before another replica may take the run over. See [what happens when things die](durable-flows.md#what-happens-when-things-die). +### A flow logs a `LedgerSizeWarningBytes` warning + +- **Symptom:** `Durable flow {id} ledger is roughly N bytes over K step(s), past the … threshold`, + once and then again each time the size doubles. +- **Cause:** step results (and values) accumulate in the ledger, and every checkpoint rewrites the + whole ledger — a run of N similar steps serializes about N²/2 step-results over its lifetime and + eventually hits the store's `MaxStateBytes` cap. +- **Fix:** keep large results out of the ledger (persist them yourself and pass references), + partition a long history into child flows, or — if the sizes are expected — raise + `DurableFlowOptions.LedgerSizeWarningBytes` (set `null` to disable). On DynamoDB lower it: the + 350 KB item cap sits under the 512 KiB default. See + [ledger growth](durable-flows.md#storage-where-flow-state-lives). + ### Every attempt of an awaited step fails with an `OnRecovery` error - **Symptom:** a flow never gets past its first `AwaitStepAsync`; each delivery throws diff --git a/samples/AsyncResponse.AppHost/AsyncResponse.AppHost.csproj b/samples/AsyncResponse.AppHost/AsyncResponse.AppHost.csproj index 36b9e5222..10a9ba429 100644 --- a/samples/AsyncResponse.AppHost/AsyncResponse.AppHost.csproj +++ b/samples/AsyncResponse.AppHost/AsyncResponse.AppHost.csproj @@ -1,18 +1,26 @@ - + Exe net10.0 false true + + $(NoWarn);ASPIRE010 asyncresponse-apphost-7253e3ff-66ad-4f3c-95f1-a44f0327ddda + + diff --git a/samples/AsyncResponse.Sample/Program.cs b/samples/AsyncResponse.Sample/Program.cs index 67b777cbc..12fabc359 100644 --- a/samples/AsyncResponse.Sample/Program.cs +++ b/samples/AsyncResponse.Sample/Program.cs @@ -601,6 +601,37 @@ var app = builder.Build(); app.Logger.LogInformation("AsyncResponse sample started: channel={Channel}, transport={Transport}.", channel, transport); +// --- Test-only, simulation, and observability routes: gated, never on in Production by default +// Two kinds of route are mapped only in the Development environment or when +// "Sample:EnableTestEndpoints=true" is configured (the integration AppHost, the in-process test +// factory, the load-test launcher, and the Native AOT gate set it): the test-only MUTATION routes +// (/seed-recovery, DELETE /test/recovery/{id}, /test/reset — they write and erase recovery +// registrations), and the SIMULATION / INJECTION / OBSERVABILITY routes (/arm, /crash, /publish, +// /lost-subscriber-flow, /emit-response, /calls, GET /durable-flow/{id}, /durable-flow/{id}/resume). +// None of them is authenticated: /publish and /emit-response inject responses and exceptions for any +// correlation id, /crash drops every local subscription on the shared channel (with Redis it calls +// UnsubscribeAll on the shared multiplexer), /calls and GET /durable-flow/{id} return recorded call +// data and a flow's full ledger (input JSON, execution metadata). Against a shared backend those +// strand real in-flight work or disclose it, so a Production instance answers 404 for all of them. +// Operational flow tooling (reading and resuming runs) belongs behind real authorization and +// ownership checks, not behind this switch. +var testEndpointsSetting = builder.Configuration["Sample:EnableTestEndpoints"]; +var enableTestEndpoints = bool.TryParse(testEndpointsSetting, out var enableTestEndpointsParsed) + ? enableTestEndpointsParsed + : app.Environment.IsDevelopment(); +if (enableTestEndpoints) +{ + app.Logger.LogWarning( + "Test-only endpoints (/seed-recovery, /test/recovery/{{correlationId}}, /test/reset, /arm, /crash, /publish, /lost-subscriber-flow, /emit-response, /calls, GET /durable-flow/{{flowId}}, /durable-flow/{{flowId}}/resume) are enabled in the {Environment} environment; they mutate recovery state, inject responses, and read flow ledgers without authorization. Set Sample:EnableTestEndpoints=false to disable them.", + app.Environment.EnvironmentName); +} +else +{ + app.Logger.LogInformation( + "Test-only endpoints are disabled in the {Environment} environment (set Sample:EnableTestEndpoints=true to map them).", + app.Environment.EnvironmentName); +} + app.MapOpenApi(); app.UseSwaggerUI(options => options.SwaggerEndpoint("/openapi/v1.json", "AsyncResponse sample v1")); @@ -1674,19 +1705,25 @@ static async Task CaptureFailureAsync(Task task) }) .WithTags("Flows"); +if (enableTestEndpoints) +{ app.MapGet("/durable-flow/{flowId}", async (IDurableFlows flows, string flowId) => { var state = await flows.GetStateAsync(flowId); return state is null ? Results.NotFound() : Results.Ok(state); }) .WithTags("Flows"); +} +if (enableTestEndpoints) +{ app.MapPost("/durable-flow/{flowId}/resume", async (IDurableFlows flows, string flowId) => { await flows.ResumeAsync(flowId); return Results.Ok(); }) .WithTags("Flows"); +} // 2b-child) Flow composition: the parent runs a local step, then AwaitChildFlowAsync starts a // child durable flow and suspends the parent (worker released) until the child reaches a @@ -1791,6 +1828,8 @@ await asyncResponse // 5a) Lost-subscriber recovery — arm: register a waiter with recovery callbacks and keep it waiting // in the background. The HTTP request returns immediately; the subscription and persisted // recovery state stay alive. The propagators capture the trace/tenant into the recovery state. +if (enableTestEndpoints) +{ app.MapPost("/arm", async (IServiceProvider services, FlowRecorder recorder, string? trace) => { var asyncResponse = services.GetService(); @@ -1826,9 +1865,12 @@ await asyncResponse return Results.Ok(new CorrelationResult(correlationId)); }) .WithTags("Recovery"); +} // 5b) Lost-subscriber recovery — crash: drop every local subscription, like a redeploy would. The // durable recovery state stays in the channel store; only the in-memory waiters die. +if (enableTestEndpoints) +{ app.MapPost("/crash", async (IServiceProvider services, CancellationToken cancellationToken) => { var multiplexer = services.GetService(); @@ -1862,9 +1904,12 @@ await asyncResponse return Results.Ok(); }) .WithTags("Recovery"); +} // 5c) Deliver a late response/exception for a correlation id through the configured channel (used by // the lost-subscriber recovery scenarios after /crash, and by the active-waiter scenarios). +if (enableTestEndpoints) +{ app.MapPost("/publish", async (IAsyncResponsePublisher publisher, string correlationId, string? status, string? message, string? exception) => { if (exception is not null) @@ -1880,10 +1925,13 @@ await asyncResponse return Results.Accepted(); }) .WithTags("Recovery"); +} // 5d) Composed lost-subscriber recovery: arm, simulate the crash, publish the late terminal signal, // and wait for the recovery callback in one request. This endpoint complements the lower-level // /arm + /crash + /publish endpoints that integration tests can still drive step by step. +if (enableTestEndpoints) +{ app.MapPost("/lost-subscriber-flow", async ( IAsyncResponsePublisher publisher, FlowRecorder recorder, @@ -1953,6 +2001,7 @@ await asyncResponse return Results.Ok(new LostSubscriberFlowResult(correlationId, normalized, callback)); }) .WithTags("Recovery"); +} static async Task DropLocalSubscriptionAsync( IServiceProvider services, @@ -2001,6 +2050,8 @@ static async Task DropLocalSubscriptionAsync( // 6) Publish a raw response to the configured broker response destination, acting as the remote // system. With useAttribute the correlation id rides broker metadata; otherwise it goes in the // JSON body so the extractor's JSON-path fallback is exercised. (broker transports only.) +if (enableTestEndpoints) +{ app.MapPost("/emit-response", async ( IServiceProvider services, string correlationId, @@ -2044,6 +2095,7 @@ static async Task DropLocalSubscriptionAsync( return Results.Conflict("Raw response ingress requires a transport such as AzureServiceBus, GooglePubSub, SQS, Kafka, RabbitMQ, Redis, NATS, PostgreSQL, SqlServer, or MongoDB."); }) .WithTags("Workers"); +} static async Task EmitSqsResponseAsync( IServiceProvider services, @@ -2444,6 +2496,8 @@ static ConnectionFactory CreateRabbitMqConnectionFactory(RabbitMqAsyncResponseOp // --- Observability / test affordances -------------------------------------------------------- // Long-poll the flow recorder for a recorded call (e.g. worker:{token}, resume:{cid}, waiter:{cid}). +if (enableTestEndpoints) +{ app.MapGet("/calls", async (FlowRecorder recorder, string key, int? timeoutMs) => { try @@ -2457,7 +2511,11 @@ static ConnectionFactory CreateRabbitMqConnectionFactory(RabbitMqAsyncResponseOp } }) .WithTags("Observability"); +} +// --- Test-only mutation routes (gated: see enableTestEndpoints above) -------------------- +if (enableTestEndpoints) +{ // Seed a stale recovery entry (no live subscriber) so the watchdog surfaces it as Degraded health. app.MapPost("/seed-recovery", async (IRecoveryStateStore store, string correlationId, int? ageMinutes) => { @@ -2501,6 +2559,7 @@ static ConnectionFactory CreateRabbitMqConnectionFactory(RabbitMqAsyncResponseOp return Results.Ok(new DeletedResult(deleted)); }) .WithTags("Observability"); +} // Health endpoint with full JSON details, including the recovery check's data payload. // Typed metadata + an explicit camelCase policy reproduce the previous anonymous-type health diff --git a/scripts/ci-retryable-failure.sh b/scripts/ci-retryable-failure.sh new file mode 100755 index 000000000..3335417f1 --- /dev/null +++ b/scripts/ci-retryable-failure.sh @@ -0,0 +1,158 @@ +#!/usr/bin/env bash +# Classify one CI job log: is this failure a known infrastructure flake that a re-run may retry? +# +# ./scripts/ci-retryable-failure.sh +# +# The ONE retry classifier, shared by ci.yml's in-job integration retry and auto-retry.yml's +# whole-run re-run. The two used to carry separate policies, and the in-job one was weaker: it +# retried on the fixture-boot signature alone, so a log carrying BOTH a fixture that failed to boot +# and an executed test's assertion failure entered the retry branch, and a passing second attempt +# turned a real correctness failure into a green job — one the standalone workflow could never +# see, because the job had already succeeded. +# +# Exit code / first stdout line: +# 0 flake: — every failed test is explained by a flake signature (or the runner +# itself went away) and nothing executed and failed on its merits +# 1 real: — an executed test failed on its merits, or the build broke (never retried) +# 2 unmatched — failed for a reason no signature explains (treated as real) +# 3 unreadable — the log cannot be read (an unreadable failure cannot be proven a flake) +# +# Two rules make the verdict: +# +# 1. PER FAILED TEST, not per log. The test runner prints one "failed ()" line +# per failed test followed by that test's exception and stack trace, so the log splits into +# failed-test blocks and each block is judged on its own: an assertion or XunitException is a +# real failure; a flake signature explains it (a fixture that failed to boot fails every test +# in its class through TestPipelineException, whose block also carries the boot error's own +# exception text); NEITHER means a test EXECUTED and failed for a reason no signature covers — +# a NullReferenceException, a timeout inside the test body — and that is real too. The earlier +# whole-log scan recognized only assertion-shaped failures, so a boot flake anywhere in the +# same log got a job with such a failure retried into green. +# +# 2. NO `grep | head` PIPELINES. Under `set -o pipefail`, `grep -o … | head -n 1` fails whenever +# head closes the pipe while grep is still writing (SIGPIPE, exit 141) — which a log carrying +# thousands of assertion failures does — and the failed pipeline made the `if` skip the +# real-failure check entirely, so exactly the loudest correctness failures were classified as +# retryable. Every match here is a single grep whose output is trimmed in bash. +# +# Signatures are matched over the raw log, ANSI colour codes and all: every signature is a +# substring that sits between colour escapes in the runner's output, never across them; the +# block splitter strips the escapes before looking for the "failed" line. +set -euo pipefail + +# Known hosted-runner infrastructure loss. "Fixture' threw in InitializeAsync" is a batch or matrix +# fixture (the original *BatchFixture types and the cross-product shards' *Fixture types, which +# carry no "Batch" infix) failing to boot its containers on a starved runner; "database is locked" +# is SQLite on a slow runner disk during the EF Core storm tests (see those tests' own comments); +# the last two are the runner itself going away. +FLAKE_SIGNATURES="Fixture' threw in InitializeAsync|SQLite Error 5: 'database is locked'|lost communication with the server|The runner has received a shutdown signal" + +# Evidence that a test EXECUTED and failed on its own merits: xunit.v3 assertion messages +# ("Assert.Equal() Failure: …", "Assert.Fail(): …") and the assertion exception base type. +REAL_FAILURE_SIGNATURES="Assert\\.[A-Za-z]+\\(\\) Failure|Assert\\.Fail\\(\\)|Xunit\\.Sdk\\.XunitException" + +# The build broke: compiler/MSBuild/NuGet error codes. Never a flake, whatever else the log holds. +BUILD_FAILURE_SIGNATURES="error CS[0-9]{4}|error MSB[0-9]{4}|error NU[0-9]{4}" + +log="${1:-}" +if [ -z "$log" ] || [ ! -r "$log" ]; then + echo "unreadable" + exit 3 +fi + +# The first match of an extended regex in the log, or empty. `-m 1` stops grep at the first +# matching LINE (it may print several matches from that one line); the first is kept in bash, so +# no second process ever closes a pipe on grep. +first_match() { + local found + found=$(grep -Eo -m 1 "$1" "$log" || true) + printf '%s' "${found%%$'\n'*}" +} + +# Splits the log into failed-test blocks and counts them by kind. Prints one record: +# blocks US real US flake US unexplained US real-evidence US flake-evidence US unexplained-evidence +# (US = the unit separator, so empty evidence fields survive `read`). The regexes travel through +# the environment, not -v: awk applies escape processing to -v values and would turn `\(` into `(`. +classify_blocks() { + CI_REAL_RE="$REAL_FAILURE_SIGNATURES" CI_FLAKE_RE="$FLAKE_SIGNATURES" awk ' + BEGIN { + esc = sprintf("%c", 27) + ansi_re = esc "\\[[0-9;]*[A-Za-z]" + real_re = ENVIRON["CI_REAL_RE"] + flake_re = ENVIRON["CI_FLAKE_RE"] + } + function flush() { + if (in_block) { + blocks++ + if (block_real != "") { + real_blocks++ + if (real_evidence == "") real_evidence = block_real " in " block_name + } else if (block_flake != "") { + flake_blocks++ + if (flake_evidence == "") flake_evidence = block_flake + } else { + unexplained_blocks++ + if (unexplained_evidence == "") + unexplained_evidence = (block_first != "" ? block_first : "no exception line") " in " block_name " (an executed test failed with no infrastructure signature)" + } + } + in_block = 0; block_real = ""; block_flake = ""; block_first = ""; block_name = "" + } + { gsub(ansi_re, "") } + /^[[:space:]]*failed[[:space:]]+[^[:space:]]/ { + flush() + in_block = 1 + block_name = $0 + sub(/^[[:space:]]*failed[[:space:]]+/, "", block_name) + sub(/[[:space:]]+\([^()]*\)[[:space:]]*$/, "", block_name) + next + } + /^[[:space:]]*(passed|skipped)[[:space:]]/ || /^[[:space:]]*(Test run summary|Test summary|Passed!|Failed!|Zero tests ran|total:)/ { + flush() + next + } + in_block { + if (block_first == "" && $0 ~ /[^[:space:]]/) { block_first = $0; sub(/^[[:space:]]+/, "", block_first) } + if (block_real == "" && match($0, real_re)) block_real = substr($0, RSTART, RLENGTH) + if (block_flake == "" && match($0, flake_re)) block_flake = substr($0, RSTART, RLENGTH) + } + END { + flush() + printf "%d\037%d\037%d\037%d\037%s\037%s\037%s\n", blocks, real_blocks, flake_blocks, unexplained_blocks, real_evidence, flake_evidence, unexplained_evidence + }' "$log" +} + +if build=$(first_match "$BUILD_FAILURE_SIGNATURES") && [ -n "$build" ]; then + echo "real: $build" + exit 1 +fi + +IFS=$'\037' read -r blocks real_blocks flake_blocks unexplained_blocks real_evidence flake_evidence unexplained_evidence < <(classify_blocks) + +if [ "$real_blocks" -gt 0 ]; then + echo "real: $real_evidence" + exit 1 +fi +if [ "$unexplained_blocks" -gt 0 ]; then + echo "real: $unexplained_evidence" + exit 1 +fi + +# No failed-test block carries a real failure. An assertion OUTSIDE any block (a fixture's own +# assertion, output the runner did not attribute to a test) is still one. +if real=$(first_match "$REAL_FAILURE_SIGNATURES") && [ -n "$real" ]; then + echo "real: $real" + exit 1 +fi + +if [ "$flake_blocks" -gt 0 ]; then + echo "flake: $flake_evidence" + exit 0 +fi +if flake=$(first_match "$FLAKE_SIGNATURES") && [ -n "$flake" ]; then + echo "flake: $flake" + exit 0 +fi + +echo "unmatched" +exit 2 diff --git a/scripts/tests/ci-retryable-failure.test.sh b/scripts/tests/ci-retryable-failure.test.sh new file mode 100755 index 000000000..097b23ea5 --- /dev/null +++ b/scripts/tests/ci-retryable-failure.test.sh @@ -0,0 +1,105 @@ +#!/usr/bin/env bash +# Self-test for scripts/ci-retryable-failure.sh: one fixture log per verdict, including the mixed +# log (a fixture-boot flake AND an executed-test assertion failure) that the in-job retry used to +# retry into green, the executed-test NullReferenceException the whole-log scan could not see, and +# the ten-thousand-assertion log that killed the old `grep | head` pipeline with SIGPIPE. Runs in +# the build-and-test CI job; no .NET involved. +set -euo pipefail + +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +classifier="$here/../ci-retryable-failure.sh" +work="$(mktemp -d)" +trap 'rm -rf "$work"' EXIT + +failures=0 +expect() { + local name="$1" expected_code="$2" expected_prefix="$3" content="$4" + local file="$work/$name.log" + printf '%b' "$content" > "$file" + local output code=0 + output=$("$classifier" "$file") || code=$? + if [ "$code" != "$expected_code" ] || [[ "$output" != "$expected_prefix"* ]]; then + echo "FAIL $name: expected exit $expected_code with '$expected_prefix…', got exit $code with '$output'" + failures=$((failures + 1)) + else + echo "ok $name: exit $code ($output)" + fi +} + +expect pure-fixture-flake 0 "flake: Fixture' threw in InitializeAsync" \ + " failed AsyncResponse.IntegrationTests.Foo\n Xunit.Sdk.TestPipelineException: Class fixture type 'AsyncResponse.IntegrationTests.DataBatchFixture' threw in InitializeAsync\n" +expect matrix-fixture-flake 0 "flake: Fixture' threw in InitializeAsync" \ + "Xunit.Sdk.TestPipelineException: Collection fixture type 'AsyncResponse.IntegrationTests.MatrixDatabaseLightFixture' threw in InitializeAsync\n" +expect sqlite-locked 0 "flake: SQLite Error 5: 'database is locked'" \ + "Microsoft.Data.Sqlite.SqliteException : SQLite Error 5: 'database is locked'.\n" +expect runner-lost 0 "flake: The runner has received a shutdown signal" \ + "The runner has received a shutdown signal. This can happen when the runner service is stopped.\n" +# The finding: a boot flake in the same log as an executed test that failed on its merits. +expect mixed-fixture-and-assertion 1 "real: Assert.Equal() Failure" \ + "Class fixture type 'AsyncResponse.IntegrationTests.BrokersBatchFixture' threw in InitializeAsync\n failed AsyncResponse.IntegrationTests.Bar\n Assert.Equal() Failure: Values differ\nExpected: 2\nActual: 1\n" +expect mixed-fixture-and-xunit-exception 1 "real: Xunit.Sdk.XunitException" \ + "Fixture' threw in InitializeAsync\nXunit.Sdk.XunitException: timed out waiting for the flow\n" +expect pure-assertion 1 "real: Assert.True() Failure" \ + " failed AsyncResponse.IntegrationTests.Baz\n Assert.True() Failure\n" +expect assert-fail 1 "real: Assert.Fail()" \ + " Assert.Fail(): Timed out waiting for a log message\n" +expect build-error 1 "real: error CS1002" \ + "Program.cs(12,4): error CS1002: ; expected\n" +expect ansi-coloured-assertion 1 "real: Assert.Equal() Failure" \ + "\e[31mFixture' threw in InitializeAsync\e[0m\n\e[91m Assert.Equal() Failure: Values differ\e[0m\n" +# A test that EXECUTED and failed for a reason no signature explains is a real failure, not an +# unmatched one: its block has neither an assertion nor a flake signature. +expect executed-test-unexplained-exception 1 "real: System.TimeoutException: the container did not answer in AsyncResponse.IntegrationTests.Qux" \ + " failed AsyncResponse.IntegrationTests.Qux\n System.TimeoutException: the container did not answer\n" +expect unmatched-without-test-blocks 2 "unmatched" \ + "System.TimeoutException: the container did not answer\n" +expect empty 2 "unmatched" "" + +# ---- Round 37: the two inputs the classifier got wrong, plus the block splitter's edge cases. ---- + +# 1. A fixture-boot flake (retryable on its own) next to an executed test that died with a +# NullReferenceException. No assertion signature anywhere, so the whole-log scan saw only the +# flake and retried the job — the correctness failure could pass on attempt 2. +expect fixture-flake-plus-executed-nre 1 "real: System.NullReferenceException: Object reference not set to an instance of an object. in AsyncResponse.IntegrationTests.Foo.Bar" \ + " failed AsyncResponse.IntegrationTests.Boot.Baz (12ms)\n Xunit.Sdk.TestPipelineException: Class fixture type 'AsyncResponse.IntegrationTests.DataBatchFixture' threw in InitializeAsync\n at Xunit.v3.FixtureMappingManager.GetFixture(Type)\n\n failed AsyncResponse.IntegrationTests.Foo.Bar (3s 40ms)\n System.NullReferenceException: Object reference not set to an instance of an object.\n at AsyncResponse.IntegrationTests.Foo.Bar() in /src/tests/Foo.cs:line 42\n" + +# 2. Ten thousand assertion failures in one log. `grep -o … | head -n 1` under pipefail: head +# closed the pipe while grep was still writing, grep died of SIGPIPE, the pipeline "failed", +# the `if` skipped the real-failure branch, and the fixture-flake line further down retried it. +ten_thousand="Class fixture type 'AsyncResponse.IntegrationTests.BrokersBatchFixture' threw in InitializeAsync\n" +for _ in $(seq 1 10000); do + ten_thousand+=" failed AsyncResponse.IntegrationTests.Many (1ms)\n Assert.Equal() Failure: Values differ\n" +done +expect ten-thousand-assertions 1 "real: Assert.Equal() Failure in AsyncResponse.IntegrationTests.Many" "$ten_thousand" + +# The boot error's own exception text lives INSIDE the fixture block; it does not make the block real. +expect fixture-block-with-inner-exception 0 "flake: Fixture' threw in InitializeAsync" \ + " failed AsyncResponse.IntegrationTests.Boot.Baz (12ms)\n Xunit.Sdk.TestPipelineException: Class fixture type 'AsyncResponse.IntegrationTests.DataBatchFixture' threw in InitializeAsync\n ---- System.Net.Http.HttpRequestException: Connection refused (localhost:8080)\n ---- System.TimeoutException: the container did not answer\n at Aspire.Hosting.DistributedApplication.StartAsync()\n" + +# The "failed" line itself wrapped in colour codes still opens a block. +expect ansi-failed-line-with-nre 1 "real: System.NullReferenceException" \ + "\e[31m failed AsyncResponse.IntegrationTests.Boot.Baz\e[0m\n\e[91m Xunit.Sdk.TestPipelineException: Class fixture type 'XFixture' threw in InitializeAsync\e[0m\n\e[31m failed AsyncResponse.IntegrationTests.Foo.Bar (3s)\e[0m\n\e[91m System.NullReferenceException: boom\e[0m\n" + +# A passed/skipped line or the summary ends a block: the assertion after the summary belongs to no +# test, and the whole-log scan still reports it. +expect block-ends-at-summary 1 "real: Assert.True() Failure" \ + " failed AsyncResponse.IntegrationTests.Boot.Baz\n Xunit.Sdk.TestPipelineException: Class fixture type 'XFixture' threw in InitializeAsync\n passed AsyncResponse.IntegrationTests.Ok (1ms)\nTest run summary: Failed!\n Assert.True() Failure\n" + +# Build errors outrank everything, blocks included. +expect build-error-with-flake-blocks 1 "real: error MSB3027" \ + " failed AsyncResponse.IntegrationTests.Boot.Baz\n Xunit.Sdk.TestPipelineException: Class fixture type 'XFixture' threw in InitializeAsync\nerror MSB3027: Could not copy\n" + +code=0 +"$classifier" "$work/does-not-exist.log" > "$work/missing.out" || code=$? +if [ "$code" != 3 ] || [ "$(cat "$work/missing.out")" != "unreadable" ]; then + echo "FAIL missing-log: expected exit 3 'unreadable', got exit $code '$(cat "$work/missing.out")'" + failures=$((failures + 1)) +else + echo "ok missing-log: exit 3 (unreadable)" +fi + +if [ "$failures" -ne 0 ]; then + echo "$failures classifier self-test(s) failed" + exit 1 +fi +echo "all classifier self-tests passed" diff --git a/src/AsyncResponse.Abstractions/AsyncResponseIndeterminateDeliveryException.cs b/src/AsyncResponse.Abstractions/AsyncResponseIndeterminateDeliveryException.cs index d571f3798..d9797fc50 100644 --- a/src/AsyncResponse.Abstractions/AsyncResponseIndeterminateDeliveryException.cs +++ b/src/AsyncResponse.Abstractions/AsyncResponseIndeterminateDeliveryException.cs @@ -1,11 +1,14 @@ namespace AsyncResponse; /// -/// Faults a waiter's when disposal abandoned an -/// in-flight delivery without learning its outcome: the drain of a dispatch that had already -/// claimed a message — typically an Until predicate still running user code — did not finish -/// within the channel's DisposalDrainTimeout. The response may or may not have been consumed -/// from the channel. +/// Faults a waiter's when the channel can no +/// longer say whether the response was delivered. Two causes: disposal abandoned an in-flight +/// delivery without learning its outcome (the drain of a dispatch that had already claimed a +/// message — typically an Until predicate still running user code — did not finish within +/// the channel's DisposalDrainTimeout), or a fire-and-forget channel was overloaded +/// (responses for the correlation id arrived faster than the wait could process them and the +/// bounded per-wait buffer filled; the next one could not be admitted). Either way the response +/// may or may not have been consumed from the channel. /// /// This is deliberately not a cancellation. A canceled response task tells the caller /// "nothing was delivered", which invites re-attaching to the correlation id — and if the wedged @@ -26,6 +29,29 @@ public AsyncResponseIndeterminateDeliveryException(string? correlationId, TimeSp CorrelationId = correlationId; } + /// + /// The overload form: responses were already queued behind + /// the wait's serial processing when the next one arrived and could not be admitted. Nothing + /// is discarded silently — the wait is faulted so the caller restarts the (idempotent) step — + /// but a terminal response may be among the queued or the refused ones. + /// + public AsyncResponseIndeterminateDeliveryException(string? correlationId, int bufferedMessages) + : base($"Responses for correlationId '{correlationId}' arrived faster than the wait could process them: " + + $"{bufferedMessages} were already queued behind its serial processing when the next one could not be " + + "admitted. A terminal response may be among them; treat delivery as indeterminate and restart the " + + "awaiting (idempotent) step instead of re-attaching to this correlation id. Speed up the completion " + + "predicate, publish fewer progress messages, or use a retained (database) channel whose backlog stays server-side.") + { + CorrelationId = correlationId; + BufferedMessages = bufferedMessages; + } + /// The correlation id whose delivery outcome is unknown. public string? CorrelationId { get; } + + /// + /// For the overload form, how many responses were queued behind the wait's serial processing + /// when the next one could not be admitted; 0 for the disposal-drain form. + /// + public int BufferedMessages { get; } } diff --git a/src/AsyncResponse.Abstractions/FlowState.cs b/src/AsyncResponse.Abstractions/FlowState.cs index f0b7c7893..ad57f3704 100644 --- a/src/AsyncResponse.Abstractions/FlowState.cs +++ b/src/AsyncResponse.Abstractions/FlowState.cs @@ -97,6 +97,18 @@ public sealed class FlowState /// which may happen in a different deployment. /// public Dictionary? Context { get; set; } + + /// + /// Retention floor: the earliest UTC instant this ledger may be allowed to expire, stamped when + /// this run — or a descendant flow waiting on this chain — parks for a window longer than the + /// ordinary idle StateExpiry. Every ledger write of a non-terminal run honors it: the + /// TTL a checkpoint stamps is raised to reach this instant, so a concurrent checkpoint that + /// knows nothing about the park (an ancestor's replay, an executor's per-attempt save) cannot + /// shrink the retention back under a wait that is still in progress. Additive wire property: + /// absent on ledgers written before it existed and on runs that never parked beyond their own + /// expiry. Ignored once the run is terminal. + /// + public DateTime? RetainUntilUtc { get; set; } } /// One step's checkpoint inside . diff --git a/src/AsyncResponse.Abstractions/IAsyncResponseBuilder.cs b/src/AsyncResponse.Abstractions/IAsyncResponseBuilder.cs index 3001d1b86..ac995ab3d 100644 --- a/src/AsyncResponse.Abstractions/IAsyncResponseBuilder.cs +++ b/src/AsyncResponse.Abstractions/IAsyncResponseBuilder.cs @@ -48,7 +48,15 @@ public interface IAsyncResponseBuilder /// Publishes a work descriptor to the configured . Prefer the /// expression-based overloads: they keep the target service's methods rooted under trimming, /// which a hand-written descriptor cannot. + /// + /// Every overload measures the serialized envelope against + /// AsyncResponseOptions.MaxInboundMessageChars before publishing and throws + /// when it would exceed what the consuming ingress + /// accepts — the ingress acknowledges an oversized message without executing it, so a + /// publish that succeeded would have been a job that silently never ran. + /// /// + /// The serialized envelope exceeds the ingress budget; nothing was published. [RequiresUnreferencedCode("The descriptor names its target service and method as strings, resolved by reflection when the " + "job executes; trimming may have removed them. Use the expression-based EnqueueWorkerAsync " + "overloads, which root the service's public methods automatically.")] diff --git a/src/AsyncResponse.Abstractions/IDurableFlow.cs b/src/AsyncResponse.Abstractions/IDurableFlow.cs index 51a619abb..187e91048 100644 --- a/src/AsyncResponse.Abstractions/IDurableFlow.cs +++ b/src/AsyncResponse.Abstractions/IDurableFlow.cs @@ -70,22 +70,19 @@ public DurableFlowIdConflictException(string message) : base(message) } /// -/// Thrown by IDurableFlows.StartAsync when the flow's ledger was committed but its worker -/// job could not be published — the run exists, in Running, with nothing scheduled to -/// execute it. -/// -/// The distinct type exists to carry out of the failure. A start called -/// without an explicit id generates one, and a plain throw discarded it: the caller was left -/// knowing a flow might exist but not which, and the store interface has no enumeration to go -/// looking. With the id in hand, recovery is a re-call of StartAsync with that same id — -/// idempotent by contract, since an identical start re-enqueues the existing run rather than -/// creating a second one. -/// +/// Thrown by IDurableFlows.StartAsync when the flow's start job could not be published to the +/// worker transport after retries. Nothing was persisted: the publish is the start's commit +/// point (the job carries the initial ledger and its execution creates the run), so a failed +/// publish leaves no orphaned Running ledger behind — the caller simply retries the start. +/// carries the id the start would have used, including a generated one, so a +/// retry can reuse it and stay idempotent: an identical start of an id that already exists +/// re-enqueues the existing run rather than creating a second one. /// /// Publication is retried before this surfaces, so it means the transport stayed unavailable, not /// that it blinked. The ambiguous case is deliberately included: a publish that may or may not -/// have landed also throws here, because a duplicate delivery of a durable flow is harmless -/// (completed steps skip via their checkpoints) while a dropped one is not. +/// have landed also throws here. If it did land, the job creates and runs the flow on its own; a +/// retried start with the SAME id then dedupes against that run, while a retry with a fresh +/// generated id starts a second, independent run — supply deterministic ids where callers retry. /// /// public sealed class DurableFlowNotDispatchedException : InvalidOperationException @@ -93,12 +90,12 @@ public sealed class DurableFlowNotDispatchedException : InvalidOperationExceptio /// Creates the failure for . public DurableFlowNotDispatchedException(string flowId, Exception? innerException = null) : base( - $"Durable flow '{flowId}' was persisted but its worker job could not be published, so nothing " + - $"is scheduled to execute it. Retry the start with this same flow id — an identical start " + - $"re-enqueues the existing run instead of creating a duplicate.", + $"Durable flow '{flowId}' could not be started: its worker job was not published, so nothing " + + $"was persisted and nothing is scheduled to execute it. Retry the start with this same flow id — an " + + $"identical start is idempotent, so a job that did land is not duplicated.", innerException) => FlowId = flowId; - /// The id of the persisted-but-undispatched run, so a caller can re-drive it. + /// The id the start would have used, so a caller can retry idempotently with it. public string FlowId { get; } } diff --git a/src/AsyncResponse.Abstractions/IDurableFlowContext.cs b/src/AsyncResponse.Abstractions/IDurableFlowContext.cs index 3f3d80750..bf5f45ee7 100644 --- a/src/AsyncResponse.Abstractions/IDurableFlowContext.cs +++ b/src/AsyncResponse.Abstractions/IDurableFlowContext.cs @@ -93,6 +93,16 @@ Task AwaitStepAsync( /// is thrown unless /// is false. /// + /// + /// The returned is that memoized snapshot on every execution — + /// the first completion and each replay alike, so a parent cannot branch differently after a + /// restart on a step it had already completed. The snapshot carries the child's status, + /// message, input, values, and step checkpoints, but not its captured ambient + /// , and the of the + /// child's own child-flow steps is elided (their , + /// completion, and fault marker stay; load a grandchild by that id through + /// while its ledger lives). + /// /// Task AwaitChildFlowAsync<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors | DynamicallyAccessedMemberTypes.PublicMethods | DynamicallyAccessedMemberTypes.Interfaces)] TFlow, TInput>( string name, diff --git a/src/AsyncResponse.Abstractions/IDurableFlows.cs b/src/AsyncResponse.Abstractions/IDurableFlows.cs index 2b6702e5f..0d5430fd1 100644 --- a/src/AsyncResponse.Abstractions/IDurableFlows.cs +++ b/src/AsyncResponse.Abstractions/IDurableFlows.cs @@ -11,6 +11,10 @@ public interface IDurableFlows { /// /// Creates a flow run and enqueues its execution on the worker transport. Returns the flow id. + /// The publish of the start job is the commit point: the job carries the initial ledger and its + /// execution creates the run if the starter's own ledger write never happened, so a process that + /// dies mid-start leaves either nothing or a run that executes — never a committed ledger that + /// nothing will ever wake. The ledger exists by the time this method returns. /// /// Pass a non-empty to make the start idempotent: starting an id that /// already exists with the same flow type and semantically identical input re-enqueues the @@ -22,10 +26,11 @@ public interface IDurableFlows /// is empty or whitespace. /// already belongs to different work. /// - /// The ledger was committed but the worker job could not be published, even after retries — the - /// run exists as with nothing scheduled to execute it. - /// carries the id (including a generated - /// one) so the orphan can be re-driven: + /// The start job could not be published to the worker transport, even after retries. Nothing + /// was persisted — the publish is the start's commit point (the job carries the initial ledger + /// and its execution creates the run), so no orphaned ledger + /// is left behind. carries the id the + /// start would have used (including a generated one) so a retry can stay idempotent: /// /// try /// { @@ -33,11 +38,17 @@ public interface IDurableFlows /// } /// catch (DurableFlowNotDispatchedException ex) /// { - /// // Idempotent: the atomic create dedupes and re-enqueues the SAME run. + /// // Idempotent: if the publish did land after all, the same id dedupes against that run. /// return await flows.StartAsync<ProvisioningFlow, ProvisionRequest>(request, ex.FlowId); /// } /// /// + /// + /// The start job — which carries the serialized initial ledger, input included — exceeds the + /// ingress's AsyncResponseOptions.MaxInboundMessageChars budget; the consuming ingress + /// would acknowledge it without ever executing it. Deterministic, so not retried and not + /// wrapped; nothing was persisted. Shrink the input or pass a reference to it. + /// Task StartAsync<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors | DynamicallyAccessedMemberTypes.PublicMethods | DynamicallyAccessedMemberTypes.Interfaces)] TFlow, TInput>( TInput input, string? flowId = null, diff --git a/src/AsyncResponse.Abstractions/IFlowStateStore.cs b/src/AsyncResponse.Abstractions/IFlowStateStore.cs index 71b922e81..19424e9cc 100644 --- a/src/AsyncResponse.Abstractions/IFlowStateStore.cs +++ b/src/AsyncResponse.Abstractions/IFlowStateStore.cs @@ -16,11 +16,13 @@ Task TryCreateAsync( /// /// Loads the state of one flow run, or null when the run is genuinely gone — unknown, - /// pruned, or expired. A row that exists but cannot be interpreted is not absence and - /// must throw : callers acknowledge a wake-up on - /// null, so reporting a live-but-unreadable ledger that way strands the run. + /// pruned, or expired. A row that exists but cannot be interpreted — malformed JSON, an + /// unknown schema version, a revision inside the JSON that disagrees with the stored one, a + /// flow id inside the JSON that is not the key — is not absence and must throw + /// : callers acknowledge a wake-up on null, + /// so reporting a live-but-unreadable ledger that way strands the run. /// - /// The ledger exists but is uninterpretable. + /// The ledger exists but is uninterpretable or inconsistent. Task LoadAsync(string flowId, CancellationToken cancellationToken = default); /// diff --git a/src/AsyncResponse.Abstractions/PublicAPI.Unshipped.txt b/src/AsyncResponse.Abstractions/PublicAPI.Unshipped.txt index 0f90e9cee..eb27f63bd 100644 --- a/src/AsyncResponse.Abstractions/PublicAPI.Unshipped.txt +++ b/src/AsyncResponse.Abstractions/PublicAPI.Unshipped.txt @@ -100,6 +100,8 @@ AsyncResponse.FlowState.ParentFlowId.get -> string? AsyncResponse.FlowState.ParentFlowId.set -> void AsyncResponse.FlowState.ParentStepName.get -> string? AsyncResponse.FlowState.ParentStepName.set -> void +AsyncResponse.FlowState.RetainUntilUtc.get -> System.DateTime? +AsyncResponse.FlowState.RetainUntilUtc.set -> void AsyncResponse.FlowState.Revision.get -> long AsyncResponse.FlowState.Revision.set -> void AsyncResponse.FlowState.SchemaVersion.get -> int @@ -362,3 +364,9 @@ static AsyncResponse.WorkerJobEnvelopeSchema.IsReadable(int entryVersion) -> boo ~override AsyncResponse.DurableFlowRunEvent.ToString() -> string ~override AsyncResponse.DurableFlowStepEvent.Equals(object obj) -> bool ~override AsyncResponse.DurableFlowStepEvent.ToString() -> string +AsyncResponse.WorkerJobTooLargeException +AsyncResponse.WorkerJobTooLargeException.Limit.get -> int +AsyncResponse.WorkerJobTooLargeException.SerializedLength.get -> int +AsyncResponse.WorkerJobTooLargeException.WorkerJobTooLargeException(int serializedLength, int limit) -> void +AsyncResponse.AsyncResponseIndeterminateDeliveryException.AsyncResponseIndeterminateDeliveryException(string? correlationId, int bufferedMessages) -> void +AsyncResponse.AsyncResponseIndeterminateDeliveryException.BufferedMessages.get -> int diff --git a/src/AsyncResponse.Abstractions/WorkerJobTooLargeException.cs b/src/AsyncResponse.Abstractions/WorkerJobTooLargeException.cs new file mode 100644 index 000000000..c018a554c --- /dev/null +++ b/src/AsyncResponse.Abstractions/WorkerJobTooLargeException.cs @@ -0,0 +1,43 @@ +namespace AsyncResponse; + +/// +/// Thrown by every EnqueueWorkerAsync overload — and by IDurableFlows.StartAsync, +/// whose start job carries the initial ledger — when the serialized worker envelope exceeds +/// AsyncResponseOptions.MaxInboundMessageChars, the budget the consuming ingress enforces. +/// Nothing was published. +/// +/// Without this producer-side check the job left the process: the transport accepted it (every +/// database transport and most brokers take far more than the engine's default 8 Mi +/// characters), the ingress then acknowledged it without executing it — an oversized +/// message never gets smaller, so redelivering it would hot-loop — and the caller held a flow id +/// for a run recorded as Running that nothing would ever execute. Failing here, in the +/// caller's stack, is the only place the mistake can still be corrected. +/// +/// +/// The measurement is exact: the envelope is serialized the way the transports serialize it and +/// compared in UTF-16 code units, which is what the ingress compares. JSON escaping counts — a +/// 5 Mi-character argument of quotes or non-ASCII text serializes to several times its length. +/// The producer's own MaxInboundMessageChars stands in for the consumer's; keep the option +/// identical across the processes of one deployment. Put large arguments behind a claim check +/// (persist the data and pass a reference) rather than raising the limit. +/// +/// +public sealed class WorkerJobTooLargeException : InvalidOperationException +{ + /// Creates the exception for an envelope of code units over . + public WorkerJobTooLargeException(int serializedLength, int limit) + : base( + $"The worker job envelope serializes to {serializedLength} UTF-16 code units, over the {limit} the consuming ingress " + + "accepts (AsyncResponseOptions.MaxInboundMessageChars); it would be acknowledged without ever executing. Nothing was " + + "published. Put large arguments behind a claim check (persist the data and pass a reference) rather than raising the limit.") + { + SerializedLength = serializedLength; + Limit = limit; + } + + /// The envelope's serialized length in UTF-16 code units, as the ingress would measure it. + public int SerializedLength { get; } + + /// The configured budget the envelope exceeded. + public int Limit { get; } +} diff --git a/src/AsyncResponse.Core/AsyncResponseBuilder.cs b/src/AsyncResponse.Core/AsyncResponseBuilder.cs index 242e5e583..2d35cd275 100644 --- a/src/AsyncResponse.Core/AsyncResponseBuilder.cs +++ b/src/AsyncResponse.Core/AsyncResponseBuilder.cs @@ -1,3 +1,4 @@ +using Microsoft.Extensions.Options; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Linq.Expressions; @@ -8,10 +9,102 @@ internal abstract class AsyncResponseBuilderBase( IWorkerTransport? _workerTransport = null, IAsyncResponseReplyTargetProvider? _replyTargetProvider = null, AsyncResponseContextPropagation? _propagation = null, - TimeProvider? _timeProvider = null) + TimeProvider? _timeProvider = null, + IOptions? _options = null) { protected IAsyncResponseReplyTargetProvider? ReplyTargetProvider => _replyTargetProvider; + // --------------------------------------------------------------------------------------- + // Producer-side size budget — the ingress's MaxInboundMessageChars, enforced before publish. + + /// + /// Worst-case UTF-16 growth of a string under System.Text.Json's default encoder: any code + /// unit can become a six-character \uXXXX escape (non-ASCII, HTML-sensitive and + /// control characters all do), and no character ever shrinks. + /// + private const int MaxJsonEscapeFactor = 6; + + /// Property names, punctuation and the fixed-width members of one envelope, generously. + private const int EnvelopeFixedOverhead = 1024; + private const int PerParamOverhead = 64; + private const int PerEntryOverhead = 16; + private const int ScalarOverhead = 64; + + /// + /// Refuses an envelope the consuming ingress would acknowledge without executing. The ingress + /// compares the delivered JSON's UTF-16 length against + /// and drops what exceeds it (an oversized message never gets smaller, so redelivery would + /// hot-loop); without this check the publish succeeded, the caller kept a flow id or a + /// fire-and-forget "success", and the work silently never ran. Measured exactly — the same + /// serialization the transports perform — but only when a cheap upper bound says it might + /// matter, so the hot path of small jobs pays no extra serialization. + /// + /// The serialized envelope exceeds the budget. + protected void ThrowIfOverInboundBudget(WorkerJobEnvelope envelope) + { + if (_options?.Value.MaxInboundMessageChars is not { } limit) + return; + + if (TryEstimateUpperBound(envelope, out var upperBound) && upperBound <= limit) + return; + + var serialized = AsyncResponseJson.Serialize(envelope); + if (serialized.Length > limit) + throw new WorkerJobTooLargeException(serialized.Length, limit); + } + + /// + /// An upper bound on the envelope's serialized UTF-16 length that never undercounts: every + /// string at its fully-escaped size, every scalar at a fixed allowance, fixed overhead for + /// the property names and punctuation. Returns false when an argument is an arbitrary + /// object whose size cannot be bounded without serializing it. + /// + internal static bool TryEstimateUpperBound(WorkerJobEnvelope envelope, out long upperBound) + { + long total = EnvelopeFixedOverhead; + total += Escaped(envelope.CorrelationId); + total += Escaped(envelope.Call.ServiceInterfaceFullName) + Escaped(envelope.Call.MethodName); + + if (envelope.ReplyTarget is { } target) + { + total += Escaped(target.Name) + Escaped(target.Transport) + Escaped(target.Address); + foreach (var (key, value) in target.Properties) + total += Escaped(key) + Escaped(value) + PerEntryOverhead; + } + + if (envelope.Context is { } context) + { + foreach (var (key, value) in context) + total += Escaped(key) + Escaped(value) + PerEntryOverhead; + } + + foreach (var param in envelope.Call.Params) + { + total += PerParamOverhead; + switch (param.Value) + { + case null: + break; + case string text: + total += Escaped(text); + break; + case bool or byte or sbyte or short or ushort or int or uint or long or ulong + or float or double or decimal or char or Guid or DateTime or DateTimeOffset or TimeSpan: + total += ScalarOverhead; + break; + default: + upperBound = 0; + return false; + } + } + + upperBound = total; + return true; + } + + private static long Escaped(string? value) + => value is null ? 8 : (long)value.Length * MaxJsonEscapeFactor + 2; + /// Validates the supplied options. protected static string ValidateCorrelationId(string correlationId) { @@ -76,6 +169,7 @@ private async Task EnqueueWorkerCoreAsync(ReflectionCallDto work, TimeSpan delay if (delay <= TimeSpan.Zero) { + ThrowIfOverInboundBudget(envelope); await transport.PublishAsync(envelope, cancellationToken).ConfigureAwait(false); return; } @@ -110,6 +204,8 @@ private async Task EnqueueWorkerCoreAsync(ReflectionCallDto work, TimeSpan delay // by the worker-job executor for the remainder, so the due time holds end to end. envelope.NotBeforeUtc = (_timeProvider ?? TimeProvider.System).GetUtcNow().UtcDateTime.Add(delay); var hop = delay <= delayedTransport.MaxPublishDelay ? delay : delayedTransport.MaxPublishDelay; + // After the due-time stamp: the check measures the envelope exactly as it is published. + ThrowIfOverInboundBudget(envelope); await delayedTransport.PublishAsync(envelope, hop, cancellationToken).ConfigureAwait(false); } catch (Exception ex) @@ -168,8 +264,9 @@ internal sealed class AsyncResponseBuilder( IWorkerTransport? workerTransport = null, IAsyncResponseReplyTargetProvider? replyTargetProvider = null, AsyncResponseContextPropagation? propagation = null, - TimeProvider? timeProvider = null) - : AsyncResponseBuilderBase(workerTransport, replyTargetProvider, propagation, timeProvider), + TimeProvider? timeProvider = null, + IOptions? options = null) + : AsyncResponseBuilderBase(workerTransport, replyTargetProvider, propagation, timeProvider, options), IAsyncResponseBuilder { /// @@ -187,8 +284,9 @@ internal sealed class RecoverableAsyncResponseBuilder( IWorkerTransport? workerTransport = null, IAsyncResponseReplyTargetProvider? replyTargetProvider = null, AsyncResponseContextPropagation? propagation = null, - TimeProvider? timeProvider = null) - : AsyncResponseBuilderBase(workerTransport, replyTargetProvider, propagation, timeProvider), + TimeProvider? timeProvider = null, + IOptions? options = null) + : AsyncResponseBuilderBase(workerTransport, replyTargetProvider, propagation, timeProvider, options), IRecoverableAsyncResponseBuilder { /// diff --git a/src/AsyncResponse.Core/AsyncResponseDiagnostics.cs b/src/AsyncResponse.Core/AsyncResponseDiagnostics.cs index 562349e5f..27d27b3d9 100644 --- a/src/AsyncResponse.Core/AsyncResponseDiagnostics.cs +++ b/src/AsyncResponse.Core/AsyncResponseDiagnostics.cs @@ -58,9 +58,94 @@ public static class AsyncResponseDiagnostics Meter.CreateCounter("asyncresponse.ingress.oversized_messages", unit: "{message}", description: "Inbound messages acknowledged without processing because they exceed AsyncResponseOptions.MaxInboundMessageChars, tagged by route."); + private static readonly Counter FlowStatePrunedRows = + Meter.CreateCounter("asyncresponse.flow_state.pruned_rows", unit: "{row}", + description: "Expired durable-flow ledger rows deleted by the relational stores' opportunistic prune, tagged by provider."); + + private static readonly Counter FlowStatePruneFailures = + Meter.CreateCounter("asyncresponse.flow_state.prune_failures", unit: "{failure}", + description: "Opportunistic durable-flow prunes that failed (the flow creation they rode on still succeeded; the next PruneInterval retries), tagged by provider."); + + private static readonly Counter FlowStatePruneBudgetExhausted = + Meter.CreateCounter("asyncresponse.flow_state.prune_budget_exhausted", unit: "{prune}", + description: "Opportunistic durable-flow prunes that stopped at PruneBudget with expired rows still remaining — the expired backlog is outgrowing the prune, tagged by provider."); + + private static readonly Counter OverloadedWaitsCounter = + Meter.CreateCounter("asyncresponse.channel.overloaded_waits", unit: "{wait}", + description: "Waits faulted as indeterminate because responses for their correlation id arrived faster than the wait could process them and the bounded per-wait buffer was full (fire-and-forget channels), tagged by channel."); + + private static readonly Counter InMemoryOverflowRejections = + Meter.CreateCounter("asyncresponse.worker.inmemory_overflow_rejections", unit: "{job}", + description: "Follow-up jobs the in-memory worker transport refused because its queue was full and the in-job overflow was at InJobOverflowCapacity; the publishing job failed and is redelivered."); + + private static readonly Counter InMemoryDelayedRejections = + Meter.CreateCounter("asyncresponse.worker.inmemory_delayed_rejections", unit: "{job}", + description: "Delayed jobs published from inside a running job that the in-memory worker transport refused because DelayedJobCapacity jobs were already scheduled; the publishing job failed and is redelivered."); + + // Every live in-memory transport in the process, for the overflow-depth gauge: the meter is + // static and a process may host several transports (test harnesses, host-per-tenant workers), + // so the gauge sums them and drops the ones that have been collected. Weak references keep a + // disposed host's transport from being pinned for the process lifetime by its own telemetry. + private static readonly List> _inMemoryTransports = []; + private static int _inMemoryOverflowGaugeRegistered; + private static int _watchdogGaugesRegistered; private static AsyncResponseWatchdogState? _watchdogState; + /// Records one follow-up publish the in-memory transport rejected at its in-job overflow capacity. + internal static void RecordInMemoryOverflowRejection() + { + if (InMemoryOverflowRejections.Enabled) + InMemoryOverflowRejections.Add(1); + } + + /// Records one in-job delayed publish the in-memory transport rejected at its delayed-job capacity. + internal static void RecordInMemoryDelayedRejection() + { + if (InMemoryDelayedRejections.Enabled) + InMemoryDelayedRejections.Add(1); + } + + /// + /// Registers a transport with the asyncresponse.worker.inmemory_overflow_depth and + /// asyncresponse.worker.inmemory_delayed_jobs gauges (created once, process-wide, on + /// first use). + /// + internal static void TrackInMemoryOverflow(InMemoryWorkerTransport transport) + { + lock (_inMemoryTransports) + { + _inMemoryTransports.RemoveAll(static reference => !reference.TryGetTarget(out _)); + _inMemoryTransports.Add(new WeakReference(transport)); + } + + if (Interlocked.Exchange(ref _inMemoryOverflowGaugeRegistered, 1) != 0) + return; + + Meter.CreateObservableGauge("asyncresponse.worker.inmemory_overflow_depth", + static () => SumOverInMemoryTransports(static transport => transport.OverflowDepth), unit: "{job}", + description: "Follow-up jobs the in-memory worker transport currently holds past QueueCapacity (summed over the process's transports); bounded by InJobOverflowCapacity."); + Meter.CreateObservableGauge("asyncresponse.worker.inmemory_delayed_jobs", + static () => SumOverInMemoryTransports(static transport => transport.DelayedJobsHeld), unit: "{job}", + description: "Delayed jobs the in-memory worker transport currently holds — waiting on their due time, or fired and waiting for queue room (summed over the process's transports); bounded by DelayedJobCapacity."); + } + + private static long SumOverInMemoryTransports(Func measure) + { + long total = 0; + lock (_inMemoryTransports) + { + _inMemoryTransports.RemoveAll(static reference => !reference.TryGetTarget(out _)); + foreach (var reference in _inMemoryTransports) + { + if (reference.TryGetTarget(out var transport)) + total += measure(transport); + } + } + + return total; + } + internal static Activity? StartActivity( string name, ActivityKind kind = ActivityKind.Internal, @@ -150,6 +235,18 @@ internal static void RecordLostSubscriber(string kind, RecoveryAction? action, b new KeyValuePair("invoked", callbackInvoked)); } + /// + /// Records one wait faulted as indeterminate because its bounded buffer overflowed: the channel + /// is fire-and-forget, the publisher was never backpressured, and admitting the next response + /// would have meant buffering without bound. Every occurrence is a saturated consumer worth + /// alerting on. + /// + internal static void RecordWaiterOverload(string channel) + { + if (OverloadedWaitsCounter.Enabled) + OverloadedWaitsCounter.Add(1, new KeyValuePair("channel", channel)); + } + /// Records one waiter timeout on the given channel kind. internal static void RecordWaiterTimeout(string channel) { @@ -195,6 +292,27 @@ internal static void RecordOversizedInboundMessage(string route) OversizedInboundCounter.Add(1, new KeyValuePair("route", route)); } + /// Records the rows one opportunistic durable-flow prune deleted (zero is not recorded). + internal static void RecordFlowStatePruned(string provider, long rows) + { + if (rows > 0 && FlowStatePrunedRows.Enabled) + FlowStatePrunedRows.Add(rows, new KeyValuePair("provider", provider)); + } + + /// Records one failed opportunistic durable-flow prune (the create it rode on succeeded). + internal static void RecordFlowStatePruneFailure(string provider) + { + if (FlowStatePruneFailures.Enabled) + FlowStatePruneFailures.Add(1, new KeyValuePair("provider", provider)); + } + + /// Records one prune that hit its budget with a full last batch — expired rows remain. + internal static void RecordFlowStatePruneBudgetExhausted(string provider) + { + if (FlowStatePruneBudgetExhausted.Enabled) + FlowStatePruneBudgetExhausted.Add(1, new KeyValuePair("provider", provider)); + } + /// /// Registers observable gauges reporting the latest watchdog scan: outstanding recovery /// registrations, those with a live waiter, and stale ones (no waiter, past the threshold). diff --git a/src/AsyncResponse.Core/AsyncResponseEnvelope.cs b/src/AsyncResponse.Core/AsyncResponseEnvelope.cs index bf0535e6c..818ed4eb5 100644 --- a/src/AsyncResponse.Core/AsyncResponseEnvelope.cs +++ b/src/AsyncResponse.Core/AsyncResponseEnvelope.cs @@ -129,7 +129,7 @@ internal sealed class AsyncResponseEnvelopeConverter : JsonConverter : JsonConverter : JsonConverter : JsonConverter : JsonConverter : JsonConverter { diff --git a/src/AsyncResponse.Core/AsyncResponseIngress.cs b/src/AsyncResponse.Core/AsyncResponseIngress.cs index bc28f6a0a..ba1a2fe8b 100644 --- a/src/AsyncResponse.Core/AsyncResponseIngress.cs +++ b/src/AsyncResponse.Core/AsyncResponseIngress.cs @@ -105,20 +105,25 @@ public async Task HandleResponseMessageAsync(string messageJson, string? correla // NAK/redeliver instead of terminally failing a waiter whose response was never lost. // Recovery resume callbacks may be re-invoked by these retries, which matches their // contract — broker redelivery re-invokes them the same way. + // + // RecoveryCallbackFailedException is excluded from both as well: the lost-subscriber + // dispatcher already ran its own ladder against the failure callback, and escalating + // through SetException would only invoke that same failing callback again. It + // propagates so the transport redelivers the still-unacknowledged terminal signal. await AsyncResponseRetry.ExecuteAsync( async _ => { await _rawPublisher.SetRawResponseJson(messageJson, correlationId).ConfigureAwait(false); return true; }, - isTransient: static ex => ex is not (System.Text.Json.JsonException or InvalidDataException or OperationCanceledException), + isTransient: static ex => ex is not (System.Text.Json.JsonException or InvalidDataException or OperationCanceledException or RecoveryCallbackFailedException), maxAttempts: 4, baseDelay: TimeSpan.FromMilliseconds(250), maxDelay: TimeSpan.FromSeconds(2), CancellationToken.None, _timeProvider).ConfigureAwait(false); } - catch (Exception ex) when (ex is not OperationCanceledException) + catch (Exception ex) when (ex is not (OperationCanceledException or RecoveryCallbackFailedException)) { _logger.LogError(ex, "Ingress failed to process the inbound response message."); AsyncResponseDiagnostics.SetError(activity, ex); diff --git a/src/AsyncResponse.Core/CallbackExpressionConverter.cs b/src/AsyncResponse.Core/CallbackExpressionConverter.cs index f1fb04807..21fb5a778 100644 --- a/src/AsyncResponse.Core/CallbackExpressionConverter.cs +++ b/src/AsyncResponse.Core/CallbackExpressionConverter.cs @@ -1,3 +1,4 @@ +using System.Diagnostics.CodeAnalysis; using System.Linq.Expressions; using System.Reflection; @@ -12,18 +13,18 @@ namespace AsyncResponse; internal static class CallbackExpressionConverter { /// Converts the callback expression to a reflection call descriptor. - public static ReflectionCallDto ToReflectionCall(Expression> expression) + public static ReflectionCallDto ToReflectionCall<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicMethods)] TService>(Expression> expression) => Build(expression.Body); /// Converts the callback expression to a reflection call descriptor. - public static ReflectionCallDto ToReflectionCall(Expression> expression) + public static ReflectionCallDto ToReflectionCall<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicMethods)] TService>(Expression> expression) => Build(expression.Body); /// Converts the callback expression to a reflection call descriptor. - public static ReflectionCallDto ToReflectionCall(Expression> expression) + public static ReflectionCallDto ToReflectionCall<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicMethods)] TService>(Expression> expression) => Build(expression.Body); - private static ReflectionCallDto Build(Expression body) + private static ReflectionCallDto Build<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicMethods)] TService>(Expression body) { if (body is not MethodCallExpression call) throw new NotSupportedException($"Only direct method calls are supported. Got: {body.NodeType}"); @@ -34,6 +35,15 @@ private static ReflectionCallDto Build(Expression body) var args = call.Arguments.Select(arg => ConvertArgument(arg, svcParam)).ToArray(); + // The compiler resolved `call.Method` from full signatures, but the descriptor persists only + // its NAME and ARITY — the wire contract every deployment reading it shares. Validate here, + // where the caller's stack is, that name + arity still select exactly this one method (no + // overload set sharing both, no by-ref or open-generic parameters). Without this an + // interface such as `Run(int)` / `Run(string)` accepted `svc => svc.Run(1)` and every + // dispatch of the job then failed as ambiguous — after publication, on a worker, burning + // the transport's retries or stranding a recovery registration. + ReflectionExtensions.EnsureBindable(typeof(TService), call.Method.Name, args.Length); + return new ReflectionCallDto { ServiceInterfaceFullName = typeof(TService).FullName diff --git a/src/AsyncResponse.Core/ChannelSerialExecutor.cs b/src/AsyncResponse.Core/ChannelSerialExecutor.cs index 93a34cf49..a85f145c3 100644 --- a/src/AsyncResponse.Core/ChannelSerialExecutor.cs +++ b/src/AsyncResponse.Core/ChannelSerialExecutor.cs @@ -128,8 +128,11 @@ private async Task EnqueueCoreAsync(Func work, CancellationToken can /// /// Synchronously queues a work delegate, returning false when the executor is already /// shutting down or full. Use when the producer can wait for capacity. + /// is false for producers that treat a full queue as + /// expected backpressure and come back later (the DB channels' dispatch sweep), so a busy + /// correlation id does not log a warning per sweep tick. /// - public bool TryEnqueue(Func work) + public bool TryEnqueue(Func work, bool logIfFull = true) { ArgumentNullException.ThrowIfNull(work); Interlocked.Increment(ref _pending); @@ -141,7 +144,10 @@ public bool TryEnqueue(Func work) } Interlocked.Decrement(ref _pending); - _logger.LogWarning("Channel executor could not enqueue work for {Channel}; queue is full or completed (pending {PendingCount}).", _channel, PendingCount); + if (logIfFull) + _logger.LogWarning("Channel executor could not enqueue work for {Channel}; queue is full or completed (pending {PendingCount}).", _channel, PendingCount); + else if (_logger.IsEnabled(LogLevel.Debug)) + _logger.LogDebug("Channel executor for {Channel} is at capacity (pending {PendingCount}); the producer will retry later.", _channel, PendingCount); return false; } diff --git a/src/AsyncResponse.Core/DurableFlowContext.cs b/src/AsyncResponse.Core/DurableFlowContext.cs index 2c293986e..e6df21c05 100644 --- a/src/AsyncResponse.Core/DurableFlowContext.cs +++ b/src/AsyncResponse.Core/DurableFlowContext.cs @@ -33,11 +33,19 @@ internal sealed class DurableFlowContext : IDurableFlowContext private bool _progressDirty; private DateTime _lastPersistenceUtc; + // The next ledger-size estimate (in chars) that logs the growth warning; long.MaxValue when + // the warning is disabled. Doubles after every warning so a long run logs O(log n) times. + private long _nextLedgerSizeWarningChars; + /// - /// How many ancestors a long park refreshes (see ); - /// far beyond any sane child-flow nesting, small enough to bound a corrupted parent cycle. + /// The deepest child-flow nesting a long park supports (see + /// ): every ancestor up to the root is refreshed, and a + /// chain longer than this fails the run terminally instead of being silently truncated — the + /// previous 16-level cap stopped walking with the root unrefreshed, so a leaf nested 17 deep + /// parked "successfully" while its root expired underneath it. Cycles are detected separately + /// (a visited set), so this bounds only the cost of a legitimately absurd nesting. /// - private const int MaxAncestorLedgerDepth = 16; + internal const int MaxAncestorLedgerDepth = 256; /// Creates the context for one execution of the given run. public DurableFlowContext( @@ -60,6 +68,7 @@ public DurableFlowContext( _builder = builder; _propagation = propagation; _options = options; + _nextLedgerSizeWarningChars = options.LedgerSizeWarningBytes ?? long.MaxValue; _subscriber = subscriber; _recoverableSubscriber = recoverableSubscriber; _logger = logger; @@ -346,88 +355,170 @@ private async Task SaveForSleepAsync(TimeSpan remaining, CancellationToken cance : remaining >= margin ? AsyncResponseChannelOptions.MaxPersistenceTtl : remaining + _options.StateExpiry; + + // The wait outlives this save's own TTL stamp only if every later write of this ledger + // carries it forward — a spurious early redelivery of the parked run stamps the plain + // StateExpiry in the executor's per-attempt save before it replays back here. The floor + // in the ledger is what those writes honor (FlowStateRetention.EffectiveTtl). + if (ttl > _options.StateExpiry) + FlowStateRetention.RaiseFloor(_state, UtcNow, ttl); await SaveAsync(cancellationToken, ttl: ttl).ConfigureAwait(false); // A parked ancestor's row must survive this run's whole wait, not just its own idle // margin: nothing refreshes an ancestor while it waits on this chain (lease renewal only // stamps the lease columns), so a descendant parking beyond the ancestor's StateExpiry // silently expired the ancestor and the eventual completion wake-up found no state. + // Part of the park, not insurance around it: a failure here propagates BEFORE any wake-up + // is published (every caller publishes after this save), so the delivery is retried from + // the checkpoint above instead of the run parking on an ancestor that will expire under it. if (ttl > _options.StateExpiry && _state.ParentFlowId is not null) await ExtendAncestorLedgersAsync(ttl, cancellationToken).ConfigureAwait(false); } /// - /// Best-effort TTL refresh of the ancestor chain when this run parks for a window its own - /// plain would not cover. Only - /// ancestors are stamped — a terminal or - /// operator-suspended run is not waiting on this chain, and an absent row is never - /// resurrected (the walk stops there and the expired-ancestor failure surfaces on wake-up, - /// as before). Failures log and return: this run's own checkpoint already latched, and - /// faulting the park over ancestor insurance would re-run the step for a write the next long - /// park retries anyway. + /// Retention extension of the WHOLE ancestor chain when this run parks for a window its own + /// plain would not cover. Each + /// ancestor gets its + /// floor raised to cover the wait and its row re-stamped with the wait's TTL — a terminal or + /// operator-suspended run is not waiting on this chain, and an absent row is never resurrected + /// (the walk stops there and the expired-ancestor failure surfaces on wake-up, as before). A + /// store failure PROPAGATES: the callers all publish their wake-up only after this returns, so + /// the park fails with nothing published and the transport redelivers the execution, which + /// replays to the same step and retries the chain. Swallowing it (an earlier behavior) let the + /// child park "successfully" — wake-up and all — while the parent it would eventually complete + /// into expired mid-wait, after which every step past the parent's child-await was lost with + /// the parent's checkpoints. The chain is walked to the root with cycle detection; a chain + /// that revisits an id or exceeds fails the run terminally + /// (deterministic on every replay) rather than being truncated in silence. + /// + /// A LOST compare-and-swap is not success. The previous design treated it as one — "a + /// concurrent writer means the ancestor is alive and re-stamping its own expiry" — but the + /// competing write was computed without this park in view: the parent replaying its + /// child-await from a snapshot taken before this run persisted its sleep stamps the plain + /// StateExpiry, and the executor's per-attempt save always does. Either one left the parent's + /// row expiring under a wait this run had just parked into, with its wake-up published. So the + /// ancestor is re-read after a lost race: when the write that won already carries a floor + /// reaching this park (another extension of the same chain, or an earlier attempt of this + /// one), the retention is proven and the walk moves on; otherwise the extension is retried + /// against the new revision, a bounded number of times. Every write here still advances the + /// ancestor's revision — it has to, the floor lives in the ledger — so a retry can cost an + /// actively-executing ancestor one checkpoint (its next save loses the compare-and-swap and + /// its delivery replays from the last one, now carrying the floor). That is the price of the + /// guarantee; the earlier eight-attempt fight + /// was avoided by ceding the race, and ceding it is what lost the parent. The attempt bound + /// keeps the fight finite: losing every attempt abandons the park (nothing published) so the + /// delivery retries it later, exactly like a store failure. + /// /// - /// SINGLE-SHOT per ancestor, deliberately NOT : - /// every write here advances the ancestor's Revision, which invalidates the - /// compare-and-swap of an ancestor execution that currently holds a lease. MutateAsync's - /// eight-attempt retry turned one such collision into a fight — each winning round killed - /// another of the live ancestor's checkpoints, abandoning its delivery for redelivery and - /// re-running everything since its last checkpoint, all for a write whose only purpose was a - /// TTL stamp. Losing the CAS is therefore treated as SUCCESS, not as something to retry: a - /// concurrent writer means the ancestor is alive and checkpointing, and every checkpoint - /// re-stamps its expiry — this insurance exists only for an ancestor that is parked and - /// therefore silent. The walk still continues upward, because a grandparent can be parked - /// behind an actively-executing parent. + /// Every write of the ancestor after this one carries the floor forward (see + /// ), so the extension has to land once, not win every race + /// from here to the wake-up. /// /// private async Task ExtendAncestorLedgersAsync(TimeSpan ttl, CancellationToken cancellationToken) { + var visited = new HashSet(StringComparer.Ordinal) { FlowId }; var ancestorId = _state.ParentFlowId; - for (var depth = 0; ancestorId is not null && depth < MaxAncestorLedgerDepth; depth++) + while (ancestorId is not null) { - try + if (!visited.Add(ancestorId)) { - var ancestor = await _store.LoadAsync(ancestorId, cancellationToken).ConfigureAwait(false); - if (ancestor is null) - { - _logger.LogWarning( - "Flow {FlowId} parked for {Ttl} but ancestor flow {AncestorFlowId} has no state (expired or deleted); its chain keeps the current expiry.", - FlowId, ttl, ancestorId); - return; - } - - if (ancestor.Status != FlowRunStatus.Running) - return; - - var expectedRevision = ancestor.Revision; - ancestor.Revision = checked(expectedRevision + 1); - ancestor.UpdatedAtUtc = UtcNow; - if (!await _store.TryUpdateAsync( - ancestorId, - ancestor, - expectedRevision, - ttl, - leaseId: null, - cancellationToken).ConfigureAwait(false)) - { - _logger.LogDebug( - "Flow {FlowId} skipped extending ancestor flow {AncestorFlowId}'s ledger TTL: a concurrent write won the revision, so the ancestor is live and re-stamping its own expiry.", - FlowId, ancestorId); - } + // Corrupted ledgers (ParentFlowId loops back into the chain). Deterministic on + // every replay, so terminal: parking would leave the run waiting on ancestors + // whose retention can never be established. + throw new DurableFlowFailedException( + $"Flow '{FlowId}' cannot park for {ttl}: its ancestor chain revisits flow '{ancestorId}' (a cycle in ParentFlowId), " + + "so the ledgers it would wait on cannot be kept alive. The stored ledgers are inconsistent; the run is failed rather than parked."); + } - ancestorId = ancestor.ParentFlowId; + if (visited.Count > MaxAncestorLedgerDepth + 1) + { + throw new DurableFlowFailedException( + $"Flow '{FlowId}' cannot park for {ttl}: it is nested more than {MaxAncestorLedgerDepth} child flows deep, and every ancestor's ledger " + + "must be kept alive for the wait. Flatten the nesting."); } - catch (OperationCanceledException) + + try { - throw; + ancestorId = await ExtendOneAncestorAsync(ancestorId, ttl, cancellationToken).ConfigureAwait(false); } - catch (Exception ex) + catch (Exception ex) when (ex is not OperationCanceledException) { + // Logged with the chain context, then rethrown AS IS: the park is abandoned with + // nothing published, the delivery retries, and the store's own exception type + // stays visible to whoever classifies it upstream. _logger.LogWarning( ex, - "Flow {FlowId} could not extend ancestor flow {AncestorFlowId}'s ledger TTL for its {Ttl} park; the ancestor keeps its current expiry.", + "Flow {FlowId} could not extend ancestor flow {AncestorFlowId}'s ledger retention for its {Ttl} park; abandoning the park (no wake-up is published) so the delivery retries it.", FlowId, ancestorId, ttl); - return; + throw; + } + } + } + + /// + /// How many times one ancestor's extension is retried against a revision a concurrent write + /// took. Each attempt re-reads the ancestor first, and a floor already reaching the park ends + /// the attempt without a write. + /// + internal const int MaxAncestorExtensionAttempts = 4; + + /// + /// Extends one ancestor (see ) and returns the id of + /// the next ancestor up, or null when the walk stops here: the row is gone, the run is + /// not , or it has no parent. + /// + private async Task ExtendOneAncestorAsync(string ancestorId, TimeSpan ttl, CancellationToken cancellationToken) + { + for (var attempt = 1; ; attempt++) + { + var ancestor = await _store.LoadAsync(ancestorId, cancellationToken).ConfigureAwait(false); + if (ancestor is null) + { + _logger.LogWarning( + "Flow {FlowId} parked for {Ttl} but ancestor flow {AncestorFlowId} has no state (expired or deleted); its chain keeps the current expiry.", + FlowId, ttl, ancestorId); + return null; + } + + if (ancestor.Status != FlowRunStatus.Running) + return null; + + var now = UtcNow; + var until = FlowStateRetention.FloorAt(now, ttl); + if (FlowStateRetention.Covers(ancestor, until)) + { + // Proven by the re-read: whoever wrote last carried a floor reaching this park + // (the write that beat a previous attempt, or a sibling park on the same chain). + return ancestor.ParentFlowId; } + + FlowStateRetention.RaiseFloor(ancestor, now, ttl); + var expectedRevision = ancestor.Revision; + ancestor.Revision = checked(expectedRevision + 1); + ancestor.UpdatedAtUtc = now; + if (await _store.TryUpdateAsync( + ancestorId, + ancestor, + expectedRevision, + FlowStateRetention.EffectiveTtl(ancestor, ttl, now), + leaseId: null, + cancellationToken).ConfigureAwait(false)) + return ancestor.ParentFlowId; + + if (attempt >= MaxAncestorExtensionAttempts) + { + // Not terminal: the ancestor is being written continuously right now, and the + // next replay of this step may find it quiet. The park is abandoned with nothing + // published, so the delivery retries it — the same route a store failure takes. + throw new InvalidOperationException( + $"Flow '{FlowId}' could not extend ancestor flow '{ancestorId}'s ledger retention for its {ttl} park: " + + $"a concurrent write advanced the ancestor's revision on each of {attempt} attempts. The park is abandoned so the delivery retries it."); + } + + _logger.LogDebug( + "Flow {FlowId} lost the revision race extending ancestor flow {AncestorFlowId}'s ledger retention (attempt {Attempt}); re-reading it.", + FlowId, ancestorId, attempt); } } @@ -598,15 +689,28 @@ public Task SetValueAsync(string key, TValue value, CancellationToken ca { // A terminal child snapshot is a settled outcome: memoize it uninterruptibly (local // and awaited-step parity) so a cancellation here cannot trip MarkLost on a healthy lease. + // The caller gets the SNAPSHOT — the reduced shape the memo holds (no ambient Context, + // nested child-step results elided) — on the first completion exactly as on every + // replay, which reads it back from the memo above. Returning the loaded child here + // handed the first execution a richer object than any re-execution would ever see, so + // parent logic could branch differently (or fail) after a restart on a step it had + // already completed; the whole point of the memo is that the two are indistinguishable. case FlowRunStatus.Succeeded: - await CompleteStepAsync(name, checkpoint, FlowStateJson.SerializeSnapshot(child), CancellationToken.None, kind: DurableFlowStepKind.ChildFlow).ConfigureAwait(false); - return child; + { + var snapshotJson = FlowStateJson.SerializeSnapshot(child); + await CompleteStepAsync(name, checkpoint, snapshotJson, CancellationToken.None, kind: DurableFlowStepKind.ChildFlow).ConfigureAwait(false); + return MaterializeChildSnapshot(name, snapshotJson); + } case FlowRunStatus.Failed: + { checkpoint.Message = child.LastMessage; - await CompleteStepAsync(name, checkpoint, FlowStateJson.SerializeSnapshot(child), CancellationToken.None, faulted: true, kind: DurableFlowStepKind.ChildFlow).ConfigureAwait(false); - ThrowIfChildFailed(child, failOnChildFailure); - return child; + var snapshotJson = FlowStateJson.SerializeSnapshot(child); + await CompleteStepAsync(name, checkpoint, snapshotJson, CancellationToken.None, faulted: true, kind: DurableFlowStepKind.ChildFlow).ConfigureAwait(false); + var snapshot = MaterializeChildSnapshot(name, snapshotJson); + ThrowIfChildFailed(snapshot, failOnChildFailure); + return snapshot; + } default: await NotifyStepAsync(static (o, e) => o.OnStepWaitingAsync(e), name, DurableFlowStepKind.ChildFlow).ConfigureAwait(false); @@ -1063,6 +1167,16 @@ private void ThrowIfSuspended() throw new DurableFlowSuspendedException(_state.LastMessage ?? $"Flow {FlowId} is suspended."); } + /// + /// Reads a just-memoized child snapshot back through the SAME deserializer the replay branch + /// uses, so the object handed to the first completion is bit-for-bit what every later + /// execution receives. + /// + private FlowState MaterializeChildSnapshot(string stepName, string snapshotJson) + => DeserializeResult(snapshotJson) + ?? throw new DurableFlowFailedException( + $"Completed child step '{stepName}' of flow '{FlowId}' has no child-state snapshot."); + private static void ThrowIfChildFailed(FlowState child, bool failOnChildFailure) { if (failOnChildFailure && child.Status == FlowRunStatus.Failed) @@ -1159,6 +1273,16 @@ private async Task CompleteStepAsync( /// than clobbering it. Best-effort by construction: on a conflict or an absent ledger the step /// simply restarts, which is the pre-existing behavior — but on the common path the response /// survives instead of being dropped. + /// + /// Fenced to THIS attempt, exactly as DurableFlowExecutor.RecoverAsync fences a recovered + /// payload: the reloaded step must still be pending on and the + /// run must still be checkpointable. Losing the lease means a takeover may already have run — + /// timed the breadcrumb out, re-triggered the step under a NEW correlation id, or failed the + /// run — and a write keyed only on "step name, not completed" would complete the newer + /// attempt's pending step with this attempt's stale response (revision CAS cannot catch it: + /// the mutation deliberately targets the freshly loaded revision). A stale response is + /// discarded with a warning; the newer attempt's own response is the one that counts. + /// /// private async Task CheckpointReceivedWithoutLeaseAsync( string name, @@ -1168,18 +1292,49 @@ private async Task CheckpointReceivedWithoutLeaseAsync( { var resultJson = AsyncResponseJson.Serialize(received); var completedAtUtc = UtcNow; + var applied = false; + string? skipReason = null; try { - await FlowStateConcurrency.MutateAsync( + var found = await FlowStateConcurrency.MutateAsync( _store, FlowId, _options.StateExpiry, _timeProvider, state => { - if (state.Steps is not { } steps || !steps.TryGetValue(name, out var current) || current.Completed) + applied = false; + skipReason = null; + + // Same eligibility as RecoverAsync: Suspended runs still take the checkpoint + // (an operator parked the run; the payload exists nowhere else and un-parking + // replays from it), terminal runs never do. + if (state.Status is not (FlowRunStatus.Running or FlowRunStatus.Suspended)) + { + skipReason = $"the run is {state.Status}"; + return false; + } + + if (state.Steps is not { } steps || !steps.TryGetValue(name, out var current)) + { + skipReason = "the step no longer exists in the ledger"; + return false; + } + + if (current.Completed) + { + skipReason = "the step is already completed"; return false; + } + + if (!string.Equals(current.PendingCorrelationId, correlationId, StringComparison.Ordinal)) + { + skipReason = current.PendingCorrelationId is null + ? "the step is no longer pending on any correlation id" + : "the step is pending on a newer correlation id (a takeover re-triggered it)"; + return false; + } current.Completed = true; current.ResultJson = resultJson; @@ -1188,15 +1343,28 @@ await FlowStateConcurrency.MutateAsync( current.Faulted = false; current.CompletedAtUtc = completedAtUtc; state.LastMessage = $"Step '{name}' completed (checkpointed after the execution lease was lost)."; + applied = true; return true; }, CancellationToken.None).ConfigureAwait(false); - _logger.LogWarning( - "Flow {FlowId} lost its execution lease while step '{Step}' held a claimed response for correlationId {CorrelationId}; the response was checkpointed without the lease so the takeover resumes from it.", - FlowId, - name, - correlationId); + if (applied) + { + _logger.LogWarning( + "Flow {FlowId} lost its execution lease while step '{Step}' held a claimed response for correlationId {CorrelationId}; the response was checkpointed without the lease so the takeover resumes from it.", + FlowId, + name, + correlationId); + } + else + { + _logger.LogWarning( + "Flow {FlowId} lost its execution lease while step '{Step}' held a claimed response for correlationId {CorrelationId}; the response was discarded because {Reason}.", + FlowId, + name, + correlationId, + found ? skipReason : "the ledger no longer exists"); + } } catch (Exception ex) { @@ -1209,6 +1377,9 @@ await FlowStateConcurrency.MutateAsync( name); } + if (!applied) + return; + step.Completed = true; step.ResultJson = resultJson; step.PendingCorrelationId = null; @@ -1226,6 +1397,35 @@ private async Task SaveAsync(CancellationToken cancellationToken, Exception? cau _progressDirty = false; _lastPersistenceUtc = UtcNow; + WarnIfLedgerLarge(); + } + + /// + /// Every checkpoint rewrites the whole ledger, so a run whose steps retain sizeable results + /// pays a persistence cost that grows with each completed step (about N²/2 step-results + /// serialized over a run of N similar steps) until it hits the store's hard cap. The + /// threshold turns that curve into an + /// early operator signal: one warning when it is first crossed, another at each doubling. + /// + private void WarnIfLedgerLarge() + { + if (_nextLedgerSizeWarningChars == long.MaxValue) + return; + + var estimate = FlowStateJson.EstimateLedgerChars(_state); + if (estimate < _nextLedgerSizeWarningChars) + return; + + _logger.LogWarning( + "Durable flow {FlowId} ledger is roughly {LedgerBytes} bytes over {StepCount} step(s), past the {Threshold}-byte LedgerSizeWarningBytes threshold. Every checkpoint rewrites the whole ledger, so persistence cost now grows with each completed step and the store's MaxStateBytes cap is the hard limit. Keep step results small (persist large data yourself and pass references) or partition a long history into child flows.", + FlowId, + estimate, + _state.Steps?.Count ?? 0, + _options.LedgerSizeWarningBytes); + + // Next warning at the next doubling of the CURRENT size (a single huge result may have + // skipped several thresholds at once), saturating instead of overflowing. + _nextLedgerSizeWarningChars = estimate > long.MaxValue / 2 ? long.MaxValue - 1 : estimate * 2; } private async Task WaitForResponseAsync(Task responseTask, CancellationToken cancellationToken) diff --git a/src/AsyncResponse.Core/DurableFlowExecutor.cs b/src/AsyncResponse.Core/DurableFlowExecutor.cs index 4100c241b..72e9e6770 100644 --- a/src/AsyncResponse.Core/DurableFlowExecutor.cs +++ b/src/AsyncResponse.Core/DurableFlowExecutor.cs @@ -23,6 +23,18 @@ public interface IDurableFlowExecutor /// Task ExecuteAsync(string flowId); + /// + /// Start target: the job publishes. Creates + /// the ledger from the serialized initial state the job carries when no ledger exists yet + /// (insert-if-absent), then runs . The publish of this job — not the + /// starter's own ledger write — is the start's commit point: a process that dies after the + /// publish leaves a job whose execution creates the run, never a committed ledger that nothing + /// will ever execute. An existing ledger for the same flow type and semantically identical + /// input is executed as an idempotent re-start; one bound to different work is logged and the + /// job dropped (the starter already reported the conflict to its caller). + /// + Task CreateAndExecuteAsync(string flowId, string initialStateJson); + /// /// Lost-subscriber resume target: re-enqueues on the worker /// transport (never runs the flow inline on a publisher's dispatch path). @@ -310,6 +322,63 @@ public async Task ExecuteAsync(string flowId) return null; } + /// + public async Task CreateAndExecuteAsync(string flowId, string initialStateJson) + { + ArgumentException.ThrowIfNullOrWhiteSpace(flowId); + ArgumentException.ThrowIfNullOrWhiteSpace(initialStateJson); + + // The carrier is the ledger wire format itself. A carrier this build cannot read is + // deterministic: FlowStateUnreadableException propagates to the transport's retry and + // dead-letter policy, which is the alarm — the same treatment an unreadable stored ledger + // gets in ExecuteAsync, and for the same reason (acknowledging it would lose the start). + var initial = FlowStateJson.Deserialize(initialStateJson, flowId); + if (!string.Equals(initial.FlowId, flowId, StringComparison.Ordinal)) + { + throw new InvalidOperationException( + $"The start job for durable flow '{flowId}' carries initial state for '{initial.FlowId}'; refusing to create a ledger under the wrong id."); + } + + await using (var scope = _scopeFactory.CreateAsyncScope()) + { + var store = scope.ServiceProvider.GetRequiredService(); + if (await FlowStateConcurrency.TryCreateAsync(store, flowId, initial, _options.StateExpiry).ConfigureAwait(false)) + { + // The starter died (or has not got there yet) between its publish and its own + // create: the job is the durable record of the start, so the ledger comes from it. + _logger.LogInformation("Durable flow {FlowId} ({FlowType}) ledger created from its start job.", flowId, initial.FlowTypeName); + } + else + { + var existing = await store.LoadAsync(flowId).ConfigureAwait(false); + if (existing is null) + { + // Created and already expired or pruned between the two calls: genuinely gone, + // the one case where acknowledging the start is right (ExecuteAsync's rule). + _logger.LogWarning("Durable flow {FlowId} exists but its ledger is expired or gone; nothing to execute.", flowId); + return; + } + + if (!FlowStateConcurrency.IsSameStart(existing, initial.FlowTypeName, initial.InputTypeName, initial.InputJson)) + { + // The id was reused for different work. The starter that published this job + // saw the same conflict on its own create and threw DurableFlowIdConflictException + // to its caller; executing the EXISTING run here would wake a flow nobody asked + // to wake, and creating a second one is impossible. Drop the job, loudly. + _logger.LogError( + "Durable flow {FlowId} start job dropped: the id is already bound to flow type {ExistingFlowType} with different input, not {RequestedFlowType}. Idempotent retries must use the same flow type, input type, and semantically identical input.", + flowId, existing.FlowTypeName, initial.FlowTypeName); + return; + } + + // Same start, ledger already there (the starter's own create won, or this is a + // redelivery / an idempotent re-start of a live run): fall through and execute it. + } + } + + await ExecuteAsync(flowId).ConfigureAwait(false); + } + /// public async Task ResumeAsync(string flowId) { diff --git a/src/AsyncResponse.Core/DurableFlowOptions.cs b/src/AsyncResponse.Core/DurableFlowOptions.cs index 270b002d5..76983a20d 100644 --- a/src/AsyncResponse.Core/DurableFlowOptions.cs +++ b/src/AsyncResponse.Core/DurableFlowOptions.cs @@ -81,6 +81,21 @@ public class DurableFlowOptions /// public TimeSpan TimerInProcessThreshold { get; set; } = TimeSpan.FromSeconds(10); + /// + /// Ledger size, in bytes (estimated from the serialized input, step results, values, and + /// context), past which the executor logs a warning naming the flow — once when the threshold + /// is first crossed and again at each doubling, so a long run logs a handful of times, not once + /// per step. Every checkpoint rewrites the whole ledger, so persistence cost grows + /// with each completed step: a run of N steps with similar result sizes serializes about N²/2 + /// step-results over its lifetime, and the store's MaxStateBytes (or the provider's + /// item cap) is the hard limit. The warning is the early signal to keep step results small + /// (persist large data yourself and pass references) or to partition a long history into child + /// flows. null disables it. Default: 512 KiB — under the smallest bundled hard cap + /// (DynamoDB's 350 KB item, whose store defaults MaxStateBytes to 350 000) users should + /// lower it accordingly. + /// + public long? LedgerSizeWarningBytes { get; set; } = 512 * 1024; + /// /// Accepts the risk of running the worker subscriber in early ACK (AckAfterEnqueue) /// while durable flows are registered, suppressing the startup error. Durable-flow wake-ups diff --git a/src/AsyncResponse.Core/DurableFlows.cs b/src/AsyncResponse.Core/DurableFlows.cs index 83bdaf7fe..62550a14d 100644 --- a/src/AsyncResponse.Core/DurableFlows.cs +++ b/src/AsyncResponse.Core/DurableFlows.cs @@ -46,6 +46,10 @@ public DurableFlowService( else ArgumentException.ThrowIfNullOrWhiteSpace(flowId); + // Every id is validated BEFORE anything is published: the publish below is the start's + // commit point, and a job for an id every store would reject must never leave the process. + FlowStateConcurrency.EnsurePortableFlowId(flowId); + await using var scope = _scopeFactory.CreateAsyncScope(); var store = scope.ServiceProvider.GetRequiredService(); @@ -61,69 +65,132 @@ public DurableFlowService( LastMessage = "Flow started.", CreatedAtUtc = now, UpdatedAtUtc = now, + Revision = 0, Context = _propagation.Capture() }; - if (await FlowStateConcurrency.TryCreateAsync( + // PUBLISH FIRST, then create. The worker job carries the whole initial ledger, and + // IDurableFlowExecutor.CreateAndExecuteAsync creates the ledger itself (insert-if-absent) + // before executing — so the publish is the single durable commit point of a start: + // - a crash before the publish leaves nothing behind (the caller sees a fault and retries); + // - a crash after the publish leaves a job whose execution creates and runs the flow. + // The previous order (create, then publish) had an unrecoverable gap: a process dying + // between the two left a committed Running ledger with Attempts = 0 that nothing would + // ever execute, and IFlowStateStore has no enumeration for a reconciler to go find it. + // The publish still runs the retry ladder the ingress uses, and a publish that fails for + // good surfaces the id (DurableFlowNotDispatchedException) — now with nothing persisted. + var id = flowId; + var initialStateJson = FlowStateJson.Serialize(state); + await PublishStartAsync( + executor => executor.CreateAndExecuteAsync(id, initialStateJson), + id, + cancellationToken).ConfigureAwait(false); + + // The starter's own create keeps the caller-facing contract: the ledger exists by the time + // StartAsync returns (GetStateAsync / ResumeAsync right after a start see it), and a + // conflicting reuse of an explicit id is reported to THIS caller. Losing the create race — + // to the executor that already picked the job up, or to a concurrent identical start — is + // the expected shape, not an error. A store fault here no longer matters for the run: the + // job is published and the executor creates the ledger; the caller gets the id. + bool created; + try + { + created = await FlowStateConcurrency.TryCreateAsync( store, flowId, state, _options.StateExpiry, - cancellationToken).ConfigureAwait(false)) + cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OperationCanceledException || !cancellationToken.IsCancellationRequested) { - _logger.LogInformation("Started durable flow {FlowId} ({FlowType}).", flowId, typeof(TFlow).Name); + _logger.LogWarning( + ex, + "Durable flow {FlowId} start job is published but the starter could not write the ledger; the executor creates it when the job is picked up.", + flowId); + return flowId; } - else + + if (created) { - var existing = await store.LoadAsync(flowId, cancellationToken).ConfigureAwait(false) - ?? throw new InvalidOperationException( - $"Durable flow '{flowId}' already exists but its ledger is expired or unreadable."); - EnsureIdempotentStart(existing, inputJson, flowId); + _logger.LogInformation("Started durable flow {FlowId} ({FlowType}).", flowId, typeof(TFlow).Name); + return flowId; + } - // A semantically identical retry re-enqueues the existing run; completed steps skip. - _logger.LogInformation("Durable flow {FlowId} already exists; re-enqueueing instead of creating a duplicate.", flowId); + var existing = await store.LoadAsync(flowId, cancellationToken).ConfigureAwait(false); + if (existing is null) + { + // Lost the create to a ledger that has since expired: the published job's create wins + // the next time round. Nothing for the caller to do. + _logger.LogWarning("Durable flow {FlowId} start job is published; the existing ledger is expired and the executor re-creates it.", flowId); + return flowId; } - // The ledger is committed; from here the run EXISTS and is Running. If the wake-up never - // gets published, nothing in the system will ever execute it — IFlowStateStore has no - // enumeration, so no reconciler can go find it either. Retry the publish through the same - // ladder the ingress uses, and if it still fails, surface the flow id rather than the bare - // transport fault: with the id, a caller can re-drive the start idempotently; without it - // (the generated-id case) the run is simply lost. - var id = flowId; + // Throws DurableFlowIdConflictException for different work; the executor drops the + // already-published job on the same test. + EnsureIdempotentStart(existing, inputJson, flowId); + + // A semantically identical retry: the published job re-enqueues the existing run + // (completed steps skip) instead of creating a duplicate. + _logger.LogInformation("Durable flow {FlowId} already exists; the start job re-enqueues the existing run instead of creating a duplicate.", flowId); + return flowId; + } + + /// + /// Publishes a start job through the ingress's retry ladder. A publish that still fails + /// surfaces as carrying the id: nothing was + /// persisted, so the caller simply retries the start (idempotent with the same id). + /// + private async Task PublishStartAsync( + System.Linq.Expressions.Expression> job, + string flowId, + CancellationToken cancellationToken) + { try { await AsyncResponseRetry.ExecuteAsync( async token => { - await _builder.EnqueueWorkerAsync( - executor => executor.ExecuteAsync(id), - token).ConfigureAwait(false); + await _builder.EnqueueWorkerAsync(job, token).ConfigureAwait(false); return true; }, // Only the CALLER's cancellation ends the ladder. An OperationCanceledException // whose token is not the caller's is a transport or SDK timeout — brokers surface // those as TaskCanceledException all the time — and that is exactly the transient // shape this retry exists for. Excluding the whole exception type meant the most - // common recoverable publish failure got zero retries and went straight to an - // orphaned Running ledger. - isTransient: ex => ex is not OperationCanceledException || !cancellationToken.IsCancellationRequested, + // common recoverable publish failure got zero retries. An envelope over the + // ingress's size budget is deterministic (the same input serializes to the same + // length): no attempt can succeed, so it is not retried either. + isTransient: ex => ex is not WorkerJobTooLargeException + && (ex is not OperationCanceledException || !cancellationToken.IsCancellationRequested), maxAttempts: 4, baseDelay: TimeSpan.FromMilliseconds(250), maxDelay: TimeSpan.FromSeconds(2), cancellationToken, _timeProvider).ConfigureAwait(false); } + catch (WorkerJobTooLargeException ex) + { + // Not a dispatch failure to retry: the start job carries the initial ledger, and this + // input serializes past what the consuming ingress accepts — it would be acknowledged + // there without ever executing. Surfaced as itself (nothing was persisted) so the + // caller can shrink the input or move it behind a claim check. + _logger.LogError( + ex, + "Durable flow {FlowId} could not be started: its start job ({SerializedLength} UTF-16 code units) exceeds the ingress budget of {Limit}. Nothing was persisted.", + flowId, + ex.SerializedLength, + ex.Limit); + throw; + } catch (Exception ex) { _logger.LogError( ex, - "Durable flow {FlowId} was persisted but its worker job could not be published; the run exists with no wake-up. Retry the start with this id to re-enqueue it.", - id); - throw new DurableFlowNotDispatchedException(id, ex); + "Durable flow {FlowId} could not be started: its worker job was not published after retries. Nothing was persisted; retry the start (idempotent with this id).", + flowId); + throw new DurableFlowNotDispatchedException(flowId, ex); } - - return flowId; } /// @@ -164,9 +231,7 @@ private static void EnsureIdempotentStart( string requestedInputJson, string flowId) { - var sameFlowType = string.Equals(existing.FlowTypeName, typeof(TFlow).FullName, StringComparison.Ordinal); - var sameInputType = string.Equals(existing.InputTypeName, typeof(TInput).FullName, StringComparison.Ordinal); - if (sameFlowType && sameInputType && FlowStateJson.JsonEquivalent(existing.InputJson, requestedInputJson)) + if (FlowStateConcurrency.IsSameStart(existing, typeof(TFlow).FullName, typeof(TInput).FullName, requestedInputJson)) return; throw new DurableFlowIdConflictException( diff --git a/src/AsyncResponse.Core/FlowStateConcurrency.cs b/src/AsyncResponse.Core/FlowStateConcurrency.cs index 1d0f56750..b6cfea9fe 100644 --- a/src/AsyncResponse.Core/FlowStateConcurrency.cs +++ b/src/AsyncResponse.Core/FlowStateConcurrency.cs @@ -14,13 +14,36 @@ public static Task TryCreateAsync( TimeSpan ttl, CancellationToken cancellationToken = default) { - if (FlowIdNotPortable(flowId) is { } rejection) - throw new ArgumentException(rejection, nameof(flowId)); + EnsurePortableFlowId(flowId); state.Revision = 0; return store.TryCreateAsync(flowId, state, ttl, cancellationToken); } + /// + /// Throws for an id that fails . + /// Called by every create, and by IDurableFlows.StartAsync BEFORE it publishes the start + /// job — the publish is the start's commit point, so a job for an id no store would accept must + /// never leave the process. + /// + internal static void EnsurePortableFlowId(string flowId) + { + if (FlowIdNotPortable(flowId) is { } rejection) + throw new ArgumentException(rejection, nameof(flowId)); + } + + /// + /// Whether an existing ledger describes the same start as the requested one: same flow type, + /// same input type (both ordinal), and semantically identical input JSON. The one idempotency + /// test for flow ids, shared by the starter (which reports a mismatch to its caller as + /// ) and the executor's start target (which drops + /// the job on a mismatch) so the two can never disagree about what "the same run" means. + /// + internal static bool IsSameStart(FlowState existing, string? flowTypeName, string? inputTypeName, string? inputJson) + => string.Equals(existing.FlowTypeName, flowTypeName, StringComparison.Ordinal) + && string.Equals(existing.InputTypeName, inputTypeName, StringComparison.Ordinal) + && FlowStateJson.JsonEquivalent(existing.InputJson, inputJson ?? string.Empty); + /// /// Enforces the portable flow-id contract on every final id at creation — the single door all /// creates walk through. Three independent limits, because the stores disagree about what an @@ -153,12 +176,15 @@ public static async Task MutateAsync( var expectedRevision = state.Revision; state.Revision = checked(expectedRevision + 1); - state.UpdatedAtUtc = (timeProvider ?? TimeProvider.System).GetUtcNow().UtcDateTime; + var nowUtc = (timeProvider ?? TimeProvider.System).GetUtcNow().UtcDateTime; + state.UpdatedAtUtc = nowUtc; if (await store.TryUpdateAsync( flowId, state, expectedRevision, - ttl, + // A lease-bypassing write (recovery, failure signal, operator) never shrinks a + // live run's ledger under a park it knows nothing about. + FlowStateRetention.EffectiveTtl(state, ttl, nowUtc), leaseId: null, cancellationToken).ConfigureAwait(false)) return true; @@ -183,6 +209,11 @@ internal static void ValidateOptions(DurableFlowOptions options) AsyncResponseChannelOptions.EnsureTimerBacked(defaultStepTimeout, nameof(DurableFlowOptions), nameof(options.DefaultStepTimeout)); AsyncResponseChannelOptions.EnsurePersistedTtl(options.ExecutionLeaseDuration, nameof(DurableFlowOptions), nameof(options.ExecutionLeaseDuration)); AsyncResponseChannelOptions.EnsureTimerBacked(options.ExecutionLeaseRenewInterval, nameof(DurableFlowOptions), nameof(options.ExecutionLeaseRenewInterval)); + if (options.LedgerSizeWarningBytes is { } ledgerWarning && ledgerWarning <= 0) + { + throw new InvalidOperationException( + $"{nameof(DurableFlowOptions)}.{nameof(options.LedgerSizeWarningBytes)} must be positive, or null to disable the warning (got {ledgerWarning})."); + } if (options.ExecutionLeaseRenewInterval >= options.ExecutionLeaseDuration) { throw new InvalidOperationException( @@ -233,6 +264,16 @@ internal sealed class FlowExecutionLease : IAsyncDisposable /// private static readonly TimeSpan DisposeJoinLimit = TimeSpan.FromSeconds(30); + /// + /// Budget for the final lease release on disposal. The release is one conditional write, so + /// ten seconds is generous; past it the call is abandoned (cancelled, its outcome observed) + /// and the server-side lease expires on its own — the same recovery the abandoned renewal + /// loops rely on. Separate from because the two hang for + /// different reasons: the loops are joined first and are usually idle, while the release is + /// a fresh store call that a wedged connection can hold indefinitely even after a clean join. + /// + private static readonly TimeSpan ReleaseLimit = TimeSpan.FromSeconds(10); + /// The flow state store the lease was acquired through. /// The flow the lease protects. /// The identity of this lease within the flow's row. @@ -313,7 +354,8 @@ public async Task SaveAsync(FlowState state, TimeSpan ttl, CancellationToken can ThrowIfLost(cause); var expectedRevision = state.Revision; state.Revision = checked(expectedRevision + 1); - state.UpdatedAtUtc = _timeProvider.GetUtcNow().UtcDateTime; + var nowUtc = _timeProvider.GetUtcNow().UtcDateTime; + state.UpdatedAtUtc = nowUtc; try { @@ -321,7 +363,11 @@ public async Task SaveAsync(FlowState state, TimeSpan ttl, CancellationToken can _flowId, state, expectedRevision, - ttl, + // Every checkpoint carries the ledger's retention floor forward (see + // FlowStateRetention): the executor's per-attempt save and an ancestor's + // re-park stamp the plain StateExpiry, and used to shrink a ledger a + // descendant had extended for a wait still in progress. + FlowStateRetention.EffectiveTtl(state, ttl, nowUtc), _leaseId, cancellationToken).ConfigureAwait(false)) return; @@ -499,16 +545,71 @@ public async ValueTask DisposeAsync() return; } + // Bounded release (see ReleaseLimit), with a token the store can honor. An unbounded, + // uncancelable release kept a FINISHED execution's disposal — and with it the executor's + // `await using`, the job's DI scope, the worker slot, and the transport acknowledgement — + // pending for as long as a wedged store took to answer, which can be forever. + var releaseCancellation = new CancellationTokenSource(); + Task? release = null; try { - await _store.ReleaseLeaseAsync(_flowId, _leaseId, CancellationToken.None).ConfigureAwait(false); + release = _store.ReleaseLeaseAsync(_flowId, _leaseId, releaseCancellation.Token); + await release.WaitAsync(ReleaseLimit, _timeProvider).ConfigureAwait(false); + releaseCancellation.Dispose(); + } + catch (TimeoutException) when (release is { IsCompleted: false }) + { + // The budget lapsed with the store still silent (a TimeoutException thrown BY the + // store completes the task first and takes the branch below). Cancel what can be + // cancelled, observe whatever the abandoned call eventually does, and move on: the + // server-side lease expires on its own, exactly as when the renewal loops are abandoned. + releaseCancellation.Cancel(); + _logger.LogWarning( + "Durable flow {FlowId} execution lease release did not complete within {ReleaseLimit}; abandoning it (the lease will expire server-side).", + _flowId, + ReleaseLimit); + ObserveAbandonedRelease(release, releaseCancellation); } catch (Exception ex) { + releaseCancellation.Dispose(); _logger.LogWarning(ex, "Failed to release durable flow {FlowId} execution lease; it will expire.", _flowId); } _stop.Dispose(); _lost.Dispose(); } + + /// + /// Attaches the one continuation an abandoned release needs: its eventual fault is observed + /// (and logged, so a store that finally answers with an error is not an unobserved-task + /// event) and the cancellation source it still holds is disposed only once it can no longer + /// be touched. + /// + private void ObserveAbandonedRelease(Task release, CancellationTokenSource releaseCancellation) + => _ = release.ContinueWith( + (task, state) => + { + var (lease, cancellation) = ((FlowExecutionLease, CancellationTokenSource))state!; + if (task.IsFaulted) + { + lease._logger.LogWarning( + task.Exception?.GetBaseException(), + "The abandoned release of durable flow {FlowId}'s execution lease eventually failed; the lease expires server-side.", + lease._flowId); + } + else + { + lease._logger.LogDebug( + "The abandoned release of durable flow {FlowId}'s execution lease eventually completed ({Status}).", + lease._flowId, + task.Status); + } + + cancellation.Dispose(); + }, + (this, releaseCancellation), + CancellationToken.None, + TaskContinuationOptions.ExecuteSynchronously, + TaskScheduler.Default); } diff --git a/src/AsyncResponse.Core/FlowStateJson.cs b/src/AsyncResponse.Core/FlowStateJson.cs index d10d770ba..916783b67 100644 --- a/src/AsyncResponse.Core/FlowStateJson.cs +++ b/src/AsyncResponse.Core/FlowStateJson.cs @@ -25,9 +25,15 @@ public static FlowState Deserialize(string json, string flowId) FlowState? state; try { - state = JsonSerializer.Deserialize(json, TypeInfo); + // Body-free failure contract (see JsonSafety): the reader's own message appends + // `Path: $.` built from the property names and dictionary keys it was reading — + // a ledger's Values or Context keys, or whatever a start job's carrier holds — and this + // exception is chained into FlowStateUnreadableException, which the worker ingress + // logs in full. Only the size and position are carried across; the raw reader + // exception is dropped, not chained. + state = JsonSafety.SafeDeserialize(json, TypeInfo); } - catch (JsonException ex) + catch (InvalidDataException ex) { throw new FlowStateUnreadableException(flowId, "the stored JSON is malformed", ex); } @@ -45,6 +51,46 @@ public static FlowState Deserialize(string json, string flowId) return state; } + /// + /// A cheap lower-bound estimate of the serialized ledger size in UTF-16 code units: the sum of + /// every string the ledger carries (input, messages, step results, values, context). O(steps + + /// values) string-length reads, against a serialization that is O(bytes) — used to decide + /// whether to warn about ledger growth without paying a second serialization per checkpoint. + /// JSON escaping and property names only add to the real size, so "over the threshold" here + /// is never a false positive. + /// + public static long EstimateLedgerChars(FlowState state) + { + long size = (state.InputJson?.Length ?? 0) + (state.LastMessage?.Length ?? 0); + + if (state.Steps is { } steps) + { + foreach (var (name, step) in steps) + { + size += name.Length + + (step.ResultJson?.Length ?? 0) + + (step.Message?.Length ?? 0) + + (step.PendingCorrelationId?.Length ?? 0) + + (step.PendingPayloadTypeFullName?.Length ?? 0) + + (step.ChildFlowId?.Length ?? 0); + } + } + + if (state.Values is { } values) + { + foreach (var (key, value) in values) + size += key.Length + (value?.Length ?? 0); + } + + if (state.Context is { } context) + { + foreach (var (key, value) in context) + size += key.Length + (value?.Length ?? 0); + } + + return size; + } + public static bool JsonEquivalent(string? left, string right) { if (string.Equals(left, right, StringComparison.Ordinal)) @@ -64,14 +110,37 @@ public static bool JsonEquivalent(string? left, string right) /// /// Serializes a child for memoization as a parent step result, without - /// the captured ambient : it is propagation machinery (it can - /// carry principal/tenant values) that the parent never needs, and dropping it keeps nested - /// child snapshots from compounding ledger size. + /// the captured ambient (propagation machinery — it can carry + /// principal/tenant values — that the parent never needs) and without the child's OWN + /// memoized child snapshots: a step whose is set has + /// its elided (the id, completion, and fault marker + /// stay). The snapshot is stored as a JSON string inside the parent's ledger, so every + /// ancestor level re-escapes the level below it; carrying grandchild snapshots along made the + /// ledger grow exponentially with nesting depth (a 72-byte leaf became ~77 KB at depth 12 and + /// ~600 KB at depth 15 — past DynamoDB's item cap — with no business payload at all). Eliding + /// them makes a memoized snapshot depth-independent: a parent holds its direct children's + /// outcomes and local step results; a grandchild's own snapshot lives in the grandchild's + /// ledger, reachable by the elided step's ChildFlowId while that ledger lives. + /// The instance handed in is restored before returning. /// public static string SerializeSnapshot(FlowState state) { var context = state.Context; state.Context = null; + + List<(FlowStepState Step, string ResultJson)>? elided = null; + if (state.Steps is { } steps) + { + foreach (var step in steps.Values) + { + if (step.ChildFlowId is null || step.ResultJson is null) + continue; + + (elided ??= []).Add((step, step.ResultJson)); + step.ResultJson = null; + } + } + try { return Serialize(state); @@ -79,6 +148,11 @@ public static string SerializeSnapshot(FlowState state) finally { state.Context = context; + if (elided is not null) + { + foreach (var (step, resultJson) in elided) + step.ResultJson = resultJson; + } } } } diff --git a/src/AsyncResponse.Core/FlowStateRetention.cs b/src/AsyncResponse.Core/FlowStateRetention.cs new file mode 100644 index 000000000..0ac994076 --- /dev/null +++ b/src/AsyncResponse.Core/FlowStateRetention.cs @@ -0,0 +1,71 @@ +namespace AsyncResponse; + +/// +/// The ledger retention floor () at the write sites. +/// +/// A park longer than the ordinary idle StateExpiry — a timer sleep, an awaited step's +/// window, a child flow's own park — needs the parked run's ledger AND every ancestor waiting on +/// it to outlive the wait. The TTL stamped by the park's own save covers that only until the +/// next write: every store recomputes expiry as "now + ttl", and the writers that can race a +/// park (an ancestor's replay re-parking on a stale child snapshot, an executor's per-attempt +/// save, a recovery or operator mutation) know nothing about the wait and stamp the plain +/// StateExpiry. The floor rides in the ledger itself, so whoever writes the ledger next +/// carries it forward: the TTL a write stamps is raised to reach the floor. That is what makes +/// a descendant's extension of an ancestor durable across the ancestor's own checkpoints, and +/// what lets the extension prove — by re-reading the ancestor — that a concurrent write which +/// beat its compare-and-swap left adequate retention behind. +/// +/// +/// Terminal runs ignore the floor: they have no wait in progress, and a failed run should not +/// be retained for the length of the sleep it never finished. +/// +/// +internal static class FlowStateRetention +{ + /// + /// The TTL a write must stamp for : , + /// raised to reach the state's retention floor when the run is live and the floor is further + /// out. Saturated at the persistence ceiling (clock skew between replicas could otherwise push + /// a floor stamped elsewhere a hair past it). + /// + public static TimeSpan EffectiveTtl(FlowState state, TimeSpan requested, DateTime nowUtc) + { + if (state.RetainUntilUtc is not { } floor || IsTerminal(state.Status)) + return requested; + + var needed = floor - nowUtc; + if (needed <= requested) + return requested; + + return needed > AsyncResponseChannelOptions.MaxPersistenceTtl + ? AsyncResponseChannelOptions.MaxPersistenceTtl + : needed; + } + + /// + /// Raises the floor of to + + /// when that is further out than the current one (never lowers it) and + /// returns the instant the floor now sits at. + /// + public static DateTime RaiseFloor(FlowState state, DateTime nowUtc, TimeSpan ttl) + { + var until = FloorAt(nowUtc, ttl); + if (state.RetainUntilUtc is not { } floor || until > floor) + state.RetainUntilUtc = until; + + return state.RetainUntilUtc!.Value; + } + + /// The instant a floor stamped now for sits at (saturating). + public static DateTime FloorAt(DateTime nowUtc, TimeSpan ttl) => AddSaturating(nowUtc, ttl); + + /// Whether the floor of already reaches . + public static bool Covers(FlowState state, DateTime until) + => state.RetainUntilUtc is { } floor && floor >= until; + + private static bool IsTerminal(FlowRunStatus status) + => status is FlowRunStatus.Succeeded or FlowRunStatus.Failed; + + private static DateTime AddSaturating(DateTime instant, TimeSpan ttl) + => ttl > DateTime.MaxValue - instant ? DateTime.MaxValue : instant + ttl; +} diff --git a/src/AsyncResponse.Core/InMemoryAsyncResponseChannel.cs b/src/AsyncResponse.Core/InMemoryAsyncResponseChannel.cs index 288be61a7..802a66bf8 100644 --- a/src/AsyncResponse.Core/InMemoryAsyncResponseChannel.cs +++ b/src/AsyncResponse.Core/InMemoryAsyncResponseChannel.cs @@ -1225,11 +1225,18 @@ private Task DispatchPayloadAsync(T payload) // deserializes its own instance case-insensitively — the same property matching the // string conversion path and every broker ingress apply. JsonElement/string/null payloads // keep the existing conversion path. + // + // Through JsonSafety, like every other reader of a body the waiter did not write: a + // publisher's payload that does not fit the waiter's type (a string-valued dictionary + // published to an int-valued waiter) fails INSIDE the payload, and the reader's own + // JsonException names the offending key ("Path: $.Values['']"). That message + // reached the waiter's task and, through SetError, the wait activity's status — the + // in-process exception to the body-free rule the broker channels enforce. private static T MaterializeAs(object? response, byte[]? wireBytes) { var payload = wireBytes is null ? response.As() - : AsyncResponseJson.DeserializeCaseInsensitive(wireBytes); + : JsonSafety.SafeDeserialize(wireBytes, AsyncResponseJson.GetTypeInfo(AsyncResponseJson.CaseInsensitive)); // A null (a published null object, a JSON-null JsonElement, a "null" string body) // must fault the waiter, never complete it — the broker channels reject the same diff --git a/src/AsyncResponse.Core/InMemoryFlowStateStore.cs b/src/AsyncResponse.Core/InMemoryFlowStateStore.cs index 3cd176768..503e8e3bc 100644 --- a/src/AsyncResponse.Core/InMemoryFlowStateStore.cs +++ b/src/AsyncResponse.Core/InMemoryFlowStateStore.cs @@ -11,8 +11,20 @@ internal sealed class InMemoryFlowStateStore : IFlowStateStore private static DateTime Expiry(DateTime now, TimeSpan ttl) => ttl >= DateTime.MaxValue - now ? DateTime.MaxValue : now.Add(ttl); + /// + /// How often sweeps expired entries whose ids are never touched + /// again. Expiry used to be enforced only on access to the SAME id (a load or a replacement), + /// which a completed run normally never gets — so a long-lived process minting unique flow + /// ids retained every expired ledger (inputs, memoized results, value bags) for its lifetime; + /// StateExpiry hid them from reads without ever bounding memory. On the engine's clock, + /// like every other stamp here: a virtual clock that never advances never sweeps, and + /// nothing has expired under it either. + /// + internal static readonly TimeSpan SweepInterval = TimeSpan.FromMinutes(1); + private readonly ConcurrentDictionary _entries = new(StringComparer.Ordinal); private readonly TimeProvider _timeProvider; + private long _nextSweepTicks; /// Creates the store; expiry and lease stamps come from the engine's clock. public InMemoryFlowStateStore(TimeProvider? timeProvider = null) @@ -30,6 +42,7 @@ public Task TryCreateAsync( throw new ArgumentException("A new flow ledger must start at revision zero.", nameof(state)); var now = _timeProvider.GetUtcNow().UtcDateTime; + SweepExpired(now); var created = CreateEntry(state, Expiry(now, ttl)); while (true) { @@ -60,15 +73,22 @@ public Task TryCreateAsync( continue; } - // Unreadable JSON or an unknown schema version throws out of here rather than - // masquerading as a deleted flow; revision/identity mismatch still reads as absent. - // Same contract as the durable stores — see DurableFlowStoreShared.ReadState. + // Unreadable JSON, an unknown schema version, and an entry whose JSON disagrees with + // its own revision or key all throw out of here rather than masquerading as a deleted + // flow: the entry is present, so acknowledging its wake-up as "gone" would strand the + // run. Same contract as the durable stores — see DurableFlowStoreShared.ReadState. var state = FlowStateJson.Deserialize(entry.StateJson, flowId); - return Task.FromResult( - state.Revision == entry.Revision - && string.Equals(state.FlowId, flowId, StringComparison.Ordinal) - ? state - : null); + if (state.Revision != entry.Revision) + { + throw new FlowStateUnreadableException( + flowId, + $"its stored revision is {entry.Revision} but the revision inside its JSON is {state.Revision}"); + } + + if (!string.Equals(state.FlowId, flowId, StringComparison.Ordinal)) + throw new FlowStateUnreadableException(flowId, "the flow id inside its JSON is not the id it is stored under"); + + return Task.FromResult(state); } return Task.FromResult(null); @@ -178,6 +198,29 @@ public Task TryDeleteAsync(string flowId, CancellationToken cancellationTo return Task.FromResult(_entries.TryRemove(flowId, out _)); } + /// + /// Removes every entry expired at , at most once per + /// ; one sweeper at a time (the interval stamp is claimed by + /// compare-exchange). Removal is conditional on the observed entry, so a concurrent + /// update/create that swapped the entry in between keeps its (unexpired) replacement. + /// + private void SweepExpired(DateTime now) + { + var due = Interlocked.Read(ref _nextSweepTicks); + if (now.Ticks < due) + return; + + var next = Expiry(now, SweepInterval).Ticks; + if (Interlocked.CompareExchange(ref _nextSweepTicks, next, due) != due) + return; + + foreach (var pair in _entries) + { + if (pair.Value.ExpiresAtUtc <= now) + _entries.TryRemove(pair); + } + } + private Task TryChangeLeaseAsync( string flowId, string leaseId, diff --git a/src/AsyncResponse.Core/InMemoryWorkerTransport.cs b/src/AsyncResponse.Core/InMemoryWorkerTransport.cs index 542aae9a2..68d80fa33 100644 --- a/src/AsyncResponse.Core/InMemoryWorkerTransport.cs +++ b/src/AsyncResponse.Core/InMemoryWorkerTransport.cs @@ -35,6 +35,21 @@ public sealed class InMemoryWorkerTransport : IWorkerTransport, IDelayedWorkerTr private int _outstanding; private volatile bool _draining; + /// + /// One slot per delayed job the transport may hold, from acceptance until the fired job has + /// entered the queue (or was dropped). Neither queue bound covered delayed jobs: every + /// scheduled publish retained its materialized envelope and captured execution context + /// against no limit at all, and when a burst's timers fired, each started an asynchronous + /// channel write that pended outside the bounded queue — so a flood of scheduled jobs grew + /// the process without either configured capacity giving a signal. Bounded by + /// : an external publisher + /// waits for a slot (honoring its cancellation token); a publish from inside a running job + /// is rejected instead, as its immediate follow-ups are at the overflow bound — a worker + /// waiting for a slot that only a fired timer entering the queue (through that worker) frees + /// would be waiting on itself. + /// + private readonly SemaphoreSlim _delayedSlots; + /// Creates a transport with default bounded-queue options. public InMemoryWorkerTransport() : this(Microsoft.Extensions.Options.Options.Create(new InMemoryWorkerTransportOptions())) @@ -54,6 +69,8 @@ public InMemoryWorkerTransport(IOptions options, FullMode = BoundedChannelFullMode.Wait, AllowSynchronousContinuations = false }); + _delayedSlots = new SemaphoreSlim(Options.DelayedJobCapacity, Options.DelayedJobCapacity); + AsyncResponseDiagnostics.TrackInMemoryOverflow(this); } internal ChannelReader Reader => _queue.Reader; @@ -62,9 +79,14 @@ public InMemoryWorkerTransport(IOptions options, /// /// Follow-up jobs published from inside a running job that did not fit the bounded queue. They /// are already counted in _outstanding, so the drain cannot complete the writer while - /// any remain; a worker moves them into the queue as soon as it frees a slot. + /// any remain; a worker moves them into the queue as soon as it frees a slot. Bounded by + /// (tracked in + /// ): unbounded, a fan-out handler could retain every follow-up + /// envelope and its captured ExecutionContext until the process ran out of memory, with the + /// configured queue capacity giving no signal at all. /// private readonly ConcurrentQueue _overflow = new(); + private int _overflowDepth; /// /// Serializes : with multiple workers, an unguarded @@ -73,6 +95,9 @@ public InMemoryWorkerTransport(IOptions options, /// private readonly object _overflowPumpGate = new(); + /// Follow-up jobs currently held past the queue's capacity (the overflow-depth gauge and test inspection). + internal int OverflowDepth => Volatile.Read(ref _overflowDepth); + /// Moves overflow jobs into the queue while it has room. Called by a worker before it reports a job finished. internal void PumpOverflow() { @@ -84,6 +109,28 @@ internal void PumpOverflow() return; _overflow.TryDequeue(out _); + Interlocked.Decrement(ref _overflowDepth); + } + } + } + + /// + /// Admits a follow-up job to the overflow if it is under its capacity. The depth is reserved + /// with a compare-and-swap BEFORE the enqueue, so concurrent in-job publishers (several + /// workers) cannot overshoot the bound between a check and an add. + /// + private bool TryEnqueueOverflow(QueuedJob queued) + { + while (true) + { + var depth = Volatile.Read(ref _overflowDepth); + if (depth >= Options.InJobOverflowCapacity) + return false; + + if (Interlocked.CompareExchange(ref _overflowDepth, depth + 1, depth) == depth) + { + _overflow.Enqueue(queued); + return true; } } } @@ -113,6 +160,12 @@ internal static class InJobScope /// Jobs accepted but not yet finished (queued + executing). Test-harness idle probe. internal int OutstandingJobs => Volatile.Read(ref _outstanding); + /// + /// Delayed jobs currently held: waiting on their due-time timer, or fired and waiting for + /// queue room (the asyncresponse.worker.inmemory_delayed_jobs gauge and test inspection). + /// + internal int DelayedJobsHeld => Options.DelayedJobCapacity - _delayedSlots.CurrentCount; + /// The delayed jobs currently waiting on their due-time timers (test inspection). internal IReadOnlyList SnapshotDelayedJobs() { @@ -187,6 +240,9 @@ internal void BeginShutdownDrain() foreach (var (job, timer) in pending) { timer.Dispose(); + // The slot is freed whether the job is retained (it is re-published into the next + // incarnation's transport, which has its own slots) or dropped. + _delayedSlots.Release(); if (retention is not null) continue; @@ -243,13 +299,21 @@ public async Task PublishAsync(WorkerJobEnvelope job, CancellationToken cancella // // Bypassing the bound for these is the safe side of the trade: the capacity exists as // backpressure on EXTERNAL producers, and follow-up work is a continuation of work the - // queue already admitted. + // queue already admitted. The bypass is itself bounded (InJobOverflowCapacity): past + // it the publish is REJECTED, never parked — the publishing job fails and rides the + // in-process redelivery ladder, which is the only backpressure a worker can be given + // without waiting on itself. The catch below undoes this publish's outstanding count. if (InJobScope.IsActive) { - if (!_queue.Writer.TryWrite(queued)) - _overflow.Enqueue(queued); + if (_queue.Writer.TryWrite(queued) || TryEnqueueOverflow(queued)) + return; - return; + AsyncResponseDiagnostics.RecordInMemoryOverflowRejection(); + throw new InvalidOperationException( + $"The in-memory worker transport rejected a follow-up job ({job.Call.ServiceInterfaceFullName}.{job.Call.MethodName}) published from inside a running job: " + + $"the queue is full ({nameof(InMemoryWorkerTransportOptions)}.{nameof(InMemoryWorkerTransportOptions.QueueCapacity)} = {Options.QueueCapacity}) and the in-job overflow is at its capacity " + + $"({nameof(InMemoryWorkerTransportOptions)}.{nameof(InMemoryWorkerTransportOptions.InJobOverflowCapacity)} = {Options.InJobOverflowCapacity}). Follow-up publishes never wait for queue room " + + "(a worker waiting on itself would deadlock), so the publishing job fails and is redelivered — make its publishes idempotent, raise the capacities, or add workers to drain the backlog."); } await _queue.Writer.WriteAsync(queued, cancellationToken).ConfigureAwait(false); @@ -285,6 +349,42 @@ public Task PublishAsync(WorkerJobEnvelope job, TimeSpan delay, CancellationToke cancellationToken.ThrowIfCancellationRequested(); job = MaterializeFromWire(job); + // Capacity is reserved BEFORE the job is accepted, and held until the fired job has + // entered the queue (WriteFiredAsync) or the drain dropped it — the delayed set and the + // fired-but-pending writes together never exceed DelayedJobCapacity. A publish from + // inside a running job never waits (see _delayedSlots): rejected, like an immediate + // follow-up at the overflow bound, so the publishing job fails and is redelivered. + if (InJobScope.IsActive) + { + if (!_delayedSlots.Wait(0)) + { + AsyncResponseDiagnostics.RecordInMemoryDelayedRejection(); + throw new InvalidOperationException( + $"The in-memory worker transport rejected a delayed job ({job.Call.ServiceInterfaceFullName}.{job.Call.MethodName}) published from inside a running job: " + + $"{nameof(InMemoryWorkerTransportOptions)}.{nameof(InMemoryWorkerTransportOptions.DelayedJobCapacity)} ({Options.DelayedJobCapacity}) delayed jobs are already scheduled. " + + "Follow-up publishes never wait for room (a worker waiting on itself would deadlock), so the publishing job fails and is redelivered — " + + "make its publishes idempotent, or raise the capacity."); + } + + ScheduleDelayed(job, delay); + return Task.CompletedTask; + } + + return PublishDelayedFromOutsideAsync(job, delay, cancellationToken); + } + + private async Task PublishDelayedFromOutsideAsync(WorkerJobEnvelope job, TimeSpan delay, CancellationToken cancellationToken) + { + // Backpressure on an external producer, exactly as the bounded queue is for its immediate + // publishes: the wait ends when a scheduled job fires and enters the queue, a drain drops + // the scheduled set, or the caller's token cancels. + await _delayedSlots.WaitAsync(cancellationToken).ConfigureAwait(false); + ScheduleDelayed(job, delay); + } + + /// Arms the timer for a job whose slot is already reserved; releases the slot when the job cannot be armed. + private void ScheduleDelayed(WorkerJobEnvelope job, TimeSpan delay) + { using var activity = AsyncResponseDiagnostics.StartActivity( "asyncresponse.worker.publish", ActivityKind.Producer, @@ -299,13 +399,16 @@ public Task PublishAsync(WorkerJobEnvelope job, TimeSpan delay, CancellationToke { if (_draining) { + // Nothing is armed here either way, so the reserved slot goes back. + _delayedSlots.Release(); + // Harness restart: a flow suspending mid-drain parks its wake-up with "the // broker" instead of faulting the draining job (and stalling the stop on the // redelivery backoff). if (_drainRetention is { } retained) { retained.Add(job); - return Task.CompletedTask; + return; } // Same contract as the shutdown drain below: delayed in-memory jobs share the @@ -322,8 +425,6 @@ public Task PublishAsync(WorkerJobEnvelope job, TimeSpan delay, CancellationToke var timer = _timeProvider.CreateTimer(static state => ((DelayedJob)state!).Fire(), delayed, delay, Timeout.InfiniteTimeSpan); _delayedJobs.Add(delayed, timer); } - - return Task.CompletedTask; } private void FireDelayed(DelayedJob delayed) @@ -368,6 +469,12 @@ private async Task WriteFiredAsync(QueuedJob queued) "Failed to enqueue fired delayed in-memory worker job {Target}.{Method}.", queued.Job.Call.ServiceInterfaceFullName, queued.Job.Call.MethodName); } + finally + { + // Held from acceptance through the pending write: the job is now either in the + // bounded queue (counted there) or dropped. + _delayedSlots.Release(); + } } // Wire parity for EVERY job, in-process included: the envelope the worker receives is @@ -398,12 +505,48 @@ private sealed class DelayedJob(InMemoryWorkerTransport owner, QueuedJob queued) /// Capacity and concurrency options for the process-local worker transport. public sealed class InMemoryWorkerTransportOptions { - /// Maximum queued jobs before publishers asynchronously wait. Default: 1024. + /// + /// Maximum queued jobs before publishers asynchronously wait. Default: 1024. Delayed jobs + /// are bounded separately by . + /// public int QueueCapacity { get; set; } = 1024; /// Number of jobs that may execute concurrently. Default: 1. public int WorkerCount { get; set; } = 1; + /// + /// Maximum number of follow-up jobs — publishes made from inside a running job, such + /// as a durable flow starting a child or a child waking its parent — held beyond + /// . Follow-up publishes never wait for queue room (the workers are + /// the only consumers, so a worker waiting for capacity would be waiting on itself; with the + /// default of 1, forever) and spill into this overflow instead. Past + /// it a follow-up publish throws : the publishing job + /// fails and is redelivered by the in-process retry ladder, so make in-job publishes + /// idempotent. Sized so an ordinary fan-out never hits it while a runaway one is bounded — + /// every held job retains its materialized envelope and captured execution context. The + /// current depth is the asyncresponse.worker.inmemory_overflow_depth gauge; rejections + /// count on asyncresponse.worker.inmemory_overflow_rejections. Default: 4096. + /// + public int InJobOverflowCapacity { get; set; } = 4096; + + /// + /// Maximum number of delayed jobs — EnqueueWorkerAsync(..., delay) and the wake-ups + /// behind suspended durable-flow timers — the transport holds at once: waiting on their due + /// time, or fired and waiting for queue room. Neither nor + /// covers them, and every held job retains its + /// materialized envelope and captured execution context. At the bound a delayed publish from + /// outside a job waits (honoring its cancellation token) until a scheduled job enters the + /// queue; one made from inside a running job — a flow parking on a timer — never + /// waits (a worker waiting for room only a fired timer draining through that worker can free + /// would be waiting on itself) and throws instead: + /// the publishing job fails and is redelivered by the in-process retry ladder, so make in-job + /// publishes idempotent. The current count is the + /// asyncresponse.worker.inmemory_delayed_jobs gauge; in-job rejections count on + /// asyncresponse.worker.inmemory_delayed_rejections. Size it above the number of + /// flows you expect to be sleeping at once on this transport. Default: 4096. + /// + public int DelayedJobCapacity { get; set; } = 4096; + /// /// Maximum number of delivery attempts before a failing job is dropped, with an error log and /// a dropped outcome on the worker-jobs counter. The process-local queue has no broker @@ -429,6 +572,10 @@ internal void Validate() throw new InvalidOperationException($"{nameof(QueueCapacity)} must be positive."); if (WorkerCount <= 0) throw new InvalidOperationException($"{nameof(WorkerCount)} must be positive."); + if (InJobOverflowCapacity < 0) + throw new InvalidOperationException($"{nameof(InJobOverflowCapacity)} must be zero (no overflow: a follow-up publish that finds the queue full is rejected) or positive."); + if (DelayedJobCapacity <= 0) + throw new InvalidOperationException($"{nameof(DelayedJobCapacity)} must be positive: durable-flow timers on this transport are delayed jobs."); if (MaxDeliveryAttempts < 0) throw new InvalidOperationException($"{nameof(MaxDeliveryAttempts)} must be zero (unlimited) or positive."); if (RetryBaseDelay <= TimeSpan.Zero) diff --git a/src/AsyncResponse.Core/JsonSafety.cs b/src/AsyncResponse.Core/JsonSafety.cs index e6c7d87ae..2b0701593 100644 --- a/src/AsyncResponse.Core/JsonSafety.cs +++ b/src/AsyncResponse.Core/JsonSafety.cs @@ -38,12 +38,31 @@ internal static class JsonSafety { return JsonSerializer.Deserialize(json, typeInfo); } - catch (JsonException jsonException) + catch (JsonException jsonException) when (!IsBodyFree(jsonException)) { throw ParseFailure(json, jsonException); } } + /// + /// UTF-8 counterpart of for readers + /// that receive bytes off the wire (the Redis channel), with the same body-free failure + /// contract: size in bytes plus the reader's position, never its message or path. + /// + public static T? SafeDeserialize(ReadOnlySpan utf8Json, JsonTypeInfo typeInfo) + { + ThrowIfClearlyNotJson(utf8Json); + + try + { + return JsonSerializer.Deserialize(utf8Json, typeInfo); + } + catch (JsonException jsonException) when (!IsBodyFree(jsonException)) + { + throw ParseFailure(utf8Json.Length, "UTF-8 bytes", jsonException); + } + } + /// /// Non-generic counterpart for callers that only know the target type at runtime (e.g. /// materializing a persisted flow input). @@ -56,12 +75,67 @@ internal static class JsonSafety { return JsonSerializer.Deserialize(json, AsyncResponseJson.GetTypeInfo(returnType, WithResolver(options))); } - catch (JsonException jsonException) + catch (JsonException jsonException) when (!IsBodyFree(jsonException)) { throw ParseFailure(json, jsonException); } } + /// + /// Converts an already-parsed (a worker-job argument, a recovery + /// payload) to with the same body-free failure contract as the + /// string overloads. The outer envelope parse is guarded elsewhere; this is the second reader + /// pass — the one that reads dictionary keys and property values off the wire into the + /// callback's parameter types — and its carries the same + /// Path: $.<key> the envelope's would, so it needs the same scrubbing. + /// + public static object? SafeDeserialize(JsonElement element, Type returnType, JsonSerializerOptions? options = null) + { + try + { + return JsonSerializer.Deserialize(element, AsyncResponseJson.GetTypeInfo(returnType, WithResolver(options))); + } + catch (JsonException jsonException) when (!IsBodyFree(jsonException)) + { + // GetRawText only on the failure path, and only for its length. + throw ParseFailure(element.GetRawText(), jsonException); + } + } + + /// + /// Key under which a the LIBRARY authored marks itself as + /// body-free, so SafeDeserialize preserves its message. Visible on + /// , harmlessly — the channels already carry + /// RemoteStackTrace there. + /// + private const string BodyFreeMessageKey = "AsyncResponse.BodyFreeMessage"; + + /// + /// A malformed-message failure whose text the library wrote itself: it names only the wire + /// contract's own property names — SchemaVersion, Success, Payload — and + /// never a byte of the inbound body, so SafeDeserialize lets it through untouched + /// instead of replacing it with the position-only failure it builds for the reader's own. + /// + /// The distinction is the whole point. System.Text.Json's own messages quote the body + /// (Path: $.Payload.Values['…'] is built from inbound dictionary keys), so they must be + /// dropped; ours are the primary operator diagnosis for the commonest malformed-envelope cause + /// in production — a foreign or mismatched producer writing to the response channel — and + /// scrubbing them to "failed at line 0, byte position 2" costs the diagnosis while protecting + /// nothing. Marked rather than subtyped so the exception REMAINS a plain + /// : every classification (the ingress treats it as permanent, no + /// retry burn), every catch, and every exact-type assertion keeps working unchanged. + /// + /// + public static JsonException WireContractFailure(string message) + { + var failure = new JsonException(message); + failure.Data[BodyFreeMessageKey] = true; + return failure; + } + + /// Whether carries a message the library authored (see ). + private static bool IsBodyFree(JsonException jsonException) => jsonException.Data.Contains(BodyFreeMessageKey); + /// /// Builds the body-free parse failure: size plus the JSON coordinates the reader stopped at. /// @@ -76,8 +150,11 @@ internal static class JsonSafety /// /// private static InvalidDataException ParseFailure(string json, JsonException jsonException) + => ParseFailure(json.Length, "UTF-16 code units", jsonException); + + private static InvalidDataException ParseFailure(int length, string unit, JsonException jsonException) => new( - $"Failed to parse JSON payload ({json.Length} UTF-16 code units) at line {Describe(jsonException.LineNumber)}, " + + $"Failed to parse JSON payload ({length} {unit}) at line {Describe(jsonException.LineNumber)}, " + $"byte position {Describe(jsonException.BytePositionInLine)}.", new JsonException( $"The JSON payload is malformed at line {Describe(jsonException.LineNumber)}, " + @@ -131,4 +208,17 @@ public static void ThrowIfClearlyNotJson(string json) if (trimmed[0] == '<') throw new InvalidDataException($"Received HTML when JSON was expected ({json.Length} UTF-16 code units)."); } + + /// UTF-8 counterpart of : the same two guards over raw bytes. + public static void ThrowIfClearlyNotJson(ReadOnlySpan utf8Json) + { + // JSON whitespace is exactly these four ASCII bytes (RFC 8259 §2), so a byte-level trim + // matches what the reader itself would skip. + var trimmed = utf8Json.TrimStart("\t\n\r "u8); + if (trimmed.IsEmpty) + throw new InvalidDataException("Empty message body when JSON was expected."); + + if (trimmed[0] == (byte)'<') + throw new InvalidDataException($"Received HTML when JSON was expected ({utf8Json.Length} UTF-8 bytes)."); + } } diff --git a/src/AsyncResponse.Core/LostSubscriberCallbackDispatcher.cs b/src/AsyncResponse.Core/LostSubscriberCallbackDispatcher.cs index b8ccc3bad..b982dcb8e 100644 --- a/src/AsyncResponse.Core/LostSubscriberCallbackDispatcher.cs +++ b/src/AsyncResponse.Core/LostSubscriberCallbackDispatcher.cs @@ -105,7 +105,7 @@ public async Task DispatchLostResponses( RecoveryAction? action = null; var routeSet = false; var routeMixed = false; - ExceptionDispatchInfo? firstException = null; + List? failures = null; foreach (var recoveryState in recoveryStates) { @@ -134,30 +134,18 @@ public async Task DispatchLostResponses( throw; // Capture rather than re-throw a bare variable so the original throw site's stack - // trace survives the dispatch to the remaining registrations. - firstException ??= ExceptionDispatchInfo.Capture(ex); + // trace survives the dispatch to the remaining registrations. EVERY failure is + // kept: settlement below classifies the whole set, not the first one. + (failures ??= []).Add(ExceptionDispatchInfo.Capture(ex)); } } - if (firstException is not null) + if (failures is not null) { if (!callbackInvoked) - firstException.Throw(); - - // A sibling registration already consumed the response and its callback succeeded - // (shared-correlation registrations are an expected shape — a worker that died - // mid-await leaves its registration beside the replacement's). Rethrowing here would - // hand the ingress a failure for a response that WAS delivered: its retry loop - // re-dispatches (the consumed registration is gone, the failing one keeps failing) - // and then escalates via SetException — terminally failing a flow that was correctly - // recovered moments earlier. The failed registration was NOT deleted, so its state - // remains for a later redelivery to retry and for the watchdog to surface; log the - // residual failure loudly instead of letting it poison the delivered response. - _logger.LogError( - firstException.SourceException, - "Lost-response dispatch for correlationId {CorrelationId} on {Channel} partially failed after another registration's callback succeeded; the failed registration stays registered for retry and watchdog visibility.", - correlationId, - channel); + ThrowUnsettled(failures); + + SettleResidualFailures(failures, correlationId, channel, "response"); } return new LostSubscriberDispatchResult(routeMixed ? null : action, callbackInvoked) { RouteMixed = routeMixed }; @@ -188,7 +176,7 @@ public async Task DispatchLostExceptions( return new LostSubscriberDispatchResult(RecoveryAction.Fail, await DispatchLostException(null, exception, channel).ConfigureAwait(false)); var callbackInvoked = false; - ExceptionDispatchInfo? firstException = null; + List? failures = null; foreach (var recoveryState in recoveryStates) { @@ -206,33 +194,117 @@ public async Task DispatchLostExceptions( throw; // Capture rather than re-throw a bare variable so the original throw site's stack - // trace survives the dispatch to the remaining registrations. - firstException ??= ExceptionDispatchInfo.Capture(ex); + // trace survives the dispatch to the remaining registrations. EVERY failure is + // kept: settlement below classifies the whole set, not the first one. + (failures ??= []).Add(ExceptionDispatchInfo.Capture(ex)); } } - if (firstException is not null) + if (failures is not null) { if (!callbackInvoked) - firstException.Throw(); - - // A sibling registration already consumed the exception and its callback succeeded - // (shared-correlation registrations are an expected shape — a worker that died - // mid-await leaves its registration beside the replacement's). Rethrowing here would - // hand the ingress a failure for an exception that WAS delivered: its retry loop - // re-dispatches (the consumed registration is gone, the failing one keeps failing) - // and the delivery never settles. The failed registration was NOT deleted, so its - // state remains for a later redelivery to retry and for the watchdog to surface; log - // the residual failure loudly instead of letting it poison the delivered exception. + ThrowUnsettled(failures); + + SettleResidualFailures(failures, correlationId, channel, "exception"); + } + + // Exception envelopes always take the failure route, so the action is fixed at Fail. + return new LostSubscriberDispatchResult(RecoveryAction.Fail, callbackInvoked); + } + + /// + /// Rethrows for a fan-out in which NO registration's callback succeeded. A sibling whose + /// failure-callback ladder was already exhausted () + /// wins whatever its position: the ingress passes that shape through untouched, so the + /// transport redelivers the still-unacknowledged signal to every registration — instead of + /// the ingress burning its own retry ladder on an earlier sibling's failure and then escalating + /// through SetException, which re-invokes the very failure callback that just gave up. + /// Otherwise the first failure propagates as before, stack trace intact, for the ingress's + /// retry-then-escalate handling. + /// + private static void ThrowUnsettled(List failures) + { + foreach (var failure in failures) + { + if (failure.SourceException is RecoveryCallbackFailedException) + failure.Throw(); + } + + failures[0].Throw(); + } + + /// + /// Settles a fan-out dispatch in which at least one registration's callback succeeded (and + /// was consumed) while one or more others failed. Shared-correlation registrations are an + /// expected shape — a worker that died mid-await leaves its registration beside the + /// replacement's — and each carries its own delivery guarantee, so the verdict is taken over + /// the WHOLE set of failures, never the first one alone. + /// + /// A deterministic failure (the target is unauthorized, unresolvable, not registered, + /// or no longer binds) is logged: redelivery cannot fix it and its registration stays for the + /// watchdog to surface. A transient one — any failure that is not deterministic — + /// propagates as , which the ingress passes + /// through untouched (no second retry ladder, no SetException escalation that would + /// invoke the FAILURE callbacks of registrations whose resume merely blipped), so the + /// transport redelivers the terminal signal; the consumed registrations are already deleted, + /// so the redelivery reaches only the registrations that failed. The message is acknowledged + /// only when EVERY failure was deterministic. Before round 39 the verdict was taken from the + /// first failure alone: a deterministic failure first in the set hid a transient sibling + /// behind it, the message was acknowledged, and the transient registration — a valid waiter + /// whose dependency was briefly down — lost the only copy of its payload, with the outcome + /// depending on the order the store returned the registrations in. + /// + /// + private void SettleResidualFailures(List failures, string correlationId, string channel, string kind) + { + ExceptionDispatchInfo? exhausted = null; + Exception? transient = null; + foreach (var failure in failures) + { + var residual = failure.SourceException; + + // Already the propagating shape (a sibling's failure-callback ladder was exhausted); + // its own ladder logged it. + if (residual is RecoveryCallbackFailedException) + { + exhausted ??= failure; + continue; + } + + if (IsPermanentCallbackFailure(residual)) + { + _logger.LogError( + residual, + "Lost-{Kind} dispatch for correlationId {CorrelationId} on {Channel} failed with a deterministic fault for one registration after another registration's callback succeeded; redelivery cannot fix it, so that registration stays registered for watchdog visibility.", + kind, + correlationId, + channel); + continue; + } + + transient ??= residual; + } + + if (exhausted is not null) + exhausted.Throw(); + + if (transient is null) + { _logger.LogError( - firstException.SourceException, - "Lost-exception dispatch for correlationId {CorrelationId} on {Channel} partially failed after another registration's callback succeeded; the failed registration stays registered for retry and watchdog visibility.", + "Lost-{Kind} dispatch for correlationId {CorrelationId} on {Channel} partially failed with deterministic faults only; the message is acknowledged.", + kind, correlationId, channel); + return; } - // Exception envelopes always take the failure route, so the action is fixed at Fail. - return new LostSubscriberDispatchResult(RecoveryAction.Fail, callbackInvoked); + _logger.LogError( + transient, + "Lost-{Kind} dispatch for correlationId {CorrelationId} on {Channel} partially failed transiently after another registration's callback succeeded; the consumed registrations are deleted, the failed ones stay armed, and the message is left unacknowledged so the transport redelivers it to those registrations alone.", + kind, + correlationId, + channel); + throw new RecoveryCallbackFailedException(correlationId, attempts: 1, transient); } /// Dispatches a successfully published payload that no subscriber received. @@ -458,7 +530,7 @@ await AsyncResponseRetry.ExecuteAsync( return true; }, isTransient: static ex => !IsPermanentCallbackFailure(ex), - maxAttempts: 4, + maxAttempts: FailureCallbackAttempts, baseDelay: TimeSpan.FromMilliseconds(250), maxDelay: TimeSpan.FromSeconds(2), CancellationToken.None, @@ -470,19 +542,39 @@ await AsyncResponseRetry.ExecuteAsync( } catch (Exception ex) { - // Deliberately not rethrown once the retries are exhausted — keep the swallow. An - // exception would bubble up to the broker ingress, which reacts with SetException and - // would invoke this same failure callback a second time; routing it to transport - // redelivery instead would hot-loop a permanently-throwing callback on RabbitMQ's - // unbounded default. The domain failure has already been dispatched (and retried - // above), and the kept recovery row is surfaced by the watchdog's staleness report, - // so the drop is operator-visible rather than silent. AsyncResponseDiagnostics.SetError(activity, ex); - _logger.LogError(ex, "Failure callback failed for channel {Channel}.", channel); - return false; + + if (IsPermanentCallbackFailure(ex)) + { + // Deterministic by construction: the same call fails the same way on every + // delivery, so redelivery would only burn the transport's attempts (or hot-loop + // on RabbitMQ's unbounded default). Swallow: the message is acknowledged, the + // kept recovery row is surfaced by the watchdog's staleness report, and the + // error log names the misconfiguration to fix. + _logger.LogError(ex, "Failure callback for channel {Channel} cannot be invoked (deterministic fault); the message is acknowledged and the registration stays for the watchdog.", channel); + return false; + } + + // Transient and exhausted: the response is a TERMINAL signal that, once acknowledged, + // exists nowhere (the recovery row keeps the callback, not the payload; the watchdog + // only reports). Propagate as a dedicated type the ingress passes through untouched — + // no retry (the ladder above already ran) and no SetException escalation (that would + // only re-invoke this same callback) — so the transport keeps the message for its own + // bounded redelivery and dead-letter policy. On RabbitMQ's default MaxDeliveryAttempts + // = 0 that is the documented unlimited requeue any failing handler gets; configure a + // cap there as for worker jobs. + _logger.LogError( + ex, + "Failure callback for channel {Channel} failed on all {Attempts} attempts; the message is left unacknowledged for transport redelivery and the registration stays armed.", + channel, + FailureCallbackAttempts); + throw new RecoveryCallbackFailedException(recoveryState.CorrelationId ?? string.Empty, FailureCallbackAttempts, ex); } } + /// In-process invocations of a failure callback per delivery before the delivery is handed back to the transport. + internal const int FailureCallbackAttempts = 4; + /// /// Whether a failed callback invocation is deterministic — the same call will fail the same /// way on every attempt, so retrying only burns the backoff ladder on the publish path. diff --git a/src/AsyncResponse.Core/PublicAPI.Unshipped.txt b/src/AsyncResponse.Core/PublicAPI.Unshipped.txt index 974cf2bde..51be38660 100644 --- a/src/AsyncResponse.Core/PublicAPI.Unshipped.txt +++ b/src/AsyncResponse.Core/PublicAPI.Unshipped.txt @@ -117,6 +117,8 @@ AsyncResponse.DurableFlowOptions.ExecutionLeaseDuration.get -> System.TimeSpan AsyncResponse.DurableFlowOptions.ExecutionLeaseDuration.set -> void AsyncResponse.DurableFlowOptions.ExecutionLeaseRenewInterval.get -> System.TimeSpan AsyncResponse.DurableFlowOptions.ExecutionLeaseRenewInterval.set -> void +AsyncResponse.DurableFlowOptions.LedgerSizeWarningBytes.get -> long? +AsyncResponse.DurableFlowOptions.LedgerSizeWarningBytes.set -> void AsyncResponse.DurableFlowOptions.ProgressPersistenceInterval.get -> System.TimeSpan AsyncResponse.DurableFlowOptions.ProgressPersistenceInterval.set -> void AsyncResponse.DurableFlowOptions.StateExpiry.get -> System.TimeSpan @@ -124,6 +126,7 @@ AsyncResponse.DurableFlowOptions.StateExpiry.set -> void AsyncResponse.DurableFlowOptions.TimerInProcessThreshold.get -> System.TimeSpan AsyncResponse.DurableFlowOptions.TimerInProcessThreshold.set -> void AsyncResponse.IDurableFlowExecutor +AsyncResponse.IDurableFlowExecutor.CreateAndExecuteAsync(string! flowId, string! initialStateJson) -> System.Threading.Tasks.Task! AsyncResponse.IDurableFlowExecutor.ExecuteAsync(string! flowId) -> System.Threading.Tasks.Task! AsyncResponse.IDurableFlowExecutor.FailAsync(string! flowId, System.Exception! exception) -> System.Threading.Tasks.Task! AsyncResponse.IDurableFlowExecutor.FailAsync(string! flowId, System.Exception! exception, string! correlationId) -> System.Threading.Tasks.Task! @@ -139,6 +142,10 @@ AsyncResponse.InMemoryWorkerTransport.PublishAsync(AsyncResponse.WorkerJobEnvelo AsyncResponse.InMemoryWorkerTransport.PublishAsync(AsyncResponse.WorkerJobEnvelope! job, System.TimeSpan delay, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! AsyncResponse.InMemoryWorkerTransportOptions AsyncResponse.InMemoryWorkerTransportOptions.InMemoryWorkerTransportOptions() -> void +AsyncResponse.InMemoryWorkerTransportOptions.InJobOverflowCapacity.get -> int +AsyncResponse.InMemoryWorkerTransportOptions.InJobOverflowCapacity.set -> void +AsyncResponse.InMemoryWorkerTransportOptions.DelayedJobCapacity.get -> int +AsyncResponse.InMemoryWorkerTransportOptions.DelayedJobCapacity.set -> void AsyncResponse.InMemoryWorkerTransportOptions.MaxDeliveryAttempts.get -> int AsyncResponse.InMemoryWorkerTransportOptions.MaxDeliveryAttempts.set -> void AsyncResponse.InMemoryWorkerTransportOptions.QueueCapacity.get -> int @@ -149,6 +156,10 @@ AsyncResponse.InMemoryWorkerTransportOptions.RetryMaxDelay.get -> System.TimeSpa AsyncResponse.InMemoryWorkerTransportOptions.RetryMaxDelay.set -> void AsyncResponse.InMemoryWorkerTransportOptions.WorkerCount.get -> int AsyncResponse.InMemoryWorkerTransportOptions.WorkerCount.set -> void +AsyncResponse.RecoveryCallbackFailedException +AsyncResponse.RecoveryCallbackFailedException.Attempts.get -> int +AsyncResponse.RecoveryCallbackFailedException.CorrelationId.get -> string! +AsyncResponse.RecoveryCallbackFailedException.RecoveryCallbackFailedException(string! correlationId, int attempts, System.Exception! innerException) -> void AsyncResponse.RecoveryStateObservation AsyncResponse.RecoveryStateObservation.$() -> AsyncResponse.RecoveryStateObservation! AsyncResponse.RecoveryStateObservation.ActiveSubscribers.get -> long @@ -165,7 +176,11 @@ AsyncResponse.RecoveryStateObservation.RegisteredAtUtc.init -> void AsyncResponse.ScheduledFlowOptions AsyncResponse.ScheduledFlowOptions.Enabled.get -> bool AsyncResponse.ScheduledFlowOptions.Enabled.set -> void +AsyncResponse.ScheduledFlowOptions.RedriveInterval.get -> System.TimeSpan +AsyncResponse.ScheduledFlowOptions.RedriveInterval.set -> void AsyncResponse.ScheduledFlowOptions.ScheduledFlowOptions() -> void +AsyncResponse.ScheduledFlowOptions.StartupRedriveWindow.get -> System.TimeSpan +AsyncResponse.ScheduledFlowOptions.StartupRedriveWindow.set -> void AsyncResponse.ScheduledFlowOptions.TimeZone.get -> System.TimeZoneInfo! AsyncResponse.ScheduledFlowOptions.TimeZone.set -> void Microsoft.Extensions.DependencyInjection.AsyncResponseCallbackAllowList diff --git a/src/AsyncResponse.Core/RecoveryCallbackFailedException.cs b/src/AsyncResponse.Core/RecoveryCallbackFailedException.cs new file mode 100644 index 000000000..30715ee40 --- /dev/null +++ b/src/AsyncResponse.Core/RecoveryCallbackFailedException.cs @@ -0,0 +1,53 @@ +namespace AsyncResponse; + +/// +/// Raised by a publish whose response (or exception) found no live subscriber and whose +/// lost-subscriber recovery callbacks could not all be invoked transiently. The message +/// was not acknowledged — every registration whose callback did not succeed stays armed, +/// and the broker ingress propagates this exception untouched (no SetException escalation, +/// which would only re-invoke the same failing callback, or fail flows whose resume merely +/// blipped) so the transport redelivers the message under its own bounded policy +/// (MaxDeliveryAttempts, then the dead-letter destination). The terminal signal therefore +/// survives in the broker until the callback's dependency recovers or an operator replays it from +/// the dead-letter queue, instead of being acknowledged into a log line while the flow stays stuck. +/// Two paths raise it: +/// +/// the failure callback failed on every attempt of its in-process retry +/// ladder ( is that ladder's length); +/// a fan-out over several registrations sharing the correlation id +/// partially failed: at least one registration's callback succeeded and was consumed, another's +/// failed transiently ( is 1 — the redelivery is the retry). Because the +/// successful registrations are deleted, the redelivery reaches only the one that failed, and a +/// caller that retries the publish once the dependency recovers completes exactly that +/// registration. +/// +/// +/// Deterministic callback faults — an unauthorized or unresolvable target, a method that no longer +/// binds — are never wrapped in this type: redelivery cannot fix them, so they are logged and the +/// message is acknowledged (the registration stays for the watchdog to surface). +/// +/// +/// A direct caller of IAsyncResponsePublisher.SetResponse/SetException (an HTTP +/// callback endpoint, for instance) sees this exception too; answering the remote system with a +/// retriable status is the equivalent of the broker's redelivery. +/// +/// +public sealed class RecoveryCallbackFailedException : Exception +{ + /// Creates the exception for after failed invocations. + public RecoveryCallbackFailedException(string correlationId, int attempts, Exception innerException) + : base( + $"A lost-subscriber recovery callback for correlationId '{correlationId}' failed transiently ({attempts} attempt(s)); " + + "the message was not acknowledged so the transport can redeliver it to the registration(s) still armed.", + innerException) + { + CorrelationId = correlationId; + Attempts = attempts; + } + + /// The correlation id whose recovery callback could not be invoked. + public string CorrelationId { get; } + + /// How many in-process invocations were attempted before handing the delivery back to the transport (1 for a partially failed fan-out). + public int Attempts { get; } +} diff --git a/src/AsyncResponse.Core/ReflectionExtensions.cs b/src/AsyncResponse.Core/ReflectionExtensions.cs index 9086b315e..cfe9591d3 100644 --- a/src/AsyncResponse.Core/ReflectionExtensions.cs +++ b/src/AsyncResponse.Core/ReflectionExtensions.cs @@ -2,6 +2,7 @@ using System.Diagnostics.CodeAnalysis; using System.Linq.Expressions; using System.Reflection; +using System.Runtime.CompilerServices; using System.Text.Json; namespace AsyncResponse; @@ -122,13 +123,8 @@ public static Task InvokeAsync(this IServiceProvider provider, ReflectionInvocat ?? throw new CallbackTargetUnresolvableException( $"Service '{dto.ServiceInterfaceFullName}' is not registered."); - // 4) Resolve and cache method metadata + compiled invocation delegate. Collectible - // (plugin) service types are planned per call: a strong Type-keyed cache entry would - // pin the plugin's AssemblyLoadContext after unload. - var planKey = new InvocationPlanKey(serviceType, dto.MethodName, dto.Params.Length); - var plan = serviceType.Assembly.IsCollectible - ? CreateInvocationPlan(planKey) - : InvocationPlans.GetOrAdd(planKey, static key => CreateInvocationPlan(key)); + // 4) Resolve and cache method metadata + compiled invocation delegate. + var plan = GetInvocationPlan(serviceType, dto.MethodName, dto.Params.Length); // 5) Convert only the arguments that need conversion, keeping already-typed arrays hot. var invocationArgs = plan.ConvertArguments(dto.Params); @@ -174,6 +170,34 @@ internal static void ThrowIfNotAuthorized( private static async Task AwaitSlow(ValueTask pending) => await pending.ConfigureAwait(false); + /// + /// The one binding rule, applied at every boundary a callback crosses: the persisted + /// (service, method name, parameter count) triple must select exactly one public + /// instance method, with no by-ref parameters and no open generics. Expression-based + /// registration calls this at conversion time (), so a + /// descriptor that could never dispatch — an overload set that shares a name and arity, which + /// the compiler resolves happily but a name-plus-arity descriptor cannot — fails at the + /// EnqueueWorkerAsync/OnLostSubscriber* call, in the caller's stack, instead of + /// after publication where it burns transport retries or strands a recovery. Dispatch calls the + /// same method, so the two can never disagree; the plan built here is the one dispatch reuses. + /// + /// The descriptor does not bind to exactly one supported method. + internal static void EnsureBindable(Type serviceType, string methodName, int parameterCount) + => GetInvocationPlan(serviceType, methodName, parameterCount); + + /// + /// Resolves (and caches) the compiled plan for a (service type, method, arity) key. + /// Collectible (plugin) service types are planned per call: a strong Type-keyed cache entry + /// would pin the plugin's AssemblyLoadContext after unload. + /// + private static InvocationPlan GetInvocationPlan(Type serviceType, string methodName, int parameterCount) + { + var planKey = new InvocationPlanKey(serviceType, methodName, parameterCount); + return serviceType.Assembly.IsCollectible + ? CreateInvocationPlan(planKey) + : InvocationPlans.GetOrAdd(planKey, static key => CreateInvocationPlan(key)); + } + // Internal: the durable-flow executor resolves persisted flow/input type names through the // same default-ALC scan + custom-resolver chain as persisted callback targets. [UnconditionalSuppressMessage("Trimming", "IL2026", @@ -295,7 +319,102 @@ private static InvocationPlan CreateInvocationPlan(InvocationPlanKey key) $"Callback method '{method.Name}' on '{key.ServiceType.Name}' has unbound generic parameters, which are not supported."); } - return new InvocationPlan(converters, CreateInvoker(method, parameters)); + // A void-returning target is awaited as "already complete" (ToValueTaskExpression), which + // is exactly right for a synchronous method and exactly wrong for an `async void` one: its + // body is still running at the first await when the invoker returns, so the worker job is + // acknowledged, the DI scope disposed, and any later exception lost to the thread pool. + // A concrete (class-typed) service exposes the implementation here, so it is rejected at + // plan time; an interface hides it behind the DI resolution, so the plan carries the + // interface method and checks the resolved implementation on invoke. + MethodInfo? voidMethod = null; + if (method.ReturnType == typeof(void)) + { + if (!key.ServiceType.IsInterface) + ThrowIfAsyncVoid(method, key.ServiceType); + voidMethod = method; + } + + return new InvocationPlan(converters, CreateInvoker(method, parameters), voidMethod); + } + + /// + /// Implementations already verified to be synchronous for a given void interface method, keyed + /// by the concrete service type. Collectible (plugin) types are never cached — a Type key would + /// pin their AssemblyLoadContext — and are re-checked per call, the plugin-host cold path. + /// + private static readonly ConcurrentDictionary<(Type Implementation, MethodInfo Method), bool> VerifiedSynchronousVoid = new(); + + /// + /// Rejects an async void implementation of a void-returning callback target before it is + /// invoked. The C# compiler marks every async method with + /// ; a void return with that marker is the one + /// shape a caller can neither await nor observe faults from. Failing open when the interface + /// map is unavailable (a runtime without it) keeps the historical behavior there. + /// + [UnconditionalSuppressMessage("Trimming", "IL2072", + Justification = "GetInterfaceMap is asked for the callback method's own declaring interface, which the registration " + + "already rooted (DynamicallyAccessedMembers(PublicMethods) on TService); no member beyond the ones the " + + "invocation itself needs is required, and an unavailable map fails open to the pre-existing behavior.")] + [UnconditionalSuppressMessage("Trimming", "IL2075", + Justification = "The implementation type is the DI-resolved service for an interface rooted at registration " + + "(DynamicallyAccessedMembers(PublicMethods|Interfaces) on TService, or WithDurableFlow's static route); " + + "the interface map needs only the members the invocation itself already requires. If the runtime cannot " + + "produce the map the check fails open — the pre-existing behavior — never closed.")] + private static void EnsureNotAsyncVoid(object service, MethodInfo voidMethod) + { + var implementationType = service.GetType(); + var cacheable = !implementationType.Assembly.IsCollectible; + if (cacheable && VerifiedSynchronousVoid.ContainsKey((implementationType, voidMethod))) + return; + + MethodInfo? implementation = null; + try + { + var declaringType = voidMethod.DeclaringType!; + if (declaringType.IsInterface) + { + if (declaringType.IsAssignableFrom(implementationType)) + { + var map = implementationType.GetInterfaceMap(declaringType); + var index = Array.IndexOf(map.InterfaceMethods, voidMethod); + if (index >= 0) + implementation = map.TargetMethods[index]; + } + } + else + { + // Class-typed service: the plan already rejected the declared method; a derived + // registration may override it, so look the override up on the resolved type. + implementation = implementationType.GetMethod( + voidMethod.Name, + BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic, + binder: null, + voidMethod.GetParameters().Select(p => p.ParameterType).ToArray(), + modifiers: null); + } + } + catch (Exception ex) when (ex is NotSupportedException or ArgumentException or TypeLoadException or AmbiguousMatchException) + { + // No map on this runtime, or a shape it cannot answer for: fail open. + implementation = null; + } + + if (implementation is not null) + ThrowIfAsyncVoid(implementation, implementationType); + + if (cacheable) + VerifiedSynchronousVoid.TryAdd((implementationType, voidMethod), true); + } + + private static void ThrowIfAsyncVoid(MethodInfo implementation, Type implementationType) + { + if (implementation.ReturnType != typeof(void) || !implementation.IsDefined(typeof(AsyncStateMachineAttribute), inherit: false)) + return; + + throw new CallbackTargetUnresolvableException( + $"'{implementationType.FullName}.{implementation.Name}' is an async void method. The dispatcher cannot await it: the worker job or " + + "recovery callback would be acknowledged, and its DI scope disposed, while the body is still running at its first await, and any " + + "later exception would escape to the thread pool. Return Task or ValueTask instead (a synchronous void method is fine)."); } /// @@ -447,7 +566,7 @@ public static ReflectionInvocationDto ResolveCallback( private readonly record struct InvocationPlanKey(Type ServiceType, string MethodName, int ParameterCount); - private sealed class InvocationPlan(ConversionPlan[] converters, AsyncMethodInvoker invoker) + private sealed class InvocationPlan(ConversionPlan[] converters, AsyncMethodInvoker invoker, MethodInfo? voidMethod) { /// Runs the ConvertArguments operation. public object?[] ConvertArguments(object?[] args) @@ -474,7 +593,14 @@ private sealed class InvocationPlan(ConversionPlan[] converters, AsyncMethodInvo /// Invokes the reflected operation. public ValueTask Invoke(object service, object?[] args) - => invoker(service, args); + { + // Only void-returning plans pay for the implementation check; Task/ValueTask targets + // are awaited for real and need none. + if (voidMethod is not null) + EnsureNotAsyncVoid(service, voidMethod); + + return invoker(service, args); + } private static object?[] CopyPrefix(object?[] args, int length) { @@ -495,10 +621,14 @@ private sealed class ConversionPlan(Type targetType) public object? Convert(object? value) { // Handle JSON payloads (contract metadata resolved through the AOT-safe chain; loose - // case-insensitive matching as before). + // case-insensitive matching as before). Through JsonSafety, not the raw serializer: + // this is the reader pass that walks the payload's own property names and dictionary + // keys into the parameter type, and a raw JsonException quotes them ("Path: + // $.") — the worker ingress logs the exception that escapes here, so the payload + // must not be in it (docs/security.md: the library never logs a message body). if (value is JsonElement je) { - return JsonSerializer.Deserialize(je, AsyncResponseJson.GetTypeInfo(targetType, AsyncResponseJson.CaseInsensitive)); + return JsonSafety.SafeDeserialize(je, targetType, AsyncResponseJson.CaseInsensitive); } // Already the correct CLR type (a boxed value also satisfies its nullable counterpart). @@ -512,10 +642,11 @@ private sealed class ConversionPlan(Type targetType) return value; } - // JSON in a string (a target the string cannot satisfy directly) + // JSON in a string (a target the string cannot satisfy directly); same body-free + // failure contract as the element branch above. if (value is string s && !_isString) { - return JsonSerializer.Deserialize(s, AsyncResponseJson.GetTypeInfo(targetType, AsyncResponseJson.CaseInsensitive)); + return JsonSafety.SafeDeserialize(s, targetType, AsyncResponseJson.CaseInsensitive); } // Null handling diff --git a/src/AsyncResponse.Core/ScheduledFlows.cs b/src/AsyncResponse.Core/ScheduledFlows.cs index 222556d04..9a32c1cbc 100644 --- a/src/AsyncResponse.Core/ScheduledFlows.cs +++ b/src/AsyncResponse.Core/ScheduledFlows.cs @@ -15,6 +15,29 @@ public sealed class ScheduledFlowOptions /// (and its flow-type routing) while pausing new occurrences — e.g. per environment. /// public bool Enabled { get; set; } = true; + + /// + /// How long the scheduler waits between attempts to re-drive an occurrence whose start job + /// could not be published (a broker outage outlasting the start's own in-process retry + /// ladder). The publish is the start's commit point, so such an occurrence has no + /// ledger yet; a re-drive is the same idempotent start — it publishes the job, and the + /// executor creates the run from it — and repeats at this interval until the job is published + /// or the run is seen to exist and to have executed (another replica started it). Default: + /// 30 seconds. + /// + public TimeSpan RedriveInterval { get; set; } = TimeSpan.FromSeconds(30); + + /// + /// How far back the scheduler looks at startup for occurrences of this schedule whose ledger + /// exists, is still , and has never been executed + /// ( is zero) — the shape a process crash between the ledger + /// commit and the job publish leaves behind, and the shape an in-process re-drive queue lost + /// with its process. Each such occurrence is re-driven (at most the 64 most recent in the + /// window). A run that is merely queued behind a busy worker looks the same and is re-driven + /// too, harmlessly: the duplicate wake-up is deduplicated by the execution lease. Default: + /// 1 hour; zero disables the probe. + /// + public TimeSpan StartupRedriveWindow { get; set; } = TimeSpan.FromHours(1); } /// @@ -42,6 +65,19 @@ internal sealed class ScheduledFlowRegistration /// A late timer fire (seconds) still starts its own occurrence — only occurrences whose successor /// is already due are skipped. /// +/// +/// A due occurrence whose start could not be published is never abandoned. Skipping +/// applies only to occurrences the loop never reached. An occurrence whose start job could not be +/// published (: the broker outage outlasted the +/// start's retry ladder — nothing was persisted, the publish is the start's commit point) is kept +/// in an in-process re-drive queue and its idempotent start repeated every +/// until the job is published. Because that +/// queue dies with the process, each loop also probes +/// of recent occurrences at startup and +/// re-drives any whose ledger is Running with zero attempts — a run whose wake-up was lost in +/// transit (an early-ACK worker subscriber, a broker that dropped it) and that nothing else will +/// find. +/// /// internal sealed class ScheduledFlowService( IDurableFlows _flows, @@ -120,6 +156,43 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) } } + /// + /// Upper bound on the in-process re-drive queue per schedule. An outage long enough to queue + /// this many undispatched occurrences (a per-minute schedule down for four hours) is an + /// operator incident; beyond it the OLDEST entries are dropped with an error log naming the + /// flow id, so they stay re-drivable by hand. + /// + internal const int MaxUndispatchedOccurrences = 256; + + /// Most recent occurrences inside the startup probe loads. + internal const int MaxStartupProbes = 64; + + private sealed class UndispatchedOccurrence + { + public required string FlowId { get; init; } + public required DateTimeOffset Occurrence { get; init; } + public required DateTimeOffset DueUtc { get; set; } + + /// + /// true when the occurrence's start job has never been published: the start's + /// publish is its commit point, so NO ledger exists for it (the shape a + /// leaves behind). A re-drive that finds no + /// ledger must then start the occurrence again, not conclude that its run has expired. + /// false for an entry the startup probe queued from an EXISTING never-executed + /// ledger, where an absent ledger on re-drive really does mean expired or deleted. + /// + public required bool AwaitingFirstPublish { get; init; } + } + + private enum RedriveOutcome + { + /// The wake-up is published, or the run no longer needs one; drop the entry. + Settled, + + /// Still undispatched; try again after . + Retry + } + private async Task RunScheduleAsync(ScheduledFlowRegistration registration, CancellationToken stoppingToken) { var timeProvider = _timeProvider ?? TimeProvider.System; @@ -140,11 +213,29 @@ private async Task RunScheduleAsync(ScheduledFlowRegistration registration, Canc "Scheduled flow '{Schedule}' ({Cron}, {TimeZone}): first occurrence at {NextOccurrence}.", registration.Name, registration.CronExpression, registration.Options.TimeZone.Id, next); + var undispatched = new List(); try { + await ProbeUndispatchedAtStartupAsync(registration, schedule, undispatched, timeProvider, stoppingToken).ConfigureAwait(false); + while (!stoppingToken.IsCancellationRequested) { - if (next is not { } occurrence) + var now = timeProvider.GetUtcNow(); + if (next is { } occurrence && occurrence <= now) + { + if (!await StartOccurrenceAsync(registration, occurrence, stoppingToken).ConfigureAwait(false)) + Enqueue(undispatched, registration, occurrence, timeProvider.GetUtcNow()); + + // Strictly after the fired occurrence, then skip anything already due (missed + // occurrences are dropped by policy, not replayed in a burst). + var resumeFrom = timeProvider.GetUtcNow(); + next = schedule.GetNextOccurrence(occurrence > resumeFrom ? occurrence : resumeFrom); + continue; + } + + await RedriveDueAsync(registration, undispatched, timeProvider, stoppingToken).ConfigureAwait(false); + + if (next is null && undispatched.Count == 0) { _logger.LogWarning( "Scheduled flow '{Schedule}' ({Cron}) has no future occurrence (unsatisfiable expression); stopping its loop.", @@ -152,20 +243,20 @@ private async Task RunScheduleAsync(ScheduledFlowRegistration registration, Canc return; } - var now = timeProvider.GetUtcNow(); - if (occurrence > now) + // Wake for whichever comes first: the next occurrence or the earliest re-drive. + var wakeAt = next ?? DateTimeOffset.MaxValue; + foreach (var entry in undispatched) { - var sleep = occurrence - now; - await Task.Delay(sleep <= MaxSleepChunk ? sleep : MaxSleepChunk, timeProvider, stoppingToken).ConfigureAwait(false); - continue; + if (entry.DueUtc < wakeAt) + wakeAt = entry.DueUtc; } - await StartOccurrenceAsync(registration, occurrence, stoppingToken).ConfigureAwait(false); - - // Strictly after the fired occurrence, then skip anything already due (missed - // occurrences are dropped by policy, not replayed in a burst). - var resumeFrom = timeProvider.GetUtcNow(); - next = schedule.GetNextOccurrence(occurrence > resumeFrom ? occurrence : resumeFrom); + now = timeProvider.GetUtcNow(); + if (wakeAt > now) + { + var sleep = wakeAt - now; + await Task.Delay(sleep <= MaxSleepChunk ? sleep : MaxSleepChunk, timeProvider, stoppingToken).ConfigureAwait(false); + } } } catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) @@ -174,7 +265,11 @@ private async Task RunScheduleAsync(ScheduledFlowRegistration registration, Canc } } - private async Task StartOccurrenceAsync( + /// + /// Starts one occurrence. Returns false only when the occurrence's ledger is committed + /// but its worker job was not published — the one outcome the loop must keep re-driving. + /// + private async Task StartOccurrenceAsync( ScheduledFlowRegistration registration, DateTimeOffset occurrence, CancellationToken stoppingToken) @@ -204,14 +299,16 @@ private async Task StartOccurrenceAsync( } catch (DurableFlowNotDispatchedException ex) { - // Worse than a failed start: the occurrence's ledger IS committed and Running, and only - // its wake-up was lost. Nothing will retry it on its own, so this is called out - // separately from the nothing-happened case below — the id is deterministic, so an - // operator (or a re-drive job) can start the same occurrence again idempotently. + // The start's publish failed after retries, so the occurrence was NOT started (the + // publish is the start's commit point; nothing was persisted). Unlike a plain failure + // this one is worth re-driving on its own: the id is deterministic and the start + // idempotent, so repeating it publishes the job once the broker is back — and if the + // publish had landed ambiguously, the same id dedupes against the run it created. _logger.LogError( ex, - "Scheduled flow '{Schedule}' persisted occurrence {FlowId} but could not publish its worker job; the run exists with no wake-up and needs to be re-driven with this id.", - registration.Name, flowId); + "Scheduled flow '{Schedule}' could not publish the start job for occurrence {FlowId}; the occurrence is not started and will be re-driven every {RedriveInterval} until it is published.", + registration.Name, flowId, registration.Options.RedriveInterval); + return false; } catch (Exception ex) { @@ -219,6 +316,194 @@ private async Task StartOccurrenceAsync( // lives on for the next one. Another replica may still have started it. _logger.LogError(ex, "Scheduled flow '{Schedule}' failed to start occurrence {FlowId}.", registration.Name, flowId); } + + return true; + } + + private void Enqueue(List undispatched, ScheduledFlowRegistration registration, DateTimeOffset occurrence, DateTimeOffset now) + { + var flowId = OccurrenceFlowId(registration.Name, occurrence); + foreach (var existing in undispatched) + { + if (string.Equals(existing.FlowId, flowId, StringComparison.Ordinal)) + return; + } + + while (undispatched.Count >= MaxUndispatchedOccurrences) + { + var dropped = undispatched[0]; + undispatched.RemoveAt(0); + _logger.LogError( + "Scheduled flow '{Schedule}' has {Count} undispatched occurrences queued for re-drive; dropping the oldest, {FlowId}. Nothing was persisted for it (the publish is the start's commit point) — start the same occurrence id by hand once the worker transport is back.", + registration.Name, MaxUndispatchedOccurrences, dropped.FlowId); + } + + undispatched.Add(new UndispatchedOccurrence + { + FlowId = flowId, + Occurrence = occurrence, + DueUtc = now + registration.Options.RedriveInterval, + AwaitingFirstPublish = true + }); + } + + private async Task RedriveDueAsync( + ScheduledFlowRegistration registration, + List undispatched, + TimeProvider timeProvider, + CancellationToken stoppingToken) + { + for (var i = 0; i < undispatched.Count;) + { + var entry = undispatched[i]; + if (entry.DueUtc > timeProvider.GetUtcNow()) + { + i++; + continue; + } + + if (await RedriveAsync(registration, entry, stoppingToken).ConfigureAwait(false) == RedriveOutcome.Retry) + { + entry.DueUtc = timeProvider.GetUtcNow() + registration.Options.RedriveInterval; + i++; + } + else + { + undispatched.RemoveAt(i); + } + } + } + + private async Task RedriveAsync( + ScheduledFlowRegistration registration, + UndispatchedOccurrence entry, + CancellationToken stoppingToken) + { + FlowState? state; + try + { + state = await _flows.GetStateAsync(entry.FlowId, stoppingToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + throw; + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Scheduled flow '{Schedule}' could not load occurrence {FlowId} to re-drive it; retrying after {RedriveInterval}.", registration.Name, entry.FlowId, registration.Options.RedriveInterval); + return RedriveOutcome.Retry; + } + + if (state is null) + { + if (!entry.AwaitingFirstPublish) + { + _logger.LogWarning("Scheduled flow '{Schedule}' occurrence {FlowId} no longer has a ledger (expired or deleted); giving up its re-drive.", registration.Name, entry.FlowId); + return RedriveOutcome.Settled; + } + + // Publish-first start: the failed publish persisted NOTHING, so "no ledger" is the + // expected shape of an occurrence still waiting for its first successful publish — not + // evidence that its run expired. Settling here (the pre-fix reading, written for the + // old create-then-publish order) permanently lost every occurrence that fell due during + // a broker outage: the queue held the id, the ledger it looked for had never existed, + // and the startup probe cannot find a run that was never persisted either. + _logger.LogInformation("Scheduled flow '{Schedule}' occurrence {FlowId} has no ledger because its start job was never published; re-driving the start.", registration.Name, entry.FlowId); + } + else if (state.Status != FlowRunStatus.Running || state.Attempts > 0) + { + // Another replica re-drove it (or its own wake-up arrived after all) and the run + // executed: nothing left to publish. + _logger.LogInformation("Scheduled flow '{Schedule}' occurrence {FlowId} has been picked up ({Status}, {Attempts} attempt(s)); no re-drive needed.", registration.Name, entry.FlowId, state.Status, state.Attempts); + return RedriveOutcome.Settled; + } + + try + { + await registration.StartOccurrenceAsync(_flows, entry.FlowId, entry.Occurrence, stoppingToken).ConfigureAwait(false); + _logger.LogInformation("Scheduled flow '{Schedule}' re-drove occurrence {FlowId}: its worker job is published.", registration.Name, entry.FlowId); + return RedriveOutcome.Settled; + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + throw; + } + catch (DurableFlowNotDispatchedException ex) + { + _logger.LogWarning(ex, "Scheduled flow '{Schedule}' could not publish the worker job for occurrence {FlowId} on re-drive; retrying after {RedriveInterval}.", registration.Name, entry.FlowId, registration.Options.RedriveInterval); + return RedriveOutcome.Retry; + } + catch (DurableFlowIdConflictException ex) + { + _logger.LogWarning(ex, "Scheduled flow '{Schedule}' occurrence {FlowId} cannot be re-driven: the input factory produced a different input than the persisted run (it is not deterministic). Giving up its re-drive; the run is still Running and needs a manual re-drive.", registration.Name, entry.FlowId); + return RedriveOutcome.Settled; + } + catch (Exception ex) + { + _logger.LogError(ex, "Scheduled flow '{Schedule}' failed to re-drive occurrence {FlowId}; retrying after {RedriveInterval}.", registration.Name, entry.FlowId, registration.Options.RedriveInterval); + return RedriveOutcome.Retry; + } + } + + /// + /// Finds recent occurrences whose ledger is committed and Running with zero attempts — never + /// executed — and queues them for an immediate re-drive: a start whose job was published and + /// then lost in transit (an early-ACK worker subscriber, a broker that dropped it), which + /// nothing else would ever look for, because the store has no enumeration. The in-process + /// re-drive queue above does not survive a restart, and an occurrence whose publish was still + /// failing when the process died left nothing persisted — this probe cannot find it either, so + /// it is skipped like any other occurrence missed while no replica was up (documented; the run + /// history shows the gap). Best-effort: a failed load ends the probe (the loop starts + /// regardless). + /// + private async Task ProbeUndispatchedAtStartupAsync( + ScheduledFlowRegistration registration, + CronSchedule schedule, + List undispatched, + TimeProvider timeProvider, + CancellationToken stoppingToken) + { + var window = registration.Options.StartupRedriveWindow; + if (window <= TimeSpan.Zero) + return; + + var now = timeProvider.GetUtcNow(); + var cursor = window >= now - DateTimeOffset.UnixEpoch ? DateTimeOffset.UnixEpoch : now - window; + var candidates = new List(); + while (schedule.GetNextOccurrence(cursor) is { } occurrence && occurrence <= now) + { + candidates.Add(occurrence); + if (candidates.Count > MaxStartupProbes) + candidates.RemoveAt(0); + cursor = occurrence; + } + + foreach (var occurrence in candidates) + { + var flowId = OccurrenceFlowId(registration.Name, occurrence); + FlowState? state; + try + { + state = await _flows.GetStateAsync(flowId, stoppingToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + throw; + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Scheduled flow '{Schedule}' could not probe occurrence {FlowId} for an undispatched run at startup; skipping the rest of the probe.", registration.Name, flowId); + return; + } + + if (state is not { Status: FlowRunStatus.Running, Attempts: 0 }) + continue; + + _logger.LogWarning( + "Scheduled flow '{Schedule}' found occurrence {FlowId} committed but never executed (Running, 0 attempts) — its worker job was lost before publish (a crash or an outage in a previous process). Re-driving it.", + registration.Name, flowId); + undispatched.Add(new UndispatchedOccurrence { FlowId = flowId, Occurrence = occurrence, DueUtc = now, AwaitingFirstPublish = false }); + } } internal static string OccurrenceFlowId(string name, DateTimeOffset occurrence) diff --git a/src/AsyncResponse.Core/SerialExecutorRegistry.cs b/src/AsyncResponse.Core/SerialExecutorRegistry.cs index f9cb20fb5..efbf901e1 100644 --- a/src/AsyncResponse.Core/SerialExecutorRegistry.cs +++ b/src/AsyncResponse.Core/SerialExecutorRegistry.cs @@ -182,6 +182,67 @@ public async ValueTask EnqueueAsync(string channel, Func work, Cance } } + /// The outcome of a non-blocking . + public enum TryEnqueueOutcome + { + /// Accepted by the channel's live executor. + Accepted, + + /// + /// Not accepted right now — the executor's bounded queue is full, or the channel's executor + /// is mid-retirement. The work was not queued; the producer should come back later. + /// + Full, + + /// Suppressed by a tombstone (retired executor, no registration left): nothing will ever run it. + Suppressed + } + + /// + /// Non-blocking counterpart of : never waits for queue capacity or + /// for a retirement to finish. Built for the DB channels' process-wide dispatch sweep, which + /// walks every subscribed correlation id in turn: awaiting one correlation id's capacity there + /// parked the whole loop — a single waiter wedged in a slow completion predicate, fed a + /// backlog of NEW progress messages, stopped every other correlation id's delivery until its + /// executor drained. A result leaves the message + /// unclaimed in the store for a later rescan of that one correlation id. + /// + public TryEnqueueOutcome TryEnqueue(string channel, Func work) + { + ArgumentException.ThrowIfNullOrWhiteSpace(channel); + ArgumentNullException.ThrowIfNull(work); + + lock (_gate) + { + if (!_executors.TryGetValue(channel, out var current)) + { + // Same tombstone rule as EnqueueAsync: no registration and a live tombstone means + // the work would run against no subscription; recreating an executor would leak it. + if (!_registrations.ContainsKey(channel) && IsTombstonedUnderLock(channel)) + { + _logger.LogWarning( + "Suppressed a delivery for channel {Channel}: the channel is tombstoned and has no registered subscription.", + channel); + return TryEnqueueOutcome.Suppressed; + } + + current = new ExecutorEntry(new ChannelSerialExecutor(_logger, channel)); + _executors[channel] = current; + } + + // Mid-retirement: EnqueueAsync would wait for the drain and then recreate; a + // non-blocking caller simply comes back after it. + if (current.Retiring) + return TryEnqueueOutcome.Full; + + // TryWrite is synchronous and never blocks, so it can run under the gate; no in-flight + // enqueue bookkeeping is needed because nothing is left waiting for capacity. + return current.Executor.TryEnqueue(work, logIfFull: false) + ? TryEnqueueOutcome.Accepted + : TryEnqueueOutcome.Full; + } + } + /// /// Retires the channel's serial executor (if present), draining its queued work. Safe to call /// concurrently with : admitted enqueues finish against the retiring diff --git a/src/AsyncResponse.Core/ServiceCollectionExtensions.cs b/src/AsyncResponse.Core/ServiceCollectionExtensions.cs index ded5f5c9f..2cd558a59 100644 --- a/src/AsyncResponse.Core/ServiceCollectionExtensions.cs +++ b/src/AsyncResponse.Core/ServiceCollectionExtensions.cs @@ -1,5 +1,6 @@ using AsyncResponse; using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Options; using System.Diagnostics.CodeAnalysis; namespace Microsoft.Extensions.DependencyInjection; @@ -46,7 +47,9 @@ public static AsyncResponseRegistrationBuilder AddAsyncResponse( provider.GetService(), provider.GetService(), provider.GetRequiredService(), - provider.GetService())); + provider.GetService(), + // The producer-side mirror of the ingress's inbound size budget (WorkerJobTooLargeException). + provider.GetService>())); // Fail fast before background services do any real work if the required channel, // transport, and durable-flow store choices were not made explicitly. TryAddEnumerable @@ -196,6 +199,11 @@ public static AsyncResponseRegistrationBuilder AddAsyncResponse( var options = new ScheduledFlowOptions(); configure?.Invoke(options); ArgumentNullException.ThrowIfNull(options.TimeZone, $"{nameof(ScheduledFlowOptions)}.{nameof(ScheduledFlowOptions.TimeZone)}"); + // The re-drive interval arms a Task.Delay, so it gets the timer ceiling; the startup window + // is only ever subtracted from "now" (zero disables the probe). + AsyncResponseChannelOptions.EnsureTimerBacked(options.RedriveInterval, nameof(ScheduledFlowOptions), nameof(ScheduledFlowOptions.RedriveInterval)); + if (options.StartupRedriveWindow < TimeSpan.Zero) + throw new ArgumentException($"{nameof(ScheduledFlowOptions)}.{nameof(ScheduledFlowOptions.StartupRedriveWindow)} cannot be negative (zero disables the startup probe).", nameof(configure)); // Validate the FINAL occurrence id against the WHOLE portable contract now — length, // bytes, characters, surrounding spaces — by running the id the scheduler will actually @@ -347,7 +355,9 @@ public static AsyncResponseRegistrationBuilder WithInMemoryChannel( provider.GetService(), provider.GetService(), provider.GetRequiredService(), - provider.GetService()))); + provider.GetService(), + // The producer-side mirror of the ingress's inbound size budget (WorkerJobTooLargeException). + provider.GetService>()))); services.Replace(ServiceDescriptor.Singleton(provider => provider.GetRequiredService())); // The resolved default waiter timeout is declared through the marker so the startup diff --git a/src/AsyncResponse.Testing/AsyncResponseTestHarness.cs b/src/AsyncResponse.Testing/AsyncResponseTestHarness.cs index bfbd3aab6..af0558dda 100644 --- a/src/AsyncResponse.Testing/AsyncResponseTestHarness.cs +++ b/src/AsyncResponse.Testing/AsyncResponseTestHarness.cs @@ -37,6 +37,22 @@ public sealed class AsyncResponseTestHarnessOptions /// public TimeSpan RealTimeGuard { get; set; } = TimeSpan.FromSeconds(10); + /// + /// What does when user code is + /// still executing after the old incarnation's graceful stop lapsed + /// (): a step body that ignored its cancellation and is blocked on + /// something the test controls, for example. A simulated restart is cooperative — it + /// discards the process-bound state a crash would lose, but it cannot terminate a running + /// delegate the way a process kill does — so such an execution would keep running beside the + /// new incarnation and perform its side effects after the "restart" returned, proving less + /// than the test claims. Default (false): the restart fails with + /// naming the count. true: the executions are + /// abandoned (their leases broken, their provider disposed) and the restart proceeds; the + /// test then owns the overlap. Engine-owned parks — an awaited step or an in-process timer + /// holding its worker slot on the virtual clock — are not user code and never trip this. + /// + public bool AbandonLingeringExecutionsOnRestart { get; set; } + /// /// Flow-execution observers installed into every incarnation (the current one and each /// simulated restart). installs its probe here. @@ -195,6 +211,12 @@ public async Task WaitForWorkerIdleAsync() /// makes cron schedules skip the occurrences that fell into the downtime, exactly as a real /// outage would. /// + /// + /// User code of the old incarnation was still executing after the graceful stop lapsed and + /// is off: + /// the restart is cooperative and cannot kill that code, so it refuses to report a restart + /// the surviving execution would contradict. + /// public async Task SimulateRestartAsync(Action? whileDown = null) { ObjectDisposedException.ThrowIf(_disposed, this); @@ -206,6 +228,29 @@ public async Task SimulateRestartAsync(Action? whileDown = null) // Resolved BEFORE the provider goes away; abandoned after, once nothing can add to it. var dyingChannel = _provider.GetService(); await StopHostedServicesAsync().ConfigureAwait(false); + + // Quiescence check BEFORE the provider is discarded. Jobs still outstanding after the + // bounded stop are executions the stop could not end. Engine-owned parks (an awaited step + // or an in-process timer holding its worker slot on the virtual clock) are expected — + // their leases are broken below and the new incarnation takes them over, as after a real + // crash. Anything beyond them is USER code still running: this restart cannot terminate + // it (there is no process to kill), so reporting a restart while it keeps executing — + // and performs side effects after the restart "completed" — would prove less than the + // test claims. Refuse unless the test opted into owning that overlap. + var lingering = Transport.OutstandingJobs + _quiesce.DirectRunsInFlight - _quiesce.ParkedCount; + if (lingering > 0 && !_options.AbandonLingeringExecutionsOnRestart) + { + throw new InvalidOperationException( + $"{nameof(SimulateRestartAsync)} could not establish quiescence: {lingering} execution(s) of the old incarnation " + + $"were still running user code after the graceful stop lapsed ({_options.RealTimeGuard} of real time). A simulated " + + "restart is cooperative — it cannot terminate a running delegate the way a process kill does — so that code would " + + "keep running beside the new incarnation and perform its side effects after the restart. Let the step observe its " + + "cancellation token or finish before restarting, inject a crash at the checkpoint boundary with " + + "FlowTestHarness.CrashBeforeStep/CrashAfterStep, or set " + + $"{nameof(AsyncResponseTestHarnessOptions)}.{nameof(AsyncResponseTestHarnessOptions.AbandonLingeringExecutionsOnRestart)} " + + "to accept the overlap."); + } + await _provider.DisposeAsync().ConfigureAwait(false); // Hard-crash semantics for whatever survived the graceful stop: a parked execution's diff --git a/src/AsyncResponse.Testing/PublicAPI.Unshipped.txt b/src/AsyncResponse.Testing/PublicAPI.Unshipped.txt index 3f1401046..bf378aca6 100644 --- a/src/AsyncResponse.Testing/PublicAPI.Unshipped.txt +++ b/src/AsyncResponse.Testing/PublicAPI.Unshipped.txt @@ -89,3 +89,5 @@ static AsyncResponse.Testing.FlowTestHarness.StartAsync(System.Action System.DateTimeOffset ~override AsyncResponse.Testing.FlowProbeEvent.Equals(object obj) -> bool ~override AsyncResponse.Testing.FlowProbeEvent.ToString() -> string +AsyncResponse.Testing.AsyncResponseTestHarnessOptions.AbandonLingeringExecutionsOnRestart.get -> bool +AsyncResponse.Testing.AsyncResponseTestHarnessOptions.AbandonLingeringExecutionsOnRestart.set -> void diff --git a/src/Channels/AsyncResponse.Channels.MongoDB/MongoDbChannelStore.cs b/src/Channels/AsyncResponse.Channels.MongoDB/MongoDbChannelStore.cs index fc27269d3..a4b9e76d4 100644 --- a/src/Channels/AsyncResponse.Channels.MongoDB/MongoDbChannelStore.cs +++ b/src/Channels/AsyncResponse.Channels.MongoDB/MongoDbChannelStore.cs @@ -9,10 +9,17 @@ namespace AsyncResponse.Channels.MongoDB; +/// One stored response envelope row/document as the channel store returns it. +/// +/// EnvelopeJson is the stored envelope, or null for a document the dispatch sweep loaded header-only (an +/// already-acknowledged one — see ); the +/// sweep hydrates the few such documents it still has to deliver through +/// before handing them to a waiter. +/// internal readonly record struct MongoDbChannelMessage( Guid Id, string CorrelationId, - string EnvelopeJson, + string? EnvelopeJson, DateTimeOffset CreatedAtUtc, DateTimeOffset? AckedAtUtc = null, long? AckedSeq = null); @@ -441,11 +448,74 @@ public async Task> LoadMessagesAsync( Builders.Filter.Gt(item => item.Id, cursorId))); } var documents = await _messages.Find(filter) + .Project(SweepProjection) .Sort(Builders.Sort .Ascending(item => item.CreatedAtUtc) .Ascending(item => item.Id)) .Limit(batchSize) .ToListAsync(cancellationToken).ConfigureAwait(false); + return ToMessages(documents); + } + + /// + /// The sweep's projection: every field but the envelope, and the envelope only for a document + /// nobody has acknowledged yet. Acknowledged documents are the consumed history the sweep + /// re-reads on every tick (they stay in the result so a fan-out waiter in ANOTHER process + /// still receives them): shipping their bodies with each sweep made a long-lived progress + /// subscription's cost grow with its whole retained history. The shared sweep fetches the + /// envelope by id for the rare acknowledged document a live subscription has not seen. + /// $ifNull folds a missing acked_at (a pre-settlement document) into null. + /// + internal static readonly ProjectionDefinition SweepProjection = + new BsonDocumentProjectionDefinition(new BsonDocument + { + ["_id"] = 1, + ["correlation_id"] = 1, + ["created_at"] = 1, + ["expires_at"] = 1, + ["acked_at"] = 1, + ["acked_seq"] = 1, + ["recovery_claimed"] = 1, + ["envelope_json"] = new BsonDocument("$cond", new BsonArray + { + new BsonDocument("$eq", new BsonArray + { + new BsonDocument("$ifNull", new BsonArray { "$acked_at", BsonNull.Value }), + BsonNull.Value + }), + "$envelope_json", + BsonNull.Value + }) + }); + + /// + /// The full documents (envelope included) for under + /// , in sweep order — how the dispatch sweep hydrates the + /// header-only acknowledged documents it still has to deliver. A document reaped between the + /// sweep's page and this read is simply absent. + /// + public async Task> LoadMessagesByIdAsync( + string correlationId, + IReadOnlyList ids, + CancellationToken cancellationToken) + { + if (ids.Count == 0) + return []; + + await EnsureCreatedAsync(cancellationToken).ConfigureAwait(false); + var filter = Builders.Filter.Eq(item => item.CorrelationId, correlationId) + & Builders.Filter.In(item => item.Id, ids) + & NotExpiredOnServerClock(); + var documents = await _messages.Find(filter) + .Sort(Builders.Sort + .Ascending(item => item.CreatedAtUtc) + .Ascending(item => item.Id)) + .ToListAsync(cancellationToken).ConfigureAwait(false); + return ToMessages(documents); + } + + private static List ToMessages(List documents) + { var messages = new List(documents.Count); foreach (var document in documents) messages.Add(new MongoDbChannelMessage( @@ -804,8 +874,9 @@ internal sealed class MongoChannelMessageDocument [BsonElement("correlation_id")] public string CorrelationId { get; set; } = ""; + /// Null only on a sweep projection of an acknowledged document (). [BsonElement("envelope_json")] - public string EnvelopeJson { get; set; } = ""; + public string? EnvelopeJson { get; set; } = ""; [BsonElement("created_at")] public DateTime CreatedAtUtc { get; set; } diff --git a/src/Channels/AsyncResponse.Channels.MongoDB/MongoDbRecoveryStateStore.cs b/src/Channels/AsyncResponse.Channels.MongoDB/MongoDbRecoveryStateStore.cs index a887896e4..a9918dc09 100644 --- a/src/Channels/AsyncResponse.Channels.MongoDB/MongoDbRecoveryStateStore.cs +++ b/src/Channels/AsyncResponse.Channels.MongoDB/MongoDbRecoveryStateStore.cs @@ -1,6 +1,7 @@ using Microsoft.Extensions.Logging; using System.Runtime.CompilerServices; using System.Text.Json; +using System.Text.Json.Serialization.Metadata; namespace AsyncResponse.Channels.MongoDB; @@ -95,11 +96,19 @@ private IReadOnlyList DeserializeStates(IReadOnlyList jso return states; } + /// The registration's metadata off the library's resolver — case-sensitive matching, as before. + private static readonly JsonTypeInfo _stateTypeInfo = + AsyncResponseJson.GetTypeInfo(AsyncResponseJson.Default); + private RecoveryState? DeserializeState(string json, string? correlationId, ref int unreadable) { try { - var state = AsyncResponseJson.Deserialize(json); + // Through JsonSafety, not the raw reader: the exception logged below is the body-free + // rebuild (size and position). The reader's own appends `Path: $.Context['']` + // built from the stored registration's context keys — tenant and auth baggage — which + // the warning then carried into the application log. + var state = JsonSafety.SafeDeserialize(json, _stateTypeInfo); if (state is null) { unreadable++; @@ -137,7 +146,7 @@ private IReadOnlyList DeserializeStates(IReadOnlyList jso return state; } - catch (JsonException ex) + catch (Exception ex) when (ex is JsonException or InvalidDataException) { _logger.LogWarning(ex, "Unreadable MongoDB recovery state for correlationId {CorrelationId}; skipping.", correlationId); unreadable++; diff --git a/src/Channels/AsyncResponse.Channels.MongoDB/ServiceCollectionExtensions.cs b/src/Channels/AsyncResponse.Channels.MongoDB/ServiceCollectionExtensions.cs index 8f8c61c57..4157fee06 100644 --- a/src/Channels/AsyncResponse.Channels.MongoDB/ServiceCollectionExtensions.cs +++ b/src/Channels/AsyncResponse.Channels.MongoDB/ServiceCollectionExtensions.cs @@ -77,7 +77,9 @@ public static AsyncResponseRegistrationBuilder WithMongoDbChannel( provider.GetService(), provider.GetService(), provider.GetRequiredService(), - provider.GetService()))); + provider.GetService(), + // The producer-side mirror of the ingress's inbound size budget (WorkerJobTooLargeException). + provider.GetService>()))); services.Replace(ServiceDescriptor.Singleton(provider => provider.GetRequiredService())); // The resolved default waiter timeout is declared through the marker so the startup diff --git a/src/Channels/AsyncResponse.Channels.NATS/NatsAsyncResponseChannel.cs b/src/Channels/AsyncResponse.Channels.NATS/NatsAsyncResponseChannel.cs index 81abf0c44..515976eb3 100644 --- a/src/Channels/AsyncResponse.Channels.NATS/NatsAsyncResponseChannel.cs +++ b/src/Channels/AsyncResponse.Channels.NATS/NatsAsyncResponseChannel.cs @@ -388,7 +388,10 @@ async Task ProcessResponseAsync(string? payload) return; } - var envelope = JsonSerializer.Deserialize(payload, AsyncResponseEnvelopeJson.TypeInfo()); + // JsonSafety, not the raw reader: a parse failure is logged below and handed to the + // waiter, and the reader's own message quotes inbound property names and dictionary + // keys (docs/security.md, "never logs a message body"). Size and position only. + var envelope = JsonSafety.SafeDeserialize(payload, AsyncResponseEnvelopeJson.TypeInfo()); if (envelope == null) { diff --git a/src/Channels/AsyncResponse.Channels.NATS/NatsRecoveryStateStore.cs b/src/Channels/AsyncResponse.Channels.NATS/NatsRecoveryStateStore.cs index bc5f7fa37..30bf46caa 100644 --- a/src/Channels/AsyncResponse.Channels.NATS/NatsRecoveryStateStore.cs +++ b/src/Channels/AsyncResponse.Channels.NATS/NatsRecoveryStateStore.cs @@ -383,9 +383,13 @@ private bool IsStateReadable(RecoveryState? state, string key, string correlatio { try { - return JsonSerializer.Deserialize(json, _envelopeTypeInfo); + // Through JsonSafety, not the raw reader: the exception logged below is the body-free + // rebuild (size and position). The reader's own appends `Path: $.States[0].Context['']` + // built from the stored registration's context keys — tenant and auth baggage — which + // this warning then carried into the application log. + return JsonSafety.SafeDeserialize(json, _envelopeTypeInfo); } - catch (JsonException ex) + catch (Exception ex) when (ex is JsonException or InvalidDataException) { _logger.LogWarning(ex, "Unreadable recovery state at key {RecoveryKey}; skipping.", key); return null; diff --git a/src/Channels/AsyncResponse.Channels.NATS/ServiceCollectionExtensions.cs b/src/Channels/AsyncResponse.Channels.NATS/ServiceCollectionExtensions.cs index 206df6c33..c837956d4 100644 --- a/src/Channels/AsyncResponse.Channels.NATS/ServiceCollectionExtensions.cs +++ b/src/Channels/AsyncResponse.Channels.NATS/ServiceCollectionExtensions.cs @@ -69,7 +69,9 @@ public static AsyncResponseRegistrationBuilder WithNatsChannel( provider.GetService(), provider.GetService(), provider.GetRequiredService(), - provider.GetService()))); + provider.GetService(), + // The producer-side mirror of the ingress's inbound size budget (WorkerJobTooLargeException). + provider.GetService>()))); services.Replace(ServiceDescriptor.Singleton(provider => provider.GetRequiredService())); // The resolved default waiter timeout is declared through the marker so the startup diff --git a/src/Channels/AsyncResponse.Channels.PostgreSQL/PostgreSqlChannelSql.cs b/src/Channels/AsyncResponse.Channels.PostgreSQL/PostgreSqlChannelSql.cs index 8a75730c6..84932c940 100644 --- a/src/Channels/AsyncResponse.Channels.PostgreSQL/PostgreSqlChannelSql.cs +++ b/src/Channels/AsyncResponse.Channels.PostgreSQL/PostgreSqlChannelSql.cs @@ -6,10 +6,17 @@ namespace AsyncResponse.Channels.PostgreSQL; +/// One stored response envelope row/document as the channel store returns it. +/// +/// EnvelopeJson is the stored envelope, or null for a row the dispatch sweep loaded header-only (an +/// already-acknowledged row — see ); the +/// sweep hydrates the few such rows it still has to deliver through +/// before handing them to a waiter. +/// internal readonly record struct PostgreSqlChannelMessage( Guid Id, string CorrelationId, - string EnvelopeJson, + string? EnvelopeJson, DateTimeOffset CreatedAtUtc, DateTimeOffset? AckedAtUtc = null, long? AckedSeq = null); @@ -471,9 +478,15 @@ public async Task> LoadMessagesAsync( await EnsureCreatedAsync(cancellationToken).ConfigureAwait(false); await using var connection = await _dataSource.OpenConnectionAsync(cancellationToken).ConfigureAwait(false); await using var command = connection.CreateCommand(); + // The envelope travels only for rows nobody has acknowledged yet. Acknowledged rows are + // the consumed history the sweep re-reads on every tick (they stay in the result set so a + // fan-out waiter in ANOTHER process still receives them): shipping their bodies with each + // sweep made a long-lived progress subscription's cost grow with its whole retained + // history. The shared sweep fetches the envelope by id for the rare acknowledged row a + // live subscription has not seen. command.CommandText = $""" - SELECT id, correlation_id, envelope_json::text, created_at, acked_at, acked_seq + SELECT id, correlation_id, CASE WHEN acked_at IS NULL THEN envelope_json::text END, created_at, acked_at, acked_seq FROM {MessageTable} WHERE correlation_id = @correlation_id AND created_at >= @since @@ -491,13 +504,49 @@ public async Task> LoadMessagesAsync( command.Parameters.AddWithValue("after_id", afterId ?? throw new ArgumentNullException(nameof(afterId))); } - var messages = new List(batchSize); + return await ReadMessagesAsync(command, batchSize, cancellationToken).ConfigureAwait(false); + } + + /// + /// The full rows (envelope included) for under + /// , in sweep order — how the dispatch sweep hydrates the + /// header-only acknowledged rows it still has to deliver. A row pruned between the sweep's + /// page and this read is simply absent. + /// + public async Task> LoadMessagesByIdAsync( + string correlationId, + IReadOnlyList ids, + CancellationToken cancellationToken) + { + if (ids.Count == 0) + return []; + + await EnsureCreatedAsync(cancellationToken).ConfigureAwait(false); + await using var connection = await _dataSource.OpenConnectionAsync(cancellationToken).ConfigureAwait(false); + await using var command = connection.CreateCommand(); + command.CommandText = + $""" + SELECT id, correlation_id, envelope_json::text, created_at, acked_at, acked_seq + FROM {MessageTable} + WHERE correlation_id = @correlation_id + AND id = ANY(@ids) + AND expires_at > now() + ORDER BY created_at, id; + """; + command.Parameters.AddWithValue("correlation_id", correlationId); + command.Parameters.AddWithValue("ids", ids is Guid[] array ? array : [.. ids]); + return await ReadMessagesAsync(command, ids.Count, cancellationToken).ConfigureAwait(false); + } + + private static async Task> ReadMessagesAsync(NpgsqlCommand command, int capacity, CancellationToken cancellationToken) + { + var messages = new List(capacity); await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) messages.Add(new PostgreSqlChannelMessage( reader.GetGuid(0), reader.GetString(1), - reader.GetString(2), + reader.IsDBNull(2) ? null : reader.GetString(2), reader.GetFieldValue(3), reader.IsDBNull(4) ? null : reader.GetFieldValue(4), reader.IsDBNull(5) ? null : reader.GetInt64(5))); diff --git a/src/Channels/AsyncResponse.Channels.PostgreSQL/PostgreSqlRecoveryStateStore.cs b/src/Channels/AsyncResponse.Channels.PostgreSQL/PostgreSqlRecoveryStateStore.cs index 75c9c58b1..43ae99531 100644 --- a/src/Channels/AsyncResponse.Channels.PostgreSQL/PostgreSqlRecoveryStateStore.cs +++ b/src/Channels/AsyncResponse.Channels.PostgreSQL/PostgreSqlRecoveryStateStore.cs @@ -1,6 +1,7 @@ using Microsoft.Extensions.Logging; using System.Runtime.CompilerServices; using System.Text.Json; +using System.Text.Json.Serialization.Metadata; namespace AsyncResponse.Channels.PostgreSQL; @@ -94,11 +95,19 @@ private IReadOnlyList DeserializeStates(IReadOnlyList jso return states; } + /// The registration's metadata off the library's resolver — case-sensitive matching, as before. + private static readonly JsonTypeInfo _stateTypeInfo = + AsyncResponseJson.GetTypeInfo(AsyncResponseJson.Default); + private RecoveryState? DeserializeState(string json, string? correlationId, ref int unreadable) { try { - var state = AsyncResponseJson.Deserialize(json); + // Through JsonSafety, not the raw reader: the exception logged below is the body-free + // rebuild (size and position). The reader's own appends `Path: $.Context['']` + // built from the stored registration's context keys — tenant and auth baggage — which + // the warning then carried into the application log. + var state = JsonSafety.SafeDeserialize(json, _stateTypeInfo); if (state is null) { unreadable++; @@ -136,7 +145,7 @@ private IReadOnlyList DeserializeStates(IReadOnlyList jso return state; } - catch (JsonException ex) + catch (Exception ex) when (ex is JsonException or InvalidDataException) { _logger.LogWarning(ex, "Unreadable PostgreSQL recovery state for correlationId {CorrelationId}; skipping.", correlationId); unreadable++; diff --git a/src/Channels/AsyncResponse.Channels.PostgreSQL/ServiceCollectionExtensions.cs b/src/Channels/AsyncResponse.Channels.PostgreSQL/ServiceCollectionExtensions.cs index da3779182..77657acf6 100644 --- a/src/Channels/AsyncResponse.Channels.PostgreSQL/ServiceCollectionExtensions.cs +++ b/src/Channels/AsyncResponse.Channels.PostgreSQL/ServiceCollectionExtensions.cs @@ -1,6 +1,7 @@ using AsyncResponse; using AsyncResponse.Channels.PostgreSQL; using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Options; namespace Microsoft.Extensions.DependencyInjection; @@ -41,7 +42,9 @@ public static AsyncResponseRegistrationBuilder WithPostgreSqlChannel( provider.GetService(), provider.GetService(), provider.GetRequiredService(), - provider.GetService()))); + provider.GetService(), + // The producer-side mirror of the ingress's inbound size budget (WorkerJobTooLargeException). + provider.GetService>()))); services.Replace(ServiceDescriptor.Singleton(provider => provider.GetRequiredService())); // The resolved default waiter timeout is declared through the marker so the startup diff --git a/src/Channels/AsyncResponse.Channels.Redis/RedisAsyncResponseChannel.cs b/src/Channels/AsyncResponse.Channels.Redis/RedisAsyncResponseChannel.cs index 3df0126a4..e65ed13b6 100644 --- a/src/Channels/AsyncResponse.Channels.Redis/RedisAsyncResponseChannel.cs +++ b/src/Channels/AsyncResponse.Channels.Redis/RedisAsyncResponseChannel.cs @@ -319,6 +319,9 @@ private sealed class RedisSubscription where T : IAsyncResponsePayload // Ensures unsubscribe, recovery-state delete, timeout disposal, and executor cleanup // happen once no matter whether completion, timeout, or waiter disposal got there first. private int _cleanupStarted; + + // Set by the overload fault: every message still queued behind it is skipped unprocessed. + private int _overloaded; private readonly object _cleanupGate = new(); private Task? _cleanupTask; @@ -391,19 +394,60 @@ await DrainThenCleanupAsync( }); /// - /// Receives pub/sub messages from the async subscription and enqueues them on the - /// per-channel executor, awaiting admission so executor backpressure reaches the - /// subscription's message loop instead of blocking a Redis reader thread. + /// Receives pub/sub messages from the async subscription and admits them to the + /// per-channel serial executor WITHOUT waiting for capacity. Redis pub/sub is + /// fire-and-forget: the publisher is never backpressured, and the SDK's + /// ChannelMessageQueue behind this callback is unbounded — so an earlier version + /// that awaited executor admission here did not slow anything down, it only moved the + /// backlog from the bounded executor into that unbounded SDK queue, where a progress-message + /// burst behind a slow Until predicate could grow process memory until failure. The + /// executor's capacity ( messages per + /// correlation id) is now the whole buffer: a message that finds it full faults the wait + /// as indeterminate () instead of being buffered without + /// bound — and never silently dropped, since a terminal response may be among the queued ones. /// public Task HandleMessageAsync(RedisChannel messageChannel, RedisValue messageValue) { // The registry coordinates create/enqueue/retire under one lock, so the message is never // enqueued onto an executor that is concurrently being torn down (no lost messages) and a // correlation-id reused mid-drain never produces two live executors for one channel. - var enqueue = _owner._executors.EnqueueAsync( - ChannelName, - () => ProcessUnderCapturedContextAsync(messageValue)); - return enqueue.IsCompletedSuccessfully ? Task.CompletedTask : enqueue.AsTask(); + return _owner._executors.TryEnqueue(ChannelName, () => ProcessUnderCapturedContextAsync(messageValue)) switch + { + // Suppressed = a tombstoned channel with no registration left: the wait is gone and + // the message would run against nobody (EnqueueAsync dropped these the same way). + SerialExecutorRegistry.TryEnqueueOutcome.Accepted or SerialExecutorRegistry.TryEnqueueOutcome.Suppressed + => Task.CompletedTask, + _ => OnOverloadedAsync() + }; + } + + /// + /// The overload outcome: the bounded per-correlation-id buffer is full and the next response + /// cannot be admitted. Faults the wait with the explicit indeterminate contract (a terminal + /// response may be queued or may be the one refused) and tears the subscription down so the + /// flood stops here. Deliberately rather than the drain: the + /// executor is full, and parking on a drain marker would block the subscriber's message + /// loop — exactly the unbounded buffering this refuses. A full executor that is merely + /// mid-retirement means cleanup already settled the task, and the message is a straggler. + /// + private async Task OnOverloadedAsync() + { + var overload = new AsyncResponseIndeterminateDeliveryException(_correlationId, ChannelSerialExecutor.DefaultCapacity); + Interlocked.Exchange(ref _overloaded, 1); + if (!_tcs.TrySetException(overload)) + { + if (_owner._logger.IsEnabled(LogLevel.Debug)) + _owner._logger.LogDebug("Dropped a late message on channel {Channel}: the wait for correlationId {CorrelationId} is already settled and its executor retiring.", ChannelName, _correlationId); + return; + } + + _owner._logger.LogError( + "Wait for correlationId {CorrelationId} is overloaded: {Buffered} responses are queued behind its serial processing and the next could not be admitted. Faulting it as indeterminate and unsubscribing; the queued responses are discarded with it.", + _correlationId, + ChannelSerialExecutor.DefaultCapacity); + AsyncResponseDiagnostics.SetError(_activity, "overloaded", "The wait's bounded response buffer overflowed."); + AsyncResponseDiagnostics.RecordWaiterOverload("redis"); + await CleanupOnceAsync().ConfigureAwait(false); } /// @@ -430,6 +474,18 @@ async Task ProcessAsync() /// Deserializes and handles a single incoming envelope, completes the TCS when terminal. private async Task ProcessMessageAsync(RedisValue messageValue) { + if (Volatile.Read(ref _overloaded) != 0) + { + // Queued behind the overload fault: the wait is settled as indeterminate and the + // subscription torn down, so running the predicate would spend user code — up to a + // full executor's worth of it — on an outcome that cannot change. Only the overload + // skips: a message admitted ahead of an ordinary terminal settlement still runs, as + // the retirement drain expects. + if (_owner._logger.IsEnabled(LogLevel.Debug)) + _owner._logger.LogDebug("Dropped a queued message on channel {Channel}: the wait for correlationId {CorrelationId} was faulted as overloaded.", ChannelName, _correlationId); + return; + } + _owner._logger.LogDebug("Received message on channel {Channel}.", ChannelName); bool finished = false; @@ -437,8 +493,12 @@ private async Task ProcessMessageAsync(RedisValue messageValue) { // The delivered value is UTF-8 bytes; deserializing them directly avoids the // ToString() detour, which paid a payload-sized UTF-16 allocation plus a - // transcode both ways on every message. - var envelope = JsonSerializer.Deserialize((ReadOnlySpan)(byte[]?)messageValue, AsyncResponseEnvelopeJson.TypeInfo()); + // transcode both ways on every message. Through JsonSafety, not the raw reader: + // a parse failure lands in the catch below, which logs it AND hands it to the + // waiter, and the reader's own message quotes the inbound body — property names + // and dictionary keys straight off the wire (docs/security.md, "never logs a + // message body"). Only the size and position survive. + var envelope = JsonSafety.SafeDeserialize((ReadOnlySpan)(byte[]?)messageValue, AsyncResponseEnvelopeJson.TypeInfo()); if (envelope == null) { diff --git a/src/Channels/AsyncResponse.Channels.Redis/RedisChannelSubscriber.cs b/src/Channels/AsyncResponse.Channels.Redis/RedisChannelSubscriber.cs index f9012ea82..f076e2c66 100644 --- a/src/Channels/AsyncResponse.Channels.Redis/RedisChannelSubscriber.cs +++ b/src/Channels/AsyncResponse.Channels.Redis/RedisChannelSubscriber.cs @@ -8,10 +8,16 @@ internal interface IRedisChannelSubscription : IAsyncDisposable; /// /// Async-capable subscribe seam over StackExchange.Redis pub/sub. The channel consumes this instead /// of 's -/// synchronous callback so message handling can await the per-channel serial executor (bounded -/// backpressure) without sync-over-async blocking a Redis reader thread. Also the unit-test seam: -/// is sealed with no public constructor, so tests fake this -/// interface rather than the queue. +/// synchronous callback — whose handlers the SDK runs on pool threads with no ordering — so +/// messages reach the channel one at a time, in order, off any Redis reader thread. Also the +/// unit-test seam: is sealed with no public constructor, so +/// tests fake this interface rather than the queue. +/// +/// The handler must not wait for downstream capacity: the SDK queue behind this seam is +/// unbounded, so a handler parked on admission does not backpressure the publisher (Redis +/// pub/sub has none), it only lets that queue grow. The channel admits non-blockingly and faults +/// the wait as indeterminate when its bounded buffer is full. +/// /// internal interface IRedisChannelSubscriber { @@ -25,8 +31,8 @@ internal interface IRedisChannelSubscriber /// /// Production over : a /// per subscription, whose OnMessage(Func<…, Task>) -/// loop awaits the handler — preserving per-channel ordering while propagating executor -/// backpressure to the queue instead of blocking a reader thread. +/// loop awaits the handler — preserving per-channel ordering off the reader thread. The queue +/// itself is unbounded (an SDK detail), which is why the channel's handler never waits in it. /// internal sealed class RedisChannelMessageQueueSubscriber(ISubscriber _subscriber) : IRedisChannelSubscriber { diff --git a/src/Channels/AsyncResponse.Channels.Redis/RedisRecoveryStateStore.cs b/src/Channels/AsyncResponse.Channels.Redis/RedisRecoveryStateStore.cs index b37e5edf9..64083daa9 100644 --- a/src/Channels/AsyncResponse.Channels.Redis/RedisRecoveryStateStore.cs +++ b/src/Channels/AsyncResponse.Channels.Redis/RedisRecoveryStateStore.cs @@ -253,11 +253,11 @@ private bool CountStoredRegistrations(RedisValue value, out int stored) if (IsLegacyShape(json)) { // Legacy blobs carry no per-entry expiry; every element is a live registration. - stored = AsyncResponseJson.Deserialize>(json)?.Count ?? 0; + stored = JsonSafety.SafeDeserialize(json, _legacyTypeInfo)?.Count ?? 0; return true; } - var parsed = JsonSerializer.Deserialize(json, _envelopeTypeInfo); + var parsed = JsonSafety.SafeDeserialize(json, _envelopeTypeInfo); var registrations = parsed?.Registrations; if (registrations is null) { @@ -274,7 +274,7 @@ private bool CountStoredRegistrations(RedisValue value, out int stored) return true; } - catch (JsonException) + catch (Exception ex) when (ex is JsonException or InvalidDataException) { // Unparseable at the top level: the blob exists and holds an unknown number of // registrations, all of them unreadable. One is enough to fail the delivery. @@ -299,6 +299,10 @@ private bool CountStoredRegistrations(RedisValue value, out int stored) }; /// The envelope's metadata off the chained options — the JsonTypeInfo overloads keep this trim/AOT-clean. + /// Legacy bare-array blobs, read with the same case-sensitive matching as before. + private static readonly JsonTypeInfo> _legacyTypeInfo = + AsyncResponseJson.GetTypeInfo>(AsyncResponseJson.Default); + private static readonly JsonTypeInfo _envelopeTypeInfo = AsyncResponseJson.GetTypeInfo(_envelopeOptions); @@ -343,7 +347,7 @@ private static TimeSpan MaxRemaining(List entries, DateTimeO // first significant character tells the shapes apart without a speculative parse. if (IsLegacyShape(json)) { - var states = AsyncResponseJson.Deserialize>(json) ?? []; + var states = JsonSafety.SafeDeserialize(json, _legacyTypeInfo) ?? []; var legacyEntries = new List(states.Count); foreach (var state in states) { @@ -356,7 +360,7 @@ private static TimeSpan MaxRemaining(List entries, DateTimeO return (legacyEntries, true); } - var stored = JsonSerializer.Deserialize(json, _envelopeTypeInfo); + var stored = JsonSafety.SafeDeserialize(json, _envelopeTypeInfo); var entries = stored?.Registrations ?? []; entries.RemoveAll(entry => entry is null || (!preserveUnreadable && !IsStateReadable(entry.State, recoveryKey, correlationId))); // An entry past its per-entry expiry is logically gone even while a longer-lived @@ -365,8 +369,12 @@ private static TimeSpan MaxRemaining(List entries, DateTimeO entries.RemoveAll(entry => entry.ExpiresAtUtc <= nowUtc); return (entries, false); } - catch (JsonException ex) + catch (Exception ex) when (ex is JsonException or InvalidDataException) { + // Through JsonSafety, so `ex` is the body-free rebuild (size and position), never the + // reader's own message: that one appends `Path: $.States[0].Context['']` built + // from the stored registration's context keys — tenant and auth baggage — and this + // log line is what carried them into the application log. if (logAsError) _logger.LogError(ex, "Failed to deserialize recovery state at {RecoveryKey}.", recoveryKey); else diff --git a/src/Channels/AsyncResponse.Channels.Redis/ServiceCollectionExtensions.cs b/src/Channels/AsyncResponse.Channels.Redis/ServiceCollectionExtensions.cs index 4dfabdb1a..aabf0b21f 100644 --- a/src/Channels/AsyncResponse.Channels.Redis/ServiceCollectionExtensions.cs +++ b/src/Channels/AsyncResponse.Channels.Redis/ServiceCollectionExtensions.cs @@ -1,6 +1,7 @@ using AsyncResponse; using AsyncResponse.Channels.Redis; using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Options; namespace Microsoft.Extensions.DependencyInjection; @@ -57,7 +58,9 @@ public static AsyncResponseRegistrationBuilder WithRedisChannel( provider.GetService(), provider.GetService(), provider.GetRequiredService(), - provider.GetService()))); + provider.GetService(), + // The producer-side mirror of the ingress's inbound size budget (WorkerJobTooLargeException). + provider.GetService>()))); services.Replace(ServiceDescriptor.Singleton(provider => provider.GetRequiredService())); // The resolved default waiter timeout is declared through the marker so the startup diff --git a/src/Channels/AsyncResponse.Channels.SqlServer/ServiceCollectionExtensions.cs b/src/Channels/AsyncResponse.Channels.SqlServer/ServiceCollectionExtensions.cs index 041e5a3cc..e1b5e7a35 100644 --- a/src/Channels/AsyncResponse.Channels.SqlServer/ServiceCollectionExtensions.cs +++ b/src/Channels/AsyncResponse.Channels.SqlServer/ServiceCollectionExtensions.cs @@ -1,6 +1,7 @@ using AsyncResponse; using AsyncResponse.Channels.SqlServer; using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Options; namespace Microsoft.Extensions.DependencyInjection; @@ -44,7 +45,9 @@ public static AsyncResponseRegistrationBuilder WithSqlServerChannel( provider.GetService(), provider.GetService(), provider.GetRequiredService(), - provider.GetService()))); + provider.GetService(), + // The producer-side mirror of the ingress's inbound size budget (WorkerJobTooLargeException). + provider.GetService>()))); services.Replace(ServiceDescriptor.Singleton(provider => provider.GetRequiredService())); // The resolved default waiter timeout is declared through the marker so the startup diff --git a/src/Channels/AsyncResponse.Channels.SqlServer/SqlServerChannelSql.cs b/src/Channels/AsyncResponse.Channels.SqlServer/SqlServerChannelSql.cs index b8777405f..ee410cd35 100644 --- a/src/Channels/AsyncResponse.Channels.SqlServer/SqlServerChannelSql.cs +++ b/src/Channels/AsyncResponse.Channels.SqlServer/SqlServerChannelSql.cs @@ -4,10 +4,17 @@ namespace AsyncResponse.Channels.SqlServer; +/// One stored response envelope row/document as the channel store returns it. +/// +/// EnvelopeJson is the stored envelope, or null for a row the dispatch sweep loaded header-only (an +/// already-acknowledged row — see ); the +/// sweep hydrates the few such rows it still has to deliver through +/// before handing them to a waiter. +/// internal readonly record struct SqlServerChannelMessage( Guid Id, string CorrelationId, - string EnvelopeJson, + string? EnvelopeJson, DateTimeOffset CreatedAtUtc, DateTimeOffset? AckedAtUtc = null, long? AckedSeq = null); @@ -488,9 +495,15 @@ public async Task> LoadMessagesAsync( await EnsureCreatedAsync(cancellationToken).ConfigureAwait(false); await using var connection = await OpenConnectionAsync(cancellationToken).ConfigureAwait(false); await using var command = connection.CreateCommand(); + // The envelope travels only for rows nobody has acknowledged yet. Acknowledged rows are + // the consumed history the sweep re-reads on every tick (they stay in the result set so a + // fan-out waiter in ANOTHER process still receives them): shipping their bodies with each + // sweep made a long-lived progress subscription's cost grow with its whole retained + // history. The shared sweep fetches the envelope by id for the rare acknowledged row a + // live subscription has not seen. command.CommandText = $""" - SELECT id, correlation_id, envelope_json, created_at, acked_at, acked_seq + SELECT id, correlation_id, CASE WHEN acked_at IS NULL THEN envelope_json END, created_at, acked_at, acked_seq FROM {MessageTable} WHERE correlation_id = @correlation_id AND created_at >= @since @@ -512,13 +525,57 @@ public async Task> LoadMessagesAsync( command.Parameters.AddWithValue("@after_id", afterId ?? throw new ArgumentNullException(nameof(afterId))); } - var messages = new List(batchSize); + return await ReadMessagesAsync(command, batchSize, cancellationToken).ConfigureAwait(false); + } + + /// + /// The full rows (envelope included) for under + /// , in sweep order — how the dispatch sweep hydrates the + /// header-only acknowledged rows it still has to deliver. A row pruned between the sweep's + /// page and this read is simply absent. + /// + public async Task> LoadMessagesByIdAsync( + string correlationId, + IReadOnlyList ids, + CancellationToken cancellationToken) + { + if (ids.Count == 0) + return []; + + await EnsureCreatedAsync(cancellationToken).ConfigureAwait(false); + await using var connection = await OpenConnectionAsync(cancellationToken).ConfigureAwait(false); + await using var command = connection.CreateCommand(); + // One parameter per id (the sweep hands over at most a page): a joined literal list + // would put ids into SQL text, and SQL Server has no array parameter to bind instead. + var placeholders = new string[ids.Count]; + for (var i = 0; i < ids.Count; i++) + { + placeholders[i] = $"@id{i}"; + command.Parameters.Add(placeholders[i], SqlDbType.UniqueIdentifier).Value = ids[i]; + } + + command.CommandText = + $""" + SELECT id, correlation_id, envelope_json, created_at, acked_at, acked_seq + FROM {MessageTable} + WHERE correlation_id = @correlation_id + AND id IN ({string.Join(", ", placeholders)}) + AND expires_at > SYSUTCDATETIME() + ORDER BY created_at, id; + """; + command.Parameters.AddWithValue("@correlation_id", correlationId); + return await ReadMessagesAsync(command, ids.Count, cancellationToken).ConfigureAwait(false); + } + + private static async Task> ReadMessagesAsync(SqlCommand command, int capacity, CancellationToken cancellationToken) + { + var messages = new List(capacity); await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) messages.Add(new SqlServerChannelMessage( reader.GetGuid(0), reader.GetString(1), - reader.GetString(2), + reader.IsDBNull(2) ? null : reader.GetString(2), new DateTimeOffset(reader.GetDateTime(3), TimeSpan.Zero), reader.IsDBNull(4) ? null : new DateTimeOffset(reader.GetDateTime(4), TimeSpan.Zero), reader.IsDBNull(5) ? null : reader.GetInt64(5))); diff --git a/src/Channels/AsyncResponse.Channels.SqlServer/SqlServerRecoveryStateStore.cs b/src/Channels/AsyncResponse.Channels.SqlServer/SqlServerRecoveryStateStore.cs index bb9ea87b3..1e0fb4548 100644 --- a/src/Channels/AsyncResponse.Channels.SqlServer/SqlServerRecoveryStateStore.cs +++ b/src/Channels/AsyncResponse.Channels.SqlServer/SqlServerRecoveryStateStore.cs @@ -1,6 +1,7 @@ using Microsoft.Extensions.Logging; using System.Runtime.CompilerServices; using System.Text.Json; +using System.Text.Json.Serialization.Metadata; namespace AsyncResponse.Channels.SqlServer; @@ -94,11 +95,19 @@ private IReadOnlyList DeserializeStates(IReadOnlyList jso return states; } + /// The registration's metadata off the library's resolver — case-sensitive matching, as before. + private static readonly JsonTypeInfo _stateTypeInfo = + AsyncResponseJson.GetTypeInfo(AsyncResponseJson.Default); + private RecoveryState? DeserializeState(string json, string? correlationId, ref int unreadable) { try { - var state = AsyncResponseJson.Deserialize(json); + // Through JsonSafety, not the raw reader: the exception logged below is the body-free + // rebuild (size and position). The reader's own appends `Path: $.Context['']` + // built from the stored registration's context keys — tenant and auth baggage — which + // the warning then carried into the application log. + var state = JsonSafety.SafeDeserialize(json, _stateTypeInfo); if (state is null) { unreadable++; @@ -136,7 +145,7 @@ private IReadOnlyList DeserializeStates(IReadOnlyList jso return state; } - catch (JsonException ex) + catch (Exception ex) when (ex is JsonException or InvalidDataException) { _logger.LogWarning(ex, "Unreadable SQL Server recovery state for correlationId {CorrelationId}; skipping.", correlationId); unreadable++; diff --git a/src/Channels/Shared/DbChannelShared.cs b/src/Channels/Shared/DbChannelShared.cs index 1d522833d..27fe10f01 100644 --- a/src/Channels/Shared/DbChannelShared.cs +++ b/src/Channels/Shared/DbChannelShared.cs @@ -957,6 +957,18 @@ private async Task DispatchPendingCorrelationAsync( afterCreatedAtUtc, afterId, cancellationToken).ConfigureAwait(false); + + // Two passes over the page. The store ships the envelope only for rows nobody has + // acknowledged; an acknowledged row — the consumed history this sweep re-reads on + // every tick and every targeted signal, retained so a fan-out waiter in ANOTHER + // process still receives it — comes back header-only. Before the split, every sweep + // re-transferred and re-materialized a long-lived progress subscription's whole + // retained history just to drop it in the pre-filter below. The first pass decides + // which rows a live subscription would actually take; the second hydrates the + // envelopes of the (rare) acknowledged rows among them in one store read and enqueues + // in page order, so nothing later is admitted ahead of an earlier row. + List? eligible = null; + List? headerOnly = null; foreach (var message in messages) { // The store was asked for ONE exact correlation id, but "exact" is the @@ -978,25 +990,20 @@ private async Task DispatchPendingCorrelationAsync( // Pre-filter BEFORE enqueueing. The work item re-checks this anyway, but the store // deliberately keeps returning acked rows (cross-process fan-out), so every sweep - // tick and every targeted signal re-enqueued one item per retained message. A - // waiter wedged in a slow user Until predicate holds its per-correlation executor, - // those items pile up against the executor's bounded queue, and the next - // EnqueueAsync then parks the PROCESS-WIDE dispatch loop — which walks correlation - // ids sequentially — so every other waiter stopped receiving and local publishers - // blocked on the same enqueue. Skipping messages no live subscription would take - // keeps a wedged predicate's blast radius inside its own correlation id. + // tick and every targeted signal re-enqueued one item per retained message. + // Skipping messages no live subscription would take keeps the already-consumed + // history out of the executor queue. if (!WouldDeliverToAnySubscription(message, subscriptions)) continue; - // Work-item class, not a lambda: a queued closure would chain display classes - // pinning this paging frame (batch list, cursors, watermark) for as long as the - // item sits in the executor's bounded queue. - await _executors.EnqueueAsync( - ChannelName(correlationId), - new LocalDispatchWorkItem(this, message, subscriptions, cancellationToken).InvokeAsync, - cancellationToken).ConfigureAwait(false); + (eligible ??= []).Add(message); + if (message.EnvelopeJson is null) + (headerOnly ??= []).Add(message.Id); } + if (eligible is not null && !await EnqueueEligibleAsync(correlationId, eligible, headerOnly, subscriptions, cancellationToken).ConfigureAwait(false)) + return; + if (messages.Count < _options.PendingMessageBatchSize) break; @@ -1006,6 +1013,72 @@ await _executors.EnqueueAsync( } } + /// + /// Second pass of one sweep page: hydrates the header-only rows among + /// and admits every row to the correlation id's executor in page order. Returns false + /// when the executor is full (the page's remaining rows are left in the store, in order, and + /// a rescan is scheduled), which ends the correlation id's scan for this sweep. + /// + private async Task EnqueueEligibleAsync( + string correlationId, + List eligible, + List? headerOnly, + List subscriptions, + CancellationToken cancellationToken) + { + Dictionary? hydrated = null; + if (headerOnly is not null) + { + var loaded = await _store.LoadMessagesByIdAsync(correlationId, headerOnly, cancellationToken).ConfigureAwait(false); + hydrated = new Dictionary(loaded.Count); + foreach (var message in loaded) + { + // The by-id read is exact on the id (unique), but a hydrated row must carry its + // envelope: a store that answered header-only here would hand the waiter nothing. + if (message.EnvelopeJson is not null) + hydrated[message.Id] = message; + } + } + + foreach (var message in eligible) + { + var deliverable = message; + if (message.EnvelopeJson is null) + { + // Pruned or expired between the page read and the hydration: nothing to deliver + // now; a row that is still there is re-evaluated by the next sweep. + if (hydrated is null || !hydrated.TryGetValue(message.Id, out deliverable)) + continue; + } + + // Work-item class, not a lambda: a queued closure would chain display classes + // pinning this paging frame (batch list, cursors, watermark) for as long as the + // item sits in the executor's bounded queue. + // + // NON-BLOCKING admission. This loop is the process-wide dispatch sweep and walks + // correlation ids sequentially, so waiting for ONE correlation id's executor + // capacity here (the old EnqueueAsync) parked delivery for every other waiter in + // the process: a waiter wedged in a slow Until predicate, fed a backlog of NEW + // progress messages (the pre-filter above only screens consumed history), filled + // its 1024-slot executor and the sweep then blocked on slot 1025 without ever + // querying the next correlation id. Its per-correlation backpressure became shared + // delivery blockage — unrelated remote/polled responses timed out behind it. At + // capacity the rest of this correlation id's messages are left unclaimed in the + // store, in order (nothing later is enqueued ahead of them), and a rescan of just + // this id is scheduled for when the executor has had a poll interval to drain. + var outcome = _executors.TryEnqueue( + ChannelName(correlationId), + new LocalDispatchWorkItem(this, deliverable, subscriptions, cancellationToken).InvokeAsync); + if (outcome == SerialExecutorRegistry.TryEnqueueOutcome.Full) + { + ScheduleBackpressureRescan(correlationId, cancellationToken); + return false; + } + } + + return true; + } + /// /// Would any live subscription actually take this message? Used both as the sweep's /// pre-enqueue filter and as the dispatch work item's own guard, so the two can never drift. @@ -1254,6 +1327,53 @@ private protected async Task TryConfirmDeliveryAsync(PendingConfirmation c private protected void SignalDispatcher(string? correlationId = null) => _signals.Writer.TryWrite(correlationId); + /// + /// Correlation ids whose executor was at capacity during a sweep and that have a rescan + /// pending. One pending rescan per id: a saturated id is re-signalled once per poll interval, + /// not once per sweep that found it full. + /// + private readonly ConcurrentDictionary _backpressureRescans = new(StringComparer.Ordinal); + + /// + /// Re-signals a targeted scan of after one poll interval — + /// the time the sweep would otherwise have waited for the saturated executor, spent letting + /// every other correlation id deliver instead. The messages themselves stay in the store + /// (unclaimed, unseen) until that scan enqueues them, in their original order. + /// + private void ScheduleBackpressureRescan(string correlationId, CancellationToken cancellationToken) + { + if (!_backpressureRescans.TryAdd(correlationId, 0)) + return; + + if (_logger.IsEnabled(LogLevel.Debug)) + { + _logger.LogDebug( + "{Provider} dispatch for correlationId {CorrelationId} is at executor capacity; the remaining messages are left in the store and this id is rescanned after the poll interval.", + _providerName, correlationId); + } + + _ = RescanAfterDelayAsync(correlationId, cancellationToken); + } + + private async Task RescanAfterDelayAsync(string correlationId, CancellationToken cancellationToken) + { + try + { + await Task.Delay(CurrentPollInterval(), cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + // Listener stopping: nothing to rescan for. + } + finally + { + _backpressureRescans.TryRemove(correlationId, out _); + } + + if (!cancellationToken.IsCancellationRequested) + SignalDispatcher(correlationId); + } + private async Task WaitForAcknowledgementAsync(PendingConfirmation confirmation, CancellationToken cancellationToken) { var deadline = _timeProvider.GetUtcNow() + _options.DeliveryConfirmationTimeout; @@ -1555,7 +1675,13 @@ public async Task ProcessAsync(DbChannelMessage message) var finished = false; try { - var envelope = JsonSerializer.Deserialize(message.EnvelopeJson, AsyncResponseEnvelopeJson.TypeInfo()); + // JsonSafety, not the raw reader: a parse failure is logged below and handed to the + // waiter, and the reader's own message quotes inbound property names and dictionary + // keys (docs/security.md, "never logs a message body"). Size and position only. + // A header-only sweep row never reaches delivery: the sweep hydrates it first. + var envelopeJson = message.EnvelopeJson + ?? throw new InvalidOperationException($"The {_owner._providerName} channel message {message.Id} reached delivery without its envelope."); + var envelope = JsonSafety.SafeDeserialize(envelopeJson, AsyncResponseEnvelopeJson.TypeInfo()); if (envelope is null) { finished = true; diff --git a/src/DurableFlows/AsyncResponse.DurableFlows.Cosmos/CosmosDurableFlows.cs b/src/DurableFlows/AsyncResponse.DurableFlows.Cosmos/CosmosDurableFlows.cs index 94e15914a..ebeb30083 100644 --- a/src/DurableFlows/AsyncResponse.DurableFlows.Cosmos/CosmosDurableFlows.cs +++ b/src/DurableFlows/AsyncResponse.DurableFlows.Cosmos/CosmosDurableFlows.cs @@ -5,7 +5,9 @@ using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Options; using Newtonsoft.Json; +using System.Buffers; using System.Net; +using System.Text; namespace Microsoft.Extensions.DependencyInjection { @@ -67,10 +69,14 @@ public sealed class CosmosDurableFlowOptions : DurableFlowOptions public int? Throughput { get; set; } /// - /// Maximum serialized flow-state size in bytes accepted by writes; oversized ledgers fail fast - /// with an actionable error instead of the raw Cosmos 413 the executor would retry into the - /// dead-letter queue. Default: 1.9 MB (headroom under Cosmos's 2 MB item cap for the sibling - /// fields); null disables the guard. + /// Maximum size in bytes of the COMPLETE ledger document accepted by writes — the item as it + /// is serialized for Cosmos, with the ledger JSON embedded (and therefore escaped a second + /// time) as its stateJson string and the sibling fields beside it. Cosmos caps the item + /// as a whole at 2 MB, not the ledger inside it: a ledger whose own JSON is well under the + /// budget can escape into a document over it, so the budget is enforced on what is actually + /// sent. Oversized ledgers fail fast with an actionable error instead of the raw Cosmos 413 + /// the executor would retry into the dead-letter queue. Default: 1.9 MB (headroom under the + /// item cap); null disables the guard. /// public long? MaxStateBytes { get; set; } = 1_900_000; @@ -187,6 +193,8 @@ public async Task TryCreateAsync(string flowId, FlowState state, TimeSpan var now = DateTime.UtcNow; var document = CreateDocument(flowId, stateJson, state.Revision, ttl, now); + if (attempt == 0) + ThrowIfDocumentTooLarge(flowId, document); try { await container.CreateItemAsync(document, new PartitionKey(flowId), cancellationToken: cancellationToken).ConfigureAwait(false); @@ -264,6 +272,7 @@ public async Task TryUpdateAsync( document.UpdatedAtUtc = now; document.Revision = state.Revision; document.Ttl = CosmosTtlSeconds(ttl); + ThrowIfDocumentTooLarge(flowId, document); await container.ReplaceItemAsync( document, flowId, @@ -283,9 +292,10 @@ await container.ReplaceItemAsync( } catch (CosmosException ex) when (ex.StatusCode == HttpStatusCode.PreconditionFailed) { - // Lease renewal also replaces the document and changes its ETag without changing - // the ledger revision. Re-read and retry so that benign race is not reported as a - // lost execution lease; a real state race fails the revision check above. + // Lease acquire/renew/release patch the document's lease fields and change its + // ETag without changing the ledger revision. Re-read and retry so that benign + // race is not reported as a lost execution lease; a real state race fails the + // revision check above. } } @@ -305,26 +315,24 @@ public async Task ReleaseLeaseAsync(string flowId, string leaseId, CancellationT { try { - var current = await container.ReadItemAsync( - flowId, - new PartitionKey(flowId), - cancellationToken: cancellationToken).ConfigureAwait(false); - if (current.Resource.LeaseId != leaseId) + var current = await ReadLeaseAsync(container, flowId, cancellationToken).ConfigureAwait(false); + if (current is null || current.LeaseId != leaseId) return; - current.Resource.LeaseId = null; - current.Resource.LeaseExpiresAtUtc = null; - // Every replace refreshes _ts — the anchor the server-side TTL counts from — so + // Every write refreshes _ts — the anchor the server-side TTL counts from — so // re-persisting the stored full-window ttl would restart the physical-retention // countdown and decouple it from the logical ExpiresAtUtc. Rewrite it from the // remaining logical window instead. (Checkpoints recompute both together in - // TryUpdateAsync; only the lease paths replace without moving ExpiresAtUtc.) - current.Resource.Ttl = CosmosTtlSeconds(current.Resource.ExpiresAtUtc, DateTime.UtcNow); - await container.ReplaceItemAsync( - current.Resource, + // TryUpdateAsync; only the lease paths write without moving ExpiresAtUtc.) + await PatchLeaseAsync( + container, flowId, - new PartitionKey(flowId), - new ItemRequestOptions { IfMatchEtag = current.ETag }, + current.ETag, + [ + PatchOperation.Set(LeaseIdPath, null), + PatchOperation.Set(LeaseExpiresAtPath, null), + PatchOperation.Set(TtlPath, CosmosTtlSeconds(current.ExpiresAtUtc, DateTime.UtcNow)) + ], cancellationToken).ConfigureAwait(false); return; } @@ -338,6 +346,76 @@ await container.ReplaceItemAsync( } } + // JSON-pointer paths of the lease fields, matching CosmosFlowStateDocument's property names. + private const string LeaseIdPath = "/leaseId"; + private const string LeaseExpiresAtPath = "/leaseExpiresAtUtc"; + private const string TtlPath = "/ttl"; + + /// + /// The lease-relevant slice of one ledger document, read with a projecting point query so a + /// lease acquire, heartbeat, or release never transfers stateJson. A point read has no + /// projection — it returned the whole document, StateJson included, and the follow-up + /// ReplaceItemAsync sent it all back — so an idle execution's every renewal (default: each + /// 20 seconds) moved and re-serialized the full ledger twice, proportional to its size. + /// Together with the conditional patches below, lease maintenance now costs O(lease fields) + /// on the wire regardless of ledger size. RU cost still depends on the service's accounting + /// for the loaded document; measure it (docs/durable-flow-state-stores.md). + /// + /// Only the SQL text is shared. A is a mutable parameter bag — + /// WithParameter replaces the named parameter in place and returns the same instance — + /// so one static definition parameterized per call handed concurrent lease operations each + /// other's ids: flow A's query could execute with @id = B under A's partition key, + /// return no document, and fail a healthy renewal (which abandons and replays the run). Every + /// call builds its own definition. + /// + /// + private const string LeaseProjectionSql = + "SELECT c.id, c._etag, c.expiresAtUtc, c.revision, c.leaseId, c.leaseExpiresAtUtc FROM c WHERE c.id = @id"; + + private static async Task ReadLeaseAsync(Container container, string flowId, CancellationToken cancellationToken) + { + using var iterator = container.GetItemQueryIterator( + new QueryDefinition(LeaseProjectionSql).WithParameter("@id", flowId), + requestOptions: new QueryRequestOptions { PartitionKey = new PartitionKey(flowId), MaxItemCount = 1 }); + while (iterator.HasMoreResults) + { + var page = await iterator.ReadNextAsync(cancellationToken).ConfigureAwait(false); + foreach (var projection in page) + { + if (string.IsNullOrEmpty(projection.ETag)) + { + // The fence for every lease write. A projection without it cannot be acted + // on safely, and silently treating it as "not held" would let the executor + // acknowledge a wake-up as a duplicate against a run nobody holds. + throw new InvalidOperationException( + $"The Cosmos DB durable-flow store's lease query for '{flowId}' returned no _etag; the registered serializer does not surface system properties, so lease writes cannot be fenced."); + } + + return projection; + } + } + + return null; + } + + /// + /// A conditional partial update of the lease fields: fenced by the projection's ETag exactly + /// as the replace was, with no document content in the response (there is nothing the + /// caller reads back). + /// + private static Task PatchLeaseAsync( + Container container, + string flowId, + string etag, + IReadOnlyList operations, + CancellationToken cancellationToken) + => container.PatchItemAsync( + flowId, + new PartitionKey(flowId), + operations, + new PatchItemRequestOptions { IfMatchEtag = etag, EnableContentResponseOnWrite = false }, + cancellationToken); + public async Task TryDeleteAsync(string flowId, CancellationToken cancellationToken = default) { ArgumentException.ThrowIfNullOrWhiteSpace(flowId); @@ -473,12 +551,8 @@ private async Task UpdateLeaseAsync( var now = DateTime.UtcNow; try { - var current = await container.ReadItemAsync( - flowId, - new PartitionKey(flowId), - cancellationToken: cancellationToken).ConfigureAwait(false); - var document = current.Resource; - if (document.ExpiresAtUtc <= now || document.Revision is null) + var document = await ReadLeaseAsync(container, flowId, cancellationToken).ConfigureAwait(false); + if (document is null || document.ExpiresAtUtc <= now || document.Revision is null) return false; if (acquire) { @@ -491,17 +565,18 @@ private async Task UpdateLeaseAsync( return false; } - document.LeaseId = leaseId; - document.LeaseExpiresAtUtc = DurableFlowStoreShared.AddSaturating(now, leaseDuration); - // Same _ts realignment as ReleaseLeaseAsync: a lease heartbeat replaces the + // Same _ts realignment as ReleaseLeaseAsync: a lease heartbeat writes the // document without moving ExpiresAtUtc, so it must not restart the server TTL's // full retention window. - document.Ttl = CosmosTtlSeconds(document.ExpiresAtUtc, now); - await container.ReplaceItemAsync( - document, + await PatchLeaseAsync( + container, flowId, - new PartitionKey(flowId), - new ItemRequestOptions { IfMatchEtag = current.ETag }, + document.ETag, + [ + PatchOperation.Set(LeaseIdPath, leaseId), + PatchOperation.Set(LeaseExpiresAtPath, DurableFlowStoreShared.AddSaturating(now, leaseDuration)), + PatchOperation.Set(TtlPath, CosmosTtlSeconds(document.ExpiresAtUtc, now)) + ], cancellationToken).ConfigureAwait(false); return true; } @@ -517,6 +592,54 @@ await container.ReplaceItemAsync( return false; } + /// + /// Enforces on the document as Cosmos + /// will receive it. already refused a + /// ledger whose own JSON is over the budget — a cheap first check, since the document can only + /// be larger — but the ledger travels inside the document as a string value, so every quote + /// and backslash in it is escaped again: a 1.2 MB ledger made of escaped quotes is a 2.4 MB + /// document, accepted by the inner check and refused by Cosmos's 2 MB item cap on every + /// retry. Measured through the host's own serializer when one is registered (its escaping + /// and property naming are what go on the wire), else through the SDK default's + /// Newtonsoft-based shape. + /// + private void ThrowIfDocumentTooLarge(string flowId, CosmosFlowStateDocument document) + { + if (_options.MaxStateBytes is not { } limit) + return; + + var size = MeasureDocumentBytes(document); + if (size > limit) + throw new FlowStateTooLargeException(flowId, size, limit, "Cosmos DB"); + } + + private long MeasureDocumentBytes(CosmosFlowStateDocument document) + { + if (_client.ClientOptions?.Serializer is { } serializer) + { + using var stream = serializer.ToStream(document); + if (stream.CanSeek) + return stream.Length; + + long total = 0; + var buffer = ArrayPool.Shared.Rent(16 * 1024); + try + { + int read; + while ((read = stream.Read(buffer, 0, buffer.Length)) > 0) + total += read; + } + finally + { + ArrayPool.Shared.Return(buffer); + } + + return total; + } + + return Encoding.UTF8.GetByteCount(JsonConvert.SerializeObject(document)); + } + private static CosmosFlowStateDocument CreateDocument(string flowId, string stateJson, long revision, TimeSpan ttl, DateTime now) => new() { @@ -603,4 +726,38 @@ internal sealed class CosmosFlowStateDocument [System.Text.Json.Serialization.JsonIgnore(Condition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull)] public int? Ttl { get; set; } } + +/// +/// The lease slice of plus the document's _etag, as +/// returned by the store's projecting lease query — everything a lease acquire, renewal, or +/// release decides on and fences with, and nothing else (no stateJson). Attributed for +/// both serializer stacks for the same reason the document is. +/// +internal sealed class CosmosLeaseProjection +{ + [JsonProperty("id")] + [System.Text.Json.Serialization.JsonPropertyName("id")] + public string Id { get; set; } = ""; + + /// The fence for every lease write; the store refuses to act on a projection without one. + [JsonProperty("_etag")] + [System.Text.Json.Serialization.JsonPropertyName("_etag")] + public string ETag { get; set; } = ""; + + [JsonProperty("expiresAtUtc")] + [System.Text.Json.Serialization.JsonPropertyName("expiresAtUtc")] + public DateTime ExpiresAtUtc { get; set; } + + [JsonProperty("revision")] + [System.Text.Json.Serialization.JsonPropertyName("revision")] + public long? Revision { get; set; } + + [JsonProperty("leaseId")] + [System.Text.Json.Serialization.JsonPropertyName("leaseId")] + public string? LeaseId { get; set; } + + [JsonProperty("leaseExpiresAtUtc")] + [System.Text.Json.Serialization.JsonPropertyName("leaseExpiresAtUtc")] + public DateTime? LeaseExpiresAtUtc { get; set; } +} } diff --git a/src/DurableFlows/AsyncResponse.DurableFlows.EFCore/EFCoreDurableFlows.cs b/src/DurableFlows/AsyncResponse.DurableFlows.EFCore/EFCoreDurableFlows.cs index d36d81e50..55a4a094e 100644 --- a/src/DurableFlows/AsyncResponse.DurableFlows.EFCore/EFCoreDurableFlows.cs +++ b/src/DurableFlows/AsyncResponse.DurableFlows.EFCore/EFCoreDurableFlows.cs @@ -6,6 +6,7 @@ using Microsoft.EntityFrameworkCore.Infrastructure; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; namespace Microsoft.Extensions.DependencyInjection @@ -51,6 +52,18 @@ public sealed class EFCoreDurableFlowOptions : DurableFlowOptions /// public TimeSpan PruneInterval { get; set; } = TimeSpan.FromMinutes(5); + /// + /// Wall-clock budget one opportunistic prune may spend draining expired rows in batches of + /// 1000 after its first batch (the first always runs). A single batch per interval capped + /// cleanup at ~3 rows/second, which any busier instance outgrew forever; the prune now drains + /// batches until one comes back short or this budget lapses, and reports the outcome on the + /// AsyncResponse meter (asyncresponse.flow_state.pruned_rows, + /// prune_failures, prune_budget_exhausted) and the store's logger. The create + /// that triggers the prune waits for it, so this bounds that create's added latency. Zero + /// keeps the historical single batch. Default: 2 seconds. + /// + public TimeSpan PruneBudget { get; set; } = DurableFlowStoreShared.DefaultPruneBudget; + /// /// Maximum serialized flow-state size in bytes accepted by writes; oversized ledgers fail fast /// with an actionable error instead of an opaque provider error. Default: null @@ -227,7 +240,7 @@ public static ModelBuilder ConfigureAsyncResponseDurableFlows( public sealed class EFCoreFlowStateStore<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors | DynamicallyAccessedMemberTypes.NonPublicConstructors | DynamicallyAccessedMemberTypes.PublicProperties)] TContext> : IFlowStateStore where TContext : DbContext { - private const int PruneBatchSize = 1000; + private readonly ILogger>? _logger; // Time authority: this store deliberately keeps the app clock (DateTime.UtcNow) for expiry // and lease comparisons. It is provider-agnostic LINQ — there is no portable way to reference @@ -239,11 +252,13 @@ public sealed class EFCoreFlowStateStore<[DynamicallyAccessedMembers(Dynamically private long _lastPruneTicks; private volatile bool _modelChecked; - public EFCoreFlowStateStore(IServiceScopeFactory scopeFactory, IOptions options) + public EFCoreFlowStateStore(IServiceScopeFactory scopeFactory, IOptions options, ILogger>? logger = null) { _scopeFactory = scopeFactory; _options = options.Value; + _logger = logger; DurableFlowStoreShared.ValidateMaxStateBytes(_options.MaxStateBytes, nameof(EFCoreDurableFlowOptions)); + DurableFlowStoreShared.ValidatePruneBudget(_options.PruneBudget, nameof(EFCoreDurableFlowOptions)); } public async Task LoadAsync(string flowId, CancellationToken cancellationToken = default) @@ -277,7 +292,7 @@ public async Task TryCreateAsync(string flowId, FlowState state, TimeSpan var now = DateTime.UtcNow; if (DurableFlowStoreShared.ShouldPrune(ref _lastPruneTicks, _options.PruneInterval)) - await DurableFlowStoreShared.PruneQuietlyAsync(() => PruneExpiredAsync(db, cancellationToken)).ConfigureAwait(false); + await DurableFlowStoreShared.PruneQuietlyAsync(() => PruneExpiredAsync(db, cancellationToken), _options.PruneBudget, "EF Core", _logger).ConfigureAwait(false); // Replace an expired ledger IN PLACE, in one statement (sibling parity: PostgreSQL // `ON CONFLICT ... DO UPDATE ... WHERE expired`, SQL Server/Oracle `MERGE ... WHEN MATCHED @@ -384,19 +399,20 @@ public async Task TryDeleteAsync(string flowId, CancellationToken cancella return deleted > 0; } - private static async Task PruneExpiredAsync(TContext db, CancellationToken cancellationToken) + private static async Task PruneExpiredAsync(TContext db, CancellationToken cancellationToken) { - // One bounded batch per prune interval (policy shared by all relational stores): an - // unbatched delete over a large expired backlog holds row locks and bloats one + // One bounded batch per call; DurableFlowStoreShared.PruneQuietlyAsync repeats it under + // the PruneBudget while batches come back full (policy shared by all relational stores): + // an unbatched delete over a large expired backlog holds row locks and bloats one // transaction for the unlucky create that triggered the prune. Loads already filter on - // expiry, so any backlog beyond the batch just waits for the next interval. The OrderBy + // expiry, so any backlog beyond the budget just waits for the next interval. The OrderBy // makes the row-limited delete deterministic (and keeps providers from warning about an // unordered Take). var now = DateTime.UtcNow; - await Records(db) + return await Records(db) .Where(r => r.ExpiresAtUtc <= now) .OrderBy(r => r.FlowId) - .Take(PruneBatchSize) + .Take(DurableFlowStoreShared.PruneBatchSize) .ExecuteDeleteAsync(cancellationToken) .ConfigureAwait(false); } diff --git a/src/DurableFlows/AsyncResponse.DurableFlows.EFCore/PublicAPI.Unshipped.txt b/src/DurableFlows/AsyncResponse.DurableFlows.EFCore/PublicAPI.Unshipped.txt index 2efe1d7cd..3a1a38464 100644 --- a/src/DurableFlows/AsyncResponse.DurableFlows.EFCore/PublicAPI.Unshipped.txt +++ b/src/DurableFlows/AsyncResponse.DurableFlows.EFCore/PublicAPI.Unshipped.txt @@ -22,8 +22,10 @@ AsyncResponse.DurableFlows.EFCore.EFCoreDurableFlowOptions.MaxStateBytes.get -> AsyncResponse.DurableFlows.EFCore.EFCoreDurableFlowOptions.MaxStateBytes.set -> void AsyncResponse.DurableFlows.EFCore.EFCoreDurableFlowOptions.PruneInterval.get -> System.TimeSpan AsyncResponse.DurableFlows.EFCore.EFCoreDurableFlowOptions.PruneInterval.set -> void +AsyncResponse.DurableFlows.EFCore.EFCoreDurableFlowOptions.PruneBudget.get -> System.TimeSpan +AsyncResponse.DurableFlows.EFCore.EFCoreDurableFlowOptions.PruneBudget.set -> void AsyncResponse.DurableFlows.EFCore.EFCoreFlowStateStore -AsyncResponse.DurableFlows.EFCore.EFCoreFlowStateStore.EFCoreFlowStateStore(Microsoft.Extensions.DependencyInjection.IServiceScopeFactory! scopeFactory, Microsoft.Extensions.Options.IOptions! options) -> void +AsyncResponse.DurableFlows.EFCore.EFCoreFlowStateStore.EFCoreFlowStateStore(Microsoft.Extensions.DependencyInjection.IServiceScopeFactory! scopeFactory, Microsoft.Extensions.Options.IOptions! options, Microsoft.Extensions.Logging.ILogger!>? logger = null) -> void AsyncResponse.DurableFlows.EFCore.EFCoreFlowStateStore.LoadAsync(string! flowId, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! AsyncResponse.DurableFlows.EFCore.EFCoreFlowStateStore.ReleaseLeaseAsync(string! flowId, string! leaseId, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! AsyncResponse.DurableFlows.EFCore.EFCoreFlowStateStore.TryAcquireLeaseAsync(string! flowId, string! leaseId, System.TimeSpan leaseDuration, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! diff --git a/src/DurableFlows/AsyncResponse.DurableFlows.MySql/MySqlDurableFlows.cs b/src/DurableFlows/AsyncResponse.DurableFlows.MySql/MySqlDurableFlows.cs index f65af5997..b94f1f3a8 100644 --- a/src/DurableFlows/AsyncResponse.DurableFlows.MySql/MySqlDurableFlows.cs +++ b/src/DurableFlows/AsyncResponse.DurableFlows.MySql/MySqlDurableFlows.cs @@ -2,6 +2,7 @@ using AsyncResponse.DurableFlows.Internal; using AsyncResponse.DurableFlows.MySql; using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using MySqlConnector; @@ -45,6 +46,18 @@ public sealed class MySqlDurableFlowOptions : DurableFlowOptions /// public TimeSpan PruneInterval { get; set; } = TimeSpan.FromMinutes(5); + /// + /// Wall-clock budget one opportunistic prune may spend draining expired rows in batches of + /// 1000 after its first batch (the first always runs). A single batch per interval capped + /// cleanup at ~3 rows/second, which any busier instance outgrew forever; the prune now drains + /// batches until one comes back short or this budget lapses, and reports the outcome on the + /// AsyncResponse meter (asyncresponse.flow_state.pruned_rows, + /// prune_failures, prune_budget_exhausted) and the store's logger. The create + /// that triggers the prune waits for it, so this bounds that create's added latency. Zero + /// keeps the historical single batch. Default: 2 seconds. + /// + public TimeSpan PruneBudget { get; set; } = DurableFlowStoreShared.DefaultPruneBudget; + /// /// Maximum serialized flow-state size in bytes accepted by writes; oversized ledgers fail fast /// with an actionable error instead of an opaque provider error. Default: null @@ -58,13 +71,14 @@ public void Validate() DurableFlowStoreShared.ValidateConnectionString(ConnectionString, nameof(MySqlDurableFlowOptions)); DurableFlowStoreShared.ValidateIdentifier(TableName, $"{nameof(MySqlDurableFlowOptions)}.{nameof(TableName)}", "MySQL", identifierCap: 64); DurableFlowStoreShared.ValidateMaxStateBytes(MaxStateBytes, nameof(MySqlDurableFlowOptions)); + DurableFlowStoreShared.ValidatePruneBudget(PruneBudget, nameof(MySqlDurableFlowOptions)); } } /// MySQL/MariaDB implementation of . public sealed class MySqlFlowStateStore : IFlowStateStore { - private const int PruneBatchSize = 1000; + private readonly ILogger? _logger; /// /// SQL expression adding a millisecond bigint parameter to the database clock. All expiry and @@ -80,8 +94,9 @@ private static string AddMilliseconds(string parameterName) private long _lastPruneTicks; private volatile bool _created; - public MySqlFlowStateStore(IOptions options) + public MySqlFlowStateStore(IOptions options, ILogger? logger = null) { + _logger = logger; _options = options.Value; _options.Validate(); } @@ -109,7 +124,7 @@ public async Task TryCreateAsync(string flowId, FlowState state, TimeSpan var stateJson = DurableFlowStoreShared.SerializeBounded(flowId, state, _options.MaxStateBytes, "MySQL"); await EnsureCreatedAsync(cancellationToken).ConfigureAwait(false); if (DurableFlowStoreShared.ShouldPrune(ref _lastPruneTicks, _options.PruneInterval)) - await DurableFlowStoreShared.PruneQuietlyAsync(() => PruneExpiredAsync(cancellationToken)).ConfigureAwait(false); + await DurableFlowStoreShared.PruneQuietlyAsync(() => PruneExpiredAsync(cancellationToken), _options.PruneBudget, "MySQL", _logger).ConfigureAwait(false); await using var connection = await OpenConnectionAsync(cancellationToken).ConfigureAwait(false); await using var command = connection.CreateCommand(); @@ -137,7 +152,7 @@ public async Task TryCreateAsync(string flowId, FlowState state, TimeSpan // verification refuses such tables, but this store also runs against schemas it did not // get to inspect first (AutoCreateSchema off, table created later), so confirm the row // is actually there before believing the error. - if (!await ExistsAsync(flowId, cancellationToken).ConfigureAwait(false)) + if (!await ExistsAsync(connection, flowId, cancellationToken).ConfigureAwait(false)) throw; // The id already exists. Only an expired row may be replaced below; do not use @@ -163,11 +178,15 @@ public async Task TryCreateAsync(string flowId, FlowState state, TimeSpan /// /// Whether a row with EXACTLY this flow id exists, expired or not — the question a 1062 does - /// not answer on its own. Opens its own connection so it is safe to call mid-operation. + /// not answer on its own. Runs on the caller's already-open connection: TryCreateAsync + /// holds its connection across the 1062 handling, and opening a SECOND one from inside that + /// window meant every duplicate create occupied one pooled connection while waiting for + /// another — with MaximumPoolSize=1 a single duplicate start timed out with "All pooled + /// connections are in use", and under concurrent idempotent starts the pool starved whatever + /// size it had. /// - private async Task ExistsAsync(string flowId, CancellationToken cancellationToken) + private async Task ExistsAsync(MySqlConnection connection, string flowId, CancellationToken cancellationToken) { - await using var connection = await OpenConnectionAsync(cancellationToken).ConfigureAwait(false); await using var command = connection.CreateCommand(); command.CommandText = $"SELECT 1 FROM {Table} WHERE flow_id = @flow_id LIMIT 1;"; command.Parameters.AddWithValue("@flow_id", flowId); @@ -238,16 +257,17 @@ public async Task TryDeleteAsync(string flowId, CancellationToken cancella return await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false) > 0; } - private async Task PruneExpiredAsync(CancellationToken cancellationToken) + private async Task PruneExpiredAsync(CancellationToken cancellationToken) { - // One bounded batch per prune interval (policy shared by all relational stores): an + // One bounded batch per call; DurableFlowStoreShared.PruneQuietlyAsync repeats it under + // the PruneBudget while batches come back full (policy shared by all relational stores): an // unbatched DELETE over a large expired backlog holds row locks and bloats one // transaction for the unlucky create that triggered the prune. Loads already filter on // expiry, so any backlog beyond the batch just waits for the next interval. await using var connection = await OpenConnectionAsync(cancellationToken).ConfigureAwait(false); await using var command = connection.CreateCommand(); - command.CommandText = $"DELETE FROM {Table} WHERE expires_at_utc <= UTC_TIMESTAMP(6) LIMIT {PruneBatchSize};"; - await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + command.CommandText = $"DELETE FROM {Table} WHERE expires_at_utc <= UTC_TIMESTAMP(6) LIMIT {DurableFlowStoreShared.PruneBatchSize};"; + return await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); } private async Task EnsureCreatedAsync(CancellationToken cancellationToken) diff --git a/src/DurableFlows/AsyncResponse.DurableFlows.MySql/PublicAPI.Unshipped.txt b/src/DurableFlows/AsyncResponse.DurableFlows.MySql/PublicAPI.Unshipped.txt index 1e47c4d9d..05a2088ee 100644 --- a/src/DurableFlows/AsyncResponse.DurableFlows.MySql/PublicAPI.Unshipped.txt +++ b/src/DurableFlows/AsyncResponse.DurableFlows.MySql/PublicAPI.Unshipped.txt @@ -9,12 +9,14 @@ AsyncResponse.DurableFlows.MySql.MySqlDurableFlowOptions.MaxStateBytes.set -> vo AsyncResponse.DurableFlows.MySql.MySqlDurableFlowOptions.MySqlDurableFlowOptions() -> void AsyncResponse.DurableFlows.MySql.MySqlDurableFlowOptions.PruneInterval.get -> System.TimeSpan AsyncResponse.DurableFlows.MySql.MySqlDurableFlowOptions.PruneInterval.set -> void +AsyncResponse.DurableFlows.MySql.MySqlDurableFlowOptions.PruneBudget.get -> System.TimeSpan +AsyncResponse.DurableFlows.MySql.MySqlDurableFlowOptions.PruneBudget.set -> void AsyncResponse.DurableFlows.MySql.MySqlDurableFlowOptions.TableName.get -> string! AsyncResponse.DurableFlows.MySql.MySqlDurableFlowOptions.TableName.set -> void AsyncResponse.DurableFlows.MySql.MySqlDurableFlowOptions.Validate() -> void AsyncResponse.DurableFlows.MySql.MySqlFlowStateStore AsyncResponse.DurableFlows.MySql.MySqlFlowStateStore.LoadAsync(string! flowId, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! -AsyncResponse.DurableFlows.MySql.MySqlFlowStateStore.MySqlFlowStateStore(Microsoft.Extensions.Options.IOptions! options) -> void +AsyncResponse.DurableFlows.MySql.MySqlFlowStateStore.MySqlFlowStateStore(Microsoft.Extensions.Options.IOptions! options, Microsoft.Extensions.Logging.ILogger? logger = null) -> void AsyncResponse.DurableFlows.MySql.MySqlFlowStateStore.ReleaseLeaseAsync(string! flowId, string! leaseId, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! AsyncResponse.DurableFlows.MySql.MySqlFlowStateStore.TryAcquireLeaseAsync(string! flowId, string! leaseId, System.TimeSpan leaseDuration, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! AsyncResponse.DurableFlows.MySql.MySqlFlowStateStore.TryCreateAsync(string! flowId, AsyncResponse.FlowState! state, System.TimeSpan ttl, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! diff --git a/src/DurableFlows/AsyncResponse.DurableFlows.Oracle/OracleDurableFlows.cs b/src/DurableFlows/AsyncResponse.DurableFlows.Oracle/OracleDurableFlows.cs index fedc50d18..b084640a4 100644 --- a/src/DurableFlows/AsyncResponse.DurableFlows.Oracle/OracleDurableFlows.cs +++ b/src/DurableFlows/AsyncResponse.DurableFlows.Oracle/OracleDurableFlows.cs @@ -2,6 +2,7 @@ using AsyncResponse.DurableFlows.Internal; using AsyncResponse.DurableFlows.Oracle; using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Oracle.ManagedDataAccess.Client; @@ -45,6 +46,18 @@ public sealed class OracleDurableFlowOptions : DurableFlowOptions /// public TimeSpan PruneInterval { get; set; } = TimeSpan.FromMinutes(5); + /// + /// Wall-clock budget one opportunistic prune may spend draining expired rows in batches of + /// 1000 after its first batch (the first always runs). A single batch per interval capped + /// cleanup at ~3 rows/second, which any busier instance outgrew forever; the prune now drains + /// batches until one comes back short or this budget lapses, and reports the outcome on the + /// AsyncResponse meter (asyncresponse.flow_state.pruned_rows, + /// prune_failures, prune_budget_exhausted) and the store's logger. The create + /// that triggers the prune waits for it, so this bounds that create's added latency. Zero + /// keeps the historical single batch. Default: 2 seconds. + /// + public TimeSpan PruneBudget { get; set; } = DurableFlowStoreShared.DefaultPruneBudget; + /// /// Maximum serialized flow-state size in bytes accepted by writes; oversized ledgers fail fast /// with an actionable error instead of an opaque provider error. Default: null @@ -67,6 +80,7 @@ public void Validate() throw new InvalidOperationException( $"{nameof(OracleDurableFlowOptions)}.{nameof(TableName)} '{TableName}' collides with its derived expiry-index name; rename the table."); DurableFlowStoreShared.ValidateMaxStateBytes(MaxStateBytes, nameof(OracleDurableFlowOptions)); + DurableFlowStoreShared.ValidatePruneBudget(PruneBudget, nameof(OracleDurableFlowOptions)); } } @@ -76,7 +90,7 @@ public sealed class OracleFlowStateStore : IFlowStateStore private const int ObjectAlreadyExists = 955; private const int ColumnListAlreadyIndexed = 1408; private const int UniqueConstraintViolated = 1; - private const int PruneBatchSize = 1000; + private readonly ILogger? _logger; /// /// SQL expression adding a millisecond bind parameter to the database clock. All expiry and @@ -94,8 +108,9 @@ private static string AddMilliseconds(string parameterName) private long _lastPruneTicks; private volatile bool _created; - public OracleFlowStateStore(IOptions options) + public OracleFlowStateStore(IOptions options, ILogger? logger = null) { + _logger = logger; _options = options.Value; _options.Validate(); } @@ -124,7 +139,7 @@ public async Task TryCreateAsync(string flowId, FlowState state, TimeSpan var stateJson = DurableFlowStoreShared.SerializeBounded(flowId, state, _options.MaxStateBytes, "Oracle"); await EnsureCreatedAsync(cancellationToken).ConfigureAwait(false); if (DurableFlowStoreShared.ShouldPrune(ref _lastPruneTicks, _options.PruneInterval)) - await DurableFlowStoreShared.PruneQuietlyAsync(() => PruneExpiredAsync(cancellationToken)).ConfigureAwait(false); + await DurableFlowStoreShared.PruneQuietlyAsync(() => PruneExpiredAsync(cancellationToken), _options.PruneBudget, "Oracle", _logger).ConfigureAwait(false); await using var connection = await OpenConnectionAsync(cancellationToken).ConfigureAwait(false); try @@ -237,17 +252,18 @@ public async Task TryDeleteAsync(string flowId, CancellationToken cancella return await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false) > 0; } - private async Task PruneExpiredAsync(CancellationToken cancellationToken) + private async Task PruneExpiredAsync(CancellationToken cancellationToken) { - // One bounded batch per prune interval (policy shared by all relational stores): an + // One bounded batch per call; DurableFlowStoreShared.PruneQuietlyAsync repeats it under + // the PruneBudget while batches come back full (policy shared by all relational stores): an // unbatched DELETE over a large expired backlog holds row locks and bloats one // transaction for the unlucky create that triggered the prune. Loads already filter on // expiry, so any backlog beyond the batch just waits for the next interval. await using var connection = await OpenConnectionAsync(cancellationToken).ConfigureAwait(false); await using var command = connection.CreateCommand(); command.BindByName = true; - command.CommandText = $"DELETE FROM {Table} WHERE expires_at_utc <= {UtcNowSql} AND ROWNUM <= {PruneBatchSize}"; - await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + command.CommandText = $"DELETE FROM {Table} WHERE expires_at_utc <= {UtcNowSql} AND ROWNUM <= {DurableFlowStoreShared.PruneBatchSize}"; + return await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); } private async Task EnsureCreatedAsync(CancellationToken cancellationToken) diff --git a/src/DurableFlows/AsyncResponse.DurableFlows.Oracle/PublicAPI.Unshipped.txt b/src/DurableFlows/AsyncResponse.DurableFlows.Oracle/PublicAPI.Unshipped.txt index c3a00281c..cde847e72 100644 --- a/src/DurableFlows/AsyncResponse.DurableFlows.Oracle/PublicAPI.Unshipped.txt +++ b/src/DurableFlows/AsyncResponse.DurableFlows.Oracle/PublicAPI.Unshipped.txt @@ -9,12 +9,14 @@ AsyncResponse.DurableFlows.Oracle.OracleDurableFlowOptions.MaxStateBytes.set -> AsyncResponse.DurableFlows.Oracle.OracleDurableFlowOptions.OracleDurableFlowOptions() -> void AsyncResponse.DurableFlows.Oracle.OracleDurableFlowOptions.PruneInterval.get -> System.TimeSpan AsyncResponse.DurableFlows.Oracle.OracleDurableFlowOptions.PruneInterval.set -> void +AsyncResponse.DurableFlows.Oracle.OracleDurableFlowOptions.PruneBudget.get -> System.TimeSpan +AsyncResponse.DurableFlows.Oracle.OracleDurableFlowOptions.PruneBudget.set -> void AsyncResponse.DurableFlows.Oracle.OracleDurableFlowOptions.TableName.get -> string! AsyncResponse.DurableFlows.Oracle.OracleDurableFlowOptions.TableName.set -> void AsyncResponse.DurableFlows.Oracle.OracleDurableFlowOptions.Validate() -> void AsyncResponse.DurableFlows.Oracle.OracleFlowStateStore AsyncResponse.DurableFlows.Oracle.OracleFlowStateStore.LoadAsync(string! flowId, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! -AsyncResponse.DurableFlows.Oracle.OracleFlowStateStore.OracleFlowStateStore(Microsoft.Extensions.Options.IOptions! options) -> void +AsyncResponse.DurableFlows.Oracle.OracleFlowStateStore.OracleFlowStateStore(Microsoft.Extensions.Options.IOptions! options, Microsoft.Extensions.Logging.ILogger? logger = null) -> void AsyncResponse.DurableFlows.Oracle.OracleFlowStateStore.ReleaseLeaseAsync(string! flowId, string! leaseId, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! AsyncResponse.DurableFlows.Oracle.OracleFlowStateStore.TryAcquireLeaseAsync(string! flowId, string! leaseId, System.TimeSpan leaseDuration, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! AsyncResponse.DurableFlows.Oracle.OracleFlowStateStore.TryCreateAsync(string! flowId, AsyncResponse.FlowState! state, System.TimeSpan ttl, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! diff --git a/src/DurableFlows/AsyncResponse.DurableFlows.PostgreSQL/PostgreSqlDurableFlows.cs b/src/DurableFlows/AsyncResponse.DurableFlows.PostgreSQL/PostgreSqlDurableFlows.cs index 5d763b695..74cf14b0e 100644 --- a/src/DurableFlows/AsyncResponse.DurableFlows.PostgreSQL/PostgreSqlDurableFlows.cs +++ b/src/DurableFlows/AsyncResponse.DurableFlows.PostgreSQL/PostgreSqlDurableFlows.cs @@ -1,7 +1,9 @@ using AsyncResponse; using AsyncResponse.DurableFlows.Internal; using AsyncResponse.DurableFlows.PostgreSQL; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Npgsql; using NpgsqlTypes; @@ -33,12 +35,13 @@ public static AsyncResponseRegistrationBuilder WithPostgreSqlDurableFlows( // bare NpgsqlDataSource service, so unrelated resolutions of that type are never // answered — or broken — by this package. var shared = provider.GetService(); + var logger = provider.GetService>(); if (shared is not null) - return new PostgreSqlFlowStateStore(shared, options); + return new PostgreSqlFlowStateStore(shared, options, logger: logger); if (string.IsNullOrWhiteSpace(options.Value.ConnectionString)) throw new InvalidOperationException($"{nameof(PostgreSqlDurableFlowOptions)}.{nameof(PostgreSqlDurableFlowOptions.ConnectionString)} must be configured when no NpgsqlDataSource is registered."); - return new PostgreSqlFlowStateStore(NpgsqlDataSource.Create(options.Value.ConnectionString), options, ownsDataSource: true); + return new PostgreSqlFlowStateStore(NpgsqlDataSource.Create(options.Value.ConnectionString), options, ownsDataSource: true, logger: logger); }); return builder.WithDurableFlows(configure); } @@ -69,6 +72,18 @@ public sealed class PostgreSqlDurableFlowOptions : DurableFlowOptions /// public TimeSpan PruneInterval { get; set; } = TimeSpan.FromMinutes(5); + /// + /// Wall-clock budget one opportunistic prune may spend draining expired rows in batches of + /// 1000 after its first batch (the first always runs). A single batch per interval capped + /// cleanup at ~3 rows/second, which any busier instance outgrew forever; the prune now drains + /// batches until one comes back short or this budget lapses, and reports the outcome on the + /// AsyncResponse meter (asyncresponse.flow_state.pruned_rows, + /// prune_failures, prune_budget_exhausted) and the store's logger. The create + /// that triggers the prune waits for it, so this bounds that create's added latency. Zero + /// keeps the historical single batch. Default: 2 seconds. + /// + public TimeSpan PruneBudget { get; set; } = DurableFlowStoreShared.DefaultPruneBudget; + /// /// Maximum serialized flow-state size in bytes accepted by writes; oversized ledgers fail fast /// with an actionable error instead of an opaque provider error. Default: null @@ -89,13 +104,14 @@ public void Validate() throw new InvalidOperationException( $"{nameof(PostgreSqlDurableFlowOptions)}.{nameof(TableName)} '{TableName}' collides with its derived expiry-index name; rename the table."); DurableFlowStoreShared.ValidateMaxStateBytes(MaxStateBytes, nameof(PostgreSqlDurableFlowOptions)); + DurableFlowStoreShared.ValidatePruneBudget(PruneBudget, nameof(PostgreSqlDurableFlowOptions)); } } /// PostgreSQL implementation of . public sealed class PostgreSqlFlowStateStore : IFlowStateStore, IDisposable, IAsyncDisposable { - private const int PruneBatchSize = 1000; + private readonly ILogger? _logger; private readonly NpgsqlDataSource _dataSource; private readonly PostgreSqlDurableFlowOptions _options; @@ -105,9 +121,10 @@ public sealed class PostgreSqlFlowStateStore : IFlowStateStore, IDisposable, IAs private long _lastPruneTicks; private volatile bool _created; - public PostgreSqlFlowStateStore(NpgsqlDataSource dataSource, IOptions options, bool ownsDataSource = false) + public PostgreSqlFlowStateStore(NpgsqlDataSource dataSource, IOptions options, bool ownsDataSource = false, ILogger? logger = null) { _dataSource = dataSource; + _logger = logger; _options = options.Value; _options.Validate(); _ownsDataSource = ownsDataSource; @@ -140,7 +157,7 @@ public async Task TryCreateAsync(string flowId, FlowState state, TimeSpan var stateJson = DurableFlowStoreShared.SerializeBounded(flowId, state, _options.MaxStateBytes, "PostgreSQL"); await EnsureCreatedAsync(cancellationToken).ConfigureAwait(false); if (DurableFlowStoreShared.ShouldPrune(ref _lastPruneTicks, _options.PruneInterval)) - await DurableFlowStoreShared.PruneQuietlyAsync(() => PruneExpiredAsync(cancellationToken)).ConfigureAwait(false); + await DurableFlowStoreShared.PruneQuietlyAsync(() => PruneExpiredAsync(cancellationToken), _options.PruneBudget, "PostgreSQL", _logger).ConfigureAwait(false); await using var connection = await _dataSource.OpenConnectionAsync(cancellationToken).ConfigureAwait(false); await using var command = connection.CreateCommand(); @@ -228,9 +245,10 @@ public async Task TryDeleteAsync(string flowId, CancellationToken cancella return await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false) > 0; } - private async Task PruneExpiredAsync(CancellationToken cancellationToken) + private async Task PruneExpiredAsync(CancellationToken cancellationToken) { - // One bounded batch per prune interval (policy shared by all relational stores): an + // One bounded batch per call; DurableFlowStoreShared.PruneQuietlyAsync repeats it under + // the PruneBudget while batches come back full (policy shared by all relational stores): an // unbatched DELETE over a large expired backlog holds row locks and bloats one // transaction for the unlucky create that triggered the prune. Loads already filter on // expiry, so any backlog beyond the batch just waits for the next interval. @@ -239,9 +257,9 @@ private async Task PruneExpiredAsync(CancellationToken cancellationToken) command.CommandText = $""" DELETE FROM {Table} - WHERE ctid IN (SELECT ctid FROM {Table} WHERE expires_at_utc <= now() LIMIT {PruneBatchSize}); + WHERE ctid IN (SELECT ctid FROM {Table} WHERE expires_at_utc <= now() LIMIT {DurableFlowStoreShared.PruneBatchSize}); """; - await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + return await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); } private async Task EnsureCreatedAsync(CancellationToken cancellationToken) diff --git a/src/DurableFlows/AsyncResponse.DurableFlows.PostgreSQL/PublicAPI.Unshipped.txt b/src/DurableFlows/AsyncResponse.DurableFlows.PostgreSQL/PublicAPI.Unshipped.txt index f1a0b97cb..d6666c715 100644 --- a/src/DurableFlows/AsyncResponse.DurableFlows.PostgreSQL/PublicAPI.Unshipped.txt +++ b/src/DurableFlows/AsyncResponse.DurableFlows.PostgreSQL/PublicAPI.Unshipped.txt @@ -9,6 +9,8 @@ AsyncResponse.DurableFlows.PostgreSQL.PostgreSqlDurableFlowOptions.MaxStateBytes AsyncResponse.DurableFlows.PostgreSQL.PostgreSqlDurableFlowOptions.PostgreSqlDurableFlowOptions() -> void AsyncResponse.DurableFlows.PostgreSQL.PostgreSqlDurableFlowOptions.PruneInterval.get -> System.TimeSpan AsyncResponse.DurableFlows.PostgreSQL.PostgreSqlDurableFlowOptions.PruneInterval.set -> void +AsyncResponse.DurableFlows.PostgreSQL.PostgreSqlDurableFlowOptions.PruneBudget.get -> System.TimeSpan +AsyncResponse.DurableFlows.PostgreSQL.PostgreSqlDurableFlowOptions.PruneBudget.set -> void AsyncResponse.DurableFlows.PostgreSQL.PostgreSqlDurableFlowOptions.SchemaName.get -> string! AsyncResponse.DurableFlows.PostgreSQL.PostgreSqlDurableFlowOptions.SchemaName.set -> void AsyncResponse.DurableFlows.PostgreSQL.PostgreSqlDurableFlowOptions.TableName.get -> string! @@ -18,7 +20,7 @@ AsyncResponse.DurableFlows.PostgreSQL.PostgreSqlFlowStateStore AsyncResponse.DurableFlows.PostgreSQL.PostgreSqlFlowStateStore.Dispose() -> void AsyncResponse.DurableFlows.PostgreSQL.PostgreSqlFlowStateStore.DisposeAsync() -> System.Threading.Tasks.ValueTask AsyncResponse.DurableFlows.PostgreSQL.PostgreSqlFlowStateStore.LoadAsync(string! flowId, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! -AsyncResponse.DurableFlows.PostgreSQL.PostgreSqlFlowStateStore.PostgreSqlFlowStateStore(Npgsql.NpgsqlDataSource! dataSource, Microsoft.Extensions.Options.IOptions! options, bool ownsDataSource = false) -> void +AsyncResponse.DurableFlows.PostgreSQL.PostgreSqlFlowStateStore.PostgreSqlFlowStateStore(Npgsql.NpgsqlDataSource! dataSource, Microsoft.Extensions.Options.IOptions! options, bool ownsDataSource = false, Microsoft.Extensions.Logging.ILogger? logger = null) -> void AsyncResponse.DurableFlows.PostgreSQL.PostgreSqlFlowStateStore.ReleaseLeaseAsync(string! flowId, string! leaseId, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! AsyncResponse.DurableFlows.PostgreSQL.PostgreSqlFlowStateStore.TryAcquireLeaseAsync(string! flowId, string! leaseId, System.TimeSpan leaseDuration, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! AsyncResponse.DurableFlows.PostgreSQL.PostgreSqlFlowStateStore.TryCreateAsync(string! flowId, AsyncResponse.FlowState! state, System.TimeSpan ttl, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! diff --git a/src/DurableFlows/AsyncResponse.DurableFlows.SqlServer/PublicAPI.Unshipped.txt b/src/DurableFlows/AsyncResponse.DurableFlows.SqlServer/PublicAPI.Unshipped.txt index 3f661a2a4..bb0fc1b04 100644 --- a/src/DurableFlows/AsyncResponse.DurableFlows.SqlServer/PublicAPI.Unshipped.txt +++ b/src/DurableFlows/AsyncResponse.DurableFlows.SqlServer/PublicAPI.Unshipped.txt @@ -8,6 +8,8 @@ AsyncResponse.DurableFlows.SqlServer.SqlServerDurableFlowOptions.MaxStateBytes.g AsyncResponse.DurableFlows.SqlServer.SqlServerDurableFlowOptions.MaxStateBytes.set -> void AsyncResponse.DurableFlows.SqlServer.SqlServerDurableFlowOptions.PruneInterval.get -> System.TimeSpan AsyncResponse.DurableFlows.SqlServer.SqlServerDurableFlowOptions.PruneInterval.set -> void +AsyncResponse.DurableFlows.SqlServer.SqlServerDurableFlowOptions.PruneBudget.get -> System.TimeSpan +AsyncResponse.DurableFlows.SqlServer.SqlServerDurableFlowOptions.PruneBudget.set -> void AsyncResponse.DurableFlows.SqlServer.SqlServerDurableFlowOptions.SchemaName.get -> string! AsyncResponse.DurableFlows.SqlServer.SqlServerDurableFlowOptions.SchemaName.set -> void AsyncResponse.DurableFlows.SqlServer.SqlServerDurableFlowOptions.SqlServerDurableFlowOptions() -> void @@ -17,7 +19,7 @@ AsyncResponse.DurableFlows.SqlServer.SqlServerDurableFlowOptions.Validate() -> v AsyncResponse.DurableFlows.SqlServer.SqlServerFlowStateStore AsyncResponse.DurableFlows.SqlServer.SqlServerFlowStateStore.LoadAsync(string! flowId, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! AsyncResponse.DurableFlows.SqlServer.SqlServerFlowStateStore.ReleaseLeaseAsync(string! flowId, string! leaseId, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! -AsyncResponse.DurableFlows.SqlServer.SqlServerFlowStateStore.SqlServerFlowStateStore(Microsoft.Extensions.Options.IOptions! options) -> void +AsyncResponse.DurableFlows.SqlServer.SqlServerFlowStateStore.SqlServerFlowStateStore(Microsoft.Extensions.Options.IOptions! options, Microsoft.Extensions.Logging.ILogger? logger = null) -> void AsyncResponse.DurableFlows.SqlServer.SqlServerFlowStateStore.TryAcquireLeaseAsync(string! flowId, string! leaseId, System.TimeSpan leaseDuration, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! AsyncResponse.DurableFlows.SqlServer.SqlServerFlowStateStore.TryCreateAsync(string! flowId, AsyncResponse.FlowState! state, System.TimeSpan ttl, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! AsyncResponse.DurableFlows.SqlServer.SqlServerFlowStateStore.TryDeleteAsync(string! flowId, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! diff --git a/src/DurableFlows/AsyncResponse.DurableFlows.SqlServer/SqlServerDurableFlows.cs b/src/DurableFlows/AsyncResponse.DurableFlows.SqlServer/SqlServerDurableFlows.cs index 0338a41a4..ac6ddd0cb 100644 --- a/src/DurableFlows/AsyncResponse.DurableFlows.SqlServer/SqlServerDurableFlows.cs +++ b/src/DurableFlows/AsyncResponse.DurableFlows.SqlServer/SqlServerDurableFlows.cs @@ -4,6 +4,7 @@ using AsyncResponse.Internal; using Microsoft.Data.SqlClient; using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; namespace Microsoft.Extensions.DependencyInjection @@ -49,6 +50,18 @@ public sealed class SqlServerDurableFlowOptions : DurableFlowOptions /// public TimeSpan PruneInterval { get; set; } = TimeSpan.FromMinutes(5); + /// + /// Wall-clock budget one opportunistic prune may spend draining expired rows in batches of + /// 1000 after its first batch (the first always runs). A single batch per interval capped + /// cleanup at ~3 rows/second, which any busier instance outgrew forever; the prune now drains + /// batches until one comes back short or this budget lapses, and reports the outcome on the + /// AsyncResponse meter (asyncresponse.flow_state.pruned_rows, + /// prune_failures, prune_budget_exhausted) and the store's logger. The create + /// that triggers the prune waits for it, so this bounds that create's added latency. Zero + /// keeps the historical single batch. Default: 2 seconds. + /// + public TimeSpan PruneBudget { get; set; } = DurableFlowStoreShared.DefaultPruneBudget; + /// /// Maximum serialized flow-state size in bytes accepted by writes; oversized ledgers fail fast /// with an actionable error instead of an opaque provider error. Default: null @@ -63,23 +76,25 @@ public void Validate() DurableFlowStoreShared.ValidateIdentifier(SchemaName, $"{nameof(SqlServerDurableFlowOptions)}.{nameof(SchemaName)}", "SQL Server", identifierCap: 128); DurableFlowStoreShared.ValidateIdentifier(TableName, $"{nameof(SqlServerDurableFlowOptions)}.{nameof(TableName)}", "SQL Server", identifierCap: 128); DurableFlowStoreShared.ValidateMaxStateBytes(MaxStateBytes, nameof(SqlServerDurableFlowOptions)); + DurableFlowStoreShared.ValidatePruneBudget(PruneBudget, nameof(SqlServerDurableFlowOptions)); } } /// SQL Server implementation of . public sealed class SqlServerFlowStateStore : IFlowStateStore { - private const int PruneBatchSize = 1000; + private readonly ILogger? _logger; private readonly SqlServerDurableFlowOptions _options; private readonly SemaphoreSlim _ensureGate = new(1, 1); private long _lastPruneTicks; private volatile bool _created; - public SqlServerFlowStateStore(IOptions options) + public SqlServerFlowStateStore(IOptions options, ILogger? logger = null) { _options = options.Value; _options.Validate(); + _logger = logger; } public async Task LoadAsync(string flowId, CancellationToken cancellationToken = default) @@ -108,7 +123,7 @@ public async Task TryCreateAsync(string flowId, FlowState state, TimeSpan var stateJson = DurableFlowStoreShared.SerializeBounded(flowId, state, _options.MaxStateBytes, "SQL Server"); await EnsureCreatedAsync(cancellationToken).ConfigureAwait(false); if (DurableFlowStoreShared.ShouldPrune(ref _lastPruneTicks, _options.PruneInterval)) - await DurableFlowStoreShared.PruneQuietlyAsync(() => PruneExpiredAsync(cancellationToken)).ConfigureAwait(false); + await DurableFlowStoreShared.PruneQuietlyAsync(() => PruneExpiredAsync(cancellationToken), _options.PruneBudget, "SQL Server", _logger).ConfigureAwait(false); await using var connection = await OpenConnectionAsync(cancellationToken).ConfigureAwait(false); await using var command = connection.CreateCommand(); @@ -198,16 +213,17 @@ public async Task TryDeleteAsync(string flowId, CancellationToken cancella return await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false) > 0; } - private async Task PruneExpiredAsync(CancellationToken cancellationToken) + private async Task PruneExpiredAsync(CancellationToken cancellationToken) { - // One bounded batch per prune interval (policy shared by all relational stores): an + // One bounded batch per call; DurableFlowStoreShared.PruneQuietlyAsync repeats it under + // the PruneBudget while batches come back full (policy shared by all relational stores): an // unbatched DELETE over a large expired backlog holds row locks and bloats one // transaction for the unlucky create that triggered the prune. Loads already filter on // expiry, so any backlog beyond the batch just waits for the next interval. await using var connection = await OpenConnectionAsync(cancellationToken).ConfigureAwait(false); await using var command = connection.CreateCommand(); - command.CommandText = $"DELETE TOP ({PruneBatchSize}) FROM {Table} WHERE expires_at_utc <= SYSUTCDATETIME();"; - await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + command.CommandText = $"DELETE TOP ({DurableFlowStoreShared.PruneBatchSize}) FROM {Table} WHERE expires_at_utc <= SYSUTCDATETIME();"; + return await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); } private async Task EnsureCreatedAsync(CancellationToken cancellationToken) diff --git a/src/DurableFlows/AsyncResponse.DurableFlows.Sqlite/PublicAPI.Unshipped.txt b/src/DurableFlows/AsyncResponse.DurableFlows.Sqlite/PublicAPI.Unshipped.txt index 211ab3db0..d1e9ba706 100644 --- a/src/DurableFlows/AsyncResponse.DurableFlows.Sqlite/PublicAPI.Unshipped.txt +++ b/src/DurableFlows/AsyncResponse.DurableFlows.Sqlite/PublicAPI.Unshipped.txt @@ -8,6 +8,8 @@ AsyncResponse.DurableFlows.Sqlite.SqliteDurableFlowOptions.MaxStateBytes.get -> AsyncResponse.DurableFlows.Sqlite.SqliteDurableFlowOptions.MaxStateBytes.set -> void AsyncResponse.DurableFlows.Sqlite.SqliteDurableFlowOptions.PruneInterval.get -> System.TimeSpan AsyncResponse.DurableFlows.Sqlite.SqliteDurableFlowOptions.PruneInterval.set -> void +AsyncResponse.DurableFlows.Sqlite.SqliteDurableFlowOptions.PruneBudget.get -> System.TimeSpan +AsyncResponse.DurableFlows.Sqlite.SqliteDurableFlowOptions.PruneBudget.set -> void AsyncResponse.DurableFlows.Sqlite.SqliteDurableFlowOptions.SqliteDurableFlowOptions() -> void AsyncResponse.DurableFlows.Sqlite.SqliteDurableFlowOptions.TableName.get -> string! AsyncResponse.DurableFlows.Sqlite.SqliteDurableFlowOptions.TableName.set -> void @@ -15,7 +17,7 @@ AsyncResponse.DurableFlows.Sqlite.SqliteDurableFlowOptions.Validate() -> void AsyncResponse.DurableFlows.Sqlite.SqliteFlowStateStore AsyncResponse.DurableFlows.Sqlite.SqliteFlowStateStore.LoadAsync(string! flowId, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! AsyncResponse.DurableFlows.Sqlite.SqliteFlowStateStore.ReleaseLeaseAsync(string! flowId, string! leaseId, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! -AsyncResponse.DurableFlows.Sqlite.SqliteFlowStateStore.SqliteFlowStateStore(Microsoft.Extensions.Options.IOptions! options) -> void +AsyncResponse.DurableFlows.Sqlite.SqliteFlowStateStore.SqliteFlowStateStore(Microsoft.Extensions.Options.IOptions! options, Microsoft.Extensions.Logging.ILogger? logger = null) -> void AsyncResponse.DurableFlows.Sqlite.SqliteFlowStateStore.TryAcquireLeaseAsync(string! flowId, string! leaseId, System.TimeSpan leaseDuration, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! AsyncResponse.DurableFlows.Sqlite.SqliteFlowStateStore.TryCreateAsync(string! flowId, AsyncResponse.FlowState! state, System.TimeSpan ttl, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! AsyncResponse.DurableFlows.Sqlite.SqliteFlowStateStore.TryDeleteAsync(string! flowId, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! diff --git a/src/DurableFlows/AsyncResponse.DurableFlows.Sqlite/SqliteDurableFlows.cs b/src/DurableFlows/AsyncResponse.DurableFlows.Sqlite/SqliteDurableFlows.cs index 8e0ca3267..129897cb6 100644 --- a/src/DurableFlows/AsyncResponse.DurableFlows.Sqlite/SqliteDurableFlows.cs +++ b/src/DurableFlows/AsyncResponse.DurableFlows.Sqlite/SqliteDurableFlows.cs @@ -3,6 +3,7 @@ using AsyncResponse.DurableFlows.Sqlite; using Microsoft.Data.Sqlite; using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; namespace Microsoft.Extensions.DependencyInjection @@ -45,6 +46,18 @@ public sealed class SqliteDurableFlowOptions : DurableFlowOptions /// public TimeSpan PruneInterval { get; set; } = TimeSpan.FromMinutes(5); + /// + /// Wall-clock budget one opportunistic prune may spend draining expired rows in batches of + /// 1000 after its first batch (the first always runs). A single batch per interval capped + /// cleanup at ~3 rows/second, which any busier instance outgrew forever; the prune now drains + /// batches until one comes back short or this budget lapses, and reports the outcome on the + /// AsyncResponse meter (asyncresponse.flow_state.pruned_rows, + /// prune_failures, prune_budget_exhausted) and the store's logger. The create + /// that triggers the prune waits for it, so this bounds that create's added latency. Zero + /// keeps the historical single batch. Default: 2 seconds. + /// + public TimeSpan PruneBudget { get; set; } = DurableFlowStoreShared.DefaultPruneBudget; + /// /// Maximum serialized flow-state size in bytes accepted by writes; oversized ledgers fail fast /// with an actionable error instead of an opaque provider error. Default: null @@ -58,13 +71,14 @@ public void Validate() DurableFlowStoreShared.ValidateConnectionString(ConnectionString, nameof(SqliteDurableFlowOptions)); DurableFlowStoreShared.ValidateIdentifier(TableName, $"{nameof(SqliteDurableFlowOptions)}.{nameof(TableName)}", "SQLite"); DurableFlowStoreShared.ValidateMaxStateBytes(MaxStateBytes, nameof(SqliteDurableFlowOptions)); + DurableFlowStoreShared.ValidatePruneBudget(PruneBudget, nameof(SqliteDurableFlowOptions)); } } /// SQLite implementation of . public sealed class SqliteFlowStateStore : IFlowStateStore { - private const int PruneBatchSize = 1000; + private readonly ILogger? _logger; // Time authority: this store deliberately keeps the app clock (DateTime.UtcNow) for expiry // and lease comparisons. A SQLite database file lives on a single machine, and every writer @@ -84,8 +98,9 @@ public sealed class SqliteFlowStateStore : IFlowStateStore private long _lastPruneTicks; private volatile bool _created; - public SqliteFlowStateStore(IOptions options) + public SqliteFlowStateStore(IOptions options, ILogger? logger = null) { + _logger = logger; _options = options.Value; _options.Validate(); } @@ -123,7 +138,7 @@ public async Task TryCreateAsync( var stateJson = DurableFlowStoreShared.SerializeBounded(flowId, state, _options.MaxStateBytes, "SQLite"); await EnsureCreatedAsync(cancellationToken).ConfigureAwait(false); if (DurableFlowStoreShared.ShouldPrune(ref _lastPruneTicks, _options.PruneInterval)) - await DurableFlowStoreShared.PruneQuietlyAsync(() => PruneExpiredAsync(cancellationToken)).ConfigureAwait(false); + await DurableFlowStoreShared.PruneQuietlyAsync(() => PruneExpiredAsync(cancellationToken), _options.PruneBudget, "SQLite", _logger).ConfigureAwait(false); await using var connection = await OpenConnectionAsync(cancellationToken).ConfigureAwait(false); await using var command = connection.CreateCommand(); command.CommandText = @@ -214,10 +229,11 @@ public async Task TryDeleteAsync(string flowId, CancellationToken cancella return await ExecuteWriteAsync(command, cancellationToken).ConfigureAwait(false) > 0; } - private async Task PruneExpiredAsync(CancellationToken cancellationToken) + private async Task PruneExpiredAsync(CancellationToken cancellationToken) { // Timestamps are stored as ISO-8601 TEXT, which compares correctly lexicographically. - // One bounded batch per prune interval (policy shared by all relational stores): an + // One bounded batch per call; DurableFlowStoreShared.PruneQuietlyAsync repeats it under + // the PruneBudget while batches come back full (policy shared by all relational stores): an // unbatched DELETE over a large expired backlog holds the single SQLite write lock for // the whole sweep. Loads already filter on expiry, so any backlog beyond the batch just // waits for the next interval. Id-subquery form because DELETE ... LIMIT needs a @@ -227,10 +243,10 @@ private async Task PruneExpiredAsync(CancellationToken cancellationToken) command.CommandText = $""" DELETE FROM {Table} - WHERE flow_id IN (SELECT flow_id FROM {Table} WHERE expires_at_utc <= $now_utc LIMIT {PruneBatchSize}); + WHERE flow_id IN (SELECT flow_id FROM {Table} WHERE expires_at_utc <= $now_utc LIMIT {DurableFlowStoreShared.PruneBatchSize}); """; command.Parameters.AddWithValue("$now_utc", DateTime.UtcNow); - await ExecuteWriteAsync(command, cancellationToken).ConfigureAwait(false); + return await ExecuteWriteAsync(command, cancellationToken).ConfigureAwait(false); } private async Task EnsureCreatedAsync(CancellationToken cancellationToken) diff --git a/src/DurableFlows/Shared/DurableFlowStoreShared.cs b/src/DurableFlows/Shared/DurableFlowStoreShared.cs index 7b7df3ffd..ecbbab58c 100644 --- a/src/DurableFlows/Shared/DurableFlowStoreShared.cs +++ b/src/DurableFlows/Shared/DurableFlowStoreShared.cs @@ -1,35 +1,97 @@ using System.Data.Common; +using System.Diagnostics; using System.Text; +using Microsoft.Extensions.Logging; namespace AsyncResponse.DurableFlows.Internal; internal static class DurableFlowStoreShared { /// - /// Runs an opportunistic prune so that its failure never fails the primitive it rides on. - /// Awaited bare inside TryCreateAsync, a prune chosen as the deadlock victim (1205) - /// or hitting a lock-wait timeout against the store's own live checkpoint traffic failed - /// StartAsync for a flow whose row would have been created without incident — and - /// had already consumed the interval, so it was not retried either. - /// Loads filter on expiry, so a skipped prune costs nothing but disk until the next interval. - /// Cancellation still propagates. + /// Rows one prune statement deletes. Every relational store deletes in batches of this size: + /// an unbatched DELETE over a large expired backlog holds row locks and bloats one transaction + /// for the unlucky create that triggered the prune. /// - public static async Task PruneQuietlyAsync(Func prune) + public const int PruneBatchSize = 1000; + + /// + /// The default PruneBudget: wall-clock time one opportunistic prune may spend draining + /// batches after the first. Two seconds at ~1000 rows per batch drains tens of thousands of + /// expired rows per interval on an ordinary database, against the ~3 rows/second a single + /// batch per five-minute interval sustained — which any instance creating more than that fell + /// behind forever. + /// + public static readonly TimeSpan DefaultPruneBudget = TimeSpan.FromSeconds(2); + + /// + /// Runs an opportunistic prune so that its failure never fails the primitive it rides on, + /// draining -row batches until a batch comes back short (the + /// backlog is gone) or lapses. The first batch always runs, so a + /// zero budget is the historical single-batch policy. Awaited bare inside + /// TryCreateAsync, a prune chosen as the deadlock victim (1205) or hitting a lock-wait + /// timeout against the store's own live checkpoint traffic failed StartAsync for a flow + /// whose row would have been created without incident — and had + /// already consumed the interval, so it was not retried either. Loads filter on expiry, so a + /// skipped prune costs nothing but disk until the next interval. The outcome is never silent: + /// deleted rows, a lapsed budget with rows remaining, and failures are counted on the + /// AsyncResponse meter and logged when the store has a logger. Cancellation still + /// propagates. + /// + /// Deletes one batch and returns the rows it deleted. + /// Wall-clock budget for batches after the first. + /// Metric/log tag for the store ("PostgreSQL", "SQL Server", …). + /// The store's logger when DI supplied one. + public static async Task PruneQuietlyAsync(Func> pruneBatch, TimeSpan budget, string providerName, ILogger? logger) { + var started = Stopwatch.GetTimestamp(); + var deleted = 0L; + var batches = 0; try { - await prune().ConfigureAwait(false); + while (true) + { + var batchDeleted = await pruneBatch().ConfigureAwait(false); + batches++; + deleted += Math.Max(batchDeleted, 0); + if (batchDeleted < PruneBatchSize) + break; + + if (Stopwatch.GetElapsedTime(started) >= budget) + { + AsyncResponseDiagnostics.RecordFlowStatePruneBudgetExhausted(providerName); + logger?.LogWarning( + "{Provider} durable-flow prune deleted {Deleted} expired rows in {Batches} batches and stopped at its {Budget} PruneBudget with expired rows remaining; the backlog is outgrowing the prune — raise PruneBudget or shorten PruneInterval.", + providerName, deleted, batches, budget); + break; + } + } + + AsyncResponseDiagnostics.RecordFlowStatePruned(providerName, deleted); } catch (OperationCanceledException) { + AsyncResponseDiagnostics.RecordFlowStatePruned(providerName, deleted); throw; } - catch (Exception) + catch (Exception ex) { - // Opportunistic maintenance; the next interval retries. + // Opportunistic maintenance; the next interval retries — but never silently. + AsyncResponseDiagnostics.RecordFlowStatePruned(providerName, deleted); + AsyncResponseDiagnostics.RecordFlowStatePruneFailure(providerName); + logger?.LogWarning( + ex, + "{Provider} durable-flow prune failed after deleting {Deleted} expired rows in {Batches} batches; the flow creation it rode on is unaffected and the next PruneInterval retries.", + providerName, deleted, batches); } } + /// A PruneBudget is a non-negative duration; zero means a single batch per interval. + public static void ValidatePruneBudget(TimeSpan budget, string optionsName) + { + if (budget < TimeSpan.Zero) + throw new InvalidOperationException($"{optionsName}.PruneBudget cannot be negative (zero limits each prune to one batch)."); + } + /// /// Upper bound for TTL values handed to server-clock date arithmetic (~68 years). SQL Server's /// DATEADD takes int seconds, and MySQL/Oracle datetime types stop at year 9999, @@ -102,27 +164,38 @@ public static string SerializeBounded(string flowId, FlowState state, long? maxS /// /// Materializes a loaded ledger row. /// - /// A revision that does not match the stored row, and an identity-mismatched ledger - /// (state.FlowId != flowId), load as absent — the read-side mirror of the write-side - /// key/identity validation in , so a row copied or restored under - /// the wrong key can never resurrect as that flow. + /// The row is never executed as anything but what it consistently says it is: a revision + /// inside the JSON that disagrees with the row's own revision column, or a ledger whose + /// FlowId is not the key it was loaded under (a row copied or restored under the + /// wrong key), is refused — the read-side mirror of the write-side key/identity validation + /// in . /// /// - /// Unreadable JSON and an unknown schema version do NOT: they say the row is there and this - /// build cannot interpret it, so they throw - /// rather than impersonating a deleted flow. That - /// distinction is what stops a rolling deployment from acknowledging a live flow's only - /// wake-up (see the exception's remarks). + /// Refused means , never null. Every built-in + /// store reads the JSON and the revision from ONE row or document, so a disagreement inside + /// that snapshot is an inconsistent — corrupt, hand-edited, mis-restored — ledger that is + /// physically present, not proof the run is gone. A null here told the executor to + /// acknowledge the wake-up as belonging to a deleted flow, and the run behind the row lost + /// its only wake-up while its row sat in the table. Unreadable JSON and an unknown schema + /// version throw for the same reason (see the exception's remarks); the delivery rides the + /// transport's retry and dead-letter path, which is the operator alarm. /// /// - /// The row is present but uninterpretable. + /// The row is present but uninterpretable or inconsistent. public static FlowState? ReadState(string flowId, string stateJson, long revision) { var state = Deserialize(stateJson, flowId); - return state.Revision == revision - && string.Equals(state.FlowId, flowId, StringComparison.Ordinal) - ? state - : null; + if (state.Revision != revision) + { + throw new FlowStateUnreadableException( + flowId, + $"its stored revision is {revision} but the revision inside its JSON is {state.Revision}"); + } + + if (!string.Equals(state.FlowId, flowId, StringComparison.Ordinal)) + throw new FlowStateUnreadableException(flowId, "the flow id inside its JSON is not the id it is stored under"); + + return state; } /// diff --git a/src/Transports/AsyncResponse.Transports.Kafka/KafkaMessageDispatcher.cs b/src/Transports/AsyncResponse.Transports.Kafka/KafkaMessageDispatcher.cs index ca9954ec9..ee276c324 100644 --- a/src/Transports/AsyncResponse.Transports.Kafka/KafkaMessageDispatcher.cs +++ b/src/Transports/AsyncResponse.Transports.Kafka/KafkaMessageDispatcher.cs @@ -26,6 +26,31 @@ internal sealed record KafkaDelivery( /// with bounded backoff; a message that exhausts its attempts is produced to the dead-letter topic /// and its offset stored so the partition keeps moving. /// +/// +/// A message exhausted its handling and its dead-letter publish failed for good, so it is +/// neither buried nor committable. Thrown out of the poll loop ON PURPOSE: Kafka commits a +/// partition position, not per-record acknowledgements, so merely leaving this message's +/// offset unstored (the previous behavior) protected nothing — the next successful settlement on +/// the same partition stored a higher offset, the auto-committer committed past the failed +/// message, and a restart skipped it with no dead-letter copy anywhere. Faulting the subscriber +/// instead stops the partition at the unresolved message: the consumer closes without ever +/// storing past it, the supervisor rebuilds it after its backoff, and the message is re-consumed +/// and its burial retried until the dead-letter topic is back. That is a loud, bounded-rate loop +/// (every restart logs this failure) and a stalled subscriber — the at-least-once outcome — rather +/// than a silent loss. +/// +internal sealed class KafkaDeadLetterPublishFailedException(string topic, int partition, long offset, Exception innerException) + : Exception( + $"Kafka message {topic}[{partition}]@{offset} could not be dead-lettered after exhausting its handling attempts. " + + "Its offset is left unstored and the subscriber is restarted so no later settlement on the partition commits past it; " + + "fix the dead-letter topic to let the partition advance.", + innerException) +{ + public string Topic { get; } = topic; + public int Partition { get; } = partition; + public long Offset { get; } = offset; +} + internal abstract class KafkaMessageDispatcher : IAsyncDisposable { private readonly Func _handler; @@ -63,6 +88,7 @@ protected KafkaMessageDispatcher( protected KafkaAsyncResponseTransportOptions TransportOptions { get; } protected ILogger Logger { get; } + protected IKafkaConsumerClient Consumer => _consumer; protected int MaxDeliveryAttempts => _subscriberOptions.MaxDeliveryAttempts; @@ -118,10 +144,10 @@ public static void ValidateOptions( ? $"{nameof(KafkaAsyncResponseTransportOptions)}.{nameof(KafkaAsyncResponseTransportOptions.WorkerSubscriber)}" : $"{nameof(KafkaAsyncResponseTransportOptions)}.{nameof(KafkaAsyncResponseTransportOptions.ResponseSubscriber)}"; - // PollTimeout goes to Consume(TimeSpan), which librdkafka takes as 32-bit milliseconds; - // the backpressure and handler-retry delays arm in-process Task.Delay timers. + // PollTimeout and BackpressurePollDelay both go to Consume(TimeSpan), which librdkafka + // takes as 32-bit milliseconds; the handler-retry delays arm in-process Task.Delay timers. KafkaTransportOptionsValidator.EnsureIntMilliseconds(subscriberOptions.PollTimeout, optionPath, nameof(KafkaSubscriberOptions.PollTimeout)); - AsyncResponseChannelOptions.EnsureTimerBacked(subscriberOptions.BackpressurePollDelay, optionPath, nameof(KafkaSubscriberOptions.BackpressurePollDelay)); + KafkaTransportOptionsValidator.EnsureIntMilliseconds(subscriberOptions.BackpressurePollDelay, optionPath, nameof(KafkaSubscriberOptions.BackpressurePollDelay)); if (subscriberOptions.MaxDeliveryAttempts < 0) throw new InvalidOperationException($"{optionPath}.{nameof(KafkaSubscriberOptions.MaxDeliveryAttempts)} cannot be negative."); AsyncResponseChannelOptions.EnsureTimerBacked(subscriberOptions.HandlerRetryBaseDelay, optionPath, nameof(KafkaSubscriberOptions.HandlerRetryBaseDelay)); @@ -134,32 +160,30 @@ public static void ValidateOptions( } KafkaTransportOptionsValidator.EnsureMaxPollInterval(subscriberOptions.MaxPollInterval, optionPath, nameof(KafkaSubscriberOptions.MaxPollInterval)); - - // The in-process retry loop runs on the poll thread, so its delays suspend Consume(); - // a poll gap reaching max.poll.interval.ms gets the consumer evicted from its group - // mid-retry and its partitions redelivered elsewhere. The retry DELAY budget (handler - // execution time is the operator's responsibility) plus one poll must fit within half - // the interval so real handler time has the other half. Unlimited retries - // (MaxDeliveryAttempts = 0) have no finite budget and stay the operator's call. - if (subscriberOptions.MaxDeliveryAttempts > 0) - { - var retryDelayBudgetMs = WorstCaseRetryDelayBudgetMs(subscriberOptions); - var pollGapMs = retryDelayBudgetMs + subscriberOptions.PollTimeout.TotalMilliseconds; - if (pollGapMs * 2 > subscriberOptions.MaxPollInterval.TotalMilliseconds) - { - throw new InvalidOperationException( - $"{optionPath}: the worst-case in-process handler retry delay budget plus one poll " + - $"({DescribeMilliseconds(pollGapMs)} across {subscriberOptions.MaxDeliveryAttempts} delivery attempts) must fit within half of " + - $"{nameof(KafkaSubscriberOptions.MaxPollInterval)} ({subscriberOptions.MaxPollInterval}) — these delays run on the poll thread, and a " + - "poll gap reaching max.poll.interval.ms gets the consumer evicted from its group mid-retry. Reduce " + - $"{nameof(KafkaSubscriberOptions.MaxDeliveryAttempts)}, {nameof(KafkaSubscriberOptions.HandlerRetryBaseDelay)}, or " + - $"{nameof(KafkaSubscriberOptions.HandlerRetryMaxDelay)}, or raise {nameof(KafkaSubscriberOptions.MaxPollInterval)}."); - } - } + AsyncResponseChannelOptions.EnsureTimerBackedAllowZero(subscriberOptions.DetachHandlerAfter, optionPath, nameof(KafkaSubscriberOptions.DetachHandlerAfter)); + AsyncResponseChannelOptions.EnsureTimerBackedAllowZero(subscriberOptions.FaultDrainTimeout, optionPath, nameof(KafkaSubscriberOptions.FaultDrainTimeout)); switch (subscriberOptions.AckMode) { case KafkaAckMode.AckAfterHandlerCompletes: + // The poll thread's longest gap in this mode is one inline handler wait plus one + // poll; a gap reaching max.poll.interval.ms gets the consumer evicted from its + // group and its partitions redelivered elsewhere. Half the interval is the margin. + // Handler execution time and the in-process retry ladder no longer count: past + // DetachHandlerAfter the handler runs detached while the poll thread keeps polling + // (the earlier rule bounded the retry DELAYS for that reason, and left real handler + // time — a flow step awaiting a remote response — to overrun the interval anyway). + var pollGapMs = subscriberOptions.DetachHandlerAfter.TotalMilliseconds + subscriberOptions.PollTimeout.TotalMilliseconds; + if (pollGapMs * 2 > subscriberOptions.MaxPollInterval.TotalMilliseconds) + { + throw new InvalidOperationException( + $"{optionPath}: {nameof(KafkaSubscriberOptions.DetachHandlerAfter)} ({subscriberOptions.DetachHandlerAfter}) plus " + + $"{nameof(KafkaSubscriberOptions.PollTimeout)} ({subscriberOptions.PollTimeout}) must fit within half of " + + $"{nameof(KafkaSubscriberOptions.MaxPollInterval)} ({subscriberOptions.MaxPollInterval}) — that sum is the poll thread's " + + "longest gap, and a gap reaching max.poll.interval.ms gets the consumer evicted from its group. Lower " + + $"{nameof(KafkaSubscriberOptions.DetachHandlerAfter)} or raise {nameof(KafkaSubscriberOptions.MaxPollInterval)}."); + } + return; case KafkaAckMode.AckAfterEnqueue: @@ -195,43 +219,60 @@ public static void ValidateOptions( } } + /// Handles the delivered message through to settlement, offset store included. + public abstract Task HandleAsync(KafkaDelivery delivery, CancellationToken subscriberCancellationToken); + /// - /// Ceiling of the retry delays a failing message can spend on the poll thread: one backoff per - /// completed attempt except the last. Mirrors 's - /// pre-jitter shape — min(max, base * 2^min(attempt-1, 10)); jitter only ever shrinks a - /// step — and is computed in milliseconds because a large attempt count times the capped step - /// overflows . + /// The poll thread's entry point for a consumed message. Returns once the message is settled + /// (offset stored, or dead-lettered and stored) or — for the awaiting dispatcher — once its + /// handler has been detached to run on while polling continues. Throws when the message cannot + /// be settled (a permanently failing burial, cancellation), which faults the poll loop so the + /// subscriber is rebuilt without ever committing past the message. /// - private static double WorstCaseRetryDelayBudgetMs(KafkaSubscriberOptions subscriberOptions) - { - var baseMs = subscriberOptions.HandlerRetryBaseDelay.TotalMilliseconds; - var maxMs = subscriberOptions.HandlerRetryMaxDelay.TotalMilliseconds; - var delays = subscriberOptions.MaxDeliveryAttempts - 1; - - var totalMs = 0d; - for (var attempt = 1; attempt <= Math.Min(delays, 11); attempt++) - totalMs += Math.Min(maxMs, baseMs * (1 << (attempt - 1))); + public virtual void Accept(KafkaDelivery delivery, CancellationToken subscriberCancellationToken) + => HandleAsync(delivery, subscriberCancellationToken).GetAwaiter().GetResult(); - // The multiplier saturates at 2^10, so every later delay is the same capped step. - if (delays > 11) - totalMs += (delays - 11) * Math.Min(maxMs, baseMs * 1024); + /// + /// The poll thread's entry point for a consumed message that could not be turned into a + /// delivery ( describes the settlement). Default: + /// settled at once — the queued dispatcher stores every offset at enqueue, in consumption + /// order, so nothing earlier on the partition is still unresolved. The awaiting dispatcher + /// overrides it to hold the message behind a detached handler of the same partition: its + /// offset must not be stored — and so committed — ahead of a message consumed before it that + /// is still being handled. + /// + public virtual void AcceptUnprocessable(KafkaIncomingMessage message, Exception failure, CancellationToken subscriberCancellationToken) + => DiscardUnprocessableAsync(message, failure, subscriberCancellationToken).GetAwaiter().GetResult(); - return totalMs; + /// + /// Poll-thread tick: settles detached handlers that have finished — offset stored, partition + /// resumed, the next held message started. Throws when one of them failed for good (the poll + /// loop faults, exactly as an inline failure would). + /// + public virtual void SettleCompleted() + { } - private static string DescribeMilliseconds(double milliseconds) - => milliseconds <= TimeSpan.MaxValue.TotalMilliseconds - ? TimeSpan.FromMilliseconds(milliseconds).ToString() - : $"more than {TimeSpan.MaxValue}"; + /// + /// The poll loop FAILED (as opposed to a stop) and the consumer is about to be closed and + /// rebuilt by the supervisor. Default: the graceful drain. The awaiting dispatcher overrides + /// it with a bounded wait () so the + /// reconnect is not parked behind an unrelated long handler. + /// + public virtual ValueTask TeardownAfterFaultAsync() => DisposeAsync(); - /// Handles the delivered message. - public abstract Task HandleAsync(KafkaDelivery delivery, CancellationToken subscriberCancellationToken); + /// + /// Whether detached handlers are in flight. The poll loop then polls in + /// slices so a completion is settled + /// promptly instead of after a full . + /// + public virtual bool HasDetachedWork => false; /// /// Whether the dispatcher can accept more deliveries right now. Awaiting dispatchers always can - /// (handlers run inline); the queued dispatcher returns false while its bounded queue is - /// saturated so the subscriber pauses partition fetching instead of buffering an unbounded - /// backlog in-process. + /// (a partition with a detached handler is paused, so nothing arrives for it); the queued + /// dispatcher returns false while its bounded queue is saturated so the subscriber + /// pauses partition fetching instead of buffering an unbounded backlog in-process. /// public virtual bool CanAcceptMore => true; @@ -372,10 +413,11 @@ public async Task DiscardUnprocessableAsync( // Settlement ignores the stopping token, as every sibling settlement path does: a shutdown // landing between the dead-letter publish and the offset store would abort the publish - // mid-flight and leave the poison message neither buried nor committed. Guarded for the - // same reason as the at-the-cap publish: this runs inside the poll loop, and a permanently - // failing dead-letter topic would otherwise fault the subscriber into a restart loop with - // the offset unstored. + // mid-flight and leave the poison message neither buried nor committed. A burial that + // fails for good FAULTS the poll loop (see KafkaDeadLetterPublishFailedException): an + // earlier round swallowed it and left the offset unstored, which looked safe but was not — + // the next settlement on the same partition committed past this message. The restart loop + // it replaces is bounded by the supervisor's backoff and is the at-least-once outcome. try { await DeadLetterCoreAsync( @@ -394,11 +436,11 @@ await DeadLetterCoreAsync( { Logger.LogError( deadLetterException, - "Failed to dead-letter unprocessable Kafka message {Topic}[{Partition}]@{Offset}; its offset is left unstored so the burial is retried after a restart or rebalance, but later settlements on the partition can commit past it — fix the dead-letter topic promptly.", + "Failed to dead-letter unprocessable Kafka message {Topic}[{Partition}]@{Offset}; its offset is left unstored and the subscriber restarts so no later settlement commits past it. The partition is stalled until the dead-letter topic is fixed.", message.Topic, message.Partition, message.Offset); - return; + throw new KafkaDeadLetterPublishFailedException(message.Topic, message.Partition, message.Offset, deadLetterException); } // Guarded like every other settlement: a rebalance revoking this partition makes @@ -436,12 +478,15 @@ private async Task DeadLetterCoreAsync( headers.Add(KafkaTransportHeader.Utf8("exceptionMessage", exception.Message)); headers.Add(KafkaTransportHeader.Utf8("occurredAtUtc", DateTimeOffset.UtcNow.ToString("O"))); - // Both burial callers block the poll thread on this, and a produce to an undeliverable - // dead-letter topic waits out librdkafka's message.timeout.ms (5 min by default) PER - // attempt — past max.poll.interval.ms, which evicted the consumer mid-burial and - // rebalanced the partition to a peer that hit the same message: a rebalance storm at - // zero throughput. Bound the whole ladder to a quarter of the poll interval; every caller - // already treats a failed burial as "offset left unstored, retried after restart/rebalance". + // The unprocessable-message discard blocks the poll thread on this (the awaiting + // dispatcher's burial runs inside the detached handler task now, but keeps the same bound + // so a partition is not parked on an undeliverable dead-letter topic for message.timeout.ms + // per attempt either): a produce to an undeliverable dead-letter topic waits out + // librdkafka's message.timeout.ms (5 min by default) PER attempt — past + // max.poll.interval.ms, which evicted the consumer mid-burial and rebalanced the partition + // to a peer that hit the same message: a rebalance storm at zero throughput. Bound the whole + // ladder to a quarter of the poll interval; every caller already treats a failed burial as + // "offset left unstored, retried after restart/rebalance". using var pollBudget = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); pollBudget.CancelAfter(TimeSpan.FromTicks(_subscriberOptions.MaxPollInterval.Ticks / 4)); @@ -490,22 +535,351 @@ await callback(new KafkaBackgroundFailureContext( } } -internal sealed class AwaitingKafkaMessageDispatcher( - Func handler, - IKafkaConsumerClient consumer, - IKafkaProducerClient producer, - KafkaAsyncResponseTransportOptions transportOptions, - KafkaSubscriberOptions subscriberOptions, - ILogger logger, - string topic, - string consumerGroup, - KafkaSubscriberRole role) - : KafkaMessageDispatcher(handler, consumer, producer, transportOptions, subscriberOptions, logger, topic, consumerGroup, role) +/// +/// Ack-after-handler mode. A message's handler is started the moment it is consumed and awaited +/// inline for up to ; a handler still +/// running past that is detached: its partition is paused (Kafka's own ordering primitive +/// — nothing for it is fetched, nothing is buffered in-process), the handler and its retry ladder +/// run on, and the poll thread returns to polling. The earlier design awaited the whole handler on +/// the poll thread: a durable-flow step awaiting a remote response or a timer for longer than +/// max.poll.interval.ms (5 minutes by default) got the consumer evicted from its group, its +/// partitions rebalanced, the message redelivered to a peer that started the same work again, +/// and every other partition assigned to this consumer stalled behind it. +/// +/// The consumer is touched only from the poll thread: detached handlers never store offsets or +/// resume partitions themselves. The poll loop calls every tick, +/// which observes finished handlers exactly as the inline path would — success stores the offset, +/// cancellation leaves it unstored for redelivery, a burial that failed for good faults the poll +/// loop so nothing is ever committed past the message — then starts the next message held for the +/// partition, or resumes it. Disposal (the poll loop has exited by then) waits for the remaining +/// detached handlers and settles them before the consumer's close commits, so finished work is +/// not redelivered by a routine stop. +/// +/// +internal sealed class AwaitingKafkaMessageDispatcher : KafkaMessageDispatcher { - /// Handles the delivered message. - public override async Task HandleAsync( - KafkaDelivery delivery, - CancellationToken subscriberCancellationToken) + private readonly TimeSpan _detachAfter; + private readonly TimeSpan _faultDrainTimeout; + private readonly string _topic; + + // Poll-thread-only: the loop is the sole caller of Accept/SettleCompleted, and DisposeAsync + // runs after it has exited. No lock. + private readonly Dictionary _detached = []; + + /// Runs the AwaitingKafkaMessageDispatcher operation. + public AwaitingKafkaMessageDispatcher( + Func handler, + IKafkaConsumerClient consumer, + IKafkaProducerClient producer, + KafkaAsyncResponseTransportOptions transportOptions, + KafkaSubscriberOptions subscriberOptions, + ILogger logger, + string topic, + string consumerGroup, + KafkaSubscriberRole role) + : base(handler, consumer, producer, transportOptions, subscriberOptions, logger, topic, consumerGroup, role) + { + _detachAfter = subscriberOptions.DetachHandlerAfter; + _faultDrainTimeout = subscriberOptions.FaultDrainTimeout; + _topic = topic; + } + + /// Partitions whose handler is currently detached (test observability). + internal int DetachedCount => _detached.Count; + + public override bool HasDetachedWork => _detached.Count > 0; + + /// Handles the delivered message inline through to the offset store (the unit-test and inline-path contract). + public override async Task HandleAsync(KafkaDelivery delivery, CancellationToken subscriberCancellationToken) + { + await SettleAsync(delivery, subscriberCancellationToken).ConfigureAwait(false); + StoreOffsetAfterSettlement(delivery); + } + + /// + public override void Accept(KafkaDelivery delivery, CancellationToken subscriberCancellationToken) + { + if (_detached.TryGetValue(delivery.Partition, out var inFlight)) + { + // A message for a partition whose handler is still running: a rebalance handed the + // partition back with its pause reset (librdkafka resets pause state on assignment), + // or the client delivered a message it had fetched before the pause. Hold it behind + // the running one — the partition's order is the contract — and re-assert the pause + // so nothing more arrives; the hold is therefore bounded by what was already in + // flight, never a queue that grows. + (inFlight.Held ??= new Queue()).Enqueue(HeldMessage.For(delivery)); + PausePartition(delivery.Partition); + return; + } + + // Started on the pool, not inline: the inline wait below is a real bound on the poll + // thread's gap even for a handler whose synchronous prefix is long. + var settlement = Task.Run(() => SettleAsync(delivery, subscriberCancellationToken), CancellationToken.None); + if (WaitInline(settlement)) + { + // The fast path, unchanged: settle in place and consume the next message. + settlement.GetAwaiter().GetResult(); + StoreOffsetAfterSettlement(delivery); + return; + } + + PausePartition(delivery.Partition); + _detached[delivery.Partition] = new DetachedPartition(delivery, settlement, subscriberCancellationToken); + Logger.LogDebug( + "Kafka handler for {Topic}[{Partition}]@{Offset} is still running after {DetachAfter}; detached it and paused the partition while polling continues.", + delivery.Topic, + delivery.Partition, + delivery.Offset, + _detachAfter); + } + + /// + public override void AcceptUnprocessable(KafkaIncomingMessage message, Exception failure, CancellationToken subscriberCancellationToken) + { + if (_detached.TryGetValue(message.Partition, out var inFlight)) + { + // Same rule as a valid delivery for the partition: the message consumed before it is + // still being handled, so this one waits its turn. Settling it now would store — and + // let the auto-committer commit — an offset PAST the unfinished message; a crash + // after that commit skipped the unfinished message for good, and the dead-letter + // copy this discard produces is of the malformed record, not of the work that was + // lost. Held, it is buried and its offset stored in order, once the handler settles. + (inFlight.Held ??= new Queue()).Enqueue(HeldMessage.Unprocessable(message, failure)); + PausePartition(message.Partition); + Logger.LogDebug( + "Kafka message {Topic}[{Partition}]@{Offset} could not be parsed into a delivery and is held behind the partition's detached handler; it is dead-lettered in order once that handler settles.", + message.Topic, + message.Partition, + message.Offset); + return; + } + + // Nothing earlier on the partition is unresolved (every earlier message settled inline + // or would be in _detached), so the discard is safe to settle at once. + base.AcceptUnprocessable(message, failure, subscriberCancellationToken); + } + + /// + public override void SettleCompleted() + { + if (_detached.Count == 0) + return; + + List? finished = null; + foreach (var (partition, work) in _detached) + { + if (work.Settlement.IsCompleted) + (finished ??= []).Add(partition); + } + + if (finished is null) + return; + + foreach (var partition in finished) + { + var work = _detached[partition]; + // Removed BEFORE it is observed: a settlement that throws faults the poll loop, and the + // entry must not be settled a second time by disposal. + _detached.Remove(partition); + work.Settlement.GetAwaiter().GetResult(); + StoreOffsetAfterSettlement(work.Delivery); + ContinueHeld(partition, work.Held, work.SubscriberCancellationToken); + } + } + + /// + /// Works through the messages held behind a settled handler, in consumption order: an + /// unprocessable one is dead-lettered and its offset stored right here (its turn has come — + /// never ahead of the handler it was consumed behind); the first valid delivery is started + /// detached with the rest still held behind it (the partition stays paused); an empty hold + /// resumes the partition. + /// + private void ContinueHeld(int partition, Queue? held, CancellationToken subscriberCancellationToken) + { + while (held is { Count: > 0 }) + { + var next = held.Dequeue(); + if (next.Delivery is { } delivery) + { + var settlement = Task.Run(() => SettleAsync(delivery, subscriberCancellationToken), CancellationToken.None); + _detached[partition] = new DetachedPartition(delivery, settlement, subscriberCancellationToken) { Held = held.Count > 0 ? held : null }; + return; + } + + // A burial that fails for good throws out of here and faults the poll loop, exactly + // as an inline discard would; whatever is still held redelivers with the partition. + DiscardUnprocessableAsync(next.Message!, next.Failure!, subscriberCancellationToken).GetAwaiter().GetResult(); + } + + ResumePartition(partition); + } + + /// + /// The poll loop has exited (a stop, or a fault). Detached handlers run on — the handler takes + /// no cancellation token the ingress would honor — so wait for each and settle it exactly as the + /// poll thread would have: an offset stored here is committed by the consumer close that + /// follows, and finished work is not redelivered by a routine stop. Unbounded, as the inline + /// path was (the host's shutdown budget bounds the stop as a whole). Messages still held behind + /// a detached handler are dropped unstarted: their offsets are unstored, so they redeliver. + /// + public override async ValueTask DisposeAsync() + { + if (_detached.Count == 0) + return; + + Logger.LogInformation( + "Waiting for {Count} detached Kafka handler(s) on {Topic} to settle before the consumer closes.", + _detached.Count, + _topic); + + foreach (var (partition, work) in _detached.ToArray()) + { + _detached.Remove(partition); + await SettleAfterLoopExitAsync(work).ConfigureAwait(false); + } + } + + /// + /// The poll loop FAILED and the consumer is about to be closed and rebuilt. Waits at most + /// for the detached handlers: those + /// that settled get their offsets stored, exactly as the poll thread would have (the close + /// that follows commits them); the rest are abandoned — offsets unstored, so their messages + /// redeliver on the rebuilt consumer while the abandoned handler may still be running — and + /// observed, so each one's eventual outcome is logged instead of vanishing. Messages held + /// behind a detached handler are dropped unstarted, as on a stop. The unbounded wait this + /// replaces on the fault path let one long handler (a durable-flow step awaiting a remote + /// response) hold the subscriber's reconnect for its whole duration, so a transient broker + /// failure disabled every partition of the subscriber for as long as that step took and the + /// configured reconnect policy never ran. + /// + public override async ValueTask TeardownAfterFaultAsync() + { + if (_detached.Count == 0) + return; + + Logger.LogInformation( + "Kafka poll loop for {Topic} failed with {Count} detached handler(s) still running; waiting up to {FaultDrainTimeout} for them before the consumer is rebuilt.", + _topic, + _detached.Count, + _faultDrainTimeout); + + if (_faultDrainTimeout > TimeSpan.Zero) + { + var settlements = new Task[_detached.Count]; + var index = 0; + foreach (var work in _detached.Values) + settlements[index++] = work.Settlement; + + try + { + await Task.WhenAll(settlements).WaitAsync(_faultDrainTimeout).ConfigureAwait(false); + } + catch (Exception) + { + // A timeout, or a settlement that faulted or was canceled: each one is observed + // individually below. + } + } + + foreach (var (partition, work) in _detached.ToArray()) + { + _detached.Remove(partition); + if (work.Settlement.IsCompleted) + { + await SettleAfterLoopExitAsync(work).ConfigureAwait(false); + continue; + } + + Logger.LogWarning( + "Abandoning detached Kafka handler for {Topic}[{Partition}]@{Offset}: still running {FaultDrainTimeout} after the poll loop failed. Its offset is left unstored, so the message redelivers on the rebuilt consumer — possibly while this handler is still running; its outcome is logged when it settles.", + work.Delivery.Topic, + work.Delivery.Partition, + work.Delivery.Offset, + _faultDrainTimeout); + ObserveAbandoned(work); + } + } + + /// + /// Settles a detached handler after the poll loop has exited (a stop, or a fault whose budget + /// it finished within): its offset is stored for the consumer close to commit, a cancellation + /// or failure leaves it unstored so the message redelivers. + /// + private async Task SettleAfterLoopExitAsync(DetachedPartition work) + { + try + { + await work.Settlement.ConfigureAwait(false); + StoreOffsetAfterSettlement(work.Delivery); + } + catch (OperationCanceledException) + { + Logger.LogInformation( + "Detached Kafka handler for {Topic}[{Partition}]@{Offset} was canceled by the stop; its offset is left unstored and the message redelivers.", + work.Delivery.Topic, + work.Delivery.Partition, + work.Delivery.Offset); + } + catch (Exception ex) + { + Logger.LogError( + ex, + "Detached Kafka handler for {Topic}[{Partition}]@{Offset} failed while the subscriber was stopping; its offset is left unstored and the message redelivers.", + work.Delivery.Topic, + work.Delivery.Partition, + work.Delivery.Offset); + } + } + + /// + /// Logs the eventual outcome of a handler the fault teardown abandoned. It never touches the + /// consumer — the one it was consumed on is closed by then — so the outcome is informational: + /// the message has already been handed back to the group for redelivery. + /// + private void ObserveAbandoned(DetachedPartition work) + => _ = work.Settlement.ContinueWith( + static (settlement, state) => + { + var (logger, delivery) = ((ILogger, KafkaDelivery))state!; + if (settlement.IsCanceled) + { + logger.LogInformation( + "Abandoned Kafka handler for {Topic}[{Partition}]@{Offset} stopped on the session's cancellation; the message redelivers on the rebuilt consumer.", + delivery.Topic, + delivery.Partition, + delivery.Offset); + } + else if (settlement.IsFaulted) + { + logger.LogWarning( + settlement.Exception!.GetBaseException(), + "Abandoned Kafka handler for {Topic}[{Partition}]@{Offset} failed after the consumer it was consumed on was rebuilt; the message redelivers there.", + delivery.Topic, + delivery.Partition, + delivery.Offset); + } + else + { + logger.LogInformation( + "Abandoned Kafka handler for {Topic}[{Partition}]@{Offset} completed after the consumer it was consumed on was rebuilt; its offset was never stored, so the message redelivers there (handlers are at-least-once).", + delivery.Topic, + delivery.Partition, + delivery.Offset); + } + }, + (Logger, work.Delivery), + CancellationToken.None, + TaskContinuationOptions.ExecuteSynchronously, + TaskScheduler.Default); + + /// + /// Runs the handler with the in-process retry ladder and, at the delivery cap, the dead-letter + /// burial — everything but the offset store, which the poll thread performs once this returns. + /// Returns normally when the message is settled (handled, or buried); throws on cancellation + /// (offset not stored: redelivered after restart or rebalance) and when the burial fails for + /// good (). + /// + private async Task SettleAsync(KafkaDelivery delivery, CancellationToken subscriberCancellationToken) { var attempt = 0; while (true) @@ -531,14 +905,17 @@ public override async Task HandleAsync( delivery.Partition, delivery.Offset, MaxDeliveryAttempts); - // Guarded like the queued dispatcher's identical publish: this runs inside the - // poll loop, and an unguarded throw (a permanently failing dead-letter topic — - // UnknownTopicOrPart with auto-create off, an over-sized payload — burns the - // publish retries and then rethrows) faulted the whole subscriber with the - // offset unstored, so the supervisor rebuilt the consumer and re-executed the - // handler MaxDeliveryAttempts more times per restart, forever. Swallow and - // leave the offset unstored: the message is re-consumed after restart or - // rebalance and the burial retried. + // A permanently failing dead-letter topic (UnknownTopicOrPart with auto-create + // off, an over-sized payload) burns the publish retries and then throws. That + // throw is deliberately NOT swallowed: an earlier round swallowed it, leaving + // the offset unstored and consumption running, and the next successful + // settlement on the same partition then stored a higher offset — the + // auto-committer committed past this message and a restart skipped it with no + // dead-letter copy. Faulting the subscriber (KafkaDeadLetterPublishFailedException) + // stalls the partition AT this message: the consumer closes without storing + // past it, the supervisor restarts it after its backoff, and the handler and + // burial are retried per restart — a loud, bounded-rate loop until the + // dead-letter topic is fixed, which is the at-least-once outcome. try { await DeadLetterAsync( @@ -552,20 +929,20 @@ await DeadLetterAsync( { Logger.LogError( deadLetterException, - "Failed to dead-letter Kafka message {Topic}[{Partition}]@{Offset} at the delivery cap; its offset is left unstored so the burial is retried after a restart or rebalance, but later settlements on the partition can commit past it — fix the dead-letter topic promptly.", + "Failed to dead-letter Kafka message {Topic}[{Partition}]@{Offset} at the delivery cap; its offset is left unstored and the subscriber restarts so no later settlement commits past it. The partition is stalled until the dead-letter topic is fixed.", delivery.Topic, delivery.Partition, delivery.Offset); - return; + throw new KafkaDeadLetterPublishFailedException(delivery.Topic, delivery.Partition, delivery.Offset, deadLetterException); } - StoreOffsetAfterSettlement(delivery); return; } // Kafka offsets cannot NACK one message, so retry in-process with backoff. This // stalls the message's partition (head-of-line), which is inherent to classic - // consumer groups. + // consumer groups — and only that partition: past DetachHandlerAfter the ladder + // runs detached from the poll thread. await Task.Delay(RetryBackoff(attempt), subscriberCancellationToken).ConfigureAwait(false); continue; } @@ -574,10 +951,78 @@ await DeadLetterAsync( // sibling transport): a StoreOffset failure after a successful handler — routine when a // rebalance revoked the partition mid-handler — must not be misread as a handler // failure that re-runs, or dead-letters, work that already succeeded. - StoreOffsetAfterSettlement(delivery); return; } } + + /// + /// Blocks the poll thread for at most the inline budget. true when the settlement task + /// finished (in any state — the caller observes it); false when it is still running. + /// + private bool WaitInline(Task settlement) + { + if (settlement.IsCompleted) + return true; + if (_detachAfter <= TimeSpan.Zero) + return false; + + try + { + return settlement.Wait(_detachAfter); + } + catch (AggregateException) + { + // Completed, faulted: the caller re-awaits it and gets the original exception. + return true; + } + } + + private void PausePartition(int partition) + { + try + { + Consumer.PausePartition(_topic, partition); + } + catch (Exception ex) + { + // Not assigned any more (a rebalance took it): nothing to pause, nothing arrives for it, + // and the running handler's outcome is settled like any other when it finishes. + Logger.LogDebug(ex, "Could not pause {Topic}[{Partition}] behind its detached handler; the partition is no longer assigned to this consumer.", _topic, partition); + } + } + + private void ResumePartition(int partition) + { + try + { + Consumer.ResumePartition(_topic, partition); + } + catch (Exception ex) + { + Logger.LogDebug(ex, "Could not resume {Topic}[{Partition}] after its detached handler settled; the partition is no longer assigned to this consumer.", _topic, partition); + } + } + + private sealed class DetachedPartition(KafkaDelivery delivery, Task settlement, CancellationToken subscriberCancellationToken) + { + public KafkaDelivery Delivery { get; } = delivery; + public Task Settlement { get; } = settlement; + public CancellationToken SubscriberCancellationToken { get; } = subscriberCancellationToken; + + /// Messages consumed for the partition while its handler was detached, in order. + public Queue? Held { get; set; } + } + + /// + /// One message consumed behind a detached handler: a valid delivery, or one that could not + /// be projected (kept with the failure that rejected it, for the dead-letter headers). + /// + private readonly record struct HeldMessage(KafkaDelivery? Delivery, KafkaIncomingMessage? Message, Exception? Failure) + { + public static HeldMessage For(KafkaDelivery delivery) => new(delivery, null, null); + + public static HeldMessage Unprocessable(KafkaIncomingMessage message, Exception failure) => new(null, message, failure); + } } internal sealed class QueuedKafkaMessageDispatcher : KafkaMessageDispatcher diff --git a/src/Transports/AsyncResponse.Transports.Kafka/KafkaSubscriberOptions.cs b/src/Transports/AsyncResponse.Transports.Kafka/KafkaSubscriberOptions.cs index 62c7a9101..4b3953939 100644 --- a/src/Transports/AsyncResponse.Transports.Kafka/KafkaSubscriberOptions.cs +++ b/src/Transports/AsyncResponse.Transports.Kafka/KafkaSubscriberOptions.cs @@ -10,7 +10,13 @@ public enum KafkaAckMode /// message is retried in-process with backoff (Kafka offsets cannot NACK a single message); /// after the message is produced to /// the dead-letter topic and its offset is committed so the partition keeps moving. Messages - /// are processed serially per assignment, preserving per-partition ordering. + /// are processed serially per partition. A handler still running after + /// is detached: its partition is + /// paused, the handler (retries included) runs on while the poll thread keeps polling — the + /// consumer's other partitions, its max.poll.interval.ms liveness, and rebalance + /// callbacks all continue — and the offset is stored once the handler settles. A durable flow + /// awaiting a remote step or sleeping on a timer for minutes therefore no longer gets the + /// consumer evicted from its group. /// AckAfterHandlerCompletes = 0, @@ -86,11 +92,47 @@ public sealed class KafkaSubscriberOptions public TimeSpan PollTimeout { get; set; } = TimeSpan.FromMilliseconds(200); /// - /// Delay between capacity re-checks while consumption is paused because the - /// background queue is full. Default: 50ms. + /// The short poll slice used while the poll thread is waiting on in-process work: capacity + /// re-checks while consumption is paused because the + /// background queue is full, and completion checks while + /// handlers run detached (a finished + /// handler's offset is stored and its partition resumed within one slice). Default: 50ms. /// public TimeSpan BackpressurePollDelay { get; set; } = TimeSpan.FromMilliseconds(50); + /// + /// In mode, how long the poll thread waits + /// for a message's handler inline before detaching it. Within the budget a fast handler settles + /// exactly as before — offset stored, next message consumed, no pause. Past it the message's + /// partition is paused (its order holds, nothing is buffered in-process), the handler and its + /// in-process retries continue on the thread pool, and the poll thread goes back to polling: + /// the consumer's other partitions keep flowing, is honored, and + /// rebalance callbacks fire. The poll thread stores the offset and resumes the partition once + /// the handler settles (checked every ). Detached handlers + /// for different partitions run concurrently; a stop waits for them so their offsets are + /// committed. detaches every handler immediately. Plus + /// this is the poll thread's longest gap, and startup validation + /// requires it to fit within half of . Default: 1s. + /// + public TimeSpan DetachHandlerAfter { get; set; } = TimeSpan.FromSeconds(1); + + /// + /// In mode, how long a subscriber whose + /// poll loop failed (a consume error, a dropped broker connection) waits for its + /// detached handlers to settle before it closes the consumer and rebuilds it. Handlers that + /// settle within the budget get their offsets stored and committed by the close, exactly as + /// after a stop; the rest are abandoned — their offsets stay unstored, the messages redeliver + /// on the rebuilt consumer (an abandoned handler may still be running then; see the + /// at-least-once notes in transport-semantics.md), and each one's eventual outcome is logged. + /// Without the bound the fault teardown waited for every detached handler with no limit, so a + /// transient broker failure disabled the subscriber for as long as an unrelated long handler + /// — a durable-flow step awaiting a remote response — took, and the configured reconnect + /// policy (SubscriberRetryBaseDelay → SubscriberRetryMaxDelay) never ran. A + /// graceful stop is not bounded here; the host's shutdown budget bounds it. + /// abandons detached handlers at once. Default: 5s. + /// + public TimeSpan FaultDrainTimeout { get; set; } = TimeSpan.FromSeconds(5); + /// /// Maximum number of in-process delivery attempts before a failing message is produced to the /// dead-letter topic and its offset committed. Kafka offsets cannot NACK a single message, so @@ -109,20 +151,21 @@ public sealed class KafkaSubscriberOptions public TimeSpan HandlerRetryBaseDelay { get; set; } = TimeSpan.FromMilliseconds(100); /// - /// Maximum delay between in-process handler retry attempts. Keep the total retry budget well - /// below or the broker will evict the consumer from its group - /// mid-retry. Default: 5s. + /// Maximum delay between in-process handler retry attempts. The retry ladder runs inside the + /// message's handler task — detached from the poll thread past + /// — so it stalls only that message's partition, never the consumer's group membership. + /// Default: 5s. /// public TimeSpan HandlerRetryMaxDelay { get; set; } = TimeSpan.FromSeconds(5); /// /// Maximum gap between consumer polls before the broker evicts this consumer from its group - /// and rebalances its partitions (the librdkafka max.poll.interval.ms). The in-process - /// handler retry delays run on the poll thread, so validation requires the worst-case retry - /// delay budget plus to fit within half this interval; handler - /// execution time itself is not bounded by the library and remains the operator's - /// responsibility, as does the unlimited-retry mode ( = - /// 0). Default: 5 minutes (the librdkafka default). + /// and rebalances its partitions (the librdkafka max.poll.interval.ms). The poll thread's + /// longest gap is one inline handler wait () plus one poll + /// (), and validation requires that sum to fit within half this + /// interval; handler execution time itself is unbounded and no longer counts, because a + /// handler that outlives the inline budget is detached while polling continues. Default: + /// 5 minutes (the librdkafka default). /// public TimeSpan MaxPollInterval { get; set; } = TimeSpan.FromMinutes(5); diff --git a/src/Transports/AsyncResponse.Transports.Kafka/KafkaSubscriberServices.cs b/src/Transports/AsyncResponse.Transports.Kafka/KafkaSubscriberServices.cs index d4d4778af..fedcc93b6 100644 --- a/src/Transports/AsyncResponse.Transports.Kafka/KafkaSubscriberServices.cs +++ b/src/Transports/AsyncResponse.Transports.Kafka/KafkaSubscriberServices.cs @@ -83,7 +83,14 @@ private async Task RunSubscriberAsync(CancellationToken stoppingToken) try { consumer.Subscribe(Topic); - await using var dispatcher = KafkaMessageDispatcher.Create( + + // One session token per consumer, linked to the host's: a stop cancels it as before. + // A poll-loop FAULT cancels it too — after the bounded fault teardown below — so a + // detached handler abandoned by that teardown stops retrying a message whose offset + // this session can no longer store, instead of running its whole retry ladder for a + // consumer that is gone. + using var session = CancellationTokenSource.CreateLinkedTokenSource(stoppingToken); + var dispatcher = KafkaMessageDispatcher.Create( HandleMessageAsync, consumer, _producer, @@ -101,16 +108,42 @@ private async Task RunSubscriberAsync(CancellationToken stoppingToken) SubscriberRole, SubscriberOptions.AckMode); - // Consume() blocks the calling thread, so the poll loop runs on a dedicated thread - // instead of starving the thread pool; the dispatcher's async work is awaited from it. - await Task.Factory.StartNew( - () => RunPollLoop(consumer, dispatcher, stoppingToken), - stoppingToken, - TaskCreationOptions.LongRunning, - TaskScheduler.Default).ConfigureAwait(false); - - // Leaving the await-using scope drains the ACK-after-enqueue background queue before - // the consumer commits its final stored offsets below. + var faulted = false; + try + { + // Consume() blocks the calling thread, so the poll loop runs on a dedicated thread + // instead of starving the thread pool; the dispatcher's settlements happen on it too + // (the consumer is touched from no other thread while the loop runs). + await Task.Factory.StartNew( + () => RunPollLoop(consumer, dispatcher, session.Token), + stoppingToken, + TaskCreationOptions.LongRunning, + TaskScheduler.Default).ConfigureAwait(false); + } + catch (Exception) when (!stoppingToken.IsCancellationRequested) + { + // The poll loop failed (a consume error, a dropped connection, a burial that + // failed for good) and the supervisor will rebuild the consumer after its backoff. + // Teardown is BOUNDED here, unlike the graceful stop's: waiting for every detached + // handler with no limit parked the reconnect behind an unrelated long handler — a + // durable-flow step awaiting a remote response — and the configured retry policy + // never ran. Handlers that settle within the budget get their offsets stored + // (the close below commits them); the rest are abandoned with their offsets + // unstored, so their messages redeliver on the rebuilt consumer. + faulted = true; + await dispatcher.TeardownAfterFaultAsync().ConfigureAwait(false); + session.Cancel(); + throw; + } + finally + { + // Graceful stop (or a fault racing one): the drain waits for the ACK-after-enqueue + // background queue — or ack-after-handler mode's detached handlers, storing their + // offsets — before the consumer commits its final stored offsets below. The host's + // shutdown budget bounds it. + if (!faulted) + await dispatcher.DisposeAsync().ConfigureAwait(false); + } } finally { @@ -127,6 +160,11 @@ private void RunPollLoop( var paused = false; while (!stoppingToken.IsCancellationRequested) { + // Handlers that outlived their inline budget are settled here, on the poll thread — + // the only thread that touches the consumer — before the next poll: offset stored, + // partition resumed. A settlement that failed for good throws and faults the loop. + dispatcher.SettleCompleted(); + KafkaIncomingMessage? message; if (!dispatcher.CanAcceptMore) { @@ -161,7 +199,12 @@ private void RunPollLoop( Topic); } - message = consumer.Consume(SubscriberOptions.PollTimeout); + // With detached handlers in flight, poll in short slices so a completion is + // settled within BackpressurePollDelay instead of after a full PollTimeout; the + // slice is what bounds the resume latency of the paused partition. + message = consumer.Consume(dispatcher.HasDetachedWork + ? SubscriberOptions.BackpressurePollDelay + : SubscriberOptions.PollTimeout); } if (message is null) @@ -183,11 +226,19 @@ private void RunPollLoop( // after every supervisor restart and the whole subscriber (all assigned partitions) // stops advancing — MaxDeliveryAttempts cannot help, because it is keyed on a // delivery this path never constructed. - dispatcher.DiscardUnprocessableAsync(message, ex, stoppingToken).GetAwaiter().GetResult(); + // + // Through the dispatcher's partition ordering, not a direct discard: storing this + // message's offset while an earlier message of the same partition is still being + // handled (detached) commits the partition PAST that unfinished message, and a + // crash after the commit skips it for good with no dead-letter copy anywhere. + dispatcher.AcceptUnprocessable(message, ex, stoppingToken); continue; } - dispatcher.HandleAsync(delivery, stoppingToken).GetAwaiter().GetResult(); + // Settles inline (queued mode, and ack-after-handler mode within DetachHandlerAfter) + // or detaches the handler and returns; either way the poll thread is back here within + // the validated poll gap. + dispatcher.Accept(delivery, stoppingToken); } } diff --git a/src/Transports/AsyncResponse.Transports.Kafka/KafkaTransportClientAdapters.cs b/src/Transports/AsyncResponse.Transports.Kafka/KafkaTransportClientAdapters.cs index ee49bf4b1..6970dc0e5 100644 --- a/src/Transports/AsyncResponse.Transports.Kafka/KafkaTransportClientAdapters.cs +++ b/src/Transports/AsyncResponse.Transports.Kafka/KafkaTransportClientAdapters.cs @@ -41,7 +41,9 @@ Task PublishAsync( /// /// Adapter seam over one Kafka consumer. Not thread-safe: each hosted subscriber owns one -/// consumer and touches it only from its own poll loop. +/// consumer and touches it only from its own poll loop — detached handlers never call it; their +/// completions are settled by the poll thread (and, after the loop has exited, by the +/// dispatcher's disposal, sequentially). /// internal interface IKafkaConsumerClient : IDisposable { @@ -67,6 +69,16 @@ internal interface IKafkaConsumerClient : IDisposable /// Resumes fetching on all currently assigned partitions. void ResumeAssignment(); + /// + /// Pauses fetching on one partition while a detached handler runs its message, so the + /// partition's order holds with nothing buffered in-process. Throws when the partition is not + /// currently assigned (revoked by a rebalance); callers treat that as informational. + /// + void PausePartition(string topic, int partition); + + /// Resumes fetching on one partition once its detached handler has settled. Throws when it is no longer assigned. + void ResumePartition(string topic, int partition); + /// Leaves the group cleanly, committing stored offsets. void Close(); } @@ -235,6 +247,14 @@ public void PauseAssignment() public void ResumeAssignment() => _consumer.Resume(_consumer.Assignment); + /// Pauses fetching on one partition. + public void PausePartition(string topic, int partition) + => _consumer.Pause([new TopicPartition(topic, new Partition(partition))]); + + /// Resumes fetching on one partition. + public void ResumePartition(string topic, int partition) + => _consumer.Resume([new TopicPartition(topic, new Partition(partition))]); + /// Leaves the group cleanly, committing stored offsets. public void Close() => _consumer.Close(); @@ -278,9 +298,9 @@ public IKafkaConsumerClient Create(KafkaSubscriberRole role) EnableAutoCommit = true, EnableAutoOffsetStore = false, AutoCommitIntervalMs = (int)Math.Max(1, _options.OffsetCommitInterval.TotalMilliseconds), - // The dispatcher's in-process retry delays run on the poll thread, so the eviction - // deadline they must fit within is set explicitly and validated against the retry - // budget instead of trusting the librdkafka default to line up. + // The poll thread's longest gap (one inline handler wait of DetachHandlerAfter plus + // one poll) is validated against this deadline at startup, so it is set explicitly + // instead of trusting the librdkafka default to line up. MaxPollIntervalMs = (int)Math.Max(1, subscriberOptions.MaxPollInterval.TotalMilliseconds), // Start new consumer groups at the beginning of the topic so messages published before // the first subscriber starts are not skipped (mirrors the other transports). diff --git a/src/Transports/AsyncResponse.Transports.Kafka/PublicAPI.Unshipped.txt b/src/Transports/AsyncResponse.Transports.Kafka/PublicAPI.Unshipped.txt index 9c14141e5..e14369fab 100644 --- a/src/Transports/AsyncResponse.Transports.Kafka/PublicAPI.Unshipped.txt +++ b/src/Transports/AsyncResponse.Transports.Kafka/PublicAPI.Unshipped.txt @@ -108,3 +108,7 @@ AsyncResponse.Transports.Kafka.KafkaWorkerTransport.PublishAsync(AsyncResponse.W const AsyncResponse.Transports.Kafka.KafkaAsyncResponseTransportOptions.TransportName = "kafka" -> string! Microsoft.Extensions.DependencyInjection.KafkaAsyncResponseTransportServiceCollectionExtensions static Microsoft.Extensions.DependencyInjection.KafkaAsyncResponseTransportServiceCollectionExtensions.WithKafkaTransport(this Microsoft.Extensions.DependencyInjection.AsyncResponseRegistrationBuilder! builder, System.Action! configure) -> Microsoft.Extensions.DependencyInjection.AsyncResponseRegistrationBuilder! +AsyncResponse.Transports.Kafka.KafkaSubscriberOptions.DetachHandlerAfter.get -> System.TimeSpan +AsyncResponse.Transports.Kafka.KafkaSubscriberOptions.DetachHandlerAfter.set -> void +AsyncResponse.Transports.Kafka.KafkaSubscriberOptions.FaultDrainTimeout.get -> System.TimeSpan +AsyncResponse.Transports.Kafka.KafkaSubscriberOptions.FaultDrainTimeout.set -> void diff --git a/src/Transports/AsyncResponse.Transports.NATS/NatsSubscriberServices.cs b/src/Transports/AsyncResponse.Transports.NATS/NatsSubscriberServices.cs index 0207b95b5..2260f8aba 100644 --- a/src/Transports/AsyncResponse.Transports.NATS/NatsSubscriberServices.cs +++ b/src/Transports/AsyncResponse.Transports.NATS/NatsSubscriberServices.cs @@ -161,18 +161,45 @@ private async Task DispatchBatchAsync( finally { renewalCancellation.Cancel(); - await renewalTask.ConfigureAwait(false); + try + { + // Cancellation exits the sweep between messages and aborts the in-flight + // heartbeat (the token reaches the SDK call), so this normally completes at once. + // The bound is the hard backstop for a heartbeat the client cannot abort — a + // write wedged on a dead socket: an unbounded join here held the loop after every + // message in the batch had settled, so no further batch was fetched and a stop + // never completed, with nothing for the supervisor to restart. Past one heartbeat + // interval the loop is abandoned; the server's AckWait settles whatever it left. + await renewalTask.WaitAsync(RenewalInterval).ConfigureAwait(false); + } + catch (TimeoutException) + { + Logger.LogWarning( + "NATS in-progress heartbeat for {Role} did not stop within {RenewalInterval} after its batch settled; abandoning it — unsettled deliveries fall back to the server-side AckWait.", + Role, + RenewalInterval); + _ = renewalTask.ContinueWith( + static (task, state) => ((ILogger)state!).LogWarning(task.Exception, "Abandoned NATS in-progress heartbeat faulted."), + Logger, + CancellationToken.None, + TaskContinuationOptions.OnlyOnFaulted | TaskContinuationOptions.ExecuteSynchronously, + TaskScheduler.Default); + } } } + /// + /// ~AckWait/3: two chances to land a renewal inside every AckWait window even when one sweep + /// is delayed by a slow round trip. Also the bound on joining the renewal loop after a batch. + /// + private TimeSpan RenewalInterval => TimeSpan.FromMilliseconds(Math.Max(1, Options.AckWait.TotalMilliseconds / 3)); + private async Task RenewInProgressLoopAsync( List batch, BatchProgress progress, CancellationToken cancellationToken) { - // ~AckWait/3: two chances to land a renewal inside every AckWait window even when one - // sweep is delayed by a slow round trip. - var interval = TimeSpan.FromMilliseconds(Math.Max(1, Options.AckWait.TotalMilliseconds / 3)); + var interval = RenewalInterval; try { while (true) @@ -198,7 +225,7 @@ private async Task RenewInProgressLoopAsync( var delivery = batch[i]; try { - await delivery.ProgressAsync().ConfigureAwait(false); + await delivery.ProgressAsync(cancellationToken).ConfigureAwait(false); } catch (Exception ex) when (ex is not OperationCanceledException) { diff --git a/src/Transports/AsyncResponse.Transports.NATS/NatsTransportClientAdapters.cs b/src/Transports/AsyncResponse.Transports.NATS/NatsTransportClientAdapters.cs index 5cafdad21..7ef3038e0 100644 --- a/src/Transports/AsyncResponse.Transports.NATS/NatsTransportClientAdapters.cs +++ b/src/Transports/AsyncResponse.Transports.NATS/NatsTransportClientAdapters.cs @@ -28,9 +28,11 @@ internal sealed record NatsJobDelivery( /// Signals "working on it" (JetStream in-progress) so the server resets this delivery's /// AckWait window without settling it or bumping its delivery count. An init property with a /// no-op default rather than a positional parameter so out-of-package constructions stay - /// source-compatible. + /// source-compatible. The token is the batch's renewal cancellation: a heartbeat still in + /// flight when the batch settles (or the subscriber stops) must abort with it, not hold the + /// batch — the SDK call it wraps takes the token for exactly that. /// - public Func ProgressAsync { get; init; } = static () => ValueTask.CompletedTask; + public Func ProgressAsync { get; init; } = static _ => ValueTask.CompletedTask; } /// @@ -235,7 +237,10 @@ private static NatsJobDelivery ToDelivery(INatsJSMsg message) delay => captured.NakAsync(delay: delay, cancellationToken: CancellationToken.None), () => captured.AckTerminateAsync(cancellationToken: CancellationToken.None)) { - ProgressAsync = () => captured.AckProgressAsync(cancellationToken: CancellationToken.None) + // Unlike the settlements above (deliberately uncancelable: a settlement decision + // already taken must reach the server), a progress heartbeat is advisory — the + // renewal loop's token cancels one that stalls, so the batch never waits on it. + ProgressAsync = cancellationToken => captured.AckProgressAsync(cancellationToken: cancellationToken) }; } diff --git a/tests/AsyncResponse.IntegrationTests.AppHost/AsyncResponse.IntegrationTests.AppHost.csproj b/tests/AsyncResponse.IntegrationTests.AppHost/AsyncResponse.IntegrationTests.AppHost.csproj index 895ac33fb..c483ec035 100644 --- a/tests/AsyncResponse.IntegrationTests.AppHost/AsyncResponse.IntegrationTests.AppHost.csproj +++ b/tests/AsyncResponse.IntegrationTests.AppHost/AsyncResponse.IntegrationTests.AppHost.csproj @@ -1,12 +1,18 @@ - + Exe net10.0 false true + + $(NoWarn);ASPIRE010 asyncresponse-integrationtests-apphost-5ff760b0-7ffb-4cd9-8fe5-6464f1286486 @@ -15,6 +21,8 @@ + + diff --git a/tests/AsyncResponse.IntegrationTests.AppHost/Program.cs b/tests/AsyncResponse.IntegrationTests.AppHost/Program.cs index 4053dcfa0..9f14f4b33 100644 --- a/tests/AsyncResponse.IntegrationTests.AppHost/Program.cs +++ b/tests/AsyncResponse.IntegrationTests.AppHost/Program.cs @@ -131,7 +131,11 @@ IResourceBuilder AddSutApp(string name, bool aotCapabl // allocated port through ASPNETCORE_HTTP_PORTS keeps Kestrel on the proxied endpoint. var exe = builder.AddExecutable(name, sutAotPath, Path.GetDirectoryName(Path.GetFullPath(sutAotPath))!) .WithHttpEndpoint(env: "ASPNETCORE_HTTP_PORTS") - .WithHttpHealthCheck("/alive"); + .WithHttpHealthCheck("/alive") + // The suite's per-test reset and recovery seeding go through the sample's test-only + // mutation routes, which the sample maps only when told to (never in Production by + // default); every SUT app is a test fixture here, so opt in explicitly. + .WithEnvironment("Sample:EnableTestEndpoints", "true"); foreach (var dependency in waitFor) exe.WaitFor(dependency); return exe; @@ -139,7 +143,8 @@ IResourceBuilder AddSutApp(string name, bool aotCapabl var project = builder.AddProject(name, launchProfileName: null) .WithHttpEndpoint() - .WithHttpHealthCheck("/alive"); + .WithHttpHealthCheck("/alive") + .WithEnvironment("Sample:EnableTestEndpoints", "true"); foreach (var dependency in waitFor) project.WaitFor(dependency); return project; diff --git a/tests/AsyncResponse.IntegrationTests/Batches.cs b/tests/AsyncResponse.IntegrationTests/Batches.cs index 77ee8035b..57ca135ca 100644 --- a/tests/AsyncResponse.IntegrationTests/Batches.cs +++ b/tests/AsyncResponse.IntegrationTests/Batches.cs @@ -242,6 +242,9 @@ protected override async ValueTask WireAsync() "itest-app-redis", "itest-app-redis-early-ack"); + // The in-process Kafka host of KafkaLongHandlerIntegrationTests addresses the broker itself. + await WireBrokerConnectionStringsAsync(); + Client = clients[0]; EarlyAckClient = clients[1]; RabbitMqClient = clients[2]; diff --git a/tests/AsyncResponse.IntegrationTests/DurableFlowStateStorePackageIntegrationTests.cs b/tests/AsyncResponse.IntegrationTests/DurableFlowStateStorePackageIntegrationTests.cs index 53b05e1f3..f87236c63 100644 --- a/tests/AsyncResponse.IntegrationTests/DurableFlowStateStorePackageIntegrationTests.cs +++ b/tests/AsyncResponse.IntegrationTests/DurableFlowStateStorePackageIntegrationTests.cs @@ -352,6 +352,48 @@ public async Task MySqlPackageStore_RoundTrips_Expires_Deletes() } } + /// + /// Round 39: a duplicate create handled MySQL's 1062 by opening a SECOND pooled connection + /// for the existence check while still holding the first — on a pool of one that timed out + /// with "All pooled connections are in use", and under concurrent idempotent starts the pool + /// starved at any size. The check now runs on the connection the create already holds. + /// Pre-fix: the second TryCreateAsync throws MySqlException after the connection timeout. + /// + [Fact] + public async Task MySqlPackageStore_DuplicateCreate_CompletesOnAOneConnectionPool() + { + await WaitForMySqlAsync(); + var table = NewIdentifier("df_mysql_pool1", 64); + var connectionString = new MySqlConnectionStringBuilder(Fixture.MySqlConnectionString) + { + MaximumPoolSize = 1, + ConnectionTimeout = 5 + }.ConnectionString; + try + { + var store = new MySqlFlowStateStore( + Options.Create(new MySqlDurableFlowOptions + { + ConnectionString = connectionString, + TableName = table + })); + var state = new FlowState { FlowId = "dup", FlowTypeName = "T", InputTypeName = "I", Status = FlowRunStatus.Running, CreatedAtUtc = DateTime.UtcNow, UpdatedAtUtc = DateTime.UtcNow }; + + Assert.True(await store.TryCreateAsync("dup", state, TimeSpan.FromMinutes(5))); + var duplicate = await store.TryCreateAsync("dup", state, TimeSpan.FromMinutes(5)).WaitAsync(TimeSpan.FromSeconds(20)); + Assert.False(duplicate); + } + finally + { + MySqlConnection.ClearAllPools(); + await using var connection = new MySqlConnection(Fixture.MySqlConnectionString); + await connection.OpenAsync(); + await using var command = connection.CreateCommand(); + command.CommandText = $"DROP TABLE IF EXISTS `{table}`;"; + await command.ExecuteNonQueryAsync(); + } + } + [Theory] [InlineData(true)] [InlineData(false)] diff --git a/tests/AsyncResponse.IntegrationTests/InMemoryEndToEndTests.cs b/tests/AsyncResponse.IntegrationTests/InMemoryEndToEndTests.cs index 75dcc0010..70dc695a9 100644 --- a/tests/AsyncResponse.IntegrationTests/InMemoryEndToEndTests.cs +++ b/tests/AsyncResponse.IntegrationTests/InMemoryEndToEndTests.cs @@ -240,6 +240,10 @@ protected override void ConfigureWebHost(IWebHostBuilder builder) { builder.UseSetting("AsyncResponse:Channel", "InMemory"); builder.UseSetting("AsyncResponse:Transport", "InMemory"); + // The per-test /test/reset is a test-only mutation route the sample maps only on + // request (or in Development); say so explicitly rather than lean on the factory's + // default environment. + builder.UseSetting("Sample:EnableTestEndpoints", "true"); } } } diff --git a/tests/AsyncResponse.IntegrationTests/KafkaLongHandlerIntegrationTests.cs b/tests/AsyncResponse.IntegrationTests/KafkaLongHandlerIntegrationTests.cs new file mode 100644 index 000000000..34a4464bc --- /dev/null +++ b/tests/AsyncResponse.IntegrationTests/KafkaLongHandlerIntegrationTests.cs @@ -0,0 +1,93 @@ +using System.Collections.Concurrent; +using AsyncResponse.Transports.Kafka; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Xunit; + +namespace AsyncResponse.IntegrationTests; + +/// +/// Round 37 (F7): the real broker's answer to a handler that outlives max.poll.interval.ms. +/// The poll thread used to await the whole handler, so past the interval librdkafka left the +/// group (MAXPOLL), the next poll rejoined, the partition was re-fetched from the last committed +/// offset — the message's offset was never stored — and the same job ran a second time. With the +/// handler detached and polling continuing, the consumer stays in its group and the job runs once. +/// +[Collection(BrokersCollection.Name)] +[Trait(Batches.Trait, Batches.Brokers)] +public sealed class KafkaLongHandlerIntegrationTests(BrokersBatchFixture fixture) : IntegrationTestBase(fixture) +{ + public interface ISlowProbe + { + Task RunAsync(string token); + } + + public sealed class SlowProbe : ISlowProbe + { + public static readonly ConcurrentDictionary Runs = new(StringComparer.Ordinal); + public static readonly TimeSpan Duration = TimeSpan.FromSeconds(12); + + public async Task RunAsync(string token) + { + Runs.AddOrUpdate(token, 1, (_, count) => count + 1); + await Task.Delay(Duration); + } + } + + [Fact] + public async Task AckAfterHandler_AHandlerOutlivingMaxPollInterval_RunsExactlyOnce() + { + var prefix = NewId("r37-long"); + var services = new ServiceCollection(); + services.AddLogging(); + services.AddSingleton(); + services.AddAsyncResponse() + .WithInMemoryChannel() + .WithInMemoryDurableFlows() + .WithKafkaTransport(options => + { + options.BootstrapServers = Fixture.KafkaBootstrapServers!; + options.TopicPrefix = prefix; + options.WorkerConsumerGroup = $"{prefix}-workers"; + options.ResponseConsumerGroup = $"{prefix}-responses"; + options.CreateTopics = true; + // A poll interval the 12-second handler comfortably outlives, with the session + // settings the broker's default minimums allow. + options.WorkerSubscriber.MaxPollInterval = TimeSpan.FromSeconds(8); + options.WorkerSubscriber.PollTimeout = TimeSpan.FromMilliseconds(100); + options.ConfigureConsumer = config => + { + config.SessionTimeoutMs = 6000; + config.HeartbeatIntervalMs = 2000; + }; + }); + + await using var provider = services.BuildServiceProvider(); + var hosted = provider.GetServices().ToList(); + foreach (var service in hosted) + await service.StartAsync(CancellationToken.None); + + try + { + var token = NewId("slow"); + await provider.GetRequiredService().EnqueueWorkerAsync(probe => probe.RunAsync(token)); + + var firstRun = await PollAsync( + () => Task.FromResult(SlowProbe.Runs.GetValueOrDefault(token)), + runs => runs >= 1, + TimeSpan.FromSeconds(90)); + Assert.Equal(1, firstRun); + + // Past the handler's own duration and the poll interval with margin: a consumer evicted + // for exceeding max.poll.interval.ms would have rejoined and re-run the job by now. + await Task.Delay(SlowProbe.Duration + TimeSpan.FromSeconds(10)); + Assert.Equal(1, SlowProbe.Runs.GetValueOrDefault(token)); + } + finally + { + hosted.Reverse(); + foreach (var service in hosted) + await service.StopAsync(CancellationToken.None); + } + } +} diff --git a/tests/AsyncResponse.IntegrationTests/MongoDbDirectIntegrationTests.cs b/tests/AsyncResponse.IntegrationTests/MongoDbDirectIntegrationTests.cs index b7e49e86a..045470745 100644 --- a/tests/AsyncResponse.IntegrationTests/MongoDbDirectIntegrationTests.cs +++ b/tests/AsyncResponse.IntegrationTests/MongoDbDirectIntegrationTests.cs @@ -149,12 +149,26 @@ await store.HeartbeatSubscribersAsync( var message = Assert.Single(messages); Assert.Equal(messageId, message.Id); Assert.Equal(firstCreatedAt, message.CreatedAtUtc); + Assert.Equal("""{"Success":true}""", message.EnvelopeJson); // unacknowledged: the page carries the body // Live delivery claims the message; recovery must then lose the arbitration. Assert.True(await store.TryClaimForDeliveryAsync(messageId, CancellationToken.None)); Assert.True(await store.IsMessageAcknowledgedAsync(messageId, CancellationToken.None)); Assert.False(await store.TryClaimForRecoveryAsync(messageId, CancellationToken.None)); + // Round 39: once acknowledged, the sweep's page carries the document header-only (the + // $cond projection nulls the envelope) and the by-id read hydrates it in full. + var ackedPage = await store.LoadMessagesAsync(messageCorrelation, since.AddSeconds(-1), 16, null, null, CancellationToken.None); + var ackedHeader = Assert.Single(ackedPage); + Assert.Null(ackedHeader.EnvelopeJson); + Assert.NotNull(ackedHeader.AckedAtUtc); + Assert.NotNull(ackedHeader.AckedSeq); + var hydrated = Assert.Single(await store.LoadMessagesByIdAsync(messageCorrelation, [messageId, Guid.NewGuid()], CancellationToken.None)); + Assert.Equal(messageId, hydrated.Id); + Assert.Equal("""{"Success":true}""", hydrated.EnvelopeJson); + Assert.Equal(ackedHeader.AckedSeq, hydrated.AckedSeq); + Assert.Empty(await store.LoadMessagesByIdAsync("some-other-correlation", [messageId], CancellationToken.None)); + // And the reverse: once recovery owns a message, live delivery must not double-handle it. var recoveryMessageId = Guid.NewGuid(); await store.InsertMessageAsync(recoveryMessageId, messageCorrelation, """{"Success":true}""", TimeSpan.FromMinutes(5), CancellationToken.None); @@ -443,7 +457,11 @@ await AssertUnreadableEnvelopeAsync( store, """{"SchemaVersion":999,"Success":true,"Payload":{"Status":2}}""", typeof(InvalidOperationException)); - await AssertUnreadableEnvelopeAsync(subscriber, store, "{not-json", typeof(JsonException)); + // A malformed BODY is scrubbed to the body-free InvalidDataException (round 36): the + // reader's own message quotes what it was reading. A contract violation the LIBRARY + // authors keeps its message — pinned on the next line. + await AssertUnreadableEnvelopeAsync(subscriber, store, "{not-json", typeof(InvalidDataException)); + await AssertUnreadableEnvelopeAsync(subscriber, store, "{}", typeof(JsonException), expectedMessageFragment: "SchemaVersion is required."); await AssertUnreadableEnvelopeAsync( subscriber, store, @@ -486,7 +504,8 @@ private static async Task AssertUnreadableEnvelopeAsync( MongoDbChannelStore store, string envelopeJson, Type expectedExceptionType, - string? expectedRemoteStack = null) + string? expectedRemoteStack = null, + string? expectedMessageFragment = null) { var correlationId = $"mongo-unreadable-{Guid.NewGuid():N}"; await using var waiter = await subscriber.CreateResponseWaiter( @@ -502,6 +521,8 @@ await store.InsertMessageAsync( var exception = await Assert.ThrowsAnyAsync( () => waiter.ResponseTask.WaitAsync(TimeSpan.FromSeconds(10))); Assert.IsAssignableFrom(expectedExceptionType, exception); + if (expectedMessageFragment is not null) + Assert.Contains(expectedMessageFragment, exception.Message, StringComparison.Ordinal); if (expectedRemoteStack is not null) Assert.Equal(expectedRemoteStack, exception.Data["RemoteStackTrace"]); } diff --git a/tests/AsyncResponse.IntegrationTests/NativeAotPublishGateTests.cs b/tests/AsyncResponse.IntegrationTests/NativeAotPublishGateTests.cs index 6284e4a14..018023dec 100644 --- a/tests/AsyncResponse.IntegrationTests/NativeAotPublishGateTests.cs +++ b/tests/AsyncResponse.IntegrationTests/NativeAotPublishGateTests.cs @@ -90,6 +90,10 @@ public async Task Sample_PublishesNativeAot_AndServesCoreScenarios() ["ASPNETCORE_HTTP_PORTS"] = port.ToString(), ["AsyncResponse__Channel"] = "InMemory", ["AsyncResponse__Transport"] = "InMemory", + // A published binary runs as Production, where the sample's simulation and + // ledger routes (GET /durable-flow/{id} below) are gated off; opt in like + // the integration AppHost does. + ["Sample__EnableTestEndpoints"] = "true", }, }; var appLog = new StringBuilder(); diff --git a/tests/AsyncResponse.IntegrationTests/OracleCosmosStoreContractTests.cs b/tests/AsyncResponse.IntegrationTests/OracleCosmosStoreContractTests.cs index f83bb2597..246e0f2f7 100644 --- a/tests/AsyncResponse.IntegrationTests/OracleCosmosStoreContractTests.cs +++ b/tests/AsyncResponse.IntegrationTests/OracleCosmosStoreContractTests.cs @@ -68,7 +68,8 @@ public async Task OraclePackageStore_RoundTrips_Expires_Deletes_WhenConnectionSt revisionCommand.CommandText = $"UPDATE {table} SET revision = revision + 1 WHERE flow_id = :flow_id"; revisionCommand.Parameters.Add(new OracleParameter("flow_id", mismatchedRevisionFlowId)); Assert.Equal(1, await revisionCommand.ExecuteNonQueryAsync()); - Assert.Null(await store.LoadAsync(mismatchedRevisionFlowId)); + // The row is present and inconsistent with itself: unreadable, never "gone" (round 38). + await Assert.ThrowsAsync(() => store.LoadAsync(mismatchedRevisionFlowId)); } finally { diff --git a/tests/AsyncResponse.IntegrationTests/PostgreSqlDirectIntegrationTests.cs b/tests/AsyncResponse.IntegrationTests/PostgreSqlDirectIntegrationTests.cs index d74a8f81c..2870488d2 100644 --- a/tests/AsyncResponse.IntegrationTests/PostgreSqlDirectIntegrationTests.cs +++ b/tests/AsyncResponse.IntegrationTests/PostgreSqlDirectIntegrationTests.cs @@ -1382,6 +1382,44 @@ private ServiceProvider BuildProvider( return services.BuildServiceProvider(); } + /// + /// Round 39: the sweep's page carries the envelope only for rows nobody has acknowledged; an + /// acknowledged row comes back header-only and is hydrated by id when a live subscription + /// still has to receive it. Pre-fix both reads returned the body for every row (and the + /// by-id read did not exist). + /// + [Fact] + public async Task LoadMessages_ShipsTheEnvelopeOnlyForUnacknowledgedRows_AndHydratesById() + { + await WithDataSourceAsync("sweep_header_only", async (schema, dataSource) => + { + var sql = new PostgreSqlChannelSql(dataSource, Options.Create(ChannelOptions(schema))); + await sql.EnsureCreatedAsync(); + var correlationId = $"header-only-{Guid.NewGuid():N}"; + var since = (await sql.GetServerTimeUtcAsync(CancellationToken.None)).AddSeconds(-1); + var pending = Guid.NewGuid(); + var acked = Guid.NewGuid(); + await sql.InsertMessageAsync(pending, correlationId, """{"Success":true,"Payload":"pending"}""", TimeSpan.FromMinutes(5), CancellationToken.None); + await sql.InsertMessageAsync(acked, correlationId, """{"Success":true,"Payload":"acked"}""", TimeSpan.FromMinutes(5), CancellationToken.None); + Assert.True(await sql.TryClaimForDeliveryAsync(acked, CancellationToken.None)); + + var page = await sql.LoadMessagesAsync(correlationId, since, 16, null, null, CancellationToken.None); + Assert.Equal(2, page.Count); + Assert.Contains("\"pending\"", Assert.Single(page, m => m.Id == pending).EnvelopeJson, StringComparison.Ordinal); + var ackedRow = Assert.Single(page, m => m.Id == acked); + Assert.Null(ackedRow.EnvelopeJson); + Assert.NotNull(ackedRow.AckedAtUtc); + Assert.NotNull(ackedRow.AckedSeq); + + var hydrated = await sql.LoadMessagesByIdAsync(correlationId, [acked, Guid.NewGuid()], CancellationToken.None); + var full = Assert.Single(hydrated); + Assert.Equal(acked, full.Id); + Assert.Contains("\"acked\"", full.EnvelopeJson, StringComparison.Ordinal); + Assert.Equal(ackedRow.AckedSeq, full.AckedSeq); + Assert.Empty(await sql.LoadMessagesByIdAsync("some-other-correlation", [acked], CancellationToken.None)); + }); + } + private async Task WithDataSourceAsync(string prefix, Func body) { var schema = NewSchema(prefix); @@ -1704,6 +1742,10 @@ await WithDataSourceAsync("channel_edges", async (schema, dataSource) => // proves ProcessUnderCapturedContextAsync ran end to end. Assert.False(subscription1.Completion.Task.IsCompleted); Assert.False(subscription2.Completion.Task.IsCompleted); + // The converter's own contract violations stay a plain JsonException AND keep their + // message: JsonSafety's body-free scrub (round 36) skips the failures the library + // authored, which name only the contract's own properties, and replaces only the + // reader's own — whose messages quote the inbound body. var liveError = await Assert.ThrowsAsync( () => subscription3.Completion.Task.WaitAsync(TimeSpan.FromSeconds(5))); Assert.Contains("SchemaVersion", liveError.Message); diff --git a/tests/AsyncResponse.IntegrationTests/SampleTestEndpointGateTests.cs b/tests/AsyncResponse.IntegrationTests/SampleTestEndpointGateTests.cs new file mode 100644 index 000000000..c422b0398 --- /dev/null +++ b/tests/AsyncResponse.IntegrationTests/SampleTestEndpointGateTests.cs @@ -0,0 +1,145 @@ +using AsyncResponse.Sample; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Mvc.Testing; +using Microsoft.AspNetCore.Routing; +using Microsoft.Extensions.DependencyInjection; +using System.Net; +using Xunit; + +namespace AsyncResponse.IntegrationTests; + +/// +/// Round 34, finding 3: the sample's test-only mutation routes (/seed-recovery, +/// /test/recovery/{correlationId}, /test/reset) were mapped unconditionally and +/// unauthenticated — a Production instance against a shared backend let any caller erase every +/// recovery registration. They are now mapped only in Development or when +/// Sample:EnableTestEndpoints says so (the integration AppHost and the load-test launcher +/// opt in). In-process, no Docker. Pre-fix failure: Production answers 200 on all three. +/// +[Trait(Batches.Trait, Batches.None)] +public sealed class SampleTestEndpointGateTests +{ + [Fact] + public async Task ProductionByDefault_DoesNotMapTheTestMutationRoutes() + { + await using var factory = new GatedAppFactory(environment: "Production", enableTestEndpoints: null); + using var client = factory.CreateClient(); + + Assert.Equal(HttpStatusCode.NotFound, (await client.PostAsync("/test/reset", content: null)).StatusCode); + Assert.Equal(HttpStatusCode.NotFound, (await client.DeleteAsync("/test/recovery/any")).StatusCode); + Assert.Equal(HttpStatusCode.NotFound, (await client.PostAsync("/seed-recovery?correlationId=any", content: null)).StatusCode); + + // The app itself is up: the operational routes are unaffected. + Assert.Equal(HttpStatusCode.OK, (await client.GetAsync("/healthz")).StatusCode); + } + + /// + /// Round 35: the switch covered three routes while the simulation, injection, and ledger + /// routes stayed mapped and unauthenticated in Production — /publish injected responses + /// for any correlation id (202), /crash was mapped (409 only because the in-memory + /// channel cannot drop subscriptions), and GET /durable-flow/{id} returned the full + /// ledger, input JSON included. Pre-fix failure: /publish answers 202 here. + /// + [Fact] + public async Task ProductionByDefault_DoesNotMapTheSimulationInjectionOrLedgerRoutes() + { + await using var factory = new GatedAppFactory(environment: "Production", enableTestEndpoints: null); + using var client = factory.CreateClient(); + + Assert.Equal(HttpStatusCode.NotFound, (await client.PostAsync("/publish?correlationId=any&status=Completed", content: null)).StatusCode); + Assert.Equal(HttpStatusCode.NotFound, (await client.PostAsync("/crash", content: null)).StatusCode); + Assert.Equal(HttpStatusCode.NotFound, (await client.PostAsync("/arm", content: null)).StatusCode); + Assert.Equal(HttpStatusCode.NotFound, (await client.PostAsync("/lost-subscriber-flow?outcome=Completed", content: null)).StatusCode); + Assert.Equal(HttpStatusCode.NotFound, (await client.PostAsync("/emit-response?correlationId=any&useAttribute=true", content: null)).StatusCode); + Assert.Equal(HttpStatusCode.NotFound, (await client.GetAsync("/calls?key=any")).StatusCode); + Assert.Equal(HttpStatusCode.NotFound, (await client.GetAsync("/durable-flow/any")).StatusCode); + Assert.Equal(HttpStatusCode.NotFound, (await client.PostAsync("/durable-flow/any/resume", content: null)).StatusCode); + + // Starting a flow is the demo itself and stays operational. + Assert.Equal(HttpStatusCode.OK, (await client.PostAsync("/durable-flow?name=acme", content: null)).StatusCode); + } + + /// + /// The whole route inventory, so a new unauthenticated affordance cannot slip into Production + /// unnoticed: everything the sample maps there must be on this list. + /// + [Fact] + public async Task ProductionByDefault_MapsExactlyTheOperationalRouteInventory() + { + await using var factory = new GatedAppFactory(environment: "Production", enableTestEndpoints: null); + using var client = factory.CreateClient(); // forces the host to build + + var mapped = factory.Services.GetRequiredService().Endpoints + .OfType() + .Select(endpoint => endpoint.RoutePattern.RawText ?? string.Empty) + .Select(pattern => pattern.StartsWith('/') ? pattern : "/" + pattern) + .Distinct(StringComparer.Ordinal) + .OrderBy(pattern => pattern, StringComparer.Ordinal) + .ToArray(); + + string[] operational = + [ + "/", + "/alive", + "/ambient-exception", + "/attach", + "/config", + "/durable-flow", + "/durable-flow-child", + "/healthz", + "/multi-step", + "/openapi/{documentName}.json", + "/reply-target", + "/request-response", + "/shared-correlation-exception", + "/worker", + ]; + string[] gated = + [ + "/arm", "/calls", "/crash", "/durable-flow/{flowId}", "/durable-flow/{flowId}/resume", "/emit-response", + "/lost-subscriber-flow", "/publish", "/seed-recovery", "/test/recovery/{correlationId}", "/test/reset", + ]; + + Assert.Empty(mapped.Intersect(gated, StringComparer.Ordinal)); + Assert.Equal(operational.OrderBy(p => p, StringComparer.Ordinal), mapped); + } + + [Fact] + public async Task ExplicitOptIn_MapsTheTestMutationRoutes_EvenInProduction() + { + await using var factory = new GatedAppFactory(environment: "Production", enableTestEndpoints: true); + using var client = factory.CreateClient(); + + Assert.Equal(HttpStatusCode.OK, (await client.PostAsync("/test/reset", content: null)).StatusCode); + Assert.Equal(HttpStatusCode.OK, (await client.DeleteAsync("/test/recovery/any")).StatusCode); + // The simulation and ledger routes ride the same switch. + Assert.Equal(HttpStatusCode.NotFound, (await client.GetAsync("/durable-flow/unknown-run")).StatusCode); + Assert.Equal(HttpStatusCode.Accepted, (await client.PostAsync("/publish?correlationId=nobody&status=Completed", content: null)).StatusCode); + } + + [Fact] + public async Task ExplicitOptOut_UnmapsTheTestMutationRoutes_EvenInDevelopment() + { + await using var factory = new GatedAppFactory(environment: "Development", enableTestEndpoints: false); + using var client = factory.CreateClient(); + + Assert.Equal(HttpStatusCode.NotFound, (await client.PostAsync("/test/reset", content: null)).StatusCode); + } + + /// + /// Boots the sample in-process on the fully in-memory providers. + /// only identifies the sample's assembly (the referenced Aspire AppHost also defines a + /// Program). + /// + private sealed class GatedAppFactory(string environment, bool? enableTestEndpoints) : WebApplicationFactory + { + protected override void ConfigureWebHost(IWebHostBuilder builder) + { + builder.UseEnvironment(environment); + builder.UseSetting("AsyncResponse:Channel", "InMemory"); + builder.UseSetting("AsyncResponse:Transport", "InMemory"); + if (enableTestEndpoints is { } enable) + builder.UseSetting("Sample:EnableTestEndpoints", enable ? "true" : "false"); + } + } +} diff --git a/tests/AsyncResponse.IntegrationTests/SqlServerDirectIntegrationTests.cs b/tests/AsyncResponse.IntegrationTests/SqlServerDirectIntegrationTests.cs index 72bf856fd..e4bbfb7ac 100644 --- a/tests/AsyncResponse.IntegrationTests/SqlServerDirectIntegrationTests.cs +++ b/tests/AsyncResponse.IntegrationTests/SqlServerDirectIntegrationTests.cs @@ -1575,6 +1575,44 @@ private ServiceProvider BuildProvider( return services.BuildServiceProvider(); } + /// + /// Round 39: the sweep's page carries the envelope only for rows nobody has acknowledged; an + /// acknowledged row comes back header-only and is hydrated by id when a live subscription + /// still has to receive it. Pre-fix both reads returned the body for every row (and the + /// by-id read did not exist). + /// + [Fact] + public async Task LoadMessages_ShipsTheEnvelopeOnlyForUnacknowledgedRows_AndHydratesById() + { + await WithSchemaAsync("sweep_header_only", async schema => + { + var sql = new SqlServerChannelSql(Options.Create(ChannelOptions(schema))); + await sql.EnsureCreatedAsync(); + var correlationId = $"header-only-{Guid.NewGuid():N}"; + var since = (await sql.GetServerTimeUtcAsync(CancellationToken.None)).AddSeconds(-1); + var pending = Guid.NewGuid(); + var acked = Guid.NewGuid(); + await sql.InsertMessageAsync(pending, correlationId, """{"Success":true,"Payload":"pending"}""", TimeSpan.FromMinutes(5), CancellationToken.None); + await sql.InsertMessageAsync(acked, correlationId, """{"Success":true,"Payload":"acked"}""", TimeSpan.FromMinutes(5), CancellationToken.None); + Assert.True(await sql.TryClaimForDeliveryAsync(acked, CancellationToken.None)); + + var page = await sql.LoadMessagesAsync(correlationId, since, 16, null, null, CancellationToken.None); + Assert.Equal(2, page.Count); + Assert.Contains("\"pending\"", Assert.Single(page, m => m.Id == pending).EnvelopeJson, StringComparison.Ordinal); + var ackedRow = Assert.Single(page, m => m.Id == acked); + Assert.Null(ackedRow.EnvelopeJson); + Assert.NotNull(ackedRow.AckedAtUtc); + Assert.NotNull(ackedRow.AckedSeq); + + var hydrated = await sql.LoadMessagesByIdAsync(correlationId, [acked, Guid.NewGuid()], CancellationToken.None); + var full = Assert.Single(hydrated); + Assert.Equal(acked, full.Id); + Assert.Contains("\"acked\"", full.EnvelopeJson, StringComparison.Ordinal); + Assert.Equal(ackedRow.AckedSeq, full.AckedSeq); + Assert.Empty(await sql.LoadMessagesByIdAsync("some-other-correlation", [acked], CancellationToken.None)); + }); + } + private async Task WithSchemaAsync(string prefix, Func body) { var schema = NewSchema(prefix); @@ -1834,6 +1872,10 @@ await WithSchemaAsync("channel_edges", async schema => // proves ProcessUnderCapturedContextAsync ran end to end. Assert.False(subscription1.Completion.Task.IsCompleted); Assert.False(subscription2.Completion.Task.IsCompleted); + // The converter's own contract violations stay a plain JsonException AND keep their + // message: JsonSafety's body-free scrub (round 36) skips the failures the library + // authored, which name only the contract's own properties, and replaces only the + // reader's own — whose messages quote the inbound body. var liveError = await Assert.ThrowsAsync( () => subscription3.Completion.Task.WaitAsync(TimeSpan.FromSeconds(5))); Assert.Contains("SchemaVersion", liveError.Message); diff --git a/tests/AsyncResponse.Tests/AsyncResponseActivityCollector.cs b/tests/AsyncResponse.Tests/AsyncResponseActivityCollector.cs index f5b161eae..07a9179d4 100644 --- a/tests/AsyncResponse.Tests/AsyncResponseActivityCollector.cs +++ b/tests/AsyncResponse.Tests/AsyncResponseActivityCollector.cs @@ -41,6 +41,13 @@ public Activity Single(string name, string tagKey, object? tagValue) return Assert.Single(_activities, activity => activity.OperationName == name && Equals(Tag(activity, tagKey), tagValue)); } + /// Every activity recorded so far (a snapshot). + public IReadOnlyList All() + { + lock (_gate) + return [.. _activities]; + } + public int Count(string name) { lock (_gate) diff --git a/tests/AsyncResponse.Tests/AsyncResponseEnvelopeTests.cs b/tests/AsyncResponse.Tests/AsyncResponseEnvelopeTests.cs index 16b0a5477..0792b577c 100644 --- a/tests/AsyncResponse.Tests/AsyncResponseEnvelopeTests.cs +++ b/tests/AsyncResponse.Tests/AsyncResponseEnvelopeTests.cs @@ -133,9 +133,15 @@ public void AbsentPayload_OnASuccessEnvelope_ThrowsJsonException(string json) Assert.Throws(() => JsonSerializer.Deserialize>( json, AsyncResponseEnvelopeOptions.Instance)); - // The ingress entry point: the parse failure surfaces as the permanently-classified - // InvalidDataException, not as an envelope the dispatcher would complete a waiter with. - Assert.Throws(() => AsyncResponseEnvelopeJson.SafeDeserialize(json)); + // The ingress entry point: the parse failure surfaces as a throw the ingress classifies + // as PERMANENT, not as an envelope the dispatcher would complete a waiter with — that is + // what this pins. Round 36: it also keeps its reason. The body-free scrub replaces only + // the reader's own messages (which quote the inbound body) and leaves the ones the + // library authored, which name the contract's own properties, so an operator still learns + // WHY a producer's envelope was rejected instead of only where the reader stopped. + var ingressFailure = Assert.ThrowsAny( + () => AsyncResponseEnvelopeJson.SafeDeserialize(json)); + Assert.Contains("Payload is null or absent", ingressFailure.Message, StringComparison.Ordinal); } /// diff --git a/tests/AsyncResponse.Tests/CoreCoverageTests.cs b/tests/AsyncResponse.Tests/CoreCoverageTests.cs index 99476fabb..dce6a7a82 100644 --- a/tests/AsyncResponse.Tests/CoreCoverageTests.cs +++ b/tests/AsyncResponse.Tests/CoreCoverageTests.cs @@ -324,12 +324,13 @@ public async Task InMemoryFlowStateStore_CoversMalformedReadsExpiredLeaseTakeove AddRawFlowEntry(store, "revision-mismatch", FlowStateJson.Serialize(State("revision-mismatch")), revision: 1); AddRawFlowEntry(store, "flow-mismatch", FlowStateJson.Serialize(State("different-id")), revision: 0); - // A present-but-unreadable row throws; only genuine absence (and the two benign mismatches) - // reads as null, because callers ack on null. See FlowStateUnreadableException. + // A present-but-unreadable row throws — malformed, and (since round 38) inconsistent with + // its own revision or key; only genuine absence reads as null, because callers ack on + // null. See FlowStateUnreadableException. await Assert.ThrowsAsync(() => store.LoadAsync("malformed")); await Assert.ThrowsAsync(() => store.LoadAsync("null-json")); - Assert.Null(await store.LoadAsync("revision-mismatch")); - Assert.Null(await store.LoadAsync("flow-mismatch")); + await Assert.ThrowsAsync(() => store.LoadAsync("revision-mismatch")); + await Assert.ThrowsAsync(() => store.LoadAsync("flow-mismatch")); Assert.Null(await store.LoadAsync("missing")); await Assert.ThrowsAsync(() => store.LoadAsync(" ")); diff --git a/tests/AsyncResponse.Tests/CosmosDurableFlowStateStoreTests.cs b/tests/AsyncResponse.Tests/CosmosDurableFlowStateStoreTests.cs index 561405f37..39e8e9505 100644 --- a/tests/AsyncResponse.Tests/CosmosDurableFlowStateStoreTests.cs +++ b/tests/AsyncResponse.Tests/CosmosDurableFlowStateStoreTests.cs @@ -93,8 +93,8 @@ public async Task Store_HandlesLeaseOutcomesAndReleaseRaces() using var harness = new CosmosHarness(); var state = CreateState("flow"); var document = Document(state, DateTime.UtcNow.AddMinutes(5)); - harness.Reads(document); - harness.ReplacesSuccessfully(); + harness.QueriesLease(document); + harness.PatchesSuccessfully(); Assert.True(await harness.Store.TryAcquireLeaseAsync("flow", "owner", TimeSpan.FromMinutes(1))); @@ -107,22 +107,30 @@ public async Task Store_HandlesLeaseOutcomesAndReleaseRaces() document.LeaseId = "owner"; await harness.Store.ReleaseLeaseAsync("flow", "owner"); - harness.ReadsException(HttpStatusCode.NotFound); + harness.QueriesNothing(); Assert.False(await harness.Store.TryAcquireLeaseAsync("flow", "owner", TimeSpan.FromMinutes(1))); await harness.Store.ReleaseLeaseAsync("flow", "owner"); - harness.Reads(document); - harness.Container - .Setup(container => container.ReplaceItemAsync( - It.IsAny(), - It.IsAny(), - It.IsAny(), - It.IsAny(), - It.IsAny())) - .ThrowsAsync(CosmosError(HttpStatusCode.PreconditionFailed)); + // The patch's own 404/0: purged between the projection read and the write. + harness.QueriesLease(document); + harness.PatchesThrowing(CosmosError(HttpStatusCode.NotFound)); Assert.False(await harness.Store.TryRenewLeaseAsync("flow", "owner", TimeSpan.FromMinutes(1))); await harness.Store.ReleaseLeaseAsync("flow", "owner"); + harness.PatchesThrowing(CosmosError(HttpStatusCode.PreconditionFailed)); + Assert.False(await harness.Store.TryRenewLeaseAsync("flow", "owner", TimeSpan.FromMinutes(1))); + await harness.Store.ReleaseLeaseAsync("flow", "owner"); + + // The lease paths never touch the document body: no point read, no replace. + harness.Container.Verify( + container => container.ReadItemAsync( + It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), + Times.Never); + harness.Container.Verify( + container => container.ReplaceItemAsync( + It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), + Times.Never); + await Assert.ThrowsAsync(() => harness.Store.TryAcquireLeaseAsync(" ", "owner", TimeSpan.FromMinutes(1))); await Assert.ThrowsAsync(() => @@ -194,10 +202,13 @@ public async Task Store_LoadsReadableStateAndHandlesDeleteOutcomes() harness.Reads(Document(state, DateTime.UtcNow.AddSeconds(-1))); Assert.Null(await harness.Store.LoadAsync("flow")); + // A document whose physical revision disagrees with the one inside its JSON is present + // and inconsistent: unreadable, never absent (round 38 — "absent" acked its wake-up). var unreadable = Document(state, DateTime.UtcNow.AddMinutes(1)); unreadable.Revision = state.Revision + 1; harness.Reads(unreadable); - Assert.Null(await harness.Store.LoadAsync("flow")); + var inconsistent = await Assert.ThrowsAsync(() => harness.Store.LoadAsync("flow")); + Assert.Contains("stored revision is 1", inconsistent.Reason, StringComparison.Ordinal); harness.Reads(Document(state, DateTime.UtcNow.AddMinutes(1))); Assert.Equal("flow", (await harness.Store.LoadAsync("flow"))?.FlowId); @@ -241,13 +252,18 @@ public async Task Store_ExhaustsOptimisticConcurrencyRetries() Assert.False(await harness.Store.TryUpdateAsync( "flow", state, expectedRevision: 0, TimeSpan.FromMinutes(1))); - harness.ReadsFactory(() => + // The lease paths read a projection and patch; every patch losing its ETag race exhausts + // the same bounded loop. + harness.QueriesLease(() => new CosmosLeaseProjection { - var document = Document(CreateState("flow"), DateTime.UtcNow.AddMinutes(5)); - document.LeaseId = "owner"; - document.LeaseExpiresAtUtc = DateTime.UtcNow.AddMinutes(1); - return document; + Id = "flow", + ETag = "etag", + ExpiresAtUtc = DateTime.UtcNow.AddMinutes(5), + Revision = 0, + LeaseId = "owner", + LeaseExpiresAtUtc = DateTime.UtcNow.AddMinutes(1) }); + harness.PatchesThrowing(CosmosError(HttpStatusCode.PreconditionFailed)); Assert.False(await harness.Store.TryRenewLeaseAsync( "flow", "owner", TimeSpan.FromMinutes(1))); await harness.Store.ReleaseLeaseAsync("flow", "owner"); @@ -287,6 +303,7 @@ public async Task Store_CoversRevisionAndLeaseEligibilityBranches() document.Revision = null; harness.Reads(document); + harness.QueriesLease(document); // Present-but-uninterpretable: the document is in the container, so reporting absence here // would ack the only wake-up of a run that still exists. await Assert.ThrowsAsync(() => harness.Store.LoadAsync("flow")); @@ -306,12 +323,13 @@ public async Task Store_CoversRevisionAndLeaseEligibilityBranches() document.LeaseId = "owner"; document.LeaseExpiresAtUtc = DateTime.UtcNow.AddMinutes(-1); harness.Reads(document); + harness.QueriesLease(document); Assert.False(await harness.Store.TryUpdateAsync( "flow", state, expectedRevision: 0, TimeSpan.FromMinutes(1), leaseId: "owner")); Assert.False(await harness.Store.TryRenewLeaseAsync("flow", "owner", TimeSpan.FromMinutes(1))); document.LeaseId = "other"; - harness.ReplacesSuccessfully(); + harness.PatchesSuccessfully(); Assert.True(await harness.Store.TryAcquireLeaseAsync("flow", "owner", TimeSpan.FromMinutes(1))); } @@ -325,31 +343,41 @@ public async Task Store_LeaseWritesRewriteTtlToRemainingLogicalWindow() // refreshes _ts (the server TTL anchor), so a lease write persisting that value unchanged // would restart the whole physical-retention countdown on each heartbeat. document.Ttl = (int)TimeSpan.FromHours(2).TotalSeconds; - harness.Reads(document); - CosmosFlowStateDocument? replaced = null; - harness.ReplacesSuccessfully(written => replaced = written); + harness.QueriesLease(document); + IReadOnlyList? patched = null; + PatchItemRequestOptions? requestOptions = null; + harness.PatchesSuccessfully((operations, options) => (patched, requestOptions) = (operations, options)); Assert.True(await harness.Store.TryAcquireLeaseAsync("flow", "owner", TimeSpan.FromMinutes(1))); - Assert.NotNull(replaced?.Ttl); - Assert.InRange(replaced!.Ttl!.Value, 540, 601); // the ~10 minutes left, never the stored 7200 - - // Release replaces too and must realign the same way. + Assert.NotNull(patched); + Assert.InRange((int)PatchValue(PatchFor(patched!, "/ttl"))!, 540, 601); // the ~10 minutes left, never the stored 7200 + Assert.Equal("owner", PatchValue(PatchFor(patched!, "/leaseId"))); + Assert.NotNull(PatchValue(PatchFor(patched!, "/leaseExpiresAtUtc"))); + // Round 36: conditional on the projection's ETag, and no document body comes back. + Assert.Equal("etag", requestOptions!.IfMatchEtag); + Assert.False(requestOptions.EnableContentResponseOnWrite); + + // Release patches too and must realign the same way. (The projection is re-read from + // `document` on every call, so the acquired state is modeled explicitly.) + document.LeaseId = "owner"; + document.LeaseExpiresAtUtc = DateTime.UtcNow.AddMinutes(1); document.ExpiresAtUtc = DateTime.UtcNow.AddMinutes(10); document.Ttl = (int)TimeSpan.FromHours(2).TotalSeconds; - replaced = null; + patched = null; await harness.Store.ReleaseLeaseAsync("flow", "owner"); - Assert.NotNull(replaced); - Assert.Null(replaced!.LeaseId); - Assert.InRange(replaced.Ttl!.Value, 540, 601); + Assert.NotNull(patched); + Assert.Null(PatchValue(PatchFor(patched!, "/leaseId"))); + Assert.Null(PatchValue(PatchFor(patched!, "/leaseExpiresAtUtc"))); + Assert.InRange((int)PatchValue(PatchFor(patched!, "/ttl"))!, 540, 601); // An already-due ledger collapses to the 1-second floor (Cosmos rejects 0) instead of the // release granting it a fresh retention window. document.LeaseId = "owner"; document.ExpiresAtUtc = DateTime.UtcNow.AddMinutes(-5); document.Ttl = (int)TimeSpan.FromHours(2).TotalSeconds; - replaced = null; + patched = null; await harness.Store.ReleaseLeaseAsync("flow", "owner"); - Assert.Equal(1, replaced!.Ttl); + Assert.Equal(1, PatchValue(PatchFor(patched!, "/ttl"))); } [Fact] @@ -409,12 +437,104 @@ public async Task LeaseFence_TreatsAMissingLeaseDeadlineAsNotHeld() current.LeaseId = "owner"; current.LeaseExpiresAtUtc = null; harness.Reads(current); + harness.QueriesLease(current); harness.ReplacesSuccessfully(); + harness.PatchesSuccessfully(); Assert.False(await harness.Store.TryUpdateAsync("flow", state, 0, TimeSpan.FromMinutes(1), leaseId: "owner")); Assert.False(await harness.Store.TryRenewLeaseAsync("flow", "owner", TimeSpan.FromMinutes(1))); } + // ---- Round 39: the size budget is enforced on the DOCUMENT Cosmos receives, not the ledger inside it. ---- + + /// + /// A ledger of escaped backslashes: its own JSON is 1.2 MB (under the 1.9 MB default), but + /// embedded as the document's stateJson string every \\ escapes again to + /// \\\\ — a 2.4 MB document Cosmos's 2 MB item cap refuses on every retry. Pre-fix the + /// guard measured the inner JSON only and the create went to the container. (Backslashes + /// rather than quotes so the arithmetic does not depend on either serializer's encoder.) + /// + private static FlowState EscapeHeavyState(string flowId) + { + var state = CreateState(flowId); + state.Steps = new Dictionary + { + ["blob"] = new FlowStepState + { + Completed = true, + // A JSON string literal of 300k escaped backslashes: ResultJson is JSON text. + ResultJson = "\"" + new string('\\', 600_000) + "\"" + } + }; + return state; + } + + [Fact] + public async Task TryCreate_RejectsALedgerWhoseEscapedDocumentExceedsTheBudget() + { + using var harness = new CosmosHarness(); + var state = EscapeHeavyState("flow"); + var innerBytes = System.Text.Encoding.UTF8.GetByteCount(JsonSerializer.Serialize(state)); + Assert.InRange(innerBytes, 1_000_000, 1_900_000); + + // The shared source's FlowStateTooLargeException is internal to each store assembly (one + // copy per package), so it is matched by name, as the other store suites do. + var ex = await Assert.ThrowsAnyAsync( + () => harness.Store.TryCreateAsync("flow", state, TimeSpan.FromMinutes(1))); + + Assert.Equal("FlowStateTooLargeException", ex.GetType().Name); + Assert.Equal("flow", ex.GetType().GetProperty("FlowId")!.GetValue(ex)); + var reported = (long)ex.GetType().GetProperty("SerializedSizeBytes")!.GetValue(ex)!; + Assert.True(reported > 1_900_000, $"reported size {reported} should be the escaped document's"); + harness.Container.Verify(container => container.CreateItemAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny()), Times.Never); + } + + [Fact] + public async Task TryUpdate_RejectsALedgerWhoseEscapedDocumentExceedsTheBudget() + { + using var harness = new CosmosHarness(); + var state = EscapeHeavyState("flow"); + state.Revision = 1; + harness.Reads(Document(CreateState("flow"), DateTime.UtcNow.AddMinutes(5))); + harness.ReplacesSuccessfully(); + + var ex = await Assert.ThrowsAnyAsync( + () => harness.Store.TryUpdateAsync("flow", state, 0, TimeSpan.FromMinutes(1))); + Assert.Equal("FlowStateTooLargeException", ex.GetType().Name); + + harness.Container.Verify(container => container.ReplaceItemAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny()), Times.Never); + } + + /// A ledger that fits as a document still writes (the guard is not just "smaller"). + [Fact] + public async Task TryCreate_AcceptsALedgerWhoseDocumentFitsTheBudget() + { + using var harness = new CosmosHarness(); + var state = CreateState("flow"); + state.Steps = new Dictionary + { + ["blob"] = new FlowStepState { Completed = true, ResultJson = "\"" + new string('x', 500_000) + "\"" } + }; + harness.Container + .Setup(container => container.CreateItemAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .ReturnsAsync(Mock.Of>()); + + Assert.True(await harness.Store.TryCreateAsync("flow", state, TimeSpan.FromMinutes(1))); + } + private static FlowState CreateState(string flowId) => new() { FlowId = flowId, @@ -485,8 +605,23 @@ public async Task Writes_NotFoundWithANonZeroSubStatus_ThrowInsteadOfReportingAb return document; }); - // The replace after a successful read answers 404/1002. + harness.QueriesLease(() => + { + var document = Document(CreateState("flow"), DateTime.UtcNow.AddMinutes(5)); + return new CosmosLeaseProjection + { + Id = "flow", + ETag = "etag", + ExpiresAtUtc = document.ExpiresAtUtc, + Revision = document.Revision, + LeaseId = "owner", + LeaseExpiresAtUtc = DateTime.UtcNow.AddMinutes(1) + }; + }); + + // The write after a successful read answers 404/1002. harness.ReplacesThrowing(ReadSessionNotAvailable()); + harness.PatchesThrowing(ReadSessionNotAvailable()); var checkpoint = await Assert.ThrowsAsync( () => harness.Store.TryUpdateAsync("flow", state, 0, TimeSpan.FromMinutes(1), leaseId: "owner")); Assert.Equal(1002, checkpoint.SubStatusCode); @@ -496,6 +631,7 @@ public async Task Writes_NotFoundWithANonZeroSubStatus_ThrowInsteadOfReportingAb // The read itself answers 404/1002 (one filter guards both calls of each path). harness.ReadsThrowing(ReadSessionNotAvailable()); + harness.QueriesThrowing(ReadSessionNotAvailable()); await Assert.ThrowsAsync(() => harness.Store.TryUpdateAsync("flow", state, 0, TimeSpan.FromMinutes(1))); await Assert.ThrowsAsync(() => harness.Store.TryAcquireLeaseAsync("flow", "owner", TimeSpan.FromMinutes(1))); await Assert.ThrowsAsync(() => harness.Store.ReleaseLeaseAsync("flow", "owner")); @@ -505,6 +641,7 @@ public async Task Writes_NotFoundWithANonZeroSubStatus_ThrowInsteadOfReportingAb // Sub-status 0 stays a genuine absence on every path. harness.ReadsException(HttpStatusCode.NotFound); + harness.QueriesNothing(); Assert.False(await harness.Store.TryUpdateAsync("flow", state, 0, TimeSpan.FromMinutes(1))); Assert.False(await harness.Store.TryAcquireLeaseAsync("flow", "owner", TimeSpan.FromMinutes(1))); await harness.Store.ReleaseLeaseAsync("flow", "owner"); @@ -512,6 +649,30 @@ public async Task Writes_NotFoundWithANonZeroSubStatus_ThrowInsteadOfReportingAb Assert.False(await harness.Store.TryDeleteAsync("flow")); } + /// + /// Round 36: the lease paths read a projection and patch. A projection without _etag + /// (a serializer that hides system properties) cannot fence a write; silently treating it as + /// "not held" would let the executor ack a wake-up as a duplicate against a run nobody holds. + /// + [Fact] + public async Task LeaseQuery_WithoutAnEtag_ThrowsInsteadOfReportingTheLeaseFree() + { + using var harness = new CosmosHarness(); + var document = Document(CreateState("flow"), DateTime.UtcNow.AddMinutes(5)); + harness.QueriesLease(() => new CosmosLeaseProjection + { + Id = "flow", + ETag = "", + ExpiresAtUtc = document.ExpiresAtUtc, + Revision = document.Revision + }); + harness.PatchesSuccessfully(); + + var ex = await Assert.ThrowsAsync( + () => harness.Store.TryAcquireLeaseAsync("flow", "owner", TimeSpan.FromMinutes(1))); + Assert.Contains("_etag", ex.Message, StringComparison.Ordinal); + } + private static CosmosException ReadSessionNotAvailable() => new("read session not available", HttpStatusCode.NotFound, 1002, "activity", 0); @@ -525,6 +686,92 @@ private static Mock ContainerResult(ContainerProperties prope return response; } + [Fact] + public async Task Round37_ConcurrentLeaseOperations_NeverExecuteWithEachOthersId() + { + // Regression (round 37, F1): the lease projection query was ONE static QueryDefinition + // parameterized per call. WithParameter replaces the named parameter in place and returns + // the same instance, so two flows' lease operations interleaving on one store instance + // raced on that one parameter bag: flow A built its query with @id = A, flow B then set + // @id = B on the same object, and A's query EXECUTED under A's partition key asking for + // B — no such document in A's partition, "no rows", and a healthy renewal reported false + // (the executor abandons the run and it replays). The mocks that hand back a fixed document + // for any query could never see it. This one answers what the query ASKS FOR, as the + // service does, observes the @id each query carries when it executes, and forces the + // interleaving: A's query is built first, B's is built before A's executes. + using var harness = new CosmosHarness(); + var documents = new Dictionary(StringComparer.Ordinal) + { + ["flow-a"] = Document(CreateState("flow-a"), DateTime.UtcNow.AddMinutes(5)), + ["flow-b"] = Document(CreateState("flow-b"), DateTime.UtcNow.AddMinutes(5)) + }; + var aQueried = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var bQueried = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var executedWith = new System.Collections.Concurrent.ConcurrentDictionary(StringComparer.Ordinal); + + harness.Container + .Setup(item => item.GetItemQueryIterator( + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns((QueryDefinition query, string? _, QueryRequestOptions options) => + { + // PartitionKey renders as a JSON array of its components: ["flow-a"]. + var partition = JsonSerializer.Deserialize(options.PartitionKey!.Value.ToString())![0]; + if (partition == "flow-a") + aQueried.TrySetResult(); + else + bQueried.TrySetResult(); + + var more = true; + var iterator = new Mock>(); + iterator.SetupGet(item => item.HasMoreResults).Returns(() => more); + iterator + .Setup(item => item.ReadNextAsync(It.IsAny())) + .Returns(async () => + { + // A executes only once B's query has been built. + if (partition == "flow-a") + await bQueried.Task.WaitAsync(TimeSpan.FromSeconds(5)); + + more = false; + var requestedId = (string?)query.GetQueryParameters().Single(parameter => parameter.Name == "@id").Value; + executedWith[partition] = requestedId; + + // WHERE c.id = @id under this partition key: a row only when the id asked + // for lives in the partition queried. + var rows = requestedId == partition && documents.TryGetValue(requestedId, out var document) + ? new[] + { + new CosmosLeaseProjection + { + Id = document.Id, + ETag = "etag", + ExpiresAtUtc = document.ExpiresAtUtc, + Revision = document.Revision, + LeaseId = document.LeaseId, + LeaseExpiresAtUtc = document.LeaseExpiresAtUtc + } + } + : []; + var page = new Mock>(); + page.Setup(item => item.GetEnumerator()).Returns(() => ((IEnumerable)rows).GetEnumerator()); + return page.Object; + }); + return iterator.Object; + }); + harness.PatchesSuccessfully(); + + var acquireA = harness.Store.TryAcquireLeaseAsync("flow-a", "owner-a", TimeSpan.FromMinutes(1)); + await aQueried.Task.WaitAsync(TimeSpan.FromSeconds(5)); + var acquireB = harness.Store.TryAcquireLeaseAsync("flow-b", "owner-b", TimeSpan.FromMinutes(1)); + + Assert.True(await acquireB.WaitAsync(TimeSpan.FromSeconds(5))); + Assert.True(await acquireA.WaitAsync(TimeSpan.FromSeconds(5))); + Assert.Equal("flow-a", executedWith["flow-a"]); + Assert.Equal("flow-b", executedWith["flow-b"]); + } + private sealed class CosmosHarness : IDisposable { private readonly Mock _containerResponse; @@ -633,6 +880,98 @@ public void ReplacesSuccessfully(Action? onReplace = nu (document, _, _, _, _) => onReplace?.Invoke(document)) .ReturnsAsync(Mock.Of>()); + // ---- Round 36: the lease paths read a projection (no stateJson) and patch the lease fields. ---- + + /// The lease query answers with the lease slice of , re-read on every call. + public void QueriesLease(CosmosFlowStateDocument document) + => QueriesLease(() => new CosmosLeaseProjection + { + Id = document.Id, + ETag = "etag", + ExpiresAtUtc = document.ExpiresAtUtc, + Revision = document.Revision, + LeaseId = document.LeaseId, + LeaseExpiresAtUtc = document.LeaseExpiresAtUtc + }); + + /// The lease query answers with no rows: the flow does not exist. + public void QueriesNothing() => QueriesLease(() => null); + + public void QueriesLease(Func projection) + => Container + .Setup(item => item.GetItemQueryIterator( + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns(() => LeaseIterator(projection())); + + public void QueriesThrowing(CosmosException exception) + => Container + .Setup(item => item.GetItemQueryIterator( + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns(() => + { + var iterator = new Mock>(); + iterator.SetupGet(item => item.HasMoreResults).Returns(true); + iterator.Setup(item => item.ReadNextAsync(It.IsAny())).ThrowsAsync(exception); + return iterator.Object; + }); + + private static FeedIterator LeaseIterator(CosmosLeaseProjection? projection) + { + var page = new Mock>(); + var rows = projection is null ? Array.Empty() : [projection]; + page.Setup(item => item.GetEnumerator()).Returns(() => ((IEnumerable)rows).GetEnumerator()); + page.SetupGet(item => item.Resource).Returns(rows); + + var iterator = new Mock>(); + var more = true; + iterator.SetupGet(item => item.HasMoreResults).Returns(() => more); + iterator + .Setup(item => item.ReadNextAsync(It.IsAny())) + .ReturnsAsync(() => + { + more = false; + return page.Object; + }); + return iterator.Object; + } + + /// Every lease patch succeeds; sees the operations and request options. + public void PatchesSuccessfully(Action, PatchItemRequestOptions>? onPatch = null) + => Container + .Setup(item => item.PatchItemAsync( + It.IsAny(), + It.IsAny(), + It.IsAny>(), + It.IsAny(), + It.IsAny())) + .Callback, PatchItemRequestOptions, CancellationToken>( + (_, _, operations, requestOptions, _) => onPatch?.Invoke(operations, requestOptions)) + .ReturnsAsync(Mock.Of>()); + + public void PatchesThrowing(CosmosException exception) + => Container + .Setup(item => item.PatchItemAsync( + It.IsAny(), + It.IsAny(), + It.IsAny>(), + It.IsAny(), + It.IsAny())) + .ThrowsAsync(exception); + public void Dispose() => Store.Dispose(); } + + /// The value a patch operation carries, read through the SDK's public surface (the value itself is not exposed). + private static object? PatchValue(PatchOperation operation) + { + var property = operation.GetType().GetProperty("Value", BindingFlags.Instance | BindingFlags.Public); + return property?.GetValue(operation); + } + + private static PatchOperation PatchFor(IReadOnlyList operations, string path) + => Assert.Single(operations, operation => operation.Path == path); } diff --git a/tests/AsyncResponse.Tests/DbChannelSharedCoverageTests.cs b/tests/AsyncResponse.Tests/DbChannelSharedCoverageTests.cs index 812800a10..8723b0eff 100644 --- a/tests/AsyncResponse.Tests/DbChannelSharedCoverageTests.cs +++ b/tests/AsyncResponse.Tests/DbChannelSharedCoverageTests.cs @@ -6,6 +6,7 @@ using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; using MongoDB.Bson; +using MongoDB.Bson.Serialization; using MongoDB.Driver; using Moq; using Npgsql; @@ -620,6 +621,138 @@ await Ignoring(() => ((IAsyncResponsePublisher)harness.Channel) activities.Single(name, "asyncresponse.channel", tag); } + /// + /// Round 35 (P1): the process-wide dispatch sweep visited correlation ids sequentially and + /// AWAITED each id's serial-executor capacity. One waiter wedged in a slow completion predicate + /// (its executor's single reader blocked on the first message) plus a backlog of NEW progress + /// messages for that id filled the 1024-slot queue, and the sweep then parked on slot 1025 + /// without ever querying the next correlation id — every other waiter in the process stopped + /// receiving. The sweep now admits work without waiting: at capacity it leaves the rest of that + /// id's messages unclaimed in the store, schedules a rescan of that id alone, and moves on. + /// Mongo harness only: its store is the real MongoDbChannelStore over a mocked + /// collection, so the backlog can be arranged (the relational harnesses' closed-port stores + /// cannot answer a query); the sweep itself is shared source, identical in all three + /// assemblies. The targeted scope is a HashSet whose enumeration follows insertion order + /// for a small, removal-free set, so the wedged id is visited first — the shape that hung. + /// Pre-fix failure: the live waiter's delivery never arrives (the sweep is parked), and the + /// sweep task never completes. + /// + [Fact] + public async Task DispatchSweep_ASaturatedCorrelationExecutor_DoesNotBlockDeliveryToOtherCorrelations() + { + await using var harness = Harness.Create(Provider.MongoDb, failing: false, pollInterval: TimeSpan.FromSeconds(30), pendingMessageBatchSize: 4096); + var startedAt = DateTimeOffset.UtcNow; + + // The wedged waiter: its dispatch hook never completes, so the first message parks its + // executor's reader and every later message for the id queues behind it. + var (blocked, _) = harness.Subscription("corr-blocked", startedAt); + var wedged = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + harness.SetProcessHook(blocked, () => wedged.Task); + harness.AddSubscription("corr-blocked", blocked); + + // The unrelated waiter whose delivery must not wait behind it. + var (live, _) = harness.Subscription("corr-live", startedAt); + var delivered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + harness.SetProcessHook(live, () => + { + delivered.TrySetResult(); + return Task.CompletedTask; + }); + harness.AddSubscription("corr-live", live); + + // The store: 1100 distinct pending progress messages for the wedged id (more than the + // executor's capacity), one for the live id — all unacked and inside both watermarks. + MongoChannelMessageDocument Pending(string correlationId, int i) => new() + { + Id = Guid.NewGuid(), + CorrelationId = correlationId, + EnvelopeJson = StaleEnvelope, + CreatedAtUtc = startedAt.AddMilliseconds(i).UtcDateTime, + ExpiresAtUtc = startedAt.AddMinutes(5).UtcDateTime + }; + var backlog = Enumerable.Range(0, ChannelSerialExecutor.DefaultCapacity + 76).Select(i => Pending("corr-blocked", i)).ToList(); + var single = new List { Pending("corr-live", 0) }; + harness.MongoMessages! + .Setup(collection => collection.FindAsync( + It.IsAny>(), + It.IsAny>(), + It.IsAny())) + .Returns((FilterDefinition filter, FindOptions _, CancellationToken _) => + Task.FromResult(Cursor(CorrelationIdOf(filter) == "corr-blocked" ? backlog : single))); + // Every delivery claim wins (the claim gates only on recovery_claimed). + harness.MongoMessages + .Setup(collection => collection.FindOneAndUpdateAsync( + It.IsAny>(), + It.IsAny>(), + It.IsAny>(), + It.IsAny())) + .ReturnsAsync(Pending("claimed", 0)); + + using var sweepCancellation = new CancellationTokenSource(); + var sweep = harness.InvokeAsync( + "DispatchPendingMessagesAsync", + new HashSet(StringComparer.Ordinal) { "corr-blocked", "corr-live" }, + sweepCancellation.Token); + + try + { + await delivered.Task.WaitAsync(TimeSpan.FromSeconds(10)); + await sweep.WaitAsync(TimeSpan.FromSeconds(10)); + } + finally + { + // On the pre-fix build the sweep is parked on the wedged id's 1025th enqueue; unpark + // it so the harness can dispose. + sweepCancellation.Cancel(); + wedged.TrySetResult(); + } + } + + /// A one-batch cursor over , for the mocked collection's FindAsync. + private static IAsyncCursor Cursor(IReadOnlyList items) + { + var cursor = new Mock>(); + var moved = false; + cursor.Setup(c => c.MoveNextAsync(It.IsAny())).ReturnsAsync(() => !moved && (moved = true)); + cursor.Setup(c => c.MoveNext(It.IsAny())).Returns(() => !moved && (moved = true)); + cursor.SetupGet(c => c.Current).Returns(items); + return cursor.Object; + } + + /// The correlation_id a rendered message filter asks for, or null. + private static string? CorrelationIdOf(FilterDefinition filter) + { + var rendered = filter.Render(new RenderArgs( + BsonSerializer.LookupSerializer(), + BsonSerializer.SerializerRegistry)); + return FindCorrelationId(rendered); + + static string? FindCorrelationId(BsonValue value) + { + switch (value) + { + case BsonDocument document: + if (document.TryGetValue("correlation_id", out var id) && id.IsString) + return id.AsString; + foreach (var element in document) + { + if (FindCorrelationId(element.Value) is { } nested) + return nested; + } + return null; + case BsonArray array: + foreach (var item in array) + { + if (FindCorrelationId(item) is { } nested) + return nested; + } + return null; + default: + return null; + } + } + } + /// /// Regression (round 33): the same-process fast path built its dispatch message with a /// fabricated AckedAtUtc = null, so a publish RETRY — the same message id landing as an @@ -720,6 +853,9 @@ public enum Provider /// base. "Failing" points the relational providers at a closed port and arms the Mongo mocks to /// throw, so every store call faults deterministically without a container. /// + /// Target of : bound to , ignores the message. + private static Task InvokeProcessHook(Func body, TMessage _) => body(); + private sealed class Harness : IAsyncDisposable { private readonly Type _channelType; @@ -752,7 +888,7 @@ private Harness( /// Mongo harness only: the messages-collection mock, for arranging what the store's upsert and claim return. public Mock>? MongoMessages { get; private set; } - public static Harness Create(Provider provider, bool failing, TimeSpan pollInterval, TimeSpan? fullSweepInterval = null, bool useChangeStreams = false) + public static Harness Create(Provider provider, bool failing, TimeSpan pollInterval, TimeSpan? fullSweepInterval = null, bool useChangeStreams = false, int? pendingMessageBatchSize = null) { var logger = new CollectingLogger(); var recoveryState = new Mock(); @@ -852,6 +988,7 @@ public static Harness Create(Provider provider, bool failing, TimeSpan pollInter UseChangeStreams = useChangeStreams, ListenerPollInterval = pollInterval, FullSweepInterval = fullSweepInterval, + PendingMessageBatchSize = pendingMessageBatchSize ?? 64, SubscriberHeartbeatInterval = heartbeat, SubscriberHeartbeatTimeout = TimeSpan.FromSeconds(5), DeliveryConfirmationTimeout = TimeSpan.FromMilliseconds(2), @@ -981,6 +1118,22 @@ public Array SubscriptionArray(object subscription) public void AddSubscription(string correlationId, object subscription) => Method("AddSubscription").Invoke(Channel, [correlationId, subscription]); + /// + /// Replaces the subscription's per-message dispatch delegate (ProcessUnderContextAsync, + /// a Func<DbChannelMessage, Task> over the provider assembly's message type) with + /// , so a test can wedge or observe delivery without an envelope. + /// + public void SetProcessHook(object subscription, Func body) + { + var property = subscription.GetType().GetProperty("ProcessUnderContextAsync", BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic)!; + var messageType = property.PropertyType.GetGenericArguments()[0]; + var hook = typeof(DbChannelSharedCoverageTests) + .GetMethod(nameof(InvokeProcessHook), BindingFlags.Static | BindingFlags.NonPublic)! + .MakeGenericMethod(messageType) + .CreateDelegate(property.PropertyType, body); + property.SetValue(subscription, hook); + } + /// The per-correlation serial executor the disposal drain has to get through. public SerialExecutorRegistry Executors => (SerialExecutorRegistry)Field("_executors").GetValue(Channel)!; diff --git a/tests/AsyncResponse.Tests/DbTransportSharedCoverageTests.cs b/tests/AsyncResponse.Tests/DbTransportSharedCoverageTests.cs index 8a41d2412..4aaa1ab12 100644 --- a/tests/AsyncResponse.Tests/DbTransportSharedCoverageTests.cs +++ b/tests/AsyncResponse.Tests/DbTransportSharedCoverageTests.cs @@ -294,12 +294,20 @@ public async Task EarlyAck_DrainBudgetLapse_DeadLettersQueuedWorkInsteadOfRunnin [InlineData(Provider.MongoDb)] public async Task EarlyAck_DrainBudgetLapse_FinishesDeadLetteringQueuedWorkBeforeDisposeReturns(Provider provider) { - // Each burial takes 25 ms to commit; the counter moves only once it has. With a 3 s budget - // the reserved quarter (750 ms) comfortably covers the three burials; the old fire-and-forget - // dispose returned with none of them committed. - var calls = new Calls { DeadLetterDelay = TimeSpan.FromMilliseconds(25) }; + // Each burial takes 10 ms to commit; the counter moves only once it has. With a 6 s budget + // the reserved quarter (1.5 s) covers the three burials (30 ms of work) ~50x over; the old + // fire-and-forget dispose returned with none of them committed. + // + // The headroom is deliberately that wide. At 25 ms per burial inside a 3 s budget the + // reserve was 750 ms for 75 ms of work — 10x — and a starved Windows CI runner still + // stalled long enough to commit only two of the three (the assertion below is synchronous + // by design, so it cannot wait the stall out). Nothing here measures speed: the fact is + // that DisposeAsync does not return until the queued entries are buried, so buying the + // margin with a smaller unit of work and a larger reserve costs a few seconds and removes + // a wall-clock race against the runner. + var calls = new Calls { DeadLetterDelay = TimeSpan.FromMilliseconds(10) }; var backgroundFailures = 0; - var drain = TimeSpan.FromSeconds(3); + var drain = TimeSpan.FromSeconds(6); var (dispatcher, handle) = CreateEarlyAckDispatcher( provider, calls, diff --git a/tests/AsyncResponse.Tests/DurableFlowStateStoreExampleTests.cs b/tests/AsyncResponse.Tests/DurableFlowStateStoreExampleTests.cs index 850aa5fd5..54897753b 100644 --- a/tests/AsyncResponse.Tests/DurableFlowStateStoreExampleTests.cs +++ b/tests/AsyncResponse.Tests/DurableFlowStateStoreExampleTests.cs @@ -450,9 +450,13 @@ private static async Task RunFlowWithStoreAsync(Action(); var builder = services.AddAsyncResponse() + // Generous, not tuned: every wait in this method is a liveness check ("did the run get + // there at all"), never a performance assertion, and the whole flow finishes in + // milliseconds on an idle machine. Tight values only decide how a starved CI runner + // fails — a Windows agent stalled past the old 5 s run wait with the flow mid-flight. .WithInMemoryChannel(options => { - options.DefaultTimeout = TimeSpan.FromSeconds(10); + options.DefaultTimeout = TimeSpan.FromSeconds(60); options.RecoveryStateExpiry = TimeSpan.FromMinutes(5); }) .WithInMemoryTransport(); @@ -466,11 +470,11 @@ private static async Task RunFlowWithStoreAsync(Action(new TestFlowInput(7)); var run = executor.ExecuteAsync(flowId); - var correlationId = await probe.TriggerFired.Task.WaitAsync(TimeSpan.FromSeconds(5)); + var correlationId = await probe.TriggerFired.Task.WaitAsync(TimeSpan.FromSeconds(30)); await publisher.SetResponse(new OperationResult { Status = OperationStatus.Running, Message = "halfway" }, correlationId); await publisher.SetResponse(new OperationResult { Status = OperationStatus.Completed }, correlationId); - await run.WaitAsync(TimeSpan.FromSeconds(5)); + await run.WaitAsync(TimeSpan.FromSeconds(30)); var state = await flows.GetStateAsync(flowId); Assert.Equal(FlowRunStatus.Succeeded, state!.Status); diff --git a/tests/AsyncResponse.Tests/DurableFlowStoreReviewFixesTests.cs b/tests/AsyncResponse.Tests/DurableFlowStoreReviewFixesTests.cs index 9fdcf65d3..51f685a3d 100644 --- a/tests/AsyncResponse.Tests/DurableFlowStoreReviewFixesTests.cs +++ b/tests/AsyncResponse.Tests/DurableFlowStoreReviewFixesTests.cs @@ -293,11 +293,12 @@ public void MongoDbStore_StateUpdate_StampsServerClockAndOptionallyResetsLease() } // --------------------------------------------------------------------------------------- - // P3-2: identity-mismatched ledgers load as absent + // P3-2: identity-mismatched ledgers are refused — as UNREADABLE since round 38, never as + // absent: the row is physically present, and "absent" acknowledged its wake-up. // --------------------------------------------------------------------------------------- [Fact] - public async Task SqliteStore_LoadsIdentityMismatchedRowAsAbsent() + public async Task SqliteStore_RefusesAnIdentityMismatchedRowAsUnreadable() { await using var database = new TempSqliteDatabase(); var store = new SqliteFlowStateStore(Options.Create(new SqliteDurableFlowOptions @@ -307,7 +308,7 @@ public async Task SqliteStore_LoadsIdentityMismatchedRowAsAbsent() Assert.True(await store.TryCreateAsync("provision", CreateState("provision"), TimeSpan.FromMinutes(5))); // A row copied/restored under the wrong key: state_json says "other-flow" but the row key - // is "hijacked-flow". Docs promise identity-mismatched records load as absent. + // is "hijacked-flow". Docs promise identity-mismatched records are refused as unreadable. await using (var connection = new SqliteConnection(database.ConnectionString)) { await connection.OpenAsync(); @@ -323,11 +324,13 @@ public async Task SqliteStore_LoadsIdentityMismatchedRowAsAbsent() await command.ExecuteNonQueryAsync(); } - Assert.Null(await store.LoadAsync("hijacked-flow")); + var ex = await Assert.ThrowsAsync(() => store.LoadAsync("hijacked-flow")); + Assert.Equal("hijacked-flow", ex.FlowId); + Assert.Contains("not the id it is stored under", ex.Reason, StringComparison.Ordinal); } [Fact] - public async Task DynamoDbStore_LoadsIdentityMismatchedItemAsAbsent() + public async Task DynamoDbStore_RefusesAnIdentityMismatchedItemAsUnreadable() { var client = new Mock(); client @@ -351,7 +354,9 @@ public async Task DynamoDbStore_LoadsIdentityMismatchedItemAsAbsent() EnableTimeToLive = false })); - Assert.Null(await store.LoadAsync("hijacked-flow")); + var ex = await Assert.ThrowsAsync(() => store.LoadAsync("hijacked-flow")); + Assert.Equal("hijacked-flow", ex.FlowId); + Assert.Contains("not the id it is stored under", ex.Reason, StringComparison.Ordinal); } // --------------------------------------------------------------------------------------- diff --git a/tests/AsyncResponse.Tests/DurableFlowStoreSharedTests.cs b/tests/AsyncResponse.Tests/DurableFlowStoreSharedTests.cs index c43a2e688..9c49f18ad 100644 --- a/tests/AsyncResponse.Tests/DurableFlowStoreSharedTests.cs +++ b/tests/AsyncResponse.Tests/DurableFlowStoreSharedTests.cs @@ -117,14 +117,20 @@ public void SharedHelpers_BoundSizeReadStateAndSaturateClocks(Type providerOptio Assert.Contains("exceeding the provider MaxStateBytes limit of 1 bytes", tooLarge.Message); // ReadState: only a readable ledger whose revision AND identity match loads as present. - // The two mismatches still read as absent — a row under the wrong key or at the wrong - // revision is a benign race or a misplaced restore, and treating it as this flow would be - // worse. An UNREADABLE row is the case that changed: it throws, because reporting it as - // absent told the executor to ack a live run's only wake-up. + // Everything else THROWS — never null. Unreadable JSON did already (reporting it as absent + // told the executor to ack a live run's only wake-up); round 38 extended that to the two + // mismatches: the JSON and the revision come from ONE row, so a disagreement inside that + // snapshot is an inconsistent row that is physically present, and a row under the wrong + // key is a misplaced restore — neither is proof the run is gone, and "absent" acked the + // wake-up of a run whose row sat in the table. Assert.Equal("flow", Assert.IsType(Invoke(shared, "ReadState", "flow", json, 0L)).FlowId); AssertInner(shared, "ReadState", "flow", "{", 0L); // unreadable - Assert.Null(Invoke(shared, "ReadState", "flow", json, 7L)); // revision mismatch - Assert.Null(Invoke(shared, "ReadState", "other", json, 0L)); // identity mismatch + var revisionMismatch = AssertInner(shared, "ReadState", "flow", json, 7L); + Assert.Contains("stored revision is 7", revisionMismatch.Reason, StringComparison.Ordinal); + Assert.Contains("inside its JSON is 0", revisionMismatch.Reason, StringComparison.Ordinal); + var identityMismatch = AssertInner(shared, "ReadState", "other", json, 0L); + Assert.Equal("other", identityMismatch.FlowId); + Assert.Contains("not the id it is stored under", identityMismatch.Reason, StringComparison.Ordinal); // Saturating adds: an absurd expiry means "never" rather than an overflow on every write. var instant = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc); @@ -174,14 +180,16 @@ public async Task PruneQuietly_ContainsPruneFailures_ButPropagatesCancellation(T var attempted = 0; await Quietly(() => { attempted++; throw new InvalidOperationException("deadlock victim"); }); - await Quietly(() => { attempted++; return Task.FromException(new TimeoutException("lock wait timeout")); }); + await Quietly(() => { attempted++; return Task.FromException(new TimeoutException("lock wait timeout")); }); Assert.Equal(2, attempted); await Assert.ThrowsAsync(() => Quietly(() => throw new OperationCanceledException())); await Assert.ThrowsAnyAsync( - () => Quietly(() => Task.FromCanceled(new CancellationToken(canceled: true)))); + () => Quietly(() => Task.FromCanceled(new CancellationToken(canceled: true)))); - Task Quietly(Func prune) => (Task)pruneQuietly!.Invoke(null, [prune])!; + // Round 34: the helper drains batches under a budget and reports; the single-batch shape + // is the zero budget. See Round34NewApiTests for the batching and reporting pins. + Task Quietly(Func> prune) => (Task)pruneQuietly!.Invoke(null, [prune, TimeSpan.Zero, "test", null])!; } public static TheoryData ProviderOptionTypes => diff --git a/tests/AsyncResponse.Tests/DynamoDbDurableFlowStateStoreTests.cs b/tests/AsyncResponse.Tests/DynamoDbDurableFlowStateStoreTests.cs index c1a5c57a3..5df8e9f56 100644 --- a/tests/AsyncResponse.Tests/DynamoDbDurableFlowStateStoreTests.cs +++ b/tests/AsyncResponse.Tests/DynamoDbDurableFlowStateStoreTests.cs @@ -177,8 +177,9 @@ public async Task Load_RejectsMalformedItemsAndReturnsMatchingRevision() // The item EXISTS in every case below, so "cannot interpret it" and "it is gone" must not // give the same answer: callers ack on null, and acking a ledger that is still in the table - // strands a Running flow with no wake-up left. Only real expiry and the deliberate - // revision-mismatch race read as absent. + // strands a Running flow with no wake-up left. Only real expiry reads as absent — a + // revision inside the JSON that disagrees with the item's own is an inconsistent item, + // not a deleted flow (round 38). await AssertUnreadableAsync("its 'expires_at' attribute is missing"); // no expires_at await AssertUnreadableAsync("not an epoch-seconds number"); // expires_at = "not-a-number" Assert.Null(await store.LoadAsync("flow")); // expired: genuinely gone @@ -186,7 +187,7 @@ public async Task Load_RejectsMalformedItemsAndReturnsMatchingRevision() await AssertUnreadableAsync("'state_json' attribute is missing or empty"); // state_json = "" await AssertUnreadableAsync("'revision' attribute is missing"); // no revision await AssertUnreadableAsync("not a number"); // revision = "bad" - Assert.Null(await store.LoadAsync("flow")); // revision mismatch: benign race + await AssertUnreadableAsync("stored revision is 1"); // revision mismatch: inconsistent item Assert.Equal("flow", (await store.LoadAsync("flow"))?.FlowId); diff --git a/tests/AsyncResponse.Tests/InMemoryAsyncResponseTests.cs b/tests/AsyncResponse.Tests/InMemoryAsyncResponseTests.cs index 742474d53..f3951f338 100644 --- a/tests/AsyncResponse.Tests/InMemoryAsyncResponseTests.cs +++ b/tests/AsyncResponse.Tests/InMemoryAsyncResponseTests.cs @@ -363,7 +363,10 @@ public async Task RawObjectResponse_WhenMaterializationFails_FaultsWaiterAndClea await rawPublisher.SetRawResponse("not-json", correlationId); - await Assert.ThrowsAsync(() => waiter.ResponseTask.WaitAsync(TimeSpan.FromSeconds(2))); + // Body-free since round 34: materialization goes through the JSON safety helper, so the + // waiter's fault names size and position, never the body (a JsonException quoted it). + var thrown = await Assert.ThrowsAsync(() => waiter.ResponseTask.WaitAsync(TimeSpan.FromSeconds(2))); + Assert.DoesNotContain("not-json", thrown.Message, StringComparison.Ordinal); Assert.Equal(0, await probe.CountActiveSubscribersAsync(correlationId)); } diff --git a/tests/AsyncResponse.Tests/KafkaDispatcherTests.cs b/tests/AsyncResponse.Tests/KafkaDispatcherTests.cs index 32fde2609..b0aa7ab83 100644 --- a/tests/AsyncResponse.Tests/KafkaDispatcherTests.cs +++ b/tests/AsyncResponse.Tests/KafkaDispatcherTests.cs @@ -1,4 +1,5 @@ using AsyncResponse.Transports.Kafka; +using System.Diagnostics; using Microsoft.Extensions.Logging.Abstractions; using Xunit; @@ -92,56 +93,83 @@ public void ValidateOptions_RejectsMaxPollIntervalAboveTheLibrdkafkaRange() } [Fact] - public void ValidateOptions_RejectsRetryDelayBudgetThatCannotFitTheMaxPollInterval() + public void ValidateOptions_RejectsDetachHandlerAfterThatCannotFitTheMaxPollInterval() { - // The in-process retry delays run on the poll thread: 4 completed attempts back off - // 20+40+80+160 = 300s of pure delay, which cannot fit within half of a 5-minute - // max.poll.interval.ms — the broker would evict the consumer mid-retry. - var subscriberOptions = new KafkaSubscriberOptions - { - MaxDeliveryAttempts = 5, - HandlerRetryBaseDelay = TimeSpan.FromSeconds(20), - HandlerRetryMaxDelay = TimeSpan.FromSeconds(160) - }; - + // Round 37: the poll thread's longest gap is one inline handler wait plus one poll. A + // 3-minute inline budget plus a 200 ms poll cannot fit within half of a 5-minute + // max.poll.interval.ms — the broker would evict the consumer while it waited inline. var ex = Assert.Throws(() => KafkaMessageDispatcher.ValidateOptions( KafkaTestData.NewOptions(), - subscriberOptions, + new KafkaSubscriberOptions { DetachHandlerAfter = TimeSpan.FromMinutes(3) }, KafkaSubscriberRole.Worker)); + Assert.Contains(nameof(KafkaSubscriberOptions.DetachHandlerAfter), ex.Message, StringComparison.Ordinal); Assert.Contains(nameof(KafkaSubscriberOptions.MaxPollInterval), ex.Message, StringComparison.Ordinal); Assert.Contains("evicted", ex.Message, StringComparison.Ordinal); } [Fact] - public void ValidateOptions_AcceptsRetryBudgetOnceMaxPollIntervalIsRaised() + public void ValidateOptions_AcceptsDetachHandlerAfterOnceMaxPollIntervalIsRaised() { - // The same budget passes when the operator raises the poll interval to hold it. + // The same inline budget passes when the operator raises the poll interval to hold it. + KafkaMessageDispatcher.ValidateOptions( + KafkaTestData.NewOptions(), + new KafkaSubscriberOptions + { + DetachHandlerAfter = TimeSpan.FromMinutes(3), + MaxPollInterval = TimeSpan.FromMinutes(15) + }, + KafkaSubscriberRole.Worker); + } + + [Fact] + public void ValidateOptions_RetryDelaysNoLongerCountAgainstTheMaxPollInterval() + { + // The retry ladder runs inside the (detached) handler task, so a delay budget of 300 s + // against a 5-minute interval — rejected before round 37 — is accepted: it stalls only the + // message's partition, never the poll thread. KafkaMessageDispatcher.ValidateOptions( KafkaTestData.NewOptions(), new KafkaSubscriberOptions { MaxDeliveryAttempts = 5, HandlerRetryBaseDelay = TimeSpan.FromSeconds(20), - HandlerRetryMaxDelay = TimeSpan.FromSeconds(160), - MaxPollInterval = TimeSpan.FromMinutes(15) + HandlerRetryMaxDelay = TimeSpan.FromSeconds(160) }, KafkaSubscriberRole.Worker); } [Fact] - public void ValidateOptions_UnlimitedRetries_SkipTheRetryBudgetCheck() + public void ValidateOptions_DetachHandlerAfter_AllowsZero_RejectsNegative() + { + // Zero detaches every handler immediately; negative is meaningless. + KafkaMessageDispatcher.ValidateOptions( + KafkaTestData.NewOptions(), + new KafkaSubscriberOptions { DetachHandlerAfter = TimeSpan.Zero }, + KafkaSubscriberRole.Worker); + + var ex = Assert.Throws(() => + KafkaMessageDispatcher.ValidateOptions( + KafkaTestData.NewOptions(), + new KafkaSubscriberOptions { DetachHandlerAfter = TimeSpan.FromMilliseconds(-1) }, + KafkaSubscriberRole.Worker)); + Assert.Contains(nameof(KafkaSubscriberOptions.DetachHandlerAfter), ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void ValidateOptions_EarlyAck_DoesNotApplyTheInlineGapRule() { - // MaxDeliveryAttempts = 0 has no finite delay budget; the option's doc pins staying - // under the ceiling as the operator's responsibility. + // Under AckAfterEnqueue the handler never runs on the poll thread, so DetachHandlerAfter + // has no gap to bound; only its own range is validated. KafkaMessageDispatcher.ValidateOptions( KafkaTestData.NewOptions(), new KafkaSubscriberOptions { - MaxDeliveryAttempts = 0, - HandlerRetryBaseDelay = TimeSpan.FromMinutes(2), - HandlerRetryMaxDelay = TimeSpan.FromMinutes(10) + DetachHandlerAfter = TimeSpan.FromMinutes(3), + BackgroundWorkerCount = 1, + BackgroundQueueCapacity = 1, + AckMode = KafkaAckMode.AckAfterEnqueue }, KafkaSubscriberRole.Worker); } @@ -492,7 +520,7 @@ public async Task Awaiting_DeadLetterPublish_RetriesTransientFailures() } [Fact] - public async Task Awaiting_DeadLetterPublishFailsPermanently_SwallowsWithoutStoringOffset() + public async Task Awaiting_DeadLetterPublishFailsPermanently_FaultsTheDispatch_SoNoLaterSettlementCommitsPastIt() { var consumer = new FakeKafkaConsumerClient(); var producer = new FakeKafkaProducerClient { PublishException = new InvalidOperationException("broker gone") }; @@ -502,14 +530,20 @@ public async Task Awaiting_DeadLetterPublishFailsPermanently_SwallowsWithoutStor consumer: consumer, producer: producer); - // A failed burial must NOT escape: this call runs inside the poll loop, and a propagated - // throw faulted the whole subscriber with the offset unstored — the supervisor rebuilt - // the consumer, re-consumed the message, and re-executed the failing handler - // MaxDeliveryAttempts more times per restart, forever. - await dispatcher.HandleAsync(KafkaTestData.Delivery(Topic, offset: 1), CancellationToken.None); - - // The offset stays unstored so a restart or rebalance redelivers the message and the - // burial is retried. + // Round 35: round 31 swallowed this burial failure and merely left the offset unstored — + // which protects nothing on Kafka, because a later successful settlement on the same + // partition stores a HIGHER offset and the auto-committer commits past the failed message + // (see the subscriber-level pin in KafkaSubscriberTests). The failure now faults the + // dispatch so the poll loop stops at this message and the supervisor restarts the + // consumer after its backoff. Pre-fix failure: HandleAsync returned normally. + var ex = await Assert.ThrowsAsync( + () => dispatcher.HandleAsync(KafkaTestData.Delivery(Topic, offset: 1), CancellationToken.None)); + + Assert.Equal(Topic, ex.Topic); + Assert.Equal(1, ex.Offset); + Assert.IsType(ex.InnerException); + // The offset stays unstored so the restarted consumer re-consumes the message and retries + // the burial. Assert.Empty(consumer.StoredOffsets); } @@ -1035,12 +1069,13 @@ await dispatcher.DiscardUnprocessableAsync( } [Fact] - public async Task DiscardUnprocessable_WhenTheDeadLetterPublishFailsPermanently_DoesNotFaultThePollLoop() + public async Task DiscardUnprocessable_WhenTheDeadLetterPublishFailsPermanently_FaultsTheDispatch_SoNoLaterSettlementCommitsPastIt() { - // Regression (round 31): the burial itself was unguarded on this path (only the offset - // store was wrapped), and this call originates inside the poll loop's own catch arm — a - // permanently failing dead-letter topic burned the publish retries, rethrew, faulted the - // subscriber with the offset unstored, and the restart re-ran the same discard forever. + // Round 31 guarded this burial and swallowed its failure with the offset unstored; round + // 35 reverses the swallow (the malformed-message path reproduced the same commit-past + // loss as the handler-failure path). The typed fault is what the poll loop lets escape so + // the supervisor restarts the consumer with backoff and the partition stays parked at + // this message. Pre-fix failure: DiscardUnprocessableAsync returned normally. var consumer = new FakeKafkaConsumerClient(); var producer = new FakeKafkaProducerClient { PublishException = new InvalidOperationException("broker gone") }; await using var dispatcher = CreateDispatcher( @@ -1051,12 +1086,13 @@ public async Task DiscardUnprocessable_WhenTheDeadLetterPublishFailsPermanently_ var message = KafkaTestData.Message(Topic, offset: 4, payload: "", ("correlationId", "corr-x")); - await dispatcher.DiscardUnprocessableAsync( + var ex = await Assert.ThrowsAsync(() => dispatcher.DiscardUnprocessableAsync( message, new InvalidDataException("no payload"), - CancellationToken.None); + CancellationToken.None)); - // No burial and no commit: the offset stays unstored so a restart or rebalance retries + Assert.Equal(4, ex.Offset); + // No burial and no commit: the offset stays unstored so the restarted consumer retries // the burial instead of dropping the message with no record. Assert.Empty(consumer.StoredOffsets); } @@ -1129,8 +1165,9 @@ public async Task DiscardUnprocessable_BoundsTheDeadLetterProduceToAFractionOfTh // produce to an undeliverable topic waits out librdkafka's message.timeout.ms (5 min by // default) PER attempt — past max.poll.interval.ms, evicting the consumer mid-burial and // rebalancing the partition to a peer that hit the same message: a rebalance storm. The - // ladder is now bounded to a quarter of the poll interval and the caller keeps treating a - // failed burial as "offset left unstored, retried later". + // ladder is now bounded to a quarter of the poll interval; once it runs out the caller + // faults the dispatch (round 35) with the offset left unstored, so the supervisor restarts + // the consumer instead of a later settlement committing past the message. var producer = new HangingKafkaProducerClient(); await using var dispatcher = KafkaMessageDispatcher.Create( (_, _) => Task.CompletedTask, @@ -1141,6 +1178,7 @@ public async Task DiscardUnprocessable_BoundsTheDeadLetterProduceToAFractionOfTh { MaxDeliveryAttempts = 0, PollTimeout = TimeSpan.FromMilliseconds(10), + DetachHandlerAfter = TimeSpan.FromMilliseconds(50), MaxPollInterval = TimeSpan.FromMilliseconds(400) }, NullLogger.Instance, @@ -1148,10 +1186,10 @@ public async Task DiscardUnprocessable_BoundsTheDeadLetterProduceToAFractionOfTh Group, KafkaSubscriberRole.Worker); - await dispatcher.DiscardUnprocessableAsync( + await Assert.ThrowsAsync(() => dispatcher.DiscardUnprocessableAsync( KafkaTestData.Message(Topic, offset: 4, payload: ""), new InvalidDataException("no payload"), - CancellationToken.None).WaitAsync(TimeSpan.FromSeconds(5)); + CancellationToken.None).WaitAsync(TimeSpan.FromSeconds(5))); Assert.True(producer.SawCancellation); } @@ -1212,6 +1250,259 @@ public void Dispose() } } + // ---------- Round 37: ack-after-handler detachment (the poll-thread API) ---------- + + [Fact] + public async Task Awaiting_Accept_SettlesInlineWithinTheBudget_WithoutPausing() + { + var consumer = new FakeKafkaConsumerClient(); + await using var dispatcher = CreateDispatcher( + (_, _) => Task.CompletedTask, + new KafkaSubscriberOptions { DetachHandlerAfter = TimeSpan.FromSeconds(5) }, + consumer: consumer); + + dispatcher.Accept(KafkaTestData.Delivery(Topic, offset: 3), CancellationToken.None); + + Assert.Equal(new FakeKafkaConsumerClient.StoredOffset(Topic, 0, 3), Assert.Single(consumer.StoredOffsets)); + Assert.False(dispatcher.HasDetachedWork); + Assert.Empty(consumer.PartitionPauses); + } + + [Fact] + public async Task Awaiting_Accept_DetachesAHandlerPastTheBudget_PausesItsPartition_AndSettlesOnATick() + { + var consumer = new FakeKafkaConsumerClient(); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + await using var dispatcher = CreateDispatcher( + async (_, _) => await release.Task, + new KafkaSubscriberOptions { DetachHandlerAfter = TimeSpan.FromMilliseconds(20) }, + consumer: consumer); + + var accepted = Stopwatch.StartNew(); + dispatcher.Accept(KafkaTestData.Delivery(Topic, offset: 9, partition: 4), CancellationToken.None); + accepted.Stop(); + + // Back on the poll thread within the budget (generous bound for a slow runner), the + // partition paused, nothing stored: the handler is still running. + Assert.True(accepted.Elapsed < TimeSpan.FromSeconds(2), $"Accept blocked for {accepted.Elapsed}."); + Assert.True(dispatcher.HasDetachedWork); + Assert.Equal(4, Assert.Single(consumer.PartitionPauses)); + Assert.True(consumer.IsPartitionPaused(4)); + Assert.Empty(consumer.StoredOffsets); + + // A tick with the handler still running settles nothing. + dispatcher.SettleCompleted(); + Assert.Empty(consumer.StoredOffsets); + Assert.True(dispatcher.HasDetachedWork); + + release.SetResult(); + await KafkaTestData.WaitUntilAsync(() => + { + dispatcher.SettleCompleted(); + return !dispatcher.HasDetachedWork; + }); + + Assert.Equal(new FakeKafkaConsumerClient.StoredOffset(Topic, 4, 9), Assert.Single(consumer.StoredOffsets)); + Assert.Equal(4, Assert.Single(consumer.PartitionResumes)); + Assert.False(consumer.IsPartitionPaused(4)); + } + + [Fact] + public async Task Awaiting_Accept_ZeroBudget_DetachesImmediately() + { + var consumer = new FakeKafkaConsumerClient(); + var started = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + await using var dispatcher = CreateDispatcher( + async (_, _) => + { + started.TrySetResult(); + await release.Task; + }, + new KafkaSubscriberOptions { DetachHandlerAfter = TimeSpan.Zero }, + consumer: consumer); + + dispatcher.Accept(KafkaTestData.Delivery(Topic, offset: 1), CancellationToken.None); + + Assert.True(dispatcher.HasDetachedWork); + await started.Task.WaitAsync(TimeSpan.FromSeconds(5)); + release.SetResult(); + await KafkaTestData.WaitUntilAsync(() => + { + dispatcher.SettleCompleted(); + return !dispatcher.HasDetachedWork; + }); + Assert.Single(consumer.StoredOffsets); + } + + [Fact] + public async Task Awaiting_SettleCompleted_RethrowsADetachedBurialFailure_WithoutStoringTheOffset() + { + // The detached path keeps the round-35 contract: a message that exhausted its attempts and + // could not be dead-lettered faults the poll loop (through the tick) with its offset + // unstored, so no later settlement on the partition commits past it. + var consumer = new FakeKafkaConsumerClient(); + var producer = new FakeKafkaProducerClient { PublishException = new InvalidOperationException("dead-letter topic gone") }; + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + await using var dispatcher = CreateDispatcher( + async (_, _) => + { + await release.Task; + throw new InvalidOperationException("handler boom"); + }, + new KafkaSubscriberOptions + { + DetachHandlerAfter = TimeSpan.FromMilliseconds(20), + MaxDeliveryAttempts = 1 + }, + consumer: consumer, + producer: producer); + + dispatcher.Accept(KafkaTestData.Delivery(Topic, offset: 7), CancellationToken.None); + Assert.True(dispatcher.HasDetachedWork); + + release.SetResult(); + Exception? faulted = null; + await KafkaTestData.WaitUntilAsync(() => + { + try + { + dispatcher.SettleCompleted(); + return false; + } + catch (Exception ex) + { + faulted = ex; + return true; + } + }); + + Assert.IsType(faulted); + Assert.Empty(consumer.StoredOffsets); + Assert.False(dispatcher.HasDetachedWork); + } + + [Fact] + public async Task Awaiting_Dispose_WaitsForDetachedHandlers_AndStoresTheirOffsets() + { + // A stop lets a detached handler finish and stores its offset before the consumer's close + // commits, so a routine deploy does not redeliver work that completed. + var consumer = new FakeKafkaConsumerClient(); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var dispatcher = CreateDispatcher( + async (_, _) => await release.Task, + new KafkaSubscriberOptions { DetachHandlerAfter = TimeSpan.FromMilliseconds(20) }, + consumer: consumer); + + dispatcher.Accept(KafkaTestData.Delivery(Topic, offset: 11), CancellationToken.None); + Assert.True(dispatcher.HasDetachedWork); + + var disposal = dispatcher.DisposeAsync().AsTask(); + await Task.Delay(100); + Assert.False(disposal.IsCompleted); + Assert.Empty(consumer.StoredOffsets); + + release.SetResult(); + await disposal.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(new FakeKafkaConsumerClient.StoredOffset(Topic, 0, 11), Assert.Single(consumer.StoredOffsets)); + } + + [Fact] + public async Task Awaiting_Dispose_CanceledDetachedHandler_LeavesTheOffsetUnstored() + { + var consumer = new FakeKafkaConsumerClient(); + using var stopping = new CancellationTokenSource(); + var dispatcher = CreateDispatcher( + async (_, token) => await Task.Delay(Timeout.InfiniteTimeSpan, token), + new KafkaSubscriberOptions { DetachHandlerAfter = TimeSpan.FromMilliseconds(20) }, + consumer: consumer); + + dispatcher.Accept(KafkaTestData.Delivery(Topic, offset: 2), stopping.Token); + Assert.True(dispatcher.HasDetachedWork); + + stopping.Cancel(); + await dispatcher.DisposeAsync().AsTask().WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.Empty(consumer.StoredOffsets); + } + + [Fact] + public async Task Awaiting_AMessageArrivingForADetachedPartition_IsHeldAndRunsAfterIt_InOrder() + { + // A rebalance can hand a paused partition back with its pause reset; a message that + // arrives for a partition whose handler is detached is held behind it — order preserved — + // and the pause re-asserted so the hold never grows. + var consumer = new FakeKafkaConsumerClient(); + var order = new List(); + var releaseFirst = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + await using var dispatcher = CreateDispatcher( + async (delivery, _) => + { + lock (order) + { + order.Add(delivery.Offset); + } + + if (delivery.Offset == 1) + await releaseFirst.Task; + }, + new KafkaSubscriberOptions { DetachHandlerAfter = TimeSpan.FromMilliseconds(20) }, + consumer: consumer); + + dispatcher.Accept(KafkaTestData.Delivery(Topic, offset: 1), CancellationToken.None); + dispatcher.Accept(KafkaTestData.Delivery(Topic, offset: 2), CancellationToken.None); + + // Held, not started: the second handler must not run while the first is in flight. + await Task.Delay(100); + lock (order) + { + Assert.Equal([1], order); + } + + Assert.Equal(2, consumer.PartitionPauses.Count); // re-asserted on the held message + Assert.Empty(consumer.StoredOffsets); + + releaseFirst.SetResult(); + await KafkaTestData.WaitUntilAsync(() => + { + dispatcher.SettleCompleted(); + return consumer.StoredOffsets.Count == 2; + }); + + lock (order) + { + Assert.Equal([1, 2], order); + } + + Assert.Equal([1L, 2L], consumer.StoredOffsets.Select(stored => stored.Offset)); + Assert.Single(consumer.PartitionResumes); // resumed only once the hold was drained + } + + [Fact] + public async Task Awaiting_PauseFailureOfARevokedPartition_DoesNotFaultTheDetach() + { + var consumer = new FakeKafkaConsumerClient + { + PartitionPauseException = new Confluent.Kafka.KafkaException(new Confluent.Kafka.Error(Confluent.Kafka.ErrorCode.Local_UnknownPartition)) + }; + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + await using var dispatcher = CreateDispatcher( + async (_, _) => await release.Task, + new KafkaSubscriberOptions { DetachHandlerAfter = TimeSpan.FromMilliseconds(20) }, + consumer: consumer); + + dispatcher.Accept(KafkaTestData.Delivery(Topic, offset: 5), CancellationToken.None); + Assert.True(dispatcher.HasDetachedWork); + + release.SetResult(); + await KafkaTestData.WaitUntilAsync(() => + { + dispatcher.SettleCompleted(); + return !dispatcher.HasDetachedWork; + }); + Assert.Single(consumer.StoredOffsets); + } + private static KafkaMessageDispatcher CreateDispatcher( Func handler, KafkaSubscriberOptions subscriberOptions, diff --git a/tests/AsyncResponse.Tests/KafkaSubscriberTests.cs b/tests/AsyncResponse.Tests/KafkaSubscriberTests.cs index 9633017f0..1780b8d9e 100644 --- a/tests/AsyncResponse.Tests/KafkaSubscriberTests.cs +++ b/tests/AsyncResponse.Tests/KafkaSubscriberTests.cs @@ -340,6 +340,313 @@ public async Task Subscriber_InvalidEarlyAckOptions_FailFastOnStart() Assert.Contains(nameof(KafkaSubscriberOptions.BackgroundWorkerCount), ex.Message, StringComparison.Ordinal); } + /// + /// Round 35: the reviewer's two-message scenario. Offset 10 exhausts its handling and its + /// dead-letter publish fails; offset 11 then succeeds. Pre-fix, the burial failure was swallowed + /// with offset 10 left unstored, offset 11's settlement stored the partition position past it, + /// and the auto-committer committed that — a restart skipped offset 10 with no dead-letter copy + /// anywhere. The subscriber must now fault at offset 10 (never storing past it) and be rebuilt by + /// the supervisor, which re-consumes from the committed position. + /// + [Fact] + public async Task WorkerSubscriber_WhenAFailedMessageCannotBeDeadLettered_NeverCommitsPastIt_AndRestarts() + { + var first = new FakeKafkaConsumerClient(); + first.Enqueue(KafkaTestData.Message("workers", offset: 10, payload: "poison", ("correlationId", "corr-poison"))); + first.Enqueue(KafkaTestData.Message("workers", offset: 11, payload: "fine", ("correlationId", "corr-fine"))); + var second = new FakeKafkaConsumerClient(); + var factory = new FakeKafkaConsumerClientFactory(first, second); + + var ingress = new Mock(); + ingress.Setup(i => i.HandleWorkerMessageAsync("poison")).ThrowsAsync(new InvalidOperationException("handler boom")); + ingress.Setup(i => i.HandleWorkerMessageAsync("fine")).Returns(Task.CompletedTask); + + var options = NewOptions(o => + { + o.WorkerTopic = "workers"; + o.WorkerConsumerGroup = "workers-group"; + o.WorkerSubscriber.MaxDeliveryAttempts = 1; + o.WorkerSubscriber.HandlerRetryBaseDelay = TimeSpan.FromMilliseconds(1); + o.WorkerSubscriber.HandlerRetryMaxDelay = TimeSpan.FromMilliseconds(2); + o.PublishRetryBaseDelay = TimeSpan.FromMilliseconds(1); + o.PublishRetryMaxDelay = TimeSpan.FromMilliseconds(2); + }); + var subscriber = new KafkaWorkerSubscriber( + Options.Create(options), + factory, + new FakeKafkaProducerClient { PublishException = new InvalidOperationException("dead-letter topic gone") }, + new FakeKafkaAdminClient(), + ingress.Object, + NullLogger.Instance); + + await subscriber.StartAsync(CancellationToken.None); + try + { + // The supervisor rebuilt the consumer: the first one faulted at offset 10. + await KafkaTestData.WaitUntilAsync(() => factory.CreatedRoles.Count >= 2, TimeSpan.FromSeconds(10)); + } + finally + { + await subscriber.StopAsync(CancellationToken.None); + } + + // Nothing was ever stored on the faulted consumer: offset 11 was never handled behind the + // unresolved offset 10, so its close committed nothing past the poison message. + Assert.Empty(first.StoredOffsets); + Assert.True(first.Closed); + ingress.Verify(i => i.HandleWorkerMessageAsync("fine"), Times.Never); + } + + // ---------- Round 37: a long handler no longer stalls the poll loop (F7) ---------- + + [Fact] + public async Task WorkerSubscriber_KeepsPollingWhileAHandlerRunsLong() + { + // Regression (round 37, F7): the poll thread awaited the whole handler, so a durable-flow + // step awaiting a remote response or a timer stopped every Consume() call for its duration + // — past max.poll.interval.ms the broker evicted the consumer, rebalanced its partitions, + // and redelivered the message to a peer that started the same work again. Past + // DetachHandlerAfter (the default second here) the handler is detached and the loop must + // keep calling Consume, which is what the broker counts as liveness. + var consumer = new FakeKafkaConsumerClient(); + consumer.Enqueue(KafkaTestData.Message("workers", offset: 1, payload: "slow-job")); + + var handlerStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseHandler = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var ingress = new Mock(); + ingress.Setup(i => i.HandleWorkerMessageAsync("slow-job")) + .Returns(async () => + { + handlerStarted.TrySetResult(); + await releaseHandler.Task.ConfigureAwait(false); + }); + + var subscriber = CreateWorkerSubscriber(consumer, ingress.Object, options => options.WorkerTopic = "workers"); + await subscriber.StartAsync(CancellationToken.None); + try + { + await handlerStarted.Task.WaitAsync(TimeSpan.FromSeconds(5)); + + // Past the inline budget: with the handler still parked, the poll count must keep + // climbing. The old loop was blocked inside the handler and froze it here. + await Task.Delay(TimeSpan.FromMilliseconds(1500)); + var pollsAfterDetach = consumer.ConsumeCalls; + await Task.Delay(TimeSpan.FromMilliseconds(500)); + Assert.True( + consumer.ConsumeCalls > pollsAfterDetach + 5, + $"Consume was called {consumer.ConsumeCalls - pollsAfterDetach} time(s) in 500 ms while the handler ran; the poll loop is stalled."); + Assert.Empty(consumer.StoredOffsets); // not settled yet + + releaseHandler.SetResult(); + await KafkaTestData.WaitUntilAsync(() => consumer.StoredOffsets.Count == 1); + Assert.Equal(new FakeKafkaConsumerClient.StoredOffset("workers", 0, 1), Assert.Single(consumer.StoredOffsets)); + } + finally + { + releaseHandler.TrySetResult(); + await subscriber.StopAsync(CancellationToken.None); + } + + ingress.Verify(i => i.HandleWorkerMessageAsync("slow-job"), Times.Once); + } + + [Fact] + public async Task WorkerSubscriber_OtherPartitionsKeepFlowing_WhileOneHandlerIsDetached() + { + // Same finding, the other consequence: with the poll thread parked in one partition's + // handler, every other partition assigned to the consumer stalled behind it. + var consumer = new FakeKafkaConsumerClient(); + consumer.Enqueue(KafkaTestData.MessageOn("workers", partition: 0, offset: 1, payload: "slow-job")); + consumer.Enqueue(KafkaTestData.MessageOn("workers", partition: 1, offset: 1, payload: "quick-job")); + + var slowStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseSlow = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var quickHandled = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var ingress = new Mock(); + ingress.Setup(i => i.HandleWorkerMessageAsync("slow-job")) + .Returns(async () => + { + slowStarted.TrySetResult(); + await releaseSlow.Task.ConfigureAwait(false); + }); + ingress.Setup(i => i.HandleWorkerMessageAsync("quick-job")) + .Returns(() => + { + quickHandled.TrySetResult(); + return Task.CompletedTask; + }); + + var subscriber = CreateWorkerSubscriber(consumer, ingress.Object, options => options.WorkerTopic = "workers"); + await subscriber.StartAsync(CancellationToken.None); + try + { + await slowStarted.Task.WaitAsync(TimeSpan.FromSeconds(5)); + + // Partition 1's message is handled while partition 0's handler is still running. + await quickHandled.Task.WaitAsync(TimeSpan.FromSeconds(5)); + await KafkaTestData.WaitUntilAsync(() => consumer.StoredOffsets.Count == 1); + Assert.Equal(new FakeKafkaConsumerClient.StoredOffset("workers", 1, 1), Assert.Single(consumer.StoredOffsets)); + Assert.False(releaseSlow.Task.IsCompleted); + + releaseSlow.SetResult(); + await KafkaTestData.WaitUntilAsync(() => consumer.StoredOffsets.Count == 2); + } + finally + { + releaseSlow.TrySetResult(); + await subscriber.StopAsync(CancellationToken.None); + } + } + + [Fact] + public async Task WorkerSubscriber_DetachedHandler_PausesItsPartition_AndResumesItOnceSettled() + { + var consumer = new FakeKafkaConsumerClient(); + consumer.Enqueue(KafkaTestData.MessageOn("workers", partition: 3, offset: 5, payload: "slow-job")); + consumer.Enqueue(KafkaTestData.MessageOn("workers", partition: 3, offset: 6, payload: "next-job")); + + var slowStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseSlow = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var handled = new List(); + var ingress = new Mock(); + ingress.Setup(i => i.HandleWorkerMessageAsync(It.IsAny())) + .Returns(async (string payload) => + { + lock (handled) + { + handled.Add(payload); + } + + if (payload == "slow-job") + { + slowStarted.TrySetResult(); + await releaseSlow.Task.ConfigureAwait(false); + } + }); + + var subscriber = CreateWorkerSubscriber(consumer, ingress.Object, options => + { + options.WorkerTopic = "workers"; + options.WorkerSubscriber.DetachHandlerAfter = TimeSpan.FromMilliseconds(20); + }); + await subscriber.StartAsync(CancellationToken.None); + try + { + await slowStarted.Task.WaitAsync(TimeSpan.FromSeconds(5)); + await KafkaTestData.WaitUntilAsync(() => consumer.IsPartitionPaused(3)); + + // Paused: the next message on the partition is NOT consumed behind the running one. + await Task.Delay(200); + lock (handled) + { + Assert.Equal(["slow-job"], handled); + } + + Assert.Empty(consumer.StoredOffsets); + + releaseSlow.SetResult(); + await KafkaTestData.WaitUntilAsync(() => consumer.StoredOffsets.Count == 2); + Assert.Equal([5L, 6L], consumer.StoredOffsets.Select(stored => stored.Offset)); + Assert.Equal(3, Assert.Single(consumer.PartitionPauses)); + Assert.Equal(3, Assert.Single(consumer.PartitionResumes)); + lock (handled) + { + Assert.Equal(["slow-job", "next-job"], handled); + } + } + finally + { + releaseSlow.TrySetResult(); + await subscriber.StopAsync(CancellationToken.None); + } + } + + [Fact] + public async Task WorkerSubscriber_StopWaitsForADetachedHandler_AndStoresItsOffsetBeforeClosing() + { + var consumer = new FakeKafkaConsumerClient(); + consumer.Enqueue(KafkaTestData.Message("workers", offset: 8, payload: "slow-job")); + + var slowStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseSlow = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var ingress = new Mock(); + ingress.Setup(i => i.HandleWorkerMessageAsync("slow-job")) + .Returns(async () => + { + slowStarted.TrySetResult(); + await releaseSlow.Task.ConfigureAwait(false); + }); + + var subscriber = CreateWorkerSubscriber(consumer, ingress.Object, options => + { + options.WorkerTopic = "workers"; + options.WorkerSubscriber.DetachHandlerAfter = TimeSpan.FromMilliseconds(20); + }); + await subscriber.StartAsync(CancellationToken.None); + await slowStarted.Task.WaitAsync(TimeSpan.FromSeconds(5)); + await KafkaTestData.WaitUntilAsync(() => consumer.IsPartitionPaused(0)); + + var stopping = subscriber.StopAsync(CancellationToken.None); + await Task.Delay(200); + Assert.False(stopping.IsCompleted); + Assert.False(consumer.Closed); + + releaseSlow.SetResult(); + await stopping.WaitAsync(TimeSpan.FromSeconds(10)); + + Assert.Equal(new FakeKafkaConsumerClient.StoredOffset("workers", 0, 8), Assert.Single(consumer.StoredOffsets)); + Assert.True(consumer.Closed); + } + + [Fact] + public async Task WorkerSubscriber_DetachedHandlerThatCannotBeDeadLettered_RestartsWithoutCommittingPastIt() + { + // The round-35 contract through the detached path: the burial failure surfaces from the + // poll thread's settlement tick, the consumer closes without storing the offset, and the + // supervisor rebuilds it. + var first = new FakeKafkaConsumerClient(); + first.Enqueue(KafkaTestData.Message("workers", offset: 10, payload: "poison")); + var second = new FakeKafkaConsumerClient(); + var factory = new FakeKafkaConsumerClientFactory(first, second); + + var ingress = new Mock(); + ingress.Setup(i => i.HandleWorkerMessageAsync("poison")) + .Returns(async () => + { + await Task.Delay(100); + throw new InvalidOperationException("handler boom"); + }); + + var options = NewOptions(o => + { + o.WorkerTopic = "workers"; + o.WorkerSubscriber.MaxDeliveryAttempts = 1; + o.WorkerSubscriber.DetachHandlerAfter = TimeSpan.FromMilliseconds(20); + o.PublishRetryBaseDelay = TimeSpan.FromMilliseconds(1); + o.PublishRetryMaxDelay = TimeSpan.FromMilliseconds(2); + }); + var subscriber = new KafkaWorkerSubscriber( + Options.Create(options), + factory, + new FakeKafkaProducerClient { PublishException = new InvalidOperationException("dead-letter topic gone") }, + new FakeKafkaAdminClient(), + ingress.Object, + NullLogger.Instance); + + await subscriber.StartAsync(CancellationToken.None); + try + { + await KafkaTestData.WaitUntilAsync(() => factory.CreatedRoles.Count >= 2, TimeSpan.FromSeconds(10)); + } + finally + { + await subscriber.StopAsync(CancellationToken.None); + } + + Assert.Empty(first.StoredOffsets); + Assert.True(first.Closed); + } + // ---------- Helpers ---------- private static readonly Dictionary Factories = []; diff --git a/tests/AsyncResponse.Tests/KafkaTestFakes.cs b/tests/AsyncResponse.Tests/KafkaTestFakes.cs index 242d6abf1..981ffe8b7 100644 --- a/tests/AsyncResponse.Tests/KafkaTestFakes.cs +++ b/tests/AsyncResponse.Tests/KafkaTestFakes.cs @@ -58,7 +58,9 @@ internal sealed record PublishCall( internal sealed class FakeKafkaConsumerClient : IKafkaConsumerClient { private readonly object _gate = new(); - private readonly Queue _messages = new(); + private readonly List _messages = []; + private readonly HashSet _pausedPartitions = []; + private int _consumeCalls; public List Subscriptions { get; } = []; public List StoredOffsets { get; } = []; @@ -68,6 +70,31 @@ internal sealed class FakeKafkaConsumerClient : IKafkaConsumerClient public bool Closed { get; private set; } public bool Disposed { get; private set; } + /// How many times Consume was called — the poll loop's liveness, as the broker sees it. + public int ConsumeCalls => Volatile.Read(ref _consumeCalls); + + /// Per-partition pause/resume calls (the ack-after-handler detach path). + public List PartitionPauses { get; } = []; + public List PartitionResumes { get; } = []; + + public bool IsPartitionPaused(int partition) + { + lock (_gate) + { + return _pausedPartitions.Contains(partition); + } + } + + /// + /// When set, a paused partition still delivers — what a rebalance does when it hands the + /// partition back with its pause state reset, or a client delivering a message it had already + /// fetched before the pause. + /// + public bool IgnorePartitionPause { get; set; } + + /// When set, PausePartition/ResumePartition throw it (the partition is no longer assigned). + public Exception? PartitionPauseException { get; set; } + /// When set, the next Consume call throws this exception once. public Exception? NextConsumeException { get; set; } @@ -81,7 +108,7 @@ public void Enqueue(KafkaIncomingMessage message) { lock (_gate) { - _messages.Enqueue(message); + _messages.Add(message); } } @@ -95,6 +122,7 @@ public void Subscribe(string topic) public KafkaIncomingMessage? Consume(TimeSpan maxWait) { + Interlocked.Increment(ref _consumeCalls); lock (_gate) { if (NextConsumeException is { } consumeException) @@ -103,9 +131,20 @@ public void Subscribe(string topic) throw consumeException; } - // Paused partitions deliver nothing, mirroring librdkafka semantics. - if (!Paused && _messages.Count > 0) - return _messages.Dequeue(); + // Paused partitions deliver nothing, mirroring librdkafka semantics; the rest deliver + // in the order they were enqueued. + if (!Paused) + { + for (var i = 0; i < _messages.Count; i++) + { + var candidate = _messages[i]; + if (!IgnorePartitionPause && _pausedPartitions.Contains(candidate.Partition)) + continue; + + _messages.RemoveAt(i); + return candidate; + } + } } // Keep the poll loop from spinning hot in tests while staying responsive. @@ -142,6 +181,30 @@ public void ResumeAssignment() } } + public void PausePartition(string topic, int partition) + { + if (PartitionPauseException is not null) + throw PartitionPauseException; + + lock (_gate) + { + _pausedPartitions.Add(partition); + PartitionPauses.Add(partition); + } + } + + public void ResumePartition(string topic, int partition) + { + if (PartitionPauseException is not null) + throw PartitionPauseException; + + lock (_gate) + { + _pausedPartitions.Remove(partition); + PartitionResumes.Add(partition); + } + } + public void Close() { Closed = true; @@ -212,9 +275,17 @@ public static KafkaIncomingMessage Message( long offset, string payload, params (string Key, string Value)[] headers) + => MessageOn(topic, partition: 0, offset, payload, headers); + + public static KafkaIncomingMessage MessageOn( + string topic, + int partition, + long offset, + string payload, + params (string Key, string Value)[] headers) => new( topic, - Partition: 0, + partition, offset, Encoding.UTF8.GetBytes(payload), headers.Select(header => KafkaTransportHeader.Utf8(header.Key, header.Value)).ToArray()); @@ -223,10 +294,11 @@ public static KafkaDelivery Delivery( string topic, long offset, string payload = "payload-json", - string? correlationId = "corr") + string? correlationId = "corr", + int partition = 0) => new( topic, - Partition: 0, + partition, offset, payload, correlationId, diff --git a/tests/AsyncResponse.Tests/LostSubscriberPartialSuccessTests.cs b/tests/AsyncResponse.Tests/LostSubscriberPartialSuccessTests.cs index f479c76d8..9c4f3f7b8 100644 --- a/tests/AsyncResponse.Tests/LostSubscriberPartialSuccessTests.cs +++ b/tests/AsyncResponse.Tests/LostSubscriberPartialSuccessTests.cs @@ -6,13 +6,20 @@ namespace AsyncResponse.Tests; /// -/// Regression (r23): with two recovery registrations sharing one correlation id (the expected -/// shape when a worker dies mid-await and its replacement re-attaches), a failure in ONE -/// registration's resume callback was rethrown even though the OTHER registration had already -/// consumed the response and resumed its flow. The ingress escalated that throw through its -/// retry loop into SetException, terminally failing a flow that was correctly recovered moments -/// earlier. A partial success now completes the dispatch: the residual failure is logged and the -/// failed registration stays registered for redelivery/watchdog visibility. +/// Two recovery registrations sharing one correlation id (the expected shape when a worker dies +/// mid-await and its replacement re-attaches), where one registration's callback succeeds and the +/// other's fails. +/// +/// History: r23/r24 made a partial success complete the dispatch — the residual failure was logged +/// and swallowed so the ingress would not escalate a delivered response through +/// SetException. Round 35 found the other half of that trade: swallowing returned success to +/// the broker for a payload the FAILED registration never received, so the broker acknowledged its +/// only copy, the registration stayed armed with nothing left to replay it, and the watchdog could +/// only report the stale row. A transient residual failure now propagates as +/// (the ingress passes it through untouched, so the +/// transport redelivers to the one registration still armed); a deterministic one keeps the +/// swallow, because redelivery cannot fix it. +/// /// public sealed class LostSubscriberPartialSuccessTests { @@ -32,6 +39,9 @@ private sealed class PartialResumeSpy : IPartialResumeSpy public int Ok => Volatile.Read(ref _ok); public int Boom => Volatile.Read(ref _boom); + /// The dependency behind ResumeBoom: down until a test brings it back. + public volatile bool DependencyUp; + public Task ResumeOk(OperationResult payload) { Interlocked.Increment(ref _ok); @@ -41,19 +51,37 @@ public Task ResumeOk(OperationResult payload) public Task ResumeBoom(OperationResult payload) { Interlocked.Increment(ref _boom); - throw new InvalidOperationException("re-enqueue failed on a publish-blocked broker"); + if (!DependencyUp) + throw new InvalidOperationException("re-enqueue failed on a publish-blocked broker"); + return Task.CompletedTask; } } - [Fact] - public async Task DispatchLostResponses_SiblingFailureAfterASuccessfulCallback_DoesNotEscalate() + /// A target interface no service implements: its callback fails deterministically at wire-up. + public interface IUnregisteredSpy + { + Task Resume(OperationResult payload); + } + + private static ServiceProvider BuildProvider(PartialResumeSpy spy) { - var spy = new PartialResumeSpy(); var services = new ServiceCollection(); services.AddSingleton(typeof(ILogger<>), typeof(NullLogger<>)); services.AddSingleton(spy); services.AddAsyncResponse().WithInMemoryChannel(); - await using var provider = services.BuildServiceProvider(); + return services.BuildServiceProvider(); + } + + /// + /// Round 35: pre-fix, this publish completed normally — the broker would have acknowledged the + /// response the failed registration never got. It now propagates for redelivery, with the + /// successful registration consumed and the failed one still armed. + /// + [Fact] + public async Task DispatchLostResponses_TransientSiblingFailureAfterASuccessfulCallback_PropagatesForRedelivery() + { + var spy = new PartialResumeSpy(); + await using var provider = BuildProvider(spy); var recoveryStateStore = provider.GetRequiredService(); var okRegistration = Guid.NewGuid(); @@ -63,23 +91,88 @@ public async Task DispatchLostResponses_SiblingFailureAfterASuccessfulCallback_D var publisher = provider.GetRequiredService(); - // On the old code this task faulted with the sibling's InvalidOperationException even - // though the response WAS delivered — and the ingress path escalated exactly that throw - // into SetException/FailAsync. - await publisher.SetResponse( + var ex = await Assert.ThrowsAsync(() => publisher.SetResponse( new OperationResult { Status = OperationStatus.Completed, Message = "late response" }, - CorrelationId); + CorrelationId)); + Assert.Equal(CorrelationId, ex.CorrelationId); + Assert.IsType(ex.InnerException); Assert.Equal(1, spy.Ok); Assert.Equal(1, spy.Boom); - // The successful registration was consumed; the failed one stays registered so a later - // redelivery can retry it and the watchdog can surface it. + // The successful registration was consumed; the failed one stays armed for the redelivery. var remaining = await recoveryStateStore.GetAllAsync(CorrelationId); var leftover = Assert.Single(remaining); Assert.Equal(boomRegistration, leftover.RegistrationId); } + /// + /// Eventual completion: once the dependency is back, the redelivery (here: the publisher's + /// caller retrying the publish) reaches only the registration that failed — the consumed one + /// is not re-invoked — and settles with nothing left armed. + /// + [Fact] + public async Task DispatchLostResponses_RedeliveryAfterTheDependencyRecovers_CompletesOnlyTheFailedRegistration() + { + var spy = new PartialResumeSpy(); + await using var provider = BuildProvider(spy); + + var recoveryStateStore = provider.GetRequiredService(); + await recoveryStateStore.SaveAsync(CorrelationId, Registration(Guid.NewGuid(), nameof(IPartialResumeSpy.ResumeOk)), TimeSpan.FromMinutes(5)); + await recoveryStateStore.SaveAsync(CorrelationId, Registration(Guid.NewGuid(), nameof(IPartialResumeSpy.ResumeBoom)), TimeSpan.FromMinutes(5)); + + var publisher = provider.GetRequiredService(); + var response = new OperationResult { Status = OperationStatus.Completed, Message = "late response" }; + + await Assert.ThrowsAsync(() => publisher.SetResponse(response, CorrelationId)); + + spy.DependencyUp = true; + await publisher.SetResponse(response, CorrelationId); + + Assert.Equal(1, spy.Ok); + Assert.Equal(2, spy.Boom); + Assert.Empty(await recoveryStateStore.GetAllAsync(CorrelationId)); + } + + /// + /// Pin (unchanged): a DETERMINISTIC residual failure — the sibling's target service is not + /// registered — is still swallowed. Redelivery cannot fix it; the message is acknowledged and + /// the failed registration stays for the watchdog to surface. + /// + [Fact] + public async Task DispatchLostResponses_DeterministicSiblingFailureAfterASuccessfulCallback_IsStillSwallowed() + { + var spy = new PartialResumeSpy(); + await using var provider = BuildProvider(spy); + + var recoveryStateStore = provider.GetRequiredService(); + var unresolvable = Guid.NewGuid(); + await recoveryStateStore.SaveAsync(CorrelationId, Registration(Guid.NewGuid(), nameof(IPartialResumeSpy.ResumeOk)), TimeSpan.FromMinutes(5)); + await recoveryStateStore.SaveAsync(CorrelationId, new RecoveryState + { + RegistrationId = unresolvable, + CorrelationId = CorrelationId, + PayloadTypeFullName = typeof(OperationResult).FullName, + RegisteredAtUtc = DateTime.UtcNow, + ResumeCallback = new ReflectionCallDto + { + ServiceInterfaceFullName = typeof(IUnregisteredSpy).FullName!, + MethodName = nameof(IUnregisteredSpy.Resume), + Params = [CallbackParam.ForPlaceholder(PlaceholderType.Payload)] + } + }, TimeSpan.FromMinutes(5)); + + var publisher = provider.GetRequiredService(); + + await publisher.SetResponse( + new OperationResult { Status = OperationStatus.Completed, Message = "late response" }, + CorrelationId); + + Assert.Equal(1, spy.Ok); + var leftover = Assert.Single(await recoveryStateStore.GetAllAsync(CorrelationId)); + Assert.Equal(unresolvable, leftover.RegistrationId); + } + private static RecoveryState Registration(Guid registrationId, string methodName) => new() { @@ -118,18 +211,18 @@ public Task FailOk(Exception exception) public Task FailBoom(Exception exception) { Interlocked.Increment(ref _boom); - throw new InvalidOperationException("failure callback hit an unregistered service"); + throw new InvalidOperationException("failure callback hit a dependency that is down"); } } + /// + /// The exception-path twin (r24 made it a swallow; round 35 turns the transient case into a + /// propagation for redelivery, exactly like the response path). Pre-fix: the publish + /// completed normally with one registration's callback never having run. + /// [Fact] - public async Task DispatchLostExceptions_SiblingFailureAfterASuccessfulCallback_DoesNotEscalate() + public async Task DispatchLostExceptions_TransientSiblingFailureAfterASuccessfulCallback_PropagatesForRedelivery() { - // Regression (r24): DispatchLostResponses received this partial-failure guard in r23, but - // its exception-path twin still rethrew unconditionally — a SetException whose FIRST - // registration's failure callback succeeded (and was consumed) faulted on the SECOND - // registration's throw, so the ingress redelivered the whole message forever (the consumed - // registration is gone, the failing one keeps failing; the delivery never settles). const string correlationId = "partial-failure-correlation-id"; var spy = new PartialFailSpy(); var services = new ServiceCollection(); @@ -146,15 +239,14 @@ public async Task DispatchLostExceptions_SiblingFailureAfterASuccessfulCallback_ var publisher = provider.GetRequiredService(); - // On the old code this faulted with the sibling's InvalidOperationException even though - // the exception WAS delivered to (and consumed by) the first registration. - await publisher.SetException(new InvalidOperationException("remote boom"), correlationId); + var ex = await Assert.ThrowsAsync( + () => publisher.SetException(new InvalidOperationException("remote boom"), correlationId)); + Assert.Equal(correlationId, ex.CorrelationId); Assert.Equal(1, spy.Ok); Assert.Equal(1, spy.Boom); - // The successful registration was consumed; the failed one stays registered so a later - // redelivery can retry it and the watchdog can surface it. + // The successful registration was consumed; the failed one stays armed for the redelivery. var remaining = await recoveryStateStore.GetAllAsync(correlationId); var leftover = Assert.Single(remaining); Assert.Equal(boomRegistration, leftover.RegistrationId); diff --git a/tests/AsyncResponse.Tests/LostSubscriberRoutingTests.cs b/tests/AsyncResponse.Tests/LostSubscriberRoutingTests.cs index 64d250e4f..321deed94 100644 --- a/tests/AsyncResponse.Tests/LostSubscriberRoutingTests.cs +++ b/tests/AsyncResponse.Tests/LostSubscriberRoutingTests.cs @@ -290,16 +290,22 @@ public async Task SetResponse_FailedPayload_WithoutFailureCallback_ErrorLogsPayl } [Fact] - public async Task SetResponse_FailureCallbackThrows_IsRetriedThenSwallowedAndRecoveryStateIsKept() + public async Task SetResponse_FailureCallbackThrows_IsRetriedThenThrowsAndRecoveryStateIsKept() { ArmRecoveryState(); _spy.FailureCallbackError = new InvalidOperationException("handler exploded"); - // Must not throw: rethrowing would loop back through the ingress's SetException safety - // net and invoke the same failure callback a second time. A persistently failing callback - // is retried in-process (bounded, mirroring the ingress policy) before the swallow. - await Publisher.SetResponse(new OperationResult { Status = OperationStatus.Failed }, CorrelationId); - + // Round 34: a persistently failing callback is retried in-process (bounded, mirroring the + // ingress policy) and then the publish THROWS the dedicated type instead of returning — + // returning normally acknowledged a terminal signal that then existed nowhere. The + // ingress passes this type through untouched (no SetException loop back into the same + // callback), so the transport redelivers; the registration stays for that redelivery. + var thrown = await Assert.ThrowsAsync( + () => Publisher.SetResponse(new OperationResult { Status = OperationStatus.Failed }, CorrelationId)); + + Assert.Equal(CorrelationId, thrown.CorrelationId); + Assert.Equal(4, thrown.Attempts); + Assert.Same(_spy.FailureCallbackError, thrown.InnerException); Assert.Equal(4, _spy.Failures.Count); _database.Verify(d => d.KeyDeleteAsync(It.IsAny(), It.IsAny()), Times.Never); } diff --git a/tests/AsyncResponse.Tests/MongoDbChannelCoverageTests.cs b/tests/AsyncResponse.Tests/MongoDbChannelCoverageTests.cs index 00ffbd83a..b636913c4 100644 --- a/tests/AsyncResponse.Tests/MongoDbChannelCoverageTests.cs +++ b/tests/AsyncResponse.Tests/MongoDbChannelCoverageTests.cs @@ -5,6 +5,7 @@ using System.Collections; using Microsoft.Extensions.Options; using MongoDB.Bson; +using MongoDB.Bson.Serialization; using MongoDB.Driver; using MongoDB.Driver.Core.Clusters; using MongoDB.Driver.Core.Connections; @@ -191,7 +192,8 @@ public async Task Subscription_ProcessesEveryEnvelopeOutcome_AndMaintainsSeenSet var malformed = fixture.Subscription(_ => new ValueTask(true)); await InvokeTaskAsync(malformed.Instance, "ProcessAsync", Message("{not-json")); - await Assert.ThrowsAsync(() => malformed.Completion.Task); + // The body-free parse failure (JsonSafety), not the raw reader's JsonException. + await Assert.ThrowsAsync(() => malformed.Completion.Task); var dropped = fixture.Subscription(_ => new ValueTask(true)); await InvokeValueTaskAsync(dropped.Instance, "DropLocalAsync", CancellationToken.None); @@ -1354,6 +1356,115 @@ public async Task DispatchPendingMessagesAsync_ScopedScanDeliversThroughTheQueue Assert.Equal("swept", delivered.Message); } + /// + /// Round 39: the sweep re-read every retained row's envelope on every tick, acknowledged + /// history included, only to drop it in the pre-filter — a long-lived progress subscription's + /// sweep cost grew with its whole history. The store now ships the envelope only for + /// unacknowledged rows; an acknowledged row a live subscription has not seen (cross-process + /// fan-out) comes back header-only and is hydrated by id before delivery. Pre-fix: the + /// header-only row was handed to the waiter as-is and its delivery faulted on a null body. + /// + [Fact] + public async Task DispatchPendingMessagesAsync_HydratesAHeaderOnlyAcknowledgedRow_BeforeDeliveringIt() + { + var fixture = new ChannelFixture(); + var channel = fixture.Channel; + var subscription = fixture.Subscription(_ => new ValueTask(true), "hydrate-corr"); + AddSubscription(channel, "hydrate-corr", subscription.Instance); + + // Acknowledged by another process AFTER this waiter registered: inside the watermark, + // unseen here, so it must be delivered — but the sweep's page carries no envelope for it. + var id = Guid.NewGuid(); + var createdAt = DateTime.UtcNow; + var ackedAt = DateTime.UtcNow.AddSeconds(2); + MongoChannelMessageDocument HeaderOnly() => new() + { + Id = id, + CorrelationId = "hydrate-corr", + EnvelopeJson = null, + CreatedAtUtc = createdAt, + AckedAtUtc = ackedAt, + AckedSeq = 42 + }; + MongoChannelMessageDocument Full() => new() + { + Id = id, + CorrelationId = "hydrate-corr", + EnvelopeJson = """{"SchemaVersion":1,"Success":true,"Payload":{"Status":2,"Message":"hydrated"}}""", + CreatedAtUtc = createdAt, + AckedAtUtc = ackedAt, + AckedSeq = 42 + }; + var byIdReads = 0; + fixture.Messages + .Setup(c => c.FindAsync( + It.IsAny>(), + It.IsAny>(), + It.IsAny())) + .Returns((FilterDefinition filter, FindOptions _, CancellationToken _) => + { + // The sweep page (correlation + keyset) vs the hydration read (id $in ...). + var byId = RenderFilter(filter).Contains("$in", StringComparison.Ordinal); + if (byId) + Interlocked.Increment(ref byIdReads); + return Task.FromResult>( + new DummyCursor([byId ? Full() : HeaderOnly()])); + }); + fixture.Messages + .Setup(c => c.FindOneAndUpdateAsync( + It.IsAny>(), + It.IsAny>(), + It.IsAny>(), + It.IsAny())) + .ReturnsAsync(Full()); + + var dispatchMethod = typeof(MongoDbAsyncResponseChannel) + .GetMethod("DispatchPendingMessagesAsync", BindingFlags.Instance | BindingFlags.NonPublic)!; + await (Task)dispatchMethod.Invoke(channel, [new HashSet { "hydrate-corr" }, CancellationToken.None])!; + + var delivered = await subscription.Completion.Task.WaitAsync(TimeSpan.FromSeconds(10)); + Assert.Equal("hydrated", delivered.Message); + Assert.Equal(1, byIdReads); + } + + /// The sweep's page query projects the envelope only for documents nobody has acknowledged. + [Fact] + public async Task LoadMessagesAsync_ProjectsTheEnvelopeOnlyForUnacknowledgedDocuments() + { + var fixture = new ChannelFixture(); + FindOptions? captured = null; + fixture.Messages + .Setup(c => c.FindAsync( + It.IsAny>(), + It.IsAny>(), + It.IsAny())) + .Callback, FindOptions, CancellationToken>( + (_, options, _) => captured = options) + .ReturnsAsync(new DummyCursor([])); + + await fixture.Store.LoadMessagesAsync("corr", DateTimeOffset.UtcNow, 16, null, null, CancellationToken.None); + + Assert.NotNull(captured?.Projection); + var rendered = captured!.Projection!.Render(new RenderArgs( + BsonSerializer.LookupSerializer(), + BsonSerializer.SerializerRegistry)).Document; + var envelope = rendered["envelope_json"].AsBsonDocument; + Assert.True(envelope.Contains("$cond"), rendered.ToJson()); + Assert.Contains("$acked_at", rendered.ToJson(), StringComparison.Ordinal); + Assert.Equal(1, rendered["acked_seq"].AsInt32); + + // The hydration read carries no projection: it must return the envelope. + captured = null; + await fixture.Store.LoadMessagesByIdAsync("corr", [Guid.NewGuid()], CancellationToken.None); + Assert.NotNull(captured); + Assert.Null(captured!.Projection); + } + + private static string RenderFilter(FilterDefinition filter) + => filter.Render(new RenderArgs( + BsonSerializer.LookupSerializer(), + BsonSerializer.SerializerRegistry)).ToJson(); + [Fact] public async Task MongoDbRecoveryStateStore_ThrowsOnMismatchedCorrelationId() { diff --git a/tests/AsyncResponse.Tests/NatsAdapterTests.cs b/tests/AsyncResponse.Tests/NatsAdapterTests.cs index 1af1ee522..c6e424315 100644 --- a/tests/AsyncResponse.Tests/NatsAdapterTests.cs +++ b/tests/AsyncResponse.Tests/NatsAdapterTests.cs @@ -546,14 +546,18 @@ public async Task FetchNoWaitAsync_MapsMessages_AndSettlementDelegatesForward() await single.AckAsync(); await single.TermAsync(); - await single.ProgressAsync(); + // Round 39: the heartbeat's token reaches the SDK call (the settlements above stay + // deliberately uncancelable — a decision already taken must reach the server). + using var progressCancellation = new CancellationTokenSource(); + await single.ProgressAsync(progressCancellation.Token); // NakAsync(delay) is a NATS.Net extension over the message (not a mockable member), so it // cannot be Moq-verified; invoking the delegate still exercises the adapter's nak path, and the // extension's internal member call on the loose mock is tolerated. try { await single.NakAsync(TimeSpan.FromSeconds(2)); } catch (Exception) { /* extension-over-mock */ } message.Verify(m => m.AckAsync(It.IsAny(), It.IsAny()), Times.Once); message.Verify(m => m.AckTerminateAsync(It.IsAny(), It.IsAny()), Times.Once); - message.Verify(m => m.AckProgressAsync(It.IsAny(), It.IsAny()), Times.Once); + message.Verify(m => m.AckProgressAsync(It.IsAny(), progressCancellation.Token), Times.Once); + message.Verify(m => m.AckAsync(It.IsAny(), CancellationToken.None), Times.Once); } [Fact] diff --git a/tests/AsyncResponse.Tests/NatsAsyncResponseChannelTests.cs b/tests/AsyncResponse.Tests/NatsAsyncResponseChannelTests.cs index c0232cade..4c9a51e42 100644 --- a/tests/AsyncResponse.Tests/NatsAsyncResponseChannelTests.cs +++ b/tests/AsyncResponse.Tests/NatsAsyncResponseChannelTests.cs @@ -393,7 +393,8 @@ public async Task CreateResponseWaiter_MalformedMessageFaultsWaiter() _client.Push("{not-json"); - await Assert.ThrowsAsync(() => waiter.ResponseTask.WaitAsync(TimeSpan.FromSeconds(2))); + // The body-free parse failure (JsonSafety), not the raw reader's JsonException. + await Assert.ThrowsAsync(() => waiter.ResponseTask.WaitAsync(TimeSpan.FromSeconds(2))); } [Fact] diff --git a/tests/AsyncResponse.Tests/NatsChannelCoverageTests.cs b/tests/AsyncResponse.Tests/NatsChannelCoverageTests.cs index d0c6efda1..6bfeb586b 100644 --- a/tests/AsyncResponse.Tests/NatsChannelCoverageTests.cs +++ b/tests/AsyncResponse.Tests/NatsChannelCoverageTests.cs @@ -315,6 +315,28 @@ await Assert.ThrowsAsync( hang.TrySetResult(); } + /// + /// Round 36: the reader deserialized the payload directly, so a payload that failed to + /// convert faulted the waiter with — and logged — the raw System.Text.Json exception, whose + /// message quotes the inbound dictionary key (Path: $.Payload.Values['…']). Pre-fix + /// failure: the marker is in the waiter's exception and in the channel's error log. + /// + [Fact] + public async Task CreateResponseWaiter_MalformedPayload_DoesNotEchoInboundKeysIntoLogsOrTheWaiter() + { + var logger = new CollectingLogger(); + var client = new FakeNatsResponseChannelClient(); + var channel = CreateChannel(client, logger.For()); + + await using var waiter = await channel.CreateResponseWaiter( + "corr-leak", + timeout: TimeSpan.FromSeconds(5)); + client.Push(Round36RegressionTests.LeakingEnvelope); + + var ex = await Assert.ThrowsAnyAsync(() => waiter.ResponseTask.WaitAsync(TimeSpan.FromSeconds(2))); + Round36RegressionTests.AssertNoMarker(ex, logger); + } + private NatsAsyncResponseChannel CreateChannel( FakeNatsResponseChannelClient client, ILogger logger, diff --git a/tests/AsyncResponse.Tests/NatsSubscriberServicesTests.cs b/tests/AsyncResponse.Tests/NatsSubscriberServicesTests.cs index 4945d51eb..a4a1b3d61 100644 --- a/tests/AsyncResponse.Tests/NatsSubscriberServicesTests.cs +++ b/tests/AsyncResponse.Tests/NatsSubscriberServicesTests.cs @@ -199,6 +199,96 @@ public async Task WorkerSubscriber_InvalidOptions_FailHostStartupSynchronously() Assert.Contains("BackgroundWorkerCount", ex.Message, StringComparison.Ordinal); } + /// + /// Round 39: the in-progress heartbeat ran with CancellationToken.None and the batch's + /// cleanup joined the renewal loop without a bound, so ONE heartbeat wedged on a dead socket + /// kept the batch pending after every message in it had settled — no further batch was + /// fetched, a stop never completed, and the supervisor had nothing to restart. The token now + /// reaches the heartbeat: a client-side stall aborts with it and the batch completes at once. + /// Pre-fix: the second delivery is never fetched. + /// + [Fact] + public async Task WorkerSubscriber_AHeartbeatThatHonorsCancellation_IsAbortedWhenTheBatchSettles() + { + var ingress = new GatedIngress(); + var first = new RecordingDelivery + { + // The heartbeat stalls until its token is cancelled. + ProgressBehavior = async cancellationToken => + { + var stalled = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + using var registration = cancellationToken.Register(() => stalled.TrySetCanceled(cancellationToken)); + await stalled.Task; + } + }; + var second = new RecordingDelivery(); + _jetStream.EnqueueDelivery(first.Create("p1", numDelivered: 1)); + var subscriber = new NatsWorkerSubscriber( + Options(o => o.AckWait = TimeSpan.FromMilliseconds(150)), + _jetStream, + ingress, + new TestLogger()); + + await subscriber.StartAsync(CancellationToken.None); + try + { + await ingress.Started.Task.WaitAsync(TimeSpan.FromSeconds(5)); + await Eventually(() => first.Progresses >= 1); // the heartbeat is now wedged + ingress.Release.TrySetResult(); + await Eventually(() => first.Acks == 1); + + // The batch settled; the wedged heartbeat must not hold the loop: the next batch is + // fetched and settled. + _jetStream.EnqueueDelivery(second.Create("p2", numDelivered: 1)); + await Eventually(() => second.Acks == 1); + } + finally + { + await subscriber.StopAsync(CancellationToken.None); + subscriber.Dispose(); + } + } + + /// + /// The backstop for a heartbeat the client cannot abort at all (it ignores its token): the + /// join is bounded by one heartbeat interval, after which the renewal loop is abandoned with + /// a warning and the loop moves on — unsettled deliveries fall back to the server's AckWait. + /// + [Fact] + public async Task WorkerSubscriber_AHeartbeatThatIgnoresCancellation_IsAbandonedAfterOneInterval() + { + var ingress = new GatedIngress(); + var never = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var first = new RecordingDelivery { ProgressBehavior = async _ => await never.Task }; + var second = new RecordingDelivery(); + _jetStream.EnqueueDelivery(first.Create("p1", numDelivered: 1)); + var logger = new RecordingThrowingLogger(); + var subscriber = new NatsWorkerSubscriber( + Options(o => o.AckWait = TimeSpan.FromMilliseconds(150)), + _jetStream, + ingress, + logger); + + await subscriber.StartAsync(CancellationToken.None); + try + { + await ingress.Started.Task.WaitAsync(TimeSpan.FromSeconds(5)); + await Eventually(() => first.Progresses >= 1); + ingress.Release.TrySetResult(); + await Eventually(() => first.Acks == 1); + + _jetStream.EnqueueDelivery(second.Create("p2", numDelivered: 1)); + await Eventually(() => second.Acks == 1); + Assert.True(logger.HasEntry(Microsoft.Extensions.Logging.LogLevel.Warning, "did not stop within"), "the abandoned heartbeat must be logged"); + } + finally + { + never.TrySetResult(); + await subscriber.StopAsync(CancellationToken.None); + subscriber.Dispose(); + } + } + private static async Task Eventually(Func condition) { using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); diff --git a/tests/AsyncResponse.Tests/NatsTestFakes.cs b/tests/AsyncResponse.Tests/NatsTestFakes.cs index 24ece5ad7..5ade57ae7 100644 --- a/tests/AsyncResponse.Tests/NatsTestFakes.cs +++ b/tests/AsyncResponse.Tests/NatsTestFakes.cs @@ -352,6 +352,13 @@ internal sealed class RecordingDelivery /// When set, every in-progress attempt is recorded and then fails with this exception. public Exception? ProgressException { get; set; } + /// + /// When set, every in-progress attempt is recorded and then runs this instead (a heartbeat + /// that stalls, honors or ignores its cancellation token, …). Takes precedence over + /// . + /// + public Func? ProgressBehavior { get; set; } + public NatsJobDelivery Create(string payload, long numDelivered, string subject = "asyncresponse.transport.worker", IReadOnlyDictionary? headers = null) => new( subject, @@ -374,9 +381,11 @@ public NatsJobDelivery Create(string payload, long numDelivered, string subject return TermException is null ? ValueTask.CompletedTask : ValueTask.FromException(TermException); }) { - ProgressAsync = () => + ProgressAsync = cancellationToken => { Interlocked.Increment(ref _progresses); + if (ProgressBehavior is { } behavior) + return behavior(cancellationToken); return ProgressException is null ? ValueTask.CompletedTask : ValueTask.FromException(ProgressException); } }; diff --git a/tests/AsyncResponse.Tests/RedisAsyncResponseChannelWaiterTests.cs b/tests/AsyncResponse.Tests/RedisAsyncResponseChannelWaiterTests.cs index ad256ee65..9e5de574d 100644 --- a/tests/AsyncResponse.Tests/RedisAsyncResponseChannelWaiterTests.cs +++ b/tests/AsyncResponse.Tests/RedisAsyncResponseChannelWaiterTests.cs @@ -266,7 +266,7 @@ await DuplicateFaultAsync( ExceptionMessage = "remote" }, AsyncResponseEnvelopeOptions.Instance), typeof(Exception)); - await DuplicateFaultAsync(channel, "duplicate-malformed", "{not-json", typeof(JsonException)); + await DuplicateFaultAsync(channel, "duplicate-malformed", "{not-json", typeof(InvalidDataException)); } [Fact] @@ -321,7 +321,34 @@ public async Task CreateResponseWaiter_MalformedRedisMessageFaultsWaiter() await _channelSubscriber.Handler!.Invoke(_channelSubscriber.SubscribedChannel, "{not-json"); - await Assert.ThrowsAsync(() => waiter.ResponseTask.WaitAsync(TimeSpan.FromSeconds(2))); + // The body-free parse failure (JsonSafety), not the raw reader's JsonException. + await Assert.ThrowsAsync(() => waiter.ResponseTask.WaitAsync(TimeSpan.FromSeconds(2))); + } + + /// + /// Round 36: the reader deserialized the wire bytes directly, so a payload that failed to + /// convert faulted the waiter with — and logged — the raw System.Text.Json exception, whose + /// message quotes the inbound dictionary key (Path: $.Payload.Values['…']). Pre-fix + /// failure: the marker is in the waiter's exception and in the channel's error log. + /// + [Fact] + public async Task CreateResponseWaiter_MalformedPayload_DoesNotEchoInboundKeysIntoLogsOrTheWaiter() + { + var logger = new CollectingLogger(); + var channel = CreateChannel(new RedisAsyncResponseOptions + { + DefaultTimeout = TimeSpan.FromSeconds(5), + RecoveryStateExpiry = TimeSpan.FromMinutes(5) + }, logger.For()); + + await using var waiter = await channel.CreateResponseWaiter( + "corr-leak", + timeout: TimeSpan.FromSeconds(5)); + + await _channelSubscriber.Handler!.Invoke(_channelSubscriber.SubscribedChannel, Round36RegressionTests.LeakingEnvelope); + + var ex = await Assert.ThrowsAnyAsync(() => waiter.ResponseTask.WaitAsync(TimeSpan.FromSeconds(2))); + Round36RegressionTests.AssertNoMarker(ex, logger); } [Fact] diff --git a/tests/AsyncResponse.Tests/RelationalChannelSubscriptionCoverageTests.cs b/tests/AsyncResponse.Tests/RelationalChannelSubscriptionCoverageTests.cs index f7f7c1341..7e8ed29af 100644 --- a/tests/AsyncResponse.Tests/RelationalChannelSubscriptionCoverageTests.cs +++ b/tests/AsyncResponse.Tests/RelationalChannelSubscriptionCoverageTests.cs @@ -124,7 +124,8 @@ await ProcessAsync(remoteFailure.Instance, message( var malformed = Subscription(channelType, subscriptionTypeName, channel, _ => new ValueTask(true)); await ProcessAsync(malformed.Instance, message("{not-json")); - await Assert.ThrowsAsync(() => malformed.Completion.Task); + // The body-free parse failure (JsonSafety), not the raw reader's JsonException. + await Assert.ThrowsAsync(() => malformed.Completion.Task); var dropped = Subscription(channelType, subscriptionTypeName, channel, _ => new ValueTask(true)); SetField(dropped.Instance, "_dropped", true); @@ -162,10 +163,18 @@ private static (object Instance, TaskCompletionSource Completio string nestedTypeName, object channel, Func> predicate) + => Subscription(channelType, nestedTypeName, channel, predicate); + + private static (object Instance, TaskCompletionSource Completion) Subscription( + Type channelType, + string nestedTypeName, + object channel, + Func> predicate) + where TPayload : IAsyncResponsePayload { - var completion = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var completion = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var type = channelType.BaseType!.GetNestedType(nestedTypeName, BindingFlags.NonPublic)! - .MakeGenericType(typeof(OperationResult)); + .MakeGenericType(typeof(TPayload)); var instance = Activator.CreateInstance( type, BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic, @@ -178,6 +187,43 @@ private static (object Instance, TaskCompletionSource Completio return (instance, completion); } + /// + /// Round 36, over the shared DbChannelShared source (SQL Server here; PostgreSQL and MongoDB + /// compile the same file): the subscription deserialized the stored envelope directly, so a + /// payload that failed to convert faulted the waiter with — and logged — the raw + /// System.Text.Json exception, whose message quotes the inbound dictionary key. Pre-fix + /// failure: the marker is in the waiter's exception and in the channel's error log. + /// + [Fact] + public async Task DbSubscription_MalformedPayload_DoesNotEchoInboundKeysIntoLogsOrTheWaiter() + { + var options = Options.Create(new SqlServerAsyncResponseChannelOptions + { + ConnectionString = "Server=localhost,1;Database=unused;User Id=unused;Password=unused;TrustServerCertificate=true;Connect Timeout=1", + AutoCreateSchema = false + }); + var logger = new CollectingLogger(); + using var provider = new ServiceCollection().BuildServiceProvider(); + var channel = new SqlServerAsyncResponseChannel( + provider.GetRequiredService(), + new SqlServerChannelSql(options), + MockRecoveryStore(), + options, + new AsyncResponseContextPropagation([]), + logger.For()); + + var subscription = Subscription( + typeof(SqlServerAsyncResponseChannel), "DbSubscription`1", channel, _ => new ValueTask(true)); + await ProcessAsync( + subscription.Instance, + new SqlServerChannelMessage(Guid.NewGuid(), "corr", Round36RegressionTests.LeakingEnvelope, DateTimeOffset.UtcNow)); + + var ex = await Assert.ThrowsAnyAsync(() => subscription.Completion.Task); + Round36RegressionTests.AssertNoMarker(ex, logger); + + await channel.DisposeAsync(); + } + private static IRecoveryStateStore MockRecoveryStore() { var store = new Moq.Mock(); @@ -268,6 +314,8 @@ public async Task SqlServerChannelSql_ExceptionCoverage() await Assert.ThrowsAnyAsync(() => sql.GetServerTimeUtcAsync(CancellationToken.None)); await Assert.ThrowsAnyAsync(() => sql.IsMessageAcknowledgedAsync(Guid.NewGuid(), CancellationToken.None)); await Assert.ThrowsAnyAsync(() => sql.LoadMessagesAsync("corr", DateTimeOffset.UtcNow, 10, null, null, CancellationToken.None)); + Assert.Empty(await sql.LoadMessagesByIdAsync("corr", [], CancellationToken.None)); // no ids, no round trip + await Assert.ThrowsAnyAsync(() => sql.LoadMessagesByIdAsync("corr", [Guid.NewGuid(), Guid.NewGuid()], CancellationToken.None)); await Assert.ThrowsAnyAsync(() => sql.HeartbeatSubscribersAsync("instance", [("corr", Guid.NewGuid())], TimeSpan.FromMinutes(1), CancellationToken.None)); await Assert.ThrowsAnyAsync(() => sql.CountActiveSubscribersAsync("corr", CancellationToken.None)); await Assert.ThrowsAnyAsync(() => sql.SaveRecoveryStateAsync("corr", new RecoveryState { RegistrationId = Guid.NewGuid(), CorrelationId = "corr" }, TimeSpan.FromMinutes(1), CancellationToken.None)); @@ -299,6 +347,8 @@ public async Task PostgreSqlChannelSql_ExceptionCoverage() await Assert.ThrowsAnyAsync(() => sql.GetServerTimeUtcAsync(CancellationToken.None)); await Assert.ThrowsAnyAsync(() => sql.IsMessageAcknowledgedAsync(Guid.NewGuid(), CancellationToken.None)); await Assert.ThrowsAnyAsync(() => sql.LoadMessagesAsync("corr", DateTimeOffset.UtcNow, 10, null, null, CancellationToken.None)); + Assert.Empty(await sql.LoadMessagesByIdAsync("corr", [], CancellationToken.None)); // no ids, no round trip + await Assert.ThrowsAnyAsync(() => sql.LoadMessagesByIdAsync("corr", [Guid.NewGuid()], CancellationToken.None)); await Assert.ThrowsAnyAsync(() => sql.HeartbeatSubscribersAsync("instance", [("corr", Guid.NewGuid())], TimeSpan.FromMinutes(1), CancellationToken.None)); await Assert.ThrowsAnyAsync(() => sql.CountActiveSubscribersAsync("corr", CancellationToken.None)); diff --git a/tests/AsyncResponse.Tests/Round26RegressionTests.cs b/tests/AsyncResponse.Tests/Round26RegressionTests.cs index aadda921d..0d1424d04 100644 --- a/tests/AsyncResponse.Tests/Round26RegressionTests.cs +++ b/tests/AsyncResponse.Tests/Round26RegressionTests.cs @@ -437,20 +437,26 @@ public void ACaseOnlyNamePlanCollision_NamesBothSpellings_SqlServer() // ----------------------------------------------------------------------------------------- [Fact] - public async Task AncestorLedgerRefresh_AgainstAConcurrentWriter_TriesOnceAndGivesUp() + public async Task AncestorLedgerRefresh_AgainstAConcurrentWriter_RetriesBoundedly_ThenAbandonsThePark() { - // Pre-fix MutateAsync retried the compare-and-swap eight times. Every round that WON + // Pre-round-26 MutateAsync retried the compare-and-swap eight times. Every round that WON // advanced the ancestor's revision, so a parent that was genuinely executing lost its next // checkpoint's CAS, called MarkLost() and abandoned its delivery for redelivery — - // re-running everything since its last checkpoint. A lost CAS now means "the ancestor is - // alive and re-stamping its own expiry", which is exactly when this insurance is not - // needed, so one attempt is enough. + // re-running everything since its last checkpoint. Round 26 made a lost CAS mean "the + // ancestor is alive and re-stamping its own expiry" and stopped after one attempt; round + // 38 found the competing write stamps the plain StateExpiry (it knows nothing about the + // park), so ceding the race expired the parent under the child's wait. The extension now + // re-reads and retries a small, fixed number of times — a re-read that already carries a + // floor reaching the park ends it without a write — and a walk that loses every attempt + // abandons the park (nothing published; the delivery retries) instead of either fighting + // without end or parking on unproven retention. var store = new CasRefusingFlowStateStore("parent"); var context = CreateContextForAncestorWalk(store, childFlowId: "child", parentFlowId: "parent"); - await InvokeExtendAncestorLedgersAsync(context, TimeSpan.FromDays(30)); + var ex = await Assert.ThrowsAsync(() => InvokeExtendAncestorLedgersAsync(context, TimeSpan.FromDays(30))); - Assert.Equal(1, store.UpdateAttempts); + Assert.Equal(DurableFlowContext.MaxAncestorExtensionAttempts, store.UpdateAttempts); + Assert.Contains("abandoned", ex.Message, StringComparison.Ordinal); } [Fact] diff --git a/tests/AsyncResponse.Tests/Round27RegressionTests.cs b/tests/AsyncResponse.Tests/Round27RegressionTests.cs index 3a11f1887..e526d2195 100644 --- a/tests/AsyncResponse.Tests/Round27RegressionTests.cs +++ b/tests/AsyncResponse.Tests/Round27RegressionTests.cs @@ -176,8 +176,14 @@ private static ServiceProvider BuildStartProvider(FailingWorkerTransport transpo return services.BuildServiceProvider(); } + /// + /// Round 27 pinned that a failed publish surfaces the generated id AFTER retrying. Round 35 + /// changed what the id points at: the publish is now the start's commit point, so nothing is + /// persisted when it fails — the id is for an idempotent retry, not for re-driving an orphan. + /// (The orphan half of this test moved to Round35RegressionTests, inverted.) + /// [Fact] - public async Task StartWithAGeneratedId_WhosePublishFails_SurfacesTheIdSoTheRunCanBeReDriven() + public async Task StartWithAGeneratedId_WhosePublishFails_SurfacesTheIdSoTheStartCanBeRetriedIdempotently() { var clock = new VirtualTimeProvider(); var transport = new FailingWorkerTransport(); @@ -198,18 +204,17 @@ public async Task StartWithAGeneratedId_WhosePublishFails_SurfacesTheIdSoTheRunC // The publish was actually retried, not given up on after one throw. Assert.True(transport.PublishAttempts > 1, $"expected the publish to be retried; saw {transport.PublishAttempts} attempt(s)"); - // The generated id survives the failure — this is what made the orphan unrecoverable. + // The generated id survives the failure, and — since round 35 — nothing was persisted + // under it: the publish is the commit point, so a failed start leaves no orphan behind. Assert.False(string.IsNullOrWhiteSpace(ex.FlowId)); - var orphan = await store.LoadAsync(ex.FlowId); - Assert.NotNull(orphan); - Assert.Equal(FlowRunStatus.Running, orphan!.Status); + Assert.Null(await store.LoadAsync(ex.FlowId)); - // And with the id, the documented recovery genuinely works: the same start re-enqueues the - // existing run rather than creating a second one. + // With the id, the documented retry is idempotent: it creates the run and publishes one job. transport.Fail = false; - var reDriven = await flows.StartAsync(new R27Input("acme"), ex.FlowId); + var retried = await flows.StartAsync(new R27Input("acme"), ex.FlowId); - Assert.Equal(ex.FlowId, reDriven); + Assert.Equal(ex.FlowId, retried); + Assert.Equal(FlowRunStatus.Running, (await store.LoadAsync(ex.FlowId))!.Status); lock (transport.Published) Assert.Single(transport.Published); } diff --git a/tests/AsyncResponse.Tests/Round34NewApiTests.cs b/tests/AsyncResponse.Tests/Round34NewApiTests.cs new file mode 100644 index 000000000..1930cc4bd --- /dev/null +++ b/tests/AsyncResponse.Tests/Round34NewApiTests.cs @@ -0,0 +1,236 @@ +using System.Diagnostics.Metrics; +using System.Reflection; +using AsyncResponse.DurableFlows.EFCore; +using AsyncResponse.DurableFlows.MySql; +using AsyncResponse.DurableFlows.Oracle; +using AsyncResponse.DurableFlows.PostgreSQL; +using AsyncResponse.DurableFlows.Sqlite; +using AsyncResponse.DurableFlows.SqlServer; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Xunit; + +namespace AsyncResponse.Tests; + +/// +/// Round-34 pins that use API introduced by the same fix pass (they cannot compile against the +/// pre-fix build; their red-on-old proof is the compile break). The behavior pins that DO compile +/// on the old build are in . +/// +public sealed class Round34NewApiTests +{ + private const string SharedTypeName = "AsyncResponse.DurableFlows.Internal.DurableFlowStoreShared"; + + // --------------------------------------------------------------------------------------------- + // F8 — the dedicated exception the ingress passes through. + + [Fact] + public void RecoveryCallbackFailedException_CarriesTheCorrelationIdAttemptsAndCause() + { + var cause = new TimeoutException("dependency down"); + + var ex = new RecoveryCallbackFailedException("corr-1", 4, cause); + + Assert.Equal("corr-1", ex.CorrelationId); + Assert.Equal(4, ex.Attempts); + Assert.Same(cause, ex.InnerException); + Assert.Contains("corr-1", ex.Message, StringComparison.Ordinal); + Assert.Contains("not acknowledged", ex.Message, StringComparison.Ordinal); + Assert.Equal(4, LostSubscriberCallbackDispatcher.FailureCallbackAttempts); + } + + // --------------------------------------------------------------------------------------------- + // F9 — the scheduler's re-drive knobs validate at registration. + + [Fact] + public void WithScheduledFlow_RejectsANonPositiveRedriveInterval_AndANegativeStartupWindow() + { + Assert.Throws(() => Register(o => o.RedriveInterval = TimeSpan.Zero)); + Assert.Throws(() => Register(o => o.RedriveInterval = TimeSpan.FromDays(60))); + Assert.Throws(() => Register(o => o.StartupRedriveWindow = TimeSpan.FromSeconds(-1))); + + // Zero disables the startup probe and is legal. + Register(o => o.StartupRedriveWindow = TimeSpan.Zero); + + var defaults = new ScheduledFlowOptions(); + Assert.Equal(TimeSpan.FromSeconds(30), defaults.RedriveInterval); + Assert.Equal(TimeSpan.FromHours(1), defaults.StartupRedriveWindow); + + static void Register(Action configure) + => new ServiceCollection() + .AddAsyncResponse() + .WithInMemoryChannel() + .WithInMemoryTransport() + .WithInMemoryDurableFlows() + .WithScheduledFlow("nightly", "0 6 * * *", occurrence => new ReportInput(occurrence), configure); + } + + // --------------------------------------------------------------------------------------------- + // F5 — the prune budget: option validation, batch draining, metrics, and failure logging. + + [Theory] + [MemberData(nameof(RelationalOptionTypes))] + public void PruneBudget_DefaultsToTwoSeconds_AndRejectsANegativeValue(Type optionsType) + { + var options = Activator.CreateInstance(optionsType)!; + var budget = optionsType.GetProperty("PruneBudget")!; + Assert.Equal(TimeSpan.FromSeconds(2), budget.GetValue(options)); + + // The relational stores validate through Options.Validate(); EF Core validates in its + // store constructor (its options type has no Validate method), pinned separately below. + if (optionsType.GetMethod("Validate", BindingFlags.Public | BindingFlags.Instance) is not { } validate) + return; + + MakeValid(options); + validate.Invoke(options, null); + budget.SetValue(options, TimeSpan.Zero); + validate.Invoke(options, null); + budget.SetValue(options, TimeSpan.FromSeconds(-1)); + var thrown = Assert.Throws(() => validate.Invoke(options, null)); + Assert.IsType(thrown.InnerException); + Assert.Contains("PruneBudget", thrown.InnerException!.Message, StringComparison.Ordinal); + + static void MakeValid(object options) + { + // The server stores validate a connection string alongside the budget; identifiers + // default valid. + options.GetType().GetProperty("ConnectionString")?.SetValue(options, "Server=localhost;Database=asyncresponse"); + } + } + + [Fact] + public void EFCorePruneBudget_NegativeValue_IsRejectedByTheStoreConstructor() + { + var thrown = Assert.Throws(() => new EFCoreFlowStateStore( + new ServiceCollection().BuildServiceProvider().GetRequiredService(), + Microsoft.Extensions.Options.Options.Create(new EFCoreDurableFlowOptions { PruneBudget = TimeSpan.FromSeconds(-1) }))); + Assert.Contains("PruneBudget", thrown.Message, StringComparison.Ordinal); + } + + public static TheoryData RelationalOptionTypes => + [ + typeof(EFCoreDurableFlowOptions), + typeof(MySqlDurableFlowOptions), + typeof(OracleDurableFlowOptions), + typeof(PostgreSqlDurableFlowOptions), + typeof(SqliteDurableFlowOptions), + typeof(SqlServerDurableFlowOptions) + ]; + + /// + /// The shared prune helper, reflected per provider assembly (each compiles its own copy): + /// full batches keep it draining, a short batch stops it, a zero budget keeps the historical + /// single batch, and the deleted rows land on the meter. + /// + [Theory] + [MemberData(nameof(RelationalOptionTypes))] + public async Task PruneQuietly_DrainsFullBatchesUntilAShortOne_AndCountsTheRows(Type providerOptionsType) + { + var pruneQuietly = PruneQuietlyMethod(providerOptionsType); + var batchSize = PruneBatchSize(providerOptionsType); + Assert.Equal(1000, batchSize); + + var batches = new Queue([batchSize, batchSize, 7, batchSize]); + var calls = 0; + var measurements = await CollectAsync(() => Quietly( + pruneQuietly, + () => { calls++; return Task.FromResult(batches.Dequeue()); }, + TimeSpan.FromMinutes(1), + $"probe-{providerOptionsType.Name}", + logger: null)); + + // Two full batches, then the short one ends the drain; the fourth is never requested. + Assert.Equal(3, calls); + var pruned = Assert.Single(measurements, m => m.Instrument == "asyncresponse.flow_state.pruned_rows" && Equals(m.Tags["provider"], $"probe-{providerOptionsType.Name}")); + Assert.Equal(2 * batchSize + 7, pruned.Value); + + // A zero budget runs exactly one batch even when it comes back full — and says so. + var logger = new CollectingLogger(); + calls = 0; + measurements = await CollectAsync(() => Quietly( + pruneQuietly, + () => { calls++; return Task.FromResult(batchSize); }, + TimeSpan.Zero, + $"zero-{providerOptionsType.Name}", + logger)); + Assert.Equal(1, calls); + Assert.Contains(measurements, m => m.Instrument == "asyncresponse.flow_state.prune_budget_exhausted" && Equals(m.Tags["provider"], $"zero-{providerOptionsType.Name}")); + Assert.Contains(logger.Messages, m => m.Contains("PruneBudget", StringComparison.Ordinal) && m.Contains("expired rows remaining", StringComparison.Ordinal)); + } + + /// A failed prune is contained (the create it rides on succeeds), but no longer silent. + [Theory] + [MemberData(nameof(RelationalOptionTypes))] + public async Task PruneQuietly_ContainsAFailure_ButLogsAndCountsIt_AndPropagatesCancellation(Type providerOptionsType) + { + var pruneQuietly = PruneQuietlyMethod(providerOptionsType); + var logger = new CollectingLogger(); + var provider = $"fail-{providerOptionsType.Name}"; + + var measurements = await CollectAsync(() => Quietly( + pruneQuietly, + () => throw new InvalidOperationException("deadlock victim"), + TimeSpan.FromSeconds(1), + provider, + logger)); + + Assert.Contains(measurements, m => m.Instrument == "asyncresponse.flow_state.prune_failures" && Equals(m.Tags["provider"], provider)); + var entry = Assert.Single(logger.Entries, e => e.Message.Contains("prune failed", StringComparison.Ordinal)); + Assert.IsType(entry.Exception); + Assert.Contains(provider, entry.Message, StringComparison.Ordinal); + + await Assert.ThrowsAsync(() => Quietly(pruneQuietly, () => throw new OperationCanceledException(), TimeSpan.Zero, provider, logger)); + await Assert.ThrowsAnyAsync(() => Quietly(pruneQuietly, () => Task.FromCanceled(new CancellationToken(canceled: true)), TimeSpan.Zero, provider, logger)); + } + + private static MethodInfo PruneQuietlyMethod(Type providerOptionsType) + { + var shared = providerOptionsType.Assembly.GetType(SharedTypeName, throwOnError: true)!; + var method = shared.GetMethod("PruneQuietlyAsync", BindingFlags.Public | BindingFlags.Static); + Assert.NotNull(method); + Assert.Equal( + [typeof(Func>), typeof(TimeSpan), typeof(string), typeof(ILogger)], + method!.GetParameters().Select(p => p.ParameterType).ToArray()); + return method; + } + + private static int PruneBatchSize(Type providerOptionsType) + { + var shared = providerOptionsType.Assembly.GetType(SharedTypeName, throwOnError: true)!; + var field = shared.GetField("PruneBatchSize", BindingFlags.Public | BindingFlags.Static); + Assert.NotNull(field); + return (int)field!.GetRawConstantValue()!; + } + + private static Task Quietly(MethodInfo pruneQuietly, Func> batch, TimeSpan budget, string provider, ILogger? logger) + => (Task)pruneQuietly.Invoke(null, [batch, budget, provider, logger])!; + + private sealed record Measurement(string Instrument, long Value, Dictionary Tags); + + private static async Task> CollectAsync(Func action) + { + var measurements = new List(); + using var listener = new MeterListener + { + InstrumentPublished = (instrument, l) => + { + if (instrument.Meter.Name == AsyncResponseDiagnostics.MeterName) + l.EnableMeasurementEvents(instrument); + } + }; + listener.SetMeasurementEventCallback((instrument, value, tags, _) => + { + var tagDict = new Dictionary(StringComparer.Ordinal); + foreach (var tag in tags) + tagDict[tag.Key] = tag.Value; + lock (measurements) + measurements.Add(new Measurement(instrument.Name, value, tagDict)); + }); + listener.Start(); + + await action(); + + lock (measurements) + return [.. measurements]; + } +} diff --git a/tests/AsyncResponse.Tests/Round34RegressionTests.cs b/tests/AsyncResponse.Tests/Round34RegressionTests.cs new file mode 100644 index 000000000..785a12001 --- /dev/null +++ b/tests/AsyncResponse.Tests/Round34RegressionTests.cs @@ -0,0 +1,775 @@ +using System.Collections.Concurrent; +using System.Text.Json; +using AsyncResponse.DurableFlows.Sqlite; +using AsyncResponse.Testing; +using Microsoft.Data.Sqlite; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Moq; +using Xunit; + +namespace AsyncResponse.Tests; + +/// +/// Regression pins for the round-34 review (2026-09-08). Every fact here compiles against the +/// pre-fix build and fails there: the file deliberately uses no new API (the new-API pins live in +/// ). +/// +public sealed class Round34RegressionTests +{ + // --------------------------------------------------------------------------------------------- + // F1 — the lease-less "won response" checkpoint must be fenced to the attempt that won it. + + /// + /// Round 34, finding 1: CheckpointReceivedWithoutLeaseAsync located its target by step + /// name and "not completed" only. After a lease loss a takeover may already have timed the + /// breadcrumb out and RE-TRIGGERED the step under a new correlation id — the reloaded step is + /// then pending on that new id, and the stale executor's response completed it with a payload + /// that answers the OLD request. The takeover skipped the step and consumed the wrong result. + /// Pre-fix failure: the persisted step is completed with the stale response; post-fix it stays + /// pending on the takeover's id and the stale response is discarded. + /// + [Fact] + public async Task AwaitStep_LeaseLostAfterATakeoverReTriggered_DoesNotCompleteTheNewerAttemptWithTheStaleResponse() + { + var store = new TakeoverStore("remote", takeover: state => + { + var step = state.Steps!["remote"]; + step.PendingCorrelationId = "takeover-cid"; + step.PendingPayloadTypeFullName = typeof(OperationResult).FullName; + step.Completed = false; + step.ResultJson = null; + state.LastMessage = "re-triggered by the takeover"; + }); + var state = new FlowState { FlowId = "r34-stale-response-vs-takeover" }; + Assert.True(await store.TryCreateAsync(state.FlowId!, state, TimeSpan.FromMinutes(5))); + + var stale = new OperationResult { Status = OperationStatus.Completed, Message = "answer-to-the-OLD-request" }; + await using (var lease = await AcquireLeaseAsync(store, state.FlowId!)) + { + var context = CreateContext(state, store, SubscriberReturning(Task.FromResult(stale)), lease); + var surfaced = await Assert.ThrowsAsync(() => context.AwaitStepAsync( + "remote", + _ => Task.CompletedTask)); + Assert.Contains("lost its execution lease", surfaced.Message, StringComparison.Ordinal); + } + + Assert.Equal(1, store.Takeovers); + var persisted = await store.LoadAsync(state.FlowId!); + Assert.NotNull(persisted); + var step = persisted!.Steps!["remote"]; + Assert.False(step.Completed); + Assert.Equal("takeover-cid", step.PendingCorrelationId); + Assert.Null(step.ResultJson); + Assert.Equal("re-triggered by the takeover", persisted.LastMessage); + } + + /// + /// Round 34, finding 1 (second shape): the same unfenced write also mutated a run the takeover + /// had already FAILED — a terminal ledger gained a completed step and a new LastMessage. + /// Pre-fix failure: the Failed run's step is completed and its message rewritten. + /// + [Fact] + public async Task AwaitStep_LeaseLostAfterTheRunWasFailed_LeavesTheTerminalLedgerUntouched() + { + var store = new TakeoverStore("remote", takeover: state => + { + state.Status = FlowRunStatus.Failed; + state.LastMessage = "failed by the takeover"; + }); + var state = new FlowState { FlowId = "r34-stale-response-vs-failed-run" }; + Assert.True(await store.TryCreateAsync(state.FlowId!, state, TimeSpan.FromMinutes(5))); + + var stale = new OperationResult { Status = OperationStatus.Completed, Message = "late" }; + await using (var lease = await AcquireLeaseAsync(store, state.FlowId!)) + { + var context = CreateContext(state, store, SubscriberReturning(Task.FromResult(stale)), lease); + await Assert.ThrowsAsync(() => context.AwaitStepAsync("remote", _ => Task.CompletedTask)); + } + + var persisted = await store.LoadAsync(state.FlowId!); + Assert.NotNull(persisted); + Assert.Equal(FlowRunStatus.Failed, persisted!.Status); + Assert.Equal("failed by the takeover", persisted.LastMessage); + Assert.False(persisted.Steps!["remote"].Completed); + Assert.Null(persisted.Steps["remote"].ResultJson); + } + + /// + /// Rejects the FIRST lease-fenced completion save for stepName exactly as a store answers + /// a lost lease — and, before answering, applies to the persisted + /// ledger through a lease-less write, so the rescue's reload sees what a real takeover wrote. + /// + private sealed class TakeoverStore(string stepName, Action takeover) : IFlowStateStore + { + private readonly InMemoryFlowStateStore _inner = new(); + private int _takeovers; + + public int Takeovers => Volatile.Read(ref _takeovers); + + public Task TryCreateAsync(string flowId, FlowState state, TimeSpan ttl, CancellationToken cancellationToken = default) + => _inner.TryCreateAsync(flowId, state, ttl, cancellationToken); + + public Task LoadAsync(string flowId, CancellationToken cancellationToken = default) + => _inner.LoadAsync(flowId, cancellationToken); + + public async Task TryUpdateAsync( + string flowId, + FlowState state, + long expectedRevision, + TimeSpan ttl, + string? leaseId = null, + CancellationToken cancellationToken = default) + { + if (leaseId is not null + && Volatile.Read(ref _takeovers) == 0 + && state.Steps is { } steps + && steps.TryGetValue(stepName, out var step) + && step.Completed) + { + Interlocked.Increment(ref _takeovers); + var current = await _inner.LoadAsync(flowId, cancellationToken) + ?? throw new InvalidOperationException("The ledger under test vanished."); + takeover(current); + var revision = current.Revision; + current.Revision = revision + 1; + Assert.True(await _inner.TryUpdateAsync(flowId, current, revision, ttl, leaseId: null, cancellationToken)); + return false; + } + + return await _inner.TryUpdateAsync(flowId, state, expectedRevision, ttl, leaseId, cancellationToken); + } + + public Task TryAcquireLeaseAsync(string flowId, string leaseId, TimeSpan leaseDuration, CancellationToken cancellationToken = default) + => _inner.TryAcquireLeaseAsync(flowId, leaseId, leaseDuration, cancellationToken); + + public Task TryRenewLeaseAsync(string flowId, string leaseId, TimeSpan leaseDuration, CancellationToken cancellationToken = default) + => _inner.TryRenewLeaseAsync(flowId, leaseId, leaseDuration, cancellationToken); + + public Task ReleaseLeaseAsync(string flowId, string leaseId, CancellationToken cancellationToken = default) + => _inner.ReleaseLeaseAsync(flowId, leaseId, cancellationToken); + + public Task TryDeleteAsync(string flowId, CancellationToken cancellationToken = default) + => _inner.TryDeleteAsync(flowId, cancellationToken); + } + + // --------------------------------------------------------------------------------------------- + // F2 — worker-argument conversion must not quote the payload in the exception it throws. + + private const string PrivateMarker = "SYNTHETIC_PRIVATE_MARKER"; + + /// + /// Round 34, finding 2: the envelope parse was sanitized, but converting a JsonElement argument + /// into the callback's parameter type used the raw serializer, whose JsonException carries + /// Path: $.<key> — dictionary keys read straight off the wire. Pre-fix failure: + /// the marker (a payload key) is in the exception chain. + /// + [Theory] + [InlineData(true)] + [InlineData(false)] + public void As_ConversionFailure_NeverQuotesThePayload(bool asElement) + { + var json = $$"""{"{{PrivateMarker}}":"not-an-int"}"""; + object value = asElement ? JsonSerializer.Deserialize(json)! : json; + + var thrown = Assert.ThrowsAny(() => value.As>()); + + for (var ex = thrown; ex is not null; ex = ex.InnerException) + { + Assert.DoesNotContain(PrivateMarker, ex.Message, StringComparison.Ordinal); + Assert.DoesNotContain("not-an-int", ex.Message, StringComparison.Ordinal); + } + + // Still diagnosable: size and reader position survive. + Assert.Contains("code units", thrown.Message, StringComparison.Ordinal); + Assert.Contains("byte position", thrown.Message, StringComparison.Ordinal); + } + + public interface IDictionaryWorker + { + Task Take(Dictionary values); + } + + private sealed class DictionaryWorker : IDictionaryWorker + { + public Task Take(Dictionary values) => Task.CompletedTask; + } + + /// + /// The path the review reproduced end to end: a worker envelope whose argument cannot convert + /// to the parameter type, executed through the ingress, which logs the failure with its + /// exception. Pre-fix failure: the logged exception chain names the payload's key. + /// + [Fact] + public async Task HandleWorkerMessageAsync_ArgumentConversionFailure_NeverLogsThePayloadKeys() + { + var logger = new CollectingLogger(); + var services = new ServiceCollection(); + services.AddSingleton(typeof(ILogger<>), typeof(NullLogger<>)); + services.AddSingleton>(logger.For()); + services.AddSingleton(new DictionaryWorker()); + services.AddAsyncResponse().WithInMemoryChannel(); + await using var provider = services.BuildServiceProvider(); + var ingress = provider.GetRequiredService(); + + var envelope = AsyncResponseJson.Serialize(new WorkerJobEnvelope + { + CorrelationId = "corr-r34-argument", + Call = new ReflectionCallDto + { + ServiceInterfaceFullName = typeof(IDictionaryWorker).FullName!, + MethodName = nameof(IDictionaryWorker.Take), + Params = [CallbackParam.ForValue(new Dictionary { [PrivateMarker] = "not-an-int" })] + } + }); + + // Deterministic, so it propagates for the transport's bounded redelivery — unchanged. + await Assert.ThrowsAnyAsync(() => ingress.HandleWorkerMessageAsync(envelope)); + + Assert.NotEmpty(logger.Entries); + foreach (var (message, exception) in logger.Entries) + { + Assert.DoesNotContain(PrivateMarker, message, StringComparison.Ordinal); + for (var ex = exception; ex is not null; ex = ex.InnerException) + Assert.DoesNotContain(PrivateMarker, ex.Message, StringComparison.Ordinal); + } + } + + // --------------------------------------------------------------------------------------------- + // F4 — expired in-memory ledgers must be swept without anyone loading their ids again. + + /// + /// Round 34, finding 4: the in-memory store dropped an expired entry only when THAT id was + /// loaded or replaced, which a completed run's id never is again — a long-lived process with + /// unique flow ids retained every expired ledger for its lifetime. Pre-fix failure: 1001 + /// entries after a create that follows 1000 expired ledgers by a month. + /// + [Fact] + public async Task InMemoryFlowStateStore_SweepsExpiredLedgersOnCreate_WithoutLoadingThem() + { + var clock = new VirtualTimeProvider(); + var store = new InMemoryFlowStateStore(clock); + for (var i = 0; i < 1000; i++) + Assert.True(await store.TryCreateAsync($"expired-{i}", new FlowState { FlowId = $"expired-{i}" }, TimeSpan.FromMinutes(1))); + Assert.Equal(1000, EntryCount(store)); + + clock.Advance(TimeSpan.FromDays(30)); + Assert.True(await store.TryCreateAsync("fresh", new FlowState { FlowId = "fresh" }, TimeSpan.FromMinutes(1))); + + Assert.Equal(1, EntryCount(store)); + Assert.NotNull(await store.LoadAsync("fresh")); + } + + /// Retained entries, read off the private dictionary so this fact compiles against the pre-fix build. + private static int EntryCount(InMemoryFlowStateStore store) + { + var entries = typeof(InMemoryFlowStateStore) + .GetField("_entries", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance)! + .GetValue(store); + return ((System.Collections.ICollection)entries!).Count; + } + + // --------------------------------------------------------------------------------------------- + // F6 — memoized child snapshots must not compound with nesting depth. + + /// + /// Round 34, finding 6: a child snapshot is stored as a JSON STRING inside the parent's ledger, + /// so every ancestor level re-escapes the level below; carrying the child's own memoized + /// grandchild snapshots along made the ledger grow exponentially with depth (a 72-byte leaf + /// reached ~600 KB at depth 15 with no business payload). Descendant snapshots are now elided + /// from the memo (the grandchild's own ledger keeps them). Pre-fix failure: the depth-15 + /// snapshot is hundreds of kilobytes. + /// + [Fact] + public void SerializeSnapshot_ElidesDescendantSnapshots_SoNestingDepthDoesNotCompoundSize() + { + var snapshot = FlowStateJson.SerializeSnapshot(new FlowState { FlowId = "leaf", Status = FlowRunStatus.Succeeded }); + var depthOne = 0; + for (var depth = 1; depth <= 15; depth++) + { + var parent = new FlowState + { + FlowId = $"depth-{depth}", + Status = FlowRunStatus.Succeeded, + Steps = new Dictionary(StringComparer.Ordinal) + { + ["child"] = new() { Completed = true, ChildFlowId = $"depth-{depth - 1}", ResultJson = snapshot } + } + }; + snapshot = FlowStateJson.SerializeSnapshot(parent); + if (depth == 1) + depthOne = snapshot.Length; + } + + // Depth-independent up to the id digits (pre-fix: ~600 KB at depth 15 against ~150 bytes). + Assert.True(snapshot.Length < 2 * depthOne, $"depth 15 snapshot is {snapshot.Length} bytes; depth 1 was {depthOne}"); + } + + /// + /// What the memo must still carry: the child's outcome, its local step results, and the + /// elided step's identity (id, completion, fault marker) — and the instance handed in is + /// restored, because the caller returns it to flow code. + /// + [Fact] + public void SerializeSnapshot_KeepsLocalResultsAndChildIdentity_AndRestoresTheInstance() + { + var child = new FlowState + { + FlowId = "child", + Status = FlowRunStatus.Failed, + LastMessage = "child failed", + Context = new Dictionary { ["tenant"] = "acme" }, + Steps = new Dictionary(StringComparer.Ordinal) + { + ["local"] = new() { Completed = true, ResultJson = "\"local-result\"" }, + ["grandchild"] = new() { Completed = true, Faulted = true, ChildFlowId = "child:grandchild", ResultJson = "{\"FlowId\":\"child:grandchild\"}" } + } + }; + + var memo = FlowStateJson.Deserialize(FlowStateJson.SerializeSnapshot(child), "child"); + + Assert.Equal(FlowRunStatus.Failed, memo.Status); + Assert.Equal("child failed", memo.LastMessage); + Assert.Null(memo.Context); + Assert.Equal("\"local-result\"", memo.Steps!["local"].ResultJson); + var elided = memo.Steps["grandchild"]; + Assert.True(elided.Completed); + Assert.True(elided.Faulted); + Assert.Equal("child:grandchild", elided.ChildFlowId); + Assert.Null(elided.ResultJson); + + // Restored: the flow code that receives this instance still sees everything. + Assert.Equal("{\"FlowId\":\"child:grandchild\"}", child.Steps["grandchild"].ResultJson); + Assert.Equal("acme", child.Context["tenant"]); + } + + // --------------------------------------------------------------------------------------------- + // F8 — a failure callback that keeps failing transiently must NOT acknowledge the response. + + private const string ExhaustedCorrelationId = "r34-failure-callback-exhausted"; + + /// + /// Round 34, finding 8: after its four in-process attempts the failure callback's fault was + /// swallowed and the publish returned normally — the transport acknowledged a TERMINAL signal + /// that then existed nowhere (the registration keeps the callback, not the payload; the + /// watchdog only reports). The exhausted transient fault now propagates as a dedicated type so + /// the transport redelivers under its own bounded policy. Pre-fix failure: SetResponse + /// completes normally. + /// + [Fact] + public async Task SetResponse_FailureCallbackExhaustsTransientRetries_ThrowsInsteadOfAcknowledging() + { + var time = new VirtualTimeProvider(); + var spy = new AlwaysThrowingFailureSpy(); + await using var provider = BuildProvider(time, spy); + var recoveryStateStore = provider.GetRequiredService(); + await RegisterFailureCallbackAsync(recoveryStateStore, ExhaustedCorrelationId); + + var publisher = provider.GetRequiredService(); + var dispatching = publisher.SetResponse( + new OperationResult { Status = OperationStatus.Failed, Message = "remote step failed" }, + ExhaustedCorrelationId); + + var thrown = await Assert.ThrowsAnyAsync(() => DriveBackoffAsync(time, dispatching)); + + Assert.Equal("RecoveryCallbackFailedException", thrown.GetType().Name); + Assert.Contains(ExhaustedCorrelationId, thrown.Message, StringComparison.Ordinal); + Assert.Equal(4, spy.Calls); + // The registration stays armed for the redelivery. + Assert.Single(await recoveryStateStore.GetAllAsync(ExhaustedCorrelationId)); + } + + /// + /// The ingress half of finding 8: the broker ingress must neither re-run its own retry ladder + /// over the dispatcher's (four attempts, not sixteen) nor escalate through SetException (which + /// would only invoke the same failing callback again) — it passes the fault to the transport. + /// Pre-fix failure: HandleResponseMessageAsync returns normally (the message would be ACKed). + /// + [Fact] + public async Task HandleResponseMessageAsync_FailureCallbackExhausted_PropagatesForRedelivery_WithoutSetException() + { + var time = new VirtualTimeProvider(); + var spy = new AlwaysThrowingFailureSpy(); + await using var provider = BuildProvider(time, spy); + var recoveryStateStore = provider.GetRequiredService(); + const string correlationId = ExhaustedCorrelationId + "-ingress"; + await RegisterFailureCallbackAsync(recoveryStateStore, correlationId); + + var ingress = provider.GetRequiredService(); + var handling = ingress.HandleResponseMessageAsync( + AsyncResponseJson.Serialize(new OperationResult { Status = OperationStatus.Failed, Message = "remote step failed" }), + correlationId); + + var thrown = await Assert.ThrowsAnyAsync(() => DriveBackoffAsync(time, handling)); + + Assert.Equal("RecoveryCallbackFailedException", thrown.GetType().Name); + Assert.Equal(4, spy.Calls); + Assert.Single(await recoveryStateStore.GetAllAsync(correlationId)); + } + + /// + /// The deliberately unchanged half: a DETERMINISTIC callback fault (the target interface is not + /// registered) is still swallowed and acknowledged — redelivery cannot fix it, and RabbitMQ's + /// unbounded default would hot-loop it — with the registration kept for the watchdog. + /// + [Fact] + public async Task SetResponse_FailureCallbackWithUnresolvableTarget_StillAcknowledges() + { + var time = new VirtualTimeProvider(); + await using var provider = BuildProvider(time, spy: null); + var recoveryStateStore = provider.GetRequiredService(); + const string correlationId = ExhaustedCorrelationId + "-unresolvable"; + await RegisterFailureCallbackAsync(recoveryStateStore, correlationId); + + var publisher = provider.GetRequiredService(); + await publisher.SetResponse(new OperationResult { Status = OperationStatus.Failed }, correlationId) + .WaitAsync(TimeSpan.FromSeconds(10)); + + Assert.Single(await recoveryStateStore.GetAllAsync(correlationId)); + } + + private static ServiceProvider BuildProvider(VirtualTimeProvider time, IAlwaysThrowingFailureSpy? spy) + { + var services = new ServiceCollection(); + services.AddSingleton(typeof(ILogger<>), typeof(NullLogger<>)); + services.AddSingleton(time); + if (spy is not null) + services.AddSingleton(spy); + services.AddAsyncResponse().WithInMemoryChannel(); + return services.BuildServiceProvider(); + } + + private static Task RegisterFailureCallbackAsync(IRecoveryStateStore recoveryStateStore, string correlationId) + => recoveryStateStore.SaveAsync( + correlationId, + new RecoveryState + { + RegistrationId = Guid.NewGuid(), + CorrelationId = correlationId, + PayloadTypeFullName = typeof(OperationResult).FullName, + RegisteredAtUtc = DateTime.UtcNow, + FailureCallback = new ReflectionCallDto + { + ServiceInterfaceFullName = typeof(IAlwaysThrowingFailureSpy).FullName!, + MethodName = nameof(IAlwaysThrowingFailureSpy.OnFailure), + Params = [CallbackParam.ForPlaceholder(PlaceholderType.Exception)] + } + }, + TimeSpan.FromMinutes(5)); + + /// + /// Fires every backoff timer the retry ladder arms on the virtual clock until + /// settles (bounded, so a ladder that never ends fails the test + /// instead of hanging it), then awaits the operation's outcome. + /// + private static async Task DriveBackoffAsync(VirtualTimeProvider time, Task operation) + { + for (var round = 0; round < 12 && !operation.IsCompleted; round++) + { + var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(5); + while (!operation.IsCompleted && time.NextTimerDueAt is null) + { + if (DateTime.UtcNow > deadline) + throw new TimeoutException("the retry ladder armed no backoff timer on the virtual clock"); + await Task.Delay(10); + } + + if (!operation.IsCompleted) + time.Advance(TimeSpan.FromSeconds(2)); + } + + await operation.WaitAsync(TimeSpan.FromSeconds(10)); + } + + public interface IAlwaysThrowingFailureSpy + { + Task OnFailure(Exception exception); + } + + private sealed class AlwaysThrowingFailureSpy : IAlwaysThrowingFailureSpy + { + private int _calls; + + public int Calls => Volatile.Read(ref _calls); + + public Task OnFailure(Exception exception) + { + Interlocked.Increment(ref _calls); + return Task.FromException(new InvalidOperationException("dependency still down")); + } + } + + // --------------------------------------------------------------------------------------------- + // F9 — a scheduled occurrence whose ledger is committed but whose job was not published must + // be re-driven, in-process and across a restart. + + /// + /// Round 34, finding 9: when the worker-job publish failed, the scheduler logged the + /// occurrence and advanced — nothing ever retried it. It is now re-driven every + /// RedriveInterval until published. Pre-fix failure: exactly one start attempt, and the + /// second never comes. The fake models the round-35 publish-first start: a failed publish + /// persists NOTHING (round 36 — the re-drive used to read that absent ledger as "expired" + /// and give up), and the successful re-publish is what creates the ledger. + /// + [Fact] + public async Task ScheduledFlow_UndispatchedOccurrence_IsRedrivenUntilItsJobIsPublished() + { + var time = new VirtualTimeProvider(new DateTimeOffset(2030, 1, 1, 0, 0, 30, TimeSpan.Zero)); + var flows = new FakeFlows(); + const string occurrenceId = "sched:hourly:20300101T010000Z"; + var failuresLeft = 1; + flows.OnStart = flowId => + { + // Publish-first: a start whose publish fails leaves no ledger behind; only the + // successful publish (the re-drive) creates one. + if (Interlocked.Decrement(ref failuresLeft) >= 0) + return new DurableFlowNotDispatchedException(flowId, new TimeoutException("broker down")); + + flows.States[flowId] = new FlowState { FlowId = flowId, Status = FlowRunStatus.Running, Attempts = 0 }; + return null; + }; + + using var scheduler = new ScheduledFlowService(flows, [HourlyRegistration()], NullLogger.Instance, time); + await scheduler.StartAsync(CancellationToken.None); + try + { + // Each advance waits for the loop to arm its sleep on the virtual clock first: an + // advance that lands between the loop reading "now" and arming would leave a timer + // aimed an hour past the occurrence. + // Land a few seconds past the 01:00 occurrence — short of the 30-second re-drive. + await WaitForArmedTimerAsync(time); + time.Advance(TimeSpan.FromMinutes(59) + TimeSpan.FromSeconds(35)); + await flows.WaitForStartsAsync(1); + Assert.Equal([occurrenceId], flows.Starts.ToArray()); + + // Well before the NEXT occurrence (02:00): the re-drive fires on RedriveInterval. + await WaitForArmedTimerAsync(time); + time.Advance(TimeSpan.FromSeconds(30)); + await flows.WaitForStartsAsync(2); + Assert.Equal([occurrenceId, occurrenceId], flows.Starts.ToArray()); + + // Settled: no third attempt after the successful re-publish. + await WaitForArmedTimerAsync(time); + time.Advance(TimeSpan.FromMinutes(5)); + await Task.Delay(100); + Assert.Equal(2, flows.Starts.Count); + } + finally + { + await scheduler.StopAsync(CancellationToken.None); + } + } + + /// + /// The cross-restart half of finding 9: an in-process re-drive queue dies with its process, + /// and a crash between the ledger commit and the publish never reached it. At startup the loop + /// probes recent occurrences and re-drives any whose ledger is Running with zero attempts. + /// Pre-fix failure: the committed, never-executed occurrence is never started again. + /// + [Fact] + public async Task ScheduledFlow_StartupProbe_RedrivesACommittedNeverExecutedOccurrence() + { + var time = new VirtualTimeProvider(new DateTimeOffset(2030, 1, 1, 0, 0, 30, TimeSpan.Zero)); + var flows = new FakeFlows(); + // Both inside the default one-hour window of a half-hourly schedule started at 00:00:30. + const string undispatched = "sched:half-hourly:20300101T000000Z"; + const string executed = "sched:half-hourly:20291231T233000Z"; + flows.States[undispatched] = new FlowState { FlowId = undispatched, Status = FlowRunStatus.Running, Attempts = 0 }; + flows.States[executed] = new FlowState { FlowId = executed, Status = FlowRunStatus.Running, Attempts = 1 }; + + using var scheduler = new ScheduledFlowService(flows, [Registration("half-hourly", "*/30 * * * *")], NullLogger.Instance, time); + await scheduler.StartAsync(CancellationToken.None); + try + { + await flows.WaitForStartsAsync(1); + Assert.Equal([undispatched], flows.Starts.ToArray()); + + // The executed one (attempts > 0) and the ledger-less ones are left alone. + await Task.Delay(100); + Assert.Single(flows.Starts); + } + finally + { + await scheduler.StopAsync(CancellationToken.None); + } + } + + private static async Task WaitForArmedTimerAsync(VirtualTimeProvider time) + { + var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(10); + while (time.NextTimerDueAt is null) + { + if (DateTime.UtcNow > deadline) + throw new TimeoutException("the scheduler loop armed no sleep on the virtual clock"); + await Task.Delay(10); + } + } + + private static ScheduledFlowRegistration HourlyRegistration() => Registration("hourly", "0 * * * *"); + + private static ScheduledFlowRegistration Registration(string name, string cron) => new() + { + Name = name, + CronExpression = cron, + Options = new ScheduledFlowOptions(), + StartOccurrenceAsync = static (flows, flowId, occurrence, cancellationToken) => + flows.StartAsync(new ReportInput(occurrence), flowId, cancellationToken) + }; + + private sealed class FakeFlows : IDurableFlows + { + public ConcurrentDictionary States { get; } = new(StringComparer.Ordinal); + + public ConcurrentQueue Starts { get; } = new(); + + /// Returns the exception a start should throw, or null to succeed. + public Func OnStart { get; set; } = _ => null; + + public Task StartAsync(TInput input, string? flowId = null, CancellationToken cancellationToken = default) + where TFlow : class, IDurableFlow + { + Starts.Enqueue(flowId!); + if (OnStart(flowId!) is { } failure) + throw failure; + return Task.FromResult(flowId!); + } + + public Task ResumeAsync(string flowId, CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task GetStateAsync(string flowId, CancellationToken cancellationToken = default) + => Task.FromResult(States.TryGetValue(flowId, out var state) ? state : null); + + public async Task WaitForStartsAsync(int count) + { + var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(10); + while (Starts.Count < count) + { + if (DateTime.UtcNow > deadline) + throw new TimeoutException($"Expected {count} start(s); saw [{string.Join(", ", Starts)}]."); + await Task.Delay(10); + } + } + } + + // --------------------------------------------------------------------------------------------- + // F5 — the relational prune must drain a backlog, not one batch per interval. + + /// + /// Round 34, finding 5, proven against the real in-process SQLite store: one 1000-row batch per + /// prune interval capped cleanup at ~3 rows/second, which any busier instance outgrew forever. + /// The prune now drains batches until one comes back short or the budget lapses. Pre-fix + /// failure: 1200 of 2200 expired rows survive the create that pruned. + /// + [Fact] + public async Task SqliteTryCreate_DrainsTheWholeExpiredBacklog_NotOneBatch() + { + await using var database = new TempSqlite(); + var options = new SqliteDurableFlowOptions + { + ConnectionString = database.ConnectionString, + // Seed without pruning: the first create consumes the interval, the rest ride it. + PruneInterval = TimeSpan.FromHours(1) + }; + var store = new SqliteFlowStateStore(Options.Create(options)); + + const int backlog = 2200; + for (var i = 0; i < backlog; i++) + Assert.True(await store.TryCreateAsync($"expired-{i}", NewState($"expired-{i}"), TimeSpan.FromMilliseconds(1))); + await Task.Delay(50); + Assert.Equal(backlog, await database.CountExpiredAsync()); + + // The same options instance the store holds: the next create prunes. + options.PruneInterval = TimeSpan.Zero; + Assert.True(await store.TryCreateAsync("fresh", NewState("fresh"), TimeSpan.FromMinutes(5))); + + Assert.Equal(0, await database.CountExpiredAsync()); + Assert.Equal("fresh", (await store.LoadAsync("fresh"))?.FlowId); + } + + private static FlowState NewState(string flowId) => new() + { + FlowId = flowId, + FlowTypeName = "PruneBacklogFlow", + Status = FlowRunStatus.Running, + Steps = [] + }; + + private sealed class TempSqlite : IAsyncDisposable + { + private readonly string _path = Path.Combine(Path.GetTempPath(), $"ar-prune-backlog-{Guid.NewGuid():N}.db"); + + public string ConnectionString => $"Data Source={_path}"; + + public async Task CountExpiredAsync() + { + await using var connection = new SqliteConnection(ConnectionString); + await connection.OpenAsync(); + await using var command = connection.CreateCommand(); + command.CommandText = """SELECT COUNT(*) FROM "asyncresponse_flow_state" WHERE flow_id LIKE 'expired-%';"""; + return (long)(await command.ExecuteScalarAsync())!; + } + + public ValueTask DisposeAsync() + { + SqliteConnection.ClearPool(new SqliteConnection(ConnectionString)); + foreach (var suffix in new[] { "", "-wal", "-shm" }) + { + try + { + File.Delete(_path + suffix); + } + catch (IOException) + { + } + } + + return ValueTask.CompletedTask; + } + } + + // --------------------------------------------------------------------------------------------- + // Shared helpers (mirroring Round33RegressionTests). + + private static DurableFlowContext CreateContext( + FlowState state, + IFlowStateStore store, + IAsyncResponseSubscriber subscriber, + FlowExecutionLease lease) + => new( + state, + store, + Mock.Of(), + new AsyncResponseContextPropagation([]), + new DurableFlowOptions(), + subscriber, + null, + NullLogger.Instance, + lease); + + private static async Task AcquireLeaseAsync(IFlowStateStore store, string flowId) + { + var lease = await FlowStateConcurrency.TryAcquireExecutionLeaseAsync( + store, + flowId, + new DurableFlowOptions(), + NullLogger.Instance); + return Assert.IsType(lease); + } + + private static IAsyncResponseSubscriber SubscriberReturning(Task responseTask) + { + var waiter = new Mock>(); + waiter.SetupGet(instance => instance.ResponseTask).Returns(responseTask); + waiter.Setup(instance => instance.DisposeAsync()).Returns(ValueTask.CompletedTask); + + var subscriber = new Mock(); + subscriber.Setup(instance => instance.CreateResponseWaiter( + It.IsAny(), + It.IsAny>?>(), + It.IsAny())) + .ReturnsAsync(waiter.Object); + return subscriber.Object; + } +} diff --git a/tests/AsyncResponse.Tests/Round35NewApiTests.cs b/tests/AsyncResponse.Tests/Round35NewApiTests.cs new file mode 100644 index 000000000..780fb2da4 --- /dev/null +++ b/tests/AsyncResponse.Tests/Round35NewApiTests.cs @@ -0,0 +1,240 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Xunit; + +namespace AsyncResponse.Tests; + +/// +/// Round 35 pins over API this round introduced — IDurableFlowExecutor.CreateAndExecuteAsync, +/// DurableFlowOptions.LedgerSizeWarningBytes, FlowStateJson.EstimateLedgerChars, and +/// SerialExecutorRegistry.TryEnqueue. They are compile-level red on the pre-fix tree, so +/// they live apart from , whose behavior pins are copied onto +/// the old source for the red-on-old proof. +/// +public sealed class Round35NewApiTests +{ + private static ServiceProvider BuildFlowProvider(IWorkerTransport transport) + { + var services = new ServiceCollection(); + services.AddSingleton(typeof(ILogger<>), typeof(NullLogger<>)); + services + .AddAsyncResponse() + .WithInMemoryChannel() + .WithInMemoryDurableFlows() + .WithDurableFlow(); + services.AddSingleton(transport); + return services.BuildServiceProvider(); + } + + private sealed class NullWorkerTransport : IWorkerTransport + { + public Task PublishAsync(WorkerJobEnvelope job, CancellationToken cancellationToken = default) => Task.CompletedTask; + } + + private static FlowState InitialState(string flowId, string inputJson) => new() + { + FlowId = flowId, + FlowTypeName = typeof(Round35RegressionTests.R35MarkerFlow).FullName, + InputTypeName = typeof(Round35RegressionTests.R35Input).FullName, + InputJson = inputJson, + Status = FlowRunStatus.Running, + LastMessage = "Flow started.", + CreatedAtUtc = DateTime.UtcNow, + UpdatedAtUtc = DateTime.UtcNow + }; + + // --------------------------------------------------------------------------------------------- + // A1 — the executor-side half of the start's idempotency contract. + + /// A start job for an id already bound to different work is dropped, loudly, without touching the existing ledger. + [Fact] + public async Task CreateAndExecute_WhenTheIdIsBoundToDifferentWork_DropsTheJobAndLeavesTheLedgerAlone() + { + await using var provider = BuildFlowProvider(new NullWorkerTransport()); + var executor = provider.GetRequiredService(); + var store = provider.GetRequiredService(); + + const string id = "flow-r35-conflict"; + Assert.True(await store.TryCreateAsync(id, InitialState(id, """{"Name":"first"}"""), TimeSpan.FromDays(1))); + + var before = Volatile.Read(ref Round35RegressionTests.R35MarkerFlow.Executions); + await executor.CreateAndExecuteAsync(id, FlowStateJson.Serialize(InitialState(id, """{"Name":"second"}"""))); + + Assert.Equal(before, Volatile.Read(ref Round35RegressionTests.R35MarkerFlow.Executions)); + var untouched = await store.LoadAsync(id); + Assert.NotNull(untouched); + Assert.Equal("""{"Name":"first"}""", untouched!.InputJson); + Assert.Equal(0, untouched.Attempts); + Assert.Equal(FlowRunStatus.Running, untouched.Status); + } + + /// The same start already persisted (a redelivery, or the starter's create winning) is simply executed. + [Fact] + public async Task CreateAndExecute_WhenTheSameStartAlreadyExists_ExecutesTheExistingRun() + { + await using var provider = BuildFlowProvider(new NullWorkerTransport()); + var executor = provider.GetRequiredService(); + var store = provider.GetRequiredService(); + + const string id = "flow-r35-same-start"; + Assert.True(await store.TryCreateAsync(id, InitialState(id, """{"Name":"acme"}"""), TimeSpan.FromDays(1))); + + var before = Volatile.Read(ref Round35RegressionTests.R35MarkerFlow.Executions); + // Semantically identical input (different formatting) is the same start. + await executor.CreateAndExecuteAsync(id, FlowStateJson.Serialize(InitialState(id, """{ "Name" : "acme" }"""))); + + Assert.Equal(before + 1, Volatile.Read(ref Round35RegressionTests.R35MarkerFlow.Executions)); + Assert.Equal(FlowRunStatus.Succeeded, (await store.LoadAsync(id))!.Status); + } + + /// A carrier for a different id than the job names is refused (and the throw propagates to the transport). + [Fact] + public async Task CreateAndExecute_WithACarrierForAnotherId_Throws() + { + await using var provider = BuildFlowProvider(new NullWorkerTransport()); + var executor = provider.GetRequiredService(); + + await Assert.ThrowsAsync( + () => executor.CreateAndExecuteAsync("flow-r35-a", FlowStateJson.Serialize(InitialState("flow-r35-b", """{"Name":"x"}""")))); + } + + // --------------------------------------------------------------------------------------------- + // P2 — every checkpoint rewrites the whole ledger, so persistence cost grows with each completed + // step until the store's hard cap. An early warning at a configurable estimated size, once + // and then per doubling. + + public sealed record ChattyInput(string Name); + + public sealed class ChattyFlow : IDurableFlow + { + public async Task ExecuteAsync(IDurableFlowContext flow, ChattyInput input) + { + for (var i = 0; i < 6; i++) + await flow.StepAsync($"step-{i}", () => Task.FromResult(new string('x', 1024))); + } + } + + [Fact] + public async Task LedgerGrowth_PastTheWarningThreshold_IsLoggedOnceThenPerDoubling() + { + var logger = new CollectingLogger(); + var services = new ServiceCollection(); + services.AddSingleton(typeof(ILogger<>), typeof(NullLogger<>)); + services.AddSingleton>(logger.For()); + services + .AddAsyncResponse() + .WithInMemoryChannel() + .WithInMemoryTransport() + .WithInMemoryDurableFlows(options => options.LedgerSizeWarningBytes = 2048) + .WithDurableFlow(); + await using var provider = services.BuildServiceProvider(); + var hosted = provider.GetServices().ToArray(); + foreach (var service in hosted) + await service.StartAsync(CancellationToken.None); + + try + { + var flows = provider.GetRequiredService(); + var id = await flows.StartAsync(new ChattyInput("acme"), "flow-r35-chatty"); + + var deadline = DateTime.UtcNow.AddSeconds(15); + FlowState? state; + do + { + state = await flows.GetStateAsync(id); + if (state?.Status == FlowRunStatus.Succeeded) + break; + await Task.Delay(20); + } + while (DateTime.UtcNow < deadline); + Assert.Equal(FlowRunStatus.Succeeded, state?.Status); + + // ~6 KiB of results over a 2 KiB threshold: crossed once (≥2 KiB), then at the doubling + // (≥4 KiB) — never once per step. + var warnings = logger.Messages.Where(m => m.Contains("LedgerSizeWarningBytes threshold", StringComparison.Ordinal)).ToArray(); + Assert.InRange(warnings.Length, 1, 3); + Assert.All(warnings, w => Assert.Contains(id, w, StringComparison.Ordinal)); + } + finally + { + foreach (var service in hosted) + await service.StopAsync(CancellationToken.None); + } + } + + [Fact] + public void LedgerSizeWarningBytes_MustBePositiveOrNull() + { + Assert.Throws(() => FlowStateConcurrency.ValidateOptions(new DurableFlowOptions { LedgerSizeWarningBytes = 0 })); + Assert.Throws(() => FlowStateConcurrency.ValidateOptions(new DurableFlowOptions { LedgerSizeWarningBytes = -1 })); + FlowStateConcurrency.ValidateOptions(new DurableFlowOptions { LedgerSizeWarningBytes = null }); + FlowStateConcurrency.ValidateOptions(new DurableFlowOptions { LedgerSizeWarningBytes = 1 }); + } + + [Fact] + public void EstimateLedgerChars_CountsEveryStringTheLedgerCarries() + { + var state = new FlowState + { + InputJson = new string('i', 10), + LastMessage = new string('m', 5), + Steps = new Dictionary(StringComparer.Ordinal) + { + ["ab"] = new() { ResultJson = new string('r', 100), Message = new string('s', 3) } + }, + Values = new Dictionary(StringComparer.Ordinal) { ["k"] = new string('v', 20) }, + Context = new Dictionary(StringComparer.Ordinal) { ["c"] = new string('x', 4) } + }; + + Assert.Equal(10 + 5 + 2 + 100 + 3 + 1 + 20 + 1 + 4, FlowStateJson.EstimateLedgerChars(state)); + } + + // --------------------------------------------------------------------------------------------- + // P1 — the registry's non-blocking admission the DB channels' dispatch sweep now uses. The + // sweep-level behavior pin runs against the shared channel source in + // DbChannelSharedCoverageTests. + + [Fact] + public async Task SerialExecutorRegistry_TryEnqueue_AtCapacity_ReportsFullInsteadOfWaiting() + { + var registry = new SerialExecutorRegistry(NullLogger.Instance); + registry.OnSubscriptionRegistered("corr"); + var gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + // The first item occupies the single reader for as long as the gate is held... + Assert.Equal(SerialExecutorRegistry.TryEnqueueOutcome.Accepted, registry.TryEnqueue("corr", () => gate.Task)); + + // ...so at most the queue capacity (plus that one item, if the reader has not pulled it + // yet) can be admitted; the next call answers Full synchronously rather than parking. + var accepted = 1; + SerialExecutorRegistry.TryEnqueueOutcome last; + do + { + last = registry.TryEnqueue("corr", static () => Task.CompletedTask); + if (last == SerialExecutorRegistry.TryEnqueueOutcome.Accepted) + accepted++; + } + while (last == SerialExecutorRegistry.TryEnqueueOutcome.Accepted && accepted <= ChannelSerialExecutor.DefaultCapacity + 2); + + Assert.Equal(SerialExecutorRegistry.TryEnqueueOutcome.Full, last); + Assert.InRange(accepted, ChannelSerialExecutor.DefaultCapacity, ChannelSerialExecutor.DefaultCapacity + 1); + + gate.SetResult(); + registry.OnSubscriptionRetired("corr"); + await registry.RemoveAsync("corr"); + } + + [Fact] + public async Task SerialExecutorRegistry_TryEnqueue_OnATombstonedChannel_IsSuppressed() + { + var registry = new SerialExecutorRegistry(NullLogger.Instance); + registry.OnSubscriptionRegistered("corr"); + Assert.Equal(SerialExecutorRegistry.TryEnqueueOutcome.Accepted, registry.TryEnqueue("corr", static () => Task.CompletedTask)); + registry.OnSubscriptionRetired("corr"); + await registry.RemoveAsync("corr"); + + Assert.Equal(SerialExecutorRegistry.TryEnqueueOutcome.Suppressed, registry.TryEnqueue("corr", static () => Task.CompletedTask)); + } +} diff --git a/tests/AsyncResponse.Tests/Round35RegressionTests.cs b/tests/AsyncResponse.Tests/Round35RegressionTests.cs new file mode 100644 index 000000000..f96ba508d --- /dev/null +++ b/tests/AsyncResponse.Tests/Round35RegressionTests.cs @@ -0,0 +1,315 @@ +using AsyncResponse.Testing; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Xunit; + +namespace AsyncResponse.Tests; + +/// +/// Regressions for round 35 (external architect review of 684a3fb): behavior pins that compile +/// against the pre-fix tree and fail there. Pins over API this round introduced +/// (CreateAndExecuteAsync, LedgerSizeWarningBytes, SerialExecutorRegistry. +/// TryEnqueue) live in ; the DB-channel sweep pin lives in +/// DbChannelSharedCoverageTests and the Kafka pins in the Kafka dispatcher/subscriber tests. +/// +public sealed class Round35RegressionTests +{ + // --------------------------------------------------------------------------------------------- + // A1 — StartAsync committed the ledger and THEN published. A process dying between the two left + // a Running ledger with Attempts = 0 that nothing would ever execute, and IFlowStateStore has + // no enumeration for a reconciler to find it. The publish is now the commit point: the job + // carries the initial ledger and its execution creates the run. + + public sealed record R35Input(string Name); + + /// A flow that records having run, so a test can prove whether it did. + public sealed class R35MarkerFlow : IDurableFlow + { + public static int Executions; + + public Task ExecuteAsync(IDurableFlowContext flow, R35Input input) + { + Interlocked.Increment(ref Executions); + return Task.CompletedTask; + } + } + + /// A worker transport that records every published job and can be told to refuse. + private sealed class CapturingWorkerTransport : IWorkerTransport + { + public int PublishAttempts; + public volatile bool Fail; + public readonly List Published = []; + + public Task PublishAsync(WorkerJobEnvelope job, CancellationToken cancellationToken = default) + { + Interlocked.Increment(ref PublishAttempts); + if (Fail) + throw new InvalidOperationException("broker unavailable"); + + lock (Published) + Published.Add(job); + return Task.CompletedTask; + } + + public WorkerJobEnvelope[] Snapshot() + { + lock (Published) + return Published.ToArray(); + } + } + + private static ServiceProvider BuildFlowProvider(CapturingWorkerTransport transport, VirtualTimeProvider? clock = null) + { + var services = new ServiceCollection(); + services.AddSingleton(typeof(ILogger<>), typeof(NullLogger<>)); + if (clock is not null) + services.AddSingleton(clock); + services + .AddAsyncResponse() + .WithInMemoryChannel() + .WithInMemoryDurableFlows() + .WithDurableFlow(); + services.AddSingleton(transport); + return services.BuildServiceProvider(); + } + + /// Walks the start's retry ladder on the virtual clock until the task settles. + private static async Task DriveRetryLadderAsync(Task start, VirtualTimeProvider clock) + { + for (var i = 0; i < 200 && !start.IsCompleted; i++) + { + if (clock.NextTimerDueAt is { } due) + clock.Advance(due - clock.GetUtcNow() + TimeSpan.FromMilliseconds(1)); + else + await Task.Delay(5); + } + } + + /// + /// Pre-fix failure: LoadAsync(ex.FlowId) returned a Running ledger — the orphan the + /// exception told the caller to re-drive, which a caller that never saw the exception (the + /// process died) could not. + /// + [Fact] + public async Task Start_WhosePublishFails_PersistsNothing_SoNoRunIsStranded() + { + var clock = new VirtualTimeProvider(); + var transport = new CapturingWorkerTransport { Fail = true }; + await using var provider = BuildFlowProvider(transport, clock); + var flows = provider.GetRequiredService(); + var store = provider.GetRequiredService(); + + var start = flows.StartAsync(new R35Input("acme"), "flow-r35-undispatched"); + await DriveRetryLadderAsync(start, clock); + + var ex = await Assert.ThrowsAsync(() => start); + Assert.Equal("flow-r35-undispatched", ex.FlowId); + Assert.True(transport.PublishAttempts > 1, $"expected the publish to be retried; saw {transport.PublishAttempts} attempt(s)"); + + // The publish is the commit point: nothing exists for a reconciler to have to find. + Assert.Null(await store.LoadAsync(ex.FlowId)); + + // And the documented retry works: the same id creates the run and publishes exactly one job. + transport.Fail = false; + var retried = await flows.StartAsync(new R35Input("acme"), ex.FlowId); + Assert.Equal(ex.FlowId, retried); + Assert.NotNull(await store.LoadAsync(ex.FlowId)); + Assert.Single(transport.Snapshot()); + } + + /// + /// The crash window itself: the job is published, the starter's own ledger write never + /// happens. Pre-fix failure: the job carried only the id (ExecuteAsync), so replaying it + /// against an absent ledger logged "no state" and acknowledged — the run never existed and + /// never ran. + /// + [Fact] + public async Task Start_PublishesAJobThatCreatesTheLedgerItself_SoACrashBeforeTheStartersWriteCannotStrandTheRun() + { + var transport = new CapturingWorkerTransport(); + await using var provider = BuildFlowProvider(transport); + var flows = provider.GetRequiredService(); + var store = provider.GetRequiredService(); + var ingress = provider.GetRequiredService(); + + var id = await flows.StartAsync(new R35Input("acme"), "flow-r35-self-creating"); + var job = Assert.Single(transport.Snapshot()); + Assert.Equal("CreateAndExecuteAsync", job.Call.MethodName); + + // Simulate the crash: the starter's ledger is gone, the published job is all that survives. + Assert.True(await store.TryDeleteAsync(id)); + Assert.Null(await store.LoadAsync(id)); + + var before = Volatile.Read(ref R35MarkerFlow.Executions); + await ingress.HandleWorkerMessageAsync(AsyncResponseJson.Serialize(job)); + + var state = await store.LoadAsync(id); + Assert.NotNull(state); + Assert.Equal(FlowRunStatus.Succeeded, state!.Status); + Assert.Equal(typeof(R35MarkerFlow).FullName, state.FlowTypeName); + Assert.Equal(before + 1, Volatile.Read(ref R35MarkerFlow.Executions)); + } + + /// Pin (unchanged behavior): an identical explicit-id start re-enqueues, a conflicting one is rejected. + [Fact] + public async Task Start_WithAnExistingIdenticalRun_ReEnqueuesIt_AndRejectsDifferentInput() + { + var transport = new CapturingWorkerTransport(); + await using var provider = BuildFlowProvider(transport); + var flows = provider.GetRequiredService(); + var store = provider.GetRequiredService(); + + var id = await flows.StartAsync(new R35Input("acme"), "flow-r35-idempotent"); + var again = await flows.StartAsync(new R35Input("acme"), id); + + Assert.Equal(id, again); + Assert.Equal(2, transport.Snapshot().Length); + Assert.NotNull(await store.LoadAsync(id)); + + await Assert.ThrowsAsync( + () => flows.StartAsync(new R35Input("other"), id)); + } + + // --------------------------------------------------------------------------------------------- + // R3 — a void-returning target was awaited as "already complete", so an `async void` + // implementation was acknowledged (and its DI scope disposed) while its body was still + // running at the first await. + + public interface IVoidWorker + { + void Run(); + } + + public sealed class AsyncVoidWorker : IVoidWorker + { + public readonly TaskCompletionSource Gate = new(TaskCreationOptions.RunContinuationsAsynchronously); + public int Started; + + public async void Run() + { + Interlocked.Increment(ref Started); + await Gate.Task; + } + } + + public sealed class SyncVoidWorker : IVoidWorker + { + public int Runs; + + public void Run() => Runs++; + } + + private static ReflectionInvocationDto VoidRunDto(Type serviceType) => new() + { + ServiceInterfaceFullName = serviceType.FullName!, + MethodName = nameof(IVoidWorker.Run), + Params = [] + }; + + /// Pre-fix failure: the invocation completed successfully and Started was 1 — the body was mid-flight. + [Fact] + public async Task AsyncVoidImplementation_BehindAnInterface_IsRejectedBeforeItStarts() + { + var worker = new AsyncVoidWorker(); + var services = new ServiceCollection(); + services.AddSingleton(worker); + await using var provider = services.BuildServiceProvider(); + + var ex = await Assert.ThrowsAsync(() => provider.InvokeAsync(VoidRunDto(typeof(IVoidWorker)))); + + Assert.Contains("async void", ex.Message, StringComparison.Ordinal); + Assert.Contains(nameof(AsyncVoidWorker), ex.Message, StringComparison.Ordinal); + Assert.Equal(0, Volatile.Read(ref worker.Started)); + worker.Gate.SetResult(); + } + + /// Pre-fix failure: same as the interface case, for a class-typed service (rejected at plan time). + [Fact] + public async Task AsyncVoidImplementation_OnAClassTypedService_IsRejectedBeforeItStarts() + { + var worker = new AsyncVoidWorker(); + var services = new ServiceCollection(); + services.AddSingleton(worker); + await using var provider = services.BuildServiceProvider(); + + var ex = await Assert.ThrowsAsync(() => provider.InvokeAsync(VoidRunDto(typeof(AsyncVoidWorker)))); + + Assert.Contains("async void", ex.Message, StringComparison.Ordinal); + Assert.Equal(0, Volatile.Read(ref worker.Started)); + worker.Gate.SetResult(); + } + + /// Synchronous void targets are still supported: the guard is about the async marker, not the return type. + [Fact] + public async Task SyncVoidImplementation_StillDispatches() + { + var worker = new SyncVoidWorker(); + var services = new ServiceCollection(); + services.AddSingleton(worker); + await using var provider = services.BuildServiceProvider(); + + await provider.InvokeAsync(VoidRunDto(typeof(IVoidWorker))); + await provider.InvokeAsync(VoidRunDto(typeof(IVoidWorker))); + + Assert.Equal(2, worker.Runs); + } + + // --------------------------------------------------------------------------------------------- + // M1 — the expression converter knew the exact MethodInfo but persisted only name + arity, so an + // interface with `Run(int)` / `Run(string)` accepted `svc => svc.Run(1)` and every dispatch + // of the job then failed as ambiguous — after publication. + + public interface IOverloaded + { + Task Run(int value); + Task Run(string value); + Task Unique(int value); + } + + /// Pre-fix failure: the converter returned a descriptor; the throw came at dispatch. + [Fact] + public void AmbiguousOverload_FailsAtConversion_NotAtDispatch() + { + var ex = Assert.Throws( + () => CallbackExpressionConverter.ToReflectionCall(svc => svc.Run(1))); + + Assert.Contains("overloads", ex.Message, StringComparison.Ordinal); + Assert.Contains(nameof(IOverloaded.Run), ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void UniqueMethod_StillConverts() + { + var dto = CallbackExpressionConverter.ToReflectionCall(svc => svc.Unique(7)); + + Assert.Equal(nameof(IOverloaded.Unique), dto.MethodName); + Assert.Equal(7, Assert.Single(dto.Params).Value); + } + + /// Pre-fix failure: the job was published and failed on the worker; here nothing reaches the transport. + [Fact] + public async Task EnqueueWorker_WithAnAmbiguousTarget_ThrowsInTheCallersStack_AndPublishesNothing() + { + var transport = new CapturingWorkerTransport(); + await using var provider = BuildFlowProvider(transport); + var builder = provider.GetRequiredService(); + + await Assert.ThrowsAsync(() => builder.EnqueueWorkerAsync(svc => svc.Run(1))); + + Assert.Empty(transport.Snapshot()); + } + + /// Pre-fix failure: the registration succeeded and the recovery callback was unresolvable when it fired. + [Fact] + public async Task RecoveryCallbackRegistration_WithAnAmbiguousTarget_ThrowsAtRegistration() + { + var transport = new CapturingWorkerTransport(); + await using var provider = BuildFlowProvider(transport); + var builder = provider.GetRequiredService(); + + Assert.Throws( + () => builder.For().OnLostSubscriberResume(svc => svc.Run(1))); + } +} diff --git a/tests/AsyncResponse.Tests/Round36NewApiTests.cs b/tests/AsyncResponse.Tests/Round36NewApiTests.cs new file mode 100644 index 000000000..60a4d2189 --- /dev/null +++ b/tests/AsyncResponse.Tests/Round36NewApiTests.cs @@ -0,0 +1,211 @@ +using AsyncResponse.Testing; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using System.Diagnostics.Metrics; +using Xunit; + +namespace AsyncResponse.Tests; + +/// +/// Round-36 pins over API the round introduced — they do not compile against the pre-fix tree, so +/// they live apart from the behavior pins in . +/// +public sealed class Round36NewApiTests +{ + private static WorkerJobEnvelope Job() => new() + { + Call = new ReflectionCallDto + { + ServiceInterfaceFullName = "AsyncResponse.Tests.IRound36Probe", + MethodName = "RunAsync", + Params = [] + } + }; + + private static Task PublishInJobAsync(InMemoryWorkerTransport transport, int count) + => Task.Run(async () => + { + InMemoryWorkerTransport.InJobScope.MarkActive(); + var accepted = 0; + for (var i = 0; i < count; i++) + { + try + { + await transport.PublishAsync(Job()); + accepted++; + } + catch (InvalidOperationException) + { + break; + } + } + + return accepted; + }); + + [Fact] + public async Task InJobOverflowCapacity_BoundsTheOverflow_AndAWorkerFreeingASlotReopensIt() + { + var transport = new InMemoryWorkerTransport(Options.Create(new InMemoryWorkerTransportOptions + { + QueueCapacity = 1, + InJobOverflowCapacity = 8 + })); + + // One queue slot plus eight overflow entries; the tenth is rejected. + Assert.Equal(9, await PublishInJobAsync(transport, 20)); + Assert.Equal(8, transport.OverflowDepth); + Assert.Equal(9, transport.OutstandingJobs); + + // A worker takes the queued job and pumps the overflow: one entry moves into the queue, + // the depth drops, and the next follow-up publish is admitted again. + Assert.True(transport.Reader.TryRead(out _)); + transport.PumpOverflow(); + Assert.Equal(7, transport.OverflowDepth); + Assert.Equal(1, await PublishInJobAsync(transport, 1)); + Assert.Equal(8, transport.OverflowDepth); + } + + [Fact] + public async Task InJobOverflowCapacity_Zero_RejectsTheFirstFollowUpThatFindsTheQueueFull() + { + var transport = new InMemoryWorkerTransport(Options.Create(new InMemoryWorkerTransportOptions + { + QueueCapacity = 2, + InJobOverflowCapacity = 0 + })); + + Assert.Equal(2, await PublishInJobAsync(transport, 5)); + Assert.Equal(0, transport.OverflowDepth); + } + + [Fact] + public void InJobOverflowCapacity_Negative_IsRejectedAtConstruction() + { + var ex = Assert.Throws(() => new InMemoryWorkerTransport( + Options.Create(new InMemoryWorkerTransportOptions { InJobOverflowCapacity = -1 }))); + + Assert.Contains(nameof(InMemoryWorkerTransportOptions.InJobOverflowCapacity), ex.Message, StringComparison.Ordinal); + } + + [Fact] + public async Task OverflowRejections_AndDepth_AreObservableOnTheMeter() + { + var measurements = new List<(string Instrument, long Value)>(); + using var listener = new MeterListener + { + InstrumentPublished = (instrument, l) => + { + if (instrument.Meter.Name == AsyncResponseDiagnostics.MeterName) + l.EnableMeasurementEvents(instrument); + } + }; + listener.SetMeasurementEventCallback((instrument, value, _, _) => + { + lock (measurements) + measurements.Add((instrument.Name, value)); + }); + listener.Start(); + + var transport = new InMemoryWorkerTransport(Options.Create(new InMemoryWorkerTransportOptions + { + QueueCapacity = 1, + InJobOverflowCapacity = 3 + })); + Assert.Equal(4, await PublishInJobAsync(transport, 6)); + listener.RecordObservableInstruments(); + + lock (measurements) + { + Assert.Contains(measurements, m => m.Instrument == "asyncresponse.worker.inmemory_overflow_rejections" && m.Value == 1); + // Summed over every live transport in the process, so at least this one's three. + Assert.Contains(measurements, m => m.Instrument == "asyncresponse.worker.inmemory_overflow_depth" && m.Value >= 3); + } + + GC.KeepAlive(transport); + } + + // --------------------------------------------------------------------------------------------- + // Ancestor-depth ceiling: past MaxAncestorLedgerDepth the run fails terminally instead of the + // chain being truncated in silence. + + private sealed record DepthInput(int Value); + + private sealed class DepthFlow : IDurableFlow + { + public Task ExecuteAsync(IDurableFlowContext flow, DepthInput input) => Task.CompletedTask; + } + + private sealed class RecordingDelayedTransport : IDelayedWorkerTransport + { + public int Published; + + public TimeSpan MaxPublishDelay => TimeSpan.FromDays(30); + + public Task PublishAsync(WorkerJobEnvelope job, CancellationToken cancellationToken = default) + { + Interlocked.Increment(ref Published); + return Task.CompletedTask; + } + + public Task PublishAsync(WorkerJobEnvelope job, TimeSpan delay, CancellationToken cancellationToken = default) + => PublishAsync(job, cancellationToken); + } + + [Fact] + public async Task AncestorExtension_ChainDeeperThanTheCeiling_FailsTheRunTerminally() + { + Assert.Equal(256, DurableFlowContext.MaxAncestorLedgerDepth); + + var clock = new VirtualTimeProvider(); + var transport = new RecordingDelayedTransport(); + var services = new ServiceCollection(); + services.AddSingleton(typeof(ILogger<>), typeof(NullLogger<>)); + services.AddSingleton(clock); + services.AddAsyncResponse().WithInMemoryChannel().WithInMemoryDurableFlows(); + services.AddSingleton(transport); + await using var provider = services.BuildServiceProvider(); + var store = provider.GetRequiredService(); + + string? parent = null; + FlowState leaf = null!; + for (var level = 0; level <= DurableFlowContext.MaxAncestorLedgerDepth + 1; level++) + { + var id = parent is null ? "r36-ceiling" : $"{parent}:c"; + leaf = new FlowState + { + FlowId = id, + ParentFlowId = parent, + ParentStepName = parent is null ? null : "c", + Status = FlowRunStatus.Running, + FlowTypeName = typeof(DepthFlow).FullName, + InputTypeName = typeof(DepthInput).FullName, + InputJson = "{\"Value\":1}" + }; + Assert.True(await store.TryCreateAsync(id, leaf, TimeSpan.FromMinutes(1))); + parent = id; + } + + var options = new DurableFlowOptions { StateExpiry = TimeSpan.FromMinutes(1), TimerInProcessThreshold = TimeSpan.Zero }; + await using var lease = (await FlowStateConcurrency.TryAcquireExecutionLeaseAsync(store, leaf.FlowId!, options, NullLogger.Instance, clock))!; + var context = new DurableFlowContext( + leaf, + store, + provider.GetRequiredService(), + provider.GetRequiredService(), + options, + provider.GetRequiredService(), + recoverableSubscriber: null, + NullLogger.Instance, + lease, + clock, + workerTransport: transport); + + var ex = await Assert.ThrowsAsync(() => context.DelayAsync("long-wait", TimeSpan.FromHours(1))); + + Assert.Contains("nested more than 256", ex.Message, StringComparison.Ordinal); + Assert.Equal(0, transport.Published); + } +} diff --git a/tests/AsyncResponse.Tests/Round36RegressionTests.cs b/tests/AsyncResponse.Tests/Round36RegressionTests.cs new file mode 100644 index 000000000..4a15ff9e9 --- /dev/null +++ b/tests/AsyncResponse.Tests/Round36RegressionTests.cs @@ -0,0 +1,681 @@ +using AsyncResponse.Testing; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using System.Text.Json; +using Xunit; + +namespace AsyncResponse.Tests; + +/// +/// Regressions for round 36 (external holistic review of 145aa8c): behavior pins that compile +/// against the pre-fix tree and fail there. Pins over API this round introduced +/// (InMemoryWorkerTransportOptions.InJobOverflowCapacity, the overflow metrics, the +/// ancestor-depth constant) live in ; the per-channel +/// body-free-reader pins live next to each channel's existing malformed-envelope tests, and the +/// Cosmos lease-patch pins in CosmosDurableFlowStateStoreTests. +/// +public sealed class Round36RegressionTests +{ + public sealed record R36Input(DateTimeOffset Occurrence); + + public sealed class R36NoopFlow : IDurableFlow + { + public Task ExecuteAsync(IDurableFlowContext flow, R36Input input) => Task.CompletedTask; + } + + /// + /// A payload whose dictionary keys come straight off the wire: a value that fails to convert + /// makes System.Text.Json report Path: $.Payload.Values['<key>'] — the key IS + /// body content, and it used to reach the application log and the waiter verbatim. + /// + public sealed class LeakProbePayload : IAsyncResponsePayload + { + public Dictionary? Values { get; set; } + + public RecoveryAction OnRecovery() => RecoveryAction.Resume; + } + + /// The marker no log line or exception chain may carry (a synthetic private identifier). + public const string Marker = "private_customer_42@example.invalid"; + + /// A success envelope whose only defect is a string where wants an int. + public const string LeakingEnvelope = + """{"SchemaVersion":1,"Success":true,"Payload":{"Values":{"private_customer_42@example.invalid":"not-a-number"}}}"""; + + /// Asserts neither the exception chain nor anything the channel logged carries the marker. + internal static void AssertNoMarker(Exception exception, CollectingLogger logger) + { + Assert.DoesNotContain(Marker, exception.ToString(), StringComparison.Ordinal); + foreach (var (message, logged) in logger.Entries) + { + Assert.DoesNotContain(Marker, message, StringComparison.Ordinal); + if (logged is not null) + Assert.DoesNotContain(Marker, logged.ToString(), StringComparison.Ordinal); + } + + // The diagnosis is still there: the failure is located by size and position. + Assert.Contains("Failed to parse JSON payload", exception.ToString(), StringComparison.Ordinal); + } + + // --------------------------------------------------------------------------------------------- + // F1 — the scheduler's re-drive settled an occurrence whose start had never been published. + // Round 35 made a start publish-first (a failed publish persists NOTHING), but the re-drive + // still read "no ledger" as "expired or deleted; give up" — so every occurrence that fell + // due during a broker outage was lost for good once the outage outlasted the start's own + // retry ladder, and the startup probe could not find a run that was never persisted either. + + /// A worker transport that can be told to refuse, and records what it accepted. + private sealed class FailableWorkerTransport : IWorkerTransport + { + private readonly List _published = []; + private int _attempts; + + public volatile bool Fail; + + public int Attempts => Volatile.Read(ref _attempts); + + public int PublishedCount + { + get { lock (_published) return _published.Count; } + } + + public Task PublishAsync(WorkerJobEnvelope job, CancellationToken cancellationToken = default) + { + Interlocked.Increment(ref _attempts); + if (Fail) + throw new TimeoutException("simulated broker outage"); + + lock (_published) + _published.Add(job); + return Task.CompletedTask; + } + } + + /// + /// Fires virtual timers one at a time — never two due instants in one advance, so a re-drive + /// timer and the next occurrence cannot collapse into a single pass — until the condition holds. + /// + private static async Task AdvanceUntilAsync(VirtualTimeProvider clock, Func done, string what) + { + var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(20); + while (!done()) + { + if (DateTime.UtcNow > deadline) + throw new TimeoutException($"Timed out waiting for {what}."); + + if (clock.NextTimerDueAt is { } due) + clock.Advance(due - clock.GetUtcNow() + TimeSpan.FromMilliseconds(1)); + else + await Task.Delay(5); + } + } + + /// + /// Pre-fix failure: after the outage the re-drive made zero further publish attempts and + /// settled the entry — the transport never sees a second job, and no ledger ever appears. + /// Proven with the REAL starter (DurableFlowService) and the real scheduler loop; the + /// round-34 pin passed because its fake created a ledger before failing, the pre-round-35 order. + /// + [Fact] + public async Task ScheduledFlow_OccurrenceWhosePublishFailed_IsRedrivenByTheRealStarterUntilPublished() + { + var clock = new VirtualTimeProvider(new DateTimeOffset(2030, 1, 1, 0, 0, 30, TimeSpan.Zero)); + var transport = new FailableWorkerTransport { Fail = true }; + var services = new ServiceCollection(); + services.AddSingleton(typeof(ILogger<>), typeof(NullLogger<>)); + services.AddSingleton(clock); + services + .AddAsyncResponse() + .WithInMemoryChannel() + .WithInMemoryDurableFlows() + .WithDurableFlow(); + services.AddSingleton(transport); + await using var provider = services.BuildServiceProvider(); + var flows = provider.GetRequiredService(); + var store = provider.GetRequiredService(); + + var registration = new ScheduledFlowRegistration + { + Name = "r36-hourly", + CronExpression = "0 * * * *", + Options = new ScheduledFlowOptions(), + StartOccurrenceAsync = static (durableFlows, flowId, occurrence, cancellationToken) => + durableFlows.StartAsync(new R36Input(occurrence), flowId, cancellationToken) + }; + var occurrenceId = ScheduledFlowService.OccurrenceFlowId("r36-hourly", new DateTimeOffset(2030, 1, 1, 1, 0, 0, TimeSpan.Zero)); + + using var scheduler = new ScheduledFlowService(flows, [registration], NullLogger.Instance, clock); + await scheduler.StartAsync(CancellationToken.None); + try + { + // 01:00 falls due inside the outage: the start's whole retry ladder (4 attempts) fails. + await AdvanceUntilAsync(clock, () => transport.Attempts >= 4, "the start's retry ladder to exhaust"); + await Task.Delay(50); + Assert.Equal(4, transport.Attempts); + Assert.Equal(0, transport.PublishedCount); + // Publish-first: the failed start persisted nothing — exactly the shape the old + // re-drive misread as "expired". + Assert.Null(await store.LoadAsync(occurrenceId)); + + // The broker is back. The re-drive (RedriveInterval later) must START the occurrence + // again; pre-fix it loaded the never-created ledger, read the null as gone, and settled. + transport.Fail = false; + await AdvanceUntilAsync(clock, () => transport.PublishedCount >= 1, "the re-drive to publish the start job"); + Assert.Equal(1, transport.PublishedCount); + Assert.Equal(5, transport.Attempts); + // The starter's own create ran after its publish, so the run now exists. + Assert.NotNull(await store.LoadAsync(occurrenceId)); + + // Settled: nothing else is published before the next occurrence (02:00 is far away). + clock.Advance(TimeSpan.FromMinutes(5)); + await Task.Delay(100); + Assert.Equal(1, transport.PublishedCount); + } + finally + { + await scheduler.StopAsync(CancellationToken.None); + } + } + + // --------------------------------------------------------------------------------------------- + // F2 — AwaitChildFlowAsync handed the first completion the FULL loaded child (ambient Context, + // grandchild results and all) but every replay the reduced snapshot read back from the memo, + // so a parent could branch differently after a restart on a step it had already completed. + + private static FlowState State(string id, string? parent = null, FlowRunStatus status = FlowRunStatus.Running) => new() + { + FlowId = id, + ParentFlowId = parent, + Status = status, + FlowTypeName = typeof(R36NoopFlow).FullName, + InputTypeName = typeof(R36Input).FullName, + InputJson = JsonSerializer.Serialize(new R36Input(DateTimeOffset.UnixEpoch)), + CreatedAtUtc = DateTime.UtcNow, + UpdatedAtUtc = DateTime.UtcNow + }; + + private static ServiceProvider BuildContextProvider(IWorkerTransport transport, TimeProvider clock) + { + var services = new ServiceCollection(); + services.AddSingleton(typeof(ILogger<>), typeof(NullLogger<>)); + services.AddSingleton(clock); + services + .AddAsyncResponse() + .WithInMemoryChannel() + .WithInMemoryDurableFlows() + .WithDurableFlow(); + services.AddSingleton(transport); + return services.BuildServiceProvider(); + } + + private static DurableFlowContext CreateContext( + ServiceProvider provider, + FlowState state, + IFlowStateStore store, + FlowExecutionLease lease, + DurableFlowOptions options, + TimeProvider clock, + IWorkerTransport transport, + ILogger? logger = null) + => new( + state, + store, + provider.GetRequiredService(), + provider.GetRequiredService(), + options, + provider.GetRequiredService(), + recoverableSubscriber: null, + logger ?? NullLogger.Instance, + lease, + clock, + workerTransport: transport); + + /// + /// Pre-fix failure: the first call returns the child with its Context (1 entry) and the + /// grandchild step's ResultJson; the replay returns neither — two different objects for + /// one completed step. + /// + [Theory] + [InlineData(FlowRunStatus.Succeeded)] + [InlineData(FlowRunStatus.Failed)] + public async Task AwaitChildFlow_FirstCompletionAndReplay_ReturnTheSameMemoizedSnapshot(FlowRunStatus childStatus) + { + var clock = new VirtualTimeProvider(); + var transport = new FailableWorkerTransport(); + await using var provider = BuildContextProvider(transport, clock); + var store = provider.GetRequiredService(); + var parentId = $"r36-snapshot-{childStatus}"; + var parent = State(parentId); + var child = State($"{parentId}:child", parentId, childStatus); + child.ParentStepName = "child"; + child.LastMessage = childStatus == FlowRunStatus.Failed ? "child failed" : "child done"; + child.Context = new Dictionary(StringComparer.Ordinal) { ["tenant"] = "demo" }; + child.Steps = new Dictionary(StringComparer.Ordinal) + { + ["local"] = new() { Completed = true, ResultJson = """{"kept":true}""" }, + ["grandchild"] = new() { Completed = true, ChildFlowId = $"{parentId}:child:grandchild", ResultJson = """{"Status":1}""" } + }; + Assert.True(await store.TryCreateAsync(parentId, parent, TimeSpan.FromDays(1))); + Assert.True(await store.TryCreateAsync(child.FlowId!, child, TimeSpan.FromDays(1))); + + var options = new DurableFlowOptions(); + await using var lease = (await FlowStateConcurrency.TryAcquireExecutionLeaseAsync(store, parentId, options, NullLogger.Instance, clock))!; + var context = CreateContext(provider, parent, store, lease, options, clock, transport); + var input = new R36Input(DateTimeOffset.UnixEpoch); + + var first = await context.AwaitChildFlowAsync("child", input, failOnChildFailure: false); + var replay = await context.AwaitChildFlowAsync("child", input, failOnChildFailure: false); + + // Both are the memoized snapshot: no ambient context, grandchild result elided, the rest kept. + Assert.Null(first.Context); + Assert.Null(first.Steps!["grandchild"].ResultJson); + Assert.Equal($"{parentId}:child:grandchild", first.Steps["grandchild"].ChildFlowId); + Assert.Equal("""{"kept":true}""", first.Steps["local"].ResultJson); + Assert.Equal(childStatus, first.Status); + Assert.Equal(child.LastMessage, first.LastMessage); + Assert.Equal(FlowStateJson.Serialize(first), FlowStateJson.Serialize(replay)); + // And the stored child itself is untouched by the memoization. + var stored = await store.LoadAsync(child.FlowId!); + Assert.Equal("demo", stored!.Context!["tenant"]); + Assert.Equal("""{"Status":1}""", stored.Steps!["grandchild"].ResultJson); + } + + // --------------------------------------------------------------------------------------------- + // F3 — the ledger reader chained the raw System.Text.Json exception, whose message carries + // `Path: $.Values['']` built from the stored dictionary keys, into + // FlowStateUnreadableException — which the worker ingress logs in full. The channel + // readers' pins live next to each channel's existing malformed-envelope tests. + + private const string LeakingLedger = + """{"SchemaVersion":1,"FlowId":"r36-leak","Values":{"private_customer_42@example.invalid":123}}"""; + + /// Pre-fix failure: the inner JsonException's message quotes the dictionary key. + [Fact] + public void FlowStateJson_MalformedLedger_DoesNotEchoStoredKeysIntoTheExceptionChain() + { + var ex = Assert.Throws(() => FlowStateJson.Deserialize(LeakingLedger, "r36-leak")); + + Assert.Equal("r36-leak", ex.FlowId); + Assert.Contains("malformed", ex.Reason, StringComparison.Ordinal); + Assert.DoesNotContain(Marker, ex.ToString(), StringComparison.Ordinal); + Assert.Contains("Failed to parse JSON payload", ex.ToString(), StringComparison.Ordinal); + } + + /// + /// The same reader, reached the way an attacker (or a schema mismatch) reaches it: a start + /// job whose carrier is malformed, delivered through the worker ingress. Pre-fix failure: the + /// key is in the exception the ingress throws and in the error it logs. + /// + [Fact] + public async Task StartCarrier_MalformedInitialState_DoesNotEchoItsKeysIntoIngressLogsOrTheException() + { + var logger = new CollectingLogger(); + var transport = new FailableWorkerTransport(); + var services = new ServiceCollection(); + services.AddSingleton(typeof(ILogger<>), typeof(NullLogger<>)); + services + .AddAsyncResponse() + .WithInMemoryChannel() + .WithInMemoryDurableFlows() + .WithDurableFlow(); + services.AddSingleton(transport); + // After the open-generic NullLogger, so the closed registration wins for the ingress. + services.AddSingleton>(logger.For()); + await using var provider = services.BuildServiceProvider(); + var ingress = provider.GetRequiredService(); + + var envelope = new WorkerJobEnvelope + { + Call = new ReflectionCallDto + { + ServiceInterfaceFullName = typeof(IDurableFlowExecutor).FullName!, + MethodName = nameof(IDurableFlowExecutor.CreateAndExecuteAsync), + Params = [CallbackParam.ForValue("r36-leak"), CallbackParam.ForValue(LeakingLedger)] + } + }; + + var ex = await Assert.ThrowsAsync( + () => ingress.HandleWorkerMessageAsync(JsonSerializer.Serialize(envelope))); + + AssertNoMarker(ex, logger); + Assert.Contains(logger.Entries, entry => entry.Exception is FlowStateUnreadableException); + } + + // --------------------------------------------------------------------------------------------- + // F3 follow-up — the first cut of the body-free reader scrubbed EVERY JsonException, including + // the six the envelope converter authors itself. Those name only the wire contract's own + // properties (SchemaVersion, Success, Payload) and never a byte of the body, and they are + // the primary operator diagnosis for the commonest malformed-envelope cause in production: + // a foreign or mismatched producer writing to the response channel. Replacing + // "SchemaVersion is required." with "failed at line 0, byte position 2" cost the diagnosis + // and protected nothing. (Caught by the integration suite, which pins these messages.) + + public static TheoryData WireContractViolations() => new() + { + { "{}", "SchemaVersion is required." }, + { """{"SchemaVersion":1,"Success":true}""", "Payload is null or absent" }, + { """{"SchemaVersion":1,"Success":true,"Payload":null}""", "Payload is null or absent" }, + { """{"SchemaVersion":"one","Success":true}""", "SchemaVersion must be an integer." }, + { """{"SchemaVersion":1,"Success":"yes"}""", "Success must be a boolean." }, + { """{"SchemaVersion":1,"Success":false,"ExceptionMessage":7}""", "ExceptionMessage must be a string or null." }, + { "[]", "must be a JSON object" }, + }; + + /// + /// Pre-fix failure (of the round-36 F3 fix itself): every one of these came back as + /// "Failed to parse JSON payload (N UTF-16 code units) at line …", with the reason gone. + /// + [Theory] + [MemberData(nameof(WireContractViolations))] + public void EnvelopeContractViolation_KeepsItsDiagnosis_BecauseItNamesNoBody(string envelopeJson, string expectedReason) + { + var ex = Assert.ThrowsAny( + () => JsonSafety.SafeDeserialize(envelopeJson, AsyncResponseEnvelopeJson.TypeInfo())); + + Assert.Contains(expectedReason, ex.Message, StringComparison.Ordinal); + Assert.DoesNotContain("Failed to parse JSON payload", ex.Message, StringComparison.Ordinal); + // Still a JsonException, so the ingress keeps classifying it as permanent (no retry burn) + // and application code catching JsonException still catches it. + Assert.IsAssignableFrom(ex); + } + + /// + /// The other half of the same contract: a failure the READER authored is still scrubbed, even + /// though it arrives through the very same call. The discriminator is who wrote the message, + /// not which reader threw it. + /// + [Fact] + public void PayloadConversionFailure_IsStillScrubbed_EvenThoughTheConverterRanFirst() + { + var ex = Assert.Throws( + () => JsonSafety.SafeDeserialize(LeakingEnvelope, AsyncResponseEnvelopeJson.TypeInfo())); + + Assert.DoesNotContain(Marker, ex.ToString(), StringComparison.Ordinal); + Assert.Contains("Failed to parse JSON payload", ex.Message, StringComparison.Ordinal); + } + + // --------------------------------------------------------------------------------------------- + // F4 — an in-job publish that found the queue full spilled into an UNBOUNDED overflow: with + // QueueCapacity = 1 a fan-out handler could park ten thousand envelopes (each with its + // captured ExecutionContext) with the configured capacity giving no signal at all. + + private static WorkerJobEnvelope Job() => new() + { + Call = new ReflectionCallDto + { + ServiceInterfaceFullName = "AsyncResponse.Tests.IRound36Probe", + MethodName = "RunAsync", + Params = [] + } + }; + + /// + /// Pre-fix failure: every one of the 6 000 follow-up publishes is accepted (outstanding = + /// 6 000, overflow = 5 999). Now the queue slot plus the default in-job overflow (4 096) are + /// accepted and the next publish is rejected with the count undone. + /// + [Fact] + public async Task InMemoryTransport_InJobPublishes_AreRejectedPastTheDefaultOverflowBound() + { + var transport = new InMemoryWorkerTransport(Options.Create(new InMemoryWorkerTransportOptions { QueueCapacity = 1 })); + var accepted = 0; + InvalidOperationException? rejection = null; + + await Task.Run(async () => + { + InMemoryWorkerTransport.InJobScope.MarkActive(); + for (var i = 0; i < 6_000; i++) + { + try + { + await transport.PublishAsync(Job()); + accepted++; + } + catch (InvalidOperationException ex) + { + rejection = ex; + break; + } + } + }); + + Assert.NotNull(rejection); + Assert.Equal(1 + 4_096, accepted); + Assert.Equal(1 + 4_096, transport.OutstandingJobs); + Assert.Contains("InJobOverflowCapacity", rejection.Message, StringComparison.Ordinal); + } + + // --------------------------------------------------------------------------------------------- + // F7 — a child's long park extended its ancestors' ledgers best-effort: a failed ancestor write + // was logged and swallowed, and the walk stopped silently after 16 levels. The child then + // published its wake-up and parked "successfully" while the parent it would complete into + // expired underneath it. + + /// A delayed-capable transport that only records what it was asked to publish. + private sealed class CapturingDelayedTransport : IDelayedWorkerTransport + { + public readonly List Jobs = []; + + public TimeSpan MaxPublishDelay => TimeSpan.FromDays(30); + + public Task PublishAsync(WorkerJobEnvelope job, CancellationToken cancellationToken = default) + { + lock (Jobs) + Jobs.Add(job); + return Task.CompletedTask; + } + + public Task PublishAsync(WorkerJobEnvelope job, TimeSpan delay, CancellationToken cancellationToken = default) + => PublishAsync(job, cancellationToken); + + public int Count + { + get { lock (Jobs) return Jobs.Count; } + } + } + + /// Fails the lease-less (ancestor TTL) writes of one flow id, counting them, and can hang a lease release. + private sealed class FaultingFlowStateStore(IFlowStateStore inner) : IFlowStateStore + { + private int _failedAncestorUpdates; + + public volatile string? FailAncestorUpdates; + public bool HangRelease; + public CancellationToken ReleaseToken { get; private set; } + public TaskCompletionSource ReleaseEntered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource ReleaseGate { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public int FailedAncestorUpdates => Volatile.Read(ref _failedAncestorUpdates); + + public Task LoadAsync(string flowId, CancellationToken cancellationToken = default) + => inner.LoadAsync(flowId, cancellationToken); + + public Task TryCreateAsync(string flowId, FlowState state, TimeSpan ttl, CancellationToken cancellationToken = default) + => inner.TryCreateAsync(flowId, state, ttl, cancellationToken); + + public Task TryUpdateAsync(string flowId, FlowState state, long expectedRevision, TimeSpan ttl, string? leaseId = null, CancellationToken cancellationToken = default) + { + if (leaseId is null && string.Equals(flowId, FailAncestorUpdates, StringComparison.Ordinal)) + { + Interlocked.Increment(ref _failedAncestorUpdates); + throw new TimeoutException("simulated ancestor write outage"); + } + + return inner.TryUpdateAsync(flowId, state, expectedRevision, ttl, leaseId, cancellationToken); + } + + public Task TryAcquireLeaseAsync(string flowId, string leaseId, TimeSpan leaseDuration, CancellationToken cancellationToken = default) + => inner.TryAcquireLeaseAsync(flowId, leaseId, leaseDuration, cancellationToken); + + public Task TryRenewLeaseAsync(string flowId, string leaseId, TimeSpan leaseDuration, CancellationToken cancellationToken = default) + => inner.TryRenewLeaseAsync(flowId, leaseId, leaseDuration, cancellationToken); + + public Task ReleaseLeaseAsync(string flowId, string leaseId, CancellationToken cancellationToken = default) + { + ReleaseToken = cancellationToken; + ReleaseEntered.TrySetResult(); + return HangRelease ? ReleaseGate.Task : inner.ReleaseLeaseAsync(flowId, leaseId, cancellationToken); + } + + public Task TryDeleteAsync(string flowId, CancellationToken cancellationToken = default) + => inner.TryDeleteAsync(flowId, cancellationToken); + } + + private static readonly DurableFlowOptions ShortLedgerOptions = new() + { + StateExpiry = TimeSpan.FromMinutes(1), + TimerInProcessThreshold = TimeSpan.Zero + }; + + /// + /// Pre-fix failure: the first attempt throws DurableFlowSuspendedException with the + /// wake-up already published and the root's TTL untouched, so two virtual minutes later the + /// root is gone while the child's park (and its wake-up) live on. Now the park fails with the + /// store's own exception and nothing published; the redelivered execution replays the parked + /// timer, extends the chain, and only then publishes. + /// + [Fact] + public async Task AncestorExtension_StoreOutage_FailsTheParkWithNothingPublished_AndTheRedeliveryExtendsTheChain() + { + var clock = new VirtualTimeProvider(); + var transport = new CapturingDelayedTransport(); + await using var provider = BuildContextProvider(transport, clock); + var inner = provider.GetRequiredService(); + var store = new FaultingFlowStateStore(inner) { FailAncestorUpdates = "r36-root" }; + var root = State("r36-root"); + root.Steps = new Dictionary(StringComparer.Ordinal) { ["child"] = new() { ChildFlowId = "r36-root:child" } }; + var child = State("r36-root:child", "r36-root"); + child.ParentStepName = "child"; + Assert.True(await inner.TryCreateAsync("r36-root", root, TimeSpan.FromMinutes(1))); + Assert.True(await inner.TryCreateAsync("r36-root:child", child, TimeSpan.FromMinutes(1))); + + // Attempt 1: the ancestor write fails. The park must fail too, before any wake-up exists. + await using (var lease = (await FlowStateConcurrency.TryAcquireExecutionLeaseAsync(store, child.FlowId!, ShortLedgerOptions, NullLogger.Instance, clock))!) + { + var context = CreateContext(provider, child, store, lease, ShortLedgerOptions, clock, transport); + await Assert.ThrowsAsync(() => context.DelayAsync("long-wait", TimeSpan.FromHours(1))); + } + + Assert.Equal(0, transport.Count); + Assert.Equal(1, store.FailedAncestorUpdates); + + // The store recovers; the redelivered execution replays the parked timer. + store.FailAncestorUpdates = null; + var replayed = (await inner.LoadAsync(child.FlowId!))!; + Assert.NotNull(replayed.Steps!["long-wait"].WakeAtUtc); + await using (var lease = (await FlowStateConcurrency.TryAcquireExecutionLeaseAsync(store, child.FlowId!, ShortLedgerOptions, NullLogger.Instance, clock))!) + { + var context = CreateContext(provider, replayed, store, lease, ShortLedgerOptions, clock, transport); + await Assert.ThrowsAsync(() => context.DelayAsync("long-wait", TimeSpan.FromHours(1))); + } + + Assert.Equal(1, transport.Count); + + // The park's window is an hour; the parent outlives its own one-minute expiry. + clock.Advance(TimeSpan.FromMinutes(2)); + Assert.NotNull(await inner.LoadAsync("r36-root")); + Assert.NotNull(await inner.LoadAsync("r36-root:child")); + } + + /// + /// Pre-fix failure: the walk stopped after 16 ancestors, so the root of a 20-deep chain kept + /// its one-minute expiry and was gone two virtual minutes into the leaf's hour-long park. + /// + [Fact] + public async Task AncestorExtension_ReachesTheRootOfAChainDeeperThanSixteen() + { + var clock = new VirtualTimeProvider(); + var transport = new CapturingDelayedTransport(); + await using var provider = BuildContextProvider(transport, clock); + var store = provider.GetRequiredService(); + + const int depth = 20; + var ids = new List(); + string? parent = null; + for (var level = 0; level <= depth; level++) + { + var id = parent is null ? "r36-deep-root" : $"{parent}:c"; + var state = State(id, parent); + if (parent is not null) + state.ParentStepName = "c"; + if (level < depth) + state.Steps = new Dictionary(StringComparer.Ordinal) { ["c"] = new() { ChildFlowId = $"{id}:c" } }; + Assert.True(await store.TryCreateAsync(id, state, TimeSpan.FromMinutes(1))); + ids.Add(id); + parent = id; + } + + var leaf = (await store.LoadAsync(ids[^1]))!; + await using (var lease = (await FlowStateConcurrency.TryAcquireExecutionLeaseAsync(store, leaf.FlowId!, ShortLedgerOptions, NullLogger.Instance, clock))!) + { + var context = CreateContext(provider, leaf, store, lease, ShortLedgerOptions, clock, transport); + await Assert.ThrowsAsync(() => context.DelayAsync("long-wait", TimeSpan.FromHours(1))); + } + + clock.Advance(TimeSpan.FromMinutes(2)); + foreach (var id in ids) + Assert.True(await store.LoadAsync(id) is not null, $"{id} expired under the leaf's park"); + } + + /// + /// A ParentFlowId cycle is corrupted state. Pre-fix it was walked 16 times and the leaf + /// parked anyway; now the run fails terminally and deterministically, naming the cycle. + /// + [Fact] + public async Task AncestorExtension_CycleInTheParentChain_FailsTheRunTerminally() + { + var clock = new VirtualTimeProvider(); + var transport = new CapturingDelayedTransport(); + await using var provider = BuildContextProvider(transport, clock); + var store = provider.GetRequiredService(); + Assert.True(await store.TryCreateAsync("r36-cycle-a", State("r36-cycle-a", "r36-cycle-b"), TimeSpan.FromMinutes(1))); + Assert.True(await store.TryCreateAsync("r36-cycle-b", State("r36-cycle-b", "r36-cycle-a"), TimeSpan.FromMinutes(1))); + var leaf = State("r36-cycle-leaf", "r36-cycle-a"); + Assert.True(await store.TryCreateAsync("r36-cycle-leaf", leaf, TimeSpan.FromMinutes(1))); + + await using var lease = (await FlowStateConcurrency.TryAcquireExecutionLeaseAsync(store, "r36-cycle-leaf", ShortLedgerOptions, NullLogger.Instance, clock))!; + var context = CreateContext(provider, leaf, store, lease, ShortLedgerOptions, clock, transport); + + var ex = await Assert.ThrowsAsync(() => context.DelayAsync("long-wait", TimeSpan.FromHours(1))); + + Assert.Contains("cycle", ex.Message, StringComparison.Ordinal); + Assert.Equal(0, transport.Count); + } + + // --------------------------------------------------------------------------------------------- + // F8 — the lease holder's disposal released the lease with an unbounded, uncancelable call: a + // store whose release never answered kept a FINISHED execution's disposal — the executor's + // `await using`, the job's scope, the worker slot, the acknowledgement — pending forever. + + /// + /// Pre-fix failure: disposal is still pending after eleven virtual seconds (and after ten + /// virtual minutes), and the release received CancellationToken.None. + /// + [Fact] + public async Task LeaseDisposal_HangingRelease_IsAbandonedAfterItsBudget_WithACancelableToken() + { + var clock = new VirtualTimeProvider(); + var inner = new InMemoryFlowStateStore(clock); + var store = new FaultingFlowStateStore(inner) { HangRelease = true }; + var logger = new CollectingLogger(); + Assert.True(await inner.TryCreateAsync("r36-release", State("r36-release"), TimeSpan.FromDays(1))); + var lease = (await FlowStateConcurrency.TryAcquireExecutionLeaseAsync(store, "r36-release", new DurableFlowOptions(), logger, clock))!; + + var dispose = lease.DisposeAsync().AsTask(); + await store.ReleaseEntered.Task.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.True(store.ReleaseToken.CanBeCanceled, "the release must receive a token the store can honor"); + Assert.False(dispose.IsCompleted); + + clock.Advance(TimeSpan.FromSeconds(11)); + await dispose.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.True(store.ReleaseToken.IsCancellationRequested); + Assert.Contains(logger.Messages, message => message.Contains("release did not complete within", StringComparison.Ordinal)); + + // The abandoned call eventually completing is observed, not fatal. + store.ReleaseGate.SetResult(); + await Task.Delay(20); + } +} diff --git a/tests/AsyncResponse.Tests/Round37NewApiTests.cs b/tests/AsyncResponse.Tests/Round37NewApiTests.cs new file mode 100644 index 000000000..a973c514b --- /dev/null +++ b/tests/AsyncResponse.Tests/Round37NewApiTests.cs @@ -0,0 +1,199 @@ +using AsyncResponse.Transports.Kafka; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using System.Diagnostics.Metrics; +using Xunit; + +namespace AsyncResponse.Tests; + +/// +/// Round-37 pins over API the round introduced — they do not compile against the pre-fix tree, so +/// they live apart from the behavior pins in . +/// +public sealed class Round37NewApiTests +{ + // ---------- F2: WorkerJobTooLargeException and the estimator behind the producer-side check ---------- + + [Fact] + public async Task EnqueueWorkerAsync_OverTheBudget_ThrowsWorkerJobTooLargeException_WithTheMeasurement() + { + var transport = new Round37RegressionTests.CapturingWorkerTransport(); + var services = new ServiceCollection(); + services.AddSingleton(typeof(ILogger<>), typeof(NullLogger<>)); + services.AddSingleton(transport); + services.AddAsyncResponse(o => o.MaxInboundMessageChars = 4096).WithInMemoryChannel(); + await using var provider = services.BuildServiceProvider(); + var builder = provider.GetRequiredService(); + + var ex = await Assert.ThrowsAsync(() => + builder.EnqueueWorkerAsync(probe => probe.RunAsync(new string('x', 8192)))); + + Assert.Equal(4096, ex.Limit); + Assert.True(ex.SerializedLength > 8192, $"SerializedLength {ex.SerializedLength} should cover the 8192-character argument."); + Assert.Contains("MaxInboundMessageChars", ex.Message, StringComparison.Ordinal); + Assert.Empty(transport.Published); + } + + [Fact] + public async Task DurableFlowStart_OverTheBudget_SurfacesWorkerJobTooLargeException_Unwrapped() + { + var transport = new Round37RegressionTests.CapturingWorkerTransport(); + var services = new ServiceCollection(); + services.AddSingleton(typeof(ILogger<>), typeof(NullLogger<>)); + services.AddSingleton(transport); + services.AddAsyncResponse(o => o.MaxInboundMessageChars = 4096).WithInMemoryChannel().WithInMemoryDurableFlows(); + await using var provider = services.BuildServiceProvider(); + + // Not DurableFlowNotDispatchedException: that one means "retry the start", and this start + // fails the same way every time. + await Assert.ThrowsAsync(() => + provider.GetRequiredService().StartAsync( + new Round37RegressionTests.R37Input(new string('x', 8192)))); + Assert.Equal(0, transport.Attempts); + } + + [Fact] + public async Task NoBudget_PublishesAnythingTheTransportTakes() + { + var transport = new Round37RegressionTests.CapturingWorkerTransport(); + var services = new ServiceCollection(); + services.AddSingleton(typeof(ILogger<>), typeof(NullLogger<>)); + services.AddSingleton(transport); + services.AddAsyncResponse(o => o.MaxInboundMessageChars = null).WithInMemoryChannel(); + await using var provider = services.BuildServiceProvider(); + + await provider.GetRequiredService() + .EnqueueWorkerAsync(probe => probe.RunAsync(new string('x', 9_000_000))); + Assert.Single(transport.Published); + } + + public static TheoryData EstimableEnvelopes() + { + static WorkerJobEnvelope Envelope(params object?[] values) => new() + { + Call = new ReflectionCallDto + { + ServiceInterfaceFullName = "AsyncResponse.Tests.Round37RegressionTests+IR37Probe", + MethodName = "RunAsync", + Params = values.Select(CallbackParam.ForValue).ToArray() + }, + CorrelationId = "corr-é中\U0001F600" + }; + + return new TheoryData + { + { "ascii", Envelope("plain ascii text") }, + { "quotes and backslashes", Envelope(new string('"', 500) + new string('\\', 500)) }, + { "non-ascii and emoji", Envelope(new string('中', 400) + string.Concat(Enumerable.Repeat("\U0001F600", 200))) }, + { "control characters", Envelope(new string((char)1, 300) + "\r\n\t") }, + { "html-sensitive", Envelope(new string('<', 200) + new string('&', 200) + new string('+', 200)) }, + { "scalars", Envelope(int.MaxValue, long.MinValue, double.MaxValue, decimal.MaxValue, Guid.NewGuid(), DateTime.MaxValue, DateTimeOffset.MinValue, TimeSpan.MaxValue, true, 'q', null) }, + { + "context and reply target", + new WorkerJobEnvelope + { + Call = new ReflectionCallDto + { + ServiceInterfaceFullName = "S", + MethodName = "M", + Params = [CallbackParam.ForPlaceholder(PlaceholderType.CorrelationId), CallbackParam.ForValue("v")] + }, + Context = new Dictionary { ["tenant-ü"] = new string('"', 100), ["trace"] = "00-abc-01" }, + ReplyTarget = new AsyncResponseReplyTarget + { + Name = "default", + Transport = "kafka", + Address = new string('中', 50), + Properties = { ["group"] = "g\"1" } + }, + NotBeforeUtc = DateTime.UtcNow, + LastRedelayRemaining = TimeSpan.FromMinutes(1), + RedelayStallCount = 2 + } + } + }; + } + + [Theory] + [MemberData(nameof(EstimableEnvelopes))] + public void UpperBoundEstimate_NeverUndercountsTheSerializedEnvelope(string label, WorkerJobEnvelope envelope) + { + // The estimate is what lets the hot path skip a second serialization; it is only safe if + // it never says "fits" for an envelope that does not. + Assert.True(AsyncResponseBuilderBase.TryEstimateUpperBound(envelope, out var upperBound), label); + var actual = AsyncResponseJson.Serialize(envelope).Length; + Assert.True(upperBound >= actual, $"{label}: estimate {upperBound} undercounts the serialized {actual}."); + } + + [Fact] + public void UpperBoundEstimate_DeclinesAnArgumentItCannotBound() + { + var envelope = new WorkerJobEnvelope + { + Call = new ReflectionCallDto + { + ServiceInterfaceFullName = "S", + MethodName = "M", + Params = [CallbackParam.ForValue(new Round37RegressionTests.R37Input("x"))] + } + }; + + Assert.False(AsyncResponseBuilderBase.TryEstimateUpperBound(envelope, out _)); + } + + // ---------- F4: the overload form of the indeterminate contract and its counter ---------- + + [Fact] + public void IndeterminateDeliveryException_OverloadForm_CarriesTheBufferedCount() + { + var ex = new AsyncResponseIndeterminateDeliveryException("corr", 1024); + + Assert.Equal("corr", ex.CorrelationId); + Assert.Equal(1024, ex.BufferedMessages); + Assert.Contains("1024", ex.Message, StringComparison.Ordinal); + Assert.Contains("indeterminate", ex.Message, StringComparison.Ordinal); + Assert.Equal(0, new AsyncResponseIndeterminateDeliveryException("corr", TimeSpan.FromSeconds(1)).BufferedMessages); + } + + [Fact] + public void RecordWaiterOverload_IncrementsTheOverloadedWaitsCounter_TaggedByChannel() + { + var measurements = new List<(string Instrument, long Value, string? Channel)>(); + using var listener = new MeterListener + { + InstrumentPublished = (instrument, l) => + { + if (instrument.Meter.Name == AsyncResponseDiagnostics.MeterName && instrument.Name == "asyncresponse.channel.overloaded_waits") + l.EnableMeasurementEvents(instrument); + } + }; + listener.SetMeasurementEventCallback((instrument, value, tags, _) => + { + string? channel = null; + foreach (var tag in tags) + { + if (tag.Key == "channel") + channel = tag.Value?.ToString(); + } + + lock (measurements) + measurements.Add((instrument.Name, value, channel)); + }); + listener.Start(); + + AsyncResponseDiagnostics.RecordWaiterOverload("redis"); + + lock (measurements) + { + var measurement = Assert.Single(measurements); + Assert.Equal(("asyncresponse.channel.overloaded_waits", 1L, "redis"), measurement); + } + } + + // ---------- F7: the new Kafka knob ---------- + + [Fact] + public void KafkaSubscriberOptions_DetachHandlerAfter_DefaultsToOneSecond() + => Assert.Equal(TimeSpan.FromSeconds(1), new KafkaSubscriberOptions().DetachHandlerAfter); +} diff --git a/tests/AsyncResponse.Tests/Round37RegressionTests.cs b/tests/AsyncResponse.Tests/Round37RegressionTests.cs new file mode 100644 index 000000000..85e4d85f3 --- /dev/null +++ b/tests/AsyncResponse.Tests/Round37RegressionTests.cs @@ -0,0 +1,312 @@ +using AsyncResponse.Channels.Redis; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Moq; +using StackExchange.Redis; +using Xunit; + +namespace AsyncResponse.Tests; + +/// +/// Regressions for round 37 (external holistic review of ba63beb): behavior pins that compile +/// against the pre-fix tree and fail there. Pins over API this round introduced +/// (WorkerJobTooLargeException, the estimator, the overload constructor and counter, +/// KafkaSubscriberOptions.DetachHandlerAfter) live in ; +/// the Kafka poll-loop pins live in KafkaSubscriberTests / KafkaDispatcherTests +/// and the Cosmos lease-parameter pin in CosmosDurableFlowStateStoreTests. +/// +public sealed class Round37RegressionTests +{ + // --------------------------------------------------------------------------------------------- + // F2 — a worker envelope the consuming ingress would refuse was published anyway. The ingress + // acknowledges a message over MaxInboundMessageChars WITHOUT executing it (an oversized + // message never gets smaller, so redelivery would hot-loop), so the producer's publish + // "succeeded", the caller kept a flow id for a Running ledger with Attempts = 0, and the + // work silently never ran. The budget is now enforced before the publish, in the + // caller's stack, measured on the serialized envelope the way the ingress measures it. + + public interface IR37Probe + { + Task RunAsync(string payload); + } + + public sealed class R37Probe : IR37Probe + { + public int Calls; + + public Task RunAsync(string payload) + { + Interlocked.Increment(ref Calls); + return Task.CompletedTask; + } + } + + public sealed record R37Input(string Payload); + + public sealed class R37Flow : IDurableFlow + { + public Task ExecuteAsync(IDurableFlowContext flow, R37Input input) => Task.CompletedTask; + } + + /// A transport that accepts everything, as every database transport and most brokers do. + internal sealed class CapturingWorkerTransport : IWorkerTransport + { + private int _attempts; + + public List Published { get; } = []; + public int Attempts => Volatile.Read(ref _attempts); + + public Task PublishAsync(WorkerJobEnvelope job, CancellationToken cancellationToken = default) + { + Interlocked.Increment(ref _attempts); + lock (Published) + { + Published.Add(job); + } + + return Task.CompletedTask; + } + } + + private static ServiceProvider BuildProducer(CapturingWorkerTransport transport, int limit, R37Probe? probe = null) + { + var services = new ServiceCollection(); + services.AddSingleton(typeof(ILogger<>), typeof(NullLogger<>)); + services.AddSingleton(transport); + if (probe is not null) + services.AddSingleton(probe); + services.AddAsyncResponse(o => o.MaxInboundMessageChars = limit).WithInMemoryChannel().WithInMemoryDurableFlows(); + return services.BuildServiceProvider(); + } + + [Fact] + public async Task EnqueueWorkerAsync_EnvelopeOverTheIngressBudget_ThrowsBeforePublishing() + { + var transport = new CapturingWorkerTransport(); + await using var provider = BuildProducer(transport, limit: 4096); + var builder = provider.GetRequiredService(); + + await Assert.ThrowsAnyAsync(() => + builder.EnqueueWorkerAsync(probe => probe.RunAsync(new string('x', 8192)))); + + Assert.Empty(transport.Published); + } + + [Fact] + public async Task EnqueueWorkerAsync_JsonEscapingCountsTowardTheBudget() + { + // 3000 quote characters are under the 4096 budget as a string and over it once serialized + // (each becomes \" on the wire) — the ingress measures the wire form. + var transport = new CapturingWorkerTransport(); + await using var provider = BuildProducer(transport, limit: 4096); + var builder = provider.GetRequiredService(); + + await Assert.ThrowsAnyAsync(() => + builder.EnqueueWorkerAsync(probe => probe.RunAsync(new string('"', 3000)))); + + Assert.Empty(transport.Published); + } + + [Fact] + public async Task EnqueueWorkerAsync_EnvelopeWithinTheBudget_PublishesAndTheIngressExecutesTheSameJson() + { + // The producer/ingress boundary end to end: what the producer let through, the ingress + // runs — the same serialization on both sides, measured the same way. + var transport = new CapturingWorkerTransport(); + var probe = new R37Probe(); + await using var provider = BuildProducer(transport, limit: 4096, probe); + var builder = provider.GetRequiredService(); + + await builder.EnqueueWorkerAsync(p => p.RunAsync(new string('x', 2048))); + + var envelope = Assert.Single(transport.Published); + var json = AsyncResponseJson.Serialize(envelope); + Assert.True(json.Length <= 4096, $"Published envelope is {json.Length} characters."); + + await provider.GetRequiredService().HandleWorkerMessageAsync(json); + Assert.Equal(1, probe.Calls); + } + + [Fact] + public async Task DurableFlowStart_InputOverTheIngressBudget_ThrowsWithNoAttemptAndNothingPersisted() + { + // The start job carries the initial ledger, input included. Before: the transport took + // it, the ingress dropped it, and the caller held an id for a run nothing would execute. + // Now: no publish attempt at all (the failure is deterministic, so the retry ladder is + // not entered) and no ledger. + var transport = new CapturingWorkerTransport(); + await using var provider = BuildProducer(transport, limit: 4096); + var flows = provider.GetRequiredService(); + + await Assert.ThrowsAnyAsync(() => + flows.StartAsync(new R37Input(new string('x', 8192)), "r37-oversized")); + + Assert.Equal(0, transport.Attempts); + Assert.Null(await flows.GetStateAsync("r37-oversized")); + } + + // --------------------------------------------------------------------------------------------- + // F4 — the Redis channel buffered a progress flood without bound. Its subscription handler + // awaited admission to the bounded per-correlation-id executor, which does not slow the + // publisher (Redis pub/sub is fire-and-forget): it only parked the SDK's message loop, + // and the SDK's ChannelMessageQueue behind it is unbounded — 20,000 messages sat there + // against an executor of 1,024. Admission is non-blocking now, and a message that finds + // the buffer full faults the wait as indeterminate and tears the subscription down. + + private sealed class RedisHarness + { + public Mock Multiplexer { get; } = new(); + public Mock RedisSubscriber { get; } = new(); + public Mock Store { get; } = new(); + public FakeRedisChannelSubscriber Subscriber { get; } = new(); + public ServiceProvider Services { get; } = new ServiceCollection().BuildServiceProvider(); + + public RedisHarness() + { + Multiplexer.Setup(m => m.GetSubscriber(It.IsAny())).Returns(RedisSubscriber.Object); + Store + .Setup(s => s.SaveAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(Task.CompletedTask); + Store + .Setup(s => s.TryDeleteAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(true); + } + + public RedisAsyncResponseChannel CreateChannel() => new( + Services.GetRequiredService(), + Multiplexer.Object, + Store.Object, + Options.Create(new RedisAsyncResponseOptions + { + DefaultTimeout = TimeSpan.FromSeconds(30), + RecoveryStateExpiry = TimeSpan.FromMinutes(5) + }), + new AsyncResponseContextPropagation([]), + NullLogger.Instance, + Subscriber); + } + + /// The channel's async subscribe seam: captures the handler so the test can push messages through it. + private sealed class FakeRedisChannelSubscriber : IRedisChannelSubscriber + { + private int _unsubscribeCount; + + public RedisChannel SubscribedChannel { get; private set; } + public Func? Handler { get; private set; } + public int UnsubscribeCount => Volatile.Read(ref _unsubscribeCount); + + public Task SubscribeAsync(RedisChannel channel, Func onMessage) + { + SubscribedChannel = channel; + Handler = onMessage; + return Task.FromResult(new Subscription(this)); + } + + private sealed class Subscription(FakeRedisChannelSubscriber owner) : IRedisChannelSubscription + { + public ValueTask DisposeAsync() + { + Interlocked.Increment(ref owner._unsubscribeCount); + return ValueTask.CompletedTask; + } + } + } + + private const string ProgressEnvelope = + """{"SchemaVersion":1,"Success":true,"Payload":{"Status":1,"Message":"progress"},"ExceptionMessage":null,"ExceptionStackTrace":null}"""; + + [Fact] + public async Task RedisChannel_ProgressFloodBehindASlowPredicate_FaultsTheWaitAsIndeterminate_InsteadOfBufferingWithoutBound() + { + var harness = new RedisHarness(); + var channel = harness.CreateChannel(); + var predicateEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releasePredicate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + await using var waiter = await channel.CreateResponseWaiter( + "corr-flood", + completionPredicate: async _ => + { + predicateEntered.TrySetResult(); + await releasePredicate.Task; + return false; + }, + timeout: TimeSpan.FromSeconds(30)); + try + { + var handler = harness.Subscriber.Handler!; + var subscribedChannel = harness.Subscriber.SubscribedChannel; + + // The first message parks the executor's reader inside the predicate. + await handler(subscribedChannel, ProgressEnvelope).WaitAsync(TimeSpan.FromSeconds(5)); + await predicateEntered.Task.WaitAsync(TimeSpan.FromSeconds(5)); + + // The flood. Every delivery must return promptly — admitted or refused — because a + // handler parked on admission is exactly what let the SDK queue grow without bound. + // The old handler parks on the delivery that finds the executor full. + var delivered = 0; + while (!waiter.ResponseTask.IsCompleted && delivered < 1100) + { + await handler(subscribedChannel, ProgressEnvelope).WaitAsync(TimeSpan.FromSeconds(2)); + delivered++; + } + + Assert.True(waiter.ResponseTask.IsFaulted, $"The wait was not faulted after {delivered} deliveries."); + var overload = await Assert.ThrowsAsync(() => waiter.ResponseTask); + Assert.Equal("corr-flood", overload.CorrelationId); + + // Torn down, so the flood stops here: unsubscribed and the recovery registration gone. + await WaitUntilAsync(() => harness.Subscriber.UnsubscribeCount == 1); + harness.Store.Verify(s => s.TryDeleteAsync("corr-flood", It.IsAny(), It.IsAny()), Times.AtLeastOnce); + } + finally + { + releasePredicate.TrySetResult(); + } + } + + [Fact] + public async Task RedisChannel_TerminalResponseAheadOfTheBuffer_StillCompletesTheWait() + { + // The bound never costs a response that was admitted: a terminal message admitted while + // the predicate is slow completes the wait once the executor reaches it. + var harness = new RedisHarness(); + var channel = harness.CreateChannel(); + var releasePredicate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var predicateCalls = 0; + + await using var waiter = await channel.CreateResponseWaiter( + "corr-admitted", + completionPredicate: async payload => + { + if (Interlocked.Increment(ref predicateCalls) == 1) + await releasePredicate.Task; + return payload.Status == OperationStatus.Completed; + }, + timeout: TimeSpan.FromSeconds(30)); + + var handler = harness.Subscriber.Handler!; + await handler(harness.Subscriber.SubscribedChannel, ProgressEnvelope); + await WaitUntilAsync(() => Volatile.Read(ref predicateCalls) == 1); + await handler( + harness.Subscriber.SubscribedChannel, + """{"SchemaVersion":1,"Success":true,"Payload":{"Status":2,"Message":"done"},"ExceptionMessage":null,"ExceptionStackTrace":null}"""); + + releasePredicate.SetResult(); + Assert.Equal(OperationStatus.Completed, (await waiter.ResponseTask.WaitAsync(TimeSpan.FromSeconds(5))).Status); + } + + private static async Task WaitUntilAsync(Func condition, TimeSpan? timeout = null) + { + var deadline = DateTimeOffset.UtcNow + (timeout ?? TimeSpan.FromSeconds(5)); + while (!condition()) + { + if (DateTimeOffset.UtcNow > deadline) + throw new TimeoutException("Condition was not reached within the timeout."); + + await Task.Delay(10); + } + } +} diff --git a/tests/AsyncResponse.Tests/Round38NewApiTests.cs b/tests/AsyncResponse.Tests/Round38NewApiTests.cs new file mode 100644 index 000000000..47dcdce95 --- /dev/null +++ b/tests/AsyncResponse.Tests/Round38NewApiTests.cs @@ -0,0 +1,536 @@ +using AsyncResponse.Testing; +using AsyncResponse.Transports.Kafka; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using System.Diagnostics.Metrics; +using System.Text.Json; +using Xunit; + +namespace AsyncResponse.Tests; + +/// +/// Round-38 pins over API the round introduced — they do not compile against the pre-fix tree, so +/// they live apart from the behavior pins in . +/// +public sealed class Round38NewApiTests +{ + // ---------- F1: FlowState.RetainUntilUtc, the retention floor every ledger write honors ---------- + + [Fact] + public void RetainUntilUtc_IsOmittedFromTheLedgerWhenUnset_AndRoundTripsWhenSet() + { + var state = new FlowState { FlowId = "r38-wire", Status = FlowRunStatus.Running }; + Assert.DoesNotContain("RetainUntilUtc", FlowStateJson.Serialize(state), StringComparison.Ordinal); + + var floor = new DateTime(2026, 9, 11, 12, 0, 0, DateTimeKind.Utc); + state.RetainUntilUtc = floor; + var json = FlowStateJson.Serialize(state); + Assert.Contains("\"RetainUntilUtc\":\"2026-09-11T12:00:00Z\"", json, StringComparison.Ordinal); + Assert.Equal(floor, FlowStateJson.Deserialize(json, "r38-wire").RetainUntilUtc); + } + + [Theory] + [InlineData(FlowRunStatus.Running, true)] + [InlineData(FlowRunStatus.Suspended, true)] + [InlineData(FlowRunStatus.Succeeded, false)] + [InlineData(FlowRunStatus.Failed, false)] + public void EffectiveTtl_RaisesTheRequestedTtlToTheFloor_ForLiveRunsOnly(FlowRunStatus status, bool honored) + { + var now = new DateTime(2026, 9, 11, 12, 0, 0, DateTimeKind.Utc); + var state = new FlowState { Status = status, RetainUntilUtc = now.AddHours(1) }; + + var ttl = FlowStateRetention.EffectiveTtl(state, TimeSpan.FromMinutes(1), now); + + Assert.Equal(honored ? TimeSpan.FromHours(1) : TimeSpan.FromMinutes(1), ttl); + } + + [Fact] + public void EffectiveTtl_KeepsARequestedTtlAlreadyPastTheFloor_AndSaturatesAtThePersistenceCeiling() + { + var now = new DateTime(2026, 9, 11, 12, 0, 0, DateTimeKind.Utc); + Assert.Equal(TimeSpan.FromHours(2), FlowStateRetention.EffectiveTtl(new FlowState { RetainUntilUtc = now.AddHours(1) }, TimeSpan.FromHours(2), now)); + Assert.Equal(TimeSpan.FromMinutes(1), FlowStateRetention.EffectiveTtl(new FlowState { RetainUntilUtc = now.AddHours(-1) }, TimeSpan.FromMinutes(1), now)); + Assert.Equal(TimeSpan.FromMinutes(1), FlowStateRetention.EffectiveTtl(new FlowState(), TimeSpan.FromMinutes(1), now)); + Assert.Equal( + AsyncResponseChannelOptions.MaxPersistenceTtl, + FlowStateRetention.EffectiveTtl(new FlowState { RetainUntilUtc = DateTime.MaxValue }, TimeSpan.FromMinutes(1), now)); + } + + [Fact] + public void RaiseFloor_NeverLowersAnExistingFloor() + { + var now = new DateTime(2026, 9, 11, 12, 0, 0, DateTimeKind.Utc); + var state = new FlowState(); + Assert.Equal(now.AddHours(2), FlowStateRetention.RaiseFloor(state, now, TimeSpan.FromHours(2))); + Assert.Equal(now.AddHours(2), FlowStateRetention.RaiseFloor(state, now, TimeSpan.FromHours(1))); + Assert.Equal(now.AddHours(3), FlowStateRetention.RaiseFloor(state, now, TimeSpan.FromHours(3))); + Assert.True(FlowStateRetention.Covers(state, now.AddHours(3))); + Assert.False(FlowStateRetention.Covers(state, now.AddHours(3).AddTicks(1))); + } + + private static readonly DurableFlowOptions ShortLedgerOptions = new() + { + StateExpiry = TimeSpan.FromMinutes(1), + DefaultStepTimeout = TimeSpan.FromSeconds(10), + TimerInProcessThreshold = TimeSpan.Zero + }; + + private static FlowState State(string id, string? parent = null) => new() + { + FlowId = id, + ParentFlowId = parent, + Status = FlowRunStatus.Running, + FlowTypeName = typeof(Round38RegressionTests.R38NoopFlow).FullName, + InputTypeName = typeof(Round38RegressionTests.R38Input).FullName, + InputJson = JsonSerializer.Serialize(new Round38RegressionTests.R38Input("x")), + CreatedAtUtc = DateTime.UtcNow, + UpdatedAtUtc = DateTime.UtcNow + }; + + /// + /// The executor's per-attempt save and every checkpoint go through the lease: a plain + /// one-minute save of a run whose floor is an hour out keeps the ledger for the hour. A + /// terminal save does not — a failed run is not retained for the sleep it never finished. + /// + [Fact] + public async Task LeaseSave_HonorsTheRetentionFloorForALiveRun_AndIgnoresItOnceTerminal() + { + var clock = new VirtualTimeProvider(); + var store = new InMemoryFlowStateStore(clock); + var now = clock.GetUtcNow().UtcDateTime; + var live = State("r38-live"); + var terminal = State("r38-terminal"); + Assert.True(await store.TryCreateAsync("r38-live", live, TimeSpan.FromMinutes(1))); + Assert.True(await store.TryCreateAsync("r38-terminal", terminal, TimeSpan.FromMinutes(1))); + + await using (var lease = (await FlowStateConcurrency.TryAcquireExecutionLeaseAsync(store, "r38-live", ShortLedgerOptions, NullLogger.Instance, clock))!) + { + live.RetainUntilUtc = now.AddHours(1); + await lease.SaveAsync(live, TimeSpan.FromMinutes(1)); + } + + await using (var lease = (await FlowStateConcurrency.TryAcquireExecutionLeaseAsync(store, "r38-terminal", ShortLedgerOptions, NullLogger.Instance, clock))!) + { + terminal.RetainUntilUtc = now.AddHours(1); + terminal.Status = FlowRunStatus.Failed; + await lease.SaveAsync(terminal, TimeSpan.FromMinutes(1)); + } + + clock.Advance(TimeSpan.FromMinutes(30)); + Assert.NotNull(await store.LoadAsync("r38-live")); + Assert.Null(await store.LoadAsync("r38-terminal")); + } + + [Fact] + public async Task MutateAsync_HonorsTheRetentionFloor() + { + var clock = new VirtualTimeProvider(); + var store = new InMemoryFlowStateStore(clock); + var state = State("r38-mutate"); + state.RetainUntilUtc = clock.GetUtcNow().UtcDateTime.AddHours(1); + Assert.True(await store.TryCreateAsync("r38-mutate", state, TimeSpan.FromMinutes(1))); + + Assert.True(await FlowStateConcurrency.MutateAsync(store, "r38-mutate", TimeSpan.FromMinutes(1), clock, s => + { + s.LastMessage = "recovered"; + return true; + })); + + clock.Advance(TimeSpan.FromMinutes(30)); + Assert.Equal("recovered", (await store.LoadAsync("r38-mutate"))?.LastMessage); + } + + private static ServiceProvider BuildProvider(IWorkerTransport transport, TimeProvider clock) + { + var services = new ServiceCollection(); + services.AddSingleton(typeof(Microsoft.Extensions.Logging.ILogger<>), typeof(NullLogger<>)); + services.AddSingleton(clock); + services + .AddAsyncResponse() + .WithInMemoryChannel() + .WithInMemoryDurableFlows() + .WithDurableFlow(); + services.AddSingleton(transport); + return services.BuildServiceProvider(); + } + + private static DurableFlowContext CreateContext(ServiceProvider provider, FlowState state, IFlowStateStore store, FlowExecutionLease lease, TimeProvider clock, IWorkerTransport transport) + => new( + state, + store, + provider.GetRequiredService(), + provider.GetRequiredService(), + ShortLedgerOptions, + provider.GetRequiredService(), + recoverableSubscriber: null, + NullLogger.Instance, + lease, + clock, + workerTransport: transport); + + /// A leaf's park stamps the floor on itself and on every Running ancestor up to the root. + [Fact] + public async Task AncestorExtension_StampsTheRetentionFloorOnTheRunAndEveryAncestor() + { + var clock = new VirtualTimeProvider(); + var transport = new Round38RegressionTests.CapturingDelayedTransport(); + await using var provider = BuildProvider(transport, clock); + var store = provider.GetRequiredService(); + Assert.True(await store.TryCreateAsync("r38-g", State("r38-g"), TimeSpan.FromMinutes(1))); + Assert.True(await store.TryCreateAsync("r38-g:p", State("r38-g:p", "r38-g"), TimeSpan.FromMinutes(1))); + var leaf = State("r38-g:p:c", "r38-g:p"); + Assert.True(await store.TryCreateAsync("r38-g:p:c", leaf, TimeSpan.FromMinutes(1))); + + await using (var lease = (await FlowStateConcurrency.TryAcquireExecutionLeaseAsync(store, "r38-g:p:c", ShortLedgerOptions, NullLogger.Instance, clock))!) + { + var context = CreateContext(provider, leaf, store, lease, clock, transport); + await Assert.ThrowsAsync(() => context.DelayAsync("long-wait", TimeSpan.FromHours(1))); + } + + var wake = clock.GetUtcNow().UtcDateTime.AddHours(1); + foreach (var id in new[] { "r38-g", "r38-g:p", "r38-g:p:c" }) + { + var floor = (await store.LoadAsync(id))!.RetainUntilUtc; + Assert.True(floor is { } f && f >= wake + ShortLedgerOptions.StateExpiry, $"{id} carries no floor covering the park (got {floor})"); + } + + // The floor is what an unrelated write of the parent honors: a lease-less recovery-style + // mutation with the plain one-minute TTL leaves the parent alive for the hour. + Assert.True(await FlowStateConcurrency.MutateAsync(store, "r38-g:p", TimeSpan.FromMinutes(1), clock, s => true)); + clock.Advance(TimeSpan.FromMinutes(30)); + Assert.NotNull(await store.LoadAsync("r38-g:p")); + Assert.NotNull(await store.LoadAsync("r38-g")); + } + + /// Wraps a store so every lease-less write of one id loses its revision race, and counts the attempts. + private sealed class AlwaysLosingFlowStateStore(IFlowStateStore inner, string loseOn, DateTime? carryFloor) : IFlowStateStore + { + public int AncestorWriteAttempts { get; private set; } + + public Task LoadAsync(string flowId, CancellationToken cancellationToken = default) + => inner.LoadAsync(flowId, cancellationToken); + + public Task TryCreateAsync(string flowId, FlowState state, TimeSpan ttl, CancellationToken cancellationToken = default) + => inner.TryCreateAsync(flowId, state, ttl, cancellationToken); + + public async Task TryUpdateAsync(string flowId, FlowState state, long expectedRevision, TimeSpan ttl, string? leaseId = null, CancellationToken cancellationToken = default) + { + if (leaseId is null && string.Equals(flowId, loseOn, StringComparison.Ordinal)) + { + AncestorWriteAttempts++; + var current = (await inner.LoadAsync(flowId, cancellationToken))!; + var old = current.Revision; + current.Revision++; + if (carryFloor is { } floor) + current.RetainUntilUtc = floor; + Assert.True(await inner.TryUpdateAsync(flowId, current, old, TimeSpan.FromMinutes(1), null, cancellationToken)); + } + + return await inner.TryUpdateAsync(flowId, state, expectedRevision, ttl, leaseId, cancellationToken); + } + + public Task TryAcquireLeaseAsync(string flowId, string leaseId, TimeSpan leaseDuration, CancellationToken cancellationToken = default) + => inner.TryAcquireLeaseAsync(flowId, leaseId, leaseDuration, cancellationToken); + + public Task TryRenewLeaseAsync(string flowId, string leaseId, TimeSpan leaseDuration, CancellationToken cancellationToken = default) + => inner.TryRenewLeaseAsync(flowId, leaseId, leaseDuration, cancellationToken); + + public Task ReleaseLeaseAsync(string flowId, string leaseId, CancellationToken cancellationToken = default) + => inner.ReleaseLeaseAsync(flowId, leaseId, cancellationToken); + + public Task TryDeleteAsync(string flowId, CancellationToken cancellationToken = default) + => inner.TryDeleteAsync(flowId, cancellationToken); + } + + /// + /// A competing write that already carries a floor reaching the park proves the retention: the + /// extension re-reads, sees it, and moves on without a second write. + /// + [Fact] + public async Task AncestorExtension_AConcurrentWriteThatCarriesTheFloor_EndsTheRetryWithoutAnotherWrite() + { + var clock = new VirtualTimeProvider(); + var transport = new Round38RegressionTests.CapturingDelayedTransport(); + await using var provider = BuildProvider(transport, clock); + var inner = provider.GetRequiredService(); + var store = new AlwaysLosingFlowStateStore(inner, "r38-floored-root", carryFloor: clock.GetUtcNow().UtcDateTime.AddDays(2)); + Assert.True(await inner.TryCreateAsync("r38-floored-root", State("r38-floored-root"), TimeSpan.FromMinutes(1))); + var child = State("r38-floored-root:c", "r38-floored-root"); + Assert.True(await inner.TryCreateAsync("r38-floored-root:c", child, TimeSpan.FromMinutes(1))); + + await using (var lease = (await FlowStateConcurrency.TryAcquireExecutionLeaseAsync(store, child.FlowId!, ShortLedgerOptions, NullLogger.Instance, clock))!) + { + var context = CreateContext(provider, child, store, lease, clock, transport); + await Assert.ThrowsAsync(() => context.DelayAsync("long-wait", TimeSpan.FromHours(1))); + } + + Assert.Equal(1, store.AncestorWriteAttempts); + Assert.Equal(1, transport.Count); + } + + /// + /// Losing every bounded attempt abandons the park with nothing published — the delivery + /// retries it later — instead of parking on an ancestor whose retention is unproven, and + /// instead of fighting a live ancestor without end. + /// + [Fact] + public async Task AncestorExtension_LosingEveryAttempt_AbandonsTheParkWithNothingPublished() + { + var clock = new VirtualTimeProvider(); + var transport = new Round38RegressionTests.CapturingDelayedTransport(); + await using var provider = BuildProvider(transport, clock); + var inner = provider.GetRequiredService(); + var store = new AlwaysLosingFlowStateStore(inner, "r38-busy-root", carryFloor: null); + Assert.True(await inner.TryCreateAsync("r38-busy-root", State("r38-busy-root"), TimeSpan.FromMinutes(1))); + var child = State("r38-busy-root:c", "r38-busy-root"); + Assert.True(await inner.TryCreateAsync("r38-busy-root:c", child, TimeSpan.FromMinutes(1))); + + await using (var lease = (await FlowStateConcurrency.TryAcquireExecutionLeaseAsync(store, child.FlowId!, ShortLedgerOptions, NullLogger.Instance, clock))!) + { + var context = CreateContext(provider, child, store, lease, clock, transport); + var ex = await Assert.ThrowsAsync(() => context.DelayAsync("long-wait", TimeSpan.FromHours(1))); + Assert.Contains("r38-busy-root", ex.Message, StringComparison.Ordinal); + Assert.Contains("abandoned", ex.Message, StringComparison.Ordinal); + } + + Assert.Equal(DurableFlowContext.MaxAncestorExtensionAttempts, store.AncestorWriteAttempts); + Assert.Equal(0, transport.Count); + } + + // ---------- F3: InMemoryWorkerTransportOptions.DelayedJobCapacity ---------- + + private static WorkerJobEnvelope DelayedJob() => new() + { + Call = new ReflectionCallDto + { + ServiceInterfaceFullName = "AsyncResponse.Tests.IRound38Probe", + MethodName = "RunAsync", + Params = [] + } + }; + + [Fact] + public async Task DelayedJobCapacity_RejectsAnInJobDelayedPublishAtTheBound_AndCountsIt() + { + var measurements = new List<(string Instrument, long Value)>(); + using var listener = new MeterListener + { + InstrumentPublished = (instrument, l) => + { + if (instrument.Meter.Name == AsyncResponseDiagnostics.MeterName) + l.EnableMeasurementEvents(instrument); + } + }; + listener.SetMeasurementEventCallback((instrument, value, _, _) => + { + lock (measurements) + measurements.Add((instrument.Name, value)); + }); + listener.Start(); + + var clock = new VirtualTimeProvider(); + var transport = new InMemoryWorkerTransport( + Options.Create(new InMemoryWorkerTransportOptions { QueueCapacity = 1, DelayedJobCapacity = 2 }), + clock); + + var rejection = await Task.Run(async () => + { + InMemoryWorkerTransport.InJobScope.MarkActive(); + await transport.PublishAsync(DelayedJob(), TimeSpan.FromMinutes(5)); + await transport.PublishAsync(DelayedJob(), TimeSpan.FromMinutes(5)); + return await Assert.ThrowsAsync(() => transport.PublishAsync(DelayedJob(), TimeSpan.FromMinutes(5))); + }); + + Assert.Contains(nameof(InMemoryWorkerTransportOptions.DelayedJobCapacity), rejection.Message, StringComparison.Ordinal); + Assert.Equal(2, transport.DelayedJobsHeld); + Assert.Equal(2, transport.SnapshotDelayedJobs().Count); + listener.RecordObservableInstruments(); + lock (measurements) + { + Assert.Contains(measurements, m => m.Instrument == "asyncresponse.worker.inmemory_delayed_rejections" && m.Value == 1); + Assert.Contains(measurements, m => m.Instrument == "asyncresponse.worker.inmemory_delayed_jobs" && m.Value >= 2); + } + + // The slot is held through the pending write: both fire into a one-slot queue, one lands + // and frees its slot, the other pends and keeps its slot. + clock.Advance(TimeSpan.FromMinutes(5)); + await Task.Delay(50); + Assert.Equal(1, transport.Reader.Count); + Assert.Equal(1, transport.DelayedJobsHeld); + Assert.True(transport.Reader.TryRead(out _)); + await Task.Delay(50); + Assert.Equal(0, transport.DelayedJobsHeld); + GC.KeepAlive(transport); + } + + [Fact] + public async Task DelayedJobCapacity_TheShutdownDrainFreesEverySlot_DroppedOrRetained() + { + var clock = new VirtualTimeProvider(); + var dropping = new InMemoryWorkerTransport(Options.Create(new InMemoryWorkerTransportOptions { DelayedJobCapacity = 2 }), clock); + await dropping.PublishAsync(DelayedJob(), TimeSpan.FromHours(1)); + await dropping.PublishAsync(DelayedJob(), TimeSpan.FromHours(1)); + Assert.Equal(2, dropping.DelayedJobsHeld); + dropping.BeginShutdownDrain(); + Assert.Equal(0, dropping.DelayedJobsHeld); + + var retaining = new InMemoryWorkerTransport(Options.Create(new InMemoryWorkerTransportOptions { DelayedJobCapacity = 2 }), clock); + await retaining.PublishAsync(DelayedJob(), TimeSpan.FromHours(1)); + var retained = retaining.BeginRetainingDelayedJobs(); + retaining.BeginShutdownDrain(); + Assert.Single(retained); + Assert.Equal(0, retaining.DelayedJobsHeld); + + // A delayed publish that arrives mid-drain is retained without holding a slot either. + await retaining.PublishAsync(DelayedJob(), TimeSpan.FromHours(1)); + Assert.Equal(2, retained.Count); + Assert.Equal(0, retaining.DelayedJobsHeld); + } + + [Fact] + public void DelayedJobCapacity_MustBePositive() + { + var ex = Assert.Throws(() => new InMemoryWorkerTransport( + Options.Create(new InMemoryWorkerTransportOptions { DelayedJobCapacity = 0 }))); + + Assert.Contains(nameof(InMemoryWorkerTransportOptions.DelayedJobCapacity), ex.Message, StringComparison.Ordinal); + } + + // ---------- F6: KafkaSubscriberOptions.FaultDrainTimeout ---------- + + private sealed class GateIngress : IAsyncResponseIngress + { + public TaskCompletionSource Entered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource Release { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public int SlowRuns; + + public Task HandleResponseMessageAsync(string json, string? correlationId) => Task.CompletedTask; + + public Task HandleWorkerMessageAsync(string json) + { + if (json != "slow-job") + return Task.CompletedTask; + + Interlocked.Increment(ref SlowRuns); + Entered.TrySetResult(); + return Release.Task; + } + } + + private static KafkaAsyncResponseTransportOptions KafkaOptions(TimeSpan faultDrainTimeout) + { + var options = KafkaTestData.NewOptions(); + options.WorkerTopic = "workers"; + options.CreateTopics = false; + options.SubscriberRetryBaseDelay = TimeSpan.FromMilliseconds(1); + options.SubscriberRetryMaxDelay = TimeSpan.FromMilliseconds(2); + options.WorkerSubscriber.DetachHandlerAfter = TimeSpan.FromMilliseconds(10); + options.WorkerSubscriber.FaultDrainTimeout = faultDrainTimeout; + return options; + } + + /// + /// Pre-fix failure: with the handler held open, the failed consumer is never closed and no + /// second consumer is created — the reconnect policy (2 ms) cannot run until the handler + /// finishes. Now the fault teardown waits FaultDrainTimeout, abandons the handler (offset + /// unstored, outcome logged when it settles), closes the consumer, and the supervisor + /// rebuilds it at once. + /// + [Fact] + public async Task KafkaWorker_APollLoopFailure_ReconnectsWithinFaultDrainTimeout_WhileAnUnrelatedHandlerRuns() + { + var first = new FakeKafkaConsumerClient(); + first.Enqueue(KafkaTestData.Message("workers", offset: 7, payload: "slow-job")); + var second = new FakeKafkaConsumerClient(); + var factory = new FakeKafkaConsumerClientFactory(first, second); + var ingress = new GateIngress(); + var logger = new CollectingLogger(); + using var subscriber = new KafkaWorkerSubscriber( + Options.Create(KafkaOptions(TimeSpan.FromMilliseconds(50))), + factory, + new FakeKafkaProducerClient(), + new FakeKafkaAdminClient(), + ingress, + logger.For()); + + await subscriber.StartAsync(CancellationToken.None); + try + { + await ingress.Entered.Task.WaitAsync(TimeSpan.FromSeconds(5)); + await KafkaTestData.WaitUntilAsync(() => first.IsPartitionPaused(0)); + + first.NextConsumeException = new IOException("simulated broker connection dropped"); + await KafkaTestData.WaitUntilAsync(() => factory.CreatedRoles.Count == 2, TimeSpan.FromSeconds(5)); + + Assert.True(first.Closed); + Assert.Empty(first.StoredOffsets); + Assert.False(ingress.Release.Task.IsCompleted); + await logger.WaitForAsync("Abandoning detached Kafka handler"); + + // The abandoned handler settles later: observed and logged, nothing stored on either consumer. + ingress.Release.SetResult(); + await logger.WaitForAsync("completed after the consumer it was consumed on was rebuilt"); + Assert.Empty(first.StoredOffsets); + Assert.Empty(second.StoredOffsets); + } + finally + { + ingress.Release.TrySetResult(); + await subscriber.StopAsync(CancellationToken.None); + } + } + + /// A handler that settles inside the budget has its offset stored before the failed consumer closes, exactly as after a stop. + [Fact] + public async Task KafkaWorker_AHandlerSettlingWithinFaultDrainTimeout_HasItsOffsetStoredBeforeTheClose() + { + var first = new FakeKafkaConsumerClient(); + first.Enqueue(KafkaTestData.Message("workers", offset: 7, payload: "slow-job")); + var factory = new FakeKafkaConsumerClientFactory(first, new FakeKafkaConsumerClient()); + var ingress = new GateIngress(); + using var subscriber = new KafkaWorkerSubscriber( + Options.Create(KafkaOptions(TimeSpan.FromSeconds(5))), + factory, + new FakeKafkaProducerClient(), + new FakeKafkaAdminClient(), + ingress, + NullLogger.Instance); + + await subscriber.StartAsync(CancellationToken.None); + try + { + await ingress.Entered.Task.WaitAsync(TimeSpan.FromSeconds(5)); + await KafkaTestData.WaitUntilAsync(() => first.IsPartitionPaused(0)); + + first.NextConsumeException = new IOException("simulated broker connection dropped"); + await KafkaTestData.WaitUntilAsync(() => first.NextConsumeException is null); + await Task.Delay(200); + Assert.False(first.Closed); + Assert.Single(factory.CreatedRoles); + + ingress.Release.SetResult(); + await KafkaTestData.WaitUntilAsync(() => factory.CreatedRoles.Count == 2, TimeSpan.FromSeconds(5)); + Assert.Equal(new FakeKafkaConsumerClient.StoredOffset("workers", 0, 7), Assert.Single(first.StoredOffsets)); + Assert.True(first.Closed); + } + finally + { + ingress.Release.TrySetResult(); + await subscriber.StopAsync(CancellationToken.None); + } + } + + [Fact] + public void FaultDrainTimeout_MustBeNonNegative_AndTimerBacked() + { + var options = KafkaTestData.NewOptions(); + options.WorkerSubscriber.FaultDrainTimeout = TimeSpan.FromMilliseconds(-1); + + var ex = Assert.Throws( + () => KafkaMessageDispatcher.ValidateOptions(options, options.WorkerSubscriber, KafkaSubscriberRole.Worker)); + + Assert.Contains(nameof(KafkaSubscriberOptions.FaultDrainTimeout), ex.Message, StringComparison.Ordinal); + Assert.Equal(TimeSpan.FromSeconds(5), new KafkaSubscriberOptions().FaultDrainTimeout); + } +} diff --git a/tests/AsyncResponse.Tests/Round38RegressionTests.cs b/tests/AsyncResponse.Tests/Round38RegressionTests.cs new file mode 100644 index 000000000..1b98da107 --- /dev/null +++ b/tests/AsyncResponse.Tests/Round38RegressionTests.cs @@ -0,0 +1,523 @@ +using AsyncResponse.Channels.MongoDB; +using AsyncResponse.Channels.NATS; +using AsyncResponse.Channels.PostgreSQL; +using AsyncResponse.Channels.Redis; +using AsyncResponse.Channels.SqlServer; +using AsyncResponse.Testing; +using AsyncResponse.Transports.Kafka; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Moq; +using StackExchange.Redis; +using System.Collections; +using System.Reflection; +using System.Text.Json; +using Xunit; + +namespace AsyncResponse.Tests; + +/// +/// Regressions for round 38 (external holistic review of 94c3ddb): behavior pins that compile +/// against the pre-fix tree and fail there. Pins over API this round introduced +/// (FlowState.RetainUntilUtc, InMemoryWorkerTransportOptions.DelayedJobCapacity, +/// KafkaSubscriberOptions.FaultDrainTimeout) live in . +/// +public sealed class Round38RegressionTests +{ + public sealed record R38Input(string Value); + + public sealed class R38NoopFlow : IDurableFlow + { + public static int Runs; + + public Task ExecuteAsync(IDurableFlowContext flow, R38Input input) + { + Interlocked.Increment(ref Runs); + return Task.CompletedTask; + } + } + + private static FlowState State(string id, string? parent = null) => new() + { + FlowId = id, + ParentFlowId = parent, + Status = FlowRunStatus.Running, + FlowTypeName = typeof(R38NoopFlow).FullName, + InputTypeName = typeof(R38Input).FullName, + InputJson = JsonSerializer.Serialize(new R38Input("x")), + CreatedAtUtc = DateTime.UtcNow, + UpdatedAtUtc = DateTime.UtcNow + }; + + private static ServiceProvider BuildProvider(IWorkerTransport transport, TimeProvider clock) + { + var services = new ServiceCollection(); + services.AddSingleton(typeof(ILogger<>), typeof(NullLogger<>)); + services.AddSingleton(clock); + services + .AddAsyncResponse() + .WithInMemoryChannel() + .WithInMemoryDurableFlows() + .WithDurableFlow(); + services.AddSingleton(transport); + return services.BuildServiceProvider(); + } + + private static DurableFlowContext CreateContext( + ServiceProvider provider, + FlowState state, + IFlowStateStore store, + FlowExecutionLease lease, + DurableFlowOptions options, + TimeProvider clock, + IWorkerTransport transport) + => new( + state, + store, + provider.GetRequiredService(), + provider.GetRequiredService(), + options, + provider.GetRequiredService(), + recoverableSubscriber: null, + NullLogger.Instance, + lease, + clock, + workerTransport: transport); + + /// A delayed-capable transport that only records what it was asked to publish. + internal sealed class CapturingDelayedTransport : IDelayedWorkerTransport + { + private readonly List _jobs = []; + + public TimeSpan MaxPublishDelay => TimeSpan.FromDays(30); + + public int Count + { + get { lock (_jobs) return _jobs.Count; } + } + + public Task PublishAsync(WorkerJobEnvelope job, CancellationToken cancellationToken = default) + { + lock (_jobs) + _jobs.Add(job); + return Task.CompletedTask; + } + + public Task PublishAsync(WorkerJobEnvelope job, TimeSpan delay, CancellationToken cancellationToken = default) + => PublishAsync(job, cancellationToken); + } + + // --------------------------------------------------------------------------------------------- + // F1 — the ancestor extension ceded a lost revision race: a concurrent write to the parent + // (its own replay re-parking on a pre-park snapshot of the child, or the executor's + // per-attempt save) stamped the plain StateExpiry, the child's extension lost the + // compare-and-swap, logged "the ancestor is live and re-stamping its own expiry", and + // parked with its wake-up published — on a parent that expired under the hour-long wait. + + /// + /// Wraps a store and, once, lets a competing writer win the parent's revision right between + /// the child's read of the parent and its extension write — the competing write carrying the + /// plain one-minute expiry of a checkpoint that knows nothing about the park. + /// + internal sealed class CollidingFlowStateStore(IFlowStateStore inner, string collideOn) : IFlowStateStore + { + private int _collisionsLeft = 1; + + public int ConcurrentWrites { get; private set; } + public int LostRaces { get; private set; } + + public Task LoadAsync(string flowId, CancellationToken cancellationToken = default) + => inner.LoadAsync(flowId, cancellationToken); + + public Task TryCreateAsync(string flowId, FlowState state, TimeSpan ttl, CancellationToken cancellationToken = default) + => inner.TryCreateAsync(flowId, state, ttl, cancellationToken); + + public async Task TryUpdateAsync(string flowId, FlowState state, long expectedRevision, TimeSpan ttl, string? leaseId = null, CancellationToken cancellationToken = default) + { + if (leaseId is null && string.Equals(flowId, collideOn, StringComparison.Ordinal) && _collisionsLeft-- > 0) + { + var current = (await inner.LoadAsync(flowId, cancellationToken))!; + var old = current.Revision; + current.Revision++; + if (await inner.TryUpdateAsync(flowId, current, old, TimeSpan.FromMinutes(1), null, cancellationToken)) + ConcurrentWrites++; + } + + var written = await inner.TryUpdateAsync(flowId, state, expectedRevision, ttl, leaseId, cancellationToken); + if (!written && leaseId is null) + LostRaces++; + return written; + } + + public Task TryAcquireLeaseAsync(string flowId, string leaseId, TimeSpan leaseDuration, CancellationToken cancellationToken = default) + => inner.TryAcquireLeaseAsync(flowId, leaseId, leaseDuration, cancellationToken); + + public Task TryRenewLeaseAsync(string flowId, string leaseId, TimeSpan leaseDuration, CancellationToken cancellationToken = default) + => inner.TryRenewLeaseAsync(flowId, leaseId, leaseDuration, cancellationToken); + + public Task ReleaseLeaseAsync(string flowId, string leaseId, CancellationToken cancellationToken = default) + => inner.ReleaseLeaseAsync(flowId, leaseId, cancellationToken); + + public Task TryDeleteAsync(string flowId, CancellationToken cancellationToken = default) + => inner.TryDeleteAsync(flowId, cancellationToken); + } + + private static readonly DurableFlowOptions ShortLedgerOptions = new() + { + StateExpiry = TimeSpan.FromMinutes(1), + DefaultStepTimeout = TimeSpan.FromSeconds(10), + TimerInProcessThreshold = TimeSpan.Zero + }; + + /// + /// Pre-fix failure: the child parks and publishes its wake-up (one job), the lost race is + /// logged and ignored, and two virtual minutes later the root is gone while the child's + /// hour-long park lives on. Now the lost race is retried against the re-read revision, so the + /// root outlives its one-minute expiry — and the park still publishes exactly one wake-up. + /// + /// The child is a REPLAYED timer (its due time already persisted): a first-pass park extends + /// the chain twice — breadcrumb save, then suspend — so a single lost race there was healed + /// by the second pass, while a replay parks through the suspend save alone and a lost race + /// there was final. A redelivered wake-up is the ordinary way a parked timer re-executes. + /// + /// + [Fact] + public async Task AncestorExtension_ALostRevisionRace_IsRetried_SoTheParentOutlivesTheChildsPark() + { + var clock = new VirtualTimeProvider(); + var transport = new CapturingDelayedTransport(); + await using var provider = BuildProvider(transport, clock); + var inner = provider.GetRequiredService(); + var store = new CollidingFlowStateStore(inner, collideOn: "r38-root"); + + var root = State("r38-root"); + root.Steps = new Dictionary(StringComparer.Ordinal) { ["child"] = new() { ChildFlowId = "r38-root:child" } }; + var child = State("r38-root:child", "r38-root"); + child.ParentStepName = "child"; + child.Steps = new Dictionary(StringComparer.Ordinal) + { + ["long-wait"] = new() { WakeAtUtc = clock.GetUtcNow().UtcDateTime.AddHours(1) } + }; + Assert.True(await inner.TryCreateAsync("r38-root", root, TimeSpan.FromMinutes(1))); + Assert.True(await inner.TryCreateAsync("r38-root:child", child, TimeSpan.FromMinutes(1))); + + await using (var lease = (await FlowStateConcurrency.TryAcquireExecutionLeaseAsync(store, child.FlowId!, ShortLedgerOptions, NullLogger.Instance, clock))!) + { + var context = CreateContext(provider, child, store, lease, ShortLedgerOptions, clock, transport); + await Assert.ThrowsAsync(() => context.DelayAsync("long-wait", TimeSpan.FromHours(1))); + } + + Assert.Equal(1, store.ConcurrentWrites); + Assert.Equal(1, store.LostRaces); + Assert.Equal(1, transport.Count); + + clock.Advance(TimeSpan.FromMinutes(2)); + Assert.NotNull(await inner.LoadAsync("r38-root:child")); + Assert.True(await inner.LoadAsync("r38-root") is not null, "the root expired under the child's hour-long park after a lost extension race"); + } + + // --------------------------------------------------------------------------------------------- + // F4 — "corrupt means absent": a ledger whose JSON disagreed with its stored revision or key + // loaded as null, the executor acknowledged the wake-up as belonging to a deleted flow, + // and the physically present run lost its only wake-up. + + private static IDictionary Entries(IFlowStateStore store) + => (IDictionary)store.GetType().GetField("_entries", BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(store)!; + + private static void CorruptRevision(IFlowStateStore store, string flowId) + { + var entry = Entries(store)[flowId]!; + var property = entry.GetType().GetProperty("StateJson")!; + var json = (string)property.GetValue(entry)!; + Assert.Contains("\"Revision\":0", json, StringComparison.Ordinal); + property.SetValue(entry, json.Replace("\"Revision\":0", "\"Revision\":7", StringComparison.Ordinal)); + } + + /// + /// Pre-fix failure: LoadAsync returned null for an entry that is still in the dictionary. + /// Now it throws FlowStateUnreadableException naming both revisions. + /// + [Fact] + public async Task InMemoryFlowStore_ARevisionMismatchInsideTheLedger_IsUnreadable_NotAbsent() + { + await using var provider = BuildProvider(new CapturingDelayedTransport(), new VirtualTimeProvider()); + var store = provider.GetRequiredService(); + Assert.True(await store.TryCreateAsync("r38-corrupt", State("r38-corrupt"), TimeSpan.FromDays(1))); + CorruptRevision(store, "r38-corrupt"); + + var ex = await Assert.ThrowsAsync(() => store.LoadAsync("r38-corrupt")); + Assert.Equal("r38-corrupt", ex.FlowId); + Assert.Contains("revision", ex.Reason, StringComparison.OrdinalIgnoreCase); + Assert.True(Entries(store).Contains("r38-corrupt")); + } + + /// + /// Pre-fix failure: ExecuteAsync returned normally (the wake-up acknowledged) without running + /// the flow, with the row still present. Now the unreadable ledger propagates to the transport's + /// retry/dead-letter path, which is the operator alarm. + /// + [Fact] + public async Task Executor_AnInconsistentLedger_PropagatesUnreadable_InsteadOfAcknowledgingTheWakeUp() + { + await using var provider = BuildProvider(new CapturingDelayedTransport(), new VirtualTimeProvider()); + var store = provider.GetRequiredService(); + Assert.True(await store.TryCreateAsync("r38-corrupt-exec", State("r38-corrupt-exec"), TimeSpan.FromDays(1))); + CorruptRevision(store, "r38-corrupt-exec"); + var runsBefore = R38NoopFlow.Runs; + + await Assert.ThrowsAsync( + () => provider.GetRequiredService().ExecuteAsync("r38-corrupt-exec")); + + Assert.Equal(runsBefore, R38NoopFlow.Runs); + Assert.True(Entries(store).Contains("r38-corrupt-exec")); + } + + // --------------------------------------------------------------------------------------------- + // F2 — the recovery-state readers logged the raw JsonException, whose Path is built from the + // stored registration's Context keys — tenant and auth baggage — so a malformed blob + // copied them into the application log. The readers now go through JsonSafety, and the + // logged failure carries size and position only. + + private const string Marker = "private_customer_42@example.invalid"; + + /// A registration whose Context has the marker as a KEY with a non-string value: STJ's failure path names the key. + private static string MalformedState() + => "{\"SchemaVersion\":1,\"RegistrationId\":\"" + Guid.NewGuid() + "\",\"CorrelationId\":\"corr\",\"Context\":{\"" + Marker + "\":123}}"; + + private static void AssertNothingLeaked(CollectingLogger logger) + { + Assert.NotEmpty(logger.Entries); + foreach (var (message, exception) in logger.Entries) + { + Assert.DoesNotContain(Marker, message, StringComparison.Ordinal); + Assert.DoesNotContain(Marker, exception?.ToString() ?? string.Empty, StringComparison.Ordinal); + } + } + + [Fact] + public void NatsRecoveryReader_AMalformedEnvelope_LogsNoContextKey() + { + var logger = new CollectingLogger(); + var store = new NatsRecoveryStateStore(new FakeNatsKvStore(), Options.Create(new NatsAsyncResponseChannelOptions()), logger.For()); + var json = "{\"States\":[" + MalformedState() + "]}"; + + var result = typeof(NatsRecoveryStateStore) + .GetMethod("TryDeserialize", BindingFlags.Instance | BindingFlags.NonPublic)! + .Invoke(store, [json, "recovery-key"]); + + Assert.Null(result); + AssertNothingLeaked(logger); + } + + [Fact] + public void RedisRecoveryReader_AMalformedEnvelope_LogsNoContextKey() + { + var logger = new CollectingLogger(); + var multiplexer = new Mock(); + multiplexer.Setup(m => m.GetDatabase(It.IsAny(), It.IsAny())).Returns(new Mock().Object); + var store = new RedisRecoveryStateStore( + multiplexer.Object, + Options.Create(new RedisAsyncResponseOptions { KeyPrefix = "ar" }), + logger.For()); + var json = "{\"Registrations\":[{\"State\":" + MalformedState() + ",\"ExpiresAtUtc\":\"2100-01-01T00:00:00Z\"}]}"; + + var deserialize = typeof(RedisRecoveryStateStore).GetMethod("DeserializeEntries", BindingFlags.Instance | BindingFlags.NonPublic)!; + deserialize.Invoke(store, [(RedisValue)json, "ar:recovery:corr", "corr", true, DateTimeOffset.UtcNow, false, false]); + + AssertNothingLeaked(logger); + } + + [Fact] + public void PostgreSqlRecoveryReader_AMalformedRow_LogsNoContextKey() + => AssertDatabaseReaderLeaksNothing( + logger => new PostgreSqlRecoveryStateStore(null!, logger.For())); + + [Fact] + public void SqlServerRecoveryReader_AMalformedRow_LogsNoContextKey() + => AssertDatabaseReaderLeaksNothing( + logger => new SqlServerRecoveryStateStore(null!, logger.For())); + + [Fact] + public void MongoDbRecoveryReader_AMalformedRow_LogsNoContextKey() + => AssertDatabaseReaderLeaksNothing( + logger => new MongoDbRecoveryStateStore(null!, logger.For())); + + private static void AssertDatabaseReaderLeaksNothing(Func createStore) + { + var logger = new CollectingLogger(); + var store = createStore(logger); + + object?[] args = [MalformedState(), "corr", 0]; + var result = store.GetType() + .GetMethod("DeserializeState", BindingFlags.Instance | BindingFlags.NonPublic)! + .Invoke(store, args); + + Assert.Null(result); + Assert.Equal(1, (int)args[2]!); // still counted as unreadable + AssertNothingLeaked(logger); + } + + // --------------------------------------------------------------------------------------------- + // F3 — delayed in-memory jobs were held against no bound at all: every scheduled publish + // retained its envelope and captured execution context, and when the timers fired each + // started a channel write that pended outside the bounded queue. Ten thousand scheduled + // jobs were accepted with QueueCapacity = 1 and InJobOverflowCapacity = 1. + + private static WorkerJobEnvelope DelayedJob() => new() + { + Call = new ReflectionCallDto + { + ServiceInterfaceFullName = "AsyncResponse.Tests.IRound38Probe", + MethodName = "RunAsync", + Params = [] + } + }; + + /// + /// Pre-fix failure: the 4 097th delayed publish completes like the 4 096 before it. Now it + /// waits for a slot (the default DelayedJobCapacity is 4 096), honors its cancellation token, + /// and is admitted once a scheduled job fires and enters the queue. + /// + [Fact] + public async Task InMemoryTransport_DelayedPublishesFromOutsideAJob_WaitPastTheDefaultBound() + { + var clock = new VirtualTimeProvider(); + var transport = new InMemoryWorkerTransport( + Options.Create(new InMemoryWorkerTransportOptions { QueueCapacity = 1, InJobOverflowCapacity = 1 }), + clock); + + for (var i = 0; i < 4_096; i++) + await transport.PublishAsync(DelayedJob(), TimeSpan.FromHours(1)); + Assert.Equal(4_096, transport.SnapshotDelayedJobs().Count); + + using var cancel = new CancellationTokenSource(); + var overflowing = transport.PublishAsync(DelayedJob(), TimeSpan.FromHours(1), cancel.Token); + await Task.Delay(100); + Assert.False(overflowing.IsCompleted, "the delayed publish past the bound was accepted"); + + cancel.Cancel(); + await Assert.ThrowsAnyAsync(() => overflowing); + Assert.Equal(4_096, transport.SnapshotDelayedJobs().Count); + + // One scheduled job fires and enters the (one-slot) queue: its slot frees and the next + // delayed publish is admitted. + clock.Advance(TimeSpan.FromHours(1)); + await Task.Delay(50); + Assert.Equal(1, transport.Reader.Count); + await transport.PublishAsync(DelayedJob(), TimeSpan.FromHours(1)).WaitAsync(TimeSpan.FromSeconds(5)); + } + + // --------------------------------------------------------------------------------------------- + // F5 — a Kafka message that could not be parsed into a delivery was dead-lettered and its + // offset stored at once, bypassing the dispatcher's per-partition order. Consumed behind + // a detached handler of the same partition (a rebalance handing the partition back with + // its pause reset delivers the next record), that stored the partition PAST the + // unfinished message; the auto-committer committed it, and a crash skipped the valid job + // for good — with only the malformed record's copy in the dead-letter topic. + + private sealed class GateIngress : IAsyncResponseIngress + { + public TaskCompletionSource Entered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource Release { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public Task HandleResponseMessageAsync(string json, string? correlationId) => Task.CompletedTask; + + public Task HandleWorkerMessageAsync(string json) + { + if (json != "slow-job") + return Task.CompletedTask; + + Entered.TrySetResult(); + return Release.Task; + } + } + + private static KafkaAsyncResponseTransportOptions KafkaOptions(Action? configure = null) + { + var options = KafkaTestData.NewOptions(); + options.WorkerTopic = "workers"; + options.CreateTopics = false; + options.SubscriberRetryBaseDelay = TimeSpan.FromMilliseconds(1); + options.SubscriberRetryMaxDelay = TimeSpan.FromMilliseconds(2); + options.WorkerSubscriber.DetachHandlerAfter = TimeSpan.FromMilliseconds(10); + configure?.Invoke(options); + return options; + } + + /// + /// Pre-fix failure: with the handler for offset 7 still running, offset 8 (empty payload) is + /// stored at once — [8] — and one dead-letter copy exists. Now nothing is stored while + /// 7 is pending; once it settles, 7 and then 8 are stored in order and the malformed record is + /// buried in its turn. + /// + [Fact] + public async Task KafkaWorker_AMalformedMessageBehindADetachedHandler_IsNotCommittedAheadOfIt() + { + var consumer = new FakeKafkaConsumerClient { IgnorePartitionPause = true }; + consumer.Enqueue(KafkaTestData.Message("workers", offset: 7, payload: "slow-job")); + consumer.Enqueue(KafkaTestData.Message("workers", offset: 8, payload: "")); + var ingress = new GateIngress(); + var producer = new FakeKafkaProducerClient(); + using var subscriber = new KafkaWorkerSubscriber( + Options.Create(KafkaOptions()), + new FakeKafkaConsumerClientFactory(consumer), + producer, + new FakeKafkaAdminClient(), + ingress, + NullLogger.Instance); + + await subscriber.StartAsync(CancellationToken.None); + try + { + await ingress.Entered.Task.WaitAsync(TimeSpan.FromSeconds(5)); + await Task.Delay(300); + + Assert.False(ingress.Release.Task.IsCompleted); + Assert.Empty(consumer.StoredOffsets); + Assert.Empty(producer.Publishes); + + ingress.Release.SetResult(); + await KafkaTestData.WaitUntilAsync(() => consumer.StoredOffsets.Count == 2); + Assert.Equal( + [new FakeKafkaConsumerClient.StoredOffset("workers", 0, 7), new FakeKafkaConsumerClient.StoredOffset("workers", 0, 8)], + consumer.StoredOffsets); + Assert.Single(producer.Publishes); + await KafkaTestData.WaitUntilAsync(() => !consumer.IsPartitionPaused(0)); + } + finally + { + ingress.Release.TrySetResult(); + await subscriber.StopAsync(CancellationToken.None); + } + } + + /// The ordinary case is unchanged: with nothing detached on the partition, a malformed record is buried and stored at once. + [Fact] + public async Task KafkaWorker_AMalformedMessageWithNothingDetached_IsStillDiscardedAtOnce() + { + var consumer = new FakeKafkaConsumerClient(); + consumer.Enqueue(KafkaTestData.Message("workers", offset: 3, payload: "")); + var producer = new FakeKafkaProducerClient(); + using var subscriber = new KafkaWorkerSubscriber( + Options.Create(KafkaOptions()), + new FakeKafkaConsumerClientFactory(consumer), + producer, + new FakeKafkaAdminClient(), + new GateIngress(), + NullLogger.Instance); + + await subscriber.StartAsync(CancellationToken.None); + try + { + await KafkaTestData.WaitUntilAsync(() => consumer.StoredOffsets.Count == 1); + Assert.Equal(new FakeKafkaConsumerClient.StoredOffset("workers", 0, 3), Assert.Single(consumer.StoredOffsets)); + Assert.Single(producer.Publishes); + } + finally + { + await subscriber.StopAsync(CancellationToken.None); + } + } +} diff --git a/tests/AsyncResponse.Tests/Round39RegressionTests.cs b/tests/AsyncResponse.Tests/Round39RegressionTests.cs new file mode 100644 index 000000000..55fc2d6c3 --- /dev/null +++ b/tests/AsyncResponse.Tests/Round39RegressionTests.cs @@ -0,0 +1,403 @@ +using AsyncResponse.Testing; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Xunit; + +namespace AsyncResponse.Tests; + +/// +/// Round 39 (2026-09-14) regressions: lost-subscriber settlement over the WHOLE failure set, +/// body-free typed in-memory delivery, and the test harness refusing a "restart" that user code +/// survived. Each pin was proven red on f92f1e7. +/// +public sealed class Round39RegressionTests +{ + // --------------------------------------------------------------------------------------- + // F1 (HIGH): shared-correlation dispatch settled on the FIRST failure only. A deterministic + // failure ahead of a transient one hid the transient sibling: the message was acknowledged + // and the transient registration — a valid waiter whose dependency was briefly down — lost + // the only copy of its payload. The verdict depended on the order the store returned the + // registrations in. + // --------------------------------------------------------------------------------------- + + public interface IOrderedResumeSpy + { + Task ResumeOk(OperationResult payload); + Task ResumeBoom(OperationResult payload); + } + + /// No implementation is registered: wiring its callback up fails deterministically. + public interface IUnregisteredResumeSpy + { + Task Resume(OperationResult payload); + } + + public interface IOrderedFailSpy + { + Task FailOk(Exception exception); + Task FailBoom(Exception exception); + } + + public interface IUnregisteredFailSpy + { + Task Fail(Exception exception); + } + + private sealed class OrderedSpy : IOrderedResumeSpy, IOrderedFailSpy + { + private int _ok; + private int _boom; + + public int Ok => Volatile.Read(ref _ok); + public int Boom => Volatile.Read(ref _boom); + + public Task ResumeOk(OperationResult payload) + { + Interlocked.Increment(ref _ok); + return Task.CompletedTask; + } + + public Task ResumeBoom(OperationResult payload) + { + Interlocked.Increment(ref _boom); + throw new InvalidOperationException("re-enqueue failed on a publish-blocked broker"); + } + + public Task FailOk(Exception exception) + { + Interlocked.Increment(ref _ok); + return Task.CompletedTask; + } + + public Task FailBoom(Exception exception) + { + Interlocked.Increment(ref _boom); + throw new InvalidOperationException("failure callback hit a dependency that is down"); + } + } + + private static ServiceProvider BuildProvider(OrderedSpy spy, TimeProvider? timeProvider = null) + { + var services = new ServiceCollection(); + services.AddSingleton(typeof(ILogger<>), typeof(NullLogger<>)); + if (timeProvider is not null) + services.AddSingleton(timeProvider); + services.AddSingleton(spy); + services.AddSingleton(spy); + services.AddAsyncResponse().WithInMemoryChannel(); + return services.BuildServiceProvider(); + } + + /// + /// Every ordering of a permanent failure (P), a transient failure (T) and a success (S) + /// across three registrations of one correlation id must end the same way: the response is + /// left for redelivery (), the success is + /// consumed, and the two failed registrations stay armed. Pre-fix, P ahead of T (with S + /// anywhere) returned normally: the message was acknowledged and T's payload was gone. + /// + [Theory] + [InlineData("PTS")] + [InlineData("PST")] + [InlineData("SPT")] + [InlineData("TPS")] + [InlineData("TSP")] + [InlineData("STP")] + public async Task DispatchLostResponses_ATransientSiblingFailure_PreservesRedelivery_WhateverTheOrder(string order) + { + var correlationId = $"round39-response-{order}"; + var spy = new OrderedSpy(); + await using var provider = BuildProvider(spy); + var store = provider.GetRequiredService(); + + var successId = Guid.NewGuid(); + foreach (var kind in order) + { + var (id, callback) = kind switch + { + 'S' => (successId, Resume(typeof(IOrderedResumeSpy), nameof(IOrderedResumeSpy.ResumeOk))), + 'T' => (Guid.NewGuid(), Resume(typeof(IOrderedResumeSpy), nameof(IOrderedResumeSpy.ResumeBoom))), + _ => (Guid.NewGuid(), Resume(typeof(IUnregisteredResumeSpy), nameof(IUnregisteredResumeSpy.Resume))) + }; + await store.SaveAsync(correlationId, new RecoveryState + { + RegistrationId = id, + CorrelationId = correlationId, + PayloadTypeFullName = typeof(OperationResult).FullName, + RegisteredAtUtc = DateTime.UtcNow, + ResumeCallback = callback + }, TimeSpan.FromMinutes(5)); + } + + var publisher = provider.GetRequiredService(); + var ex = await Assert.ThrowsAsync(() => publisher.SetResponse( + new OperationResult { Status = OperationStatus.Completed, Message = "late response" }, + correlationId)); + + Assert.Equal(correlationId, ex.CorrelationId); + Assert.IsType(ex.InnerException, exactMatch: true); + Assert.Equal(1, spy.Ok); + Assert.Equal(1, spy.Boom); + + var remaining = await store.GetAllAsync(correlationId); + Assert.Equal(2, remaining.Count); + Assert.DoesNotContain(remaining, registration => registration.RegistrationId == successId); + } + + /// The exception-envelope twin: every registration's FAILURE callback, same verdict. + [Theory] + [InlineData("PTS")] + [InlineData("PST")] + [InlineData("SPT")] + [InlineData("TPS")] + [InlineData("TSP")] + [InlineData("STP")] + public async Task DispatchLostExceptions_ATransientSiblingFailure_PreservesRedelivery_WhateverTheOrder(string order) + { + var correlationId = $"round39-exception-{order}"; + var spy = new OrderedSpy(); + await using var provider = BuildProvider(spy); + var store = provider.GetRequiredService(); + + var successId = Guid.NewGuid(); + foreach (var kind in order) + { + var (id, callback) = kind switch + { + 'S' => (successId, Failure(typeof(IOrderedFailSpy), nameof(IOrderedFailSpy.FailOk))), + 'T' => (Guid.NewGuid(), Failure(typeof(IOrderedFailSpy), nameof(IOrderedFailSpy.FailBoom))), + _ => (Guid.NewGuid(), Failure(typeof(IUnregisteredFailSpy), nameof(IUnregisteredFailSpy.Fail))) + }; + await store.SaveAsync(correlationId, new RecoveryState + { + RegistrationId = id, + CorrelationId = correlationId, + PayloadTypeFullName = typeof(OperationResult).FullName, + RegisteredAtUtc = DateTime.UtcNow, + FailureCallback = callback + }, TimeSpan.FromMinutes(5)); + } + + var publisher = provider.GetRequiredService(); + var ex = await Assert.ThrowsAsync( + () => publisher.SetException(new InvalidOperationException("remote boom"), correlationId)); + + Assert.Equal(correlationId, ex.CorrelationId); + Assert.Equal(1, spy.Ok); + Assert.Equal(1, spy.Boom); + + var remaining = await store.GetAllAsync(correlationId); + Assert.Equal(2, remaining.Count); + Assert.DoesNotContain(remaining, registration => registration.RegistrationId == successId); + } + + /// + /// No success at all, with a deterministic failure AHEAD of a sibling whose failure-callback + /// ladder was exhausted: the exhausted sibling's + /// must be what propagates, whatever its position. Pre-fix the first failure propagated, so + /// the ingress would have burned its own retry ladder on the deterministic fault and then + /// escalated through SetException — re-invoking the failure callback that had just given up. + /// + [Fact] + public async Task DispatchLostResponses_WithNoSuccess_AnExhaustedSiblingPropagatesOverAnEarlierDeterministicFault() + { + const string correlationId = "round39-exhausted-precedence"; + var time = new VirtualTimeProvider(); + var spy = new OrderedSpy(); + await using var provider = BuildProvider(spy, time); + var store = provider.GetRequiredService(); + + // First: a resume callback nothing can wire up. Second: no resume callback, so the + // resumable payload is routed to the failure callback, whose ladder exhausts. + await store.SaveAsync(correlationId, new RecoveryState + { + RegistrationId = Guid.NewGuid(), + CorrelationId = correlationId, + PayloadTypeFullName = typeof(OperationResult).FullName, + RegisteredAtUtc = DateTime.UtcNow, + ResumeCallback = Resume(typeof(IUnregisteredResumeSpy), nameof(IUnregisteredResumeSpy.Resume)) + }, TimeSpan.FromMinutes(5)); + await store.SaveAsync(correlationId, new RecoveryState + { + RegistrationId = Guid.NewGuid(), + CorrelationId = correlationId, + PayloadTypeFullName = typeof(OperationResult).FullName, + RegisteredAtUtc = DateTime.UtcNow, + FailureCallback = Failure(typeof(IOrderedFailSpy), nameof(IOrderedFailSpy.FailBoom)) + }, TimeSpan.FromMinutes(5)); + + var publisher = provider.GetRequiredService(); + var dispatching = publisher.SetResponse( + new OperationResult { Status = OperationStatus.Completed, Message = "late response" }, + correlationId); + + // The failure-callback ladder backs off on the virtual clock; walk it. + var deadline = DateTime.UtcNow.AddSeconds(10); + while (!dispatching.IsCompleted && DateTime.UtcNow < deadline) + { + time.Advance(TimeSpan.FromSeconds(3)); + await Task.Delay(10); + } + + var ex = await Assert.ThrowsAsync(() => dispatching); + Assert.Equal(correlationId, ex.CorrelationId); + Assert.Equal(4, spy.Boom); + Assert.Equal(2, (await store.GetAllAsync(correlationId)).Count); + } + + private static ReflectionCallDto Resume(Type service, string method) => new() + { + ServiceInterfaceFullName = service.FullName!, + MethodName = method, + Params = [CallbackParam.ForPlaceholder(PlaceholderType.Payload)] + }; + + private static ReflectionCallDto Failure(Type service, string method) => new() + { + ServiceInterfaceFullName = service.FullName!, + MethodName = method, + Params = [CallbackParam.ForPlaceholder(PlaceholderType.Exception)] + }; + + // --------------------------------------------------------------------------------------- + // F3: typed in-memory delivery materialized each waiter's payload through the raw reader. + // A publisher's payload that does not fit the waiter's type fails INSIDE the payload, and + // the reader's own message named the offending dictionary key — into the waiter's task and, + // through SetError, into the wait activity's status. + // --------------------------------------------------------------------------------------- + + public sealed class KeyedStringsPayload : IAsyncResponsePayload + { + public Dictionary Values { get; set; } = []; + public RecoveryAction OnRecovery() => RecoveryAction.Resume; + } + + public sealed class KeyedIntsPayload : IAsyncResponsePayload + { + public Dictionary Values { get; set; } = []; + public RecoveryAction OnRecovery() => RecoveryAction.Resume; + } + + [Fact] + public async Task InMemoryTypedDelivery_APayloadThatDoesNotFitTheWaiter_FaultsWithoutTheBody() + { + const string customerKey = "private_customer_42@example.invalid"; + using var activities = new AsyncResponseActivityCollector(); + var services = new ServiceCollection(); + services.AddSingleton(typeof(ILogger<>), typeof(NullLogger<>)); + services.AddAsyncResponse().WithInMemoryChannel(); + await using var provider = services.BuildServiceProvider(); + var channel = provider.GetRequiredService(); + var publisher = provider.GetRequiredService(); + var correlationId = $"round39-typed-{Guid.NewGuid():N}"; + + await using var waiter = await channel.CreateResponseWaiter(correlationId, timeout: TimeSpan.FromSeconds(5)); + await publisher.SetResponse(new KeyedStringsPayload { Values = { [customerKey] = "not an int" } }, correlationId); + + // Same failure shape as every broker channel: InvalidDataException, size and position + // only, nothing of the body anywhere in the chain. + var ex = await Assert.ThrowsAsync(() => waiter.ResponseTask.WaitAsync(TimeSpan.FromSeconds(5))); + for (Exception? current = ex; current is not null; current = current.InnerException) + { + Assert.DoesNotContain(customerKey, current.Message, StringComparison.Ordinal); + Assert.DoesNotContain("Values", current.Message, StringComparison.Ordinal); + } + + foreach (var activity in activities.All()) + { + Assert.DoesNotContain(customerKey, activity.StatusDescription ?? "", StringComparison.Ordinal); + foreach (var tag in activity.TagObjects) + Assert.DoesNotContain(customerKey, tag.Value?.ToString() ?? "", StringComparison.Ordinal); + } + } + + // --------------------------------------------------------------------------------------- + // F7: SimulateRestartAsync proceeded past a step body that outlived the graceful stop — the + // "dead" execution kept running beside the new incarnation and performed its side effect + // after the restart had returned. The restart is cooperative and cannot kill it; it now + // refuses to report a restart such an execution contradicts unless the test opts in. + // --------------------------------------------------------------------------------------- + + public sealed record LingerInput(string Name); + + /// The step's dependency: blocks until the test releases it, ignoring cancellation. + public sealed class LingerGate + { + private int _sideEffects; + + public TaskCompletionSource Entered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource Release { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public int SideEffects => Volatile.Read(ref _sideEffects); + public void RecordSideEffect() => Interlocked.Increment(ref _sideEffects); + } + + public sealed class LingeringStepFlow(LingerGate _gate) : IDurableFlow + { + public async Task ExecuteAsync(IDurableFlowContext flow, LingerInput input) + { + await flow.StepAsync("blocked", async () => + { + _gate.Entered.TrySetResult(); + await _gate.Release.Task; + _gate.RecordSideEffect(); + }); + } + } + + [Fact] + public async Task SimulateRestart_UserCodeStillRunningAfterTheStopLapsed_IsRefused() + { + var gate = new LingerGate(); + await using var harness = await AsyncResponseTestHarness.StartAsync(options => + { + options.RealTimeGuard = TimeSpan.FromMilliseconds(300); + options.ConfigureServices = services => services.AddSingleton(gate); + options.ConfigureAsyncResponse = builder => builder.WithDurableFlow(); + }); + + await harness.Flows.StartAsync(new LingerInput("linger")); + await gate.Entered.Task.WaitAsync(TimeSpan.FromSeconds(5)); + + // Released in finally: a failed assertion must not leave the step blocked forever under + // the harness's disposal. + InvalidOperationException ex; + try + { + ex = await Assert.ThrowsAsync(() => harness.SimulateRestartAsync()); + } + finally + { + gate.Release.TrySetResult(); + } + + Assert.Contains("could not establish quiescence", ex.Message, StringComparison.Ordinal); + Assert.Contains(nameof(AsyncResponseTestHarnessOptions.AbandonLingeringExecutionsOnRestart), ex.Message, StringComparison.Ordinal); + } + + [Fact] + public async Task SimulateRestart_WithTheOptIn_AbandonsTheExecution_WhichStillRunsAfterTheRestart() + { + var gate = new LingerGate(); + await using var harness = await AsyncResponseTestHarness.StartAsync(options => + { + options.RealTimeGuard = TimeSpan.FromMilliseconds(300); + options.AbandonLingeringExecutionsOnRestart = true; + options.ConfigureServices = services => services.AddSingleton(gate); + options.ConfigureAsyncResponse = builder => builder.WithDurableFlow(); + }); + + await harness.Flows.StartAsync(new LingerInput("linger")); + await gate.Entered.Task.WaitAsync(TimeSpan.FromSeconds(5)); + + await harness.SimulateRestartAsync(); + Assert.Equal(0, gate.SideEffects); + + // The documented overlap the opt-in accepts: the abandoned execution is not dead, and its + // side effect lands after the "restart". + gate.Release.TrySetResult(); + var deadline = DateTime.UtcNow.AddSeconds(5); + while (gate.SideEffects == 0 && DateTime.UtcNow < deadline) + await Task.Delay(10); + Assert.Equal(1, gate.SideEffects); + } +}