Bump OpenTelemetry.Exporter.OpenTelemetryProtocol and OpenTelemetry.Extensions.Hosting

.github/workflows/auto-retry.yml

@@ -2,13 +2,24 @@ name: Auto-retry failed jobs
 
 # Hosted runners are flaky and this repo runs 24 job legs per push to main. When a run on main
 # fails, re-run only its failed jobs (and their dependents) — the same button as "Re-run failed
-# jobs" in the UI, pressed automatically. The FIRST re-run is unconditional. Later re-runs need
-# EVERY failed job's log to match a known infrastructure signature: an unconditional five-attempt
-# loop gave an intermittently failing product bug — a race, a lease, a watchdog — five chances to
-# land green on main, and publish.yml relies on main's check as the reviewed gate. Any-job
-# matching was not enough either: rerun-failed-jobs re-runs ALL failed jobs, so one genuine
-# fixture flake re-ran a co-failing real bug alongside it. The run_attempt guard is what stops
-# this recursing: each re-run fires `workflow_run: completed` again when it finishes.
+# jobs" in the UI, pressed automatically — but ONLY when every failed job is provably an
+# infrastructure flake. Two gates, applied from the FIRST re-run on (an unconditional first re-run
+# gave an intermittently failing product bug — a lease race, a timing defect, a watchdog — a free
+# second chance to land green on main, and publish.yml relies on main's check as the reviewed
+# gate):
+#   1. every failed job's log must match a known infrastructure signature, and
+#   2. no failed job's log may contain an executed-test assertion failure or a build error — a
+#      fixture-boot flake alongside a genuine assertion failure in the same job is a real
+#      failure, whatever else the log says.
+# Both signature sets live in ONE place, scripts/ci-retryable-failure.sh, which ci.yml's in-job
+# integration retry consults too — the in-job retry used to carry a weaker copy (gate 1 only), so a
+# mixed log was retried into a green job this workflow never got to see. Any-job matching is not
+# enough either: rerun-failed-jobs re-runs ALL failed jobs, so one genuine fixture flake would
+# re-run a co-failing real bug alongside it. A log that cannot be fetched counts as unmatched.
+# Every automatic re-run leaves a ::warning:: annotation on the run naming the flake evidence, so
+# a run that went green on a re-run is never indistinguishable from one that was green on its own.
+# The run_attempt guard is what stops this recursing: each re-run fires `workflow_run: completed`
+# again when it finishes.
 on:
   workflow_run:
     workflows: [CI, CodeQL]
@@ -28,36 +39,30 @@ jobs:
       github.event.workflow_run.run_attempt < 5
     runs-on: ubuntu-latest
     steps:
+      # The classifier is repo-tracked (scripts/ci-retryable-failure.sh); a workflow_run job
+      # checks out the default branch's copy, which is the one ci.yml on main runs with.
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
       - name: Re-run failed jobs
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           GH_REPO: ${{ github.repository }}
           RUN_ID: ${{ github.event.workflow_run.id }}
           ATTEMPT: ${{ github.event.workflow_run.run_attempt }}
           WORKFLOW: ${{ github.event.workflow_run.name }}
-          # ci.yml's own in-job retry gates on the same fixture-boot flake. "database is locked"
-          # is SQLite on a slow runner disk during the EF Core storm tests (see the tests' own
-          # comments); the last two are hosted-runner infrastructure loss.
-          FLAKE_SIGNATURES: >-
-            Fixture' threw in InitializeAsync|SQLite Error 5: 'database is locked'|lost communication with the server|The runner has received a shutdown signal
         run: |
           set -euo pipefail
           failed_jobs=$(gh api --paginate "repos/$GH_REPO/actions/runs/$RUN_ID/jobs?per_page=100" \
             --jq '.jobs[] | select(.conclusion == "failure") | "\(.id) \(.name)"')
           echo "Failed jobs of $WORKFLOW run $RUN_ID (attempt $ATTEMPT):"; echo "${failed_jobs:-<none listed>}"
 
-          if [ "$ATTEMPT" -lt 2 ]; then
-            echo "::notice::Re-running failed jobs of $WORKFLOW run $RUN_ID (attempt $ATTEMPT of 5): the first re-run is unconditional."
-            gh api --method POST "repos/$GH_REPO/actions/runs/$RUN_ID/rerun-failed-jobs"
-            exit 0
-          fi
-
-          # From the second re-run on, EVERY failed job's log must match a known flake signature:
-          # the gate is per run but rerun-failed-jobs is per job, so one matching job must not
-          # carry an unmatched (real) failure along for another attempt. A log that cannot be
+          # EVERY failed job's log must match a known flake signature and none may carry a real
+          # failure: the gate is per run but rerun-failed-jobs is per job, so one matching job must
+          # not carry an unmatched (real) failure along for another attempt. A log that cannot be
           # fetched counts as unmatched — an unreadable failure cannot be proven a flake.
           matched=""
           unmatched=""
+          real=""
           fetched=0
           while read -r job_id job_name; do
             [ -n "$job_id" ] || continue
@@ -76,20 +81,28 @@ jobs:
               continue
             fi
             fetched=$((fetched + 1))
-            if grep -Eq "$FLAKE_SIGNATURES" "job-$job_id.log"; then
-              matched="${matched:+$matched, }$job_id ($job_name)"
-            else
-              unmatched="${unmatched:+$unmatched, }$job_id ($job_name)"
-            fi
+            # Exit 0 = flake (retryable), 1 = a real executed-test or build failure, 2 = unmatched,
+            # 3 = unreadable; the first line names the evidence.
+            verdict=0
+            evidence=$(./scripts/ci-retryable-failure.sh "job-$job_id.log") || verdict=$?
+            case "$verdict" in
+              0) matched="${matched:+$matched, }$job_id ($job_name: $evidence)" ;;
+              1) real="${real:+$real, }$job_id ($job_name: $evidence)" ;;
+              *) unmatched="${unmatched:+$unmatched, }$job_id ($job_name: $evidence)" ;;
+            esac
           done <<< "$failed_jobs"
 
           if [ "$fetched" -eq 0 ]; then
             echo "::warning::Not re-running $WORKFLOW run $RUN_ID (attempt $ATTEMPT): could not read any failed job log, so the gate cannot tell a flake from a bug. Fix the log fetch above."
             exit 0
           fi
+          if [ -n "$real" ]; then
+            echo "::warning::Not re-running $WORKFLOW run $RUN_ID (attempt $ATTEMPT): failed job(s) $real contain an executed-test or build failure; a correctness failure is never retried into green (flake-matching jobs alongside: ${matched:-none})."
+            exit 0
+          fi
           if [ -n "$unmatched" ]; then
-            echo "::warning::Not re-running $WORKFLOW run $RUN_ID (attempt $ATTEMPT): failed job(s) $unmatched match no known flake signature; a failure that survives one re-run is treated as real (flake-matching jobs alongside: ${matched:-none})."
+            echo "::warning::Not re-running $WORKFLOW run $RUN_ID (attempt $ATTEMPT): failed job(s) $unmatched match no known flake signature; an unexplained failure is treated as real (flake-matching jobs alongside: ${matched:-none})."
             exit 0
           fi
-          echo "::notice::Re-running failed jobs of $WORKFLOW run $RUN_ID (attempt $ATTEMPT of 5): every failed job matched a known flake signature ($matched)."
+          echo "::warning::Re-running failed jobs of $WORKFLOW run $RUN_ID (attempt $ATTEMPT of 5): every failed job matched a known infrastructure flake signature and none carried a test or build failure ($matched). A green re-run of this run is a flake, not a clean pass."
           gh api --method POST "repos/$GH_REPO/actions/runs/$RUN_ID/rerun-failed-jobs"

.github/workflows/ci.yml

@@ -42,6 +42,13 @@ jobs:
       - name: Build
         run: dotnet build --configuration Release --no-restore
 
+      # The one retry classifier both integration legs and auto-retry.yml consult
+      # (scripts/ci-retryable-failure.sh) is plain bash; its fixture logs — including the mixed
+      # "fixture failed to boot AND an executed test failed" log that must never be retried — are
+      # checked here so a signature edit cannot silently widen what CI retries into green.
+      - name: CI retry classifier self-test
+        run: ./scripts/tests/ci-retryable-failure.test.sh
+
       # Coverage runs through coverlet rather than MTP's `--coverage` flag or `dotnet-coverage`:
       # pre-instrumenting with `dotnet-coverage instrument` discards condition data, so the published
       # report measured zero branches, and the only Microsoft mode that keeps it on linux-x64 is the
@@ -215,9 +222,13 @@ jobs:
         run: |
           # The suite runs in batches, each booting its own Aspire AppHost; on 2-core hosted runners a
           # boot intermittently times out or a resource fails to start before any test executes. Retry
-          # once on that exact signature only — genuine test failures never produce it, so they still
-          # fail the job on the first pass. The signature matches the *BatchFixture types in
-          # Batches.cs; keep it in sync if a batch fixture is ever renamed.
+          # once, but only when scripts/ci-retryable-failure.sh — the same classifier auto-retry.yml
+          # applies to whole runs — says the log is a known infrastructure flake AND carries no
+          # executed-test or build failure. Keying on the boot signature alone let a log that also
+          # held an assertion failure into the retry, and a passing second attempt made that
+          # correctness failure a green job. Each attempt's console log is kept in the results
+          # directory (itest-console.attempt<N>.log), so a green re-run is never indistinguishable
+          # from a clean pass.
           run_integration_tests() {
             # The cross-product shards are excluded deliberately. Everything in them builds its hosts
             # in this test process and never touches the sample app, so running them here proves
@@ -228,19 +239,21 @@ jobs:
             # Excluded by trait rather than class name on purpose: the transport contract classes live
             # in the matrix collections too but are not named Matrix*, and a name-based filter would
             # leave them behind to boot a shard fleet here.
+            mkdir -p "${{ github.workspace }}/TestResults/IntegrationAot"
             dotnet test --project tests/AsyncResponse.IntegrationTests/AsyncResponse.IntegrationTests.csproj \
               --configuration Release --no-build \
               --filter-not-trait "batch=matrix-*" \
               --report-trx \
               --results-directory "${{ github.workspace }}/TestResults/IntegrationAot" \
-              2>&1 | tee itest-console.log
+              2>&1 | tee "${{ github.workspace }}/TestResults/IntegrationAot/itest-console.attempt$1.log"
             return "${PIPESTATUS[0]}"
           }
-          if ! run_integration_tests; then
-            if grep -q "BatchFixture' threw in InitializeAsync" itest-console.log; then
-              echo "::warning::Integration fixture failed to boot (known hosted-runner resource flake) — retrying the suite once."
-              run_integration_tests
+          if ! run_integration_tests 1; then
+            if evidence=$(./scripts/ci-retryable-failure.sh "${{ github.workspace }}/TestResults/IntegrationAot/itest-console.attempt1.log"); then
+              echo "::warning::Integration run failed on a known hosted-runner infrastructure flake ($evidence) with no executed-test or build failure — retrying the suite once. A green second attempt is a flake, not a clean pass; attempt 1's log is in the results artifact."
+              run_integration_tests 2
             else
+              echo "::error::Integration run failed and is not retried ($evidence): a correctness failure is never retried into green."
               exit 1
             fi
           fi
@@ -342,27 +355,29 @@ jobs:
         # in docs/operations.md. "none" boots nothing at all — it is the in-memory suite, the Native
         # AOT publish gate, and the batch guards.
         run: |
-          # Same targeted boot-flake retry as the AOT integration job: rerun once only when the
-          # failure signature is a batch fixture failing to initialize, never for test failures.
-          # coverlet returns non-zero when the wrapped command fails, so the retry logic keys off the
-          # same exit code as before; a second attempt re-instruments and overwrites the same output.
+          # Same retry gate as the AOT integration job — scripts/ci-retryable-failure.sh, shared with
+          # auto-retry.yml: rerun once only when the log is a known infrastructure flake AND carries
+          # no executed-test or build failure (a boot flake next to an assertion failure is a real
+          # failure). coverlet returns non-zero when the wrapped command fails, so the retry keys off
+          # the same exit code; a second attempt re-instruments and overwrites the same coverage
+          # output, while each attempt keeps its own console log in the results directory.
           run_integration_tests() {
+            mkdir -p "${{ github.workspace }}/TestResults/Integration"
             ./scripts/coverage-collect.sh \
               "${{ github.workspace }}/TestResults/Integration/integration.${{ matrix.batch }}.cobertura.xml" \
               tests/AsyncResponse.IntegrationTests/bin/Release/net10.0 \
               --share tests/AsyncResponse.IntegrationTests.AppHost/bin/Release/net10.0 \
               --share samples/AsyncResponse.Sample/bin/Release/net10.0 \
               -- dotnet test --project tests/AsyncResponse.IntegrationTests/AsyncResponse.IntegrationTests.csproj --configuration Release --no-build --filter-trait batch=${{ matrix.batch }} --report-trx --results-directory ${{ github.workspace }}/TestResults/Integration \
-              2>&1 | tee itest-console.log
+              2>&1 | tee "${{ github.workspace }}/TestResults/Integration/itest-console.${{ matrix.batch }}.attempt$1.log"
             return "${PIPESTATUS[0]}"
           }
-          if ! run_integration_tests; then
-            # Matches both the original *BatchFixture types and the cross-product shards' fixtures
-            # (MatrixDatabaseLightFixture and friends), which carry no "Batch" infix.
-            if grep -q "Fixture' threw in InitializeAsync" itest-console.log; then
-              echo "::warning::Batch '${{ matrix.batch }}' failed to boot (known hosted-runner resource flake) — retrying once."
-              run_integration_tests
+          if ! run_integration_tests 1; then
+            if evidence=$(./scripts/ci-retryable-failure.sh "${{ github.workspace }}/TestResults/Integration/itest-console.${{ matrix.batch }}.attempt1.log"); then
+              echo "::warning::Batch '${{ matrix.batch }}' failed on a known hosted-runner infrastructure flake ($evidence) with no executed-test or build failure — retrying once. A green second attempt is a flake, not a clean pass; attempt 1's log is in the results artifact."
+              run_integration_tests 2
             else
+              echo "::error::Batch '${{ matrix.batch }}' failed and is not retried ($evidence): a correctness failure is never retried into green."
               exit 1
             fi
           fi

.github/workflows/loadtest.yml

@@ -106,7 +106,17 @@ jobs:
 
           rate="${LOADTEST_RATE:-20}"
           if [ "${GITHUB_EVENT_NAME:-}" = "push" ]; then
-            rate="${LOADTEST_RATE:-5}"
+            # The broad profile is 60 scenarios, so this is a PER-SCENARIO rate: 5/s offered 300
+            # requests per second to 14 SUT apps plus ~10 broker/database containers on one runner,
+            # and the request/response scenarios each carry ~1 s of simulated remote work. That put
+            # the fleet ~30x past its knee: even on runs that PASSED, the Redis-channel scenarios
+            # averaged ~31 s per request against the client's ~35 s ceiling, so the gate was decided
+            # by how fast that particular runner happened to be — a 5 % slower one pushed the mean
+            # past the ceiling and the failure rate from 2.5 % to 7.2 % with no code change behind
+            # it (the sub-second worker scenarios were unchanged at 0.65 s across both runs).
+            # 3/s keeps every scenario and both gate thresholds while restoring headroom, so a
+            # failure means the library, not the runner. Override with LOADTEST_RATE to push harder.
+            rate="${LOADTEST_RATE:-3}"
           fi
 
           args=(