ci: dogfood ddtest across riot test suites

.github/CODEOWNERS

@@ -113,6 +113,8 @@ tests/smoke_test.py                 @DataDog/python-guild
 tests/suitespec.py                  @DataDog/python-guild  @DataDog/apm-core-python
 tests/suitespec.yml                 @DataDog/python-guild
 tests/utils.py                      @DataDog/python-guild
+tests/_ddtest_conftest_helpers.py   @DataDog/python-guild
+tests/_ddtest_env_helpers.py        @DataDog/python-guild
 tests/__init__.py                   @DataDog/python-guild
 tests/README.md                     @DataDog/python-guild
 tests/ddtrace_run.py                @DataDog/python-guild

.gitlab/tests.yml

@@ -115,4 +115,255 @@ include:
       ./scripts/check-diff ".riot/requirements/" \
         "Changes detected in committed test locks. Run scripts/compile-and-prune-test-requirements and commit the result."
 
+# ---------------------------------------------------------------------------
+# AIDEV-NOTE: DDTest must receive the complete suitespec command. Falling back
+# to bare pytest bypasses per-suite exclusions and can select unavailable tools.
+# DDTest's uv backend is intentionally self-contained: it creates an ephemeral
+# environment from the suite lock and runs the exact suite command emitted by
+# suitespec. This keeps DDTest out of scripts/run-tests.
+.ddtest_base_uv:
+  extends: .test_base_uv
+
+.ddtest_plan_uv:
+  extends: .ddtest_base_uv
+  script:
+    - |
+      set -euo pipefail
+      for metadata in ${TEST_ENVIRONMENT_HASH_PYTHON}; do
+        IFS=: read -r environment_hash python_version lockfile tests_location <<< "${metadata}"
+        command_var="DDTEST_UV_COMMAND_${environment_hash}"
+        env_var="DDTEST_UV_ENV_${environment_hash}"
+        requirements="$(mktemp)"
+        grep -vi '^ddtrace==' "${lockfile}" > "${requirements}"
+        wheel="$(find ".cache/ddtrace-test-wheels/${python_version}" -name 'ddtrace-*.whl' -print -quit)"
+        set -f
+        export ${!env_var}
+        set +f
+        uv run --isolated --no-project --python "python${python_version}" --no-python-downloads \
+          --with-requirements "${requirements}" --with "${wheel}" \
+          ddtest plan --platform python --framework pytest --command "${!command_var}" \
+          --tests-location "${tests_location}" --min-parallelism "${DDTEST_NODES}" --max-parallelism "${DDTEST_NODES}"
+        rm -f "${requirements}"
+        mv .testoptimization ".testoptimization-${environment_hash}"
+      done
+  artifacts:
+    when: always
+    paths:
+      - .testoptimization-*/
+    expire_in: 1 week
+
+.ddtest_run_uv:
+  extends: .ddtest_base_uv
+  script:
+    - |
+      set -euo pipefail
+      rm -rf .testoptimization
+      mv ".testoptimization-${TEST_ENVIRONMENT_HASH}" .testoptimization
+      command_var="DDTEST_UV_COMMAND_${TEST_ENVIRONMENT_HASH}"
+      env_var="DDTEST_UV_ENV_${TEST_ENVIRONMENT_HASH}"
+      requirements="$(mktemp)"
+      grep -vi '^ddtrace==' "${TEST_ENVIRONMENT_LOCKFILE}" > "${requirements}"
+      wheel="$(find ".cache/ddtrace-test-wheels/${PYTHON_VERSION}" -name 'ddtrace-*.whl' -print -quit)"
+      set -f
+      export ${!env_var}
+      set +f
+      export COVERAGE_FILE=".coverage.${TEST_ENVIRONMENT_HASH}.${CI_NODE_INDEX}"
+      uv run --isolated --no-project --python "python${PYTHON_VERSION}" --no-python-downloads \
+        --with-requirements "${requirements}" --with "${wheel}" \
+        ddtest run --platform python --framework pytest --command "${!command_var}" \
+          --ci-node "${CI_NODE_INDEX}" --ci-node-workers 1
+      rm -f "${requirements}"
+      coverage combine || true
+
+# ---------------------------------------------------------------------------
+# ddtest job templates. Used by gen_gitlab_config.py for suites with
+# `ddtest: true` in suitespec. ddtest owns plan/run + file splitting; riot
+# knowledge stays in Riot's command override.
+#
+# before_script and script share the same shell session in GitLab. Venv
+# preparation and activation are handled by Riot for each command.
+# ---------------------------------------------------------------------------
+.ddtest_base:
+  extends: .testrunner
+  services:
+    - !reference [.services, ddagent]
+  before_script:
+    - !reference [.testrunner, before_script]
+    - unset DD_SERVICE
+    - unset DD_ENV
+    - unset DD_TAGS
+    - unset DD_TRACE_REMOVE_INTEGRATION_SERVICE_NAMES_ENABLED
+
+.ddtest_base_snapshot:
+  extends: .ddtest_base
+  services:
+    - !reference [.ddtest_base, services]
+    - !reference [.services, testagent]
+  before_script:
+    - !reference [.ddtest_base, before_script]
+    - ln -s "${CI_PROJECT_DIR}" "/home/bits/project"
+
+.ddtest_base_gpu:
+  extends: .ddtest_base
+  image: !reference [.testrunner_gpu, image]
+  tags: !reference [.testrunner_gpu, tags]
+  timeout: 40m
+  before_script:
+    - !reference [.testrunner_gpu, before_script]
+    - unset DD_SERVICE
+    - unset DD_ENV
+    - unset DD_TAGS
+    - unset DD_TRACE_REMOVE_INTEGRATION_SERVICE_NAMES_ENABLED
+
+.ddtest_base_gpu_snapshot:
+  extends: .ddtest_base_gpu
+  services:
+    - !reference [.ddtest_base_gpu, services]
+    - !reference [.services, testagent]
+  before_script:
+    - !reference [.ddtest_base_gpu, before_script]
+    - ln -s "${CI_PROJECT_DIR}" "/home/bits/project"
+
+# Keep GPU ddtest templates hidden until a suite opts in with gpu: true.
+.ddtest_plan_gpu:
+  extends:
+    - .ddtest_plan
+    - .ddtest_base_gpu
+
+.ddtest_run_gpu:
+  extends:
+    - .ddtest_run
+    - .ddtest_base_gpu
+
+.ddtest_run_gpu_snapshot:
+  extends:
+    - .ddtest_run_snapshot
+    - .ddtest_base_gpu_snapshot
+
+# Plan: discover the venv's suite files and write .testoptimization/
+# No snapshot variant — plan only collects (glob + Datadog API) and sends
+# no traces, so the testagent is never needed at plan time.
+# --tests-location is a raw glob (ddtest does not combine it with python_files),
+# so each Riot venv declares its complete location in DDTEST_TESTS_LOCATION.
+.ddtest_plan:
+  extends: .ddtest_base
+  script:
+    - |
+      set -euo pipefail
+      export DDTEST_PLAN_AGENT_URL="${_CI_DD_AGENT_URL}"
+      riot_cmd=(riot -P -v run --pass-env -s)
+      # Keep at most one planner active per Python version: Riot's nspkgs()
+      # temporarily modifies namespace-package .pth files in the shared base
+      # venv. Each hash gets its own worktree, so its hash-specific venv and
+      # planner output are isolated, but same-Python planners cannot overlap.
+      declare -A hashes_by_python=()
+      declare -A plan_workdirs=()
+      cleanup_worktrees() {
+        local workdir
+        for workdir in "${plan_workdirs[@]}"; do
+          git worktree remove --force "${workdir}" >/dev/null 2>&1 || rm -rf "${workdir}"
+        done
+      }
+      trap cleanup_worktrees EXIT
+
+      # Each plan writes fixed paths below .testoptimization, so give every
+      # hash its own Git worktree and output directory.
+      for hash_py in ${TEST_ENVIRONMENT_HASH_PYTHON}; do
+        _hash="${hash_py%%:*}"
+        _py="${hash_py#*:}"
+        _workdir="$(mktemp -d "${TMPDIR:-/tmp}/ddtest-plan.XXXXXX")"
+        rmdir "${_workdir}"
+        git worktree add --quiet --detach "${_workdir}" HEAD
+        # .riot/requirements is tracked, so the directory already exists in
+        # the worktree. Share only the built base venvs from the artifact;
+        # hash-specific venvs and their outputs remain worktree-local.
+        base_venv_count=0
+        for _venv in "${CI_PROJECT_DIR}"/.riot/venv_*; do
+          [[ -d "${_venv}" ]] || continue
+          ln -s "${_venv}" "${_workdir}/.riot/$(basename "${_venv}")"
+          ((base_venv_count += 1))
+        done
+        if ((base_venv_count == 0)); then
+          echo "No base Riot venvs found under ${CI_PROJECT_DIR}/.riot" >&2
+          exit 1
+        fi
+        plan_workdirs["${_hash}"]="${_workdir}"
+        hashes_by_python["${_py}"]+="${_hash} "
+      done
+
+      # Limit concurrent Python groups because each group may install a
+      # hash-specific dependency set in the same pod. Four groups preserve
+      # useful cross-version parallelism without allowing all six versions to
+      # peak during pip installs and exhaust the planner's memory limit.
+      plan_parallelism=4
+      declare -a plan_pids=()
+      plan_status=0
+      for _py in "${!hashes_by_python[@]}"; do
+        (
+          for _hash in ${hashes_by_python[${_py}]}; do
+            _workdir="${plan_workdirs[${_hash}]}"
+            (
+              cd "${_workdir}"
+              "${riot_cmd[@]}" --command \
+                'export DD_TRACE_AGENT_URL="$DDTEST_PLAN_AGENT_URL"; export PYTEST_ADDOPTS="$DDTEST_PYTEST_ADDOPTS"; case "$NIGHTLY_BUILD" in true) ;; *) case "$PYTEST_ADDOPTS" in *--no-cov*) ;; *) export PYTEST_ADDOPTS="$PYTEST_ADDOPTS --no-cov" ;; esac ;; esac; ddtest plan --platform python --framework pytest --tests-location "$DDTEST_TESTS_LOCATION" --min-parallelism "$DDTEST_NODES" --max-parallelism "$DDTEST_NODES"' \
+                "${_hash}"
+            )
+            mv "${_workdir}/.testoptimization" "${CI_PROJECT_DIR}/.testoptimization-${_hash}"
+          done
+        ) &
+        plan_pids+=("$!")
+        if ((${#plan_pids[@]} >= plan_parallelism)); then
+          if ! wait "${plan_pids[0]}"; then
+            plan_status=1
+          fi
+          plan_pids=("${plan_pids[@]:1}")
+        fi
+      done
+
+      for plan_pid in "${plan_pids[@]}"; do
+        if ! wait "${plan_pid}"; then
+          plan_status=1
+        fi
+      done
+      exit "${plan_status}"
+  artifacts:
+    when: always
+    paths:
+      - .testoptimization-*/
+    expire_in: 1 week
+
+# Run: execute this CI node's files in the activated venv.
+# 4 workers (not ncpu): the testrunner image runs on many-core k8s pods
+# where ncpu spawns too many workers and corrupts pytest-cov's sqlite DB;
+# matching PYTEST_XDIST_AUTO_NUM_WORKERS=4 from testrunner.yml.
+.ddtest_run:
+  extends: .ddtest_base
+  script:
+    - |
+      # Restore this environment's plan artifact from the plan job.
+      rm -rf .testoptimization
+      mv ".testoptimization-${TEST_ENVIRONMENT_HASH}" .testoptimization
+      export COVERAGE_FILE=".coverage.${TEST_ENVIRONMENT_HASH}.${CI_NODE_INDEX}"
+      riot_cmd=(riot -P -v run --pass-env -s)
+      # Keep CI Visibility session names tied to the CI job and test command;
+      # ddtest expands these placeholders for each local worker.
+      "${riot_cmd[@]}" --command \
+        'export PYTEST_ADDOPTS="$DDTEST_PYTEST_ADDOPTS"; export DD_TEST_SESSION_NAME="${{CI_JOB_NAME}}-pytest ${{DDTEST_SUITE_PATH}} ${{DDTEST_PYTEST_ADDOPTS}} node-{{{{nodeIndex}}}}-worker-{{{{workerIndex}}}}"; case "$NIGHTLY_BUILD" in true) ;; *) case "$PYTEST_ADDOPTS" in *--no-cov*) ;; *) export PYTEST_ADDOPTS="$PYTEST_ADDOPTS --no-cov" ;; esac ;; esac; export COVERAGE_FILE="$COVERAGE_FILE"; ddtest run --platform python --framework pytest --ci-node "$CI_NODE_INDEX" --ci-node-workers 4; coverage_files=(.coverage.*); if [[ -e "${{coverage_files[0]}}" ]]; then coverage combine; fi' \
+        "${TEST_ENVIRONMENT_HASH}"
+
+.ddtest_run_snapshot:
+  extends: .ddtest_base_snapshot
+  script:
+    - |
+      rm -rf .testoptimization
+      mv ".testoptimization-${TEST_ENVIRONMENT_HASH}" .testoptimization
+      export COVERAGE_FILE=".coverage.${TEST_ENVIRONMENT_HASH}.${CI_NODE_INDEX}"
+      riot_cmd=(riot -P -v run --pass-env -s)
+      # Snapshot suites use --ci-node-workers 1 (not 4): the shared testagent
+      # is session-token-keyed, and parallel xdist workers cause cross-session
+      # contamination (wrong stats, '400: list index out of range').
+      "${riot_cmd[@]}" --command \
+        'export PYTEST_ADDOPTS="$DDTEST_PYTEST_ADDOPTS"; export DD_TEST_SESSION_NAME="${{CI_JOB_NAME}}-pytest ${{DDTEST_SUITE_PATH}} ${{DDTEST_PYTEST_ADDOPTS}} node-{{{{nodeIndex}}}}-worker-{{{{workerIndex}}}}"; case "$NIGHTLY_BUILD" in true) ;; *) case "$PYTEST_ADDOPTS" in *--no-cov*) ;; *) export PYTEST_ADDOPTS="$PYTEST_ADDOPTS --no-cov" ;; esac ;; esac; export COVERAGE_FILE="$COVERAGE_FILE"; export DD_TRACE_AGENT_URL="http://testagent:9126"; ddtest run --platform python --framework pytest --ci-node "$CI_NODE_INDEX" --ci-node-workers 1; coverage_files=(.coverage.*); if [[ -e "${{coverage_files[0]}}" ]]; then coverage combine; fi' \
+        "${TEST_ENVIRONMENT_HASH}"
+
 # Required jobs will appear here

pyproject.toml

@@ -123,7 +123,7 @@ lint = [
     "grpcio==1.82.1; python_version>='3.11'",
     "packaging==23.1",
     "pygments==2.16.1",
-    "riot==0.22.0",
+    "riot==0.23.0",
     "ruff==0.14.10",
     "clang-format==18.1.5",
     "cmake-format==0.6.13",

riotfile.py

@@ -435,6 +435,10 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT
                 "freezegun": latest,
             },
             env={
+                "DDTEST_SUITE_PATH": "tests/tracer",
+                "DDTEST_TESTS_LOCATION": "tests/tracer/**/test*.py",
+                "DD_TEST_OPTIMIZATION_RUNNER_TESTS_EXCLUDE_PATTERN": "tests/tracer/test_uwsgi_shutdown.py",
+                "_DD_PYTEST_XDIST_INFERRED_SERVICE": "tests.tracer",
                 "DD_CIVISIBILITY_LOG_LEVEL": "none",
                 "DD_INSTRUMENTATION_TELEMETRY_ENABLED": "0",
                 "_DD_CIVISIBILITY_PARTIAL_FLUSH_MIN_SPANS": "50",
@@ -503,9 +507,14 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT
         ),
         Venv(
             name="integration",
+            env={
+                "DDTEST_SUITE_PATH": "tests/integration",
+                "DDTEST_TESTS_LOCATION": "tests/integration/**/test*.py",
+                "DDTEST_PYTEST_ADDOPTS": "-vv --ignore-glob='*civisibility*'",
+            },
             # Enabling coverage for integration tests breaks certain tests in CI
             # Also, running two separate pytest sessions, the ``civisibility`` one with --no-ddtrace
-            command="pytest -vv --no-cov --ignore-glob='*civisibility*' {cmdargs} tests/integration/",
+            command="pytest -vv --no-cov --ignore-glob='*civisibility*' {cmdargs} ${{DDTEST_SUITE_PATH}}/",
             pkgs={"msgpack": [latest], "coverage": latest, "pytest-randomly": latest},
             pys=select_pys(),
             venvs=[
@@ -525,9 +534,13 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT
         ),
         Venv(
             name="integration-civisibility",
+            env={
+                "DDTEST_SUITE_PATH": "tests/integration/test_integration_civisibility.py",
+                "DDTEST_TESTS_LOCATION": "tests/integration/test_integration_civisibility.py",
+            },
             # Enabling coverage for integration tests breaks certain tests in CI
             # Also, running two separate pytest sessions, the ``civisibility`` one with --no-ddtrace
-            command="pytest --no-cov {cmdargs} tests/integration/test_integration_civisibility.py",
+            command="pytest --no-cov {cmdargs} ${{DDTEST_SUITE_PATH}}",
             pkgs={"msgpack": [latest], "coverage": latest, "pytest-randomly": latest},
             pys=select_pys(),
             venvs=[