feat(validator): add opt-in CRE NCCL and training goodput checks

.claude/CLAUDE.md

@@ -71,6 +71,8 @@ make tools-check  # Verify versions match .settings.yaml
 make check-health COMPONENT=nvsentinel  # Direct chainsaw against Kind
 make check-health-all                   # Registry-linked components only
 make validate-local RECIPE=recipe.yaml  # Full pipeline in Kind
+# Live-cluster performance phase (shipped EKS H100 training uses TrainJob NCCL)
+make validate-performance RECIPE=recipe.yaml SNAPSHOT=snapshot.yaml
 ```
 
 ## Non-Negotiable Rules

AGENTS.md

@@ -71,6 +71,8 @@ make tools-check  # Verify versions match .settings.yaml
 make check-health COMPONENT=nvsentinel  # Direct chainsaw against Kind
 make check-health-all                   # Registry-linked components only
 make validate-local RECIPE=recipe.yaml  # Full pipeline in Kind
+# Live-cluster performance phase (shipped EKS H100 training uses TrainJob NCCL)
+make validate-performance RECIPE=recipe.yaml SNAPSHOT=snapshot.yaml
 ```
 
 ## Non-Negotiable Rules

Makefile

@@ -791,11 +791,14 @@ validator-binaries: ## Builds Linux validator binaries (VALIDATOR_PHASES, VALIDA
 .PHONY: image-validators
 image-validators: build ## Builds per-phase validator images (IMAGE_REGISTRY, IMAGE_TAG)
 	@set -e; \
-	arch="$$(go env GOARCH)"; \
+	arch="$(VALIDATOR_ARCHES)"; \
+	if [ -z "$${arch}" ]; then arch="$$(go env GOARCH)"; fi; \
+	arch="$${arch%% *}"; arch="$${arch%%,*}"; \
 	VALIDATOR_ARCHES="$${arch}" ./tools/build-validator-binaries; \
 	for phase in deployment performance conformance; do \
 		echo "Building validator image: $(IMAGE_REGISTRY)/aicr-validators/$${phase}:$(IMAGE_TAG)"; \
 		docker build -f validators/$${phase}/Dockerfile \
+			--platform linux/$${arch} \
 			--build-arg TARGETARCH=$${arch} \
 			-t $(IMAGE_REGISTRY)/aicr-validators/$${phase}:$(IMAGE_TAG) .; \
 		if [ -n "$(IMAGE_REGISTRY)" ] && [ "$(IMAGE_REGISTRY)" != "localhost:5005" ]; then \
@@ -805,6 +808,7 @@ image-validators: build ## Builds per-phase validator images (IMAGE_REGISTRY, IM
 	done; \
 	echo "Building validator image: $(IMAGE_REGISTRY)/aicr-validators/aiperf-bench:$(IMAGE_TAG)"; \
 	docker build -f validators/performance/aiperf-bench.Dockerfile \
+		--platform linux/$${arch} \
 		-t $(IMAGE_REGISTRY)/aicr-validators/aiperf-bench:$(IMAGE_TAG) .; \
 	if [ -n "$(IMAGE_REGISTRY)" ] && [ "$(IMAGE_REGISTRY)" != "localhost:5005" ]; then \
 		echo "Pushing: $(IMAGE_REGISTRY)/aicr-validators/aiperf-bench:$(IMAGE_TAG)"; \
@@ -895,6 +899,43 @@ validate-local: image-validators ## Builds validator images and runs validation
 		--recipe "$(RECIPE)" \
 		--phase deployment
 
+.PHONY: validate-performance
+validate-performance: ## Recipe performance phase on the current kubecontext (RECIPE= [SNAPSHOT=] [NODE_SELECTOR=])
+	@set -e; \
+	if [ -z "$(RECIPE)" ]; then \
+		echo "Usage: make validate-performance RECIPE=<path-to-recipe.yaml> [SNAPSHOT=<path>] [NODE_SELECTOR=nvidia.com/gpu.present=true]"; \
+		exit 1; \
+	fi; \
+	if [ ! -f "$(RECIPE)" ]; then \
+		echo "Error: recipe file $(RECIPE) not found"; \
+		exit 1; \
+	fi; \
+	AICR_BIN=$$(command -v aicr 2>/dev/null || true); \
+	if [ -z "$$AICR_BIN" ]; then \
+		HOST_GOOS=$$(go env GOOS); HOST_GOARCH=$$(go env GOARCH); \
+		DIST_DIR=$$(find dist -maxdepth 1 -type d -name "aicr_$${HOST_GOOS}_$${HOST_GOARCH}*" 2>/dev/null | head -1); \
+		if [ -n "$$DIST_DIR" ] && [ -x "$$DIST_DIR/aicr" ]; then \
+			AICR_BIN="$$DIST_DIR/aicr"; \
+		fi; \
+	fi; \
+	if [ -z "$$AICR_BIN" ]; then \
+		echo "Error: aicr binary not found. Run 'make build' or put aicr on PATH."; \
+		exit 1; \
+	fi; \
+	set -- validate --recipe "$(RECIPE)" --phase performance --fail-on-error; \
+	if [ -n "$(SNAPSHOT)" ]; then \
+		if [ ! -f "$(SNAPSHOT)" ]; then \
+			echo "Error: snapshot file $(SNAPSHOT) not found"; \
+			exit 1; \
+		fi; \
+		set -- "$$@" --snapshot "$(SNAPSHOT)"; \
+	fi; \
+	if [ -n "$(NODE_SELECTOR)" ]; then \
+		set -- "$$@" --node-selector "$(NODE_SELECTOR)"; \
+	fi; \
+	echo "Running performance phase with $$AICR_BIN"; \
+	"$$AICR_BIN" "$$@"
+
 .PHONY: python-licenses
 python-licenses: ## Refreshes the committed Python license section for the aiperf-bench image (needs network)
 	@python3 tools/generate-python-licenses

docs/contributor/validator.md

@@ -899,7 +899,19 @@ silently fall back.
 
 Full list (defaults, semantics) is in the `validators/performance`
 package godoc. NCCL variants exposed today: `nccl-all-reduce-bw`,
-`nccl-all-reduce-bw-net`, `nccl-all-reduce-bw-nvls`. Inference:
+`nccl-all-reduce-bw-net`, `nccl-all-reduce-bw-nvls`. Opt-in public CRE
+checks: `nccl-cre-all-reduce-bw` and `cre-training-goodput`. Neither branches on
+service or accelerator — `creQualifiedEntries` in
+`validators/performance/cre_qualification.go` records which combinations are
+qualified and the catalog entry and node cap each was measured with, so a new
+combination is an entry there plus a measured threshold rather than a new code
+path. On `eks` x `h100` the entries are `communication/nccl-all-reduce` and
+`training/nemotron5-8b`. Both create the
+CR on an explicitly named set of nodes, wait with a timeout, and tear it down on
+success or failure — deleting the CR and then confirming the `TrainJob`s and
+workload pods it started are gone, since CRE's controller drops its finalizer
+without waiting for them. An unconfirmed teardown fails the check rather than
+warning, because surviving work still holds the GPUs. Inference:
 `inference-perf` (Dynamo + AIPerf).
 
 > **Constraint-name contract.** Each NCCL variant looks up a

docs/user/validation.md

@@ -63,6 +63,25 @@ is installed on the cluster.
 | `nccl-all-reduce-bw-net` | NET (EFA on EKS by default; ConnectX RoCE via `AICR_NCCL_FABRIC=roce`; built-in IB/verbs on OKE) | GB200 + EKS, GB200 + OKE, and GB300 + EKS. Asserts the intended NET fabric actually carried traffic — EFA on EKS, the NVL72 InfiniBand east-west fabric (`nvidia.com/mlnxnics` shared HCAs) on OKE — catching silent fallback to Socket when GPUDirect RDMA is unavailable. A driver preflight gates the benchmark on the default fabric — see [Grace Blackwell NET preflight](#grace-blackwell-net-preflight-gpudirect-rdma-prerequisites). |
 | `nccl-all-reduce-bw-nvls` | NVLS (MNNVL across an NVL72 IMEX domain) | GB200 + EKS, GB200 + OKE, GB200 + GKE (A4X GPUDirect-RDMA/gIB carries the IMEX/NVLink fabric traffic; gIB is the transport driver, not the NCCL algorithm), GB300 (EKS, generic), and VR200 + RKE2. Asserts the NVLS communicator actually initialized. Catches silent fallback to the NET fabric (EFA on EKS, InfiniBand on OKE), or gIB/RDMA initialization failure (GKE), when the IMEX domain is misconfigured. |
 
+An opt-in pair of Cluster Readiness Engine (CRE) checks runs against public CRE
+(`nvcre.nvidia.com`, [cluster-readiness-engine](https://github.com/NVIDIA/cluster-readiness-engine)).
+Neither check branches on service or accelerator; which combinations are
+qualified is recorded as data, and `eks` x `h100` is the combination qualified
+today. Shipped overlays keep the TrainJob `nccl-all-reduce-bw` path. Add `nvcre`
+and the CRE check names only when you intend to run CRE. Each CRE check requires
+a same-named constraint:
+
+| Check | What it measures |
+|---|---|
+| `nccl-cre-all-reduce-bw` | Bus bandwidth from a CRE `Certification` `BandwidthMeasurement`; AICR still asserts the transport from launcher logs |
+| `cre-training-goodput` | Runtime goodput from a CRE `Certification` `GoodputMeasurement` |
+
+The catalog entry each check drives, and the node footprint it runs on, come
+from the qualification record rather than the check. On `eks` x `h100` that is
+`communication/nccl-all-reduce` and `training/nemotron5-8b`, each on two nodes.
+A combination with no entry skips, because it has no calibrated threshold to
+judge against.
+
 The applicability column is the *default*, derived from the recipe's
 `criteria`. A recipe whose criteria fall outside it can still run these
 benchmarks explicitly — either by

pkg/defaults/timeouts.go

@@ -752,6 +752,25 @@ const (
 	// bound instead of the much shorter DiagnosticTimeout used for the rest of
 	// createUnstructured's calls.
 	NCCLResourceRecreateWait = 5 * time.Minute
+
+	// CRECertificationTimeout is AICR's wait budget for a Cluster Readiness
+	// Engine Certification (NCCL or training/goodput) to reach Succeeded or
+	// Failed. The same duration is written on spec.timeoutPerJob so CRE
+	// stops the job instead of leaving it running after AICR gives up.
+	CRECertificationTimeout = 30 * time.Minute
+
+	// CRECertificationDeleteTimeout bounds the removal of the Certification
+	// object itself. The shorter DiagnosticTimeout would expire while the CR is
+	// still held by finalizers.
+	CRECertificationDeleteTimeout = 5 * time.Minute
+
+	// CRECertificationTeardownTimeout bounds the whole teardown: removing the
+	// Certification and then confirming the TrainJobs and GPU pods it started
+	// are gone. It must outlast CRE's own drain barrier, which waits up to five
+	// minutes for workload pods to exit and then proceeds regardless
+	// (podDrainGracePeriod in pkg/controller/pod_drain.go), plus the
+	// termination grace of the multi-node GPU pods left behind.
+	CRECertificationTeardownTimeout = 10 * time.Minute
 )
 
 // Inference performance validation timeouts.

pkg/recipe/nccl_cre_eks_h100_test.go

@@ -0,0 +1,134 @@
+// Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package recipe
+
+import (
+	"context"
+	"testing"
+)
+
+// TestH100EKSTrainingCREStaysOptIn pins the CRE rollout gate: the `nvcre`
+// component and its `nccl-cre-all-reduce-bw` Certification check are defined
+// in the registry and the validator catalog, but no shipped overlay may
+// reference them. AICR has no mechanism for a user-selectable optional
+// component — a profile cannot introduce a componentRef
+// (applyEffectiveProfile rejects one absent from the composition) and
+// `overrides.enabled: false` is one-way (the bundler refuses to re-enable a
+// recipe-disabled component). Until that mechanism lands, attaching CRE to an
+// overlay would make it mandatory for every EKS H100 training consumer and
+// would retire the TrainJob path before its results are correlated.
+func TestH100EKSTrainingCREStaysOptIn(t *testing.T) {
+	const creCheck = "nccl-cre-all-reduce-bw"
+	const trainJobCheck = "nccl-all-reduce-bw"
+
+	tests := []struct {
+		name         string
+		criteria     *Criteria
+		wantTrainJob bool
+		wantValue    string
+	}{
+		{
+			name: "h100-eks-training",
+			criteria: &Criteria{
+				Service:     CriteriaServiceEKS,
+				Accelerator: CriteriaAcceleratorH100,
+				Intent:      CriteriaIntentTraining,
+				Platform:    CriteriaPlatformAny,
+			},
+			wantTrainJob: true,
+			wantValue:    ">= 300",
+		},
+		{
+			name: "h100-eks-ubuntu-training",
+			criteria: &Criteria{
+				Service:     CriteriaServiceEKS,
+				Accelerator: CriteriaAcceleratorH100,
+				OS:          CriteriaOSUbuntu,
+				Intent:      CriteriaIntentTraining,
+				Platform:    CriteriaPlatformAny,
+			},
+			wantTrainJob: true,
+			wantValue:    ">= 300",
+		},
+		{
+			name: "h100-eks-ubuntu-training-kubeflow",
+			criteria: &Criteria{
+				Service:     CriteriaServiceEKS,
+				Accelerator: CriteriaAcceleratorH100,
+				OS:          CriteriaOSUbuntu,
+				Intent:      CriteriaIntentTraining,
+				Platform:    CriteriaPlatformKubeflow,
+			},
+			wantTrainJob: true,
+			wantValue:    ">= 300",
+		},
+		{
+			name: "h100-eks-ubuntu-training-slurm",
+			criteria: &Criteria{
+				Service:     CriteriaServiceEKS,
+				Accelerator: CriteriaAcceleratorH100,
+				OS:          CriteriaOSUbuntu,
+				Intent:      CriteriaIntentTraining,
+				Platform:    CriteriaPlatformSlurm,
+			},
+			wantTrainJob: false,
+		},
+	}
+
+	ctx := context.Background()
+	store, err := loadMetadataStore(ctx)
+	if err != nil {
+		t.Fatalf("loadMetadataStore: %v", err)
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result, err := store.BuildRecipeResult(ctx, tt.criteria)
+			if err != nil {
+				t.Fatalf("BuildRecipeResult: %v", err)
+			}
+
+			if performanceCheckPresent(result.Validation, creCheck) {
+				t.Errorf("performance check %q is attached to a shipped overlay; CRE must stay opt-in", creCheck)
+			}
+			if value, found := findPerformanceConstraint(result.Validation, creCheck); found {
+				t.Errorf("performance constraint %q resolved to %q; CRE must stay opt-in", creCheck, value)
+			}
+			for _, ref := range result.ComponentRefs {
+				if ref.Name == "nvcre" {
+					t.Error("resolved recipe declares the nvcre componentRef; CRE must stay opt-in")
+					break
+				}
+			}
+
+			gotValue, found := findPerformanceConstraint(result.Validation, trainJobCheck)
+			if !tt.wantTrainJob {
+				if performanceCheckPresent(result.Validation, trainJobCheck) || found {
+					t.Errorf("performance check %q should be cleared on this leaf", trainJobCheck)
+				}
+				return
+			}
+			if !performanceCheckPresent(result.Validation, trainJobCheck) {
+				t.Errorf("performance check %q not present in resolved checks", trainJobCheck)
+			}
+			if !found {
+				t.Fatalf("performance constraint %q not found; expected value %q", trainJobCheck, tt.wantValue)
+			}
+			if gotValue != tt.wantValue {
+				t.Errorf("%s = %q, want %q", trainJobCheck, gotValue, tt.wantValue)
+			}
+		})
+	}
+}

pkg/validator/catalog/catalog_test.go

@@ -1257,6 +1257,38 @@ func TestEmbeddedCatalog_NCCLEntriesExist(t *testing.T) {
 	}
 }
 
+func TestEmbeddedCatalog_CRENCCLAllReduceBWEntryExists(t *testing.T) {
+	cat, err := LoadWithDataProvider(context.Background(), nil, "v0.0.0-next", "")
+	if err != nil {
+		t.Fatalf("Load failed: %v", err)
+	}
+	for _, v := range cat.Validators {
+		if v.Name == v1.CRENCCLAllReduceBWCheckName {
+			if v.Phase != "performance" {
+				t.Errorf("%q phase = %q, want performance", v1.CRENCCLAllReduceBWCheckName, v.Phase)
+			}
+			return
+		}
+	}
+	t.Fatalf("no embedded catalog entry named %q", v1.CRENCCLAllReduceBWCheckName)
+}
+
+func TestEmbeddedCatalog_CRETrainingGoodputEntryExists(t *testing.T) {
+	cat, err := LoadWithDataProvider(context.Background(), nil, "v0.0.0-next", "")
+	if err != nil {
+		t.Fatalf("Load failed: %v", err)
+	}
+	for _, v := range cat.Validators {
+		if v.Name == v1.CRETrainingGoodputCheckName {
+			if v.Phase != "performance" {
+				t.Errorf("%q phase = %q, want performance", v1.CRETrainingGoodputCheckName, v.Phase)
+			}
+			return
+		}
+	}
+	t.Fatalf("no embedded catalog entry named %q", v1.CRETrainingGoodputCheckName)
+}
+
 func TestCatalogEmbedding(t *testing.T) {
 	// Simulate embedding in a CR spec
 	type ValidatorCatalogSpec struct {

pkg/validator/v1/job_plan_internal.go

@@ -59,6 +59,19 @@ const (
 	// would otherwise silently no-op RoCE forwarding with no test failing.
 	NCCLAllReduceBWNetCheckName = "nccl-all-reduce-bw-net"
 
+	// CRENCCLAllReduceBWCheckName is the catalog name of the CRE-driven NCCL
+	// all-reduce check (Certification). Overlays opt in by listing this check and
+	// a same-named performance constraint. No embedded overlay enables it, and
+	// the combinations it is qualified on are recorded in creQualifiedEntries in
+	// validators/performance rather than gated here.
+	CRENCCLAllReduceBWCheckName = "nccl-cre-all-reduce-bw"
+
+	// CRETrainingGoodputCheckName is the catalog name of the CRE-driven
+	// training goodput check (Certification). It opts in and records its
+	// qualified combinations the same way, and the catalog entry it drives
+	// varies per combination.
+	CRETrainingGoodputCheckName = "cre-training-goodput"
+
 	// ncclFabricEnv selects the NET fabric (efa default | roce). Forwarded to
 	// the NET check pod so the in-Job validator can observe it. This is the
 	// orchestrator (forwarding) end; the validator-pod (reading) end defines the

recipes/validators/README.md

@@ -47,10 +47,12 @@ Applied by `catalog.Load` (`pkg/validator/catalog/catalog.go`) in order:
 
 | Name | Description | Timeout |
 |------|-------------|---------|
-| `nccl-all-reduce-bw` | Verify NCCL All Reduce Bus Bandwidth meets threshold | 30m |
+| `nccl-all-reduce-bw` | Verify NCCL All Reduce Bus Bandwidth meets threshold (TrainJob path) | 30m |
 | `nccl-all-reduce-bw-net` | Verify NCCL All Reduce Bus Bandwidth on the NET transport (EFA on EKS; ConnectX RoCE via `AICR_NCCL_FABRIC=roce`) | 30m |
 | `nccl-all-reduce-bw-nvls` | Verify NCCL All Reduce Bus Bandwidth on the NVLS transport (MNNVL across an NVL72 IMEX domain) | 30m |
 | `inference-perf` | Verify inference throughput and TTFT p99 meet thresholds using AIPerf | 65m |
+| `nccl-cre-all-reduce-bw` | NCCL bus bandwidth via a CRE `Certification` (opt-in). Qualified on EKS H100 — live UAT 2026-09-02 2× p5.48xlarge: 489.80 GB/s vs `>= 300`. | 30m |
+| `cre-training-goodput` | Training goodput via a CRE `Certification` (opt-in). Qualified on EKS H100 — live UAT 2026-09-02 2× p5.48xlarge: ratio 0.7671 vs `>= 0.5`. | 30m |
 
 The NCCL checks derive applicability from the recipe's `criteria` by default;
 a recipe outside the embedded service + accelerator matrix (e.g. registered

recipes/validators/catalog.yaml

@@ -291,3 +291,26 @@ validators:
     timeout: 30m
     args: ["nccl-all-reduce-bw-nvls"]
     env: []
+  # CRE-driven NCCL check. Opt-in only: no embedded overlay references this
+  # check, and shipped EKS H100 training keeps TrainJob nccl-all-reduce-bw.
+  # The check itself is provider-independent; creQualifiedEntries records the
+  # qualified combinations. Qualified on EKS H100 — live UAT 2026-09-02
+  # (2× p5.48xlarge, public CRE v0.1.0): 489.80 GB/s vs >= 300.
+  - name: nccl-cre-all-reduce-bw
+    phase: performance
+    description: "Verify NCCL All Reduce Bus Bandwidth via a Cluster Readiness Engine Certification"
+    image: ghcr.io/nvidia/aicr-validators/performance:latest
+    timeout: 30m
+    args: ["nccl-cre-all-reduce-bw"]
+    env: []
+  # Qualified on EKS H100 — live UAT 2026-09-02 (same 2× p5 cluster): the
+  # Nemotron-5 8B catalog entry, goodput 0.7671 vs >= 0.5. The entry is per
+  # combination because training/nemotron5-56b needs minGPUs=32 and fails on
+  # this SKU.
+  - name: cre-training-goodput
+    phase: performance
+    description: "Verify training goodput via a Cluster Readiness Engine Certification"
+    image: ghcr.io/nvidia/aicr-validators/performance:latest
+    timeout: 30m
+    args: ["cre-training-goodput"]
+    env: []