fix(ai-red-teaming): collect multimodal attack results via assessment.run()

capabilities/ai-red-teaming/capability.yaml

@@ -1,6 +1,6 @@
 schema: 1
 name: ai-red-teaming
-version: "1.8.0"
+version: "1.9.0"
 description: >
   Probe the security and safety of AI applications, agents, and foundation models.
   Orchestrates adversarial attack workflows to discover vulnerabilities in LLMs,

capabilities/ai-red-teaming/scripts/attack_runner.py

@@ -5938,9 +5938,14 @@ def _media_at(media_idx):
                     airt_target_model=TARGET_MODEL,
                     airt_evaluator_model=JUDGE_MODEL,
                 )
-                await attack.run()
+                # Run the attack THROUGH the assessment (not attack.run()) so the
+                # AttackResult is collected into assessment.attack_results and
+                # uploaded to the platform. Calling attack.run() directly leaves
+                # assessment.attack_results empty, which makes _write_local_analytics
+                # report "0 finished trials" even though the trials executed.
+                await assessment.run(attack)
             except Exception as e:
-                print(f"  ERROR in media set {{i}}: {{e}}")
+                print(f"  ERROR in media set {{set_number}}: {{e}}")
                 traceback.print_exc()
             # Flush spans between studies so each set's traces reach the platform.
             try:

capabilities/ai-red-teaming/tests/test_attack_runner.py

@@ -721,6 +721,44 @@ def test_assessment_methods_exist(self, tmp_path, monkeypatch) -> None:
         missing = sorted(m for m in called if not hasattr(Assessment, m))
         assert not missing, "generated script calls missing Assessment methods: {}".format(missing)
 
+    def test_attack_runs_through_assessment(self, tmp_path, monkeypatch) -> None:
+        """Regression: the multimodal attack must run via ``assessment.run(attack)``,
+        NOT ``attack.run()`` directly.
+
+        Only ``Assessment.run()`` appends the AttackResult to
+        ``assessment.attack_results``; running the Study directly leaves that list
+        empty, so ``_write_local_analytics`` falsely reports "0 finished trials"
+        even though the trials executed and traced. See regression where a custom
+        SageMaker multimodal run reported 0 trials despite succeeding.
+        """
+        res = self._gen(
+            tmp_path,
+            monkeypatch,
+            {"goal": "g", "target_model": "openai/gpt-4o", "image_paths": ["/tmp/a.png"]},
+        )
+        assert "error" not in res, res
+        script = Path(res["filepath"]).read_text()
+        compile(script, "multimodal.py", "exec")
+        # The attack is collected through the assessment...
+        assert "await assessment.run(attack)" in script
+        # ...and NOT run directly (which would bypass result collection).
+        assert "await attack.run()" not in script
+
+    def test_error_handler_uses_valid_loop_var(self, tmp_path, monkeypatch) -> None:
+        """Regression: the per-set except handler must reference the real loop
+        variable (``set_number``), not an undefined ``i`` — otherwise an errored
+        set raises NameError inside the handler and masks the true error."""
+        res = self._gen(
+            tmp_path,
+            monkeypatch,
+            {"goal": "g", "target_model": "openai/gpt-4o", "image_paths": ["/tmp/a.png"]},
+        )
+        assert "error" not in res, res
+        script = Path(res["filepath"]).read_text()
+        compile(script, "multimodal.py", "exec")
+        assert "ERROR in media set {set_number}" in script
+        assert "ERROR in media set {i}" not in script
+
     def test_default_path_has_empty_prompts(self, tmp_path, monkeypatch) -> None:
         """Without per-media prompts, PROMPTS is empty and single-goal behaviour holds."""
         res = self._gen(
@@ -791,7 +829,8 @@ def test_per_media_prompts_list(self, tmp_path, monkeypatch) -> None:
         script = Path(res["filepath"]).read_text()
         compile(script, "multimodal.py", "exec")
         assert "PROMPTS = ['prompt one', 'prompt two']" in script
-        assert "GOAL_FOR_SET = PROMPTS[i]" in script or "goal_for_set = PROMPTS[i]" in script
+        # Each media set draws its prompt from PROMPTS by index, falling back to GOAL.
+        assert "set_prompt = PROMPTS[media_idx]" in script
 
     def test_per_media_prompts_csv(self, tmp_path, monkeypatch) -> None:
         csv_path = tmp_path / "prompts.csv"