diff --git a/pipeline/pipelines/ci-pipeline-dev.yaml b/pipeline/pipelines/ci-pipeline-dev.yaml index 611d3f61..bb82f337 100644 --- a/pipeline/pipelines/ci-pipeline-dev.yaml +++ b/pipeline/pipelines/ci-pipeline-dev.yaml @@ -478,13 +478,10 @@ spec: # ====================================================================== # PHASE 4: ANALYZE (statistical analysis and report generation) - # Note: For MCPChecker, evaluate phase produces the final report directly + # MCPChecker skips A/B statistics. The scorecard step still runs and + # pushes Compass facts when metadata.yaml gate_policy requests it. # ====================================================================== - name: analyze - when: - - input: $(params.eval-engine) - operator: notin - values: ["mcpchecker"] runAfter: ["evaluate"] taskRef: name: analyze-and-check-degradation diff --git a/pipeline/pipelines/ci-pipeline.yaml b/pipeline/pipelines/ci-pipeline.yaml index 46b1bdad..9425d44f 100644 --- a/pipeline/pipelines/ci-pipeline.yaml +++ b/pipeline/pipelines/ci-pipeline.yaml @@ -545,13 +545,10 @@ spec: # ====================================================================== # PHASE 4: ANALYZE (statistical analysis and report generation) - # Note: For MCPChecker, evaluate phase produces the final report directly + # MCPChecker skips A/B statistics. The scorecard step still runs and + # pushes Compass facts when metadata.yaml gate_policy requests it. # ====================================================================== - name: analyze - when: - - input: $(params.eval-engine) - operator: notin - values: ["mcpchecker"] runAfter: ["evaluate"] taskRef: name: analyze-and-check-degradation diff --git a/pipeline/tasks/phases/evaluate.yaml b/pipeline/tasks/phases/evaluate.yaml index a50eff96..aef5b824 100644 --- a/pipeline/tasks/phases/evaluate.yaml +++ b/pipeline/tasks/phases/evaluate.yaml @@ -926,7 +926,7 @@ spec: when: - input: "$(params.eval-engine)" operator: in - values: ["harbor", "both"] + values: ["mcpchecker"] image: registry.access.redhat.com/ubi9/python-311:9.6 env: - name: MCP_URL @@ -996,7 +996,7 @@ spec: when: - input: "$(params.eval-engine)" operator: in - values: ["harbor", "both"] + values: ["mcpchecker"] image: registry.access.redhat.com/ubi9/python-311:9.6 script: | #!/bin/bash diff --git a/pipeline/tasks/post/analyze-and-check-degradation.yaml b/pipeline/tasks/post/analyze-and-check-degradation.yaml index 3140f6e2..6fa9460f 100644 --- a/pipeline/tasks/post/analyze-and-check-degradation.yaml +++ b/pipeline/tasks/post/analyze-and-check-degradation.yaml @@ -20,7 +20,7 @@ spec: type: string default: "harbor" description: >- - Evaluation engine used: 'harbor', 'ase', 'a2a', 'aeh', + Evaluation engine used: 'harbor', 'ase', 'a2a', 'mcpchecker', 'aeh', 'aeh_openshell_openclaw', or 'both'. Controls which analysis path runs. - name: commit-sha type: string @@ -220,6 +220,14 @@ spec: python scripts/analyze.py "${ARGS[@]}" echo "=== Both mode: ASE report already at $REPORT_DIR ===" + + elif [ "$EVAL_ENGINE" = "mcpchecker" ]; then + echo "=== MCPChecker mode: using report from evaluate ===" + if [ ! -f "$REPORT_DIR/mcpchecker-report.json" ]; then + echo "ERROR: MCPChecker report not found at $REPORT_DIR/mcpchecker-report.json" + exit 1 + fi + echo "Found MCPChecker report at $REPORT_DIR" fi # --- 3. Enrich report with PR/LLM metadata (ASE mode) --- @@ -246,7 +254,7 @@ spec: fi # --- 5. Regenerate markdown from enriched JSON (ASE/both only) --- - if [ "$EVAL_ENGINE" != "harbor" ] && [ "$EVAL_ENGINE" != "a2a" ]; then + if [ "$EVAL_ENGINE" != "harbor" ] && [ "$EVAL_ENGINE" != "a2a" ] && [ "$EVAL_ENGINE" != "mcpchecker" ]; then echo "Regenerating report.md from enriched JSON..." python3 -c "import sys;sys.path.insert(0,'$PIPELINE_DIR');from pathlib import Path;from abevalflow.report import AnalysisResult;from scripts.analyze import render_markdown;d=Path(sys.argv[1]);r=AnalysisResult.model_validate_json((d/'report.json').read_text());(d/'report.md').write_text(render_markdown(r))" "$REPORT_DIR" fi @@ -261,18 +269,31 @@ spec: # --- 6. Extract results for Tekton --- echo -n "$REPORT_DIR" > "$(results.report-path.path)" - python3 -c "import json,sys;r=json.loads(open(sys.argv[1]).read());s=r['summary'];open(sys.argv[2],'w').write(s['recommendation']);t,c=s['treatment'].get('mean_reward'),s['control'].get('mean_reward');open(sys.argv[3],'w').write(f'{t:.4f}' if t is not None else 'N/A');open(sys.argv[4],'w').write(f'{c:.4f}' if c is not None else 'N/A');g=s.get('mean_reward_gap');open(sys.argv[5],'w').write(f'{g:+.4f}' if g is not None else 'N/A');tt,fi=s.get('ttest_p_value'),s.get('fisher_p_value');open(sys.argv[6],'w').write(f'{tt:.4f}' if tt is not None else 'N/A');open(sys.argv[7],'w').write(f'{fi:.4f}' if fi is not None else 'N/A')" \ - "$REPORT_DIR/report.json" \ - "$(results.recommendation.path)" \ - "$(results.treatment-mean-reward.path)" \ - "$(results.control-mean-reward.path)" \ - "$(results.mean-reward-gap.path)" \ - "$(results.ttest-p-value.path)" \ - "$(results.fisher-p-value.path)" + if [ "$EVAL_ENGINE" = "mcpchecker" ]; then + python3 -c "import json,sys;r=json.load(open(sys.argv[1]));score=r.get('overall_score');rec=r.get('recommendation') or ('pass' if isinstance(score,(int,float)) and score>=0.7 else 'fail');open(sys.argv[2],'w').write(rec);open(sys.argv[3],'w').write(f'{score:.4f}' if isinstance(score,(int,float)) else 'N/A');open(sys.argv[4],'w').write('N/A');open(sys.argv[5],'w').write('N/A');open(sys.argv[6],'w').write('N/A');open(sys.argv[7],'w').write('N/A')" \ + "$REPORT_DIR/mcpchecker-report.json" \ + "$(results.recommendation.path)" \ + "$(results.treatment-mean-reward.path)" \ + "$(results.control-mean-reward.path)" \ + "$(results.mean-reward-gap.path)" \ + "$(results.ttest-p-value.path)" \ + "$(results.fisher-p-value.path)" + else + python3 -c "import json,sys;r=json.loads(open(sys.argv[1]).read());s=r['summary'];open(sys.argv[2],'w').write(s['recommendation']);t,c=s['treatment'].get('mean_reward'),s['control'].get('mean_reward');open(sys.argv[3],'w').write(f'{t:.4f}' if t is not None else 'N/A');open(sys.argv[4],'w').write(f'{c:.4f}' if c is not None else 'N/A');g=s.get('mean_reward_gap');open(sys.argv[5],'w').write(f'{g:+.4f}' if g is not None else 'N/A');tt,fi=s.get('ttest_p_value'),s.get('fisher_p_value');open(sys.argv[6],'w').write(f'{tt:.4f}' if tt is not None else 'N/A');open(sys.argv[7],'w').write(f'{fi:.4f}' if fi is not None else 'N/A')" \ + "$REPORT_DIR/report.json" \ + "$(results.recommendation.path)" \ + "$(results.treatment-mean-reward.path)" \ + "$(results.control-mean-reward.path)" \ + "$(results.mean-reward-gap.path)" \ + "$(results.ttest-p-value.path)" \ + "$(results.fisher-p-value.path)" + fi echo "=== Analysis complete ===" echo "Report: $REPORT_DIR" - cat "$REPORT_DIR/report.md" + if [ -f "$REPORT_DIR/report.md" ]; then + cat "$REPORT_DIR/report.md" + fi - name: aggregate-scorecard image: registry.access.redhat.com/ubi9/python-311:9.6