From 92761cae0bf96c716347ea9ce1c40d47e9e37fa8 Mon Sep 17 00:00:00 2001 From: gziv Date: Mon, 20 Jul 2026 09:55:32 +0300 Subject: [PATCH 1/5] feat(aeh): add Agent-Eval-Harness evaluation engine Add the AEH engine adapter, runner/aggregate scripts, shared scoring helpers, validation, Containerfile, in-repo samples, and unit tests. Co-authored-by: Cursor --- abevalflow/aeh_scoring.py | 123 +++ abevalflow/engines/__init__.py | 2 + abevalflow/engines/aeh.py | 424 ++++++++++ abevalflow/report.py | 20 + abevalflow/schemas.py | 1 + containers/agent-eval-harness/Containerfile | 102 +++ scripts/aggregate_aeh.py | 526 +++++++++++++ scripts/run_aeh.py | 740 ++++++++++++++++++ scripts/validate.py | 333 +++++++- .../cases/case-001/annotations.yaml | 14 + .../aeh-hello-world/cases/case-001/input.yaml | 9 + submissions/aeh-hello-world/eval.yaml | 51 ++ submissions/aeh-hello-world/metadata.yaml | 11 + .../cases/case-001/annotations.yaml | 31 + .../cases/case-001/input.yaml | 14 + .../aeh-pairwise-example/eval-control.yaml | 51 ++ .../aeh-pairwise-example/eval-treatment.yaml | 72 ++ .../aeh-pairwise-example/metadata.yaml | 12 + .../skills/aeh-pairwise-example/SKILL.md | 10 + tests/test_aeh_engine.py | 472 +++++++++++ tests/test_aeh_scoring.py | 66 ++ tests/test_aggregate_aeh.py | 236 ++++++ tests/test_run_aeh.py | 470 +++++++++++ tests/test_validate_aeh.py | 304 +++++++ tests/test_validate_aeh_plugin_dirs.py | 32 + 25 files changed, 4119 insertions(+), 7 deletions(-) create mode 100644 abevalflow/aeh_scoring.py create mode 100644 abevalflow/engines/aeh.py create mode 100644 containers/agent-eval-harness/Containerfile create mode 100644 scripts/aggregate_aeh.py create mode 100644 scripts/run_aeh.py create mode 100644 submissions/aeh-hello-world/cases/case-001/annotations.yaml create mode 100644 submissions/aeh-hello-world/cases/case-001/input.yaml create mode 100644 submissions/aeh-hello-world/eval.yaml create mode 100644 submissions/aeh-hello-world/metadata.yaml create mode 100644 submissions/aeh-pairwise-example/cases/case-001/annotations.yaml create mode 100644 submissions/aeh-pairwise-example/cases/case-001/input.yaml create mode 100644 submissions/aeh-pairwise-example/eval-control.yaml create mode 100644 submissions/aeh-pairwise-example/eval-treatment.yaml create mode 100644 submissions/aeh-pairwise-example/metadata.yaml create mode 100644 submissions/aeh-pairwise-example/skills/aeh-pairwise-example/SKILL.md create mode 100644 tests/test_aeh_engine.py create mode 100644 tests/test_aeh_scoring.py create mode 100644 tests/test_aggregate_aeh.py create mode 100644 tests/test_run_aeh.py create mode 100644 tests/test_validate_aeh.py create mode 100644 tests/test_validate_aeh_plugin_dirs.py diff --git a/abevalflow/aeh_scoring.py b/abevalflow/aeh_scoring.py new file mode 100644 index 0000000..1d8f421 --- /dev/null +++ b/abevalflow/aeh_scoring.py @@ -0,0 +1,123 @@ +"""Shared AEH scoring helpers used by the engine adapter and aggregate script. + +Keeps pairwise win-rate / gate logic and numeric judge pass/fail heuristics +aligned across call sites (engine, aggregate_aeh, Tekton PARSE scripts). +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import yaml + +DEFAULT_AEH_THRESHOLD = 0.5 + +# Import path for AEH HarborRunner --environment-import-path (emptyDir mounts). +OPENSHIFT_ENVIRONMENT_IMPORT_PATH = "abevalflow.harbor_extensions.openshift_environment:OpenShiftEnvironment" + + +def resolve_evaluation_threshold( + metadata_path: Path | str | None = None, + *, + default: float = DEFAULT_AEH_THRESHOLD, +) -> float: + """Read ``gate_policy.gates.evaluation.threshold`` from metadata.yaml. + + Returns ``default`` when metadata is missing or the threshold is unset. + """ + if metadata_path is None: + return default + path = Path(metadata_path) + if not path.is_file(): + return default + try: + data = yaml.safe_load(path.read_text()) or {} + except (OSError, yaml.YAMLError): + return default + if not isinstance(data, dict): + return default + gate_policy = data.get("gate_policy") or {} + if not isinstance(gate_policy, dict): + return default + gates = gate_policy.get("gates") or {} + if not isinstance(gates, dict): + return default + evaluation = gates.get("evaluation") or {} + if not isinstance(evaluation, dict): + return default + threshold = evaluation.get("threshold") + if threshold is None: + return default + try: + return float(threshold) + except (TypeError, ValueError): + return default + + +def pairwise_outcome( + wins_a: int, + wins_b: int, + ties: int = 0, + errors: int = 0, + *, + threshold: float = DEFAULT_AEH_THRESHOLD, +) -> dict[str, Any]: + """Compute honest pairwise win-rate and pass/fail. + + - Denominator includes ties and errors (errors are non-wins). + - All-ties (no decisive outcomes, no errors) still passes. + - All-errors does not get the all-ties exception. + """ + wins_a = int(wins_a or 0) + wins_b = int(wins_b or 0) + ties = int(ties or 0) + errors = int(errors or 0) + total = wins_a + wins_b + ties + errors + decisive = wins_a + wins_b + win_rate = (wins_a / total) if total > 0 else 0.0 + all_ties = decisive == 0 and ties > 0 and errors == 0 + passed = all_ties or (win_rate >= threshold) + return { + "wins_a": wins_a, + "wins_b": wins_b, + "ties": ties, + "errors": errors, + "total": total, + "decisive": decisive, + "win_rate": win_rate, + "all_ties": all_ties, + "passed": passed, + "threshold": threshold, + "recommendation": "pass" if passed else "fail", + } + + +def numeric_judge_passes(value: int | float) -> bool: + """Whether a numeric judge value counts as a pass. + + Scale detection: + - Integer 1–5 → Likert; pass when ``>= 3`` + - Float in [0, 1] → normalized; pass when ``>= 0.5`` + - Float in (1, 5] → Likert; pass when ``>= 3`` + - Otherwise → normalized fallback (``>= 0.5``) + + Callers must handle booleans separately (``bool`` is a subclass of ``int``). + """ + if isinstance(value, bool): + return value + if isinstance(value, int) and 1 <= value <= 5: + return value >= 3 + v = float(value) + if 0.0 <= v <= 1.0: + return v >= 0.5 + if 1.0 < v <= 5.0: + return v >= 3.0 + return v >= 0.5 + + +def numeric_judge_is_low(value: int | float) -> bool: + """Inverse of :func:`numeric_judge_passes` for finding extraction.""" + if isinstance(value, bool): + return not value + return not numeric_judge_passes(value) diff --git a/abevalflow/engines/__init__.py b/abevalflow/engines/__init__.py index 3cd86c3..eeb337a 100644 --- a/abevalflow/engines/__init__.py +++ b/abevalflow/engines/__init__.py @@ -48,6 +48,7 @@ def get_all_engines() -> list[str]: from abevalflow.engines.a2a import A2AEngine +from abevalflow.engines.aeh import AEHEngine from abevalflow.engines.ase import ASEEngine from abevalflow.engines.harbor import HarborEngine from abevalflow.engines.mcpchecker import MCPCheckerEngine @@ -59,5 +60,6 @@ def get_all_engines() -> list[str]: "HarborEngine", "ASEEngine", "A2AEngine", + "AEHEngine", "MCPCheckerEngine", ] diff --git a/abevalflow/engines/aeh.py b/abevalflow/engines/aeh.py new file mode 100644 index 0000000..a2eff7c --- /dev/null +++ b/abevalflow/engines/aeh.py @@ -0,0 +1,424 @@ +"""Agent-Eval-Harness (AEH) evaluation engine adapter. + +AEH is a generic evaluation framework for agents and skills. This adapter +handles both single-run and pairwise comparison modes. + +Result files from AEH: + - summary.yaml: Per-judge means, per-case results, run metadata, pairwise results + - run_result.json: Execution metadata (duration, cost, tokens, mean_reward) + +The adapter reads report.json (produced by aggregate_aeh.py) or falls back +to reading summary.yaml + run_result.json directly. + +Judge types supported (passthrough - no interpretation): + - check: Inline Python snippets + - llm: LLM-based judges with prompt/prompt_file + - builtin: Registry-based judges + - code: External Python modules + - pairwise: Position-swapped LLM comparison (in pairwise mode) +""" + +from __future__ import annotations + +import json +import logging +from pathlib import Path +from typing import Any + +import yaml + +from abevalflow.aeh_scoring import ( + DEFAULT_AEH_THRESHOLD, + numeric_judge_is_low, + pairwise_outcome, +) +from abevalflow.engines import register_engine +from abevalflow.engines.base import EvalEngine +from abevalflow.gates.base import Finding, GateResult, GateType, Severity +from abevalflow.schemas import GatePolicy + +logger = logging.getLogger(__name__) + +# Back-compat alias for imports/tests +DEFAULT_PAIRWISE_THRESHOLD = DEFAULT_AEH_THRESHOLD + + +@register_engine("aeh") +class AEHEngine(EvalEngine): + """Agent-Eval-Harness evaluation engine. + + Reads AEH output (summary.yaml, run_result.json) or the unified + report.json produced by aggregate_aeh.py. + + Supports both single-run and pairwise A/B comparison modes. + """ + + name = "aeh" + + def read_result(self, reports_dir: Path) -> dict[str, Any] | None: + """Read AEH results from reports directory. + + Priority order: + 1. report.json (unified format from aggregate_aeh.py) + 2. summary.yaml + run_result.json (raw AEH output) + 3. Search subdirectories for run outputs + + Args: + reports_dir: Path to reports/{submission-name}/ or + reports/{submission-name}/{run-id}/ + + Returns: + Raw result dict, or None if not found + """ + report_path = reports_dir / "report.json" + if report_path.exists(): + try: + return json.loads(report_path.read_text()) + except (json.JSONDecodeError, OSError) as e: + logger.error("Failed to read AEH report.json: %s", e) + + summary_path = reports_dir / "summary.yaml" + run_result_path = reports_dir / "run_result.json" + + if summary_path.exists(): + try: + summary = yaml.safe_load(summary_path.read_text()) + run_result = {} + if run_result_path.exists(): + run_result = json.loads(run_result_path.read_text()) + + return self._merge_aeh_output(summary, run_result, reports_dir) + except (yaml.YAMLError, json.JSONDecodeError, OSError) as e: + logger.error("Failed to read AEH output files: %s", e) + return None + + run_dirs = [d for d in reports_dir.iterdir() if d.is_dir()] + for run_dir in sorted(run_dirs, reverse=True): + result = self._read_from_run_dir(run_dir) + if result is not None: + return result + + logger.warning("AEH results not found in: %s", reports_dir) + return None + + def _read_from_run_dir(self, run_dir: Path) -> dict[str, Any] | None: + """Read AEH results from a specific run directory.""" + summary_path = run_dir / "summary.yaml" + run_result_path = run_dir / "run_result.json" + + if not summary_path.exists(): + return None + + try: + summary = yaml.safe_load(summary_path.read_text()) + run_result = {} + if run_result_path.exists(): + run_result = json.loads(run_result_path.read_text()) + return self._merge_aeh_output(summary, run_result, run_dir) + except (yaml.YAMLError, json.JSONDecodeError, OSError) as e: + logger.error("Failed to read AEH run dir %s: %s", run_dir, e) + return None + + def _merge_aeh_output( + self, + summary: dict[str, Any], + run_result: dict[str, Any], + source_dir: Path, + ) -> dict[str, Any]: + """Merge summary.yaml and run_result.json into unified format.""" + mean_reward = run_result.get("mean_reward", summary.get("mean_reward", 0.0)) + + # Detect mode from pairwise presence + mode = "pairwise" if "pairwise" in summary else "single" + + return { + "eval_engine": "aeh", + "mode": mode, + "run_id": summary.get("run_id", source_dir.name), + "mean_reward": mean_reward, + "judges": summary.get("judges", {}), + "per_case": summary.get("per_case", {}), + "run_metrics": summary.get("run_metrics"), + "pairwise": summary.get("pairwise"), + "execution": { + "duration_s": run_result.get("duration_s"), + "cost_usd": run_result.get("cost_usd"), + "tokens": run_result.get("token_usage"), + "harbor_job_dir": run_result.get("harbor_job_dir"), + }, + "aeh_warnings": [], + "source_dir": str(source_dir), + } + + def to_gate_result( + self, + raw_result: dict[str, Any], + policy: GatePolicy, + ) -> GateResult: + """Convert AEH result to GateResult. + + For single-run mode, uses mean_reward vs threshold. + For pairwise mode, uses win_rate (treatment wins / total) vs threshold. + """ + mode = raw_result.get("mode", "single") + + if mode == "pairwise": + return self._handle_pairwise_result(raw_result, policy) + + return self._handle_single_result(raw_result, policy) + + def _handle_single_result( + self, + raw_result: dict[str, Any], + policy: GatePolicy, + ) -> GateResult: + """Handle single-run result.""" + gate_policy = policy.get_gate_policy("evaluation") + threshold = gate_policy.threshold if gate_policy.threshold is not None else self.get_default_threshold() + + mean_reward = raw_result.get("mean_reward") + if mean_reward is None: + summary = raw_result.get("summary") or {} + treatment = summary.get("treatment") or {} + mean_reward = treatment.get("mean_reward") + passed = mean_reward is not None and mean_reward >= threshold + # GateResult.score is a required float; floor missing rewards at 0.0. + score = float(mean_reward) if mean_reward is not None else 0.0 + findings = self._extract_findings_single(raw_result) + + # Format judge summary preserving all types + judges = raw_result.get("judges", {}) + judge_summary = self._format_judge_summary(judges) + + reward_str = f"{mean_reward:.3f}" if mean_reward is not None else "None" + message = f"AEH single: mean_reward={reward_str} (threshold={threshold:.2f})" + if judge_summary: + message += f" | {judge_summary}" + + return GateResult( + gate_type=GateType.ENGINE, + gate_name="evaluation", + policy_key=self.name, + passed=passed, + score=score, + mode=gate_policy.mode, + threshold=threshold, + findings=findings, + details={ + "engine": self.name, + "mode": "single", + "judges": judges, + "per_case": raw_result.get("per_case", {}), + "run_metrics": raw_result.get("run_metrics"), + "execution": raw_result.get("execution"), + "aeh_warnings": raw_result.get("aeh_warnings", []), + }, + message=message, + ) + + def _handle_pairwise_result( + self, + raw_result: dict[str, Any], + policy: GatePolicy, + ) -> GateResult: + """Handle pairwise comparison result. + + In pairwise mode: + - wins_a = treatment wins (treatment is run_a in AEH's compare_runs) + - wins_b = control wins + - Win rate = wins_a / (wins_a + wins_b + ties + errors) + - Ties and errors count as non-wins + - All-ties (no decisive outcomes, no errors) still passes + - Default threshold is 0.5 (treatment must win majority of cases) + """ + gate_policy = policy.get_gate_policy("evaluation") + + # Use default pairwise threshold if not specified + threshold = gate_policy.threshold + if threshold is None: + threshold = DEFAULT_PAIRWISE_THRESHOLD + + pairwise = raw_result.get("pairwise", {}) + outcome = pairwise_outcome( + pairwise.get("wins_a", 0), + pairwise.get("wins_b", 0), + pairwise.get("ties", 0), + pairwise.get("errors", 0), + threshold=threshold, + ) + wins_a = outcome["wins_a"] + ties = outcome["ties"] + errors = outcome["errors"] + total = outcome["total"] + win_rate = outcome["win_rate"] + passed = outcome["passed"] + + findings = self._extract_findings_pairwise(raw_result) + + # Include stability info if present + stability = pairwise.get("stability") + stability_note = "" + if stability: + agreement = stability.get("agreement_rate", 0) * 100 + stability_note = f" | stability={agreement:.0f}%" + + message = ( + f"AEH pairwise: treatment wins {wins_a}/{total} ({win_rate:.0%}) " + f"| ties={ties}, errors={errors}{stability_note}" + ) + + return GateResult( + gate_type=GateType.ENGINE, + gate_name="evaluation", + policy_key=self.name, + passed=passed, + score=win_rate, + mode=gate_policy.mode, + threshold=threshold, + findings=findings, + details={ + "engine": self.name, + "mode": "pairwise", + "pairwise": pairwise, + "treatment": raw_result.get("treatment"), + "control": raw_result.get("control"), + "mean_reward": raw_result.get("mean_reward"), + "aeh_warnings": raw_result.get("aeh_warnings", []), + }, + message=message, + ) + + def _format_judge_summary(self, judges: dict[str, Any]) -> str: + """Format judge results without assuming type. + + Handles all judge types: check, llm, builtin, code. + Each judge entry can be a dict with mean/pass_rate or a raw value. + """ + parts = [] + for name, data in judges.items(): + if isinstance(data, dict): + if "pass_rate" in data and data["pass_rate"] is not None: + parts.append(f"{name}={data['pass_rate']:.0%}") + elif "mean" in data and data["mean"] is not None: + parts.append(f"{name}={data['mean']:.2f}") + elif isinstance(data, (int, float)): + parts.append(f"{name}={data:.2f}") + return ", ".join(parts) if parts else "" + + def _extract_findings_single(self, raw_result: dict[str, Any]) -> list[Finding]: + """Extract findings from single-run per-case results. + + Handles two per_case formats: + 1. Simple format (from old tests/simple aggregation): + {"case-001": {"reward": 0.3}} + 2. Nested AEH format (from real AEH output): + {"case-001": {"judge_name": {"value": True/False or 1-5, ...}}} + """ + findings = [] + per_case = raw_result.get("per_case", {}) + + for case_id, case_data in per_case.items(): + if not isinstance(case_data, dict): + continue + + # Check for simple format first (has 'reward' key directly) + if "reward" in case_data or "mean_reward" in case_data: + reward = case_data.get("reward", case_data.get("mean_reward", 1.0)) + if isinstance(reward, (int, float)) and reward < 0.5: + severity = Severity.HIGH if reward < 0.25 else Severity.MEDIUM + findings.append( + Finding( + severity=severity, + message=f"Case {case_id} scored low: reward={reward:.2f}", + location=case_id, + rule_id="aeh-low-reward", + ) + ) + continue + + # Nested AEH format - iterate over judges + for judge_name, judge_result in case_data.items(): + if not isinstance(judge_result, dict): + continue + + value = judge_result.get("value") + error = judge_result.get("error") + + if error: + findings.append( + Finding( + severity=Severity.HIGH, + message=f"Case {case_id} judge '{judge_name}' error: {error}", + location=f"{case_id}/{judge_name}", + rule_id="aeh-judge-error", + ) + ) + elif value is False: + findings.append( + Finding( + severity=Severity.MEDIUM, + message=f"Case {case_id} judge '{judge_name}' failed", + location=f"{case_id}/{judge_name}", + rule_id="aeh-judge-failed", + ) + ) + elif isinstance(value, (int, float)) and not isinstance(value, bool) and numeric_judge_is_low(value): + severity = ( + Severity.HIGH + if (isinstance(value, int) and value < 2) or (isinstance(value, float) and value < 0.25) + else Severity.MEDIUM + ) + findings.append( + Finding( + severity=severity, + message=f"Case {case_id} judge '{judge_name}' scored low: {value}", + location=f"{case_id}/{judge_name}", + rule_id="aeh-low-score", + ) + ) + + return findings + + def _extract_findings_pairwise(self, raw_result: dict[str, Any]) -> list[Finding]: + """Extract findings from pairwise comparison results.""" + findings = [] + pairwise = raw_result.get("pairwise", {}) + per_case = pairwise.get("per_case", []) + + for case_result in per_case: + if not isinstance(case_result, dict): + continue + + case_id = case_result.get("case_id", "unknown") + winner = case_result.get("winner") + error = case_result.get("error") + + if error: + findings.append( + Finding( + severity=Severity.HIGH, + message=f"Case {case_id} comparison error: {error}", + location=case_id, + rule_id="aeh-pairwise-error", + ) + ) + elif winner == "B": + findings.append( + Finding( + severity=Severity.MEDIUM, + message=f"Case {case_id}: control beat treatment", + location=case_id, + rule_id="aeh-control-wins", + ) + ) + + return findings + + def get_default_threshold(self) -> float: + """AEH default threshold is 0.5. + + This aligns with the recommendation logic in aggregate_aeh.py. + For single mode: mean_reward >= 0.5 to pass. + For pairwise mode: win_rate >= 0.5, or all-ties, to pass. + """ + return 0.5 diff --git a/abevalflow/report.py b/abevalflow/report.py index 38cd3aa..8e30df1 100644 --- a/abevalflow/report.py +++ b/abevalflow/report.py @@ -9,6 +9,7 @@ from datetime import UTC, datetime from enum import StrEnum +from typing import Any from pydantic import BaseModel, Field, computed_field @@ -174,6 +175,21 @@ class EdgeCaseResult(BaseModel): score: float | None = Field(default=None, description="Edge case score (0.0-1.0)") +class PairwiseComparisonResult(BaseModel): + """AEH pairwise LLM-judge comparison (from score.py pairwise).""" + + run_a: str = "" + run_b: str = "" + cases_compared: int = 0 + wins_a: int = 0 + wins_b: int = 0 + ties: int = 0 + errors: int = 0 + win_rate: float = 0.0 + per_case: list[dict[str, Any]] = Field(default_factory=list) + stability: dict[str, Any] | None = None + + class AnalysisResult(BaseModel): """Top-level report model written to report.json.""" @@ -195,3 +211,7 @@ class AnalysisResult(BaseModel): default_factory=list, description="Results from edge case evaluations. Empty if no edge cases defined.", ) + pairwise: PairwiseComparisonResult | None = Field( + default=None, + description="AEH pairwise comparison results. Present for aeh-mode=pairwise.", + ) diff --git a/abevalflow/schemas.py b/abevalflow/schemas.py index 97eaedf..5820cf9 100644 --- a/abevalflow/schemas.py +++ b/abevalflow/schemas.py @@ -29,6 +29,7 @@ class EvalEngine(StrEnum): ASE = "ase" MCPCHECKER = "mcpchecker" A2A = "a2a" + AEH = "aeh" # Agent-Eval-Harness BOTH = "both" # Harbor + ASE diff --git a/containers/agent-eval-harness/Containerfile b/containers/agent-eval-harness/Containerfile new file mode 100644 index 0000000..93ff308 --- /dev/null +++ b/containers/agent-eval-harness/Containerfile @@ -0,0 +1,102 @@ +# Agent-Eval-Harness (AEH) runner image for ABEvalFlow +# +# This image contains the agent-eval-harness framework pre-installed, +# along with Harbor and necessary dependencies for running evaluations +# in a Kubernetes/OpenShift environment. +# +# Build (from repo root — COPY abevalflow requires repo-root context): +# podman build -t quay.io/ecosystem-appeng/agent-eval-harness:v1.0.3 \ +# --build-arg AEH_SHA= \ +# -f containers/agent-eval-harness/Containerfile . +# +# Required build args (PIN BEFORE BUILDING): +# AEH_SHA: Commit SHA of agent-eval-harness repo (7-40 hex chars; no branch/HEAD) +# +# Published smoke-tested image: quay.io/ecosystem-appeng/agent-eval-harness:v1.0.3 +# Rebuilds from this Containerfile must pin AEH_SHA to the same upstream commit +# used for that tag (inspect image labels / build logs) — do not leave the +# REPLACE_WITH_COMMIT_SHA placeholder. +# +# Note: Uses official Harbor from PyPI (>=0.13.0), not the skills_eval_corrections fork. +# The fork's OpenShift backend is only needed for Harbor engine evaluations, not AEH. + +FROM registry.access.redhat.com/ubi9/python-312:latest + +# Build args - MUST be pinned to specific commits for reproducibility +# Build will FAIL if SHA is not provided (no default branch names allowed) +# Example: --build-arg AEH_SHA=abc123def456 +ARG AEH_REPO=https://github.com/opendatahub-io/agent-eval-harness.git +ARG AEH_SHA=REPLACE_WITH_COMMIT_SHA + +LABEL name="agent-eval-harness" \ + vendor="Red Hat Ecosystem App Engineering" \ + description="Agent-Eval-Harness runner for ABEvalFlow pipeline" \ + io.k8s.display-name="AEH Runner" \ + io.openshift.tags="abevalflow,evaluation,agent-eval-harness" + +USER 0 + +# System dependencies +# Node.js + npm for claude-code agent CLI +# git for cloning repos at build time +# tar for layer operations +RUN dnf install -y nodejs npm git tar && dnf clean all + +# Agent CLIs (Claude Code) +RUN npm install -g @anthropic-ai/claude-code + +# Validate SHA build arg - must be 7-40 char hex commit SHA +# Rejects HEAD, main, master, branch names, or placeholder defaults +RUN if ! echo "${AEH_SHA}" | grep -qE '^[0-9a-fA-F]{7,40}$'; then \ + echo "ERROR: AEH_SHA must be a 7-40 char hex commit SHA, got '${AEH_SHA}'" >&2; \ + echo "Usage: --build-arg AEH_SHA=abc123def456..." >&2; \ + exit 1; \ + fi + +# Clone agent-eval-harness at pinned SHA +# Fetch main branch then checkout specific commit (can't shallow-fetch a SHA directly) +RUN git init /opt/agent-eval-harness \ + && cd /opt/agent-eval-harness \ + && git remote add origin ${AEH_REPO} \ + && git fetch origin main \ + && git checkout ${AEH_SHA} \ + && rm -rf .git + +# Copy abevalflow module for OpenShift environment extensions +# This provides OpenShiftEnvironment class that adds emptyDir mounts for /workspace and /tmp +# Required by trial pods to work with OpenShift's read-only root filesystem (restricted-v2 SCC) +COPY --chown=1001:0 abevalflow /opt/agent-eval-harness/abevalflow + +# Python dependencies +# - Harbor (official PyPI package >=0.13.0 required by agent-eval-harness) +# - Kubernetes client for K8s environment +# - YAML/JSON handling +# - Anthropic SDK with Vertex AI support +# - Jinja2 for template rendering +# - SciPy for statistical functions +RUN pip install --no-cache-dir \ + "harbor>=0.13.0" \ + "kubernetes>=32.0.0" \ + pyyaml \ + "anthropic[vertex]" \ + jinja2 \ + scipy + +# Set up environment +ENV PYTHONPATH=/opt/agent-eval-harness \ + HOME=/workspace + +# OpenShift-compatible permissions +# - /workspace: Working directory for evaluations +# - /logs: Harbor log output +# - /tests: Test artifacts +# - /solution: Agent solution artifacts +# - /opt/agent-eval-harness: AEH package (needs read access) +RUN mkdir -p /workspace /logs /tests /solution \ + && chgrp -R 0 /workspace /logs /tests /solution /opt/agent-eval-harness \ + && chmod -R g=u /workspace /logs /tests /solution /opt/agent-eval-harness + +WORKDIR /workspace + +# Run as non-root (OpenShift assigns arbitrary UID in restricted-v2 SCC) +USER 1001 diff --git a/scripts/aggregate_aeh.py b/scripts/aggregate_aeh.py new file mode 100644 index 0000000..c7ea3ec --- /dev/null +++ b/scripts/aggregate_aeh.py @@ -0,0 +1,526 @@ +#!/usr/bin/env python3 +"""Map AEH output to ABEvalFlow report format. + +Reads agent-eval-harness output files (summary.yaml, run_result.json) and +produces a unified report.json compatible with ABEvalFlow's scorecard logic. + +Supports both single-run and pairwise modes: + - Single: One run directory + - Pairwise: Treatment and control directories with pairwise comparison results + +Usage: + # Single mode (default) + python scripts/aggregate_aeh.py [--output ] + + # Pairwise mode + python scripts/aggregate_aeh.py --mode pairwise --control-dir + +Where is the AEH output directory containing: + - summary.yaml: Per-judge means, per-case results, run metadata, pairwise results + - run_result.json: Execution metadata (duration, cost, tokens) +""" + +import argparse +import json +import logging +import sys +from pathlib import Path +from typing import Any + +import yaml + +from abevalflow.aeh_scoring import ( + DEFAULT_AEH_THRESHOLD, + numeric_judge_passes, + pairwise_outcome, +) + +logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") +logger = logging.getLogger(__name__) + + +def _load_execution_metadata(run_dir: Path) -> dict[str, Any]: + """Load execution metadata from run_result.json.""" + run_result_path = run_dir / "run_result.json" + if not run_result_path.exists(): + return {} + + try: + run_result = json.loads(run_result_path.read_text()) + return { + "duration_s": run_result.get("duration_s"), + "cost_usd": run_result.get("cost_usd"), + "tokens": run_result.get("token_usage"), + "harbor_job_dir": run_result.get("harbor_job_dir"), + "num_turns": run_result.get("num_turns"), + "n_infra_errors": run_result.get("n_infra_errors"), + "n_trial_errors": run_result.get("n_trial_errors"), + } + except (json.JSONDecodeError, OSError) as e: + logger.warning("Failed to load run_result.json: %s", e) + return {} + + +def _extract_mean_reward(run_dir: Path) -> float: + """Extract mean_reward from run_result.json or summary.yaml.""" + run_result_path = run_dir / "run_result.json" + summary_path = run_dir / "summary.yaml" + + mean_reward = 0.0 + + if summary_path.exists(): + try: + summary = yaml.safe_load(summary_path.read_text()) + mean_reward = summary.get("mean_reward", 0.0) + except yaml.YAMLError: + pass + + if run_result_path.exists(): + try: + run_result = json.loads(run_result_path.read_text()) + if "mean_reward" in run_result: + mean_reward = run_result["mean_reward"] + except json.JSONDecodeError: + pass + + return mean_reward + + +def _case_reward(case_data: Any) -> float | None: + """Derive a single reward from an AEH per_case entry. + + Prefers the mean of numeric judge values; falls back to 1.0/0.0 from + boolean judge passes. Returns None when no parseable judge value exists. + """ + if not isinstance(case_data, dict): + return None + + if isinstance(case_data.get("reward"), (int, float)): + return float(case_data["reward"]) + + numeric: list[float] = [] + bools: list[bool] = [] + for key, result in case_data.items(): + if key == "reward": + continue + if isinstance(result, dict) and "value" in result: + value = result.get("value") + else: + value = result + if isinstance(value, bool): + bools.append(value) + elif isinstance(value, (int, float)): + numeric.append(float(value)) + + if numeric: + return sum(numeric) / len(numeric) + if bools: + return 1.0 if any(bools) else 0.0 + return None + + +def _trials_from_per_case(per_case: Any) -> list[dict[str, Any]]: + """Map AEH per_case dict → TrialResult-shaped list for report.json.""" + if not isinstance(per_case, dict): + return [] + trials: list[dict[str, Any]] = [] + for case_id, case_data in per_case.items(): + trials.append({"trial_name": str(case_id), "reward": _case_reward(case_data)}) + return trials + + +def aggregate_single_run( + run_dir: Path, + *, + submission_name: str | None = None, + threshold: float = DEFAULT_AEH_THRESHOLD, +) -> dict[str, Any]: + """Read one AEH harness run directory and produce report dict. + + AEH output layout (from harbor.run): + /summary.yaml # run_id, judges, per_case, run_metrics + /run_result.json # duration, cost, tokens, mean_reward + /cases/... # per-case artifacts + + Args: + run_dir: Path to the AEH run output directory + submission_name: Override for report submission_name (Tekton param) + threshold: Pass/fail threshold for mean_reward (matches GatePolicy default) + + Returns: + Dict in ABEvalFlow report format with full judge metadata + """ + summary_path = run_dir / "summary.yaml" + + if not summary_path.exists(): + raise FileNotFoundError(f"summary.yaml not found in {run_dir}") + + summary = yaml.safe_load(summary_path.read_text()) + mean_reward = _extract_mean_reward(run_dir) + + # Preserve full judge structure (not just means) + judges_full = summary.get("judges", {}) + per_case_full = summary.get("per_case", {}) + run_metrics = summary.get("run_metrics") + + # Calculate pass rate from per_case data + total_cases = len(per_case_full) + passed_cases = 0 + for case_id, case_data in per_case_full.items(): + if isinstance(case_data, dict): + # Check if any judge reported a passing value + case_passed = False + for judge_name, judge_result in case_data.items(): + if isinstance(judge_result, dict): + value = judge_result.get("value") + if isinstance(value, bool) and value: + case_passed = True + break + if isinstance(value, (int, float)) and not isinstance(value, bool): + if numeric_judge_passes(value): + case_passed = True + break + if case_passed: + passed_cases += 1 + + pass_rate = passed_cases / total_cases if total_cases > 0 else 0.0 + recommendation = "pass" if (mean_reward is not None and mean_reward >= threshold) else "fail" + resolved_name = submission_name or (run_dir.parent.name if run_dir.parent != run_dir else run_dir.name) + mean_for_gap = mean_reward if mean_reward is not None else 0.0 + + # AnalysisResult-compatible shape (analyze task + scorecard) plus AEH extras. + return { + "submission_name": resolved_name, + "provenance": { + "eval_engine": "aeh", + "pipeline_run_id": summary.get("run_id", run_dir.name), + }, + "summary": { + "treatment": { + "n_trials": total_cases, + "n_passed": passed_cases, + "n_failed": max(total_cases - passed_cases, 0), + "pass_rate": pass_rate, + "mean_reward": mean_reward, + }, + "control": { + "n_trials": 0, + "n_passed": 0, + "n_failed": 0, + "pass_rate": 0.0, + "mean_reward": 0.0, + }, + "uplift": pass_rate, + "mean_reward_gap": mean_for_gap, + "recommendation": recommendation, + }, + "trials": { + "treatment": _trials_from_per_case(per_case_full), + "control": [], + }, + "eval_engine": "aeh", + "mode": "single", + "run_id": summary.get("run_id", run_dir.name), + "mean_reward": mean_reward, + "pass_rate": pass_rate, + "total_cases": total_cases, + "passed_cases": passed_cases, + "judges": judges_full, + "per_case": per_case_full, + "run_metrics": run_metrics, + "execution": _load_execution_metadata(run_dir), + "aeh_warnings": [], + "recommendation": recommendation, + } + + +def aggregate_pairwise_run( + treatment_dir: Path, + control_dir: Path, + *, + submission_name: str | None = None, + threshold: float = DEFAULT_AEH_THRESHOLD, +) -> dict[str, Any]: + """Read pairwise AEH run directories and produce report dict. + + Pairwise mode expects: + - Two run directories (treatment and control) + - Treatment summary.yaml contains `pairwise` section from score.py pairwise + + Args: + treatment_dir: Path to the treatment (A) run directory + control_dir: Path to the control (B) run directory + submission_name: Override for report submission_name (Tekton param) + threshold: Pass/fail win-rate threshold (matches GatePolicy / engine) + + Returns: + Dict in ABEvalFlow report format with pairwise results + """ + treatment_summary_path = treatment_dir / "summary.yaml" + control_summary_path = control_dir / "summary.yaml" + + if not treatment_summary_path.exists(): + raise FileNotFoundError(f"summary.yaml not found in {treatment_dir}") + + treatment_summary = yaml.safe_load(treatment_summary_path.read_text()) + treatment_mean_reward = _extract_mean_reward(treatment_dir) + + control_summary = {} + control_mean_reward = 0.0 + if control_summary_path.exists(): + control_summary = yaml.safe_load(control_summary_path.read_text()) + control_mean_reward = _extract_mean_reward(control_dir) + + # Extract pairwise results from treatment summary + pairwise = treatment_summary.get("pairwise", {}) + outcome = pairwise_outcome( + pairwise.get("wins_a", 0), + pairwise.get("wins_b", 0), + pairwise.get("ties", 0), + pairwise.get("errors", 0), + threshold=threshold, + ) + wins_a = outcome["wins_a"] + wins_b = outcome["wins_b"] + ties = outcome["ties"] + errors = outcome["errors"] + total = outcome["total"] + win_rate = outcome["win_rate"] + recommendation = outcome["recommendation"] + cases_compared = pairwise.get("cases_compared", total) + + # Preserve full judge and per_case structures + treatment_judges = treatment_summary.get("judges", {}) + treatment_per_case = treatment_summary.get("per_case", {}) + control_judges = control_summary.get("judges", {}) + control_per_case = control_summary.get("per_case", {}) + + resolved_name = submission_name or ( + treatment_dir.parent.name if treatment_dir.parent != treatment_dir else treatment_dir.name + ) + t_mean = treatment_mean_reward if treatment_mean_reward is not None else 0.0 + c_mean = control_mean_reward if control_mean_reward is not None else 0.0 + treatment_cases = len(treatment_per_case) if isinstance(treatment_per_case, dict) else 0 + control_cases = len(control_per_case) if isinstance(control_per_case, dict) else 0 + + return { + "submission_name": resolved_name, + "provenance": { + "eval_engine": "aeh", + "pipeline_run_id": treatment_summary.get("run_id", treatment_dir.name), + }, + "summary": { + "treatment": { + "n_trials": treatment_cases, + "n_passed": wins_a, + "n_failed": max(treatment_cases - wins_a, 0), + "pass_rate": win_rate, + "mean_reward": treatment_mean_reward, + }, + "control": { + "n_trials": control_cases, + "n_passed": wins_b, + "n_failed": max(control_cases - wins_b, 0), + "pass_rate": (wins_b / total) if total > 0 else 0.0, + "mean_reward": control_mean_reward, + }, + "uplift": win_rate - ((wins_b / total) if total > 0 else 0.0), + "mean_reward_gap": t_mean - c_mean, + "recommendation": recommendation, + }, + "trials": { + "treatment": _trials_from_per_case(treatment_per_case), + "control": _trials_from_per_case(control_per_case), + }, + "eval_engine": "aeh", + "mode": "pairwise", + "treatment": { + "run_id": treatment_summary.get("run_id", treatment_dir.name), + "mean_reward": treatment_mean_reward, + "judges": treatment_judges, + "per_case": treatment_per_case, + "run_metrics": treatment_summary.get("run_metrics"), + "execution": _load_execution_metadata(treatment_dir), + }, + "control": { + "run_id": control_summary.get("run_id", control_dir.name), + "mean_reward": control_mean_reward, + "judges": control_judges, + "per_case": control_per_case, + "run_metrics": control_summary.get("run_metrics"), + "execution": _load_execution_metadata(control_dir), + }, + "pairwise": { + "run_a": pairwise.get("run_a", treatment_dir.name), + "run_b": pairwise.get("run_b", control_dir.name), + "cases_compared": cases_compared, + "wins_a": wins_a, + "wins_b": wins_b, + "ties": ties, + "errors": errors, + "win_rate": win_rate, + "per_case": pairwise.get("per_case", []), + "stability": pairwise.get("stability"), + }, + "mean_reward": treatment_mean_reward, + "aeh_warnings": (["pairwise section missing from treatment summary.yaml"] if not pairwise else []), + "recommendation": recommendation, + } + + +def aggregate_aeh_results( + run_dir: Path, + mode: str = "single", + control_dir: Path | None = None, + *, + submission_name: str | None = None, + threshold: float = DEFAULT_AEH_THRESHOLD, +) -> dict[str, Any]: + """Aggregate AEH results into ABEvalFlow report format. + + Args: + run_dir: Path to the AEH run output directory (treatment in pairwise mode) + mode: Either "single" or "pairwise" + control_dir: Path to control directory (required for pairwise mode) + submission_name: Override for report submission_name + threshold: Pass/fail threshold aligned with AEHEngine / GatePolicy default + + Returns: + Dict in ABEvalFlow report format + """ + if mode == "pairwise": + if control_dir is None: + raise ValueError("control_dir is required for pairwise mode") + return aggregate_pairwise_run( + run_dir, + control_dir, + submission_name=submission_name, + threshold=threshold, + ) + return aggregate_single_run( + run_dir, + submission_name=submission_name, + threshold=threshold, + ) + + +def find_latest_run_dir(reports_dir: Path, submission_name: str) -> Path | None: + """Find the most recent run directory for a submission. + + Layout: reports///summary.yaml + + Args: + reports_dir: Root reports directory + submission_name: Name of the submission + + Returns: + Path to the latest run dir, or None if not found + """ + submission_dir = reports_dir / submission_name + if not submission_dir.exists(): + return None + + run_dirs = [d for d in submission_dir.iterdir() if d.is_dir() and (d / "summary.yaml").exists()] + if not run_dirs: + return None + + return sorted(run_dirs, key=lambda d: d.stat().st_mtime, reverse=True)[0] + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Aggregate AEH results into ABEvalFlow report format") + parser.add_argument( + "run_dir", + type=Path, + help="Path to AEH run directory containing summary.yaml (treatment dir in pairwise mode)", + ) + parser.add_argument( + "--output", + "-o", + type=Path, + default=None, + help="Output path for report.json (default: /report.json)", + ) + parser.add_argument( + "--mode", + type=str, + choices=["single", "pairwise"], + default="single", + help="Aggregation mode (single or pairwise)", + ) + parser.add_argument( + "--control-dir", + type=Path, + default=None, + help="Path to control run directory (required for pairwise mode)", + ) + parser.add_argument( + "--submission-name", + type=str, + default=None, + help="Override submission_name in report.json (Tekton submission-name)", + ) + parser.add_argument( + "--threshold", + type=float, + default=DEFAULT_AEH_THRESHOLD, + help=f"Pass/fail threshold (default: {DEFAULT_AEH_THRESHOLD})", + ) + args = parser.parse_args(argv) + + run_dir: Path = args.run_dir + if not run_dir.is_dir(): + logger.error("Not a directory: %s", run_dir) + return 1 + + if args.mode == "pairwise" and args.control_dir is None: + logger.error("--control-dir is required for pairwise mode") + return 1 + + if args.control_dir and not args.control_dir.is_dir(): + logger.error("Control directory not found: %s", args.control_dir) + return 1 + + try: + report = aggregate_aeh_results( + run_dir, + mode=args.mode, + control_dir=args.control_dir, + submission_name=args.submission_name, + threshold=args.threshold, + ) + except FileNotFoundError as e: + logger.error(str(e)) + return 1 + + output_path = args.output or (run_dir / "report.json") + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(json.dumps(report, indent=2)) + logger.info("Wrote report to: %s", output_path) + + if args.mode == "pairwise": + pairwise = report.get("pairwise", {}) + logger.info( + "Pairwise: treatment wins %d/%d (%.0f%%), ties=%d, errors=%d", + pairwise.get("wins_a", 0), + pairwise.get("cases_compared", 0), + pairwise.get("win_rate", 0) * 100, + pairwise.get("ties", 0), + pairwise.get("errors", 0), + ) + else: + mean_reward = report["mean_reward"] + mean_reward_str = f"{mean_reward:.3f}" if mean_reward is not None else "None" + logger.info( + "Summary: mean_reward=%s, pass_rate=%.2f (%d/%d cases)", + mean_reward_str, + report.get("pass_rate", 0), + report.get("passed_cases", 0), + report.get("total_cases", 0), + ) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/run_aeh.py b/scripts/run_aeh.py new file mode 100644 index 0000000..4fb5460 --- /dev/null +++ b/scripts/run_aeh.py @@ -0,0 +1,740 @@ +#!/usr/bin/env python3 +"""Unified AEH runner - abstracts execution backend (harbor, vanilla, future). + +This dispatcher provides a consistent interface for running AEH evaluations +regardless of the underlying execution mode. The pairwise comparison logic +is shared across all runners since score.py is execution-mode agnostic. + +Usage: + # Single run + python run_aeh.py single --runner harbor --config eval.yaml --output /path/to/output + + # Pairwise run + python run_aeh.py pairwise --runner harbor \ + --control-config eval-control.yaml \ + --treatment-config eval-treatment.yaml \ + --output /path/to/output + +Environment variables (for kubernetes/harbor mode): + AGENT_EVAL_K8S_CREDENTIALS_SECRET: Secret name for LLM credentials + AGENT_EVAL_RUNS_DIR: Base directory for run outputs +""" + +from __future__ import annotations + +import argparse +import os +import shutil +import subprocess +import sys +from abc import ABC, abstractmethod +from pathlib import Path +from typing import Any + +import yaml + +from abevalflow.aeh_scoring import OPENSHIFT_ENVIRONMENT_IMPORT_PATH + + +class RunnerError(Exception): + """Raised when a runner encounters an error.""" + + +def _output_paths_from_config(config: Path) -> list[str]: + """Read outputs[].path from eval.yaml (default: ['output']).""" + try: + raw = yaml.safe_load(config.read_text()) or {} + except (OSError, yaml.YAMLError): + return ["output"] + paths: list[str] = [] + for out in raw.get("outputs") or []: + if isinstance(out, dict) and out.get("path"): + paths.append(str(out["path"]).strip().strip("/")) + elif isinstance(out, str) and out.strip(): + paths.append(out.strip().strip("/")) + return paths or ["output"] + + +def _case_id_from_trial_dir(trial_dir: Path) -> str: + """Harbor trial dirs are often case-001__AbCdEf; score.py wants case-001.""" + name = trial_dir.name + return name.split("__", 1)[0] if "__" in name else name + + +def _iter_harbor_trial_dirs(jobs_dir: Path | None) -> list[Path]: + """Return trial directories from the newest Harbor job under jobs_dir.""" + if jobs_dir is None or not jobs_dir.is_dir(): + return [] + job_dirs = sorted( + (d for d in jobs_dir.iterdir() if d.is_dir()), + key=lambda d: d.stat().st_mtime, + ) + if not job_dirs: + return [] + job = job_dirs[-1] + trials: list[Path] = [] + for child in job.iterdir(): + if not child.is_dir(): + continue + if (child / "verifier").is_dir() or (child / "result.json").is_file(): + trials.append(child) + return trials + + +def materialize_aeh_case_outputs( + config: Path, + output_dir: Path, + jobs_dir: Path | None = None, +) -> int: + """Copy Harbor verifier/ into output_dir/cases//. + + AEH score.py pairwise expects ``cases///…`` under the + run directory. Upstream Harbor only mirrors ``verifier/artifacts``, while + generated test.sh writes ``verifier/``. + + Returns: + Number of distinct case directories that received at least one copy. + """ + output_paths = _output_paths_from_config(config) + materialized: set[str] = set() + + for trial in _iter_harbor_trial_dirs(jobs_dir): + case_id = _case_id_from_trial_dir(trial) + copied_any = False + for out_path in output_paths: + src = trial / "verifier" / out_path + if not src.is_dir(): + # Fall back to verifier/artifacts when present + art = trial / "verifier" / "artifacts" + if art.is_dir() and any(art.iterdir()): + src = art + else: + continue + dst = output_dir / "cases" / case_id / out_path + if dst.exists(): + shutil.rmtree(dst) + dst.parent.mkdir(parents=True, exist_ok=True) + shutil.copytree(src, dst) + copied_any = True + if copied_any: + materialized.add(case_id) + + n = len(materialized) + if n: + print(f"Materialized case outputs for {n} case(s) under {output_dir / 'cases'}") + else: + print( + f"WARN: no verifier/{'/'.join(output_paths)} (or artifacts) found under " + f"jobs_dir={jobs_dir}; score.py pairwise will fail without cases/" + ) + return n + + +class BaseRunner(ABC): + """Base class for AEH execution backends. + + All runners share the same pairwise comparison logic since score.py + is execution-mode agnostic - it only cares about output directories. + """ + + name: str = "base" + + def __init__( + self, + model: str | None = None, + judge_model: str | None = None, + image: str | None = None, + env_type: str = "kubernetes", + ): + self.model = model + self.judge_model = judge_model + self.image = image + self.env_type = env_type + + def run_single( + self, + config: Path, + output: Path, + run_id: str | None = None, + **opts: Any, + ) -> int: + """Run a single evaluation. + + Args: + config: Path to eval.yaml + output: Output directory for results + run_id: Optional run identifier + **opts: Additional runner-specific options + + Returns: + Exit code (0 = success, 1 = threshold warning, other = error) + """ + output.mkdir(parents=True, exist_ok=True) + return self._execute(config, output, run_id=run_id, **opts) + + def run_pairwise( + self, + control_config: Path, + treatment_config: Path, + output_base: Path, + run_id: str, + score_py_path: Path | None = None, + judge: str | None = None, + tasks_dir: Path | None = None, + jobs_dir: Path | None = None, + **opts: Any, + ) -> dict[str, Any]: + """Run pairwise A/B comparison. + + Executes control and treatment runs, then runs pairwise comparison. + The pairwise step is SHARED across all runners. + + Args: + control_config: Path to control variant eval.yaml + treatment_config: Path to treatment variant eval.yaml + output_base: Base output directory + run_id: Run identifier (used for directory naming) + score_py_path: Path to score.py (auto-detected if None) + judge: Judge name to use for pairwise comparison + tasks_dir: Base tasks directory (separate control/treatment subdirs created) + jobs_dir: Base jobs directory (separate control/treatment subdirs created) + **opts: Additional runner-specific options + + Returns: + Dictionary with control_dir, treatment_dir, exit_codes, pairwise_exit + """ + # Read skill name from treatment config for consistent directory naming + skill_name = self._read_skill_name(treatment_config) + + control_run_id = f"control-{run_id}" + treatment_run_id = f"treatment-{run_id}" + + control_dir = output_base / skill_name / control_run_id + treatment_dir = output_base / skill_name / treatment_run_id + + # Create separate tasks/jobs dirs for control and treatment to avoid conflicts + # Harbor requires these dirs and they must be distinct + base_tmp = tasks_dir.parent if tasks_dir else output_base.parent / "_eval_tmp" + control_tasks_dir = base_tmp / f"aeh-control-tasks-{run_id}" + control_jobs_dir = base_tmp / f"aeh-control-jobs-{run_id}" + treatment_tasks_dir = base_tmp / f"aeh-treatment-tasks-{run_id}" + treatment_jobs_dir = base_tmp / f"aeh-treatment-jobs-{run_id}" + + for d in [control_tasks_dir, control_jobs_dir, treatment_tasks_dir, treatment_jobs_dir]: + d.mkdir(parents=True, exist_ok=True) + + # 1. Execute control (mode-specific) + print(f"=== Running control: {control_config} -> {control_dir}") + control_exit = self._execute( + control_config, + control_dir, + run_id=control_run_id, + tasks_dir=control_tasks_dir, + jobs_dir=control_jobs_dir, + **opts, + ) + + # Fail-fast if control didn't produce summary.yaml + if not (control_dir / "summary.yaml").exists(): + raise RunnerError(f"Control run failed - no summary.yaml in {control_dir}") + + # 2. Execute treatment (mode-specific) + print(f"=== Running treatment: {treatment_config} -> {treatment_dir}") + treatment_exit = self._execute( + treatment_config, + treatment_dir, + run_id=treatment_run_id, + tasks_dir=treatment_tasks_dir, + jobs_dir=treatment_jobs_dir, + **opts, + ) + + # Fail-fast if treatment didn't produce summary.yaml + if not (treatment_dir / "summary.yaml").exists(): + raise RunnerError(f"Treatment run failed - no summary.yaml in {treatment_dir}") + + # 3. Pairwise comparison (SHARED - same for all runners) + # Set AGENT_EVAL_RUNS_DIR so score.py can find the run directories + # score.py resolves: $AGENT_EVAL_RUNS_DIR// + print("=== Running pairwise comparison") + pairwise_exit = self._run_pairwise_comparison( + control_run_id=control_run_id, + treatment_run_id=treatment_run_id, + treatment_config=treatment_config, + output_base=output_base, + score_py_path=score_py_path, + judge=judge, + ) + + # Harbor writes report.html before pairwise; regenerate so HTML includes + # the pairwise: section merged into treatment summary.yaml. + if pairwise_exit == 0: + self._regenerate_report_with_baseline( + treatment_run_id=treatment_run_id, + control_run_id=control_run_id, + treatment_config=treatment_config, + output_base=output_base, + score_py_path=score_py_path, + ) + + return { + "control_dir": str(control_dir), + "treatment_dir": str(treatment_dir), + "control_exit": control_exit, + "treatment_exit": treatment_exit, + "pairwise_exit": pairwise_exit, + "skill_name": skill_name, + "control_run_id": control_run_id, + "treatment_run_id": treatment_run_id, + } + + def _run_pairwise_comparison( + self, + control_run_id: str, + treatment_run_id: str, + treatment_config: Path, + output_base: Path, + score_py_path: Path | None = None, + judge: str | None = None, + ) -> int: + """Run score.py pairwise - shared across all runners. + + Args: + control_run_id: Control run identifier + treatment_run_id: Treatment run identifier + treatment_config: Path to treatment config (contains judge definitions) + output_base: Base output directory (set as AGENT_EVAL_RUNS_DIR) + score_py_path: Path to score.py (auto-detected if None) + judge: Judge name to use (default: "pairwise") + + Returns: + Exit code from score.py pairwise + """ + if score_py_path is None: + # Try common locations + candidates = [ + Path("/opt/agent-eval-harness/skills/eval-run/scripts/score.py"), + Path("skills/eval-run/scripts/score.py"), + Path(os.environ.get("AEH_SCORE_PY", "")), + ] + for candidate in candidates: + if candidate.exists(): + score_py_path = candidate + break + else: + raise RunnerError("score.py not found. Set AEH_SCORE_PY or provide --score-py-path") + + cmd = [ + sys.executable, + str(score_py_path), + "pairwise", + "--run-id", + treatment_run_id, + "--baseline", + control_run_id, + "--config", + str(treatment_config), + ] + + if judge: + cmd.extend(["--judge", judge]) + + # Set AGENT_EVAL_RUNS_DIR so score.py can resolve run directories + # score.py looks for: $AGENT_EVAL_RUNS_DIR// + env = os.environ.copy() + env["AGENT_EVAL_RUNS_DIR"] = str(output_base) + + print(f"Running: {' '.join(cmd)}") + print(f" AGENT_EVAL_RUNS_DIR={output_base}") + result = subprocess.run(cmd, env=env) + return result.returncode + + def _find_aeh_script(self, name: str, score_py_path: Path | None = None) -> Path | None: + """Locate an AEH eval-run script (score.py / report.py) beside score.py.""" + candidates: list[Path] = [] + if score_py_path is not None: + candidates.append(score_py_path.parent / name) + candidates.extend( + [ + Path(f"/opt/agent-eval-harness/skills/eval-run/scripts/{name}"), + Path(f"skills/eval-run/scripts/{name}"), + ] + ) + env_score = os.environ.get("AEH_SCORE_PY", "") + if env_score: + candidates.append(Path(env_score).parent / name) + for candidate in candidates: + if candidate.is_file(): + return candidate + return None + + def _regenerate_report_with_baseline( + self, + treatment_run_id: str, + control_run_id: str, + treatment_config: Path, + output_base: Path, + score_py_path: Path | None = None, + ) -> None: + """Rewrite treatment report.html including pairwise vs control baseline. + + Best-effort: log and continue if report.py is missing or fails. + """ + report_py = self._find_aeh_script("report.py", score_py_path=score_py_path) + if report_py is None: + print("WARNING: report.py not found — skipping pairwise HTML regenerate") + return + + cmd = [ + sys.executable, + str(report_py), + "--run-id", + treatment_run_id, + "--baseline", + control_run_id, + "--config", + str(treatment_config), + ] + env = os.environ.copy() + env["AGENT_EVAL_RUNS_DIR"] = str(output_base) + + print("=== Regenerating treatment report.html with pairwise baseline") + print(f"Running: {' '.join(cmd)}") + result = subprocess.run(cmd, env=env) + if result.returncode != 0: + print(f"WARNING: report.py exited {result.returncode} — treatment report.html may lack pairwise section") + + def _read_skill_name(self, config_path: Path) -> str: + """Read skill name from eval.yaml config.""" + try: + import yaml + + config = yaml.safe_load(config_path.read_text()) + return config.get("skill", config_path.stem) + except Exception: + return config_path.stem + + @abstractmethod + def _execute( + self, + config: Path, + output: Path, + run_id: str | None = None, + **opts: Any, + ) -> int: + """Execute evaluation - mode-specific implementation. + + Args: + config: Path to eval.yaml + output: Output directory + run_id: Optional run identifier + **opts: Additional options + + Returns: + Exit code + """ + ... + + +class HarborRunner(BaseRunner): + """Harbor/Kubernetes execution backend. + + Uses python -m agent_eval.harbor.run for containerized execution. + """ + + name = "harbor" + + def _execute( + self, + config: Path, + output: Path, + run_id: str | None = None, + tasks_dir: Path | None = None, + jobs_dir: Path | None = None, + **opts: Any, + ) -> int: + """Execute via harbor.run.""" + # Harbor requires --model; fail fast with clear message + if not self.model: + raise RunnerError("Harbor runner requires --model. Pass --model explicitly via CLI or pipeline parameter.") + + # For kubernetes environment, patch eval.yaml to use OpenShiftEnvironment + # This adds emptyDir mounts for /workspace and /tmp required by OpenShift + patched_config = config + use_patched_config = False + if self.env_type == "kubernetes": + patched_config = self._patch_eval_config_for_openshift(config, tasks_dir) + use_patched_config = True + + # Pre-generate + enrich task packages so Harbor skips bare upstream + # generation (missing skills/ + annotations wiring in AEH v1.0.3). + if tasks_dir is not None and self.image: + self._prepare_enriched_tasks(patched_config, Path(tasks_dir)) + + cmd = [ + sys.executable, + "-m", + "agent_eval.harbor.run", + "--config", + str(patched_config), + "--output", + str(output), + ] + + # For patched config (kubernetes mode), pass --environment-import-path explicitly + # to override agent_eval.harbor.run's default environment mapping + if use_patched_config: + cmd.extend(["--environment-import-path", OPENSHIFT_ENVIRONMENT_IMPORT_PATH]) + else: + # For other modes, use --env to select environment + cmd.extend(["--env", self.env_type]) + + if self.model: + cmd.extend(["--model", self.model]) + if self.judge_model: + cmd.extend(["--judge-model", self.judge_model]) + if self.image: + cmd.extend(["--image", self.image]) + if tasks_dir: + cmd.extend(["--tasks-dir", str(tasks_dir)]) + if jobs_dir: + cmd.extend(["--jobs-dir", str(jobs_dir)]) + + print(f"Running: {' '.join(cmd)}") + result = subprocess.run(cmd) + + # Bridge Harbor verifier outputs into the cases/ layout score.py expects. + # Use the original config for outputs[].path (patched config is equivalent). + try: + materialize_aeh_case_outputs( + Path(config), + Path(output), + Path(jobs_dir) if jobs_dir else None, + ) + except Exception as exc: # noqa: BLE001 — best-effort bridge; don't hide harbor rc + print(f"WARN: failed to materialize AEH case outputs: {exc}") + + return result.returncode + + def _prepare_enriched_tasks(self, config: Path, tasks_dir: Path) -> None: + """Generate Harbor tasks then apply ABEvalFlow skill/annotation fixes.""" + import shutil + + from agent_eval.config import EvalConfig + from agent_eval.harbor import tasks as tasks_mod + + from abevalflow.harbor_extensions.aeh_task_enrichment import ( + enrich_harbor_tasks, + ) + + if tasks_dir.exists(): + for child in tasks_dir.iterdir(): + if child.is_dir(): + shutil.rmtree(child) + else: + child.unlink() + else: + tasks_dir.mkdir(parents=True, exist_ok=True) + + eval_config = EvalConfig.from_yaml(config) + print(f"Generating Harbor tasks into {tasks_dir}") + tasks_mod.generate_tasks( + eval_config, + Path(config), + tasks_dir, + self.image, + judge_model=self.judge_model, + ) + n = enrich_harbor_tasks(tasks_dir, config_path=Path(config)) + print(f"Enriched {n} Harbor task package(s) (skills + annotations)") + + def _patch_eval_config_for_openshift(self, config: Path, tasks_dir: Path | None) -> Path: + """Patch eval.yaml for OpenShift emptyDir support. + + Sets environment.type to a valid enum value (``kubernetes``). The custom + OpenShiftEnvironment class is applied via ``--environment-import-path``. + + Args: + config: Original eval.yaml path + tasks_dir: Tasks directory (not used - kept for backward compatibility) + + Returns: + Path to patched eval.yaml + """ + import yaml + + # Load original config + with open(config) as f: + eval_config = yaml.safe_load(f) + + # Keep a valid enum type in YAML; the custom emptyDir environment is + # selected via --environment-import-path (Harbor clears type when set). + if "environment" not in eval_config: + eval_config["environment"] = {} + + eval_config["environment"]["type"] = "kubernetes" + + # Write patched config to same directory as original to preserve relative paths + # This is critical - if we write to a different directory, relative paths in + # the config (like cases/, skills/, etc.) will break + patched_config = config.parent / f"{config.stem}-openshift.yaml" + with open(patched_config, "w") as f: + yaml.dump(eval_config, f) + + print(f"Patched eval config for OpenShift: {patched_config}") + return patched_config + + +class VanillaRunner(BaseRunner): + """Vanilla (direct/local) execution backend. + + NOT YET IMPLEMENTED. Agent-eval-harness does not currently have a + standalone CLI for local execution. The eval-run skill is designed + to be driven by an LLM agent, not invoked directly. + + This runner exists as a placeholder for future local execution support. + Use 'harbor' runner with --env podman for local containerized execution. + """ + + name = "vanilla" + + def _execute( + self, + config: Path, + output: Path, + run_id: str | None = None, + **opts: Any, + ) -> int: + """Execute via vanilla mode - NOT YET IMPLEMENTED.""" + raise RunnerError( + "Vanilla runner is not yet implemented. " + "Agent-eval-harness does not have a standalone local CLI. " + "Use 'harbor' runner instead:\n" + " - For Kubernetes: --runner harbor --env-type kubernetes\n" + " - For local containers: --runner harbor --env-type podman\n" + "\n" + "See: https://github.com/agent-eval-harness for AEH documentation." + ) + + +# Registry of available runners +RUNNERS: dict[str, type[BaseRunner]] = { + "harbor": HarborRunner, + "vanilla": VanillaRunner, +} + + +def get_runner(name: str, **kwargs: Any) -> BaseRunner: + """Get a runner instance by name. + + Args: + name: Runner name (harbor, vanilla, etc.) + **kwargs: Arguments passed to runner constructor + + Returns: + Runner instance + + Raises: + ValueError: If runner name is not recognized + """ + if name not in RUNNERS: + available = ", ".join(RUNNERS.keys()) + raise ValueError(f"Unknown runner '{name}'. Available: {available}") + return RUNNERS[name](**kwargs) + + +def main() -> int: + """CLI entrypoint.""" + parser = argparse.ArgumentParser(description="Unified AEH runner - abstracts execution backend") + subparsers = parser.add_subparsers(dest="command", required=True) + + # Common arguments + common = argparse.ArgumentParser(add_help=False) + common.add_argument( + "--runner", + choices=list(RUNNERS.keys()), + default="harbor", + help=("Execution backend (default: harbor). 'vanilla' is a placeholder and raises RunnerError if selected."), + ) + common.add_argument("--model", help="Model for skill execution") + common.add_argument("--judge-model", help="Model for LLM judges") + common.add_argument("--image", help="Container image for trial pods (harbor mode)") + common.add_argument( + "--env-type", + default="kubernetes", + help="Environment type for harbor mode (default: kubernetes)", + ) + + # Single run command + single = subparsers.add_parser("single", parents=[common], help="Run single evaluation") + single.add_argument("--config", required=True, help="Path to eval.yaml") + single.add_argument("--output", required=True, help="Output directory") + single.add_argument("--run-id", help="Run identifier") + single.add_argument("--tasks-dir", help="Tasks directory (harbor mode)") + single.add_argument("--jobs-dir", help="Jobs directory (harbor mode)") + + # Pairwise run command + pairwise = subparsers.add_parser("pairwise", parents=[common], help="Run pairwise A/B comparison") + pairwise.add_argument("--control-config", required=True, help="Path to control eval.yaml") + pairwise.add_argument("--treatment-config", required=True, help="Path to treatment eval.yaml") + pairwise.add_argument("--output", required=True, help="Base output directory") + pairwise.add_argument("--run-id", required=True, help="Run identifier") + pairwise.add_argument("--score-py-path", help="Path to score.py") + pairwise.add_argument("--judge", default="pairwise", help="Judge name for comparison") + pairwise.add_argument("--tasks-dir", help="Tasks directory (harbor mode)") + pairwise.add_argument("--jobs-dir", help="Jobs directory (harbor mode)") + + args = parser.parse_args() + + try: + runner = get_runner( + args.runner, + model=args.model, + judge_model=args.judge_model, + image=getattr(args, "image", None), + env_type=getattr(args, "env_type", "kubernetes"), + ) + + if args.command == "single": + exit_code = runner.run_single( + config=Path(args.config), + output=Path(args.output), + run_id=args.run_id, + tasks_dir=Path(args.tasks_dir) if args.tasks_dir else None, + jobs_dir=Path(args.jobs_dir) if args.jobs_dir else None, + ) + return exit_code + + elif args.command == "pairwise": + result = runner.run_pairwise( + control_config=Path(args.control_config), + treatment_config=Path(args.treatment_config), + output_base=Path(args.output), + run_id=args.run_id, + score_py_path=Path(args.score_py_path) if args.score_py_path else None, + judge=args.judge, + tasks_dir=Path(args.tasks_dir) if args.tasks_dir else None, + jobs_dir=Path(args.jobs_dir) if args.jobs_dir else None, + ) + + # Print results summary + print("\n=== Pairwise Results ===") + print(f"Control: {result['control_dir']} (exit: {result['control_exit']})") + print(f"Treatment: {result['treatment_dir']} (exit: {result['treatment_exit']})") + print(f"Pairwise exit: {result['pairwise_exit']}") + + # Return pairwise exit code (0 = pass, 1 = regression) + return result["pairwise_exit"] + + except RunnerError as e: + print(f"ERROR: {e}", file=sys.stderr) + return 1 + except Exception as e: + print(f"FATAL: {e}", file=sys.stderr) + return 2 + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/validate.py b/scripts/validate.py index a6f2add..541af3a 100644 --- a/scripts/validate.py +++ b/scripts/validate.py @@ -328,6 +328,278 @@ def _check_mcpchecker_tasks(submission_dir: Path) -> list[str]: return errors +# --------------------------------------------------------------------------- +# AEH-specific checks +# --------------------------------------------------------------------------- + + +def _check_aeh_eval_yaml_file(eval_path: Path) -> list[str]: + """Validate a single AEH eval.yaml file exists and has valid structure. + + AEH eval.yaml requires: + - models.skill (or models block) + - judges section (dict or list) + - Optional: thresholds, mlflow, etc. + """ + if not eval_path.is_file(): + return [f"{eval_path.name} is required for AEH evaluation"] + + try: + data = yaml.safe_load(eval_path.read_text()) + except yaml.YAMLError as exc: + return [f"{eval_path.name} is not valid YAML: {exc}"] + + if not isinstance(data, dict): + return [f"{eval_path.name} must be a YAML mapping"] + + errors: list[str] = [] + filename = eval_path.name + + models = data.get("models") + if not models: + errors.append(f"{filename}: 'models' section is required") + elif isinstance(models, dict): + if not models.get("skill"): + errors.append(f"{filename}: 'models.skill' is required (e.g., 'claude-sonnet-4-5')") + elif isinstance(models, str): + pass + else: + errors.append(f"{filename}: 'models' must be a mapping or string") + + if not data.get("judges"): + errors.append(f"{filename}: 'judges' section is required") + + return errors + + +def _check_aeh_eval_yaml(submission_dir: Path) -> list[str]: + """Validate eval.yaml exists and has valid AEH structure (single mode).""" + return _check_aeh_eval_yaml_file(submission_dir / "eval.yaml") + + +def _check_aeh_plugin_dirs(submission_dir: Path, eval_path: Path) -> list[str]: + """If eval.yaml lists plugin_dirs, each must exist and contain a SKILL.md.""" + if not eval_path.is_file(): + return [] + try: + data = yaml.safe_load(eval_path.read_text()) or {} + except yaml.YAMLError: + return [] + if not isinstance(data, dict): + return [] + + runner = data.get("runner") or {} + if not isinstance(runner, dict): + return [] + plugin_dirs = runner.get("plugin_dirs") or [] + if not plugin_dirs: + return [] + + errors: list[str] = [] + skill_name = data.get("skill") + for rel in plugin_dirs: + root = submission_dir / str(rel) + if not root.is_dir(): + errors.append( + f"{eval_path.name}: plugin_dirs entry '{rel}/' is missing " + f"(required for Claude slash-skill /{skill_name or '…'})" + ) + continue + has_top = (root / "SKILL.md").is_file() + has_nested = any((child / "SKILL.md").is_file() for child in root.iterdir() if child.is_dir()) + if not has_top and not has_nested: + errors.append( + f"{eval_path.name}: plugin_dirs '{rel}/' must contain SKILL.md (flat or nested skills//SKILL.md)" + ) + elif skill_name and not has_top: + named = root / str(skill_name) / "SKILL.md" + if not named.is_file(): + errors.append( + f"{eval_path.name}: skill '{skill_name}' requires " + f"{rel}/{skill_name}/SKILL.md for /{skill_name} slash command" + ) + return errors + + +def _check_aeh_cases(submission_dir: Path) -> list[str]: + """Validate cases/ directory exists with at least one case. + + Each case directory must contain: + - input.yaml (required) + - annotations.yaml (optional) + """ + cases_dir = submission_dir / "cases" + if not cases_dir.is_dir(): + return ["cases/ directory is required for AEH evaluation"] + + case_dirs = [d for d in cases_dir.iterdir() if d.is_dir()] + if not case_dirs: + return ["cases/ must contain at least one case directory (e.g., cases/case-001/)"] + + errors: list[str] = [] + for case_dir in sorted(case_dirs): + rel = case_dir.relative_to(submission_dir) + input_path = case_dir / "input.yaml" + if not input_path.is_file(): + errors.append(f"{rel}: missing required input.yaml") + continue + + try: + data = yaml.safe_load(input_path.read_text()) + except yaml.YAMLError as exc: + errors.append(f"{rel}/input.yaml: invalid YAML: {exc}") + continue + + if not isinstance(data, dict): + errors.append(f"{rel}/input.yaml: must be a YAML mapping") + + return errors + + +def _check_aeh_pairwise_contract(eval_path: Path) -> list[str]: + """Pairwise mode requires outputs: and a pairwise LLM judge for score.py.""" + if not eval_path.is_file(): + return [] + try: + data = yaml.safe_load(eval_path.read_text()) or {} + except yaml.YAMLError: + return [] + + errors: list[str] = [] + filename = eval_path.name + + outputs = data.get("outputs") or [] + has_path = False + if isinstance(outputs, list): + for out in outputs: + if isinstance(out, dict) and out.get("path"): + has_path = True + break + if isinstance(out, str) and out.strip(): + has_path = True + break + if not has_path: + errors.append( + f"{filename}: pairwise mode requires outputs: with at least one path " + "(e.g. path: output) so score.py can compare case artifacts" + ) + + judges = data.get("judges") or [] + has_pairwise = False + if isinstance(judges, list): + for judge in judges: + if not isinstance(judge, dict): + continue + name = str(judge.get("name") or "").lower() + jtype = str(judge.get("type") or "").lower() + if name == "pairwise" or (jtype == "llm" and "pairwise" in name): + has_pairwise = True + break + if jtype == "llm" and (judge.get("prompt") or judge.get("prompt_file")): + # Accept any LLM judge as fallback (score.py does too) + has_pairwise = True + break + elif isinstance(judges, dict): + has_pairwise = "pairwise" in judges or any( + isinstance(v, dict) and (v.get("type") == "llm" or v.get("prompt")) for v in judges.values() + ) + if not has_pairwise: + errors.append( + f"{filename}: pairwise mode requires a 'pairwise' LLM judge " + "(or another LLM judge with prompt) for score.py pairwise" + ) + + return errors + + +def _check_aeh_structure( + submission_dir: Path, + aeh_mode: str = "single", + control_config: str = "eval-control.yaml", + treatment_config: str = "eval-treatment.yaml", +) -> list[str]: + """Run all AEH-specific validation checks. + + Args: + submission_dir: Path to the submission directory + aeh_mode: Either "single" or "pairwise" + control_config: Control variant config filename for pairwise mode + treatment_config: Treatment variant config filename for pairwise mode + """ + errors: list[str] = [] + + if aeh_mode == "pairwise": + control_path = submission_dir / control_config + treatment_path = submission_dir / treatment_config + + if not control_path.is_file(): + errors.append(f"AEH pairwise: missing {control_config}") + else: + errors.extend(_check_aeh_eval_yaml_file(control_path)) + errors.extend(_check_aeh_plugin_dirs(submission_dir, control_path)) + + if not treatment_path.is_file(): + errors.append(f"AEH pairwise: missing {treatment_config}") + else: + errors.extend(_check_aeh_eval_yaml_file(treatment_path)) + errors.extend(_check_aeh_plugin_dirs(submission_dir, treatment_path)) + # Treatment config drives score.py pairwise — enforce contract there + errors.extend(_check_aeh_pairwise_contract(treatment_path)) + else: + errors.extend(_check_aeh_eval_yaml(submission_dir)) + errors.extend(_check_aeh_plugin_dirs(submission_dir, submission_dir / "eval.yaml")) + + errors.extend(_check_aeh_cases(submission_dir)) + errors.extend(_check_aeh_skill_matches_metadata(submission_dir, aeh_mode, control_config, treatment_config)) + return errors + + +def _check_aeh_skill_matches_metadata( + submission_dir: Path, + aeh_mode: str, + control_config: str, + treatment_config: str, +) -> list[str]: + """Require eval.yaml skill to match metadata.name (analyze uses submission-name). + + Pipeline report.json is keyed by submission-name (usually metadata.name). AEH + run dirs use the skill field for score.py. Mismatches break analyze after a + successful eval. + """ + meta_path = submission_dir / "metadata.yaml" + if not meta_path.is_file(): + return [] + try: + meta = yaml.safe_load(meta_path.read_text()) or {} + except yaml.YAMLError: + return [] + meta_name = meta.get("name") + if not meta_name: + return [] + + config_paths: list[Path] + if aeh_mode == "pairwise": + config_paths = [submission_dir / control_config, submission_dir / treatment_config] + else: + config_paths = [submission_dir / "eval.yaml"] + + errors: list[str] = [] + for path in config_paths: + if not path.is_file(): + continue + try: + data = yaml.safe_load(path.read_text()) or {} + except yaml.YAMLError: + continue + skill = data.get("skill") + if skill and skill != meta_name: + errors.append( + f"{path.name}: skill '{skill}' must match metadata.name '{meta_name}' " + "(analyze reads reports//report.json)" + ) + return errors + + # --------------------------------------------------------------------------- # Main validation # --------------------------------------------------------------------------- @@ -336,8 +608,19 @@ def _check_mcpchecker_tasks(submission_dir: Path) -> list[str]: def validate_submission( submission_dir: Path, eval_engine: EvalEngine = EvalEngine.HARBOR, + aeh_mode: str = "single", + aeh_control_config: str = "eval-control.yaml", + aeh_treatment_config: str = "eval-treatment.yaml", ) -> list[str]: - """Run validation checks based on the eval engine and return error strings.""" + """Run validation checks based on the eval engine and return error strings. + + Args: + submission_dir: Path to the submission directory + eval_engine: Which evaluation engine to validate for + aeh_mode: AEH mode - "single" or "pairwise" + aeh_control_config: Control config filename for AEH pairwise mode + aeh_treatment_config: Treatment config filename for AEH pairwise mode + """ logger.info("Validating submission: %s (eval_engine=%s)", submission_dir, eval_engine) errors: list[str] = [] @@ -345,13 +628,14 @@ def validate_submission( run_ase = eval_engine in (EvalEngine.ASE, EvalEngine.BOTH) run_mcpchecker = eval_engine == EvalEngine.MCPCHECKER run_a2a = eval_engine == EvalEngine.A2A + run_aeh = eval_engine == EvalEngine.AEH # Common: metadata.yaml is always required metadata_errors, metadata = _check_metadata_yaml(submission_dir) errors.extend(metadata_errors) - # MCPChecker and A2A have their own structure - skip skills/ check - if not run_mcpchecker and not run_a2a: + # MCPChecker, A2A, and AEH have their own structure - skip skills/ check + if not run_mcpchecker and not run_a2a and not run_aeh: errors.extend(_check_skills_dir(submission_dir)) if run_harbor: @@ -384,8 +668,18 @@ def validate_submission( if not has_instruction and not has_task_toml and not has_tasks_dir: errors.append("A2A evaluation requires instruction.md, task.toml, or tasks/ directory") - # Common: supportive/ size check (skip for mcpchecker and a2a) - if not run_mcpchecker and not run_a2a: + if run_aeh: + errors.extend( + _check_aeh_structure( + submission_dir, + aeh_mode=aeh_mode, + control_config=aeh_control_config, + treatment_config=aeh_treatment_config, + ) + ) + + # Common: supportive/ size check (skip for mcpchecker, a2a, and aeh) + if not run_mcpchecker and not run_a2a and not run_aeh: errors.extend(_check_supportive_size(submission_dir)) if errors: @@ -401,10 +695,29 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument( "--eval-engine", type=str, - choices=["harbor", "ase", "mcpchecker", "a2a", "both"], + choices=["harbor", "ase", "mcpchecker", "a2a", "aeh", "both"], default="harbor", help="Evaluation engine (controls which checks run)", ) + parser.add_argument( + "--aeh-mode", + type=str, + choices=["single", "pairwise"], + default="single", + help="AEH evaluation mode (single or pairwise)", + ) + parser.add_argument( + "--aeh-control-config", + type=str, + default="eval-control.yaml", + help="Control variant eval.yaml filename for AEH pairwise mode", + ) + parser.add_argument( + "--aeh-treatment-config", + type=str, + default="eval-treatment.yaml", + help="Treatment variant eval.yaml filename for AEH pairwise mode", + ) args = parser.parse_args(argv) submission_dir: Path = args.submission_dir @@ -414,7 +727,13 @@ def main(argv: list[str] | None = None) -> int: return 1 engine = EvalEngine(args.eval_engine) - errors = validate_submission(submission_dir, eval_engine=engine) + errors = validate_submission( + submission_dir, + eval_engine=engine, + aeh_mode=args.aeh_mode, + aeh_control_config=args.aeh_control_config, + aeh_treatment_config=args.aeh_treatment_config, + ) result = {"valid": len(errors) == 0, "errors": errors} print(json.dumps(result, indent=2)) return 0 if result["valid"] else 1 diff --git a/submissions/aeh-hello-world/cases/case-001/annotations.yaml b/submissions/aeh-hello-world/cases/case-001/annotations.yaml new file mode 100644 index 0000000..0b9c81d --- /dev/null +++ b/submissions/aeh-hello-world/cases/case-001/annotations.yaml @@ -0,0 +1,14 @@ +# Annotations for case-001 +# These provide ground truth for automated judges + +expected_behavior: + - The program should print exactly "Hello, World!" + - Should use Python's print function + - Should be syntactically correct Python + +key_elements: + - print statement + - string literal + +difficulty: easy +category: basic-output diff --git a/submissions/aeh-hello-world/cases/case-001/input.yaml b/submissions/aeh-hello-world/cases/case-001/input.yaml new file mode 100644 index 0000000..c2bfad7 --- /dev/null +++ b/submissions/aeh-hello-world/cases/case-001/input.yaml @@ -0,0 +1,9 @@ +# Test case: Simple hello world program +prompt: | + Write a Python program that prints "Hello, World!" to the console. + The program should be simple and follow Python best practices. + +context: + language: python + difficulty: easy + expected_output: "Hello, World!" diff --git a/submissions/aeh-hello-world/eval.yaml b/submissions/aeh-hello-world/eval.yaml new file mode 100644 index 0000000..bf0f8d6 --- /dev/null +++ b/submissions/aeh-hello-world/eval.yaml @@ -0,0 +1,51 @@ +# Agent-Eval-Harness evaluation configuration +# This is a minimal example for testing the AEH pipeline integration + +skill: aeh-hello-world + +# Runner configuration +runner: + type: claude-code + effort: low + settings: + permission_mode: bypassPermissions + +# Model configuration +models: + skill: claude-sonnet-4-5 + judge: claude-sonnet-4-5 + +# Dataset location (relative to submission) +dataset: + path: cases + +# Judges evaluate each case +judges: + - name: correctness + type: llm + prompt: | + Evaluate whether the agent's response correctly addresses the user's request. + Score 1.0 if fully correct, 0.5 if partially correct, 0.0 if incorrect. + + User request: {input} + Agent response: {output} + + Respond with a JSON object: {"score": , "reason": ""} + + - name: helpfulness + type: llm + prompt: | + Evaluate how helpful and complete the agent's response is. + Score 1.0 if very helpful, 0.5 if somewhat helpful, 0.0 if not helpful. + + User request: {input} + Agent response: {output} + + Respond with a JSON object: {"score": , "reason": ""} + +# Thresholds for pass/fail +thresholds: + correctness: + min_mean: 0.7 + helpfulness: + min_mean: 0.5 diff --git a/submissions/aeh-hello-world/metadata.yaml b/submissions/aeh-hello-world/metadata.yaml new file mode 100644 index 0000000..78c4d52 --- /dev/null +++ b/submissions/aeh-hello-world/metadata.yaml @@ -0,0 +1,11 @@ +schema_version: "1.0" +name: aeh-hello-world +description: Sample AEH (Agent-Eval-Harness) submission for testing the pipeline integration +persona: rh-developer +version: "0.1.0" +author: ABEvalFlow Team +eval_engine: aeh +tags: + - sample + - aeh + - hello-world diff --git a/submissions/aeh-pairwise-example/cases/case-001/annotations.yaml b/submissions/aeh-pairwise-example/cases/case-001/annotations.yaml new file mode 100644 index 0000000..77f804a --- /dev/null +++ b/submissions/aeh-pairwise-example/cases/case-001/annotations.yaml @@ -0,0 +1,31 @@ +# Expected annotations for this test case +expected_output: | + A factorial function with error handling and documentation. + +golden_example: | + def factorial(n: int) -> int: + """Calculate the factorial of a non-negative integer. + + Args: + n: A non-negative integer + + Returns: + The factorial of n (n!) + + Raises: + ValueError: If n is negative + TypeError: If n is not an integer + """ + if not isinstance(n, int): + raise TypeError("Input must be an integer") + if n < 0: + raise ValueError("Input must be non-negative") + if n == 0: + return 1 + return n * factorial(n - 1) + +evaluation_criteria: + - Has docstring + - Handles negative input + - Handles 0 correctly + - Uses recursion or iteration correctly diff --git a/submissions/aeh-pairwise-example/cases/case-001/input.yaml b/submissions/aeh-pairwise-example/cases/case-001/input.yaml new file mode 100644 index 0000000..185c478 --- /dev/null +++ b/submissions/aeh-pairwise-example/cases/case-001/input.yaml @@ -0,0 +1,14 @@ +# Test case: Simple coding task +prompt: | + Write a Python function that calculates the factorial of a given number. + Include proper error handling for negative numbers and non-integer inputs. + The function should be well-documented with a docstring. + +context: + language: python + difficulty: medium + expected_behavior: | + - Accept an integer n >= 0 + - Return n! (n factorial) + - Raise ValueError for negative numbers + - Handle edge case of 0! = 1 diff --git a/submissions/aeh-pairwise-example/eval-control.yaml b/submissions/aeh-pairwise-example/eval-control.yaml new file mode 100644 index 0000000..6449887 --- /dev/null +++ b/submissions/aeh-pairwise-example/eval-control.yaml @@ -0,0 +1,51 @@ +# AEH Control configuration (baseline - without skill) +# This represents the baseline agent without any skill enhancements + +skill: aeh-pairwise-example + +# Runner configuration - standard Claude Code without skill plugins +runner: + type: claude-code + effort: low + settings: + permission_mode: bypassPermissions + # No plugin_dirs - raw Claude Code capability + +# Model configuration +models: + skill: claude-sonnet-4-5 + judge: claude-sonnet-4-5 + +# Dataset location (relative to submission) +dataset: + path: cases + +# Case artifact path (aligned with treatment for pairwise) +outputs: + - path: output + +# Judges evaluate each case +judges: + - name: task_completion + type: check + check: | + exit_code = outputs.get("exit_code", 1) + return exit_code == 0, f"Exit code: {exit_code}" + + - name: output_quality + type: llm + prompt: | + Rate the quality of the agent's output on a scale of 1-5. + + Task: {input} + Output: {output} + + Consider correctness, completeness, and clarity. + Respond with a JSON object: {"score": <1-5>, "reason": ""} + +# Thresholds for pass/fail +thresholds: + task_completion: + min_pass_rate: 0.8 + output_quality: + min_mean: 3.0 diff --git a/submissions/aeh-pairwise-example/eval-treatment.yaml b/submissions/aeh-pairwise-example/eval-treatment.yaml new file mode 100644 index 0000000..808008b --- /dev/null +++ b/submissions/aeh-pairwise-example/eval-treatment.yaml @@ -0,0 +1,72 @@ +# AEH Treatment configuration (with skill) +# This represents the agent with skill enhancements that we're testing + +skill: aeh-pairwise-example + +# Runner configuration - Claude Code with skill loaded +runner: + type: claude-code + effort: low + settings: + permission_mode: bypassPermissions + # Plugin directories load the skill being tested (relative to submission root) + plugin_dirs: + - skills + +# Model configuration +models: + skill: claude-sonnet-4-5 + judge: claude-sonnet-4-5 + +# Dataset location (relative to submission) +dataset: + path: cases + +# Case artifact path required by score.py pairwise comparison +outputs: + - path: output + +# Judges evaluate each case - same judges as control for fair comparison +judges: + - name: task_completion + type: check + check: | + exit_code = outputs.get("exit_code", 1) + return exit_code == 0, f"Exit code: {exit_code}" + + - name: output_quality + type: llm + prompt: | + Rate the quality of the agent's output on a scale of 1-5. + + Task: {input} + Output: {output} + + Consider correctness, completeness, and clarity. + Respond with a JSON object: {"score": <1-5>, "reason": ""} + + # Explicit pairwise comparison judge for A/B testing + # Used by score.py pairwise --judge pairwise to determine winner + # Note: score.py builds its own "Output A / Output B" message format; + # this prompt provides evaluation criteria only + - name: pairwise + type: llm + prompt: | + Compare the two agent outputs and determine which better accomplishes the task. + + Evaluation criteria: + - Correctness: Does it solve the problem accurately? + - Completeness: Does it fully address all requirements? + - Quality: Is the solution well-structured and clear? + - Efficiency: Is it concise without sacrificing correctness? + + Prefer the output that scores higher across these criteria. + +# Thresholds for pass/fail +thresholds: + task_completion: + min_pass_rate: 0.8 + output_quality: + min_mean: 3.0 + pairwise: + min_win_rate: 0.5 diff --git a/submissions/aeh-pairwise-example/metadata.yaml b/submissions/aeh-pairwise-example/metadata.yaml new file mode 100644 index 0000000..3cdaa78 --- /dev/null +++ b/submissions/aeh-pairwise-example/metadata.yaml @@ -0,0 +1,12 @@ +schema_version: "1.0" +name: aeh-pairwise-example +description: Sample AEH pairwise A/B comparison submission (treatment vs control) +persona: rh-developer +version: "0.1.0" +author: ABEvalFlow Team +eval_engine: aeh +tags: + - sample + - aeh + - pairwise + - a-b-testing diff --git a/submissions/aeh-pairwise-example/skills/aeh-pairwise-example/SKILL.md b/submissions/aeh-pairwise-example/skills/aeh-pairwise-example/SKILL.md new file mode 100644 index 0000000..ef37d88 --- /dev/null +++ b/submissions/aeh-pairwise-example/skills/aeh-pairwise-example/SKILL.md @@ -0,0 +1,10 @@ +--- +name: aeh-pairwise-example +description: Minimal sample skill for AEH pairwise A/B demo submissions +--- + +# aeh-pairwise-example + +Sample skill package for in-repo pairwise smoke validation. + +When invoked, briefly greet the user and confirm the skill is loaded. diff --git a/tests/test_aeh_engine.py b/tests/test_aeh_engine.py new file mode 100644 index 0000000..a6f14cb --- /dev/null +++ b/tests/test_aeh_engine.py @@ -0,0 +1,472 @@ +"""Tests for AEH (Agent-Eval-Harness) engine adapter.""" + +import json + +import pytest +import yaml + +from abevalflow.engines import get_all_engines, get_engine +from abevalflow.engines.aeh import AEHEngine +from abevalflow.gates.base import GateType +from abevalflow.schemas import GatePolicy, GatePolicyItem + + +class TestAEHEngineRegistry: + """Tests for AEH engine registration.""" + + def test_aeh_registered(self): + engines = get_all_engines() + assert "aeh" in engines + + def test_get_engine_aeh(self): + engine = get_engine("aeh") + assert isinstance(engine, AEHEngine) + assert engine.name == "aeh" + + +class TestAEHEngine: + """Tests for AEH engine adapter.""" + + def test_read_result_not_found(self, tmp_path): + engine = AEHEngine() + result = engine.read_result(tmp_path) + assert result is None + + def test_read_result_from_report_json(self, tmp_path): + """Test reading unified report.json format.""" + report = { + "eval_engine": "aeh", + "mode": "single", + "run_id": "test-run-001", + "mean_reward": 0.85, + "judges": {"correctness": 0.9, "style": 0.8}, + "per_case": {"case-001": {"reward": 0.85}}, + } + (tmp_path / "report.json").write_text(json.dumps(report)) + + engine = AEHEngine() + result = engine.read_result(tmp_path) + assert result is not None + assert result["mean_reward"] == 0.85 + assert result["mode"] == "single" + + def test_read_result_from_summary_yaml(self, tmp_path): + """Test reading raw AEH output (summary.yaml).""" + summary = { + "run_id": "test-run-001", + "mean_reward": 0.75, + "judges": {"correctness": {"mean": 0.8}}, + "per_case": {"case-001": {"reward": 0.75}}, + } + (tmp_path / "summary.yaml").write_text(yaml.dump(summary)) + + engine = AEHEngine() + result = engine.read_result(tmp_path) + assert result is not None + assert result["mean_reward"] == 0.75 + assert result["run_id"] == "test-run-001" + + def test_read_result_from_summary_with_run_result(self, tmp_path): + """Test reading summary.yaml with run_result.json (overrides mean_reward).""" + summary = { + "run_id": "test-run-001", + "mean_reward": 0.75, + "judges": {}, + "per_case": {}, + } + run_result = { + "mean_reward": 0.85, + "duration_s": 120.5, + "cost_usd": 0.15, + } + (tmp_path / "summary.yaml").write_text(yaml.dump(summary)) + (tmp_path / "run_result.json").write_text(json.dumps(run_result)) + + engine = AEHEngine() + result = engine.read_result(tmp_path) + assert result is not None + assert result["mean_reward"] == 0.85 # From run_result.json + assert result["execution"]["duration_s"] == 120.5 + + def test_read_result_from_nested_run_dir(self, tmp_path): + """Test reading from reports/// structure.""" + run_dir = tmp_path / "test-run-001" + run_dir.mkdir() + summary = { + "run_id": "test-run-001", + "mean_reward": 0.9, + "judges": {}, + "per_case": {}, + } + (run_dir / "summary.yaml").write_text(yaml.dump(summary)) + + engine = AEHEngine() + result = engine.read_result(tmp_path) + assert result is not None + assert result["mean_reward"] == 0.9 + + def test_to_gate_result_single_pass(self): + """Test single-run mode with passing result.""" + report = { + "eval_engine": "aeh", + "mode": "single", + "mean_reward": 0.85, + "judges": {"correctness": 0.9}, + "per_case": {}, + } + policy = GatePolicy() + engine = AEHEngine() + gate_result = engine.to_gate_result(report, policy) + + assert gate_result.gate_type == GateType.ENGINE + assert gate_result.gate_name == "evaluation" + assert gate_result.policy_key == "aeh" + assert gate_result.details["engine"] == "aeh" + assert gate_result.passed is True + assert gate_result.score == 0.85 + + def test_to_gate_result_single_fail(self): + """Test single-run mode with failing result (below threshold).""" + report = { + "eval_engine": "aeh", + "mode": "single", + "mean_reward": 0.3, # Below custom threshold of 0.5 + "judges": {}, + "per_case": {}, + } + # Set threshold to 0.5 so 0.3 fails + policy = GatePolicy(gates={"evaluation": GatePolicyItem(threshold=0.5)}) + engine = AEHEngine() + gate_result = engine.to_gate_result(report, policy) + + assert gate_result.passed is False # 0.3 < 0.5 + assert gate_result.score == 0.3 + + def test_to_gate_result_uses_default_threshold(self): + """Test that default policy uses get_default_threshold() (0.5).""" + report = { + "eval_engine": "aeh", + "mode": "single", + "mean_reward": 0.3, # Below default threshold of 0.5 + "judges": {}, + "per_case": {}, + } + # Default policy with no threshold set + policy = GatePolicy() + engine = AEHEngine() + gate_result = engine.to_gate_result(report, policy) + + # With default threshold of 0.5, mean_reward=0.3 should fail + assert gate_result.passed is False + assert gate_result.score == 0.3 + + def test_to_gate_result_with_custom_threshold(self): + """Test that custom threshold is respected.""" + report = { + "eval_engine": "aeh", + "mode": "single", + "mean_reward": 0.6, + "judges": {}, + "per_case": {}, + } + # Set threshold to 0.7 + policy = GatePolicy(gates={"evaluation": GatePolicyItem(threshold=0.7)}) + engine = AEHEngine() + gate_result = engine.to_gate_result(report, policy) + + assert gate_result.passed is False # 0.6 < 0.7 + assert gate_result.threshold == 0.7 + + def test_to_gate_result_with_findings(self): + """Test that low-reward cases are reported as findings.""" + report = { + "eval_engine": "aeh", + "mode": "single", + "mean_reward": 0.5, + "judges": {}, + "per_case": { + "case-001": {"reward": 0.9}, + "case-002": {"reward": 0.3}, # Low reward + "case-003": {"reward": 0.1}, # Very low reward + }, + } + policy = GatePolicy() + engine = AEHEngine() + gate_result = engine.to_gate_result(report, policy) + + assert len(gate_result.findings) == 2 # case-002 and case-003 + + def test_to_gate_result_pairwise(self): + """Test pairwise comparison mode.""" + report = { + "eval_engine": "aeh", + "mode": "pairwise", + "mean_reward": 0.0, # Not used in pairwise + "pairwise": { + "run_a": "treatment-001", + "run_b": "control-001", + "wins_a": 7, + "wins_b": 2, + "ties": 1, + }, + } + policy = GatePolicy() + engine = AEHEngine() + gate_result = engine.to_gate_result(report, policy) + + # Win rate = 7/10 = 0.7 (ties count in denominator) + assert gate_result.passed is True + assert gate_result.score == pytest.approx(0.7) + + def test_default_threshold(self): + engine = AEHEngine() + assert engine.get_default_threshold() == 0.5 + + def test_message_includes_judge_summary(self): + """Test that message includes judge scores.""" + report = { + "eval_engine": "aeh", + "mode": "single", + "mean_reward": 0.85, + "judges": {"correctness": 0.9, "style": 0.8}, + "per_case": {}, + } + policy = GatePolicy() + engine = AEHEngine() + gate_result = engine.to_gate_result(report, policy) + + assert "correctness=0.90" in gate_result.message + assert "style=0.80" in gate_result.message + + def test_pairwise_with_threshold(self): + """Test pairwise mode with custom threshold.""" + report = { + "eval_engine": "aeh", + "mode": "pairwise", + "pairwise": { + "wins_a": 3, + "wins_b": 4, + "ties": 3, + }, + } + # Win rate = 3/10 = 0.3, fails default threshold of 0.5 + policy = GatePolicy() + engine = AEHEngine() + gate_result = engine.to_gate_result(report, policy) + + assert gate_result.passed is False + assert gate_result.score == pytest.approx(0.3) + assert gate_result.threshold == 0.5 + + def test_pairwise_all_ties_passes(self): + """All-ties: score is 0%, but gate still passes.""" + report = { + "eval_engine": "aeh", + "mode": "pairwise", + "pairwise": { + "wins_a": 0, + "wins_b": 0, + "ties": 1, + }, + } + policy = GatePolicy() + engine = AEHEngine() + gate_result = engine.to_gate_result(report, policy) + + assert gate_result.passed is True + assert gate_result.score == 0.0 + + def test_single_none_mean_reward_score_is_zero(self): + """Missing mean_reward must not crash GateResult; score floors at 0.0.""" + report = { + "eval_engine": "aeh", + "mode": "single", + "mean_reward": None, + "summary": {}, + "judges": {}, + "per_case": {}, + } + policy = GatePolicy() + engine = AEHEngine() + gate_result = engine.to_gate_result(report, policy) + + assert gate_result.passed is False + assert gate_result.score == 0.0 + + def test_pairwise_errors_in_denominator(self): + """Errors count as non-wins so 1/10 with 9 errors fails the gate.""" + report = { + "eval_engine": "aeh", + "mode": "pairwise", + "pairwise": { + "wins_a": 1, + "wins_b": 0, + "ties": 0, + "errors": 9, + }, + } + policy = GatePolicy() + engine = AEHEngine() + gate_result = engine.to_gate_result(report, policy) + + assert gate_result.score == pytest.approx(0.1) + assert gate_result.passed is False + + def test_pairwise_with_stability(self): + """Test pairwise mode with stability metrics.""" + report = { + "eval_engine": "aeh", + "mode": "pairwise", + "pairwise": { + "wins_a": 7, + "wins_b": 2, + "ties": 1, + "stability": { + "runs": 3, + "agreement_rate": 0.9, + "stable_cases": 9, + "total_cases": 10, + }, + }, + } + policy = GatePolicy() + engine = AEHEngine() + gate_result = engine.to_gate_result(report, policy) + + assert "stability=90%" in gate_result.message + assert gate_result.details["pairwise"]["stability"]["agreement_rate"] == 0.9 + + def test_pairwise_findings_extraction(self): + """Test that pairwise findings include control wins and errors.""" + report = { + "eval_engine": "aeh", + "mode": "pairwise", + "pairwise": { + "wins_a": 5, + "wins_b": 3, + "ties": 2, + "per_case": [ + {"case_id": "case-001", "winner": "A", "error": None}, + {"case_id": "case-002", "winner": "B", "error": None}, + {"case_id": "case-003", "winner": "error", "error": "API timeout"}, + {"case_id": "case-004", "winner": "tie", "error": None}, + ], + }, + } + policy = GatePolicy() + engine = AEHEngine() + gate_result = engine.to_gate_result(report, policy) + + # Should have finding for case-002 (control wins) and case-003 (error) + assert len(gate_result.findings) == 2 + finding_messages = [f.message for f in gate_result.findings] + assert any("case-002" in m and "control beat treatment" in m for m in finding_messages) + assert any("case-003" in m and "error" in m for m in finding_messages) + + def test_judge_passthrough_dict_format(self): + """Test that full judge metadata (dict format) is preserved.""" + report = { + "eval_engine": "aeh", + "mode": "single", + "mean_reward": 0.8, + "judges": { + "exit_success": { + "mean": 1.0, + "pass_rate": 1.0, + }, + "output_quality": { + "mean": 3.8, + "pass_rate": None, + "stability": { + "samples": 3, + "stable_cases": 8, + "total_cases": 10, + }, + }, + }, + "per_case": {}, + } + policy = GatePolicy() + engine = AEHEngine() + gate_result = engine.to_gate_result(report, policy) + + # Check full metadata is preserved in details + judges = gate_result.details["judges"] + assert judges["exit_success"]["pass_rate"] == 1.0 + assert judges["output_quality"]["stability"]["samples"] == 3 + + # Check message formats pass_rate as percentage + assert "exit_success=100%" in gate_result.message + + def test_per_case_findings_with_judge_types(self): + """Test findings extraction from per_case with AEH's nested judge structure.""" + report = { + "eval_engine": "aeh", + "mode": "single", + "mean_reward": 0.5, + "judges": {}, + "per_case": { + "case-001": { + "exit_success": { + "value": True, + "rationale": "Success", + "judge_type": "check", + }, + }, + "case-002": { + "exit_success": { + "value": False, + "rationale": "Failed", + "judge_type": "check", + }, + }, + "case-003": { + "output_quality": { + "value": 2, + "rationale": "Low quality", + "judge_type": "llm", + }, + }, + "case-004": { + "syntax_check": { + "value": None, + "error": "Parser failed", + "judge_type": "check", + }, + }, + }, + } + policy = GatePolicy() + engine = AEHEngine() + gate_result = engine.to_gate_result(report, policy) + + # Should have findings for: case-002 (failed), case-003 (low score), case-004 (error) + assert len(gate_result.findings) == 3 + finding_rules = [f.rule_id for f in gate_result.findings] + assert "aeh-judge-failed" in finding_rules + assert "aeh-low-score" in finding_rules + assert "aeh-judge-error" in finding_rules + + def test_read_result_detects_pairwise_mode(self, tmp_path): + """Test that mode is correctly detected from summary.yaml pairwise key.""" + summary = { + "run_id": "treatment-001", + "mean_reward": 0.8, + "judges": {}, + "per_case": {}, + "pairwise": { + "run_a": "treatment-001", + "run_b": "control-001", + "wins_a": 5, + "wins_b": 3, + "ties": 2, + }, + } + (tmp_path / "summary.yaml").write_text(yaml.dump(summary)) + + engine = AEHEngine() + result = engine.read_result(tmp_path) + + assert result is not None + assert result["mode"] == "pairwise" + assert result["pairwise"]["wins_a"] == 5 diff --git a/tests/test_aeh_scoring.py b/tests/test_aeh_scoring.py new file mode 100644 index 0000000..bd8d3f9 --- /dev/null +++ b/tests/test_aeh_scoring.py @@ -0,0 +1,66 @@ +"""Unit tests for shared AEH scoring helpers.""" + +from __future__ import annotations + +from pathlib import Path + +import yaml + +from abevalflow.aeh_scoring import ( + DEFAULT_AEH_THRESHOLD, + numeric_judge_is_low, + numeric_judge_passes, + pairwise_outcome, + resolve_evaluation_threshold, +) + + +class TestPairwiseOutcome: + def test_includes_errors_in_denominator(self): + out = pairwise_outcome(wins_a=1, wins_b=0, ties=0, errors=9) + assert out["total"] == 10 + assert out["win_rate"] == 0.1 + assert out["passed"] is False + + def test_all_ties_passes(self): + out = pairwise_outcome(wins_a=0, wins_b=0, ties=5, errors=0) + assert out["all_ties"] is True + assert out["passed"] is True + assert out["win_rate"] == 0.0 + + def test_all_errors_fails(self): + out = pairwise_outcome(wins_a=0, wins_b=0, ties=0, errors=5) + assert out["all_ties"] is False + assert out["passed"] is False + + +class TestNumericJudge: + def test_normalized_scale(self): + assert numeric_judge_passes(0.5) is True + assert numeric_judge_passes(0.49) is False + + def test_likert_int(self): + assert numeric_judge_passes(1) is False + assert numeric_judge_passes(3) is True + assert numeric_judge_is_low(2) is True + + def test_likert_float(self): + assert numeric_judge_passes(2.5) is False + assert numeric_judge_passes(3.0) is True + + +class TestResolveEvaluationThreshold: + def test_default_when_missing(self, tmp_path: Path): + assert resolve_evaluation_threshold(tmp_path / "missing.yaml") == DEFAULT_AEH_THRESHOLD + + def test_reads_gate_policy(self, tmp_path: Path): + meta = tmp_path / "metadata.yaml" + meta.write_text( + yaml.dump( + { + "name": "demo", + "gate_policy": {"gates": {"evaluation": {"threshold": 0.6}}}, + } + ) + ) + assert resolve_evaluation_threshold(meta) == 0.6 diff --git a/tests/test_aggregate_aeh.py b/tests/test_aggregate_aeh.py new file mode 100644 index 0000000..28c15d5 --- /dev/null +++ b/tests/test_aggregate_aeh.py @@ -0,0 +1,236 @@ +"""Tests for AEH report aggregation (scripts/aggregate_aeh.py).""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +import yaml + +from abevalflow.report import AnalysisResult +from scripts.aggregate_aeh import ( + _case_reward, + _trials_from_per_case, + aggregate_pairwise_run, + aggregate_single_run, +) +from scripts.analyze import render_markdown + + +def _write_summary(run_dir: Path, summary: dict) -> None: + run_dir.mkdir(parents=True, exist_ok=True) + (run_dir / "summary.yaml").write_text(yaml.dump(summary)) + + +class TestCaseReward: + def test_boolean_pass(self): + assert _case_reward({"exit_success": {"value": True}}) == 1.0 + + def test_boolean_fail(self): + assert _case_reward({"exit_success": {"value": False}}) == 0.0 + + def test_numeric_mean(self): + assert ( + _case_reward( + { + "a": {"value": 0.5}, + "b": {"value": 1.0}, + } + ) + == 0.75 + ) + + def test_top_level_reward(self): + assert _case_reward({"reward": 0.9}) == 0.9 + + def test_empty(self): + assert _case_reward({}) is None + + +class TestTrialsFromPerCase: + def test_maps_case_ids(self): + trials = _trials_from_per_case( + { + "case-001": {"exit_success": {"value": True}}, + "case-002": {"exit_success": {"value": False}}, + } + ) + assert len(trials) == 2 + assert trials[0]["trial_name"] == "case-001" + assert trials[0]["reward"] == 1.0 + assert trials[1]["reward"] == 0.0 + + +class TestAggregateSingleRun: + def test_trials_match_n_trials(self, tmp_path: Path): + run_dir = tmp_path / "aeh-hello" / "run-1" + _write_summary( + run_dir, + { + "run_id": "run-1", + "mean_reward": 1.0, + "per_case": { + "case-001": {"exit_success": {"value": True}}, + }, + "judges": {}, + }, + ) + report = aggregate_single_run(run_dir) + assert report["summary"]["treatment"]["n_trials"] == 1 + assert len(report["trials"]["treatment"]) == 1 + assert report["trials"]["treatment"][0]["trial_name"] == "case-001" + assert report["trials"]["control"] == [] + + result = AnalysisResult.model_validate(report) + md = render_markdown(result) + assert "Treatment (1 trials)" in md + assert "case-001" in md + + +class TestAggregatePairwiseRun: + def test_trials_and_pairwise_section(self, tmp_path: Path): + treatment = tmp_path / "skill" / "treatment-1" + control = tmp_path / "skill" / "control-1" + _write_summary( + treatment, + { + "run_id": "treatment-1", + "mean_reward": 1.0, + "per_case": {"case-001": {"exit_success": {"value": True}}}, + "pairwise": { + "run_a": "treatment-1", + "run_b": "control-1", + "cases_compared": 1, + "wins_a": 1, + "wins_b": 0, + "ties": 0, + "errors": 0, + "per_case": [{"case_id": "case-001", "winner": "a"}], + }, + }, + ) + _write_summary( + control, + { + "run_id": "control-1", + "mean_reward": 0.0, + "per_case": {"case-001": {"exit_success": {"value": False}}}, + }, + ) + report = aggregate_pairwise_run(treatment, control) + assert len(report["trials"]["treatment"]) == 1 + assert len(report["trials"]["control"]) == 1 + assert report["pairwise"]["wins_a"] == 1 + assert report["aeh_warnings"] == [] + + result = AnalysisResult.model_validate(report) + md = render_markdown(result) + assert "## Pairwise Comparison" in md + assert "**Treatment wins:** 1" in md + assert "case-001" in md + assert "Treatment (1 trials)" in md + assert "Control (1 trials)" in md + + def test_missing_pairwise_warns(self, tmp_path: Path): + treatment = tmp_path / "skill" / "treatment-1" + control = tmp_path / "skill" / "control-1" + _write_summary( + treatment, + { + "run_id": "treatment-1", + "mean_reward": 0.0, + "per_case": {}, + }, + ) + _write_summary(control, {"run_id": "control-1", "mean_reward": 0.0, "per_case": {}}) + report = aggregate_pairwise_run(treatment, control) + assert report["aeh_warnings"] + assert report["pairwise"]["cases_compared"] == 0 + + def test_all_ties_is_pass_with_rewards_from_run_result(self, tmp_path: Path): + treatment = tmp_path / "skill" / "treatment-1" + control = tmp_path / "skill" / "control-1" + _write_summary( + treatment, + { + "run_id": "treatment-1", + "per_case": {"case-001": {"exit_success": {"value": True}}}, + "pairwise": { + "wins_a": 0, + "wins_b": 0, + "ties": 1, + "errors": 0, + "cases_compared": 1, + "per_case": [{"case_id": "case-001", "winner": "tie"}], + }, + }, + ) + (treatment / "run_result.json").write_text(json.dumps({"mean_reward": 1.0})) + _write_summary( + control, + { + "run_id": "control-1", + "per_case": {"case-001": {"exit_success": {"value": True}}}, + }, + ) + (control / "run_result.json").write_text(json.dumps({"mean_reward": 1.0})) + report = aggregate_pairwise_run(treatment, control) + assert report["summary"]["treatment"]["mean_reward"] == 1.0 + assert report["summary"]["control"]["mean_reward"] == 1.0 + assert report["pairwise"]["win_rate"] == 0.0 # ties are non-wins + assert report["recommendation"] == "pass" # all-ties is still pass + + def test_errors_lower_win_rate(self, tmp_path: Path): + treatment = tmp_path / "skill" / "treatment-1" + control = tmp_path / "skill" / "control-1" + _write_summary( + treatment, + { + "run_id": "treatment-1", + "per_case": {}, + "pairwise": { + "wins_a": 1, + "wins_b": 0, + "ties": 0, + "errors": 9, + "cases_compared": 10, + "per_case": [], + }, + }, + ) + _write_summary(control, {"run_id": "control-1", "per_case": {}}) + report = aggregate_pairwise_run(treatment, control) + assert report["pairwise"]["win_rate"] == pytest.approx(0.1) + assert report["recommendation"] == "fail" + + def test_submission_name_override(self, tmp_path: Path): + run_dir = tmp_path / "skill-from-path" / "run-1" + _write_summary( + run_dir, + { + "run_id": "run-1", + "mean_reward": 1.0, + "per_case": {"case-001": {"exit_success": {"value": True}}}, + "judges": {}, + }, + ) + report = aggregate_single_run(run_dir, submission_name="pipeline-submission") + assert report["submission_name"] == "pipeline-submission" + + def test_likert_one_is_not_a_pass(self, tmp_path: Path): + run_dir = tmp_path / "skill" / "run-1" + _write_summary( + run_dir, + { + "run_id": "run-1", + "mean_reward": 0.2, + "per_case": { + "case-001": {"output_quality": {"value": 1}}, + }, + "judges": {}, + }, + ) + report = aggregate_single_run(run_dir) + assert report["passed_cases"] == 0 + assert report["pass_rate"] == 0.0 diff --git a/tests/test_run_aeh.py b/tests/test_run_aeh.py new file mode 100644 index 0000000..4c23eaf --- /dev/null +++ b/tests/test_run_aeh.py @@ -0,0 +1,470 @@ +"""Tests for scripts/run_aeh.py - AEH runner dispatcher.""" + +from __future__ import annotations + +from pathlib import Path +from unittest.mock import MagicMock, patch + +import yaml + +from scripts.run_aeh import ( + RUNNERS, + BaseRunner, + HarborRunner, + RunnerError, + VanillaRunner, + get_runner, + materialize_aeh_case_outputs, +) + + +class TestRunnerRegistry: + """Test runner registration and lookup.""" + + def test_harbor_runner_registered(self): + """Harbor runner should be registered.""" + assert "harbor" in RUNNERS + assert RUNNERS["harbor"] is HarborRunner + + def test_vanilla_runner_registered(self): + """Vanilla runner should be registered.""" + assert "vanilla" in RUNNERS + assert RUNNERS["vanilla"] is VanillaRunner + + def test_get_runner_harbor(self): + """get_runner should return HarborRunner for 'harbor'.""" + runner = get_runner("harbor") + assert isinstance(runner, HarborRunner) + + def test_get_runner_vanilla(self): + """get_runner should return VanillaRunner for 'vanilla'.""" + runner = get_runner("vanilla") + assert isinstance(runner, VanillaRunner) + + def test_get_runner_unknown_raises(self): + """get_runner should raise ValueError for unknown runner.""" + try: + get_runner("unknown") + assert False, "Should have raised ValueError" + except ValueError as e: + assert "unknown" in str(e).lower() + assert "harbor" in str(e).lower() + assert "vanilla" in str(e).lower() + + def test_get_runner_passes_kwargs(self): + """get_runner should pass kwargs to runner constructor.""" + runner = get_runner("harbor", model="test-model", judge_model="test-judge") + assert runner.model == "test-model" + assert runner.judge_model == "test-judge" + + +class TestMaterializeAehCaseOutputs: + """Bridge Harbor verifier/output → run_dir/cases//output.""" + + def test_copies_verifier_output(self, tmp_path: Path): + config = tmp_path / "eval.yaml" + config.write_text( + yaml.dump( + { + "skill": "demo", + "outputs": [{"path": "output"}], + } + ) + ) + jobs = tmp_path / "jobs" / "job-1" + trial = jobs / "case-001__AbCdEf" + src = trial / "verifier" / "output" + src.mkdir(parents=True) + (src / "greeting.txt").write_text("Hello, World!") + + out = tmp_path / "run-out" + n = materialize_aeh_case_outputs(config, out, jobs_dir=tmp_path / "jobs") + assert n == 1 + copied = out / "cases" / "case-001" / "output" / "greeting.txt" + assert copied.is_file() + assert copied.read_text() == "Hello, World!" + + def test_falls_back_to_artifacts(self, tmp_path: Path): + config = tmp_path / "eval.yaml" + config.write_text(yaml.dump({"skill": "demo", "outputs": [{"path": "output"}]})) + jobs = tmp_path / "jobs" / "job-1" + trial = jobs / "case-001" + art = trial / "verifier" / "artifacts" + art.mkdir(parents=True) + (art / "out.txt").write_text("x") + out = tmp_path / "run-out" + n = materialize_aeh_case_outputs(config, out, jobs_dir=tmp_path / "jobs") + assert n == 1 + assert (out / "cases" / "case-001" / "output" / "out.txt").read_text() == "x" + + +class TestHarborRunner: + """Test HarborRunner execution.""" + + def test_harbor_runner_name(self): + """Harbor runner should have correct name.""" + runner = HarborRunner() + assert runner.name == "harbor" + + def test_harbor_runner_default_env_type(self): + """Harbor runner should default to kubernetes env.""" + runner = HarborRunner() + assert runner.env_type == "kubernetes" + + def test_harbor_runner_custom_env_type(self): + """Harbor runner should accept custom env type.""" + runner = HarborRunner(env_type="podman") + assert runner.env_type == "podman" + + @patch("subprocess.run") + def test_harbor_runner_execute_basic(self, mock_run, tmp_path): + """Harbor runner should call harbor.run with correct args.""" + mock_run.return_value = MagicMock(returncode=0) + runner = HarborRunner(model="test-model") + + config = tmp_path / "eval.yaml" + config.write_text(yaml.dump({"skill": "demo", "models": {"skill": "m"}})) + output = tmp_path / "output" + + exit_code = runner.run_single(config, output) + + assert exit_code == 0 + mock_run.assert_called_once() + call_args = mock_run.call_args[0][0] + + assert "-m" in call_args + assert "agent_eval.harbor.run" in call_args + assert "--config" in call_args + assert "--output" in call_args + # kubernetes mode uses OpenShiftEnvironment import path (not --env) + assert "--environment-import-path" in call_args + assert "OpenShiftEnvironment" in " ".join(call_args) + assert "--model" in call_args + assert "test-model" in call_args + + @patch("subprocess.run") + def test_harbor_runner_execute_with_image(self, mock_run, tmp_path): + """Harbor runner should pass image flag when set.""" + mock_run.return_value = MagicMock(returncode=0) + runner = HarborRunner(model="test-model", image="quay.io/test/image:v1") + + config = tmp_path / "eval.yaml" + config.write_text(yaml.dump({"skill": "demo", "models": {"skill": "m"}})) + output = tmp_path / "output" + + runner.run_single(config, output) + + call_args = mock_run.call_args[0][0] + assert "--image" in call_args + assert "quay.io/test/image:v1" in call_args + + @patch("subprocess.run") + def test_harbor_runner_prepares_enriched_tasks(self, mock_run, tmp_path): + """With image + tasks_dir, runner generates and enriches Harbor tasks.""" + mock_run.return_value = MagicMock(returncode=0) + runner = HarborRunner(model="test-model", image="quay.io/test/image:v1") + config = tmp_path / "eval.yaml" + config.write_text(yaml.dump({"skill": "demo", "models": {"skill": "m"}})) + output = tmp_path / "output" + tasks_dir = tmp_path / "tasks" + + with patch.object(runner, "_prepare_enriched_tasks") as prepare: + runner.run_single(config, output, tasks_dir=tasks_dir) + + prepare.assert_called_once() + assert prepare.call_args[0][1] == tasks_dir + + def test_harbor_runner_requires_model(self): + """Harbor runner should fail fast without model.""" + runner = HarborRunner() # No model + config = Path("/tmp/eval.yaml") + output = Path("/tmp/output") + + try: + with patch.object(Path, "mkdir"): + runner.run_single(config, output) + assert False, "Should have raised RunnerError" + except RunnerError as e: + assert "model" in str(e).lower() + + +class TestVanillaRunner: + """Test VanillaRunner execution.""" + + def test_vanilla_runner_name(self): + """Vanilla runner should have correct name.""" + runner = VanillaRunner() + assert runner.name == "vanilla" + + def test_vanilla_runner_raises_runner_error(self, tmp_path): + """Vanilla runner should raise RunnerError with helpful message.""" + runner = VanillaRunner() + config = tmp_path / "eval.yaml" + config.write_text("skill: test") + output = tmp_path / "output" + + try: + runner.run_single(config, output) + assert False, "Should have raised RunnerError" + except RunnerError as e: + # Verify helpful message + msg = str(e).lower() + assert "not yet implemented" in msg + assert "harbor" in msg # Should suggest using harbor instead + + +class TestBaseRunner: + """Test BaseRunner common functionality.""" + + def test_read_skill_name_from_config(self, tmp_path): + """_read_skill_name should extract skill from eval.yaml.""" + config_path = tmp_path / "eval.yaml" + config_path.write_text(yaml.dump({"skill": "my-test-skill"})) + + runner = HarborRunner() + skill_name = runner._read_skill_name(config_path) + + assert skill_name == "my-test-skill" + + def test_read_skill_name_fallback_to_stem(self, tmp_path): + """_read_skill_name should fall back to filename stem if no skill field.""" + config_path = tmp_path / "my-config.yaml" + config_path.write_text(yaml.dump({"models": {"skill": "claude"}})) + + runner = HarborRunner() + skill_name = runner._read_skill_name(config_path) + + assert skill_name == "my-config" + + def test_read_skill_name_handles_missing_file(self, tmp_path): + """_read_skill_name should fall back to stem for missing file.""" + config_path = tmp_path / "missing.yaml" + + runner = HarborRunner() + skill_name = runner._read_skill_name(config_path) + + assert skill_name == "missing" + + +class TestPairwiseExecution: + """Test pairwise A/B comparison execution.""" + + @patch("subprocess.run") + def test_pairwise_creates_correct_directories(self, mock_run, tmp_path): + """run_pairwise should create control and treatment directories.""" + mock_run.return_value = MagicMock(returncode=0) + + control_config = tmp_path / "eval-control.yaml" + treatment_config = tmp_path / "eval-treatment.yaml" + output_base = tmp_path / "output" + + control_config.write_text(yaml.dump({"skill": "test-skill"})) + treatment_config.write_text(yaml.dump({"skill": "test-skill"})) + + runner = HarborRunner(model="test-model") + + # Mock summary.yaml creation + def create_summary(*args, **kwargs): + result = MagicMock(returncode=0) + # Create summary.yaml after "execution" + control_dir = output_base / "test-skill" / "control-run-123" + treatment_dir = output_base / "test-skill" / "treatment-run-123" + control_dir.mkdir(parents=True, exist_ok=True) + treatment_dir.mkdir(parents=True, exist_ok=True) + (control_dir / "summary.yaml").write_text(yaml.dump({"mean_reward": 0.5})) + (treatment_dir / "summary.yaml").write_text(yaml.dump({"mean_reward": 0.6})) + return result + + mock_run.side_effect = create_summary + + # Mock score.py path + score_py = tmp_path / "score.py" + score_py.write_text("# mock") + + result = runner.run_pairwise( + control_config=control_config, + treatment_config=treatment_config, + output_base=output_base, + run_id="run-123", + score_py_path=score_py, + ) + + assert "control_dir" in result + assert "treatment_dir" in result + assert "control-run-123" in result["control_dir"] + assert "treatment-run-123" in result["treatment_dir"] + + @patch("subprocess.run") + def test_pairwise_regenerates_report_html_with_baseline(self, mock_run, tmp_path): + """After pairwise, call report.py --baseline so HTML includes pairwise.""" + mock_run.return_value = MagicMock(returncode=0) + + control_config = tmp_path / "eval-control.yaml" + treatment_config = tmp_path / "eval-treatment.yaml" + output_base = tmp_path / "output" + control_config.write_text(yaml.dump({"skill": "test-skill"})) + treatment_config.write_text(yaml.dump({"skill": "test-skill"})) + + score_py = tmp_path / "score.py" + score_py.write_text("# mock") + report_py = tmp_path / "report.py" + report_py.write_text("# mock") + + def create_summary(*args, **kwargs): + control_dir = output_base / "test-skill" / "control-run-123" + treatment_dir = output_base / "test-skill" / "treatment-run-123" + control_dir.mkdir(parents=True, exist_ok=True) + treatment_dir.mkdir(parents=True, exist_ok=True) + (control_dir / "summary.yaml").write_text(yaml.dump({"mean_reward": 0.5})) + (treatment_dir / "summary.yaml").write_text( + yaml.dump({"mean_reward": 0.6, "pairwise": {"wins_a": 0, "ties": 1}}) + ) + return MagicMock(returncode=0) + + mock_run.side_effect = create_summary + runner = HarborRunner(model="test-model") + + # HarborRunner._execute also uses subprocess; stub it so only pairwise+report run. + with patch.object(HarborRunner, "_execute", return_value=0): + # Still need summary.yaml present before pairwise — create them up front. + control_dir = output_base / "test-skill" / "control-run-123" + treatment_dir = output_base / "test-skill" / "treatment-run-123" + control_dir.mkdir(parents=True, exist_ok=True) + treatment_dir.mkdir(parents=True, exist_ok=True) + (control_dir / "summary.yaml").write_text(yaml.dump({"mean_reward": 0.5})) + (treatment_dir / "summary.yaml").write_text(yaml.dump({"mean_reward": 0.6})) + + runner.run_pairwise( + control_config=control_config, + treatment_config=treatment_config, + output_base=output_base, + run_id="run-123", + score_py_path=score_py, + ) + + cmds = [" ".join(str(x) for x in call.args[0]) for call in mock_run.call_args_list] + report_cmds = [c for c in cmds if "report.py" in c] + assert report_cmds, f"expected report.py invoke, got: {cmds}" + assert "--baseline" in report_cmds[0] + assert "control-run-123" in report_cmds[0] + assert "treatment-run-123" in report_cmds[0] + + def test_pairwise_fails_fast_on_missing_summary(self, tmp_path): + """run_pairwise should fail fast if control produces no summary.yaml.""" + control_config = tmp_path / "eval-control.yaml" + treatment_config = tmp_path / "eval-treatment.yaml" + output_base = tmp_path / "output" + + control_config.write_text(yaml.dump({"skill": "test-skill"})) + treatment_config.write_text(yaml.dump({"skill": "test-skill"})) + + runner = HarborRunner(model="test-model") + + with patch("subprocess.run", return_value=MagicMock(returncode=0)): + try: + runner.run_pairwise( + control_config=control_config, + treatment_config=treatment_config, + output_base=output_base, + run_id="run-123", + ) + assert False, "Should have raised RunnerError" + except RunnerError as e: + assert "summary.yaml" in str(e).lower() + + +class TestCLI: + """Test CLI argument parsing.""" + + def test_cli_single_missing_config_fails(self, tmp_path): + """CLI single command should fail without --config.""" + import sys + from unittest.mock import patch as mock_patch + + from scripts.run_aeh import main + + output = tmp_path / "output" + + with mock_patch.object( + sys, + "argv", + ["run_aeh.py", "single", "--output", str(output)], + ): + try: + main() + except SystemExit as e: + # argparse exits with 2 for missing required args + assert e.code == 2 + + def test_cli_pairwise_missing_configs_fails(self, tmp_path): + """CLI pairwise command should fail without control/treatment configs.""" + import sys + from unittest.mock import patch as mock_patch + + from scripts.run_aeh import main + + output = tmp_path / "output" + + with mock_patch.object( + sys, + "argv", + ["run_aeh.py", "pairwise", "--output", str(output), "--run-id", "test"], + ): + try: + main() + except SystemExit as e: + # argparse exits with 2 for missing required args + assert e.code == 2 + + def test_cli_harbor_requires_model(self, tmp_path): + """CLI harbor runner should fail without --model.""" + import sys + from unittest.mock import patch as mock_patch + + from scripts.run_aeh import main + + config = tmp_path / "eval.yaml" + config.write_text(yaml.dump({"skill": "test"})) + output = tmp_path / "output" + + with mock_patch.object( + sys, + "argv", + [ + "run_aeh.py", + "single", + "--runner", + "harbor", + "--config", + str(config), + "--output", + str(output), + ], + ): + # Should fail because harbor requires --model + exit_code = main() + assert exit_code == 1 # RunnerError exits with 1 + + +class TestRunnerExtensibility: + """Test that new runners can be added easily.""" + + def test_custom_runner_registration(self): + """Custom runners can be added to RUNNERS dict.""" + + class CustomRunner(BaseRunner): + name = "custom" + + def _execute(self, config, output, **opts): + return 0 + + # Register + RUNNERS["custom"] = CustomRunner + + try: + runner = get_runner("custom") + assert isinstance(runner, CustomRunner) + assert runner.name == "custom" + finally: + # Cleanup + del RUNNERS["custom"] diff --git a/tests/test_validate_aeh.py b/tests/test_validate_aeh.py new file mode 100644 index 0000000..fb1c8f6 --- /dev/null +++ b/tests/test_validate_aeh.py @@ -0,0 +1,304 @@ +"""Tests for AEH (Agent-Eval-Harness) validation in scripts/validate.py.""" + +from pathlib import Path + +import yaml + +from abevalflow.schemas import EvalEngine +from scripts.validate import validate_submission + +VALID_METADATA = { + "schema_version": "1.0", + "name": "my-aeh-eval", + "eval_engine": "aeh", +} + +VALID_EVAL_YAML = { + "models": {"skill": "claude-sonnet-4-5", "judge": "claude-opus-4-6"}, + "judges": [ + {"name": "correctness", "type": "llm", "prompt": "Is this correct?"}, + ], + "thresholds": {"correctness": 0.7}, + # Pairwise score.py needs case artifact paths under cases/// + "outputs": [{"path": "output"}], +} + +VALID_INPUT_YAML = { + "prompt": "Write a hello world program", + "context": {"language": "python"}, +} + + +def create_aeh_submission( + tmp_path: Path, + *, + include_metadata: bool = True, + include_eval_yaml: bool = True, + include_cases: bool = True, + eval_yaml_content: dict | None = None, + metadata_content: dict | None = None, + case_names: list[str] | None = None, +) -> Path: + """Create a minimal AEH submission directory.""" + if include_metadata: + content = metadata_content or VALID_METADATA + (tmp_path / "metadata.yaml").write_text(yaml.dump(content)) + + if include_eval_yaml: + content = eval_yaml_content or VALID_EVAL_YAML + (tmp_path / "eval.yaml").write_text(yaml.dump(content)) + + if include_cases: + cases_dir = tmp_path / "cases" + cases_dir.mkdir() + for case_name in case_names or ["case-001"]: + case_dir = cases_dir / case_name + case_dir.mkdir() + (case_dir / "input.yaml").write_text(yaml.dump(VALID_INPUT_YAML)) + (case_dir / "annotations.yaml").write_text(yaml.dump({"expected": "OK"})) + + return tmp_path + + +class TestAEHValidation: + """Tests for AEH submission validation.""" + + def test_valid_aeh_submission(self, tmp_path): + """A valid AEH submission should pass validation.""" + submission = create_aeh_submission(tmp_path) + errors = validate_submission(submission, eval_engine=EvalEngine.AEH) + assert errors == [] + + def test_skill_must_match_metadata_name(self, tmp_path): + """skill field must match metadata.name so analyze finds report.json.""" + eval_yaml = {**VALID_EVAL_YAML, "skill": "other-name"} + create_aeh_submission(tmp_path, eval_yaml_content=eval_yaml) + errors = validate_submission(tmp_path, eval_engine=EvalEngine.AEH) + assert any("must match metadata.name" in e for e in errors) + + def test_missing_eval_yaml(self, tmp_path): + """eval.yaml is required for AEH.""" + submission = create_aeh_submission(tmp_path, include_eval_yaml=False) + errors = validate_submission(submission, eval_engine=EvalEngine.AEH) + assert any("eval.yaml is required" in e for e in errors) + + def test_missing_cases_dir(self, tmp_path): + """cases/ directory is required for AEH.""" + submission = create_aeh_submission(tmp_path, include_cases=False) + errors = validate_submission(submission, eval_engine=EvalEngine.AEH) + assert any("cases/ directory is required" in e for e in errors) + + def test_empty_cases_dir(self, tmp_path): + """cases/ must contain at least one case.""" + submission = create_aeh_submission(tmp_path, include_cases=False) + (submission / "cases").mkdir() # Empty cases dir + errors = validate_submission(submission, eval_engine=EvalEngine.AEH) + assert any("at least one case directory" in e for e in errors) + + def test_case_missing_input_yaml(self, tmp_path): + """Each case must have input.yaml.""" + submission = create_aeh_submission(tmp_path, include_cases=False) + cases_dir = submission / "cases" + cases_dir.mkdir() + (cases_dir / "case-001").mkdir() + # No input.yaml in case-001 + errors = validate_submission(submission, eval_engine=EvalEngine.AEH) + assert any("missing required input.yaml" in e for e in errors) + + def test_eval_yaml_missing_models(self, tmp_path): + """eval.yaml must have models section.""" + bad_eval = {"judges": [{"name": "test"}]} + submission = create_aeh_submission(tmp_path, eval_yaml_content=bad_eval) + errors = validate_submission(submission, eval_engine=EvalEngine.AEH) + assert any("'models' section is required" in e for e in errors) + + def test_eval_yaml_missing_models_skill(self, tmp_path): + """eval.yaml models must have skill key.""" + bad_eval = {"models": {"judge": "claude-opus-4-6"}, "judges": []} + submission = create_aeh_submission(tmp_path, eval_yaml_content=bad_eval) + errors = validate_submission(submission, eval_engine=EvalEngine.AEH) + assert any("'models.skill' is required" in e for e in errors) + + def test_eval_yaml_missing_judges(self, tmp_path): + """eval.yaml must have judges section.""" + bad_eval = {"models": {"skill": "claude-sonnet-4-5"}} + submission = create_aeh_submission(tmp_path, eval_yaml_content=bad_eval) + errors = validate_submission(submission, eval_engine=EvalEngine.AEH) + assert any("'judges' section is required" in e for e in errors) + + def test_eval_yaml_models_string_format(self, tmp_path): + """eval.yaml can have models as a string (shorthand).""" + eval_yaml = {"models": "claude-sonnet-4-5", "judges": []} + submission = create_aeh_submission(tmp_path, eval_yaml_content=eval_yaml) + errors = validate_submission(submission, eval_engine=EvalEngine.AEH) + # Should not error on models being a string + assert not any("models" in e.lower() for e in errors) + + def test_eval_yaml_invalid_yaml(self, tmp_path): + """Invalid YAML in eval.yaml should be caught.""" + submission = create_aeh_submission(tmp_path, include_eval_yaml=False) + (submission / "eval.yaml").write_text("invalid: yaml: content: [") + errors = validate_submission(submission, eval_engine=EvalEngine.AEH) + assert any("not valid YAML" in e for e in errors) + + def test_case_invalid_input_yaml(self, tmp_path): + """Invalid YAML in input.yaml should be caught.""" + submission = create_aeh_submission(tmp_path, include_cases=False) + cases_dir = submission / "cases" + cases_dir.mkdir() + case_dir = cases_dir / "case-001" + case_dir.mkdir() + (case_dir / "input.yaml").write_text("invalid: [yaml") + errors = validate_submission(submission, eval_engine=EvalEngine.AEH) + assert any("invalid YAML" in e for e in errors) + + def test_multiple_cases(self, tmp_path): + """Multiple cases should all be validated.""" + submission = create_aeh_submission(tmp_path, case_names=["case-001", "case-002", "case-003"]) + errors = validate_submission(submission, eval_engine=EvalEngine.AEH) + assert errors == [] + + def test_aeh_does_not_require_skills_dir(self, tmp_path): + """AEH submissions don't require skills/ directory.""" + submission = create_aeh_submission(tmp_path) + # No skills/ directory + errors = validate_submission(submission, eval_engine=EvalEngine.AEH) + assert not any("skills/" in e for e in errors) + + def test_aeh_does_not_require_instruction_md(self, tmp_path): + """AEH submissions don't require instruction.md.""" + submission = create_aeh_submission(tmp_path) + errors = validate_submission(submission, eval_engine=EvalEngine.AEH) + assert not any("instruction.md" in e for e in errors) + + def test_aeh_does_not_require_test_outputs(self, tmp_path): + """AEH submissions don't require tests/test_outputs.py.""" + submission = create_aeh_submission(tmp_path) + errors = validate_submission(submission, eval_engine=EvalEngine.AEH) + assert not any("test_outputs.py" in e for e in errors) + + +class TestAEHPairwiseValidation: + """Tests for AEH pairwise mode validation.""" + + def test_pairwise_valid_submission(self, tmp_path): + """A valid pairwise submission has both control and treatment configs.""" + submission = create_aeh_submission(tmp_path, include_eval_yaml=False) + # Create both config files + (submission / "eval-control.yaml").write_text(yaml.dump(VALID_EVAL_YAML)) + (submission / "eval-treatment.yaml").write_text(yaml.dump(VALID_EVAL_YAML)) + + errors = validate_submission( + submission, + eval_engine=EvalEngine.AEH, + aeh_mode="pairwise", + ) + assert errors == [] + + def test_pairwise_missing_control_config(self, tmp_path): + """Pairwise mode requires eval-control.yaml.""" + submission = create_aeh_submission(tmp_path, include_eval_yaml=False) + (submission / "eval-treatment.yaml").write_text(yaml.dump(VALID_EVAL_YAML)) + + errors = validate_submission( + submission, + eval_engine=EvalEngine.AEH, + aeh_mode="pairwise", + ) + assert any("missing eval-control.yaml" in e for e in errors) + + def test_pairwise_missing_treatment_config(self, tmp_path): + """Pairwise mode requires eval-treatment.yaml.""" + submission = create_aeh_submission(tmp_path, include_eval_yaml=False) + (submission / "eval-control.yaml").write_text(yaml.dump(VALID_EVAL_YAML)) + + errors = validate_submission( + submission, + eval_engine=EvalEngine.AEH, + aeh_mode="pairwise", + ) + assert any("missing eval-treatment.yaml" in e for e in errors) + + def test_pairwise_invalid_control_config(self, tmp_path): + """Control config must have valid AEH structure.""" + submission = create_aeh_submission(tmp_path, include_eval_yaml=False) + # Missing models.skill + bad_config = {"models": {"judge": "claude-opus"}, "judges": []} + (submission / "eval-control.yaml").write_text(yaml.dump(bad_config)) + (submission / "eval-treatment.yaml").write_text(yaml.dump(VALID_EVAL_YAML)) + + errors = validate_submission( + submission, + eval_engine=EvalEngine.AEH, + aeh_mode="pairwise", + ) + assert any("eval-control.yaml" in e and "models.skill" in e for e in errors) + + def test_pairwise_requires_outputs(self, tmp_path): + """Treatment config must declare outputs: for pairwise artifact comparison.""" + submission = create_aeh_submission(tmp_path, include_eval_yaml=False) + no_outputs = { + "models": {"skill": "claude-sonnet", "judge": "claude-sonnet"}, + "judges": [{"name": "pairwise", "type": "llm", "prompt": "Who wins?"}], + } + (submission / "eval-control.yaml").write_text(yaml.dump(VALID_EVAL_YAML)) + (submission / "eval-treatment.yaml").write_text(yaml.dump(no_outputs)) + errors = validate_submission( + submission, + eval_engine=EvalEngine.AEH, + aeh_mode="pairwise", + ) + assert any("outputs" in e for e in errors) + + def test_pairwise_custom_config_filenames(self, tmp_path): + """Custom config filenames should be respected.""" + submission = create_aeh_submission(tmp_path, include_eval_yaml=False) + (submission / "baseline.yaml").write_text(yaml.dump(VALID_EVAL_YAML)) + (submission / "variant.yaml").write_text(yaml.dump(VALID_EVAL_YAML)) + + errors = validate_submission( + submission, + eval_engine=EvalEngine.AEH, + aeh_mode="pairwise", + aeh_control_config="baseline.yaml", + aeh_treatment_config="variant.yaml", + ) + assert errors == [] + + def test_pairwise_still_requires_cases(self, tmp_path): + """Pairwise mode still requires cases/ directory.""" + submission = create_aeh_submission(tmp_path, include_eval_yaml=False, include_cases=False) + (submission / "eval-control.yaml").write_text(yaml.dump(VALID_EVAL_YAML)) + (submission / "eval-treatment.yaml").write_text(yaml.dump(VALID_EVAL_YAML)) + + errors = validate_submission( + submission, + eval_engine=EvalEngine.AEH, + aeh_mode="pairwise", + ) + assert any("cases/ directory is required" in e for e in errors) + + def test_single_mode_ignores_pairwise_configs(self, tmp_path): + """Single mode should look for eval.yaml, not pairwise configs.""" + submission = create_aeh_submission(tmp_path, include_eval_yaml=True) + # Single mode should work with just eval.yaml + errors = validate_submission( + submission, + eval_engine=EvalEngine.AEH, + aeh_mode="single", + ) + assert errors == [] + + +class TestAEHEvalEngineEnum: + """Tests for AEH in EvalEngine enum.""" + + def test_aeh_in_enum(self): + assert EvalEngine.AEH == "aeh" + + def test_validate_with_aeh_engine(self, tmp_path): + """validate_submission accepts EvalEngine.AEH.""" + submission = create_aeh_submission(tmp_path) + # Should not raise + errors = validate_submission(submission, eval_engine=EvalEngine.AEH) + assert isinstance(errors, list) diff --git a/tests/test_validate_aeh_plugin_dirs.py b/tests/test_validate_aeh_plugin_dirs.py new file mode 100644 index 0000000..3fc349a --- /dev/null +++ b/tests/test_validate_aeh_plugin_dirs.py @@ -0,0 +1,32 @@ +"""AEH validation requires plugin_dirs skills when declared.""" + +from pathlib import Path + +from abevalflow.schemas import EvalEngine +from scripts.validate import validate_submission +from tests.test_validate_aeh import VALID_EVAL_YAML, create_aeh_submission + + +def test_aeh_plugin_dirs_missing_skills_fails(tmp_path: Path): + eval_yaml = { + **VALID_EVAL_YAML, + "skill": "demo-skill", + "runner": {"type": "claude-code", "plugin_dirs": ["skills"]}, + } + create_aeh_submission(tmp_path, eval_yaml_content=eval_yaml) + errors = validate_submission(tmp_path, eval_engine=EvalEngine.AEH) + assert any("plugin_dirs" in e and "missing" in e for e in errors) + + +def test_aeh_plugin_dirs_with_named_skill_passes(tmp_path: Path): + eval_yaml = { + **VALID_EVAL_YAML, + "skill": "demo-skill", + "runner": {"type": "claude-code", "plugin_dirs": ["skills"]}, + } + create_aeh_submission(tmp_path, eval_yaml_content=eval_yaml) + skill = tmp_path / "skills" / "demo-skill" + skill.mkdir(parents=True) + (skill / "SKILL.md").write_text("---\nname: demo-skill\n---\n") + errors = validate_submission(tmp_path, eval_engine=EvalEngine.AEH) + assert not any("plugin_dirs" in e or "SKILL.md" in e for e in errors) From 3afe0da4a5acd1c9acbc797c58ea9093e41c1a37 Mon Sep 17 00:00:00 2001 From: gziv Date: Mon, 20 Jul 2026 09:55:32 +0300 Subject: [PATCH 2/5] feat(aeh): wire AEH into Tekton evaluate and monitoring Integrate AEH single/pairwise steps into evaluate, and extend prepare, test, analyze, store, publish, and scorecard paths for the new engine. Co-authored-by: Cursor --- config/rbac.yaml | 4 + pipeline/pipelines/ci-pipeline-dev.yaml | 52 +- pipeline/pipelines/ci-pipeline.yaml | 52 +- pipeline/tasks/components/validate.yaml | 20 +- pipeline/tasks/phases/evaluate.yaml | 490 +++++++++++++++++- pipeline/tasks/phases/prepare.yaml | 17 +- pipeline/tasks/phases/test.yaml | 65 ++- .../post/analyze-and-check-degradation.yaml | 8 + pipeline/tasks/post/cleanup_pvc.yaml | 10 +- pipeline/tasks/post/store.yaml | 8 +- scripts/aggregate_scorecard.py | 2 +- scripts/analyze.py | 27 +- scripts/publish.py | 242 ++++++++- scripts/save_harbor_debug.py | 212 ++++++++ scripts/test_quality_review.py | 263 ++++++++-- tests/test_publish.py | 183 +++++++ tests/test_quality_review_aeh.py | 103 ++++ 17 files changed, 1670 insertions(+), 88 deletions(-) create mode 100644 scripts/save_harbor_debug.py create mode 100644 tests/test_quality_review_aeh.py diff --git a/config/rbac.yaml b/config/rbac.yaml index 1d8feef..8aec87f 100644 --- a/config/rbac.yaml +++ b/config/rbac.yaml @@ -26,9 +26,13 @@ metadata: name: pipeline-trial-manager namespace: ab-eval-flow rules: + # Pod management for Harbor trial pods and AEH KubernetesEnvironment + # AEH uses: create_namespaced_pod, read_namespaced_pod, delete_namespaced_pod - apiGroups: [""] resources: [pods] verbs: [create, get, list, watch, delete] + # Pod exec for Harbor/AEH to run commands in trial pods + # AEH uses: connect_get_namespaced_pod_exec - apiGroups: [""] resources: [pods/exec] verbs: [create] diff --git a/pipeline/pipelines/ci-pipeline-dev.yaml b/pipeline/pipelines/ci-pipeline-dev.yaml index 439e16c..bbb2318 100644 --- a/pipeline/pipelines/ci-pipeline-dev.yaml +++ b/pipeline/pipelines/ci-pipeline-dev.yaml @@ -32,7 +32,7 @@ spec: - name: eval-engine type: string default: "harbor" - description: Evaluation engine (harbor, ase, mcpchecker, both) + description: Evaluation engine (harbor, ase, mcpchecker, a2a, aeh, both) # Harbor params - name: harbor-fork-url type: string @@ -138,6 +138,35 @@ spec: - name: agent-chart-url type: string default: "https://github.com/RHEcosystemAppEng/google-lightspeed-agent.git" + # AEH (Agent-Eval-Harness) params + - name: aeh-model-override + type: string + default: "" + description: Override models.skill from eval.yaml (empty = use eval.yaml value) + - name: aeh-judge-model-override + type: string + default: "" + description: Override models.judge from eval.yaml (empty = use eval.yaml value) + - name: aeh-mode + type: string + default: "single" + description: AEH evaluation mode (single, pairwise) + - name: aeh-control-config + type: string + default: "eval-control.yaml" + description: Control variant eval.yaml filename for pairwise mode (relative to submission) + - name: aeh-treatment-config + type: string + default: "eval-treatment.yaml" + description: Treatment variant eval.yaml filename for pairwise mode (relative to submission) + - name: aeh-image + type: string + default: "quay.io/ecosystem-appeng/agent-eval-harness:v1.0.3" + description: Container image for AEH trial pods (required for harbor.run) + - name: aeh-runner + type: string + default: "harbor" + description: AEH execution backend (harbor, vanilla) # Security scanning - name: security-scan type: string @@ -219,6 +248,12 @@ spec: value: $(params.agent-type) - name: max-generation-retries value: $(params.max-generation-retries) + - name: aeh-mode + value: $(params.aeh-mode) + - name: aeh-control-config + value: $(params.aeh-control-config) + - name: aeh-treatment-config + value: $(params.aeh-treatment-config) workspaces: - name: source workspace: shared-workspace @@ -343,6 +378,21 @@ spec: value: $(params.agent-release-name) - name: agent-chart-url value: $(params.agent-chart-url) + # AEH params + - name: aeh-model-override + value: $(params.aeh-model-override) + - name: aeh-judge-model-override + value: $(params.aeh-judge-model-override) + - name: aeh-mode + value: $(params.aeh-mode) + - name: aeh-control-config + value: $(params.aeh-control-config) + - name: aeh-treatment-config + value: $(params.aeh-treatment-config) + - name: aeh-image + value: $(params.aeh-image) + - name: aeh-runner + value: $(params.aeh-runner) workspaces: - name: source workspace: shared-workspace diff --git a/pipeline/pipelines/ci-pipeline.yaml b/pipeline/pipelines/ci-pipeline.yaml index a119c46..a841250 100644 --- a/pipeline/pipelines/ci-pipeline.yaml +++ b/pipeline/pipelines/ci-pipeline.yaml @@ -32,7 +32,7 @@ spec: - name: eval-engine type: string default: "harbor" - description: Evaluation engine (harbor, ase, mcpchecker, both) + description: Evaluation engine (harbor, ase, mcpchecker, a2a, aeh, both) # Harbor params - name: harbor-fork-url type: string @@ -138,6 +138,35 @@ spec: - name: agent-chart-url type: string default: "https://github.com/RHEcosystemAppEng/google-lightspeed-agent.git" + # AEH (Agent-Eval-Harness) params + - name: aeh-model-override + type: string + default: "" + description: Override models.skill from eval.yaml (empty = use eval.yaml value) + - name: aeh-judge-model-override + type: string + default: "" + description: Override models.judge from eval.yaml (empty = use eval.yaml value) + - name: aeh-mode + type: string + default: "single" + description: AEH evaluation mode (single, pairwise) + - name: aeh-control-config + type: string + default: "eval-control.yaml" + description: Control variant eval.yaml filename for pairwise mode (relative to submission) + - name: aeh-treatment-config + type: string + default: "eval-treatment.yaml" + description: Treatment variant eval.yaml filename for pairwise mode (relative to submission) + - name: aeh-image + type: string + default: "quay.io/ecosystem-appeng/agent-eval-harness:v1.0.0" + description: Container image for AEH trial pods (required for harbor.run) + - name: aeh-runner + type: string + default: "harbor" + description: AEH execution backend (harbor, vanilla) # Security scanning - name: security-scan type: string @@ -219,6 +248,12 @@ spec: value: $(params.agent-type) - name: max-generation-retries value: $(params.max-generation-retries) + - name: aeh-mode + value: $(params.aeh-mode) + - name: aeh-control-config + value: $(params.aeh-control-config) + - name: aeh-treatment-config + value: $(params.aeh-treatment-config) workspaces: - name: source workspace: shared-workspace @@ -343,6 +378,21 @@ spec: value: $(params.agent-release-name) - name: agent-chart-url value: $(params.agent-chart-url) + # AEH params + - name: aeh-model-override + value: $(params.aeh-model-override) + - name: aeh-judge-model-override + value: $(params.aeh-judge-model-override) + - name: aeh-mode + value: $(params.aeh-mode) + - name: aeh-control-config + value: $(params.aeh-control-config) + - name: aeh-treatment-config + value: $(params.aeh-treatment-config) + - name: aeh-image + value: $(params.aeh-image) + - name: aeh-runner + value: $(params.aeh-runner) workspaces: - name: source workspace: shared-workspace diff --git a/pipeline/tasks/components/validate.yaml b/pipeline/tasks/components/validate.yaml index b26e77a..e946471 100644 --- a/pipeline/tasks/components/validate.yaml +++ b/pipeline/tasks/components/validate.yaml @@ -28,9 +28,22 @@ spec: type: string default: "harbor" description: >- - Evaluation engine (harbor, ase, both). Controls which validation + Evaluation engine (harbor, ase, both, aeh). Controls which validation checks are run — Harbor checks require instruction.md and tests/, - ASE checks require evals/evals.json and SKILL.md frontmatter. + ASE checks require evals/evals.json and SKILL.md frontmatter, + AEH checks require eval.yaml and cases/. + - name: aeh-mode + type: string + default: "single" + description: AEH evaluation mode (single, pairwise) + - name: aeh-control-config + type: string + default: "eval-control.yaml" + description: Control variant eval.yaml filename for AEH pairwise mode + - name: aeh-treatment-config + type: string + default: "eval-treatment.yaml" + description: Treatment variant eval.yaml filename for AEH pairwise mode - name: pipeline-run-name type: string description: Name of the PipelineRun (for building report prefix) @@ -102,6 +115,9 @@ spec: # Run validation and capture output python scripts/validate.py "$SUBMISSION_PATH" \ --eval-engine "$(params.eval-engine)" \ + --aeh-mode "$(params.aeh-mode)" \ + --aeh-control-config "$(params.aeh-control-config)" \ + --aeh-treatment-config "$(params.aeh-treatment-config)" \ | tee /tmp/validation-output.json # Write to Tekton result diff --git a/pipeline/tasks/phases/evaluate.yaml b/pipeline/tasks/phases/evaluate.yaml index aa27f27..0cd3b26 100644 --- a/pipeline/tasks/phases/evaluate.yaml +++ b/pipeline/tasks/phases/evaluate.yaml @@ -10,7 +10,8 @@ spec: description: >- Evaluation phase: dispatches to the appropriate evaluation engine based on eval-engine parameter. Handles Harbor (scaffold+build+eval), ASE (LLM-as-judge), - MCPChecker (MCP server testing), and A2A (agent evaluation) in a single composite task. + MCPChecker (MCP server testing), A2A (agent evaluation), and AEH (agent-eval-harness) + in a single composite task. stepTemplate: computeResources: requests: @@ -22,7 +23,7 @@ spec: params: - name: eval-engine type: string - description: Evaluation engine (harbor, ase, mcpchecker, a2a, both) + description: Evaluation engine (harbor, ase, mcpchecker, a2a, aeh, both) - name: submission-dir type: string description: Submission directory name under submissions/ @@ -129,6 +130,35 @@ spec: type: string default: "https://github.com/RHEcosystemAppEng/google-lightspeed-agent.git" description: Agent Helm chart Git URL + # AEH (Agent-Eval-Harness) params + - name: aeh-model-override + type: string + default: "" + description: Override models.skill from eval.yaml + - name: aeh-judge-model-override + type: string + default: "" + description: Override models.judge from eval.yaml + - name: aeh-mode + type: string + default: "single" + description: AEH evaluation mode (single, pairwise) + - name: aeh-control-config + type: string + default: "eval-control.yaml" + description: Control variant eval.yaml filename for pairwise mode + - name: aeh-treatment-config + type: string + default: "eval-treatment.yaml" + description: Treatment variant eval.yaml filename for pairwise mode + - name: aeh-image + type: string + default: "quay.io/ecosystem-appeng/agent-eval-harness:v1.0.3" + description: Container image for AEH trial pods (required for harbor.run) + - name: aeh-runner + type: string + default: "harbor" + description: AEH execution backend (harbor, vanilla) workspaces: - name: source description: Workspace containing the cloned submissions repository @@ -215,13 +245,16 @@ spec: fi echo "=== EVALUATE PHASE: Harbor Build Treatment (crane) ===" - + # Install crane (lightweight tool that appends layers without extracting) curl -sL https://github.com/google/go-containerregistry/releases/download/v0.20.1/go-containerregistry_Linux_x86_64.tar.gz | tar xz -C /tmp crane - + + # Get current namespace from service account + CURRENT_NS=$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace) + LOCAL_BASE_IMAGE="local-registry.ab-eval-flow.svc.cluster.local:5000/eval-base:latest" REGISTRY="$(params.registry-url)" - NS="$(params.registry-namespace)" + NS="${CURRENT_NS}" NAME="$(params.submission-name)" SHA="$(echo '$(params.commit-sha)' | tr '/' '-')" IMAGE_TAG="${REGISTRY}/${NS}/${NAME}:treatment-${SHA}" @@ -265,13 +298,16 @@ spec: fi echo "=== EVALUATE PHASE: Harbor Build Control (crane) ===" - + # Install crane curl -sL https://github.com/google/go-containerregistry/releases/download/v0.20.1/go-containerregistry_Linux_x86_64.tar.gz | tar xz -C /tmp crane - + + # Get current namespace from service account + CURRENT_NS=$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace) + LOCAL_BASE_IMAGE="local-registry.ab-eval-flow.svc.cluster.local:5000/eval-base:latest" REGISTRY="$(params.registry-url)" - NS="$(params.registry-namespace)" + NS="${CURRENT_NS}" NAME="$(params.submission-name)" SHA="$(echo '$(params.commit-sha)' | tr '/' '-')" IMAGE_TAG="${REGISTRY}/${NS}/${NAME}:control-${SHA}" @@ -1135,7 +1171,7 @@ spec: return sum(rewards) / len(rewards) if rewards else 0.0 mean_reward = compute_mean_reward(results_dir) - rec = "pass" if mean_reward >= 0.5 else "fail" + rec = "pass" if (mean_reward is not None and mean_reward >= 0.5) else "fail" output_dir.mkdir(parents=True, exist_ok=True) (output_dir / "treatment-reward").write_text(f"{mean_reward:.4f}") @@ -1146,7 +1182,429 @@ spec: print(f"Recommendation: {rec}") COMPUTE_REWARDS - # Step 12: Finalize results + # Step 12: AEH - Agent-Eval-Harness Evaluation (single mode) + # Depends on: prepare (pipeline clone), validate. Skipped unless eval-engine=aeh + # and aeh-mode=single. Writes run dirs under reports// for AEH layout, + # but report.json under reports// for analyze/scorecard. + - name: aeh-eval + when: + - input: "$(params.eval-engine)" + operator: in + values: ["aeh"] + image: $(params.aeh-image) + timeout: "2h" + computeResources: + requests: + cpu: "500m" + memory: "512Mi" + limits: + cpu: "1000m" + memory: "1Gi" + env: + - name: AGENT_EVAL_K8S_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: AGENT_EVAL_K8S_CREDENTIALS_SECRET + value: llm-credentials + - name: AGENT_EVAL_RUNS_DIR + value: $(workspaces.source.path)/reports + # Forwarded into AEH trial pods via agent_eval.harbor.kubernetes._FORWARD_ENV + # so Claude Code uses in-cluster LiteLLM (mock key is fine with the proxy). + - name: ANTHROPIC_BASE_URL + value: "$(params.llm-api-base)" + - name: ANTHROPIC_API_KEY + value: "$(params.llm-api-key)" + # Set to "1" to keep Harbor trial pods for oc exec diagnostics. + - name: AGENT_EVAL_K8S_KEEP_RUN + value: "0" + script: | + #!/usr/bin/env bash + set -euo pipefail + + EVAL_ENGINE="$(params.eval-engine)" + AEH_MODE="$(params.aeh-mode)" + if [ "$EVAL_ENGINE" != "aeh" ] || [ "$AEH_MODE" != "single" ]; then + echo "Skipping AEH single eval (eval-engine=$EVAL_ENGINE, aeh-mode=$AEH_MODE)" + exit 0 + fi + + echo "=== EVALUATE PHASE: AEH Single Mode ===" + echo "AGENT_EVAL_K8S_KEEP_RUN=${AGENT_EVAL_K8S_KEEP_RUN:-unset}" + echo "ANTHROPIC_BASE_URL=${ANTHROPIC_BASE_URL:-unset}" + + SUBMISSION_DIR="$(workspaces.source.path)/submissions/$(params.submission-dir)" + SUBMISSION_NAME="$(params.submission-name)" + RUN_ID="$(params.pipeline-run-id)" + + # Skill namespaced run dir (AEH layout); report.json uses submission-name for analyze + SKILL_NAME=$(python3 -c " + import yaml + config = yaml.safe_load(open('$SUBMISSION_DIR/eval.yaml')) + print(config.get('skill', '$SUBMISSION_NAME')) + ") + + OUT_DIR="$(workspaces.source.path)/reports/${SKILL_NAME}/${RUN_ID}" + REPORT_JSON="$(workspaces.source.path)/reports/${SUBMISSION_NAME}/report.json" + + mkdir -p "$OUT_DIR" \ + "$(workspaces.source.path)/reports/${SUBMISSION_NAME}" \ + "$(workspaces.source.path)/_eval_tmp/aeh-tasks" \ + "$(workspaces.source.path)/_eval_tmp/aeh-jobs" + + # --model is required by harbor.run; use override or value from eval.yaml + MODEL="$(params.aeh-model-override)" + if [ -z "$MODEL" ]; then + MODEL=$(python3 -c " + import yaml + config = yaml.safe_load(open('$SUBMISSION_DIR/eval.yaml')) + models = config.get('models', {}) + if isinstance(models, str): + print(models) + else: + print(models.get('skill', 'claude-sonnet-4-5')) + ") + fi + + JUDGE_OVERRIDE="$(params.aeh-judge-model-override)" + + echo "Submission: $SUBMISSION_DIR" + echo "Output: $OUT_DIR" + echo "Report: $REPORT_JSON" + echo "Model: $MODEL" + echo "Judge override: ${JUDGE_OVERRIDE:-none}" + echo "Runner: $(params.aeh-runner)" + + # Run AEH evaluation via dispatcher (supports harbor/vanilla backends) + PIPELINE_DIR="$(workspaces.source.path)/_pipeline" + AEH_IMAGE="$(params.aeh-image)" + AEH_RUNNER="$(params.aeh-runner)" + + # Add pipeline dir to PYTHONPATH so abevalflow module is importable + export PYTHONPATH="$PIPELINE_DIR:${PYTHONPATH:-}" + + # Capture exit (harbor rc=1 can be threshold warning) then always aggregate + # when summary/run_result exist so analyze gets report.json. + set +e + python "$PIPELINE_DIR/scripts/run_aeh.py" single \ + --runner "$AEH_RUNNER" \ + --config "$SUBMISSION_DIR/eval.yaml" \ + --output "$OUT_DIR" \ + ${MODEL:+--model "$MODEL"} \ + ${JUDGE_OVERRIDE:+--judge-model "$JUDGE_OVERRIDE"} \ + ${AEH_IMAGE:+--image "$AEH_IMAGE"} \ + --tasks-dir "$(workspaces.source.path)/_eval_tmp/aeh-tasks" \ + --jobs-dir "$(workspaces.source.path)/_eval_tmp/aeh-jobs" + AEH_EXIT=$? + set -e + + # Aggregate results using our script + PIPELINE_DIR="$(workspaces.source.path)/_pipeline" + export PYTHONPATH="$PIPELINE_DIR" + pip install --quiet --no-cache-dir pyyaml 2>&1 | tail -1 + + AEH_THRESHOLD=$(python3 -c " +from pathlib import Path +from abevalflow.aeh_scoring import resolve_evaluation_threshold +print(resolve_evaluation_threshold(Path('$SUBMISSION_DIR') / 'metadata.yaml')) +") + echo "AEH evaluation threshold: $AEH_THRESHOLD" + + if [ -f "$OUT_DIR/summary.yaml" ] || [ -f "$OUT_DIR/run_result.json" ]; then + python "$PIPELINE_DIR/scripts/aggregate_aeh.py" "$OUT_DIR" \ + --submission-name "$SUBMISSION_NAME" \ + --threshold "$AEH_THRESHOLD" \ + --output "$REPORT_JSON" + else + echo "ERROR: AEH single produced no summary.yaml or run_result.json in $OUT_DIR" + exit 1 + fi + + # Parse results for finalize step (even when AEH_EXIT != 0) + python3 - "$OUT_DIR" "$(workspaces.source.path)/_eval_tmp" "$AEH_THRESHOLD" <<'PARSE_AEH' + import json, sys, yaml + from pathlib import Path + + out_dir = Path(sys.argv[1]) + tmp_dir = Path(sys.argv[2]) + threshold = float(sys.argv[3]) + + mean_reward = 0.0 + + # Try summary.yaml first + summary_path = out_dir / "summary.yaml" + if summary_path.exists(): + summary = yaml.safe_load(summary_path.read_text()) + mean_reward = summary.get("mean_reward", 0.0) + + # Override from run_result.json if present + run_result_path = out_dir / "run_result.json" + if run_result_path.exists(): + run_result = json.loads(run_result_path.read_text()) + if "mean_reward" in run_result: + mean_reward = run_result["mean_reward"] + + rec = "pass" if (mean_reward is not None and mean_reward >= threshold) else "fail" + + tmp_dir.mkdir(parents=True, exist_ok=True) + reward_str = f"{mean_reward:.4f}" if mean_reward is not None else "0.0000" + (tmp_dir / "treatment-reward").write_text(reward_str) + (tmp_dir / "control-reward").write_text("0.0000") + (tmp_dir / "recommendation").write_text(rec) + + print(f"AEH mean reward: {reward_str} (threshold={threshold})") + print(f"Recommendation: {rec}") + PARSE_AEH + + if [ "$AEH_EXIT" -ne 0 ]; then + echo "ERROR: AEH single evaluation exited $AEH_EXIT (report written if outputs existed)" + exit "$AEH_EXIT" + fi + + # Step 13: AEH - Pairwise A/B Evaluation (consolidated) + # Depends on: prepare, validate. Skipped unless eval-engine=aeh and aeh-mode=pairwise. + # Run dirs stay under reports// for score.py; report.json uses submission-name. + - name: aeh-pairwise + when: + - input: "$(params.eval-engine)" + operator: in + values: ["aeh"] + image: $(params.aeh-image) + timeout: "4h" + computeResources: + requests: + cpu: "500m" + memory: "512Mi" + limits: + cpu: "1000m" + memory: "1Gi" + env: + - name: AGENT_EVAL_K8S_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: AGENT_EVAL_K8S_CREDENTIALS_SECRET + value: llm-credentials + - name: AGENT_EVAL_RUNS_DIR + value: $(workspaces.source.path)/reports + # Forwarded into AEH trial pods via agent_eval.harbor.kubernetes._FORWARD_ENV + - name: ANTHROPIC_BASE_URL + value: "$(params.llm-api-base)" + - name: ANTHROPIC_API_KEY + value: "$(params.llm-api-key)" + script: | + #!/usr/bin/env bash + set -euo pipefail + + EVAL_ENGINE="$(params.eval-engine)" + AEH_MODE="$(params.aeh-mode)" + if [ "$EVAL_ENGINE" != "aeh" ] || [ "$AEH_MODE" != "pairwise" ]; then + echo "Skipping AEH pairwise (eval-engine=$EVAL_ENGINE, aeh-mode=$AEH_MODE)" + exit 0 + fi + + echo "=== EVALUATE PHASE: AEH Pairwise A/B Evaluation ===" + echo "ANTHROPIC_BASE_URL=${ANTHROPIC_BASE_URL:-unset}" + + SUBMISSION_DIR="$(workspaces.source.path)/submissions/$(params.submission-dir)" + SUBMISSION_NAME="$(params.submission-name)" + PIPELINE_DIR="$(workspaces.source.path)/_pipeline" + RUN_ID="$(params.pipeline-run-id)" + CONTROL_CONFIG="$(params.aeh-control-config)" + TREATMENT_CONFIG="$(params.aeh-treatment-config)" + REPORTS_DIR="$(workspaces.source.path)/reports" + + # Get model from treatment config or override + MODEL="$(params.aeh-model-override)" + if [ -z "$MODEL" ]; then + MODEL=$(python3 -c " + import yaml + config = yaml.safe_load(open('$SUBMISSION_DIR/$TREATMENT_CONFIG')) + models = config.get('models', {}) + if isinstance(models, str): + print(models) + else: + print(models.get('skill', 'claude-sonnet-4-5')) + ") + fi + + JUDGE_OVERRIDE="$(params.aeh-judge-model-override)" + AEH_IMAGE="$(params.aeh-image)" + AEH_RUNNER="$(params.aeh-runner)" + + # Create base tmp dir for tasks/jobs (dispatcher creates separate control/treatment subdirs) + EVAL_TMP="$(workspaces.source.path)/_eval_tmp" + mkdir -p "$EVAL_TMP" "$REPORTS_DIR/${SUBMISSION_NAME}" + + echo "Submission: $SUBMISSION_DIR" + echo "Control config: $CONTROL_CONFIG" + echo "Treatment config: $TREATMENT_CONFIG" + echo "Model: $MODEL" + echo "Runner: $AEH_RUNNER" + + # Add pipeline dir to PYTHONPATH so abevalflow module is importable + export PYTHONPATH="$PIPELINE_DIR:${PYTHONPATH:-}" + + # Run pairwise evaluation via dispatcher + # This handles: control run -> treatment run -> score.py pairwise comparison + # Dispatcher creates separate control/treatment tasks/jobs subdirs automatically + set +e + python "$PIPELINE_DIR/scripts/run_aeh.py" pairwise \ + --runner "$AEH_RUNNER" \ + --control-config "$SUBMISSION_DIR/$CONTROL_CONFIG" \ + --treatment-config "$SUBMISSION_DIR/$TREATMENT_CONFIG" \ + --output "$REPORTS_DIR" \ + --run-id "$RUN_ID" \ + --model "$MODEL" \ + ${JUDGE_OVERRIDE:+--judge-model "$JUDGE_OVERRIDE"} \ + ${AEH_IMAGE:+--image "$AEH_IMAGE"} \ + --tasks-dir "$EVAL_TMP/aeh-tasks" \ + --jobs-dir "$EVAL_TMP/aeh-jobs" \ + --judge pairwise + PAIRWISE_EXIT=$? + set -e + + # Skill namespaced run dirs (score.py); report.json uses submission-name + SKILL_NAME=$(python3 -c " + import yaml + config = yaml.safe_load(open('$SUBMISSION_DIR/$TREATMENT_CONFIG')) + print(config.get('skill', '$SUBMISSION_NAME')) + ") + + CONTROL_DIR="$REPORTS_DIR/$SKILL_NAME/control-$RUN_ID" + TREATMENT_DIR="$REPORTS_DIR/$SKILL_NAME/treatment-$RUN_ID" + REPORT_JSON="$REPORTS_DIR/${SUBMISSION_NAME}/report.json" + + # Verify outputs exist before aggregate + if [ ! -f "$CONTROL_DIR/summary.yaml" ]; then + echo "ERROR: Control run failed - no summary.yaml in $CONTROL_DIR" + exit 1 + fi + if [ ! -f "$TREATMENT_DIR/summary.yaml" ]; then + echo "ERROR: Treatment run failed - no summary.yaml in $TREATMENT_DIR" + exit 1 + fi + + # Aggregate even when PAIRWISE_EXIT != 0 so analyze gets report.json + export PYTHONPATH="$PIPELINE_DIR" + pip install --quiet --no-cache-dir pyyaml 2>&1 | tail -1 + + AEH_THRESHOLD=$(python3 -c " +from pathlib import Path +from abevalflow.aeh_scoring import resolve_evaluation_threshold +print(resolve_evaluation_threshold(Path('$SUBMISSION_DIR') / 'metadata.yaml')) +") + echo "AEH evaluation threshold: $AEH_THRESHOLD" + + python "$PIPELINE_DIR/scripts/aggregate_aeh.py" "$TREATMENT_DIR" \ + --mode pairwise \ + --control-dir "$CONTROL_DIR" \ + --submission-name "$SUBMISSION_NAME" \ + --threshold "$AEH_THRESHOLD" \ + --output "$REPORT_JSON" + + # score.py pairwise must have merged a pairwise: section into treatment summary + if ! python3 -c " + import sys, yaml + from pathlib import Path + s = yaml.safe_load(Path(sys.argv[1]).read_text()) or {} + sys.exit(0 if isinstance(s.get('pairwise'), dict) else 1) + " "$TREATMENT_DIR/summary.yaml"; then + echo "ERROR: treatment summary.yaml missing pairwise: section (LLM judge did not run)" + exit 1 + fi + echo "Pairwise section present in treatment summary.yaml" + + # Parse results for finalize step (shared pairwise_outcome helper) + python3 - "$TREATMENT_DIR" "$CONTROL_DIR" "$(workspaces.source.path)/_eval_tmp" "$AEH_THRESHOLD" <<'PARSE_PAIRWISE' + import json, sys, yaml + from pathlib import Path + + from abevalflow.aeh_scoring import pairwise_outcome + + treatment_dir = Path(sys.argv[1]) + control_dir = Path(sys.argv[2]) + tmp_dir = Path(sys.argv[3]) + threshold = float(sys.argv[4]) + + def mean_reward(run_dir: Path) -> float: + # Harbor writes mean_reward to run_result.json; summary.yaml often omits it. + for name in ("run_result.json", "summary.yaml"): + path = run_dir / name + if not path.is_file(): + continue + try: + data = json.loads(path.read_text()) if name.endswith(".json") else yaml.safe_load(path.read_text()) + except Exception: + continue + if isinstance(data, dict) and data.get("mean_reward") is not None: + return float(data["mean_reward"]) + return 0.0 + + t_reward = mean_reward(treatment_dir) + c_reward = mean_reward(control_dir) + outcome = { + "wins_a": 0, + "wins_b": 0, + "ties": 0, + "errors": 0, + "win_rate": 0.0, + "recommendation": "fail", + } + + summary_path = treatment_dir / "summary.yaml" + if summary_path.exists(): + summary = yaml.safe_load(summary_path.read_text()) or {} + pairwise = summary.get("pairwise", {}) or {} + outcome = pairwise_outcome( + pairwise.get("wins_a", 0), + pairwise.get("wins_b", 0), + pairwise.get("ties", 0), + pairwise.get("errors", 0), + threshold=threshold, + ) + + rec = outcome["recommendation"] + win_rate = outcome["win_rate"] + wins_a = outcome["wins_a"] + wins_b = outcome["wins_b"] + ties = outcome["ties"] + errors = outcome["errors"] + + tmp_dir.mkdir(parents=True, exist_ok=True) + (tmp_dir / "treatment-reward").write_text(f"{t_reward:.4f}") + (tmp_dir / "control-reward").write_text(f"{c_reward:.4f}") + (tmp_dir / "recommendation").write_text(rec) + + print(f"AEH Pairwise Results:") + print(f" Treatment mean reward: {t_reward:.4f}") + print(f" Control mean reward: {c_reward:.4f}") + print(f" Pairwise: A(wins)={wins_a} B(wins)={wins_b} ties={ties} errors={errors}") + print(f" Treatment win rate: {win_rate:.2%} (threshold={threshold})") + print(f" Recommendation: {rec}") + PARSE_PAIRWISE + + if [ "$PAIRWISE_EXIT" -ne 0 ]; then + echo "ERROR: pairwise evaluation exited $PAIRWISE_EXIT (report written if outputs existed)" + exit "$PAIRWISE_EXIT" + fi + + # Step 15: Evaluate-time MinIO debug save (disabled — store/publish handles it) + - name: save-harbor-debug + when: + - input: "$(params.eval-engine)" + operator: in + # Disabled: dual prefixes + tar uploads. Debug is uploaded at store time + # as individual files under prepare's report-prefix (see publish.py). + values: ["__disabled__"] + onError: continue + image: registry.access.redhat.com/ubi9/python-311:9.6 + script: | + #!/usr/bin/env bash + echo "Evaluate-time Harbor debug save is disabled; store task uploads debug/" + exit 0 + + # Step 16: Finalize results - name: finalize image: registry.access.redhat.com/ubi9/python-311:9.6 script: | @@ -1163,13 +1621,23 @@ spec: echo -n "$T_REWARD" > "$(results.treatment-mean-reward.path)" echo -n "$C_REWARD" > "$(results.control-mean-reward.path)" echo -n "$REC" > "$(results.recommendation.path)" + + # AEH Harbor jobs live under _eval_tmp/: + # single: aeh-jobs/YYYY-MM-DD__… + # pairwise: aeh-control-jobs-/… and aeh-treatment-jobs-/… + # Point store at _eval_tmp so publish.py can discover both layouts. + if [ "$(params.eval-engine)" = "aeh" ]; then + echo -n "$EVAL_TMP" > "$(results.results-dir.path)" + echo "AEH results-dir: $EVAL_TMP" + ls -la "$EVAL_TMP" 2>/dev/null | head -40 || true + fi echo "Treatment reward: $T_REWARD" echo "Control reward: $C_REWARD" echo "Recommendation: $REC" echo "Results: $(cat "$(results.results-dir.path)")" - # Step 13: Cleanup A2A agent deployment (always runs) + # Step 17: Cleanup A2A agent deployment (always runs) - name: a2a-cleanup image: image-registry.openshift-image-registry.svc:5000/openshift/cli:latest script: | diff --git a/pipeline/tasks/phases/prepare.yaml b/pipeline/tasks/phases/prepare.yaml index 5884103..3bd32f9 100644 --- a/pipeline/tasks/phases/prepare.yaml +++ b/pipeline/tasks/phases/prepare.yaml @@ -25,7 +25,19 @@ spec: - name: eval-engine type: string default: "harbor" - description: Evaluation engine (harbor, ase, mcpchecker, both) + description: Evaluation engine (harbor, ase, mcpchecker, aeh, both) + - name: aeh-mode + type: string + default: "single" + description: AEH evaluation mode (single, pairwise) + - name: aeh-control-config + type: string + default: "eval-control.yaml" + description: Control variant eval.yaml filename for AEH pairwise mode + - name: aeh-treatment-config + type: string + default: "eval-treatment.yaml" + description: Treatment variant eval.yaml filename for AEH pairwise mode - name: pipeline-repo-url type: string default: "https://github.com/RHEcosystemAppEng/ABEvalFlow.git" @@ -234,6 +246,9 @@ spec: python scripts/validate.py "$SUBMISSION_PATH" \ --eval-engine "$(params.eval-engine)" \ + --aeh-mode "$(params.aeh-mode)" \ + --aeh-control-config "$(params.aeh-control-config)" \ + --aeh-treatment-config "$(params.aeh-treatment-config)" \ | tee /tmp/validation-output.json cat /tmp/validation-output.json | tr -d '\n' > "$(results.validation-result.path)" diff --git a/pipeline/tasks/phases/test.yaml b/pipeline/tasks/phases/test.yaml index 411dde2..38e65ce 100644 --- a/pipeline/tasks/phases/test.yaml +++ b/pipeline/tasks/phases/test.yaml @@ -10,7 +10,7 @@ spec: description: >- Testing phase: runs security scanning (Cisco skill-scanner) and quality review checks. Results are persisted immediately to MinIO/DB after each - check completes. Skipped entirely for MCPChecker eval-engine. + check completes. Identical for harbor, ASE, and AEH; skipped for MCPChecker. params: - name: submission-dir type: string @@ -20,7 +20,7 @@ spec: description: Validated submission name from metadata.yaml - name: eval-engine type: string - description: Evaluation engine (harbor, ase, mcpchecker, both) + description: Evaluation engine (harbor, ase, mcpchecker, a2a, aeh, both) - name: report-prefix type: string description: MinIO report prefix (timestamp_name_runid) @@ -91,9 +91,9 @@ spec: EVAL_ENGINE="$(params.eval-engine)" - # MCPChecker doesn't need testing phase + # MCPChecker has no skill submission layout to scan if [ "$EVAL_ENGINE" = "mcpchecker" ]; then - echo "MCPChecker mode: skipping test phase" + echo "Skipping test phase for eval-engine=$EVAL_ENGINE" echo -n "true" > "$(results.security-passed.path)" echo -n "disabled" > "$(results.security-mode.path)" echo -n "0" > "$(results.security-findings.path)" @@ -150,7 +150,7 @@ spec: EVAL_ENGINE="$(params.eval-engine)" if [ "$EVAL_ENGINE" = "mcpchecker" ]; then - echo "MCPChecker mode: security scan skipped" + echo "Skipping security scan for eval-engine=$EVAL_ENGINE" exit 0 fi @@ -174,9 +174,23 @@ spec: SARIF_PATH="$REPORT_DIR/security-scan.sarif" mkdir -p "$REPORT_DIR" - # Handle ASE structure - if [ -d "$SUBMISSION_PATH/skills" ] && [ -f "$SUBMISSION_PATH/skills/SKILL.md" ]; then + # Point Cisco at the skill package root that contains SKILL.md: + # - ASE flat: skills/SKILL.md → scan skills/ + # - AEH nested: skills//SKILL.md → scan skills// + # skill-scanner scan expects a skill dir, not the whole submission tree. + if [ -f "$SUBMISSION_PATH/skills/SKILL.md" ]; then SUBMISSION_PATH="$SUBMISSION_PATH/skills" + else + SKILL_MD=$(find "$SUBMISSION_PATH" -name 'SKILL.md' -type f 2>/dev/null | head -1 || true) + if [ -n "$SKILL_MD" ]; then + SUBMISSION_PATH="$(dirname "$SKILL_MD")" + echo "Using nested skill package for Cisco scan: $SUBMISSION_PATH" + else + echo "WARN: SKILL.md was not found in submission; skipping Cisco security scan" + echo -n "true" > "$(results.security-passed.path)" + echo -n "0" > "$(results.security-findings.path)" + exit 0 + fi fi # Install scanner @@ -257,7 +271,8 @@ spec: if not db_url: sys.exit(0) - findings_json = None + # Column is NOT NULL — use empty list when the scanner produced no JSON. + findings_json = "[]" counts = {"total": 0, "critical": 0, "high": 0} if os.path.isfile(json_path): with open(json_path) as f: @@ -274,7 +289,7 @@ spec: conn.execute(text(""" INSERT INTO security_scans (id, pipeline_run_id, submission_name, scanner, scan_mode, passed, findings_count, critical_count, high_count, findings_json, created_at) VALUES (:id, :run, :name, 'cisco', :mode, :passed, :total, :critical, :high, :json, NOW()) - ON CONFLICT (pipeline_run_id) DO UPDATE SET findings_count=EXCLUDED.findings_count, critical_count=EXCLUDED.critical_count, high_count=EXCLUDED.high_count, passed=EXCLUDED.passed + ON CONFLICT (pipeline_run_id) DO UPDATE SET findings_count=EXCLUDED.findings_count, critical_count=EXCLUDED.critical_count, high_count=EXCLUDED.high_count, passed=EXCLUDED.passed, findings_json=EXCLUDED.findings_json """), {"id": str(uuid.uuid4()), "run": run_id, "name": sub_name, "mode": mode, "passed": passed == "true", "total": counts["total"], "critical": counts["critical"], "high": counts["high"], "json": findings_json}) conn.commit() print("DB record written") @@ -296,7 +311,7 @@ spec: EVAL_ENGINE="$(params.eval-engine)" if [ "$EVAL_ENGINE" = "mcpchecker" ]; then - echo "MCPChecker mode: SKILL.md scan skipped" + echo "Skipping SKILL.md scan for eval-engine=$EVAL_ENGINE" exit 0 fi @@ -315,9 +330,12 @@ spec: JSON_PATH="$REPORT_DIR/skillmd-security-scan.json" mkdir -p "$REPORT_DIR" - if [ -d "$SUBMISSION_PATH/skills" ] && [ -f "$SUBMISSION_PATH/skills/SKILL.md" ]; then - SUBMISSION_PATH="$SUBMISSION_PATH/skills" + SKILL_MD=$(find "$SUBMISSION_PATH" -name 'SKILL.md' -type f 2>/dev/null | head -1 || true) + if [ -z "$SKILL_MD" ]; then + echo "WARN: SKILL.md was not found in submission; skipping SKILL.md security scan" + exit 0 fi + echo "Found SKILL.md at: $SKILL_MD" PIPELINE_DIR="$(workspaces.source.path)/_pipeline" pip install --quiet --no-cache-dir pydantic pyyaml openai 2>&1 | tail -3 @@ -363,6 +381,13 @@ spec: WORKSPACE_ROOT="$(workspaces.source.path)" JSON_PATH="$WORKSPACE_ROOT/skillmd-quality-scan.json" + SKILL_MD=$(find "$SUBMISSION_PATH" -name 'SKILL.md' -type f 2>/dev/null | head -1 || true) + if [ -z "$SKILL_MD" ]; then + echo "WARN: SKILL.md was not found in submission; skipping SKILL.md quality scan" + exit 0 + fi + echo "Found SKILL.md at: $SKILL_MD" + PIPELINE_DIR="$(workspaces.source.path)/_pipeline" pip install --quiet --no-cache-dir pydantic pyyaml 2>&1 | tail -3 export PYTHONPATH="$PIPELINE_DIR" @@ -397,7 +422,7 @@ spec: SUBMISSION_SKIP="$(params.submission-skip-quality-review)" if [ "$EVAL_ENGINE" = "mcpchecker" ]; then - echo "MCPChecker mode: quality review skipped" + echo "Skipping quality review for eval-engine=$EVAL_ENGINE" exit 0 fi @@ -424,10 +449,16 @@ spec: # Save to workspace for later use cp /tmp/review-output.json "$(workspaces.source.path)/_ai_review.json" 2>/dev/null || true - PASSED=$(python3 -c "import json; print(str(json.load(open('/tmp/review-output.json')).get('passed', False)).lower())" 2>/dev/null || echo "true") - echo -n "$PASSED" > "$(results.quality-passed.path)" - - echo "Quality review: passed=$PASSED" + REC=$(python3 -c "import json; print(json.load(open('/tmp/review-output.json')).get('recommendation', 'warn'))" 2>/dev/null || echo "warn") + # AEH quality review is advisory: never fail the test phase on recommendation. + if [ "$EVAL_ENGINE" = "aeh" ]; then + echo -n "true" > "$(results.quality-passed.path)" + echo "Quality review (AEH advisory): recommendation=$REC, quality-passed=true" + else + PASSED=$(python3 -c "import json; print(str(json.load(open('/tmp/review-output.json')).get('passed', False)).lower())" 2>/dev/null || echo "true") + echo -n "$PASSED" > "$(results.quality-passed.path)" + echo "Quality review: passed=$PASSED recommendation=$REC" + fi # Step 6: Finalize results - name: finalize diff --git a/pipeline/tasks/post/analyze-and-check-degradation.yaml b/pipeline/tasks/post/analyze-and-check-degradation.yaml index aa4ea50..6e39abe 100644 --- a/pipeline/tasks/post/analyze-and-check-degradation.yaml +++ b/pipeline/tasks/post/analyze-and-check-degradation.yaml @@ -155,6 +155,14 @@ spec: fi echo "Found ASE report at $REPORT_DIR" + elif [ "$EVAL_ENGINE" = "aeh" ]; then + echo "=== AEH mode: reading pre-generated report ===" + if [ ! -f "$REPORT_DIR/report.json" ]; then + echo "ERROR: AEH report.json not found at $REPORT_DIR/report.json" + exit 1 + fi + echo "Found AEH report at $REPORT_DIR" + elif [ "$EVAL_ENGINE" = "harbor" ]; then echo "=== Harbor mode: running analysis ===" ARGS=( diff --git a/pipeline/tasks/post/cleanup_pvc.yaml b/pipeline/tasks/post/cleanup_pvc.yaml index afc6485..97db796 100644 --- a/pipeline/tasks/post/cleanup_pvc.yaml +++ b/pipeline/tasks/post/cleanup_pvc.yaml @@ -17,13 +17,17 @@ spec: script: | #!/usr/bin/env bash set -euo pipefail - echo "Cleaning up PVCs for PipelineRun $(params.pipelinerun-name)..." - oc get pvc -n ab-eval-flow \ + + # Get current namespace from service account + NAMESPACE=$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace) + + echo "Cleaning up PVCs for PipelineRun $(params.pipelinerun-name) in namespace ${NAMESPACE}..." + oc get pvc -n "${NAMESPACE}" \ -o jsonpath='{range .items[?(@.metadata.ownerReferences[0].name=="$(params.pipelinerun-name)")]}{.metadata.name}{"\n"}{end}' \ | while read -r pvc; do if [ -n "$pvc" ]; then echo "Deleting PVC: $pvc" - oc delete pvc "$pvc" -n ab-eval-flow --wait=false + oc delete pvc "$pvc" -n "${NAMESPACE}" --wait=false fi done echo "PVC cleanup complete" diff --git a/pipeline/tasks/post/store.yaml b/pipeline/tasks/post/store.yaml index 0ea39da..4a62502 100644 --- a/pipeline/tasks/post/store.yaml +++ b/pipeline/tasks/post/store.yaml @@ -217,7 +217,13 @@ spec: ARGS+=(--pr-number "$(params.pr-number)") fi RESULTS_DIR="$(params.results-dir)" - if [ -z "$RESULTS_DIR" ] && [ "$(params.eval-engine)" != "harbor" ]; then + if [ "$(params.eval-engine)" = "aeh" ]; then + # Prefer evaluate's AEH results-dir (_eval_tmp); fall back to workspace. + # Pairwise jobs are under aeh-*-jobs-*, not aeh-jobs alone. + if [ -z "$RESULTS_DIR" ] || [ ! -d "$RESULTS_DIR" ]; then + RESULTS_DIR="$(workspaces.source.path)/_eval_tmp" + fi + elif [ -z "$RESULTS_DIR" ] && [ "$(params.eval-engine)" != "harbor" ]; then RESULTS_DIR="$(workspaces.source.path)/eval-results/$(params.submission-name)" fi if [ -n "$RESULTS_DIR" ]; then diff --git a/scripts/aggregate_scorecard.py b/scripts/aggregate_scorecard.py index 8527a04..af94988 100644 --- a/scripts/aggregate_scorecard.py +++ b/scripts/aggregate_scorecard.py @@ -534,7 +534,7 @@ def main() -> int: "--eval-engine", type=str, required=True, - choices=["harbor", "ase", "a2a", "mcpchecker", "both"], + choices=["harbor", "ase", "a2a", "mcpchecker", "aeh", "both"], help="Primary evaluation engine", ) parser.add_argument( diff --git a/scripts/analyze.py b/scripts/analyze.py index a8a6c61..98a71dc 100644 --- a/scripts/analyze.py +++ b/scripts/analyze.py @@ -536,6 +536,31 @@ def render_markdown(result: AnalysisResult) -> str: lines.append("\n\n") lines.append("") + # --- Pairwise comparison (AEH) --- + if result.pairwise is not None: + pw = result.pairwise + lines.append("## Pairwise Comparison\n") + lines.append(f"- **Treatment run:** `{pw.run_a}`") + lines.append(f"- **Control run:** `{pw.run_b}`") + lines.append(f"- **Cases compared:** {pw.cases_compared}") + lines.append(f"- **Treatment wins:** {pw.wins_a}") + lines.append(f"- **Control wins:** {pw.wins_b}") + lines.append(f"- **Ties:** {pw.ties}") + lines.append(f"- **Errors:** {pw.errors}") + lines.append(f"- **Treatment win rate:** {_fmt(pw.win_rate)}") + if pw.per_case: + lines.append("\n
\nPer-case winners\n") + lines.append("| Case | Winner |") + lines.append("|------|--------|") + for entry in pw.per_case: + if not isinstance(entry, dict): + continue + case_id = entry.get("case_id") or entry.get("case") or entry.get("id") or "?" + winner = entry.get("winner") or entry.get("verdict") or entry.get("result") or "?" + lines.append(f"| {case_id} | {winner} |") + lines.append("\n
\n") + lines.append("") + # --- Per-trial details --- lines.append("## Trial Details\n") if is_a2a: @@ -614,7 +639,7 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument( "--eval-engine", type=str, - choices=["harbor", "ase", "both", "a2a"], + choices=["harbor", "ase", "both", "a2a", "aeh"], default="harbor", help="Evaluation engine used (for provenance tagging and analysis path)", ) diff --git a/scripts/publish.py b/scripts/publish.py index b133019..2a7cdd6 100644 --- a/scripts/publish.py +++ b/scripts/publish.py @@ -191,6 +191,231 @@ def _upload_trial_tree(base_dir: Path, rel_root: Path) -> None: return uploaded +def _discover_aeh_harbor_job_dirs(results_dir: Path) -> list[tuple[str, Path]]: + """Discover Harbor job dirs for AEH single and pairwise layouts. + + Returns ``(object_prefix, job_dir)`` pairs where ``object_prefix`` is the + path under ``debug/harbor/`` (no trailing slash). + + Supported layouts (uploaded under ``debug/harbor/``): + - ``aeh-jobs/YYYY-MM-DD__…`` → ``{timestamp}/…`` (single) + - ``_eval_tmp/aeh-jobs/YYYY-MM-DD__…`` → ``{timestamp}/…`` (no aeh-jobs segment) + - ``_eval_tmp/aeh-*-jobs-*/YYYY-MM-DD__…`` → ``control|treatment/{timestamp}/…`` + - a single job dir passed directly (contains ``result.json`` / ``job.log``) + """ + + def _timestamp_jobs(parent: Path, prefix: str) -> list[tuple[str, Path]]: + jobs = sorted( + (d for d in parent.iterdir() if d.is_dir() and d.name.startswith("20")), + key=lambda d: d.stat().st_mtime, + ) + return [(f"{prefix}/{d.name}" if prefix else d.name, d) for d in jobs] + + found: list[tuple[str, Path]] = [] + + # Direct timestamp job dirs (single: results_dir == aeh-jobs) + found.extend(_timestamp_jobs(results_dir, "")) + + # Nested aeh-jobs/ under _eval_tmp — flatten (no "aeh-jobs/" segment) + nested = results_dir / "aeh-jobs" + if nested.is_dir(): + found.extend(_timestamp_jobs(nested, "")) + + # Pairwise: aeh-control-jobs-* / aeh-treatment-jobs-* → control/ / treatment/ + for sibling in sorted(results_dir.iterdir()): + if not sibling.is_dir(): + continue + name = sibling.name + if name.startswith("aeh-") and "-jobs" in name and name != "aeh-jobs": + if "control" in name: + label = "control" + elif "treatment" in name: + label = "treatment" + else: + label = name + found.extend(_timestamp_jobs(sibling, label)) + + # Allow passing a single job dir directly. + if not found and any((results_dir / name).exists() for name in ("result.json", "job.log", "config.json")): + found = [(results_dir.name, results_dir)] + + # De-dupe by resolved path while preserving order + seen: set[Path] = set() + unique: list[tuple[str, Path]] = [] + for prefix, job in found: + key = job.resolve() + if key in seen: + continue + seen.add(key) + unique.append((prefix, job)) + return unique + + +def upload_aeh_debug_artifacts( + results_dir: Path, + prefix: str, + endpoint: str, + access_key: str, + secret_key: str, + bucket: str = "ab-eval-reports", + secure: bool | None = None, +) -> int: + """Upload AEH Harbor job trees as individual files under ``debug/harbor/``. + + ``results_dir`` is typically ``_eval_tmp`` (pairwise) or ``_eval_tmp/aeh-jobs`` + (single). Each file is uploaded unpacked (not tarred) to:: + + {prefix}/debug/harbor/{job_path}/{relative_path} + + For pairwise, ``job_path`` includes the control/treatment jobs dir name so + both variants land under the same report prefix. + """ + from minio import Minio + + if not results_dir.is_dir(): + logger.warning("AEH jobs dir does not exist, skipping debug upload: %s", results_dir) + return 0 + + parsed = urlparse(endpoint) + host = parsed.netloc or parsed.path + if secure is None: + secure = parsed.scheme == "https" + + client = Minio(host, access_key=access_key, secret_key=secret_key, secure=secure) + + if not client.bucket_exists(bucket): + client.make_bucket(bucket) + + job_dirs = _discover_aeh_harbor_job_dirs(results_dir) + if not job_dirs: + logger.warning("No AEH Harbor job directories under %s", results_dir) + return 0 + + uploaded = 0 + for job_prefix, job_dir in job_dirs: + for fpath in sorted(job_dir.rglob("*")): + if not fpath.is_file(): + continue + rel = fpath.relative_to(job_dir) + object_name = f"{prefix}/debug/harbor/{job_prefix}/{rel.as_posix()}" + try: + client.fput_object(bucket, object_name, str(fpath)) + uploaded += 1 + except Exception as exc: + logger.warning("Failed to upload %s: %s", fpath, exc) + + logger.info( + "Uploaded %d AEH debug artifact files to s3://%s/%s/debug/harbor/", + uploaded, + bucket, + prefix, + ) + return uploaded + + +def _discover_aeh_run_dirs( + report_dir: Path, + workspace_root: Path | None = None, +) -> list[Path]: + """Find AEH harness run directories (summary.yaml / run_result.json). + + Layout (both single and pairwise):: + + reports///summary.yaml + reports//control-/… + reports//treatment-/… + + ``report_dir`` is typically ``reports/``. When the skill + name differs, also scan ``workspace_root/reports/*``. + """ + roots: list[Path] = [] + if report_dir.is_dir(): + roots.append(report_dir) + if workspace_root is not None: + reports_root = workspace_root / "reports" + if reports_root.is_dir(): + for child in sorted(reports_root.iterdir()): + if child.is_dir() and child.resolve() not in {r.resolve() for r in roots}: + roots.append(child) + + run_dirs: list[Path] = [] + seen: set[Path] = set() + for root in roots: + for child in sorted(root.iterdir()): + if not child.is_dir(): + continue + if not ((child / "summary.yaml").is_file() or (child / "run_result.json").is_file()): + continue + key = child.resolve() + if key in seen: + continue + seen.add(key) + run_dirs.append(child) + return run_dirs + + +def upload_aeh_run_artifacts( + report_dir: Path, + prefix: str, + endpoint: str, + access_key: str, + secret_key: str, + bucket: str = "ab-eval-reports", + secure: bool | None = None, + workspace_root: Path | None = None, +) -> int: + """Upload AEH harness run trees under ``debug/aeh/``. + + Uploads each run directory (``summary.yaml``, ``report.html``, + ``run_result.json``, ``cases/``, …) to:: + + {prefix}/debug/aeh/{run_dir_name}/{relative_path} + + Works for single (one run dir) and pairwise (control-*/treatment-*). + """ + from minio import Minio + + run_dirs = _discover_aeh_run_dirs(report_dir, workspace_root=workspace_root) + if not run_dirs: + logger.warning( + "No AEH run directories under %s (summary.yaml/run_result.json) — skipping debug/aeh/", + report_dir, + ) + return 0 + + parsed = urlparse(endpoint) + host = parsed.netloc or parsed.path + if secure is None: + secure = parsed.scheme == "https" + + client = Minio(host, access_key=access_key, secret_key=secret_key, secure=secure) + + if not client.bucket_exists(bucket): + client.make_bucket(bucket) + + uploaded = 0 + for run_dir in run_dirs: + for fpath in sorted(run_dir.rglob("*")): + if not fpath.is_file(): + continue + rel = fpath.relative_to(run_dir) + object_name = f"{prefix}/debug/aeh/{run_dir.name}/{rel.as_posix()}" + try: + client.fput_object(bucket, object_name, str(fpath)) + uploaded += 1 + except Exception as exc: + logger.warning("Failed to upload %s: %s", fpath, exc) + + logger.info( + "Uploaded %d AEH run artifact files to s3://%s/%s/debug/aeh/ (%d run dir(s))", + uploaded, + bucket, + prefix, + len(run_dirs), + ) + return uploaded + + def upload_ase_debug_artifacts( results_dir: Path, prefix: str, @@ -610,7 +835,7 @@ def main() -> int: "--eval-engine", type=str, default="harbor", - choices=["harbor", "ase", "mcpchecker", "a2a", "both"], + choices=["harbor", "ase", "mcpchecker", "a2a", "aeh", "both"], help="Evaluation engine used — determines debug artifact layout", ) parser.add_argument( @@ -659,7 +884,10 @@ def main() -> int: _upload_fn = upload_mcpchecker_debug_artifacts elif args.eval_engine in ("ase", "both"): _upload_fn = upload_ase_debug_artifacts + elif args.eval_engine == "aeh": + _upload_fn = upload_aeh_debug_artifacts else: + # harbor, a2a — treatment/control Harbor trial layout _upload_fn = upload_debug_artifacts _upload_fn( results_dir=args.results_dir, @@ -669,6 +897,18 @@ def main() -> int: secret_key=minio_secret_key, bucket=args.minio_bucket, ) + # AEH harness run dirs (summary.yaml, report.html, cases/, …) + # live under reports// — upload for both single and pairwise. + if prefix and args.eval_engine == "aeh": + upload_aeh_run_artifacts( + report_dir=args.report_dir, + prefix=prefix, + endpoint=minio_endpoint, + access_key=minio_access_key, + secret_key=minio_secret_key, + bucket=args.minio_bucket, + workspace_root=args.workspace_root, + ) if prefix and args.workspace_root: upload_scaffolded_configs( workspace_root=args.workspace_root, diff --git a/scripts/save_harbor_debug.py b/scripts/save_harbor_debug.py new file mode 100644 index 0000000..6200ce5 --- /dev/null +++ b/scripts/save_harbor_debug.py @@ -0,0 +1,212 @@ +"""Save Harbor job directory to MinIO for debugging (tarball). + +NOTE: Evaluate-time use is disabled. Prefer ``publish.upload_aeh_debug_artifacts`` +at store time — individual files under prepare's report-prefix (no tar / no +second prefix). + +Usage:: + + python scripts/save_harbor_debug.py \ + --jobs-dir /workspace/jobs \ + --submission-name my-skill \ + --pipeline-run-id abevalflow-xyz \ + --report-prefix YYYYMMDD_HHMMSS_name_run-id +""" + +from __future__ import annotations + +import argparse +import logging +import os +import sys +import tarfile +import tempfile +from datetime import UTC, datetime +from pathlib import Path +from urllib.parse import urlparse + +logger = logging.getLogger(__name__) + + +def _build_artifact_prefix( + submission_name: str, + pipeline_run_id: str, +) -> str: + """Build the MinIO object prefix: YYYYMMDD_hhmmss__.""" + datestamp = datetime.now(UTC).strftime("%Y%m%d_%H%M%S") + return f"{datestamp}_{submission_name}_{pipeline_run_id}" + + +def find_latest_job_dir(jobs_dir: Path) -> Path | None: + """Find the most recent Harbor job directory (timestamp-based). + + Harbor creates job directories with names like: YYYY-MM-DD__HH-MM-SS + Returns the most recent one or None if not found. + """ + if not jobs_dir.is_dir(): + logger.warning("Jobs directory not found: %s", jobs_dir) + return None + + # Find all timestamp-based directories (start with "20") + job_dirs = [d for d in jobs_dir.iterdir() if d.is_dir() and d.name.startswith("20")] + + if not job_dirs: + logger.warning("No Harbor job directories found in %s", jobs_dir) + return None + + # Sort by name (timestamp format sorts chronologically) and return latest + latest = sorted(job_dirs)[-1] + logger.info("Found latest Harbor job: %s", latest) + return latest + + +def create_job_tarball(job_dir: Path) -> Path: + """Create a compressed tarball of the Harbor job directory. + + Returns the path to the temporary tarball file. + """ + tarball = Path(tempfile.mktemp(suffix=".tar.gz", prefix="harbor-debug-")) + logger.info("Creating tarball: %s", tarball) + + with tarfile.open(tarball, "w:gz") as tar: + # Add the job directory with its basename as the archive root + tar.add(job_dir, arcname=job_dir.name) + + logger.info("Tarball created: %s (%d bytes)", tarball, tarball.stat().st_size) + return tarball + + +def upload_to_minio( + tarball: Path, + job_dir_name: str, + report_prefix: str, + endpoint: str, + access_key: str, + secret_key: str, + bucket: str = "ab-eval-reports", + secure: bool | None = None, +) -> bool: + """Upload tarball to MinIO following publish.py pattern. + + Target path: s3://{bucket}/{report-prefix}/debug/harbor/{job-dir-name}.tar.gz + Returns True on success. + """ + from minio import Minio + + parsed = urlparse(endpoint) + host = parsed.netloc or parsed.path + if secure is None: + secure = parsed.scheme == "https" + + client = Minio( + host, + access_key=access_key, + secret_key=secret_key, + secure=secure, + ) + + if not client.bucket_exists(bucket): + client.make_bucket(bucket) + logger.info("Created bucket: %s", bucket) + + object_name = f"{report_prefix}/debug/harbor/{job_dir_name}.tar.gz" + + try: + client.fput_object(bucket, object_name, str(tarball)) + logger.info("✓ Harbor debug artifacts saved to: s3://%s/%s", bucket, object_name) + logger.info(" Download with: mc cp %s/%s/%s .", host, bucket, object_name) + logger.info(" Extract with: tar -xzf harbor-job.tar.gz") + return True + except Exception as exc: + logger.error("Failed to upload to MinIO: %s", exc) + return False + + +def main() -> int: + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(name)s: %(message)s", + ) + + parser = argparse.ArgumentParser(description="Save Harbor debug artifacts to MinIO") + parser.add_argument("--jobs-dir", type=Path, required=True, help="Harbor jobs directory") + parser.add_argument("--submission-name", type=str, required=True, help="Submission name") + parser.add_argument("--pipeline-run-id", type=str, required=True, help="Pipeline run ID") + parser.add_argument( + "--report-prefix", + type=str, + default="", + help="MinIO prefix (e.g., YYYYMMDD_HHMMSS_name_run-id). If empty, builds from submission/run-id.", + ) + parser.add_argument( + "--minio-endpoint", + type=str, + default=None, + help="MinIO endpoint (default: MINIO_ENDPOINT env var)", + ) + parser.add_argument("--minio-bucket", type=str, default="ab-eval-reports", help="MinIO bucket name") + + args = parser.parse_args() + + # Get MinIO credentials from environment + minio_endpoint = args.minio_endpoint or os.environ.get("MINIO_ENDPOINT", "") + minio_access_key = os.environ.get("MINIO_ACCESS_KEY", "") + minio_secret_key = os.environ.get("MINIO_SECRET_KEY", "") + + if not minio_endpoint: + logger.error("MINIO_ENDPOINT not set") + return 1 + + if not minio_access_key: + logger.error("MINIO_ACCESS_KEY not set") + return 1 + + if not minio_secret_key: + logger.error("MINIO_SECRET_KEY not set") + return 1 + + logger.info("=== Saving Harbor debug artifacts to MinIO ===") + logger.info("Jobs dir: %s", args.jobs_dir) + logger.info("Submission: %s", args.submission_name) + logger.info("Pipeline run: %s", args.pipeline_run_id) + + # Find latest job directory + job_dir = find_latest_job_dir(args.jobs_dir) + if not job_dir: + logger.warning("No Harbor job directory found - nothing to upload") + return 0 # Exit successfully (not an error if no jobs exist) + + # Build report prefix if not provided + report_prefix = args.report_prefix or _build_artifact_prefix( + args.submission_name, + args.pipeline_run_id, + ) + logger.info("Report prefix: %s", report_prefix) + + # Create tarball + tarball = None + try: + tarball = create_job_tarball(job_dir) + + # Upload to MinIO + success = upload_to_minio( + tarball=tarball, + job_dir_name=job_dir.name, + report_prefix=report_prefix, + endpoint=minio_endpoint, + access_key=minio_access_key, + secret_key=minio_secret_key, + bucket=args.minio_bucket, + ) + + return 0 if success else 1 + + finally: + # Cleanup temporary tarball + if tarball and tarball.exists(): + tarball.unlink() + logger.debug("Cleaned up temporary tarball: %s", tarball) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/test_quality_review.py b/scripts/test_quality_review.py index 25a5513..9c8b423 100644 --- a/scripts/test_quality_review.py +++ b/scripts/test_quality_review.py @@ -150,6 +150,73 @@ Respond with the JSON assessment. """ +REVIEW_SYSTEM_PROMPT_AEH = """\ +You are a senior QA engineer reviewing Agent-Eval-Harness (AEH) submissions. +You must assess the quality of a submission that consists of: +- metadata.yaml (pipeline/experiment config) +- eval.yaml (single mode) and/or eval-control.yaml + eval-treatment.yaml (pairwise) +- cases/ with input.yaml (and optional annotations.yaml) +- Optionally skills/**/SKILL.md when plugin_dirs are used + +Evaluate the following dimensions and assign each a score from 0.0 to 1.0: + +1. **coherence** — Do the eval configs and cases faithfully test the intended skill/behavior? +2. **coverage** — Do the cases and judges adequately verify expected outcomes? +3. **clarity** — Are prompts, judges, and thresholds clear and unambiguous? +4. **feasibility** — Can an AI agent reasonably complete the cases? +5. **robustness** — Do cases/judges cover edge cases and failure modes? +6. **specificity** — Are case prompts concrete enough? +7. **completeness** — Are required AEH fields present (dataset, judges, models, outputs)? + +For each dimension, provide a brief finding (1-2 sentences). + +Then provide an overall recommendation: +- "pass" — Submission is ready for evaluation (all scores >= 0.6) +- "warn" — Minor issues but can proceed (some scores 0.4-0.6) +- "fail" — Significant problems, should not proceed (any score < 0.4) + +Output ONLY valid JSON with this structure: +{ + "dimensions": { + "coherence": {"score": 0.0, "finding": "..."}, + "coverage": {"score": 0.0, "finding": "..."}, + "clarity": {"score": 0.0, "finding": "..."}, + "feasibility": {"score": 0.0, "finding": "..."}, + "robustness": {"score": 0.0, "finding": "..."}, + "specificity": {"score": 0.0, "finding": "..."}, + "completeness": {"score": 0.0, "finding": "..."} + }, + "overall_score": 0.0, + "recommendation": "pass|warn|fail", + "summary": "One paragraph overall assessment" +} +""" + +REVIEW_USER_TEMPLATE_AEH = """\ +Review the following AEH submission. + +## Skill Name +{skill_name} + +## metadata.yaml +```yaml +{metadata_content} +``` + +## Eval configs +{eval_configs_section} + +## Sample cases +{cases_section} + +## Optional SKILL.md +``` +{skill_content} +``` + +Respond with the JSON assessment. +""" + def _read_file_safe(path: Path) -> str: if path.is_file(): @@ -163,7 +230,7 @@ def _find_skill_md(submission_dir: Path) -> Path | None: direct = submission_dir / "skills" / "SKILL.md" if direct.is_file(): return direct - # Older Harbor format: skills//SKILL.md + # Nested / AEH format: skills//SKILL.md skills_dir = submission_dir / "skills" if skills_dir.is_dir(): for subdir in skills_dir.iterdir(): @@ -171,7 +238,101 @@ def _find_skill_md(submission_dir: Path) -> Path | None: nested = subdir / "SKILL.md" if nested.is_file(): return nested - return None + # Any SKILL.md under submission (fallback) + matches = sorted(submission_dir.rglob("SKILL.md")) + return matches[0] if matches else None + + +def _is_aeh_submission(submission_dir: Path, metadata: SubmissionMetadata) -> bool: + """Detect AEH submissions via metadata or eval config files.""" + if getattr(metadata, "eval_engine", None) == "aeh": + return True + return ( + (submission_dir / "eval.yaml").is_file() + or (submission_dir / "eval-treatment.yaml").is_file() + or (submission_dir / "eval-control.yaml").is_file() + ) + + +def _aeh_eval_configs_section(submission_dir: Path) -> str: + parts: list[str] = [] + for name in ("eval.yaml", "eval-control.yaml", "eval-treatment.yaml"): + path = submission_dir / name + if path.is_file(): + parts.append(f"### {name}\n```yaml\n{path.read_text()}\n```") + return "\n\n".join(parts) if parts else "(no eval.yaml / eval-*.yaml present)" + + +def _aeh_cases_section(submission_dir: Path, max_cases: int = 3) -> str: + cases_dir = submission_dir / "cases" + if not cases_dir.is_dir(): + return "(cases/ not present)" + case_dirs = sorted(d for d in cases_dir.iterdir() if d.is_dir())[:max_cases] + if not case_dirs: + return "(cases/ is empty)" + parts: list[str] = [] + for case_dir in case_dirs: + input_path = case_dir / "input.yaml" + ann_path = case_dir / "annotations.yaml" + block = f"### {case_dir.name}\n" + block += f"**input.yaml**\n```yaml\n{_read_file_safe(input_path)}\n```\n" + if ann_path.is_file(): + block += f"**annotations.yaml**\n```yaml\n{ann_path.read_text()}\n```\n" + parts.append(block) + return "\n".join(parts) + + +def _advisory_aeh_missing_files(submission_dir: Path, skill_name: str) -> dict | None: + """Return a warn/pass assessment when required AEH files are missing.""" + has_eval = any( + (submission_dir / name).is_file() for name in ("eval.yaml", "eval-control.yaml", "eval-treatment.yaml") + ) + has_meta = (submission_dir / "metadata.yaml").is_file() + has_cases = (submission_dir / "cases").is_dir() and any((submission_dir / "cases").iterdir()) + if has_eval and has_meta and has_cases: + return None + missing = [] + if not has_meta: + missing.append("metadata.yaml") + if not has_eval: + missing.append("eval.yaml or eval-control/treatment.yaml") + if not has_cases: + missing.append("cases/") + return { + "dimensions": {}, + "overall_score": 0.5, + "recommendation": "warn", + "summary": ( + f"AEH submission '{skill_name}' is missing required files for a full " + f"quality review: {', '.join(missing)}. Proceeding with advisory warn." + ), + "passed": True, + "engine": "aeh", + } + + +def _normalize_assessment(assessment: dict, *, engine: str | None = None) -> dict: + if "overall_score" not in assessment: + dims = assessment.get("dimensions", {}) + scores = [d.get("score", 0.0) for d in dims.values() if isinstance(d, dict)] + assessment["overall_score"] = sum(scores) / len(scores) if scores else 0.0 + + if "recommendation" not in assessment: + score = assessment["overall_score"] + if score >= 0.6: + assessment["recommendation"] = "pass" + elif score >= 0.4: + assessment["recommendation"] = "warn" + else: + assessment["recommendation"] = "fail" + + # AEH reviews are advisory for the test phase; keep passed=true by default. + if engine == "aeh": + assessment["passed"] = True + assessment["engine"] = "aeh" + else: + assessment["passed"] = assessment["recommendation"] != "fail" + return assessment def review_submission(submission_dir: Path) -> dict: @@ -184,37 +345,50 @@ def review_submission(submission_dir: Path) -> dict: skill_md_path = _find_skill_md(submission_dir) skill_content = _read_file_safe(skill_md_path) if skill_md_path else "(file not present)" - # Detect ASE vs Harbor format - evals_path = submission_dir / "evals" / "evals.json" - is_ase = evals_path.is_file() - - if is_ase: - # ASE format: uses evals/evals.json - evals_content = _read_file_safe(evals_path) - system_prompt = REVIEW_SYSTEM_PROMPT_ASE - user_prompt = REVIEW_USER_TEMPLATE_ASE.format( + if _is_aeh_submission(submission_dir, metadata): + advisory = _advisory_aeh_missing_files(submission_dir, metadata.name) + if advisory is not None: + return advisory + system_prompt = REVIEW_SYSTEM_PROMPT_AEH + user_prompt = REVIEW_USER_TEMPLATE_AEH.format( skill_name=metadata.name, + metadata_content=_read_file_safe(meta_path), + eval_configs_section=_aeh_eval_configs_section(submission_dir), + cases_section=_aeh_cases_section(submission_dir), skill_content=skill_content, - evals_content=evals_content, ) + engine = "aeh" else: - # Harbor format: uses instruction.md and tests/ - instruction_content = _read_file_safe(submission_dir / "instruction.md") - test_content = _read_file_safe(submission_dir / "tests" / "test_outputs.py") - - llm_judge_path = submission_dir / "tests" / "llm_judge.py" - llm_judge_section = "" - if llm_judge_path.is_file(): - llm_judge_section = f"## tests/llm_judge.py\n```python\n{llm_judge_path.read_text()}\n```" - - system_prompt = REVIEW_SYSTEM_PROMPT_HARBOR - user_prompt = REVIEW_USER_TEMPLATE_HARBOR.format( - skill_name=metadata.name, - skill_content=skill_content, - instruction_content=instruction_content, - test_content=test_content, - llm_judge_section=llm_judge_section, - ) + # Detect ASE vs Harbor format + evals_path = submission_dir / "evals" / "evals.json" + is_ase = evals_path.is_file() + + if is_ase: + evals_content = _read_file_safe(evals_path) + system_prompt = REVIEW_SYSTEM_PROMPT_ASE + user_prompt = REVIEW_USER_TEMPLATE_ASE.format( + skill_name=metadata.name, + skill_content=skill_content, + evals_content=evals_content, + ) + else: + instruction_content = _read_file_safe(submission_dir / "instruction.md") + test_content = _read_file_safe(submission_dir / "tests" / "test_outputs.py") + + llm_judge_path = submission_dir / "tests" / "llm_judge.py" + llm_judge_section = "" + if llm_judge_path.is_file(): + llm_judge_section = f"## tests/llm_judge.py\n```python\n{llm_judge_path.read_text()}\n```" + + system_prompt = REVIEW_SYSTEM_PROMPT_HARBOR + user_prompt = REVIEW_USER_TEMPLATE_HARBOR.format( + skill_name=metadata.name, + skill_content=skill_content, + instruction_content=instruction_content, + test_content=test_content, + llm_judge_section=llm_judge_section, + ) + engine = None response_text = llm_client.chat_completion( messages=[ @@ -236,23 +410,7 @@ def review_submission(submission_dir: Path) -> dict: else: raise ValueError("LLM review response is not valid JSON") - if "overall_score" not in assessment: - dims = assessment.get("dimensions", {}) - scores = [d.get("score", 0.0) for d in dims.values() if isinstance(d, dict)] - assessment["overall_score"] = sum(scores) / len(scores) if scores else 0.0 - - if "recommendation" not in assessment: - score = assessment["overall_score"] - if score >= 0.6: - assessment["recommendation"] = "pass" - elif score >= 0.4: - assessment["recommendation"] = "warn" - else: - assessment["recommendation"] = "fail" - - assessment["passed"] = assessment["recommendation"] != "fail" - - return assessment + return _normalize_assessment(assessment, engine=engine) def main(argv: list[str] | None = None) -> int: @@ -271,9 +429,18 @@ def main(argv: list[str] | None = None) -> int: assessment = review_submission(args.submission_dir) except Exception as exc: logger.error("Review failed: %s", exc) - result = {"passed": False, "error": str(exc)} + # Prefer non-blocking for AEH even on unexpected errors + is_aeh = (args.submission_dir / "eval.yaml").is_file() or ( + args.submission_dir / "eval-treatment.yaml" + ).is_file() + result = { + "passed": True if is_aeh else False, + "recommendation": "warn" if is_aeh else "fail", + "error": str(exc), + "engine": "aeh" if is_aeh else None, + } print(json.dumps(result, indent=2)) - return 1 + return 0 if is_aeh else 1 print(json.dumps(assessment, indent=2)) # Always return 0 - this review is advisory (non-blocking) diff --git a/tests/test_publish.py b/tests/test_publish.py index 290c4c7..477f459 100644 --- a/tests/test_publish.py +++ b/tests/test_publish.py @@ -14,6 +14,8 @@ cleanup_images, post_pr_comment, promote_to_quay, + upload_aeh_debug_artifacts, + upload_aeh_run_artifacts, upload_debug_artifacts, upload_reports, upload_scaffolded_configs, @@ -478,6 +480,187 @@ def test_creates_bucket_if_missing(self, mock_minio_cls, results_dir): mock_client.make_bucket.assert_called_once_with("ab-eval-reports") +class TestUploadAehDebugArtifacts: + @pytest.fixture + def aeh_jobs_dir(self, tmp_path: Path) -> Path: + job = tmp_path / "2026-07-19__08-44-52" + trial = job / "case-001__abc" + (trial / "agent").mkdir(parents=True) + (trial / "verifier").mkdir(parents=True) + (trial / "agent" / "claude-code.txt").write_text("{}") + (trial / "verifier" / "reward.json").write_text('{"reward": 1.0}') + (trial / "exception.txt").write_text("boom") + (job / "result.json").write_text("{}") + (job / "job.log").write_text("ok") + return tmp_path + + @patch("minio.Minio") + def test_uploads_unpacked_job_tree(self, mock_minio_cls, aeh_jobs_dir): + mock_client = MagicMock() + mock_minio_cls.return_value = mock_client + mock_client.bucket_exists.return_value = True + + count = upload_aeh_debug_artifacts( + results_dir=aeh_jobs_dir, + prefix="20260719_prep_aeh-hello_run1", + endpoint="http://minio:9000", + access_key="key", + secret_key="secret", + ) + + assert count == 5 + names = [call[0][1] for call in mock_client.fput_object.call_args_list] + assert all(n.startswith("20260719_prep_aeh-hello_run1/debug/harbor/") for n in names) + assert any("2026-07-19__08-44-52/case-001__abc/agent/claude-code.txt" in n for n in names) + assert any("2026-07-19__08-44-52/result.json" in n for n in names) + assert not any(n.endswith(".tar.gz") for n in names) + + def test_missing_jobs_dir(self, tmp_path): + assert ( + upload_aeh_debug_artifacts( + results_dir=tmp_path / "missing", + prefix="p", + endpoint="http://minio:9000", + access_key="k", + secret_key="s", + ) + == 0 + ) + + @patch("minio.Minio") + def test_uploads_pairwise_control_and_treatment_jobs(self, mock_minio_cls, tmp_path: Path): + """Pairwise jobs live under aeh-*-jobs-* siblings of _eval_tmp.""" + eval_tmp = tmp_path / "_eval_tmp" + for label in ("control", "treatment"): + job = eval_tmp / f"aeh-{label}-jobs-run1" / "2026-07-19__12-00-00" + trial = job / "case-001__abc" + (trial / "agent").mkdir(parents=True) + (trial / "agent" / "out.txt").write_text(label) + (job / "result.json").write_text("{}") + + mock_client = MagicMock() + mock_minio_cls.return_value = mock_client + mock_client.bucket_exists.return_value = True + + count = upload_aeh_debug_artifacts( + results_dir=eval_tmp, + prefix="prefix1", + endpoint="http://minio:9000", + access_key="key", + secret_key="secret", + ) + + assert count == 4 + names = [call[0][1] for call in mock_client.fput_object.call_args_list] + assert any("debug/harbor/control/2026-07-19__12-00-00/" in n for n in names) + assert any("debug/harbor/treatment/2026-07-19__12-00-00/" in n for n in names) + assert not any("aeh-control-jobs" in n or "aeh-treatment-jobs" in n for n in names) + + @patch("minio.Minio") + def test_single_flattens_aeh_jobs_segment(self, mock_minio_cls, tmp_path: Path): + """Single mode: _eval_tmp/aeh-jobs/ → debug/harbor// (no aeh-jobs/).""" + eval_tmp = tmp_path / "_eval_tmp" + job = eval_tmp / "aeh-jobs" / "2026-07-19__12-00-00" + job.mkdir(parents=True) + (job / "result.json").write_text("{}") + (job / "job.log").write_text("ok") + + mock_client = MagicMock() + mock_minio_cls.return_value = mock_client + mock_client.bucket_exists.return_value = True + + count = upload_aeh_debug_artifacts( + results_dir=eval_tmp, + prefix="prefix1", + endpoint="http://minio:9000", + access_key="key", + secret_key="secret", + ) + assert count == 2 + names = [call[0][1] for call in mock_client.fput_object.call_args_list] + assert any(n.endswith("debug/harbor/2026-07-19__12-00-00/result.json") for n in names) + assert not any("/aeh-jobs/" in n for n in names) + + +class TestUploadAehRunArtifacts: + def _make_run(self, root: Path, name: str, *, pairwise: bool = False) -> Path: + run = root / name + (run / "cases" / "case-001" / "artifacts").mkdir(parents=True) + (run / "summary.yaml").write_text( + "run_id: x\n" + ("pairwise:\n wins_a: 0\n ties: 1\n" if pairwise else "judges: {}\n") + ) + (run / "run_result.json").write_text('{"mean_reward": 1.0}') + (run / "report.html").write_text("ok") + (run / "cases" / "case-001" / "artifacts" / "greeting.txt").write_text("Hello") + return run + + @patch("minio.Minio") + def test_uploads_single_run_tree(self, mock_minio_cls, tmp_path: Path): + report_dir = tmp_path / "reports" / "aeh-hello-world-single" + report_dir.mkdir(parents=True) + self._make_run(report_dir, "aeh-single-run1") + (report_dir / "report.json").write_text("{}") + + mock_client = MagicMock() + mock_minio_cls.return_value = mock_client + mock_client.bucket_exists.return_value = True + + count = upload_aeh_run_artifacts( + report_dir=report_dir, + prefix="prefix-single", + endpoint="http://minio:9000", + access_key="k", + secret_key="s", + ) + assert count == 4 + names = [call[0][1] for call in mock_client.fput_object.call_args_list] + assert any(n.endswith("debug/aeh/aeh-single-run1/summary.yaml") for n in names) + assert any(n.endswith("debug/aeh/aeh-single-run1/report.html") for n in names) + assert any(n.endswith("debug/aeh/aeh-single-run1/run_result.json") for n in names) + assert any("debug/aeh/aeh-single-run1/cases/case-001/artifacts/greeting.txt" in n for n in names) + # Top-level aggregate report.json must not be nested under debug/aeh/ + assert not any("/debug/aeh/report.json" in n for n in names) + + @patch("minio.Minio") + def test_uploads_pairwise_control_and_treatment_runs(self, mock_minio_cls, tmp_path: Path): + report_dir = tmp_path / "reports" / "aeh-hello-world-pairwise" + report_dir.mkdir(parents=True) + self._make_run(report_dir, "control-run1") + self._make_run(report_dir, "treatment-run1", pairwise=True) + + mock_client = MagicMock() + mock_minio_cls.return_value = mock_client + mock_client.bucket_exists.return_value = True + + count = upload_aeh_run_artifacts( + report_dir=report_dir, + prefix="prefix-pw", + endpoint="http://minio:9000", + access_key="k", + secret_key="s", + ) + assert count == 8 + names = [call[0][1] for call in mock_client.fput_object.call_args_list] + assert any("debug/aeh/control-run1/summary.yaml" in n for n in names) + assert any("debug/aeh/treatment-run1/summary.yaml" in n for n in names) + assert any("debug/aeh/treatment-run1/report.html" in n for n in names) + + def test_missing_runs_returns_zero(self, tmp_path: Path): + report_dir = tmp_path / "reports" / "empty" + report_dir.mkdir(parents=True) + (report_dir / "report.json").write_text("{}") + assert ( + upload_aeh_run_artifacts( + report_dir=report_dir, + prefix="p", + endpoint="http://minio:9000", + access_key="k", + secret_key="s", + ) + == 0 + ) + + class TestUpliftThresholdLogic: """Verify that -1.0 threshold disables Quay promotion.""" diff --git a/tests/test_quality_review_aeh.py b/tests/test_quality_review_aeh.py new file mode 100644 index 0000000..93f7d94 --- /dev/null +++ b/tests/test_quality_review_aeh.py @@ -0,0 +1,103 @@ +"""Tests for AEH-aware quality review detection and advisory paths.""" + +from __future__ import annotations + +from pathlib import Path +from unittest.mock import patch + +import yaml + +from abevalflow.schemas import SubmissionMetadata +from scripts.test_quality_review import ( + _advisory_aeh_missing_files, + _is_aeh_submission, + review_submission, +) + + +def _write_meta(submission: Path, **extra) -> None: + data = { + "name": "aeh-hello", + "description": "test", + "persona": "general", + "version": "0.1.0", + "cpus": 1, + "memory_mb": 512, + "storage_mb": 1024, + **extra, + } + (submission / "metadata.yaml").write_text(yaml.dump(data)) + + +class TestAehDetection: + def test_detect_via_eval_yaml(self, tmp_path: Path): + sub = tmp_path / "sub" + sub.mkdir() + _write_meta(sub) + (sub / "eval.yaml").write_text("skill: x\n") + meta = SubmissionMetadata(**yaml.safe_load((sub / "metadata.yaml").read_text())) + assert _is_aeh_submission(sub, meta) is True + + def test_detect_via_metadata_engine(self, tmp_path: Path): + sub = tmp_path / "sub" + sub.mkdir() + _write_meta(sub, eval_engine="aeh") + meta = SubmissionMetadata(**yaml.safe_load((sub / "metadata.yaml").read_text())) + assert _is_aeh_submission(sub, meta) is True + + def test_harbor_not_aeh(self, tmp_path: Path): + sub = tmp_path / "sub" + sub.mkdir() + _write_meta(sub) + (sub / "instruction.md").write_text("do the thing") + meta = SubmissionMetadata(**yaml.safe_load((sub / "metadata.yaml").read_text())) + assert _is_aeh_submission(sub, meta) is False + + +class TestAehAdvisoryMissing: + def test_missing_cases_is_warn_pass(self, tmp_path: Path): + sub = tmp_path / "sub" + sub.mkdir() + _write_meta(sub, eval_engine="aeh") + (sub / "eval.yaml").write_text("skill: x\n") + result = _advisory_aeh_missing_files(sub, "aeh-hello") + assert result is not None + assert result["passed"] is True + assert result["recommendation"] == "warn" + + def test_complete_aeh_no_advisory(self, tmp_path: Path): + sub = tmp_path / "sub" + sub.mkdir() + _write_meta(sub, eval_engine="aeh") + (sub / "eval.yaml").write_text("skill: x\n") + case = sub / "cases" / "case-001" + case.mkdir(parents=True) + (case / "input.yaml").write_text("prompt: hi\n") + assert _advisory_aeh_missing_files(sub, "aeh-hello") is None + + +class TestReviewSubmissionAeh: + def test_aeh_review_uses_aeh_prompt_and_stays_passed(self, tmp_path: Path): + sub = tmp_path / "sub" + sub.mkdir() + _write_meta(sub, eval_engine="aeh") + (sub / "eval.yaml").write_text("skill: x\njudges: []\n") + case = sub / "cases" / "case-001" + case.mkdir(parents=True) + (case / "input.yaml").write_text("prompt: hi\n") + + fake = { + "dimensions": {"coherence": {"score": 0.2, "finding": "weak"}}, + "overall_score": 0.2, + "recommendation": "fail", + "summary": "bad", + } + payload = __import__("json").dumps(fake) + with patch( + "scripts.test_quality_review.llm_client.chat_completion", + return_value=payload, + ): + assessment = review_submission(sub) + assert assessment["engine"] == "aeh" + assert assessment["passed"] is True # AEH advisory + assert assessment["recommendation"] == "fail" From 9a4ab35d9b184b9737e5c972c5c2f01150fae4c9 Mon Sep 17 00:00:00 2001 From: gziv Date: Mon, 20 Jul 2026 09:55:32 +0300 Subject: [PATCH 3/5] fix(aeh): OpenShift Harbor trial-pod hardening Add OpenShiftEnvironment emptyDir/verifier-path support, AEH task enrichment, and keep classic Harbor job configs on environment.type openshift. Co-authored-by: Cursor --- abevalflow/harbor_extensions/__init__.py | 5 + .../harbor_extensions/aeh_task_enrichment.py | 176 ++++++++++++++++++ .../openshift_environment.py | 106 +++++++++++ scripts/generate_eval_config.py | 3 + tests/test_aeh_task_enrichment.py | 100 ++++++++++ 5 files changed, 390 insertions(+) create mode 100644 abevalflow/harbor_extensions/__init__.py create mode 100644 abevalflow/harbor_extensions/aeh_task_enrichment.py create mode 100644 abevalflow/harbor_extensions/openshift_environment.py create mode 100644 tests/test_aeh_task_enrichment.py diff --git a/abevalflow/harbor_extensions/__init__.py b/abevalflow/harbor_extensions/__init__.py new file mode 100644 index 0000000..16b5f7e --- /dev/null +++ b/abevalflow/harbor_extensions/__init__.py @@ -0,0 +1,5 @@ +"""Harbor environment extensions for ABEvalFlow OpenShift deployment. + +Includes OpenShiftEnvironment emptyDir mounts and AEH task enrichment +(skills + annotations) applied before ``harbor run``. +""" diff --git a/abevalflow/harbor_extensions/aeh_task_enrichment.py b/abevalflow/harbor_extensions/aeh_task_enrichment.py new file mode 100644 index 0000000..353a049 --- /dev/null +++ b/abevalflow/harbor_extensions/aeh_task_enrichment.py @@ -0,0 +1,176 @@ +"""Post-process AEH-generated Harbor task packages for OpenShift runs. + +Upstream ``agent_eval.harbor.tasks.generate_tasks`` (v1.0.3) omits: + +1. ``annotations.yaml`` in ``environment/`` and blanks ``dataset.path``, so + Harbor verifiers cannot load judge annotations (``case_dir`` is the + workdir basename, typically ``workspace``). +2. Submission ``plugin_dirs`` / ``skills/`` trees, so Claude Code gets + ``Unknown command: /`` when the instruction uses a slash-skill. + +This module patches generated task packages in-place before ``harbor run``. +It does not modify upstream agent-eval-harness source. +""" + +from __future__ import annotations + +import re +import shutil +from pathlib import Path + +import yaml + +_ANNOTATIONS_STAGE = """ +# ABEvalFlow: stage annotations for Harbor verifier judges. +# reward.py uses case_dir basename (e.g. "workspace") under dataset.path. +if [ -f "@@WORKDIR@@/annotations.yaml" ]; then + _aeh_case="$(basename "@@WORKDIR@@")" + mkdir -p "/tests/cases/${_aeh_case}" + cp "@@WORKDIR@@/annotations.yaml" "/tests/cases/${_aeh_case}/annotations.yaml" +fi +""".lstrip() + + +def enrich_harbor_tasks(tasks_dir: Path, *, config_path: Path) -> int: + """Enrich every Harbor task package under *tasks_dir*. + + Returns the number of task packages updated. + """ + config_path = Path(config_path) + tasks_dir = Path(tasks_dir) + if not tasks_dir.is_dir(): + return 0 + + raw = yaml.safe_load(config_path.read_text()) or {} + submission_root = config_path.parent + dataset_rel = "" + dataset = raw.get("dataset") or {} + if isinstance(dataset, dict): + dataset_rel = str(dataset.get("path") or "").strip() + + plugin_dirs = [] + runner = raw.get("runner") or {} + if isinstance(runner, dict): + plugin_dirs = list(runner.get("plugin_dirs") or []) + + updated = 0 + for task_dir in sorted(p for p in tasks_dir.iterdir() if p.is_dir()): + if not (task_dir / "task.toml").is_file(): + continue + _enrich_one_task( + task_dir, + case_id=task_dir.name, + submission_root=submission_root, + dataset_rel=dataset_rel, + plugin_dirs=plugin_dirs, + ) + updated += 1 + return updated + + +def _enrich_one_task( + task_dir: Path, + *, + case_id: str, + submission_root: Path, + dataset_rel: str, + plugin_dirs: list[str], +) -> None: + env_dir = task_dir / "environment" + env_dir.mkdir(parents=True, exist_ok=True) + tests_dir = task_dir / "tests" + tests_dir.mkdir(parents=True, exist_ok=True) + + # 1) annotations.yaml → environment/ (uploaded to workdir) + annotations_src = _find_annotations(submission_root, dataset_rel, case_id) + if annotations_src is not None: + shutil.copy2(annotations_src, env_dir / "annotations.yaml") + + # 2) Restore dataset.path so load_case_record can resolve annotations + bundled = tests_dir / "eval.yaml" + if bundled.is_file(): + cfg = yaml.safe_load(bundled.read_text()) or {} + if isinstance(cfg.get("dataset"), dict): + cfg["dataset"]["path"] = "cases" + elif "dataset" not in cfg: + cfg["dataset"] = {"path": "cases"} + bundled.write_text(yaml.safe_dump(cfg, sort_keys=False, allow_unicode=True)) + + # 3) Stage annotations in test.sh before reward runs + test_sh = tests_dir / "test.sh" + if test_sh.is_file(): + _inject_annotations_stage(test_sh) + + # 4) Copy plugin skill trees + point Harbor at them + copied_skills = False + for rel in plugin_dirs: + src = (submission_root / rel).resolve() + if not src.is_dir(): + continue + if not _dir_has_skill_md(src): + continue + dest = env_dir / "skills" + if dest.exists(): + shutil.rmtree(dest) + shutil.copytree(src, dest) + copied_skills = True + break # Harbor expects a single skills_dir root + + if copied_skills: + _ensure_skills_dir_in_task_toml(task_dir / "task.toml", "/workspace/skills") + + +def _find_annotations(submission_root: Path, dataset_rel: str, case_id: str) -> Path | None: + candidates: list[Path] = [] + if dataset_rel: + candidates.append(submission_root / dataset_rel / case_id / "annotations.yaml") + candidates.append(submission_root / "cases" / case_id / "annotations.yaml") + for path in candidates: + if path.is_file(): + return path + return None + + +def _dir_has_skill_md(skills_root: Path) -> bool: + if (skills_root / "SKILL.md").is_file(): + return True + for child in skills_root.iterdir(): + if child.is_dir() and (child / "SKILL.md").is_file(): + return True + return False + + +def _inject_annotations_stage(test_sh: Path) -> None: + text = test_sh.read_text() + if "ABEvalFlow: stage annotations" in text: + return + + workdir = "/workspace" + m = re.search(r'--case-dir\s+"([^"]+)"', text) + if m: + workdir = m.group(1) + stage = _ANNOTATIONS_STAGE.replace("@@WORKDIR@@", workdir) + + marker = "mkdir -p /logs/verifier" + if marker in text: + text = text.replace(marker, marker + "\n\n" + stage, 1) + else: + text = stage + "\n" + text + test_sh.write_text(text) + + +def _ensure_skills_dir_in_task_toml(task_toml: Path, skills_dir: str) -> None: + text = task_toml.read_text() + if re.search(r"^\s*skills_dir\s*=", text, flags=re.MULTILINE): + return + if re.search(r'^\s*workdir\s*=\s*"[^"]*"', text, flags=re.MULTILINE): + text = re.sub( + r'^(\s*workdir\s*=\s*"[^"]*")\s*$', + rf'\1\nskills_dir = "{skills_dir}"', + text, + count=1, + flags=re.MULTILINE, + ) + else: + text = text.rstrip() + f'\n\n[environment]\nskills_dir = "{skills_dir}"\n' + task_toml.write_text(text) diff --git a/abevalflow/harbor_extensions/openshift_environment.py b/abevalflow/harbor_extensions/openshift_environment.py new file mode 100644 index 0000000..fda69c0 --- /dev/null +++ b/abevalflow/harbor_extensions/openshift_environment.py @@ -0,0 +1,106 @@ +"""OpenShift-compatible KubernetesEnvironment for Harbor trials. + +Extends agent-eval-harness KubernetesEnvironment with: + +1. emptyDir mounts for /workspace and /tmp (RO-rootfs-friendly workdirs) +2. Ensuring Harbor EnvironmentPaths exist after pod start — required because + Harbor shared-verifier mode redirects test stdout to + ``/logs/verifier/test-stdout.txt`` *before* ``test.sh`` runs. If that + directory is missing, the redirect fails, ``test.sh`` never executes, and + ``download_dir`` returns empty stdout → ``DownloadVerifierDirError``. + +Usage (AEH / agent_eval.harbor.run): + environment: + type: kubernetes + # plus CLI: + # --environment-import-path \\ + # abevalflow.harbor_extensions.openshift_environment:OpenShiftEnvironment + +Classic Harbor (skills_eval_corrections) uses environment.type: openshift and the +fork's built-in OpenShiftEnvironment — do not set an import-path type string in +job config YAML (stock Harbor rejects non-enum types at JobConfig validation). +""" + +from __future__ import annotations + +import base64 +import io +import logging +import shlex +import tarfile +from pathlib import Path + +from agent_eval.harbor.kubernetes import KubernetesEnvironment + +logger = logging.getLogger(__name__) + +# Harbor EnvironmentPaths (+ AEH workdirs) that must exist before verifier runs. +_HARBOR_PATHS = ( + "/logs/agent", + "/logs/verifier", + "/logs/artifacts", + "/tests", + "/solution", + "/workspace", + "/tmp", +) + + +class OpenShiftEnvironment(KubernetesEnvironment): + """KubernetesEnvironment with OpenShift trial-pod prep.""" + + def _pod_manifest(self, image: str, env: dict) -> dict: + """Build pod manifest with emptyDir volumes for writable workdirs.""" + manifest = super()._pod_manifest(image, env) + + pod_spec = manifest["spec"] + container = pod_spec["containers"][0] + + container.setdefault("volumeMounts", []).extend( + [ + {"name": "workspace", "mountPath": "/workspace"}, + {"name": "tmp", "mountPath": "/tmp"}, + ] + ) + pod_spec.setdefault("volumes", []).extend( + [ + {"name": "workspace", "emptyDir": {}}, + {"name": "tmp", "emptyDir": {}}, + ] + ) + + mount_paths = [m.get("mountPath") for m in container.get("volumeMounts", [])] + logger.info("OpenShiftEnvironment injected mounts: %s", mount_paths) + + return manifest + + async def start(self, force_build: bool) -> None: + """Start the pod, then create Harbor paths needed for verifier redirect.""" + await super().start(force_build) + dirs = " ".join(shlex.quote(p) for p in _HARBOR_PATHS) + # mkdir only — image paths are already group-writable; chmod can fail + # with EPERM under OpenShift SCC-assigned UIDs (including on /solution). + await self._checked_exec( + f"mkdir -p {dirs}", + "ensure Harbor EnvironmentPaths before verifier redirect", + ) + logger.info("Ensured Harbor paths exist: %s", ", ".join(_HARBOR_PATHS)) + + async def download_dir(self, source_dir: str, target_dir: Path | str) -> None: + """Download a remote directory using pipefail so missing dirs fail clearly. + + Parent implementation uses ``tar | base64`` without pipefail, so a missing + source directory yields empty stdout with exit code 0 and then + ``tarfile.ReadError: empty file`` wrapped as DownloadVerifierDirError. + """ + res = await self.exec(f"set -o pipefail; tar cf - -C {shlex.quote(source_dir)} . | base64 -w0") + if res.return_code != 0: + raise RuntimeError(f"download_dir {source_dir}: rc={res.return_code} stderr={res.stderr}") + if not res.stdout: + raise RuntimeError( + f"download_dir {source_dir}: empty archive (directory missing or tar produced no stdout)" + ) + Path(target_dir).mkdir(parents=True, exist_ok=True) + raw = base64.b64decode(res.stdout) + with tarfile.open(fileobj=io.BytesIO(raw), mode="r") as tf: + tf.extractall(target_dir, filter="data") diff --git a/scripts/generate_eval_config.py b/scripts/generate_eval_config.py index 830471c..5afe956 100644 --- a/scripts/generate_eval_config.py +++ b/scripts/generate_eval_config.py @@ -121,6 +121,9 @@ def build_variant_config( task: dict[str, Any] = {"path": task_dir} + # Stock Harbor JobConfig validates environment.type as a strict enum + # (openshift|docker|...). The fork's OpenShiftEnvironment is selected via + # this enum — do not put an import path here (that fails Pydantic validation). env_block: dict[str, Any] = { "type": "openshift", "delete": True, diff --git a/tests/test_aeh_task_enrichment.py b/tests/test_aeh_task_enrichment.py new file mode 100644 index 0000000..3e1dce2 --- /dev/null +++ b/tests/test_aeh_task_enrichment.py @@ -0,0 +1,100 @@ +"""Tests for AEH Harbor task enrichment (skills + annotations).""" + +from __future__ import annotations + +from pathlib import Path + +import yaml + +from abevalflow.harbor_extensions.aeh_task_enrichment import enrich_harbor_tasks + + +def _write_generated_task(task_dir: Path, *, workdir: str = "/workspace") -> None: + (task_dir / "environment").mkdir(parents=True) + (task_dir / "tests").mkdir(parents=True) + (task_dir / "task.toml").write_text(f'[environment]\ndocker_image = "img:latest"\nworkdir = "{workdir}"\n') + (task_dir / "tests" / "eval.yaml").write_text( + yaml.safe_dump( + { + "dataset": {"path": "", "schema": "x"}, + "judges": [{"name": "file_created", "check": "return True, 'ok'"}], + }, + sort_keys=False, + ) + ) + (task_dir / "tests" / "test.sh").write_text( + f"""#!/bin/bash +set -o pipefail +mkdir -p /logs/verifier +python3 -m agent_eval.harbor.reward \\ + --config /tests/eval.yaml \\ + --case-dir "{workdir}" \\ + --out-dir /logs/verifier +""" + ) + (task_dir / "tests" / "test.sh").chmod(0o755) + + +def test_enrich_copies_annotations_and_restores_dataset_path(tmp_path: Path): + submission = tmp_path / "submission" + cases = submission / "cases" / "case-001" + cases.mkdir(parents=True) + (cases / "annotations.yaml").write_text( + yaml.safe_dump({"expected_file": "greeting.txt", "expected_content": "Hello"}) + ) + (submission / "eval.yaml").write_text( + yaml.safe_dump( + { + "skill": "demo", + "dataset": {"path": "cases"}, + "runner": {"type": "claude-code"}, + "judges": [{"name": "file_created"}], + } + ) + ) + + tasks = tmp_path / "tasks" + task = tasks / "case-001" + _write_generated_task(task) + + n = enrich_harbor_tasks(tasks, config_path=submission / "eval.yaml") + assert n == 1 + + ann = task / "environment" / "annotations.yaml" + assert ann.is_file() + assert yaml.safe_load(ann.read_text())["expected_file"] == "greeting.txt" + + bundled = yaml.safe_load((task / "tests" / "eval.yaml").read_text()) + assert bundled["dataset"]["path"] == "cases" + + test_sh = (task / "tests" / "test.sh").read_text() + assert "ABEvalFlow: stage annotations" in test_sh + assert "/tests/cases/" in test_sh + + +def test_enrich_copies_skills_and_sets_skills_dir(tmp_path: Path): + submission = tmp_path / "submission" + skill_dir = submission / "skills" / "demo-skill" + skill_dir.mkdir(parents=True) + (skill_dir / "SKILL.md").write_text("---\nname: demo-skill\n---\n# demo\n") + (submission / "cases" / "case-001").mkdir(parents=True) + (submission / "eval.yaml").write_text( + yaml.safe_dump( + { + "skill": "demo-skill", + "dataset": {"path": "cases"}, + "runner": {"type": "claude-code", "plugin_dirs": ["skills"]}, + "judges": [{"name": "exit_success"}], + } + ) + ) + + tasks = tmp_path / "tasks" + task = tasks / "case-001" + _write_generated_task(task) + + enrich_harbor_tasks(tasks, config_path=submission / "eval.yaml") + + assert (task / "environment" / "skills" / "demo-skill" / "SKILL.md").is_file() + toml = (task / "task.toml").read_text() + assert 'skills_dir = "/workspace/skills"' in toml From f024d1bced5c44f3fc292944abbb377576bfdafc Mon Sep 17 00:00:00 2001 From: gziv Date: Mon, 20 Jul 2026 09:55:32 +0300 Subject: [PATCH 4/5] docs(aeh): triggers, investigation notes, and samples Document AEH single/pairwise triggers, MinIO debug layout, and the DownloadVerifierDirError investigation under Docs/investigations/. Co-authored-by: Cursor --- .../aeh_downloadverifiererror.md | 205 ++++++ Docs/trigger_guide.md | 252 ++++++- README.md | 60 +- scripts/misc/trigger_test_runs.sh | 655 +++++++++++------- 4 files changed, 908 insertions(+), 264 deletions(-) create mode 100644 Docs/investigations/aeh_downloadverifiererror.md diff --git a/Docs/investigations/aeh_downloadverifiererror.md b/Docs/investigations/aeh_downloadverifiererror.md new file mode 100644 index 0000000..b05b4fd --- /dev/null +++ b/Docs/investigations/aeh_downloadverifiererror.md @@ -0,0 +1,205 @@ +# AEH DownloadVerifierDirError Investigation + +## Issue Description + +When running AEH (Agent-Eval-Harness) evaluations on OpenShift, Harbor trial pods fail with: + +``` +DownloadVerifierDirError: Failed to download verifier directory from environment +``` + +**Root Cause**: OpenShift's `restricted-v2` Security Context Constraint (SCC) enforces `readOnlyRootFilesystem: true` for all containers. Harbor trial pods need to write to `/workspace` and `/tmp` directories, but these are read-only in the trial pod's root filesystem. + +**Solution Requirement**: Add `emptyDir` volume mounts for `/workspace` and `/tmp` to provide writable storage that complies with OpenShift security policies. + +## Constraints + +1. **Cannot modify upstream code**: We must NOT modify agent-eval-harness or Harbor upstream code +2. **Must use upstream versions**: Work with upstream Harbor (v0.0.18) and agent-eval-harness +3. **Available resources**: We have cloned repositories and virtual environments available for investigation + +## Environment Setup + +- **AEH Image**: `quay.io/ecosystem-appeng/agent-eval-harness:v1.0.3` +- **Harbor Version**: 0.0.18 (from agent-eval-harness) +- **OpenShift SCC**: restricted-v2 (enforces read-only root filesystem) +- **Local Development**: + - agent-eval-harness cloned at `/Users/gziv/Dev/agent-eval-harness` + - ABEvalFlow pipeline repo at `/Users/gziv/Dev/ABEvalFlow` + +## Attempts and Results + +### Attempt 1: Custom OpenShiftEnvironment Class ❌ + +**Approach**: Created custom `OpenShiftEnvironment` class extending `KubernetesEnvironment` to add emptyDir mounts. + +**Implementation**: +1. Created `abevalflow/harbor_extensions/openshift_environment.py`: + ```python + from agent_eval.harbor.kubernetes import KubernetesEnvironment + + class OpenShiftEnvironment(KubernetesEnvironment): + def _pod_manifest(self, image: str, env: dict) -> dict: + manifest = super()._pod_manifest(image, env) + pod_spec = manifest["spec"] + container = pod_spec["containers"][0] + + container.setdefault("volumeMounts", []).extend([ + {"name": "workspace", "mountPath": "/workspace"}, + {"name": "tmp", "mountPath": "/tmp"}, + ]) + pod_spec.setdefault("volumes", []).extend([ + {"name": "workspace", "emptyDir": {}}, + {"name": "tmp", "emptyDir": {}}, + ]) + + return manifest + ``` + +2. Added `abevalflow` module to AEH v1.0.3 image: + ```dockerfile + COPY --chown=1001:0 abevalflow /opt/agent-eval-harness/abevalflow + ``` + +3. Modified `scripts/run_aeh.py` to pass `--environment-import-path`: + ```python + cmd.extend(["--environment-import-path", + "abevalflow.harbor_extensions.openshift_environment:OpenShiftEnvironment"]) + ``` + +4. Updated Tekton evaluate task to export PYTHONPATH: + ```bash + export PYTHONPATH="$PIPELINE_DIR:${PYTHONPATH:-}" + ``` + +**Evidence from logs**: +``` +Running: /opt/app-root/bin/python -m agent_eval.harbor.run --config ... + --environment-import-path abevalflow.harbor_extensions.openshift_environment:OpenShiftEnvironment + +harbor: harbor run -p ... --environment-import-path + abevalflow.harbor_extensions.openshift_environment:OpenShiftEnvironment +``` + +**Result**: ❌ **Still fails with DownloadVerifierDirError** + +The custom environment class is being used (confirmed in logs), but the error persists. + +### Attempt 2: Patching eval.yaml environment.type ❌ + +**Approach**: Patch eval.yaml to set `environment.type` to our custom class, relying on config file instead of CLI arguments. + +**Implementation**: +```python +def _patch_eval_config_for_openshift(self, config: Path, tasks_dir: Path | None) -> Path: + eval_config["environment"]["type"] = \ + "abevalflow.harbor_extensions.openshift_environment:OpenShiftEnvironment" + + # Write to same directory to preserve relative paths + patched_config = config.parent / f"{config.stem}-openshift.yaml" + yaml.dump(eval_config, open(patched_config, "w")) + return patched_config +``` + +**Problem Discovered**: agent_eval.harbor.run has a **default value** for `--env` parameter (`"kubernetes"`), so even without passing `--env` explicitly, it uses the default environment mapping which overrides the config file. + +**Result**: ❌ **Did not work** - reverted to explicit `--environment-import-path` approach + +## Current Status + +**Commit**: `a8263f6` - `fix: pass --environment-import-path explicitly for OpenShiftEnvironment` + +**Test Run**: `aeh-v103-test-whsz7` + +**Status**: ❌ **Still failing with DownloadVerifierDirError** + +Despite successfully: +- Creating custom OpenShiftEnvironment class +- Including abevalflow module in v1.0.3 image +- Passing --environment-import-path correctly +- Confirming the custom class is being used (from logs and trial `config.json`) + +### Step 1 evidence (debug tarball `2026-07-16__13-51-50.tar.gz`) + +Confirmed from `case-001__XpJxhiR/exception.txt` / `result.json`: + +- Environment import path used: `abevalflow.harbor_extensions.openshift_environment:OpenShiftEnvironment` +- Wrapper: `DownloadVerifierDirError: Failed to download verifier directory from environment` +- **`__cause__`**: `tarfile.ReadError: empty file` while AEH `KubernetesEnvironment.download_dir` opens the tar stream from: + `tar cf - -C /logs/verifier . | base64 -w0` +- Host `verifier/` directory in the job artifact is **empty** +- Related: `download_file /logs/artifacts: No such file or directory` (best-effort artifact download) +- Agent did reach `/workspace` and teed to `/logs/agent/claude-code.txt` (agent then failed with `Unknown command: /aeh-hello-world-single` — separate from verifier FS) + +Interpretation: download of `/logs/verifier` returned empty stdout (typical when `tar` fails on a missing/unreadable directory but `base64` still exits 0). Current `OpenShiftEnvironment` mounts only `/workspace` and `/tmp`, not Harbor paths `/logs`, `/tests`, `/solution`. + +## Step 2 KEEP_RUN probe (`aeh-keeprun-probe-zdksm`) + +Trial pod `aeh-case-001-bmkubbm-env`: + +| Check | Result | +|---|---| +| `OpenShiftEnvironment` mounts present | Yes: emptyDir `/workspace`, `/tmp` | +| `readOnlyRootFilesystem` | **Not set** — root overlay is `rw` | +| Writability `/logs|/tests|/solution|/tmp|/workspace` | **All writable** without extra mounts | +| `/logs/verifier` after failed trial | **Missing** | +| `tar … \| base64` on missing dir | `out_len=0` (base64 masks tar failure) | + +**Decision gate:** emptyDir expansion for `/logs` is **not** the root cause on this cluster. + +## Confirmed root cause + +Harbor **shared-verifier** mode builds: +`/tests/test.sh > /logs/verifier/test-stdout.txt` +**before** `test.sh` runs. Shared verifier does **not** call `empty_dirs(/logs/verifier)` (only separate-verifier mode does). + +Reproduced on the kept pod: +1. Without `/logs/verifier` → redirect fails → `test.sh` never runs → download empty → `DownloadVerifierDirError` +2. With `mkdir -p /logs/verifier` first → `test.sh` produces `reward.json` successfully + +## Fix + +In `abevalflow/harbor_extensions/openshift_environment.py`: +1. After `start()`, `mkdir -p` Harbor paths including `/logs/verifier` (no chmod — SCC UIDs get EPERM) +2. Override `download_dir` with `set -o pipefail` so missing dirs fail clearly +3. Keep `/workspace`+`/tmp` emptyDirs; log injected mounts +4. `aggregate_aeh.py` emits `AnalysisResult`-compatible fields so analyze does not crash +5. `aggregate_scorecard.py` accepts `eval-engine=aeh`; `AEHEngine` handles `None` mean_reward + +## Retest (`aeh-verifier-fix3-k9mbp`) + +Blocked initially by PVC quota (`persistentvolumeclaims=10`); freed by deleting 5 old failed PipelineRuns and clearing stuck Terminating PVC finalizers. + +**Result:** `DownloadVerifierDirError` **gone**. Trial completed with rewards: +- Exceptions: 0 +- Exit_Success: 1.000 +- File_Created: 0.000 / Reward: 0.000 (real eval miss — agent did not create `output/greeting.txt`) +- `step-aeh-eval` exits 1 due to AEH `REGRESSIONS` detection (not infra FS) + +Remaining non-blocker: evaluate task fails on regression exit code; analyze skipped because evaluate failed. + +## Files Changed + +- `abevalflow/harbor_extensions/__init__.py` (created) +- `abevalflow/harbor_extensions/openshift_environment.py` (created) +- `containers/agent-eval-harness/Containerfile` (modified - added abevalflow COPY) +- `scripts/run_aeh.py` (modified - added environment-import-path logic) +- `pipeline/tasks/phases/evaluate.yaml` (modified - added PYTHONPATH export) +- `pipeline/pipelines/ci-pipeline-dev.yaml` (modified - updated to v1.0.3) + +## Key Learnings + +1. **agent_eval.harbor.run argument precedence**: + - `--environment-import-path` (highest) + - `--env` with default value "kubernetes" + - config file `environment.type` (lowest) + +2. **Relative paths in config files**: When patching eval.yaml, must write to same directory as original to preserve relative paths (cases/, skills/, etc.) + +3. **Module availability**: Custom environment classes must be importable where agent_eval.harbor.run executes (dispatcher pod), achieved via PYTHONPATH and image COPY + +## References + +- OpenShift restricted-v2 SCC: https://docs.openshift.com/container-platform/4.x/authentication/managing-security-context-constraints.html +- Harbor KubernetesEnvironment: `/Users/gziv/Dev/agent-eval-harness/agent_eval/harbor/kubernetes.py` +- agent_eval.harbor.run: `/Users/gziv/Dev/agent-eval-harness/agent_eval/harbor/run.py` diff --git a/Docs/trigger_guide.md b/Docs/trigger_guide.md index c317e99..8d2c64b 100644 --- a/Docs/trigger_guide.md +++ b/Docs/trigger_guide.md @@ -261,6 +261,251 @@ tkn pipeline start abevalflow-pipeline \ A2A evaluation tests both agent functionality and protocol compliance. See `examples/a2a-skill/` for a complete example. +### AEH Format (Agent-Eval-Harness) + +For evaluating agents using the Agent-Eval-Harness framework, use the AEH format. +This mode provides flexible judge-based evaluation with support for LLM judges, +custom scoring, and detailed per-case analysis. Trials run as Harbor jobs on +OpenShift (OpenShiftEnvironment). + +**Verified smoke samples** (in [skill-submissions](https://github.com/RHEcosystemAppEng/skill-submissions)): + +| Mode | Branch | `submission-dir` | +|------|--------|------------------| +| Single | `eval/aeh-hello-world-single` | `aeh-hello-world-single` | +| Pairwise | `eval/aeh-hello-world-pairwise` | `aeh-hello-world-pairwise` | + +#### AEH single mode + +``` +submissions// +├── metadata.yaml # Required — eval_engine: aeh +├── eval.yaml # Required — AEH evaluation config (judges, thresholds) +├── skills/ # Optional — nested or flat SKILL.md for treatment +│ └── …/SKILL.md +└── cases/ + └── case-001/ + ├── input.yaml # Required — task prompt / agent instruction + └── annotations.yaml # Optional — ground truth for judges +``` + +**eval.yaml essentials** (LiteLLM model ids, not Anthropic native names): + +```yaml +name: aeh-hello-world-single +skill: aeh-hello-world-single + +runner: + type: claude-code + effort: low + settings: + permission_mode: bypassPermissions + +models: + skill: claude-sonnet # LiteLLM alias (override via aeh-model-override) + judge: claude-sonnet + +dataset: + path: cases + +outputs: + - path: output # Required for artifact collection / pairwise + +judges: + - name: exit_success + type: check + # … + - name: file_created + type: check + # … +``` + +**Trigger single AEH** (params that work on OpenShift with in-cluster LiteLLM): + +```bash +# Dev pipeline + personal namespace example (guy-ziv-evalflow). +# Prod: abevalflow-pipeline / ab-eval-flow +oc create -n guy-ziv-evalflow -f - <<'YAML' +apiVersion: tekton.dev/v1 +kind: PipelineRun +metadata: + generateName: aeh-single- +spec: + pipelineRef: + name: abevalflow-pipeline-dev + params: + - name: repo-url + value: "https://github.com/RHEcosystemAppEng/skill-submissions.git" + - name: revision + value: "eval/aeh-hello-world-single" + - name: submission-dir + value: "aeh-hello-world-single" + - name: eval-engine + value: "aeh" + - name: aeh-mode + value: "single" + - name: pipeline-repo-revision + value: "APPENG-5300/aeh-engine-integration" # or main once merged + - name: llm-model + value: "claude-sonnet" + - name: llm-api-base + value: "http://litellm.ab-eval-flow.svc.cluster.local:4000" + - name: llm-api-key + value: "mock" + - name: aeh-model-override + value: "claude-sonnet" + - name: aeh-image + value: "quay.io/ecosystem-appeng/agent-eval-harness:v1.0.3" + taskRunTemplate: + serviceAccountName: pipeline + timeouts: + pipeline: 2h0m0s + tasks: 1h30m0s + workspaces: + - name: shared-workspace + volumeClaimTemplate: + spec: + accessModes: [ReadWriteOnce] + resources: + requests: + storage: 5Gi +YAML +``` + +Or with `tkn`: + +```bash +tkn pipeline start abevalflow-pipeline-dev \ + -p repo-url=https://github.com/RHEcosystemAppEng/skill-submissions.git \ + -p revision=eval/aeh-hello-world-single \ + -p submission-dir=aeh-hello-world-single \ + -p eval-engine=aeh \ + -p aeh-mode=single \ + -p pipeline-repo-revision=APPENG-5300/aeh-engine-integration \ + -p llm-model=claude-sonnet \ + -p llm-api-base=http://litellm.ab-eval-flow.svc.cluster.local:4000 \ + -p llm-api-key=mock \ + -p aeh-model-override=claude-sonnet \ + -p aeh-image=quay.io/ecosystem-appeng/agent-eval-harness:v1.0.3 \ + -w name=shared-workspace,volumeClaimTemplateFile=pipeline/triggers/pvc-template.yaml \ + -n guy-ziv-evalflow +``` + +AEH-specific parameters: +- `aeh-model-override`: Override `models.skill` from eval.yaml (use LiteLLM aliases such as `claude-sonnet`) +- `aeh-judge-model-override`: Override `models.judge` from eval.yaml +- `aeh-mode`: `single` (default) or `pairwise` +- `aeh-control-config` / `aeh-treatment-config`: Pairwise config filenames (defaults: `eval-control.yaml` / `eval-treatment.yaml`) +- `aeh-image`: Harbor trial image (use `quay.io/ecosystem-appeng/agent-eval-harness:v1.0.3` or newer) +- `aeh-runner`: Execution backend — currently `harbor` only + +**Note on execution backends:** +- **harbor** (default): Containerized execution in OpenShift trial pods via AEH’s OpenShiftEnvironment. +- **vanilla**: Not yet implemented in ABEvalFlow. + +#### AEH Pairwise A/B Testing + +Pairwise runs control then treatment on the same cases, then `score.py pairwise` +(LLM judge, position-swapped). After compare, the pipeline regenerates treatment +`report.html` with `--baseline` so the HTML includes the pairwise section. + +**Pairwise submission structure:** + +``` +submissions// +├── metadata.yaml # Required — eval_engine: aeh +├── eval-control.yaml # Required — control/baseline (often skill: "") +├── eval-treatment.yaml # Required — treatment (with skill package) +├── skills//SKILL.md # Treatment skill (optional nested layout) +└── cases/ + └── case-001/ + └── input.yaml +``` + +Both configs must share the same `skill:` namespace used for +`$AGENT_EVAL_RUNS_DIR///`. Control may set `skill: ""` for an +unskilled baseline while treatment sets the real skill package name — the +pipeline still keys runs under the treatment skill name. + +**outputs:** is required when using a pairwise LLM judge (artifacts must be +bridged into `cases///` for the judge). + +**Trigger pairwise** (same LiteLLM / image params as single): + +```bash +oc create -n guy-ziv-evalflow -f - <<'YAML' +apiVersion: tekton.dev/v1 +kind: PipelineRun +metadata: + generateName: aeh-pairwise- +spec: + pipelineRef: + name: abevalflow-pipeline-dev + params: + - name: repo-url + value: "https://github.com/RHEcosystemAppEng/skill-submissions.git" + - name: revision + value: "eval/aeh-hello-world-pairwise" + - name: submission-dir + value: "aeh-hello-world-pairwise" + - name: eval-engine + value: "aeh" + - name: aeh-mode + value: "pairwise" + - name: pipeline-repo-revision + value: "APPENG-5300/aeh-engine-integration" + - name: llm-model + value: "claude-sonnet" + - name: llm-api-base + value: "http://litellm.ab-eval-flow.svc.cluster.local:4000" + - name: llm-api-key + value: "mock" + - name: aeh-model-override + value: "claude-sonnet" + - name: aeh-image + value: "quay.io/ecosystem-appeng/agent-eval-harness:v1.0.3" + taskRunTemplate: + serviceAccountName: pipeline + timeouts: + pipeline: 2h0m0s + tasks: 1h30m0s + workspaces: + - name: shared-workspace + volumeClaimTemplate: + spec: + accessModes: [ReadWriteOnce] + resources: + requests: + storage: 5Gi +YAML +``` + +**Pairwise results:** + +| Field | Meaning | +|-------|---------| +| `wins_a` | Treatment preferred | +| `wins_b` | Control preferred | +| `ties` | No preference | +| `win_rate` | `wins_a / (wins_a + wins_b + ties)` — ties are non-wins | + +Recommendation: pass when treatment wins ≥50% of cases, **or** when the run is +all-ties (no decisive losses). Scores still report the honest `win_rate` (e.g. +`0/1` tie → `0%` with recommendation `pass`). + +**Where pairwise appears in MinIO:** + +| Path | Contents | +|------|----------| +| `debug/aeh/treatment-*/summary.yaml` → `pairwise:` | Wins/ties + LLM reasoning (AEH native) | +| `debug/aeh/treatment-*/report.html` | HTML regenerated with `--baseline` (includes pairwise) | +| `report.json` → `pairwise` | Aggregated ABEvalFlow report | +| `debug/harbor/control|treatment//` | Raw Harbor trial trees | +| `debug/aeh/control-*/` / `treatment-*/` | AEH mapped runs (`summary.yaml`, `run_result.json`, `cases/`) | + +See skill-submissions branches above for complete samples. Or use +`./scripts/misc/trigger_test_runs.sh` (includes AEH single + pairwise). + ### AI-Assisted Mode If you only have the skill definition and want the pipeline to generate the @@ -508,7 +753,10 @@ Typical runtime: **5-30 minutes** depending on evaluation engine and task comple - `scorecard.json` — unified verdict combining all evaluation gates (see below) - `security-scan.json` / `security-scan.sarif` — security scan findings - `generated/` — AI-generated files (instruction.md, test_outputs.py, evals.json) -- `debug/` — trial logs, agent outputs, error traces +- `debug/` — engine-specific trial / run trees + - **Harbor / A2A:** `debug/` trial trees (`agent/`, `verifier/`, …) + - **AEH:** `debug/harbor/{control,treatment}//` (raw Harbor jobs) + and `debug/aeh//` (`summary.yaml`, `report.html`, `run_result.json`, `cases/`) **PostgreSQL database:** - `analysis_results` table — evaluation summaries (pass rates, uplift, p-values) @@ -522,7 +770,7 @@ a different aspect: | Gate Type | Gate Name | What it checks | |-----------|-----------|----------------| -| **Engine** | `evaluation` | A/B evaluation results (Harbor, ASE, A2A, MCPChecker) | +| **Engine** | `evaluation` | A/B evaluation results (Harbor, ASE, A2A, MCPChecker, AEH) | | **Security** | `security` | Security vulnerabilities (Cisco scanner) | | **Quality** | `quality` | Test coherence, coverage, clarity (LLM review) | diff --git a/README.md b/README.md index 91beca3..c48dfd2 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ The pipeline executes in six main stages, with engine-specific steps within each - **Security Scan** — Cisco AI Defense scan for prompt injection, data exfiltration risks ### 3. Evaluate -Four evaluation engines, each suited for different artifact types: +Five evaluation engines, each suited for different artifact types: | Engine | Evaluates | Comparison Mode | Container Isolation | |--------|-----------|-----------------|---------------------| @@ -41,6 +41,7 @@ Four evaluation engines, each suited for different artifact types: | **ASE** | Skills only | A/B (treatment vs control) | No | | **A2A** | A2A-protocol agents | A/B (treatment vs control) | Yes | | **MCPChecker** | MCP servers | Single-agent task verification | No | +| **AEH** | Agents, skills | Judge-based evaluation | Yes (K8s pods) | ### 4. Analyze - Compute pass rates, uplift (gap), statistical significance (p-value) @@ -61,7 +62,7 @@ All flow configuration is defined in `metadata.yaml` within each submission: ```yaml name: my-submission -eval_engine: harbor # harbor, ase, a2a, or mcpchecker +eval_engine: harbor # harbor, ase, a2a, mcpchecker, or aeh persona: general # Agent persona for Harbor/A2A experiment: @@ -115,7 +116,7 @@ ABEvalFlow/ ## Evaluation Engines -The pipeline supports four evaluation engines, each suited for different artifact types: +The pipeline supports five evaluation engines, each suited for different artifact types: | Engine | Artifact Type | Use Case | Comparison | Container Isolation | |--------|---------------|----------|------------|---------------------| @@ -123,6 +124,7 @@ The pipeline supports four evaluation engines, each suited for different artifac | **ASE** | Skills only | Lightweight LLM-as-judge assertions | A/B (with vs without skill) | No | | **A2A** | A2A Agents | A2A-protocol compliant agent evaluation | A/B (treatment vs control) | Yes | | **MCPChecker** | MCP Servers | MCP server/tool verification | Single-agent task verification | No | +| **AEH** | Agents, Skills | Agent-Eval-Harness judge-based evaluation | Single or pairwise | Yes (K8s pods) | Engines are implemented in `abevalflow/engines/` using a registry pattern: @@ -133,6 +135,7 @@ abevalflow/engines/ ├── harbor.py # Harbor A/B evaluation ├── ase.py # ASE LLM-as-judge evaluation ├── a2a.py # A2A protocol evaluation +├── aeh.py # Agent-Eval-Harness evaluation └── mcpchecker.py # MCPChecker task verification ``` @@ -144,7 +147,7 @@ Gates are evaluation checkpoints that produce standardized results. The unified | Category | Policy Key | Purpose | Implementation | |----------|------------|---------|----------------| -| **evaluation** | `evaluation` | Results from the selected eval engine | Harbor, ASE, A2A, or MCPChecker | +| **evaluation** | `evaluation` | Results from the selected eval engine | Harbor, ASE, A2A, MCPChecker, or AEH | | **security** | `security` | Security scanning results | Cisco AI Defense scanner | | **quality** | `quality` | Quality review results | LLM-powered review | @@ -611,7 +614,54 @@ my-agent-eval/ Harbor creates treatment/control container variants and runs A/B comparison. -See [Trigger Guide](Docs/trigger_guide.md) for detailed submission and trigger instructions. +### AEH Submission (Agent-Eval-Harness) + +For evaluating agents using the Agent-Eval-Harness framework with flexible LLM judges. +Trials run as Harbor jobs on OpenShift. + +**Single** (`aeh-mode=single`): + +``` +my-aeh-eval/ +├── metadata.yaml # eval_engine: aeh (required) +├── eval.yaml # AEH config (models, judges, thresholds, outputs) +├── skills/…/SKILL.md # Optional skill package +└── cases/ + └── case-001/ + └── input.yaml +``` + +**Pairwise** (`aeh-mode=pairwise`): + +``` +my-aeh-pairwise/ +├── metadata.yaml +├── eval-control.yaml # Baseline (often unskilled) +├── eval-treatment.yaml # Treatment + pairwise LLM judge +├── skills//SKILL.md +└── cases/ + └── case-001/ + └── input.yaml +``` + +Verified smoke samples in [skill-submissions](https://github.com/RHEcosystemAppEng/skill-submissions): +- Branch `eval/aeh-hello-world-single` → `aeh-hello-world-single` +- Branch `eval/aeh-hello-world-pairwise` → `aeh-hello-world-pairwise` + +Working trigger defaults: LiteLLM `claude-sonnet` via +`http://litellm.ab-eval-flow.svc.cluster.local:4000`, AEH image +`quay.io/ecosystem-appeng/agent-eval-harness:v1.0.3`. + +MinIO layout for AEH: `debug/harbor/` (raw Harbor jobs) + `debug/aeh/` +(`summary.yaml`, `report.html`, `cases/`). Pairwise HTML is regenerated with +`--baseline` so treatment `report.html` includes the pairwise section. + +See [Trigger Guide](Docs/trigger_guide.md) for full YAML examples, or run: + +```bash +./scripts/misc/trigger_test_runs.sh # all engines including AEH +./scripts/misc/trigger_test_runs.sh "$(git branch --show-current)" aeh # AEH only +``` ## LLM Access diff --git a/scripts/misc/trigger_test_runs.sh b/scripts/misc/trigger_test_runs.sh index d7c5219..b715094 100755 --- a/scripts/misc/trigger_test_runs.sh +++ b/scripts/misc/trigger_test_runs.sh @@ -1,257 +1,398 @@ -#!/usr/bin/env bash -set -euo pipefail - -# Trigger all 6 test pipeline runs for validating ABEvalFlow changes -# Usage: ./scripts/trigger_test_runs.sh [branch-name] -# If branch-name is not provided, uses current git branch - -BRANCH="${1:-$(git rev-parse --abbrev-ref HEAD)}" -NAMESPACE="ab-eval-flow" - -echo "Triggering 6 test runs with pipeline-repo-revision: $BRANCH" -echo "" - -cat << EOF | oc create -f - -# Monitoring pipelines (3) -apiVersion: tekton.dev/v1 -kind: PipelineRun -metadata: - generateName: harbor-test- - namespace: $NAMESPACE -spec: - pipelineRef: - name: abevalflow-monitoring-pipeline - params: - - name: repo-url - value: "https://github.com/RHEcosystemAppEng/skill-submissions.git" - - name: revision - value: "main" - - name: submission-dir - value: "hello-world" - - name: eval-engine - value: "harbor" - - name: pipeline-repo-revision - value: "$BRANCH" - - name: llm-model - value: "claude-sonnet" - - name: llm-api-base - value: "http://litellm.ab-eval-flow.svc:4000" - - name: llm-api-key - value: "mock" - timeouts: - pipeline: "2h" - tasks: "1h30m" - taskRunTemplate: - serviceAccountName: pipeline - workspaces: - - name: shared-workspace - volumeClaimTemplate: - spec: - accessModes: [ReadWriteOnce] - resources: - requests: - storage: 1Gi ---- -apiVersion: tekton.dev/v1 -kind: PipelineRun -metadata: - generateName: a2a-verify- - namespace: $NAMESPACE -spec: - pipelineRef: - name: abevalflow-monitoring-pipeline - params: - - name: repo-url - value: "https://github.com/RHEcosystemAppEng/ABEvalFlow.git" - - name: revision - value: "main" - - name: submission-dir - value: "a2a-agent-eval" - - name: eval-engine - value: "a2a" - - name: pipeline-repo-revision - value: "$BRANCH" - - name: agent-endpoint - value: "http://lightspeed-agent.ab-eval-flow.svc:8000" - - name: llm-model - value: "claude-sonnet" - - name: llm-api-base - value: "http://litellm.ab-eval-flow.svc:4000" - - name: llm-api-key - value: "mock" - timeouts: - pipeline: "2h" - tasks: "1h30m" - taskRunTemplate: - serviceAccountName: pipeline - workspaces: - - name: shared-workspace - volumeClaimTemplate: - spec: - accessModes: [ReadWriteOnce] - resources: - requests: - storage: 1Gi ---- -apiVersion: tekton.dev/v1 -kind: PipelineRun -metadata: - generateName: ase-verify- - namespace: $NAMESPACE -spec: - pipelineRef: - name: abevalflow-monitoring-pipeline - params: - - name: repo-url - value: "https://github.com/RHEcosystemAppEng/skill-submissions.git" - - name: revision - value: "eval/hello-world-full" - - name: submission-dir - value: "hello-world-full" - - name: eval-engine - value: "ase" - - name: pipeline-repo-revision - value: "$BRANCH" - - name: llm-model - value: "claude-sonnet" - - name: llm-api-base - value: "http://litellm.ab-eval-flow.svc:4000" - - name: llm-api-key - value: "mock" - timeouts: - pipeline: "2h" - tasks: "1h30m" - taskRunTemplate: - serviceAccountName: pipeline - workspaces: - - name: shared-workspace - volumeClaimTemplate: - spec: - accessModes: [ReadWriteOnce] - resources: - requests: - storage: 1Gi ---- -# CI pipelines (3) -apiVersion: tekton.dev/v1 -kind: PipelineRun -metadata: - generateName: ci-harbor- - namespace: $NAMESPACE -spec: - pipelineRef: - name: abevalflow-pipeline - params: - - name: repo-url - value: "https://github.com/RHEcosystemAppEng/skill-submissions.git" - - name: revision - value: "main" - - name: submission-dir - value: "hello-world" - - name: eval-engine - value: "harbor" - - name: pipeline-repo-revision - value: "$BRANCH" - - name: llm-model - value: "claude-sonnet" - - name: llm-api-base - value: "http://litellm.ab-eval-flow.svc:4000" - - name: llm-api-key - value: "mock" - timeouts: - pipeline: "2h" - tasks: "1h30m" - taskRunTemplate: - serviceAccountName: pipeline - workspaces: - - name: shared-workspace - volumeClaimTemplate: - spec: - accessModes: [ReadWriteOnce] - resources: - requests: - storage: 1Gi ---- -apiVersion: tekton.dev/v1 -kind: PipelineRun -metadata: - generateName: ci-a2a- - namespace: $NAMESPACE -spec: - pipelineRef: - name: abevalflow-pipeline - params: - - name: repo-url - value: "https://github.com/RHEcosystemAppEng/ABEvalFlow.git" - - name: revision - value: "main" - - name: submission-dir - value: "a2a-agent-eval" - - name: eval-engine - value: "a2a" - - name: pipeline-repo-revision - value: "$BRANCH" - - name: agent-endpoint - value: "http://lightspeed-agent.ab-eval-flow.svc:8000" - - name: llm-model - value: "claude-sonnet" - - name: llm-api-base - value: "http://litellm.ab-eval-flow.svc:4000" - - name: llm-api-key - value: "mock" - timeouts: - pipeline: "2h" - tasks: "1h30m" - taskRunTemplate: - serviceAccountName: pipeline - workspaces: - - name: shared-workspace - volumeClaimTemplate: - spec: - accessModes: [ReadWriteOnce] - resources: - requests: - storage: 1Gi ---- -apiVersion: tekton.dev/v1 -kind: PipelineRun -metadata: - generateName: ci-ase- - namespace: $NAMESPACE -spec: - pipelineRef: - name: abevalflow-pipeline - params: - - name: repo-url - value: "https://github.com/RHEcosystemAppEng/skill-submissions.git" - - name: revision - value: "eval/hello-world-full" - - name: submission-dir - value: "hello-world-full" - - name: eval-engine - value: "ase" - - name: pipeline-repo-revision - value: "$BRANCH" - - name: llm-model - value: "claude-sonnet" - - name: llm-api-base - value: "http://litellm.ab-eval-flow.svc:4000" - - name: llm-api-key - value: "mock" - timeouts: - pipeline: "2h" - tasks: "1h30m" - taskRunTemplate: - serviceAccountName: pipeline - workspaces: - - name: shared-workspace - volumeClaimTemplate: - spec: - accessModes: [ReadWriteOnce] - resources: - requests: - storage: 1Gi -EOF - -echo "" -echo "Done! Monitor with: oc get pipelineruns -n $NAMESPACE --sort-by=.metadata.creationTimestamp | tail -10" +#!/usr/bin/env bash +set -euo pipefail + +# Trigger ABEvalFlow test PipelineRuns. +# +# Usage: +# ./scripts/misc/trigger_test_runs.sh [branch-name] [all|aeh|legacy] +# +# branch-name Pipeline repo revision (default: current git branch) +# mode all — Harbor/A2A/ASE monitoring+CI plus AEH single+pairwise (default) +# aeh — AEH single + pairwise only (verified smoke samples) +# legacy — original 6 Harbor/A2A/ASE runs only +# +# Env overrides: +# NAMESPACE OpenShift namespace (default: guy-ziv-evalflow) +# PIPELINE_CI CI pipeline name (default: abevalflow-pipeline-dev) +# PIPELINE_MONITOR Monitoring pipeline name (default: abevalflow-monitoring-pipeline) +# LLM_API_BASE LiteLLM base URL +# AEH_IMAGE AEH/Harbor trial image +# SKILL_REPO skill-submissions git URL +# +# Verified AEH samples (skill-submissions): +# single: revision=eval/aeh-hello-world-single submission-dir=aeh-hello-world-single +# pairwise: revision=eval/aeh-hello-world-pairwise submission-dir=aeh-hello-world-pairwise + +BRANCH="${1:-}" +if [[ -z "$BRANCH" ]]; then + BRANCH="$(git rev-parse --abbrev-ref HEAD)" +fi +MODE="${2:-all}" + +NAMESPACE="${NAMESPACE:-guy-ziv-evalflow}" +PIPELINE_CI="${PIPELINE_CI:-abevalflow-pipeline-dev}" +PIPELINE_MONITOR="${PIPELINE_MONITOR:-abevalflow-monitoring-pipeline}" +LLM_API_BASE="${LLM_API_BASE:-http://litellm.ab-eval-flow.svc.cluster.local:4000}" +AEH_IMAGE="${AEH_IMAGE:-quay.io/ecosystem-appeng/agent-eval-harness:v1.0.3}" +SKILL_REPO="${SKILL_REPO:-https://github.com/RHEcosystemAppEng/skill-submissions.git}" +LLM_MODEL="${LLM_MODEL:-claude-sonnet}" + +echo "pipeline-repo-revision: $BRANCH" +echo "namespace: $NAMESPACE" +echo "mode: $MODE" +echo "ci pipeline: $PIPELINE_CI" +echo "llm-api-base: $LLM_API_BASE" +echo "" + +trigger_aeh() { + echo "=== Triggering AEH single + pairwise (verified smoke samples) ===" + cat <&2 + exit 1 + ;; +esac + +echo "" +echo "Done! Monitor with:" +echo " oc get pipelineruns -n $NAMESPACE --sort-by=.metadata.creationTimestamp | tail -15" From 7b7896b38d0d75839f262cc05df31572166af393 Mon Sep 17 00:00:00 2001 From: gziv Date: Mon, 20 Jul 2026 13:14:52 +0300 Subject: [PATCH 5/5] fix(aeh): make evaluate.yaml AEH threshold helper YAML-safe Multiline python -c broke Tekton Task apply; use a one-liner instead. Co-authored-by: Cursor --- pipeline/tasks/phases/evaluate.yaml | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/pipeline/tasks/phases/evaluate.yaml b/pipeline/tasks/phases/evaluate.yaml index 0cd3b26..e536bcc 100644 --- a/pipeline/tasks/phases/evaluate.yaml +++ b/pipeline/tasks/phases/evaluate.yaml @@ -1303,11 +1303,7 @@ spec: export PYTHONPATH="$PIPELINE_DIR" pip install --quiet --no-cache-dir pyyaml 2>&1 | tail -1 - AEH_THRESHOLD=$(python3 -c " -from pathlib import Path -from abevalflow.aeh_scoring import resolve_evaluation_threshold -print(resolve_evaluation_threshold(Path('$SUBMISSION_DIR') / 'metadata.yaml')) -") + AEH_THRESHOLD=$(python3 -c "from pathlib import Path; from abevalflow.aeh_scoring import resolve_evaluation_threshold; print(resolve_evaluation_threshold(Path('$SUBMISSION_DIR') / 'metadata.yaml'))") echo "AEH evaluation threshold: $AEH_THRESHOLD" if [ -f "$OUT_DIR/summary.yaml" ] || [ -f "$OUT_DIR/run_result.json" ]; then @@ -1489,11 +1485,7 @@ print(resolve_evaluation_threshold(Path('$SUBMISSION_DIR') / 'metadata.yaml')) export PYTHONPATH="$PIPELINE_DIR" pip install --quiet --no-cache-dir pyyaml 2>&1 | tail -1 - AEH_THRESHOLD=$(python3 -c " -from pathlib import Path -from abevalflow.aeh_scoring import resolve_evaluation_threshold -print(resolve_evaluation_threshold(Path('$SUBMISSION_DIR') / 'metadata.yaml')) -") + AEH_THRESHOLD=$(python3 -c "from pathlib import Path; from abevalflow.aeh_scoring import resolve_evaluation_threshold; print(resolve_evaluation_threshold(Path('$SUBMISSION_DIR') / 'metadata.yaml'))") echo "AEH evaluation threshold: $AEH_THRESHOLD" python "$PIPELINE_DIR/scripts/aggregate_aeh.py" "$TREATMENT_DIR" \