diff --git a/.github/scripts/dependency_watch_completion.py b/.github/scripts/dependency_watch_completion.py new file mode 100644 index 0000000..78e3d74 --- /dev/null +++ b/.github/scripts/dependency_watch_completion.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +"""Emit the dependency-watch AutomationTerminalStateV1 completion line.""" + +from __future__ import annotations + +import json +import os +from datetime import datetime, timezone +from pathlib import Path +from typing import Mapping + + +AUTOMATION_ID = "github:AssistSupport/dependency-watch" + + +def build_payload(env: Mapping[str, str], observed_at: str) -> dict: + job_ok = env.get("JOB_STATUS") == "success" + action = env.get("ISSUE_ACTION", "") + issue_number = env.get("ISSUE_NUMBER", "") + issue_outcome = env.get("ISSUE_OUTCOME", "") + issue_attempted = issue_outcome not in ("", "skipped") + readback_required = issue_attempted + readback_verified = readback_required and env.get("ISSUE_READBACK") == "true" + partial = readback_required and not readback_verified + destination_id = None + if readback_required: + destination_id = ( + f"issue:{issue_number}" + if issue_number + else "repo:saagpatel/AssistSupport/issues#Dependency Watch Alerts" + ) + + if partial: + state = "partial" + elif job_ok: + state = "succeeded" + else: + state = "failed" + + succeeded = state == "succeeded" + return { + "schema": "AutomationTerminalStateV1", + "automation_id": AUTOMATION_ID, + "state": state, + "completed": succeeded, + "partial": partial, + "skipped": False, + "mutation_count": 1 if action else 0, + "destination_readback": { + "required": readback_required, + "verified": readback_verified, + "destination_id": destination_id, + "observed_at": observed_at if readback_required else None, + "evidence": { + "issue_action": action or ("unknown" if issue_attempted else "not_required"), + "issue_step_outcome": issue_outcome or "unknown", + }, + }, + "operator_action_required": not succeeded, + "can_auto_archive": succeeded, + "observed_at": observed_at, + } + + +def main() -> int: + observed_at = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + payload = build_payload(os.environ, observed_at) + line = "automation_completion: " + json.dumps(payload, separators=(",", ":")) + print(line) + + summary_path = os.environ.get("GITHUB_STEP_SUMMARY") + if summary_path: + with Path(summary_path).open("a", encoding="utf-8") as summary: + summary.write("\n" + line + "\n") + + readback = payload["destination_readback"] + if readback["required"] and not readback["verified"]: + print("dependency issue destination readback was not verified", file=os.sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/workflows/dependency-watch.yml b/.github/workflows/dependency-watch.yml index 3ff9cbf..a6828f1 100644 --- a/.github/workflows/dependency-watch.yml +++ b/.github/workflows/dependency-watch.yml @@ -213,43 +213,4 @@ jobs: ISSUE_ACTION: ${{ steps.alert_issue.outputs.issue_action }} ISSUE_NUMBER: ${{ steps.alert_issue.outputs.issue_number }} ISSUE_READBACK: ${{ steps.alert_issue.outputs.destination_readback_verified }} - run: | - python3 <<'PY' - import json - import os - from datetime import datetime, timezone - - job_ok = os.environ.get("JOB_STATUS") == "success" - action = os.environ.get("ISSUE_ACTION", "") - issue_number = os.environ.get("ISSUE_NUMBER", "") - issue_outcome = os.environ.get("ISSUE_OUTCOME", "") - issue_attempted = issue_outcome not in ("", "skipped") - readback_required = issue_attempted - readback_verified = os.environ.get("ISSUE_READBACK") == "true" if readback_required else True - skipped = job_ok and not action - partial = issue_attempted and not readback_verified - payload = { - "schema": "AutomationTerminalStateV1", - "automation_id": "github:AssistSupport/dependency-watch", - "state": "succeeded" if job_ok else "failed", - "completed": job_ok, - "partial": partial, - "skipped": skipped, - "mutation_count": 1 if action else 0, - "destination_readback": { - "required": readback_required, - "verified": readback_verified, - "destination_id": f"issue:{issue_number}" if issue_number else None, - "evidence": {"issue_action": action or "unknown", "issue_step_outcome": issue_outcome}, - }, - "operator_action_required": not job_ok, - "can_auto_archive": job_ok, - "observed_at": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), - } - line = "automation_completion: " + json.dumps(payload, separators=(",", ":")) - print(line) - with open(os.environ["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8") as summary: - summary.write("\n" + line + "\n") - if readback_required and not readback_verified: - raise SystemExit("dependency issue destination readback was not verified") - PY + run: python3 .github/scripts/dependency_watch_completion.py diff --git a/scripts/ci/check-dependency-watch-contract.mjs b/scripts/ci/check-dependency-watch-contract.mjs index 9a2520d..f3d6322 100644 --- a/scripts/ci/check-dependency-watch-contract.mjs +++ b/scripts/ci/check-dependency-watch-contract.mjs @@ -2,21 +2,56 @@ import fs from "node:fs"; -const workflow = fs.readFileSync(new URL("../../.github/workflows/dependency-watch.yml", import.meta.url), "utf8"); +const workflow = fs.readFileSync( + new URL("../../.github/workflows/dependency-watch.yml", import.meta.url), + "utf8", +); +const completionHelper = fs.readFileSync( + new URL( + "../../.github/scripts/dependency_watch_completion.py", + import.meta.url, + ), + "utf8", +); const requirements = [ - ["workflow concurrency", /^concurrency:\n {2}group:/m], - ["job timeout", /^ {4}timeout-minutes: \d+/m], - ["mutation action output", /core\.setOutput\('issue_action'/], - ["destination id output", /core\.setOutput\('issue_number'/], - ["destination readback", /github\.rest\.issues\.get\(/], - ["readback mismatch failure", /throw new Error\(`Dependency alert issue readback mismatch/], - ["machine completion state", /AutomationTerminalStateV1/], - ["partial mutation accounting", /partial = issue_attempted and not readback_verified/], + ["workflow concurrency", workflow, /^concurrency:\n {2}group:/m], + ["job timeout", workflow, /^ {4}timeout-minutes: \d+/m], + ["mutation action output", workflow, /core\.setOutput\('issue_action'/], + ["destination id output", workflow, /core\.setOutput\('issue_number'/], + ["destination readback", workflow, /github\.rest\.issues\.get\(/], + [ + "readback mismatch failure", + workflow, + /throw new Error\(`Dependency alert issue readback mismatch/, + ], + [ + "completion helper invocation", + workflow, + /run: python3 \.github\/scripts\/dependency_watch_completion\.py/, + ], + [ + "machine completion state", + completionHelper, + /"schema": "AutomationTerminalStateV1"/, + ], + [ + "partial mutation accounting", + completionHelper, + /partial = readback_required and not readback_verified/, + ], ]; -const missing = requirements.filter(([, pattern]) => !pattern.test(workflow)).map(([name]) => name); +const missing = requirements + .filter(([, source, pattern]) => !pattern.test(source)) + .map(([name]) => name); if (missing.length) { console.error(`dependency-watch contract missing: ${missing.join(", ")}`); process.exit(1); } -console.log(JSON.stringify({ schema: "DependencyWatchContractTestV1", ok: true, tests: requirements.length })); +console.log( + JSON.stringify({ + schema: "DependencyWatchContractTestV1", + ok: true, + tests: requirements.length, + }), +); diff --git a/tests/automation/test_dependency_watch_completion.py b/tests/automation/test_dependency_watch_completion.py new file mode 100644 index 0000000..dbb7eb2 --- /dev/null +++ b/tests/automation/test_dependency_watch_completion.py @@ -0,0 +1,176 @@ +import importlib.util +import json +import os +import subprocess +import sys +import unittest +from pathlib import Path + + +SCRIPT = ( + Path(__file__).resolve().parents[2] + / ".github" + / "scripts" + / "dependency_watch_completion.py" +) +WORKFLOW = Path(__file__).resolve().parents[2] / ".github" / "workflows" / "dependency-watch.yml" + + +def load_module(): + spec = importlib.util.spec_from_file_location("dependency_watch_completion", SCRIPT) + if spec is None or spec.loader is None: + raise RuntimeError(f"cannot load {SCRIPT}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class DependencyWatchCompletionContractTests(unittest.TestCase): + observed_at = "2026-07-14T06:30:00Z" + + def payload(self, **overrides): + env = { + "JOB_STATUS": "success", + "ISSUE_OUTCOME": "skipped", + "ISSUE_ACTION": "", + "ISSUE_NUMBER": "", + "ISSUE_READBACK": "", + } + env.update(overrides) + return load_module().build_payload(env, self.observed_at) + + def assert_contract(self, payload): + self.assertEqual(payload["schema"], "AutomationTerminalStateV1") + self.assertEqual(payload["automation_id"], "github:AssistSupport/dependency-watch") + self.assertEqual((payload["state"] == "partial"), payload["partial"]) + self.assertEqual((payload["state"] == "skipped"), payload["skipped"]) + self.assertTrue(payload["destination_readback"]["evidence"]) + if payload["destination_readback"]["required"]: + self.assertTrue(payload["destination_readback"]["destination_id"]) + if payload["can_auto_archive"]: + self.assertIn(payload["state"], {"succeeded", "skipped"}) + self.assertTrue(payload["completed"]) + self.assertFalse(payload["operator_action_required"]) + + def test_clean_run_is_success_not_skipped(self): + payload = self.payload() + self.assert_contract(payload) + self.assertEqual(payload["state"], "succeeded") + self.assertTrue(payload["completed"]) + self.assertFalse(payload["skipped"]) + self.assertEqual(payload["mutation_count"], 0) + self.assertEqual( + payload["destination_readback"], + { + "required": False, + "verified": False, + "destination_id": None, + "observed_at": None, + "evidence": {"issue_action": "not_required", "issue_step_outcome": "skipped"}, + }, + ) + + def test_issue_mutation_with_readback_is_success(self): + payload = self.payload( + ISSUE_OUTCOME="success", + ISSUE_ACTION="updated", + ISSUE_NUMBER="42", + ISSUE_READBACK="true", + ) + self.assert_contract(payload) + self.assertEqual(payload["state"], "succeeded") + self.assertEqual(payload["mutation_count"], 1) + self.assertEqual(payload["destination_readback"]["destination_id"], "issue:42") + self.assertEqual(payload["destination_readback"]["observed_at"], self.observed_at) + + def test_actionable_failure_with_verified_issue_is_failed_not_partial(self): + payload = self.payload( + JOB_STATUS="failure", + ISSUE_OUTCOME="success", + ISSUE_ACTION="created", + ISSUE_NUMBER="43", + ISSUE_READBACK="true", + ) + self.assert_contract(payload) + self.assertEqual(payload["state"], "failed") + self.assertFalse(payload["completed"]) + self.assertFalse(payload["partial"]) + self.assertTrue(payload["operator_action_required"]) + self.assertFalse(payload["can_auto_archive"]) + + def test_unverified_attempt_is_partial(self): + payload = self.payload(JOB_STATUS="failure", ISSUE_OUTCOME="failure") + self.assert_contract(payload) + self.assertEqual(payload["state"], "partial") + self.assertFalse(payload["completed"]) + self.assertTrue(payload["partial"]) + self.assertTrue(payload["destination_readback"]["required"]) + self.assertFalse(payload["destination_readback"]["verified"]) + self.assertEqual( + payload["destination_readback"]["destination_id"], + "repo:saagpatel/AssistSupport/issues#Dependency Watch Alerts", + ) + self.assertEqual(payload["destination_readback"]["observed_at"], self.observed_at) + self.assertTrue(payload["operator_action_required"]) + self.assertFalse(payload["can_auto_archive"]) + + def test_failure_before_issue_attempt_is_failed(self): + payload = self.payload(JOB_STATUS="failure") + self.assert_contract(payload) + self.assertEqual(payload["state"], "failed") + self.assertFalse(payload["destination_readback"]["required"]) + + def test_workflow_always_invokes_tested_completion_emitter(self): + workflow = WORKFLOW.read_text(encoding="utf-8") + self.assertIn("- name: Emit machine-readable completion state\n if: always()", workflow) + self.assertIn("run: python3 .github/scripts/dependency_watch_completion.py", workflow) + + def test_cli_emits_one_standalone_completion_line_and_exits_zero(self): + env = { + **os.environ, + "JOB_STATUS": "success", + "ISSUE_OUTCOME": "skipped", + "ISSUE_ACTION": "", + "ISSUE_NUMBER": "", + "ISSUE_READBACK": "", + } + env.pop("GITHUB_STEP_SUMMARY", None) + result = subprocess.run( + [sys.executable, str(SCRIPT)], + check=False, + capture_output=True, + text=True, + env=env, + ) + self.assertEqual(result.returncode, 0, result.stderr) + lines = result.stdout.splitlines() + self.assertEqual(len(lines), 1) + self.assertTrue(lines[0].startswith("automation_completion: ")) + payload = json.loads(lines[0].removeprefix("automation_completion: ")) + self.assertEqual(payload["state"], "succeeded") + + def test_cli_exits_nonzero_when_required_readback_is_unverified(self): + env = { + **os.environ, + "JOB_STATUS": "failure", + "ISSUE_OUTCOME": "failure", + "ISSUE_ACTION": "", + "ISSUE_NUMBER": "", + "ISSUE_READBACK": "", + } + env.pop("GITHUB_STEP_SUMMARY", None) + result = subprocess.run( + [sys.executable, str(SCRIPT)], + check=False, + capture_output=True, + text=True, + env=env, + ) + self.assertEqual(result.returncode, 1) + self.assertIn("destination readback was not verified", result.stderr) + payload = json.loads(result.stdout.removeprefix("automation_completion: ")) + self.assertEqual(payload["state"], "partial") + + +if __name__ == "__main__": + unittest.main()