diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 03e01ed..7ceed47 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -72,3 +72,19 @@ jobs: - name: Test the JS linter action run: ./test/js-linter/execution.sh + + definition-of-done-test: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v6 + + - name: Test definition-of-done action + run: ./definition-of-done/tests/run.sh + + pull-request-compliance-test: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v6 + + - name: Test pull-request-compliance action + run: ./pull-request-compliance/tests/run.sh diff --git a/.gitignore b/.gitignore index 70e7ccf..337f3f7 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,7 @@ !.env.test # environment variables with secrets .env*.local + +# Python caches +__pycache__/ +*.pyc diff --git a/README.md b/README.md index 344057b..f52cdfc 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,13 @@ Two actions are present to release services which are go binaries: Example of usage is presents in the `actions.yml` of each action +## Pull Request Governance + +Two actions are available for pull request quality gates: + +- [definition-of-done](/definition-of-done): validate that all checklist items are checked when a `Definition of Done` section is present in the PR body. +- [pull-request-compliance](/pull-request-compliance): validate PR title/body compliance with deterministic checks and optional OpenAI policy/template validation. + ## Automatically Merge Dependabot Pull Requests GitHub action to automatically merge the Dependabot PRs. It merges the dependency upgrade if it upgrades a minor or patch version. diff --git a/definition-of-done/README.md b/definition-of-done/README.md new file mode 100644 index 0000000..b74156e --- /dev/null +++ b/definition-of-done/README.md @@ -0,0 +1,35 @@ +# Definition of Done Action + +Checks the pull request description for a `Definition of Done` section. If that section is present, all checklist items must be checked. + +## Inputs + +- `pr-body` (required): pull request body markdown. +- `section-heading` (optional): heading text to match, default `Definition of Done`. + +## Example + +Reusable workflow example file: `examples/workflow.yml`. + +```yaml +name: definition-of-done +on: + pull_request: + types: [opened, edited, synchronize, reopened, ready_for_review] + +jobs: + check: + runs-on: ubuntu-latest + steps: + - uses: Scalingo/actions/definition-of-done@main + with: + pr-body: ${{ github.event.pull_request.body }} +``` + +## Tests + +Run: + +```bash +./definition-of-done/tests/run.sh +``` diff --git a/definition-of-done/action.yml b/definition-of-done/action.yml new file mode 100644 index 0000000..0014805 --- /dev/null +++ b/definition-of-done/action.yml @@ -0,0 +1,21 @@ +name: "Scalingo Definition of Done GitHub Action" +description: "Validate that all checklist items are checked in the PR Definition of Done section" + +inputs: + pr-body: + description: "Pull request body markdown" + required: true + section-heading: + description: "Heading text to match for the DoD section (case-insensitive)" + required: false + default: "Definition of Done" + +runs: + using: "composite" + steps: + - name: Validate Definition of Done checklist + shell: bash + env: + PR_BODY: ${{ inputs.pr-body }} + SECTION_HEADING: ${{ inputs.section-heading }} + run: python3 "${GITHUB_ACTION_PATH}/scripts/check_definition_of_done.py" diff --git a/definition-of-done/docs/inline_dod_block_Version3.py b/definition-of-done/docs/inline_dod_block_Version3.py new file mode 100644 index 0000000..775f96d --- /dev/null +++ b/definition-of-done/docs/inline_dod_block_Version3.py @@ -0,0 +1,24 @@ +# ----------------------------- +# Deterministic: DoD checklist +# ----------------------------- +import re, sys + +heading_re = re.compile(r"(?im)^#{1,6}\s*definition\s+of\s+done\s*$") +m = heading_re.search(body) +if m: + start = m.end() + next_heading_re = re.compile(r"(?im)^#{1,6}\s+\S.*$") + m2 = next_heading_re.search(body, pos=start) + section = body[start:] if not m2 else body[start:m2.start()] + + items = re.findall(r"(?m)^\s*-\s*\[(?P[ xX])\]\s+(?P.+?)\s*$", section) + if not items: + print("::error::A 'Definition of Done' heading is present, but no checklist items were found under it.") + sys.exit(1) + + unchecked = [text for mark, text in items if mark.strip() == ""] + if unchecked: + print("::error::Definition of Done checklist has unchecked items:") + for t in unchecked: + print(f" - {t}") + sys.exit(1) \ No newline at end of file diff --git a/definition-of-done/docs/pr_compliance_dod_check_Version3.py b/definition-of-done/docs/pr_compliance_dod_check_Version3.py new file mode 100644 index 0000000..4cde93f --- /dev/null +++ b/definition-of-done/docs/pr_compliance_dod_check_Version3.py @@ -0,0 +1,43 @@ +import re, sys + +def enforce_definition_of_done(body: str) -> None: + """ + If a Markdown heading (any level) matches 'definition of done' (case-insensitive), + require all task list items under that section to be checked. + + Section runs until the next Markdown heading of any level or end of text. + """ + # Match headings like: + # # Definition of Done + # ## definition of done + # ###### DeFiNiTiOn Of DoNe + heading_re = re.compile( + r"(?im)^(?P#{1,6})\s*(?Pdefinition\s+of\s+done)\s*$" + ) + + m = heading_re.search(body) + if not m: + return # DoD section not present => no enforcement + + start = m.end() + + # Find next heading after this one (any level) + next_heading_re = re.compile(r"(?im)^#{1,6}\s+\S.*$") + m2 = next_heading_re.search(body, pos=start) + section = body[start:] if not m2 else body[start:m2.start()] + + # Task list items in Markdown + items = re.findall( + r"(?m)^\s*-\s*\[(?P<mark>[ xX])\]\s+(?P<text>.+?)\s*$", + section + ) + if not items: + print("::error::A 'Definition of Done' heading is present, but no checklist items were found under it.") + sys.exit(1) + + unchecked = [text for mark, text in items if mark.strip() == ""] + if unchecked: + print("::error::Definition of Done checklist has unchecked items:") + for t in unchecked: + print(f" - {t}") + sys.exit(1) \ No newline at end of file diff --git a/definition-of-done/examples/workflow.yml b/definition-of-done/examples/workflow.yml new file mode 100644 index 0000000..b50f39a --- /dev/null +++ b/definition-of-done/examples/workflow.yml @@ -0,0 +1,13 @@ +name: definition-of-done + +on: + pull_request: + types: [opened, edited, synchronize, reopened, ready_for_review] + +jobs: + check: + runs-on: ubuntu-latest + steps: + - uses: Scalingo/actions/definition-of-done@main + with: + pr-body: ${{ github.event.pull_request.body }} diff --git a/definition-of-done/scripts/check_definition_of_done.py b/definition-of-done/scripts/check_definition_of_done.py new file mode 100644 index 0000000..7172d36 --- /dev/null +++ b/definition-of-done/scripts/check_definition_of_done.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +"""Validate Definition of Done checklist sections in Markdown text.""" + +import os +import re +import sys + + +def heading_pattern(heading_text: str) -> str: + tokens = heading_text.strip().split() + if not tokens: + return r"definition\s+of\s+done" + return r"\s+".join(re.escape(token) for token in tokens) + + +def find_dod_sections(body: str, heading_text: str) -> list[str]: + title_pattern = heading_pattern(heading_text) + heading_re = re.compile(rf"(?im)^#{{1,6}}\s*{title_pattern}\s*$") + generic_heading_re = re.compile(r"(?im)^#{1,6}\s+\S.*$") + + sections: list[str] = [] + for match in heading_re.finditer(body): + start = match.end() + next_heading = generic_heading_re.search(body, pos=start) + end = next_heading.start() if next_heading else len(body) + sections.append(body[start:end]) + + return sections + + +def validate_section(section: str) -> list[str]: + items = re.findall(r"(?m)^\s*-\s*\[(?P<mark>[ xX])\]\s+(?P<text>.+?)\s*$", section) + if not items: + return ["A 'Definition of Done' heading is present, but no checklist items were found under it."] + + unchecked = [text for mark, text in items if mark.strip() == ""] + if not unchecked: + return [] + + errors = ["Definition of Done checklist has unchecked items:"] + errors.extend(f" - {item}" for item in unchecked) + return errors + + +def main() -> int: + body = os.environ.get("PR_BODY", "") + heading = os.environ.get("SECTION_HEADING", "Definition of Done") + + if not body.strip(): + print("::error::PR description is empty.") + return 1 + + sections = find_dod_sections(body, heading) + if not sections: + return 0 + + for section in sections: + errors = validate_section(section) + if errors: + for error in errors: + print(f"::error::{error}") + return 1 + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/definition-of-done/tests/run.sh b/definition-of-done/tests/run.sh new file mode 100755 index 0000000..102a7be --- /dev/null +++ b/definition-of-done/tests/run.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd "$(dirname "$0")/../.." +python3 -m unittest discover -s definition-of-done/tests -p 'test_*.py' -v diff --git a/definition-of-done/tests/test_check_definition_of_done.py b/definition-of-done/tests/test_check_definition_of_done.py new file mode 100644 index 0000000..7c3d2c4 --- /dev/null +++ b/definition-of-done/tests/test_check_definition_of_done.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 + +import importlib.util +import io +import os +import pathlib +import unittest +from contextlib import redirect_stdout +from unittest.mock import patch + +SCRIPT_PATH = pathlib.Path(__file__).resolve().parents[1] / "scripts" / "check_definition_of_done.py" +SPEC = importlib.util.spec_from_file_location("check_definition_of_done", SCRIPT_PATH) +MODULE = importlib.util.module_from_spec(SPEC) +assert SPEC and SPEC.loader +SPEC.loader.exec_module(MODULE) + + +class DefinitionOfDoneTests(unittest.TestCase): + def run_main(self, body: str, heading: str = "Definition of Done"): + env = { + "PR_BODY": body, + "SECTION_HEADING": heading, + } + output = io.StringIO() + with patch.dict(os.environ, env, clear=False), redirect_stdout(output): + rc = MODULE.main() + return rc, output.getvalue() + + def test_empty_body_fails(self): + rc, output = self.run_main(" ") + self.assertEqual(rc, 1) + self.assertIn("PR description is empty", output) + + def test_missing_dod_heading_passes(self): + rc, output = self.run_main("## Summary\nNo DoD section here") + self.assertEqual(rc, 0) + self.assertEqual(output, "") + + def test_checked_dod_passes(self): + body = """## Definition of Done +- [x] Requirements understood +- [X] Tests updated +""" + rc, output = self.run_main(body) + self.assertEqual(rc, 0) + self.assertEqual(output, "") + + def test_heading_matching_is_case_insensitive_and_spacing_tolerant(self): + body = """### DeFiNiTiOn Of DoNe +- [x] item +""" + rc, output = self.run_main(body) + self.assertEqual(rc, 0) + self.assertEqual(output, "") + + def test_unchecked_item_fails(self): + body = """## Definition of Done +- [x] One +- [ ] Two +""" + rc, output = self.run_main(body) + self.assertEqual(rc, 1) + self.assertIn("unchecked items", output) + self.assertIn("Two", output) + + def test_dod_without_checklist_fails(self): + body = """## Definition of Done +This section has no checklist +""" + rc, output = self.run_main(body) + self.assertEqual(rc, 1) + self.assertIn("no checklist items", output) + + def test_custom_heading_can_be_used(self): + body = """## Custom Done Block +- [x] All good +""" + rc, output = self.run_main(body, heading="Custom Done Block") + self.assertEqual(rc, 0) + self.assertEqual(output, "") + + +if __name__ == "__main__": + unittest.main() diff --git a/pull-request-compliance/README.md b/pull-request-compliance/README.md new file mode 100644 index 0000000..bda6f39 --- /dev/null +++ b/pull-request-compliance/README.md @@ -0,0 +1,54 @@ +# Pull Request Compliance Action + +Validates pull request title/body with deterministic checks and optional OpenAI policy/template validation. + +## Deterministic checks + +- Empty PR description is rejected. +- `Definition of Done` checklist must be fully checked when the section is present. +- Placeholder tokens `TBD`, `TODO`, and `??` are rejected. + +## Inputs + +- `pr-title` (required): pull request title. +- `pr-body` (required): pull request body markdown. +- `template-path` (optional): default `.github/pull_request_template.md`. +- `policy-path` (optional): default `.github/pr_compliance_policy.md`. +- `openai-api-key` (optional): enables policy/template LLM validation when set. +- `openai-model` (optional): default `gpt-4o-mini`. + +## Example + +Reusable workflow example file: `examples/workflow.yml`. + +```yaml +name: pr-compliance +on: + pull_request: + types: [opened, edited, synchronize, reopened, ready_for_review] + +jobs: + check: + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read + steps: + - uses: actions/checkout@v4 + + - uses: Scalingo/actions/pull-request-compliance@main + with: + pr-title: ${{ github.event.pull_request.title }} + pr-body: ${{ github.event.pull_request.body }} + openai-api-key: ${{ secrets.OPENAI_API_KEY }} +``` + +## Tests + +Run: + +```bash +./pull-request-compliance/tests/run.sh +``` + +Unit tests mock OpenAI calls, so no API key is required. diff --git a/pull-request-compliance/action.yml b/pull-request-compliance/action.yml new file mode 100644 index 0000000..778d277 --- /dev/null +++ b/pull-request-compliance/action.yml @@ -0,0 +1,40 @@ +name: "Scalingo Pull Request Compliance GitHub Action" +description: "Validate pull request metadata and description compliance against template and policy" + +inputs: + pr-title: + description: "Pull request title" + required: true + pr-body: + description: "Pull request body markdown" + required: true + template-path: + description: "Path to pull request template markdown" + required: false + default: ".github/pull_request_template.md" + policy-path: + description: "Path to pull request compliance policy markdown" + required: false + default: ".github/pr_compliance_policy.md" + openai-api-key: + description: "OpenAI API key used for policy/template validation" + required: false + default: "" + openai-model: + description: "OpenAI model to use for compliance inference" + required: false + default: "gpt-4o-mini" + +runs: + using: "composite" + steps: + - name: Validate PR compliance + shell: bash + env: + PR_TITLE: ${{ inputs.pr-title }} + PR_BODY: ${{ inputs.pr-body }} + TEMPLATE_PATH: ${{ inputs.template-path }} + POLICY_PATH: ${{ inputs.policy-path }} + OPENAI_API_KEY: ${{ inputs.openai-api-key }} + OPENAI_MODEL: ${{ inputs.openai-model }} + run: python3 "${GITHUB_ACTION_PATH}/scripts/check_pr_compliance.py" diff --git a/pull-request-compliance/docs/docs_pr-compliance-validation_Version3.md b/pull-request-compliance/docs/docs_pr-compliance-validation_Version3.md new file mode 100644 index 0000000..e817280 --- /dev/null +++ b/pull-request-compliance/docs/docs_pr-compliance-validation_Version3.md @@ -0,0 +1,84 @@ +# PR Validation Actions for Compliance (WIP) + +Status: **Work in progress** +Goal: Add GitHub Actions checks to enforce PR metadata and description compliance before merge. + +## Objectives +- Validate PR **title** and **description** against organization compliance rules. +- Enforce a **Definition of Done** (DoD) checklist when present. +- Fail the check on **errors** only (warnings are allowed and should not block merging). +- Minimize dependencies: use GitHub-maintained actions and simple scripting; avoid “shady” third-party actions. + +## Inputs / Sources of Truth +- PR template (user-facing): `.github/pull_request_template.md` +- Compliance policy (machine-readable rubric, can be markdown): `.github/pr_compliance_policy.md` +- PR data from event payload: + - `pull_request.title` + - `pull_request.body` + +## Compliance Rules (current) +### Ticket references +Acceptable ticket reference formats: +- Jira key: `ABCDEF-12345` + - Pattern: `[A-Z]{2,10}-[0-9]{1,6}` +- GitHub issue/PR references (including other repos): + - URL: `https://github.com/<owner>/<repo>/issues/<number>` + - URL: `https://github.com/<owner>/<repo>/pull/<number>` + - Cross-repo shorthand: `<owner>/<repo>#<number>` + - Same-repo shorthand: `#<number>` (optional; decide whether to accept) + +### PR title (mandatory, error) +- MUST contain either: + - a ticket reference (Jira key or GitHub issue/PR reference), OR + - an explicit “no ticket” explanation, e.g.: + - `[no-ticket: <reason>]` + - `(no ticket - <reason>)` + - `No ticket: <reason>` +- If “no ticket”, the reason must be explanatory (not just “N/A”). + +### PR description structure (mandatory, error) +- Must include all required section headings defined in `.github/pull_request_template.md`. +- Where a section contains instructions, those instructions must be followed. + +### N/A handling +- `N/A` is allowed. +- When used, it must include an explanation of **at least one sentence** (e.g., `N/A - <one sentence explanation>`). + +### Prohibited placeholders (error) +- Disallow placeholder tokens such as: `TBD`, `TODO`, `??` (case-insensitive). + +## Definition of Done (DoD) Checklist Enforcement (mandatory when present) +Requirement: +- If a DoD section is present in the PR description, **all checklist items under it must be checked**. + +Detection: +- Accept **any Markdown heading level** (`#` through `######`) with text matching + - `definition of done` (case-insensitive; allow arbitrary capitalization) +- The DoD section content runs until the next Markdown heading of any level or end-of-document. +- Checklist items are Markdown task list items: + - checked: `- [x] ...` + - unchecked: `- [ ] ...` + +Failure conditions: +- DoD heading exists but contains **no** checklist items => **error** +- Any unchecked DoD checklist item => **error** + +## Implementation Notes (current approach) +- Deterministic checks (no AI): + - DoD checklist fully checked if present + - Block placeholder tokens + - (Optionally) regex-based ticket-in-title check +- LLM-based checks (OpenAI): + - Evaluate title/body against policy + template for “follow instructions” and completeness + - Enforce “errors fail, warnings do not” + - Require strict JSON output from the model (machine-parsable) + +## Workflow Expectations +- Trigger on: PR opened/edited/synchronize/reopened/ready_for_review +- Result: a required status check that blocks merge when failing +- Output: clear error messages in workflow logs (and optionally PR comment in later iteration) + +## Open Questions / Next Decisions +- Should same-repo shorthand `#123` be accepted as a ticket reference? +- Should ticket references in the PR body be allowed to satisfy the title rule, or title-only? +- Should multiple DoD sections be supported (enforce all) or just the first occurrence? \ No newline at end of file diff --git a/pull-request-compliance/docs/github_pr_compliance_policy_Version3.md b/pull-request-compliance/docs/github_pr_compliance_policy_Version3.md new file mode 100644 index 0000000..961aff3 --- /dev/null +++ b/pull-request-compliance/docs/github_pr_compliance_policy_Version3.md @@ -0,0 +1,3 @@ +## Definition of Done (mandatory when present) +- If the PR description contains a "Definition of Done" section, every checklist item under it must be checked (`- [x]`). +- Unchecked items (`- [ ]`) are an error. \ No newline at end of file diff --git a/pull-request-compliance/docs/github_pull_request_template_Version2.md b/pull-request-compliance/docs/github_pull_request_template_Version2.md new file mode 100644 index 0000000..420f4b8 --- /dev/null +++ b/pull-request-compliance/docs/github_pull_request_template_Version2.md @@ -0,0 +1,23 @@ +## Summary +<!-- What does this change do? --> + +## Ticket +<!-- Link the external ticket, or write N/A - <explanation>. --> + +## Definition of Done +<!-- If this section is present, all items must be checked before merge. --> +- [ ] Requirements understood / documented +- [ ] Tests added/updated (unit/integration as appropriate) +- [ ] Documentation updated (if needed) +- [ ] Security/privacy impact assessed (if applicable) +- [ ] Observability/logging considered (if applicable) +- [ ] Rollback plan included (or N/A with explanation) + +## Risk / Impact +<!-- N/A allowed with explanation --> + +## Testing +<!-- Describe what was run, or N/A - <explanation> (docs-only only). --> + +## Rollback plan +<!-- Actionable, or N/A - <explanation> --> \ No newline at end of file diff --git a/pull-request-compliance/docs/github_workflows_pr-description-compliance_Version3.yml b/pull-request-compliance/docs/github_workflows_pr-description-compliance_Version3.yml new file mode 100644 index 0000000..c0e9661 --- /dev/null +++ b/pull-request-compliance/docs/github_workflows_pr-description-compliance_Version3.yml @@ -0,0 +1,198 @@ +name: PR Description Compliance + +on: + pull_request: + types: [opened, edited, synchronize, reopened, ready_for_review] + +permissions: + contents: read + pull-requests: read + +jobs: + compliance: + runs-on: ubuntu-latest + steps: + - name: Checkout (for policy/template files) + uses: actions/checkout@v4 + + - name: Evaluate PR title/body vs policy+template (OpenAI + deterministic DoD check) + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + PR_BODY: ${{ github.event.pull_request.body }} + PR_TITLE: ${{ github.event.pull_request.title }} + run: | + set -euo pipefail + + TEMPLATE_FILE=".github/pull_request_template.md" + POLICY_FILE=".github/pr_compliance_policy.md" + + if [ ! -f "$TEMPLATE_FILE" ]; then + echo "::error::Missing $TEMPLATE_FILE." + exit 1 + fi + if [ ! -f "$POLICY_FILE" ]; then + echo "::error::Missing $POLICY_FILE." + exit 1 + fi + + python3 - << 'PY' + import json, os, re, sys, urllib.request + + api_key = os.environ.get("OPENAI_API_KEY", "") + template = open(".github/pull_request_template.md", "r", encoding="utf-8").read() + policy = open(".github/pr_compliance_policy.md", "r", encoding="utf-8").read() + body = os.environ.get("PR_BODY", "") or "" + title = os.environ.get("PR_TITLE", "") or "" + + if not body.strip(): + print("::error::PR description is empty.") + sys.exit(1) + + # ----------------------------- + # Deterministic: DoD checklist + # ----------------------------- + # If "Definition of Done" section exists in the PR body, require all checkboxes under it to be checked. + # We treat the section as running until the next H2 (## ) heading or end of text. + dod_heading_re = re.compile(r"^##\s+Definition of Done\s*$", re.IGNORECASE | re.MULTILINE) + + if dod_heading_re.search(body): + # Extract section content + parts = dod_heading_re.split(body, maxsplit=1) + after = parts[1] if len(parts) > 1 else "" + # Stop at next H2 heading + after = re.split(r"^\s*##\s+", after, maxsplit=1, flags=re.MULTILINE)[0] + + # Find markdown task list items + items = re.findall(r"^\s*-\s*\[(?P<mark>[ xX])\]\s+(?P<text>.+?)\s*$", after, flags=re.MULTILINE) + if not items: + print("::error::'Definition of Done' section is present but contains no checklist items.") + sys.exit(1) + + unchecked = [text for mark, text in items if mark.strip() == ""] + if unchecked: + print("::error::Definition of Done checklist has unchecked items:") + for t in unchecked: + print(f" - {t}") + sys.exit(1) + + # -------------------------------- + # Deterministic: placeholders block + # -------------------------------- + forbidden_placeholders = ["TBD", "TODO", "??"] + for token in forbidden_placeholders: + if re.search(rf"\b{re.escape(token)}\b", body, flags=re.IGNORECASE): + print(f"::error::Forbidden placeholder '{token}' found in PR description.") + sys.exit(1) + + # -------------------------------- + # LLM: policy/template compliance + # -------------------------------- + if not api_key: + print("::error::Missing OPENAI_API_KEY secret.") + sys.exit(1) + + prompt = f""" + You are a strict compliance checker for pull requests. + + Evaluate the PR TITLE and PR DESCRIPTION against: + (1) The PR template (structure/sections) + (2) The PR compliance policy (rules) + + TEMPLATE: + --- + {template} + --- + + POLICY (source of truth for rules): + --- + {policy} + --- + + PR TITLE: + --- + {title} + --- + + PR DESCRIPTION: + --- + {body} + --- + + IMPORTANT INTERPRETATION RULES: + - "N/A" is allowed only when accompanied by an explanation (at least one sentence). + - If policy and template conflict, policy wins. + - Be conservative: if something is unclear or missing, mark it as a violation. + - Return ONLY valid JSON matching the schema. No markdown, no extra text. + + JSON SCHEMA (must match exactly): + {{ + "pass": boolean, + "violations": [ + {{ + "id": string, + "severity": "error" | "warning", + "location": "title" | "description", + "message": string, + "suggested_fix": string + }} + ] + }} + """ + + payload = { + "model": "gpt-4o-mini", + "response_format": {"type": "json_object"}, + "messages": [ + {"role": "system", "content": "Return only JSON. No extra text."}, + {"role": "user", "content": prompt} + ], + "temperature": 0 + } + + req = urllib.request.Request( + "https://api.openai.com/v1/chat/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + }, + method="POST", + ) + + try: + with urllib.request.urlopen(req, timeout=60) as resp: + data = json.loads(resp.read().decode("utf-8")) + except Exception as e: + print(f"::error::OpenAI request failed: {e}") + sys.exit(1) + + content = data["choices"][0]["message"]["content"] + try: + result = json.loads(content) + except Exception: + print("::error::Model did not return valid JSON.") + print("Raw output:") + print(content) + sys.exit(1) + + print("Compliance result:") + print(json.dumps(result, indent=2)) + + passed = bool(result.get("pass", False)) + violations = result.get("violations", []) + error_violations = [v for v in violations if v.get("severity") == "error"] + + if (not passed) or error_violations: + print("::error::PR failed compliance checks.") + for v in violations: + sev = v.get("severity", "error") + loc = v.get("location", "description") + msg = v.get("message", "") + fix = v.get("suggested_fix", "") + print(f"- [{sev}] ({loc}) {msg}") + if fix: + print(f" suggested_fix: {fix}") + sys.exit(1) + + print("PR passed compliance checks.") + PY \ No newline at end of file diff --git a/pull-request-compliance/examples/workflow.yml b/pull-request-compliance/examples/workflow.yml new file mode 100644 index 0000000..bcef574 --- /dev/null +++ b/pull-request-compliance/examples/workflow.yml @@ -0,0 +1,20 @@ +name: pr-compliance + +on: + pull_request: + types: [opened, edited, synchronize, reopened, ready_for_review] + +jobs: + check: + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read + steps: + - uses: actions/checkout@v4 + + - uses: Scalingo/actions/pull-request-compliance@main + with: + pr-title: ${{ github.event.pull_request.title }} + pr-body: ${{ github.event.pull_request.body }} + openai-api-key: ${{ secrets.OPENAI_API_KEY }} diff --git a/pull-request-compliance/scripts/check_pr_compliance.py b/pull-request-compliance/scripts/check_pr_compliance.py new file mode 100644 index 0000000..1162332 --- /dev/null +++ b/pull-request-compliance/scripts/check_pr_compliance.py @@ -0,0 +1,212 @@ +#!/usr/bin/env python3 +"""Validate pull-request title/body against deterministic and policy-based checks.""" + +import json +import os +import re +import sys +import urllib.error +import urllib.request + + +def heading_pattern(heading_text: str) -> str: + tokens = heading_text.strip().split() + if not tokens: + return r"definition\s+of\s+done" + return r"\s+".join(re.escape(token) for token in tokens) + + +def validate_definition_of_done(body: str, heading_text: str = "Definition of Done") -> list[str]: + title_pattern = heading_pattern(heading_text) + heading_re = re.compile(rf"(?im)^#{{1,6}}\s*{title_pattern}\s*$") + generic_heading_re = re.compile(r"(?im)^#{1,6}\s+\S.*$") + + errors: list[str] = [] + matches = list(heading_re.finditer(body)) + for match in matches: + start = match.end() + next_heading = generic_heading_re.search(body, pos=start) + end = next_heading.start() if next_heading else len(body) + section = body[start:end] + + items = re.findall(r"(?m)^\s*-\s*\[(?P<mark>[ xX])\]\s+(?P<text>.+?)\s*$", section) + if not items: + errors.append("A 'Definition of Done' heading is present, but no checklist items were found under it.") + continue + + unchecked = [text for mark, text in items if mark.strip() == ""] + if unchecked: + errors.append("Definition of Done checklist has unchecked items:") + errors.extend(f" - {item}" for item in unchecked) + + return errors + + +def validate_placeholders(body: str) -> list[str]: + errors: list[str] = [] + for token in ("TBD", "TODO"): + if re.search(rf"\b{re.escape(token)}\b", body, flags=re.IGNORECASE): + errors.append(f"Forbidden placeholder '{token}' found in PR description.") + if "??" in body: + errors.append("Forbidden placeholder '??' found in PR description.") + return errors + + +def read_file(path: str) -> str: + try: + with open(path, "r", encoding="utf-8") as file_obj: + return file_obj.read() + except OSError as exc: + print(f"::error::Cannot read {path}: {exc}") + sys.exit(1) + + +def call_openai(model: str, api_key: str, prompt: str) -> dict: + payload = { + "model": model, + "response_format": {"type": "json_object"}, + "messages": [ + {"role": "system", "content": "Return only JSON. No extra text."}, + {"role": "user", "content": prompt}, + ], + "temperature": 0, + } + + request = urllib.request.Request( + "https://api.openai.com/v1/chat/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + }, + method="POST", + ) + + try: + with urllib.request.urlopen(request, timeout=60) as response: + body = response.read().decode("utf-8") + except urllib.error.HTTPError as exc: + details = exc.read().decode("utf-8", errors="replace") + print(f"::error::OpenAI request failed with HTTP {exc.code}: {details}") + sys.exit(1) + except Exception as exc: # noqa: BLE001 + print(f"::error::OpenAI request failed: {exc}") + sys.exit(1) + + try: + data = json.loads(body) + content = data["choices"][0]["message"]["content"] + return json.loads(content) + except Exception as exc: # noqa: BLE001 + print(f"::error::Model output parsing failed: {exc}") + print(body) + sys.exit(1) + + +def main() -> int: + title = os.environ.get("PR_TITLE", "") + body = os.environ.get("PR_BODY", "") + template_path = os.environ.get("TEMPLATE_PATH", ".github/pull_request_template.md") + policy_path = os.environ.get("POLICY_PATH", ".github/pr_compliance_policy.md") + openai_key = os.environ.get("OPENAI_API_KEY", "") + model = os.environ.get("OPENAI_MODEL", "gpt-4o-mini") + + if not body.strip(): + print("::error::PR description is empty.") + return 1 + + template = read_file(template_path) + policy = read_file(policy_path) + + deterministic_errors = [] + deterministic_errors.extend(validate_definition_of_done(body)) + deterministic_errors.extend(validate_placeholders(body)) + + if deterministic_errors: + for error in deterministic_errors: + print(f"::error::{error}") + return 1 + + if not openai_key: + print("::warning::OPENAI_API_KEY is not configured. Skipping policy/template LLM validation.") + return 0 + + prompt = f""" +You are a strict compliance checker for pull requests. + +Evaluate the PR TITLE and PR DESCRIPTION against: +1. The PR template (required sections and intent) +2. The PR compliance policy (source of truth) + +Template: +--- +{template} +--- + +Policy: +--- +{policy} +--- + +PR title: +--- +{title} +--- + +PR description: +--- +{body} +--- + +Rules: +- N/A is allowed only when accompanied by at least one sentence of explanation. +- If policy and template conflict, policy wins. +- Be conservative: unclear or missing details are violations. +- Return JSON only with this schema: +{{ + "pass": boolean, + "violations": [ + {{ + "id": string, + "severity": "error" | "warning", + "location": "title" | "description", + "message": string, + "suggested_fix": string + }} + ] +}} +""" + + result = call_openai(model, openai_key, prompt) + + passed = bool(result.get("pass", False)) + violations = result.get("violations", []) + error_violations = [entry for entry in violations if entry.get("severity") == "error"] + + print("Compliance result:") + print(json.dumps(result, indent=2)) + + if (not passed) or error_violations: + print("::error::PR failed compliance checks.") + for violation in violations: + severity = violation.get("severity", "error") + location = violation.get("location", "description") + message = violation.get("message", "") + fix = violation.get("suggested_fix", "") + print(f"- [{severity}] ({location}) {message}") + if fix: + print(f" suggested_fix: {fix}") + return 1 + + for violation in violations: + if violation.get("severity") == "warning": + location = violation.get("location", "description") + message = violation.get("message", "") + print(f"::warning::({location}) {message}") + + print("PR passed compliance checks.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/pull-request-compliance/tests/run.sh b/pull-request-compliance/tests/run.sh new file mode 100755 index 0000000..7e4ac74 --- /dev/null +++ b/pull-request-compliance/tests/run.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd "$(dirname "$0")/../.." +python3 -m unittest discover -s pull-request-compliance/tests -p 'test_*.py' -v diff --git a/pull-request-compliance/tests/test_check_pr_compliance.py b/pull-request-compliance/tests/test_check_pr_compliance.py new file mode 100644 index 0000000..59998eb --- /dev/null +++ b/pull-request-compliance/tests/test_check_pr_compliance.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 + +import importlib.util +import io +import os +import pathlib +import tempfile +import unittest +from contextlib import redirect_stdout +from unittest.mock import patch + +SCRIPT_PATH = pathlib.Path(__file__).resolve().parents[1] / "scripts" / "check_pr_compliance.py" +SPEC = importlib.util.spec_from_file_location("check_pr_compliance", SCRIPT_PATH) +MODULE = importlib.util.module_from_spec(SPEC) +assert SPEC and SPEC.loader +SPEC.loader.exec_module(MODULE) + + +class PullRequestComplianceTests(unittest.TestCase): + def run_main( + self, + body: str, + title: str = "Test title", + api_key: str = "", + openai_result: dict | None = None, + ): + with tempfile.TemporaryDirectory() as tempdir: + template_path = pathlib.Path(tempdir) / "template.md" + policy_path = pathlib.Path(tempdir) / "policy.md" + template_path.write_text("## Summary\n## Ticket\n", encoding="utf-8") + policy_path.write_text("Policy content", encoding="utf-8") + + env = { + "PR_TITLE": title, + "PR_BODY": body, + "TEMPLATE_PATH": str(template_path), + "POLICY_PATH": str(policy_path), + "OPENAI_API_KEY": api_key, + "OPENAI_MODEL": "gpt-4o-mini", + } + + output = io.StringIO() + with patch.dict(os.environ, env, clear=False), redirect_stdout(output): + if openai_result is None: + rc = MODULE.main() + else: + with patch.object(MODULE, "call_openai", return_value=openai_result) as mocked: + rc = MODULE.main() + self.assertEqual(mocked.call_count, 1) + + return rc, output.getvalue() + + def test_empty_body_fails(self): + rc, output = self.run_main(" ") + self.assertEqual(rc, 1) + self.assertIn("PR description is empty", output) + + def test_dod_unchecked_item_fails(self): + body = """## Definition of Done +- [x] Good +- [ ] Missing +""" + rc, output = self.run_main(body) + self.assertEqual(rc, 1) + self.assertIn("unchecked items", output) + + def test_placeholders_fail(self): + rc, output = self.run_main("## Summary\nTODO later\n") + self.assertEqual(rc, 1) + self.assertIn("Forbidden placeholder 'TODO'", output) + + rc, output = self.run_main("## Summary\nNeed clarification ??\n") + self.assertEqual(rc, 1) + self.assertIn("Forbidden placeholder '??'", output) + + def test_no_api_key_skips_llm_and_passes_when_deterministic_checks_pass(self): + body = """## Summary +Looks good. + +## Definition of Done +- [x] Requirement understood +""" + rc, output = self.run_main(body, api_key="") + self.assertEqual(rc, 0) + self.assertIn("Skipping policy/template LLM validation", output) + + def test_llm_error_violation_fails(self): + body = "## Summary\nLooks good\n" + llm_result = { + "pass": False, + "violations": [ + { + "id": "missing-ticket", + "severity": "error", + "location": "description", + "message": "Ticket reference missing", + "suggested_fix": "Add a ticket link", + } + ], + } + rc, output = self.run_main(body, api_key="test-key", openai_result=llm_result) + self.assertEqual(rc, 1) + self.assertIn("PR failed compliance checks", output) + self.assertIn("Ticket reference missing", output) + + def test_llm_warning_only_passes(self): + body = "## Summary\nLooks good\n" + llm_result = { + "pass": True, + "violations": [ + { + "id": "clarity", + "severity": "warning", + "location": "description", + "message": "Could add more detail", + "suggested_fix": "Explain rollout", + } + ], + } + rc, output = self.run_main(body, api_key="test-key", openai_result=llm_result) + self.assertEqual(rc, 0) + self.assertIn("PR passed compliance checks", output) + self.assertIn("Could add more detail", output) + + +if __name__ == "__main__": + unittest.main()