From 748f0310b564990e365b7031333ad9b0268e6e2c Mon Sep 17 00:00:00 2001 From: Sall Date: Sun, 26 Jul 2026 18:44:33 +0100 Subject: [PATCH] feat(agent): add portable learning capture gate --- .github/README.md | 24 +-- .github/instruction-surfaces.json | 24 +++ .../skills/review-project-learning/SKILL.md | 25 +++ AGENTS.md | 15 +- runbooks/learning-capture.md | 147 ++++++++++++++++++ scripts/test_validate_agent_policy.py | 43 +++++ 6 files changed, 262 insertions(+), 16 deletions(-) create mode 100644 .github/skills/review-project-learning/SKILL.md create mode 100644 runbooks/learning-capture.md diff --git a/.github/README.md b/.github/README.md index 69c4084e7..70259e33f 100644 --- a/.github/README.md +++ b/.github/README.md @@ -37,16 +37,16 @@ The `.github` repository is a [special GitHub repository](https://docs.github.co ## Repository Structure -| Path | Purpose | -| ------------------------------------------------- | ----------------------------------------------------------------------------------------------------------- | -| [`AGENTS.md`](../AGENTS.md) | Canonical org-wide instructions for AI coding agents and maintainers | -| [`PATTERNS.md`](../PATTERNS.md) | Cross-repo implementation idioms grounded in real repositories | -| [`decisions/`](../decisions/) | Architectural decision records for non-obvious org-wide choices | -| [`runbooks/`](../runbooks/) | Repeatable operational workflows such as org review, triage, ADR drafting, and release coordination | -| [`profile/`](../profile/) | Organization profile — the README and visual assets displayed on the [org page](https://github.com/z-shell) | -| [`actions/`](../actions/) | Reusable composite GitHub Actions shared across all org repositories | -| [`workflow-templates/`](../workflow-templates/) | Starter workflow templates available in the **Actions > New workflow** tab | -| [`renovate-config.json`](../renovate-config.json) | Shared Renovate preset for routine dependency version updates | +| Path | Purpose | +| ------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------- | +| [`AGENTS.md`](../AGENTS.md) | Canonical org-wide instructions for AI coding agents and maintainers | +| [`PATTERNS.md`](../PATTERNS.md) | Cross-repo implementation idioms grounded in real repositories | +| [`decisions/`](../decisions/) | Architectural decision records for non-obvious org-wide choices | +| [`runbooks/`](../runbooks/) | Repeatable operational workflows such as learning capture, org review, triage, ADR drafting, and release coordination | +| [`profile/`](../profile/) | Organization profile — the README and visual assets displayed on the [org page](https://github.com/z-shell) | +| [`actions/`](../actions/) | Reusable composite GitHub Actions shared across all org repositories | +| [`workflow-templates/`](../workflow-templates/) | Starter workflow templates available in the **Actions > New workflow** tab | +| [`renovate-config.json`](../renovate-config.json) | Shared Renovate preset for routine dependency version updates | ## Instruction Architecture @@ -59,6 +59,10 @@ The organization uses a portable, manifest-backed instruction architecture: adapter to `AGENTS.md`; it is not a policy owner. - [`instruction-update.md`](../runbooks/instruction-update.md) is the required impact review for every material instruction change. +- [`learning-capture.md`](../runbooks/learning-capture.md) defines the + evidence, destination, authority, and completion-review workflow. +- [`review-project-learning`](skills/review-project-learning/SKILL.md) is the + advisory reusable skill for that workflow. - [`validate-agent-policy.py`](../scripts/validate-agent-policy.py) and [`agent-instructions.yml`](workflows/agent-instructions.yml) enforce the public instruction contract. diff --git a/.github/instruction-surfaces.json b/.github/instruction-surfaces.json index debfffdc9..2c3a7708f 100644 --- a/.github/instruction-surfaces.json +++ b/.github/instruction-surfaces.json @@ -351,6 +351,18 @@ "review_owner": "z-shell maintainers", "canonical_for": [] }, + { + "id": "skill-review-project-learning", + "path": ".github/skills/review-project-learning/SKILL.md", + "kind": "skill", + "authority": "advisory", + "consumers": ["copilot"], + "tasks": ["learning-capture", "completion-review"], + "file_patterns": ["**"], + "required": false, + "review_owner": "z-shell maintainers", + "canonical_for": [] + }, { "id": "runbook-adr", "path": "runbooks/adr.md", @@ -399,6 +411,18 @@ "review_owner": "z-shell maintainers", "canonical_for": ["instruction-impact-review"] }, + { + "id": "runbook-learning-capture", + "path": "runbooks/learning-capture.md", + "kind": "runbook", + "authority": "canonical-detail", + "consumers": ["codex", "claude-code", "copilot", "gemini-cli", "human"], + "tasks": ["learning-capture", "completion-review"], + "file_patterns": ["**"], + "required": true, + "review_owner": "z-shell maintainers", + "canonical_for": ["learning-capture"] + }, { "id": "runbook-labels", "path": "runbooks/labels.md", diff --git a/.github/skills/review-project-learning/SKILL.md b/.github/skills/review-project-learning/SKILL.md new file mode 100644 index 000000000..aa34452a5 --- /dev/null +++ b/.github/skills/review-project-learning/SKILL.md @@ -0,0 +1,25 @@ +--- +name: review-project-learning +description: Use when non-trivial project work is about to be declared complete, after unexpected failures or corrected assumptions, when a reusable procedure emerged, or when meaningful work is being handed off. +--- + +# Review Project Learning + +1. Locate and read the complete canonical `runbooks/learning-capture.md` + selected by the current repository's instruction routing. +2. Review the actual diff, commands, failures, corrections, decisions, and + deferred work for the task being reviewed. Do not rely only on the + completion summary or import evidence from unrelated workspace state. +3. Search the proposed canonical owner for equivalent or contradictory + guidance. +4. Choose exactly one outcome from the runbook's `Valid outcomes`. +5. Prefer executable prevention over prose and an existing owner over a new + artifact. +6. Keep `No durable learning` internal. +7. For a real candidate, report the finding, evidence, proposed owner, and + authority status. +8. Do not treat a final response or handoff as durable capture unless it links + to the authorized canonical owner. +9. Do not write memory without explicit maintainer request or consent. +10. Do not let the review expand repository, publication, merge, release, or + external-write authority. diff --git a/AGENTS.md b/AGENTS.md index 4d5bc4098..921432381 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -132,14 +132,17 @@ Unless a maintainer asks otherwise, these workflows produce drafts only. ## Learning capture -Non-trivial sessions should end with durable follow-up, not silent local memory. If you discover: +Before claiming non-trivial work complete, perform a learning and reuse review +using `runbooks/learning-capture.md`. -- a pattern used in at least two repositories -- a decision that should be recorded -- a runbook gap -- a tooling gap +`No durable learning` is a valid silent result. Promote a finding only when it +is evidence-backed, likely to recur, and routed to the smallest existing +canonical owner. Prefer executable checks over prose. Do not create memory, +instructions, skills, issues, ADRs, runbooks, or documentation merely to show +that the review happened. -capture it in the relevant issue, PR, or draft change to `PATTERNS.md`, `decisions/`, or `runbooks/` for human review. +Hooks and skills may remind or guide the review, but they are optional and +cannot own this mandatory rule. ## When this file is wrong diff --git a/runbooks/learning-capture.md b/runbooks/learning-capture.md new file mode 100644 index 000000000..2384d2124 --- /dev/null +++ b/runbooks/learning-capture.md @@ -0,0 +1,147 @@ +# Runbook - Learning capture + +Use this workflow before claiming non-trivial project work complete. Its purpose +is to prevent repeated investigation while keeping project knowledge concise, +evidence-based, and attached to the correct source of truth. + +## Scope + +Run the review after implementation, debugging, research, release, incident, +policy, architecture, documentation, or multi-repository work that required +meaningful judgment. + +Treat a quick factual answer, routine formatting, a simple read-only lookup, or +an unchanged status check as trivial unless it revealed a reusable correction +or project constraint. + +## Review questions + +1. What was unexpectedly difficult, incorrect, repeated, or newly discovered? +2. Is the finding likely to affect a future task? +3. What evidence supports it? +4. Who needs the knowledge and at what repository scope? +5. Can the recurrence be prevented mechanically? +6. Which existing canonical surface owns the finding? +7. Does that surface already contain equivalent or contradictory guidance? + +## Strong candidate signals + +- an unexpected test, build, release, deployment, or tooling failure +- a maintainer correction or a disproved assumption +- a plausible approach that was rejected for a durable reason +- repeated investigation or the same finding in more than one repository +- a missing, stale, duplicated, or contradictory instruction +- a reusable command sequence or operational procedure +- meaningful deferred scope or a blocker that another session must resume +- a security, privacy, publication, or repository-boundary discovery + +## Valid outcomes + +- `No durable learning`: nothing should be written or reported. +- `Candidate identified`: report the evidence, scope, proposed owner, and + required authority. +- `Existing owner updated`: name the canonical artifact and verification. +- `Deferred follow-up recorded`: link the owning issue, pull request, or + tracker item. + +`No durable learning` is a successful outcome. Never manufacture a lesson to +satisfy the workflow. + +## Destination matrix + +| Finding | Preferred owner | +| ------------------------------------------------------------- | -------------------------------------------------------------- | +| A recurrence that can be detected mechanically | Code, test, type, schema, lint rule, validator, or script | +| A mandatory rule that applies broadly | Root `AGENTS.md` | +| Mandatory detail limited by task or path | Routed scoped instruction | +| A reusable multi-step agent workflow | Skill | +| An implementation idiom observed in at least two repositories | `PATTERNS.md` | +| A significant or difficult-to-reverse decision | Proposed ADR | +| A repeatable operational procedure | Runbook | +| Active work, blocker, or deferred work | Owning GitHub issue or pull request, plus Linear when required | +| An unfinished session another agent must resume | GitHub-native agent handoff | +| Durable long-form user or maintainer guidance | Appropriate documentation or wiki area | +| A private maintainer preference or private heuristic | Private memory, with explicit maintainer request or consent | +| A one-time detail already clear from code or history | No additional artifact | + +Prefer executable prevention over prose. Extend an existing canonical owner +instead of creating a second source of truth. + +## Candidate quality gate + +A candidate is ready for promotion only when all of these are true: + +1. Evidence from the reviewed task is named, such as a failing command, review + finding, corrected assumption, repeated search, or repository examples. +2. Future applicability is stated without claiming a universal rule from one + unusual event. +3. The intended consumers and repository scope are explicit. +4. The canonical owner has been searched for duplicates and contradictions. +5. The smallest effective destination has been selected. +6. The proposed change is within the current task's authority. +7. Verification and an owner exist for any resulting action. + +If any item is missing, keep the result as a candidate or discard it. + +## Authority and privacy gate + +- Do not write private memory unless the maintainer explicitly requested it or + explicitly consented in the current session. +- Do not create external issues, pull requests, tracker updates, or published + documentation without authority for that workflow. +- Do not edit another repository merely because it is the ideal destination. + Report the candidate and request the required scope. +- Never store secrets, credentials, personal data, private hosts or addresses, + or machine-specific state in public artifacts. +- A learning review does not broaden implementation, publication, merge, + release, or cleanup authority. + +## Completion reporting + +Keep `No durable learning` internal unless the maintainer requests an audit +trail. + +When a candidate matters, include a concise completion note: + +```text +Learning candidate: +Evidence: +Proposed owner: +Status: promoted | recorded for follow-up | approval required +``` + +A final response or handoff is not a durable owner by itself. When authority is +missing, identify the candidate and required destination without implying it +was captured. + +Do not append this block when no candidate exists. + +## Pilot evaluation + +Review the first 30 non-trivial tasks or four weeks of use, whichever occurs +first. Measure: + +- the number of reviewed tasks +- silent `No durable learning` outcomes +- candidates proposed +- candidates promoted, rejected, or merged into an existing owner +- repeated investigations that still occurred +- later tasks that reused a promoted item +- median review overhead +- stale or contradictory artifacts introduced + +Do not add a command hook merely because one review was missed. Consider a +runtime reminder only when the measured omission rate exceeds 10 percent and a +prototype can keep false reminders below 10 percent without parsing an +unstable transcript format. + +## See also + +- `AGENTS.md` +- `.github/AGENT_MEMORY.md` +- `.github/instruction-surfaces.json` +- `PATTERNS.md` +- `decisions/` +- `runbooks/adr.md` +- `runbooks/instruction-update.md` +- `runbooks/project-tracker.md` diff --git a/scripts/test_validate_agent_policy.py b/scripts/test_validate_agent_policy.py index da5ae4493..ef3a08149 100644 --- a/scripts/test_validate_agent_policy.py +++ b/scripts/test_validate_agent_policy.py @@ -1550,6 +1550,49 @@ def test_public_repository_requires_manifest_routing_for_all_runtimes(self) -> N for fragment in required_fragments: self.assertIn(fragment, policy) + def test_public_repository_declares_learning_capture_surfaces(self) -> None: + manifest = json.loads( + (PUBLIC_ROOT / ".github/instruction-surfaces.json").read_text() + ) + surfaces = {item["id"]: item for item in manifest["surfaces"]} + + self.assertEqual( + surfaces["runbook-learning-capture"], + { + "id": "runbook-learning-capture", + "path": "runbooks/learning-capture.md", + "kind": "runbook", + "authority": "canonical-detail", + "consumers": [ + "codex", + "claude-code", + "copilot", + "gemini-cli", + "human", + ], + "tasks": ["learning-capture", "completion-review"], + "file_patterns": ["**"], + "required": True, + "review_owner": "z-shell maintainers", + "canonical_for": ["learning-capture"], + }, + ) + self.assertEqual( + surfaces["skill-review-project-learning"], + { + "id": "skill-review-project-learning", + "path": ".github/skills/review-project-learning/SKILL.md", + "kind": "skill", + "authority": "advisory", + "consumers": ["copilot"], + "tasks": ["learning-capture", "completion-review"], + "file_patterns": ["**"], + "required": False, + "review_owner": "z-shell maintainers", + "canonical_for": [], + }, + ) + def test_public_repository_prohibits_vendor_root_instruction_files(self) -> None: policy = (PUBLIC_ROOT / "AGENTS.md").read_text() self.assertIn(