From 75cdd7687cd1201d3750659796b712255601f52a Mon Sep 17 00:00:00 2001 From: Keith Kirkwood Date: Fri, 10 Jul 2026 16:13:27 +0100 Subject: [PATCH 01/23] OPENR-174: Check and update product and distribution metas, and suggest them as changes --- .github/workflows/enhance.yml | 70 +++++++ conf.py | 1 + tools/README.md | 96 ++++++++++ tools/ensure_meta_tags.py | 167 +++++++++++++++++ tools/meta_tags.yaml | 5 + tools/rst_utils.py | 339 ++++++++++++++++++++++++++++++++++ 6 files changed, 678 insertions(+) create mode 100644 .github/workflows/enhance.yml create mode 100644 tools/README.md create mode 100644 tools/ensure_meta_tags.py create mode 100644 tools/meta_tags.yaml create mode 100644 tools/rst_utils.py diff --git a/.github/workflows/enhance.yml b/.github/workflows/enhance.yml new file mode 100644 index 00000000000..bb2f74874ba --- /dev/null +++ b/.github/workflows/enhance.yml @@ -0,0 +1,70 @@ +name: Enhance + +# pull_request_target grants GITHUB_TOKEN write access on fork PRs. +# PR code is checked out as data only; scripts run from the trusted base checkout. +on: + pull_request_target: + types: + - opened + - synchronize + - reopened + +permissions: + contents: read + pull-requests: write + +jobs: + ensure-meta-tags: + runs-on: ubuntu-24.04 + steps: + - name: Checkout PR code + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + repository: ${{ github.event.pull_request.head.repo.full_name }} + fetch-depth: 0 + + - name: Checkout trusted base scripts + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.base.sha }} + path: .trusted-base + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install PyYAML + run: pip install --no-warn-script-location pyyaml + + - name: Ensure product and distribution meta tags + run: | + git fetch origin "${{ github.event.pull_request.base.sha }}" + + mapfile -t changed_rst < <( + git diff --name-only --diff-filter=ACMR \ + "${{ github.event.pull_request.base.sha }}...${{ github.event.pull_request.head.sha }}" \ + -- '*.rst' + ) + + if [ "${#changed_rst[@]}" -eq 0 ]; then + echo "No changed RST files in this pull request." + exit 0 + fi + + python3 .trusted-base/tools/ensure_meta_tags.py \ + --config .trusted-base/tools/meta_tags.yaml \ + "${changed_rst[@]}" + + echo "Working tree after ensure_meta_tags:" + git status --short -- "${changed_rst[@]}" + git diff -- "${changed_rst[@]}" + + - name: Suggest meta tag changes + uses: parkerbxyz/suggest-changes@v3 + with: + comment: >- + Missing product/distribution meta tags were added using values from + config. Please review and commit the suggestions. + event: COMMENT diff --git a/conf.py b/conf.py index 941d0b951ee..88c03790edf 100644 --- a/conf.py +++ b/conf.py @@ -185,6 +185,7 @@ 'DISTRO_UBUNTU_DEB_PLATFORM': distro_ubuntu_deb_platform['rolling'], 'DISTRO_ARM_STATUS_SUFFIX': distro_arm_status_suffix.get('rolling', 'unv8'), 'REPOS_FILE_BRANCH': 'rolling', + 'PRODUCT': 'ROS 2', } html_favicon = 'favicon.ico' diff --git a/tools/README.md b/tools/README.md new file mode 100644 index 00000000000..51f1d5c6fca --- /dev/null +++ b/tools/README.md @@ -0,0 +1,96 @@ +# Documentation tools + +Helpers for ensuring reStructuredText (`.rst`) metadata on documentation pull requests. + +## Layout + +| File | Purpose | +|------|---------| +| [`rst_utils.py`](rst_utils.py) | Regex-based read/write of `.. meta::` and `.. short-description::` directives | +| [`meta_tags.yaml`](meta_tags.yaml) | Default values for missing meta fields | +| [`ensure_meta_tags.py`](ensure_meta_tags.py) | CLI that adds missing meta fields from the config | + +## Configuration + +[`meta_tags.yaml`](meta_tags.yaml) defines which `.. meta::` fields to ensure and the value to inject when each is missing: + +```yaml +meta: + product: "{PRODUCT}" + distribution: "{DISTRO}" +``` + +The `meta` map lists every field the script checks. Add a new key to extend coverage without changing Python code. + +`{PRODUCT}` and `{DISTRO}` are Sphinx substitution macros expanded at build time from [`conf.py`](../conf.py). Edit this file when you need different default values for suggested meta tags. + +## `rst_utils.py` + +Low-level utilities for locating and editing Sphinx directives in RST source: + +- **`get_meta_names_from_content`** — field names already present in the first `.. meta::` block +- **`inject_metadata_to_content`** — append missing `:name: value` lines to an existing block, or prepend a new `.. meta::` block; never overwrites existing fields + +The module also contains helpers for `.. short-description::` directives for future use. + +## `ensure_meta_tags.py` + +Checks each given `.rst` file for the fields listed in `meta_tags.yaml`. When a field is missing, it is added with the configured value. Files that already have all configured fields are left unchanged. + +### Usage + +From the repository root: + +```bash +python3 tools/ensure_meta_tags.py path/to/article.rst +``` + +Multiple files: + +```bash +python3 tools/ensure_meta_tags.py source/Topic/A.rst source/Topic/B.rst +``` + +Options: + +- `--config PATH` — YAML config file (default: `tools/meta_tags.yaml`) +- `-v` / `--verbose` — enable debug logging + +### Example + +Before: + +```rst +My Article +========== + +Some content. +``` + +After: + +```rst +.. meta:: + :product: {PRODUCT} + :distribution: {DISTRO} + +My Article +========== + +Some content. +``` + +If a `.. meta::` block already exists, missing fields are appended to it rather than creating a new block. + +## Continuous integration + +The workflow [`.github/workflows/enhance.yml`](../.github/workflows/enhance.yml) runs on every pull request (including from forks): + +1. Checks out the PR’s `.rst` files as untrusted data +2. Checks out the base branch into `.trusted-base/` for the script and config +3. Runs the trusted `ensure_meta_tags.py` against changed RST files +4. Posts any resulting edits as review suggestions via [`parkerbxyz/suggest-changes`](https://github.com/marketplace/actions/suggest-changes-action) + +Contributors can review and accept the suggested meta tags before merging. No GitHub App or repository secrets are required. + +The workflow uses [`pull_request_target`](https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request_target) so the default `GITHUB_TOKEN` can post review suggestions on fork PRs. To avoid running untrusted code with elevated permissions, only the base-branch copy of `ensure_meta_tags.py` and `meta_tags.yaml` is executed; PR content is treated as data to read and update. See [Mitigating the risks of untrusted code checkout](https://docs.github.com/en/actions/reference/security/secure-use#mitigating-the-risks-of-untrusted-code-checkout). diff --git a/tools/ensure_meta_tags.py b/tools/ensure_meta_tags.py new file mode 100644 index 00000000000..e9ee530785c --- /dev/null +++ b/tools/ensure_meta_tags.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 +""" +Ensure configured metadata fields exist in RST ``.. meta::`` blocks. + +Missing fields are injected with values from ``meta_tags.yaml``. +Existing fields are never overwritten. +""" + +from __future__ import annotations + +import argparse +import logging +import sys +from pathlib import Path + +import yaml + +# Allow ``python3 tools/ensure_meta_tags.py`` from the repository root. +_TOOLS_DIR = Path(__file__).resolve().parent +if str(_TOOLS_DIR) not in sys.path: + sys.path.insert(0, str(_TOOLS_DIR)) + +from rst_utils import get_meta_names_from_content, inject_metadata_to_content + +logger = logging.getLogger(__name__) + +DEFAULT_CONFIG_PATH = _TOOLS_DIR / "meta_tags.yaml" +RST_EXTENSION = ".rst" + + +def load_meta_config(config_path: Path) -> dict[str, str]: + """ + Load the ``meta`` mapping from a YAML config file. + + Returns: + Field name to value mapping. + + Raises: + SystemExit: If the file is missing, invalid, or has no usable ``meta`` map. + """ + if not config_path.is_file(): + logger.error("Config file not found: %s", config_path) + raise SystemExit(1) + + try: + raw = yaml.safe_load(config_path.read_text(encoding="utf-8")) + except yaml.YAMLError as exc: + logger.error("Invalid YAML in %s: %s", config_path, exc) + raise SystemExit(1) from exc + + if not isinstance(raw, dict): + logger.error("Config %s must be a YAML mapping", config_path) + raise SystemExit(1) + + meta = raw.get("meta") + if not isinstance(meta, dict) or not meta: + logger.error("Config %s must contain a non-empty 'meta' mapping", config_path) + raise SystemExit(1) + + validated: dict[str, str] = {} + for key, value in meta.items(): + if not isinstance(key, str) or not key.strip(): + logger.error("Config %s: meta keys must be non-empty strings", config_path) + raise SystemExit(1) + if not isinstance(value, str): + logger.error( + "Config %s: meta value for %r must be a string, got %s", + config_path, + key, + type(value).__name__, + ) + raise SystemExit(1) + validated[key] = value + + return validated + + +def _missing_meta_fields(content: str, meta_config: dict[str, str]) -> list[str]: + present = get_meta_names_from_content(content) + return [field for field in meta_config if field not in present] + + +def ensure_meta_tags_in_file(path: Path, meta_config: dict[str, str]) -> bool: + """ + Add missing configured meta fields to one RST file. + + Returns: + True if the file was updated, False otherwise. + """ + content = path.read_text(encoding="utf-8") + missing = _missing_meta_fields(content, meta_config) + if not missing: + logger.info("%s: all configured meta fields present", path) + return False + + metadata = {field: meta_config[field] for field in missing} + new_content, changed = inject_metadata_to_content(content, metadata) + if not changed: + return False + + path.write_text(new_content, encoding="utf-8") + logger.info("%s: added meta fields %s", path, ", ".join(missing)) + return True + + +def _collect_rst_paths(paths: list[str]) -> list[Path]: + rst_paths: list[Path] = [] + for raw in paths: + path = Path(raw) + if path.suffix.lower() != RST_EXTENSION: + logger.debug("Skipping non-RST path: %s", raw) + continue + if not path.is_file(): + logger.warning("Skipping missing file: %s", raw) + continue + rst_paths.append(path) + return rst_paths + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description=( + "Ensure configured meta tags exist in RST files. " + "Missing fields are added with values from a YAML config file." + ), + ) + parser.add_argument( + "paths", + nargs="+", + help="One or more .rst file paths to check", + ) + parser.add_argument( + "--config", + type=Path, + default=DEFAULT_CONFIG_PATH, + help=f"YAML config file (default: {DEFAULT_CONFIG_PATH})", + ) + parser.add_argument( + "-v", + "--verbose", + action="store_true", + help="Enable debug logging", + ) + args = parser.parse_args(argv) + + logging.basicConfig( + level=logging.DEBUG if args.verbose else logging.INFO, + format="%(levelname)s: %(message)s", + ) + + meta_config = load_meta_config(args.config) + rst_paths = _collect_rst_paths(args.paths) + if not rst_paths: + logger.info("No RST files to process") + return 0 + + updated = 0 + for path in rst_paths: + if ensure_meta_tags_in_file(path, meta_config): + updated += 1 + + logger.info("Updated %d of %d file(s)", updated, len(rst_paths)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/meta_tags.yaml b/tools/meta_tags.yaml new file mode 100644 index 00000000000..4dca7064556 --- /dev/null +++ b/tools/meta_tags.yaml @@ -0,0 +1,5 @@ +# Default values for missing .. meta:: fields injected by ensure_meta_tags.py. +# Sphinx expands {PRODUCT} and {DISTRO} at build time (see conf.py). +meta: + product: "{PRODUCT}" + distribution: "{DISTRO}" diff --git a/tools/rst_utils.py b/tools/rst_utils.py new file mode 100644 index 00000000000..ec35e17c2d4 --- /dev/null +++ b/tools/rst_utils.py @@ -0,0 +1,339 @@ +""" +Utilities for editing reStructuredText source, in particular ``.. meta::`` and +``.. short-description::`` directives. +""" + +import logging +import re + +logger = logging.getLogger(__name__) + + +def _find_meta_block(content: str) -> tuple[int, int, int, str, str]: + """ + Locate the first ``.. meta::`` directive in RST source. + + The directive block consists of the explicit marker line followed by + contiguous indented lines; a blank line or a less-indented line ends the + block (per reStructuredText directive block rules). + + Args: + content: The RST file content to search. + + Returns: + Tuple of ``(start, marker_end, block_end, inner, indent)``. + If no directive is found, ``start``, ``marker_end``, and ``block_end`` + are ``-1``, ``inner`` is ``''``, and ``indent`` defaults to three spaces. + """ + # Explicit markup + directive name; block body starts on the following line only + match = re.search(r"^\.\.\s+meta::\s*\n", content, re.MULTILINE) + if not match: + return -1, -1, -1, "", " " + + start = match.start() # Byte index of ``.. meta::`` (for whole-directive splice) + marker_end = match.end() # First character after the marker line's newline + indent = " " # Default field indent when the block is empty or we prepend a new block + inner_parts: list[str] = [] + consumed = 0 # Length of directive body in ``content`` (may omit final ``\n`` on last line) + remainder = content[marker_end:] # Scan forward only inside this file slice + + for line in remainder.splitlines(keepends=True): + if line.strip() == "": + break # Blank line terminates the directive block + if not line.startswith((" ", "\t")): + break # Body element at column 0 ends the block + if not inner_parts: + ws_len = len(line) - len(line.lstrip(" \t")) + indent = line[:ws_len] # Reuse the author's indent for new ``:name:`` lines + inner_parts.append(line) + consumed += len(line) + + block_end = marker_end + consumed # Exclusive end of the directive in ``content`` + inner = "".join(inner_parts) + # EOF without ``\n`` yields a last ``splitlines`` element with no newline—append one before new fields + if inner and not inner.endswith("\n"): + inner += "\n" + return start, marker_end, block_end, inner, indent + + +def _extract_meta_names_from_block(meta_block_inner: str) -> set[str]: + """ + Collect field names from the body of a ``.. meta::`` directive. + + Each line of the form ``:name: value`` contributes ``name`` (Docutils also + allows forms such as ``:name attr=value:``; the captured segment matches + that usage). + + Args: + meta_block_inner: The inner text of the meta block. + + Returns: + A set of field names found in the block. + """ + names: set[str] = set() + # Field list lines only; group 1 is the name segment (includes ``attr=value`` forms before the final ``:``) + for field_match in re.finditer(r"^[ \t]+:([^:\n]+?):", meta_block_inner, re.MULTILINE): + names.add(field_match.group(1).strip()) + return names + + +def get_meta_names_from_content(content: str) -> set[str]: + """ + Return the set of field names already present in the first ``.. meta::`` block. + + If no ``.. meta::`` directive exists, returns an empty set. + + Args: + content: The RST file content to search. + + Returns: + A set of field names present in the meta block. + """ + _start, _marker_end, _block_end, inner, _indent = _find_meta_block(content) + return _extract_meta_names_from_block(inner) + + +def _normalise_meta_field_value(value: str) -> str: + """ + Collapse whitespace so the meta field body stays a single logical line. + + Args: + value: The raw field value. + + Returns: + The normalised field value. + """ + return " ".join(value.split()) # Docutils treats the field body as one string; keep it one physical line + + +def inject_metadata_to_content(content: str, metadata: dict[str, str]) -> tuple[str, bool]: + """ + Insert or append ``.. meta::`` field entries for the given name/value pairs. + + Appends to an existing ``.. meta::`` block when present; otherwise prepends + a new block at the start of the document (leading whitespace is stripped so + the directive is the first element). Skips keys that already appear in the + block. + + Returns: + Updated source and whether any change was made. + """ + start, marker_end, block_end, inner, indent = _find_meta_block(content) + names = _extract_meta_names_from_block(inner) # Snapshot before we add keys from this same batch + additions: list[str] = [] + + for key, raw_value in metadata.items(): + if key in names: + logger.warning( + "Existing meta field %r in .. meta:: block; skipping", + key, + ) + continue + value = _normalise_meta_field_value(raw_value) + additions.append(f"{indent}:{key}: {value}\n") + names.add(key) # Prevent duplicate inserts if ``metadata`` repeats a key + + if not additions: + return content, False # Nothing new to write; leave the file untouched + + new_inner = inner + "".join(additions) # Existing fields unchanged, then appended lines + + if start >= 0: + # Replace only the directive body slice; ``marker_end``/``block_end`` bracket the original inner + # Normalise trailing whitespace: one blank line after the block + remainder = content[block_end:].lstrip() + new_content = content[:marker_end] + new_inner + "\n" + remainder + else: + # No ``.. meta::`` yet: insert at document start; strip leading whitespace so the block is truly first + remainder = content.lstrip() + new_content = ".. meta::\n" + "".join(additions) + "\n" + remainder # Blank line after block separates it from the body + + return new_content, True + + +def _find_short_description_block(content: str) -> tuple[int, int, int, str, str]: + """ + Locate the first ``.. short-description::`` directive in RST source. + + Uses the same block-boundary rules as ``_find_meta_block``: the body is + contiguous indented lines until a blank line or a line starting at column 0. + + Args: + content: The RST file content to search. + + Returns: + Tuple of ``(start, marker_end, block_end, inner, indent)``. + If no directive is found, ``start``, ``marker_end``, and ``block_end`` + are ``-1``, ``inner`` is ``''``, and ``indent`` defaults to three spaces. + """ + match = re.search(r"^\.\.\s+short-description::\s*\n", content, re.MULTILINE) + if not match: + return -1, -1, -1, "", " " + + start = match.start() + marker_end = match.end() + indent = " " + inner_parts: list[str] = [] + consumed = 0 + remainder = content[marker_end:] + + for line in remainder.splitlines(keepends=True): + if line.strip() == "": + break + if not line.startswith((" ", "\t")): + break + if not inner_parts: + ws_len = len(line) - len(line.lstrip(" \t")) + indent = line[:ws_len] + inner_parts.append(line) + consumed += len(line) + + block_end = marker_end + consumed + inner = "".join(inner_parts) + if inner and not inner.endswith("\n"): + inner += "\n" + return start, marker_end, block_end, inner, indent + + +def _short_description_inner_has_content(inner: str) -> bool: + """ + True when the directive body contains non-whitespace text. + + Args: + inner: The inner text of the short-description block. + + Returns: + True if the body has content, False otherwise. + """ + for line in inner.splitlines(): + if line.strip(): + return True + return False + + +def has_short_description_content(content: str) -> bool: + """ + Return whether the document already has a non-empty ``.. short-description::`` body. + + Args: + content: The RST file content to search. + + Returns: + True if a non-empty short-description block exists, False otherwise. + """ + _s, _m, _b, inner, _i = _find_short_description_block(content) + return _short_description_inner_has_content(inner) + + +def get_short_description_body(content: str) -> str | None: + """ + Return the normalised inner body text of the first ``.. short-description::`` block. + + Returns ``None`` if the directive is missing or the body is empty. + """ + _s, _m, _b, inner, _i = _find_short_description_block(content) + if not _short_description_inner_has_content(inner): + return None + paragraphs: list[str] = [] + current: list[str] = [] + for line in inner.splitlines(): + stripped = line.strip() + if not stripped: + if current: + paragraphs.append(" ".join(current)) + current = [] + continue + current.append(stripped) + if current: + paragraphs.append(" ".join(current)) + return "\n\n".join(paragraphs) if paragraphs else None + + +def _format_short_description_inner(text: str, indent: str) -> str: + """ + Turn model output into RST directive body lines (indented paragraphs). + + Args: + text: The model-generated prose. + indent: The indentation string to use. + + Returns: + The formatted and indented inner text for the directive. + """ + chunks = [p.strip() for p in text.split("\n\n") if p.strip()] + lines_out: list[str] = [] + for i, para in enumerate(chunks): + for line in para.split("\n"): + s = line.strip() + if s: + lines_out.append(f"{indent}{s}\n") + if i < len(chunks) - 1: + lines_out.append(f"{indent}\n") + return "".join(lines_out) + + +def _find_insertion_point_after_title(content: str) -> int: + """ + Return the index in ``content`` immediately after the first document title block. + + A title block is a non-blank text line followed by a line of ``=``, ``-``, or ``~`` + underline characters (classic reStructuredText transition marker). + If no title is found, returns ``0``. + + Args: + content: The RST file content to search. + + Returns: + The byte index where the title block ends. + """ + lines = content.splitlines(keepends=True) + i = 0 + while i + 1 < len(lines): + title_line = lines[i] + underline_line = lines[i + 1] + title_stripped = title_line.strip() + ul_match = re.match(r"^([=\-~]+)\s*$", underline_line.rstrip("\n")) + if title_stripped and ul_match is not None: + ul = ul_match.group(1) + if len(ul) >= len(title_stripped): + pos = 0 + for j in range(i + 2): + pos += len(lines[j]) + return pos + i += 1 + return 0 + + +def inject_short_description_to_content(content: str, text: str) -> tuple[str, bool]: + """ + Insert or fill the first ``.. short-description::`` directive with the given prose. + + If the directive exists and already has body text, logs a warning and returns + the original content unchanged. If the directive exists with an empty body, + fills the body. If the directive is missing, inserts a new block after the + first detected document title (or at the start of the file if none). + + Returns: + Updated source and whether any change was made. + """ + start, marker_end, block_end, inner, indent = _find_short_description_block(content) + new_inner = _format_short_description_inner(text, indent) + + if start >= 0: + if _short_description_inner_has_content(inner): + logger.warning( + "Existing .. short-description:: body has content; skipping replacement", + ) + return content, False + # Normalise trailing whitespace: one blank line after the block + remainder = content[block_end:].lstrip() + new_content = content[:marker_end] + new_inner + "\n" + remainder + return new_content, True + + insert_at = _find_insertion_point_after_title(content) + # Normalise trailing whitespace: one blank line before and after the block + remainder = content[insert_at:].lstrip() + block = f"\n.. short-description::\n{new_inner}\n" + new_content = content[:insert_at] + block + remainder + return new_content, True + From 23b5e6e7f3ad0b23832136a7c56177f74438071e Mon Sep 17 00:00:00 2001 From: Keith Kirkwood Date: Mon, 13 Jul 2026 17:27:27 +0100 Subject: [PATCH 02/23] OPENR-174: Improved handling and use suggestions when possible --- .github/workflows/enhance.yml | 27 ++- tools/README.md | 38 ++++- tools/ensure_meta_tags.py | 306 ++++++++++++++++++++++++++++++++-- tools/rst_utils.py | 95 +++++++++-- 4 files changed, 425 insertions(+), 41 deletions(-) diff --git a/.github/workflows/enhance.yml b/.github/workflows/enhance.yml index bb2f74874ba..7aebbdc897a 100644 --- a/.github/workflows/enhance.yml +++ b/.github/workflows/enhance.yml @@ -39,22 +39,28 @@ jobs: run: pip install --no-warn-script-location pyyaml - name: Ensure product and distribution meta tags + id: ensure run: | - git fetch origin "${{ github.event.pull_request.base.sha }}" + BASE_SHA="${{ github.event.pull_request.base.sha }}" + git fetch origin "$BASE_SHA" mapfile -t changed_rst < <( git diff --name-only --diff-filter=ACMR \ - "${{ github.event.pull_request.base.sha }}...${{ github.event.pull_request.head.sha }}" \ + "${BASE_SHA}...${{ github.event.pull_request.head.sha }}" \ -- '*.rst' ) if [ "${#changed_rst[@]}" -eq 0 ]; then echo "No changed RST files in this pull request." + echo "suggestable=false" >> "$GITHUB_OUTPUT" + echo "fallback=false" >> "$GITHUB_OUTPUT" exit 0 fi python3 .trusted-base/tools/ensure_meta_tags.py \ --config .trusted-base/tools/meta_tags.yaml \ + --diff-base "$BASE_SHA" \ + --status-file "$GITHUB_OUTPUT" \ "${changed_rst[@]}" echo "Working tree after ensure_meta_tags:" @@ -62,9 +68,20 @@ jobs: git diff -- "${changed_rst[@]}" - name: Suggest meta tag changes + if: steps.ensure.outputs.suggestable == 'true' uses: parkerbxyz/suggest-changes@v3 with: - comment: >- - Missing product/distribution meta tags were added using values from - config. Please review and commit the suggestions. + comment: ${{ steps.ensure.outputs.comment }} event: COMMENT + + - name: Comment fallback meta tag instructions + if: >- + steps.ensure.outputs.fallback == 'true' + && steps.ensure.outputs.suggestable != 'true' + env: + GH_TOKEN: ${{ github.token }} + REVIEW_COMMENT: ${{ steps.ensure.outputs.comment }} + run: | + gh pr review "${{ github.event.pull_request.number }}" \ + --comment \ + --body "$REVIEW_COMMENT" diff --git a/tools/README.md b/tools/README.md index 51f1d5c6fca..478e5a3c6e8 100644 --- a/tools/README.md +++ b/tools/README.md @@ -29,7 +29,7 @@ The `meta` map lists every field the script checks. Add a new key to extend cove Low-level utilities for locating and editing Sphinx directives in RST source: - **`get_meta_names_from_content`** — field names already present in the first `.. meta::` block -- **`inject_metadata_to_content`** — append missing `:name: value` lines to an existing block, or prepend a new `.. meta::` block; never overwrites existing fields +- **`inject_metadata_to_content`** — append missing `:name: value` lines to an existing block, or insert a new `.. meta::` block (`after_heading` or `at_top`); never overwrites existing fields The module also contains helpers for `.. short-description::` directives for future use. @@ -54,6 +54,8 @@ python3 tools/ensure_meta_tags.py source/Topic/A.rst source/Topic/B.rst Options: - `--config PATH` — YAML config file (default: `tools/meta_tags.yaml`) +- `--diff-base SHA` — PR base commit; only write edits that overlap the PR diff (CI) +- `--status-file PATH` — write `suggestable=`, `fallback=`, and the review comment for CI - `-v` / `--verbose` — enable debug logging ### Example @@ -67,16 +69,16 @@ My Article Some content. ``` -After: +After a local run (new `.. meta::` after the first heading by default): ```rst +My Article +========== + .. meta:: :product: {PRODUCT} :distribution: {DISTRO} -My Article -========== - Some content. ``` @@ -88,9 +90,29 @@ The workflow [`.github/workflows/enhance.yml`](../.github/workflows/enhance.yml) 1. Checks out the PR’s `.rst` files as untrusted data 2. Checks out the base branch into `.trusted-base/` for the script and config -3. Runs the trusted `ensure_meta_tags.py` against changed RST files -4. Posts any resulting edits as review suggestions via [`parkerbxyz/suggest-changes`](https://github.com/marketplace/actions/suggest-changes-action) +3. Runs the trusted `ensure_meta_tags.py` with `--diff-base` against changed RST files +4. Posts inline suggestions and/or a review comment + +Priority: **inline suggestions wherever GitHub allows them**. Copy-paste in the review body is only a fallback. + +### When inline suggestions appear + +GitHub only allows review suggestions on [lines already in the pull request diff](https://github.com/marketplace/actions/suggest-changes-action). The script compares each needed edit to that diff: + +| Situation | What happens | +|-----------|----------------| +| Missing fields; existing `.. meta::` overlaps the PR diff | Write append to the working tree → inline “Commit suggestion” via [`suggest-changes`](https://github.com/marketplace/actions/suggest-changes-action). Review text asks the submitter to accept it and why. | +| No `.. meta::`; first-heading region overlaps the PR diff | Insert after the first heading → inline suggestion + same review text. | +| No `.. meta::`; top of file overlaps the PR diff | Insert at top of file → inline suggestion + same review text. | +| Needed edit does **not** overlap the PR diff (new block or out-of-diff append) | **Do not** write an unsuggestable hunk. Review / comment only, with a copy-paste `.. meta::` block to place at the **top of the file**. | +| All configured fields already present | No action | +| No changed `.rst` files in the PR | Workflow exits early | + +Mixed PRs are supported: suggestable files get working-tree edits + inline suggestions; fallback-only files appear only in the review body. If every file is fallback-only, a `COMMENT` review is posted without `suggest-changes`. + +Typical examples: -Contributors can review and accept the suggested meta tags before merging. No GitHub App or repository secrets are required. +- PR edits near an existing `.. meta::` → inline suggestions for missing `product` / `distribution`. +- PR only changes a mid-file paragraph and the file has no meta → out of diff → copy-paste block at top of file in the review body. The workflow uses [`pull_request_target`](https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request_target) so the default `GITHUB_TOKEN` can post review suggestions on fork PRs. To avoid running untrusted code with elevated permissions, only the base-branch copy of `ensure_meta_tags.py` and `meta_tags.yaml` is executed; PR content is treated as data to read and update. See [Mitigating the risks of untrusted code checkout](https://docs.github.com/en/actions/reference/security/secure-use#mitigating-the-risks-of-untrusted-code-checkout). diff --git a/tools/ensure_meta_tags.py b/tools/ensure_meta_tags.py index e9ee530785c..571bfaf2986 100644 --- a/tools/ensure_meta_tags.py +++ b/tools/ensure_meta_tags.py @@ -4,12 +4,18 @@ Missing fields are injected with values from ``meta_tags.yaml``. Existing fields are never overwritten. + +When ``--diff-base`` is set, edits are only written to disk when they overlap +the pull request diff (so GitHub can offer inline suggestions). Otherwise the +review comment carries a top-of-file copy-paste fallback. """ from __future__ import annotations import argparse import logging +import re +import subprocess import sys from pathlib import Path @@ -20,12 +26,19 @@ if str(_TOOLS_DIR) not in sys.path: sys.path.insert(0, str(_TOOLS_DIR)) -from rst_utils import get_meta_names_from_content, inject_metadata_to_content +from rst_utils import ( + first_heading_line_span, + get_meta_names_from_content, + has_meta_block, + inject_metadata_to_content, + meta_block_line_span, +) logger = logging.getLogger(__name__) DEFAULT_CONFIG_PATH = _TOOLS_DIR / "meta_tags.yaml" RST_EXTENSION = ".rst" +_HUNK_HEADER = re.compile(r"^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@") def load_meta_config(config_path: Path) -> dict[str, str]: @@ -75,32 +88,218 @@ def load_meta_config(config_path: Path) -> dict[str, str]: return validated +def format_meta_block(meta_config: dict[str, str], fields: list[str]) -> str: + """Return an RST ``.. meta::`` block for the given fields.""" + lines = [".. meta::"] + for field in fields: + lines.append(f" :{field}: {meta_config[field]}") + lines.append("") + return "\n".join(lines) + + def _missing_meta_fields(content: str, meta_config: dict[str, str]) -> list[str]: present = get_meta_names_from_content(content) return [field for field in meta_config if field not in present] -def ensure_meta_tags_in_file(path: Path, meta_config: dict[str, str]) -> bool: +def parse_diff_new_side_lines(diff_text: str) -> set[int]: + """ + Parse a unified diff and return 1-based line numbers on the new (``+``) side. + + Both added and context lines within hunks are included so nearby suggestion + anchors count as overlapping the pull request diff. + + File headers (``---`` / ``+++``) are only skipped outside hunks. Inside a + hunk those prefixes are ordinary ``-`` / ``+`` lines (e.g. RST table rows of + ``+`` characters), and must advance ``new_line`` accordingly. + """ + lines: set[int] = set() + new_line = 0 + in_hunk = False + for line in diff_text.splitlines(): + match = _HUNK_HEADER.match(line) + if match: + in_hunk = True + new_line = int(match.group(1)) + continue + if not in_hunk and (line.startswith("---") or line.startswith("+++")): + continue + if line.startswith("\\"): + continue + if line.startswith("+"): + lines.add(new_line) + new_line += 1 + elif line.startswith("-"): + continue + elif line.startswith(" ") or line == "": + if new_line > 0: + lines.add(new_line) + new_line += 1 + elif line.startswith("diff "): + # Next file in a multi-file diff; subsequent ---/+++ are headers again. + in_hunk = False + new_line = 0 + return lines + + +def pr_diff_lines_for_file(diff_base: str, path: Path) -> set[int]: + """Return PR-diff line numbers for ``path`` on the head side vs ``diff_base``.""" + result = subprocess.run( + ["git", "diff", "-U3", f"{diff_base}...HEAD", "--", str(path)], + check=False, + capture_output=True, + text=True, + ) + if result.returncode not in (0, 1): + logger.warning( + "git diff failed for %s (exit %s): %s", + path, + result.returncode, + result.stderr.strip(), + ) + return set() + return parse_diff_new_side_lines(result.stdout) + + +def _span_overlaps(span: tuple[int, int] | None, pr_lines: set[int]) -> bool: + if span is None or not pr_lines: + return False + start, end = span + return any(line in pr_lines for line in range(start, end + 1)) + + +def choose_suggestable_placement( + content: str, + pr_lines: set[int], +) -> str | None: + """ + Return how to apply an in-diff edit, or ``None`` if only fallback is possible. + + Returns: + ``append``, ``after_heading``, ``at_top``, or ``None``. """ - Add missing configured meta fields to one RST file. + if has_meta_block(content): + span = meta_block_line_span(content) + if span is None: + return None + start, end = span + # Include the line after the block where fields would be appended. + if _span_overlaps((start, end + 1), pr_lines): + return "append" + return None + + heading = first_heading_line_span(content) + if heading is not None: + start, end = heading + if _span_overlaps((start, end + 1), pr_lines): + return "after_heading" + + if _span_overlaps((1, 1), pr_lines): + return "at_top" + + return None + + +def ensure_meta_tags_in_file( + path: Path, + meta_config: dict[str, str], + *, + pr_lines: set[int] | None = None, +) -> dict[str, object] | None: + """ + Add missing configured meta fields, preferring pull-request-diff overlap. + + When ``pr_lines`` is provided, the file is only modified when the edit can + land on lines already in the PR diff (inline-suggestion path). Otherwise the + result is a fallback entry without writing the file. + + When ``pr_lines`` is ``None`` (local CLI use), behaviour is unconditional + write using append / after-heading placement. Returns: - True if the file was updated, False otherwise. + A result dict when action is needed, otherwise ``None``. + Keys include ``path``, ``fields``, ``mode`` (``suggestable`` or + ``fallback``), ``placement``, and ``snippet``. """ content = path.read_text(encoding="utf-8") missing = _missing_meta_fields(content, meta_config) if not missing: logger.info("%s: all configured meta fields present", path) - return False + return None metadata = {field: meta_config[field] for field in missing} - new_content, changed = inject_metadata_to_content(content, metadata) + snippet = format_meta_block(meta_config, missing) + path_str = str(path).replace("\\", "/") + + if pr_lines is None: + if has_meta_block(content): + placement = "append" + elif first_heading_line_span(content) is not None: + placement = "after_heading" + else: + placement = "at_top" + + if placement == "append": + new_content, changed = inject_metadata_to_content(content, metadata) + else: + new_content, changed = inject_metadata_to_content( + content, + metadata, + new_block_placement=placement, + ) + + if not changed: + return None + path.write_text(new_content, encoding="utf-8") + logger.info("%s: added meta fields %s (%s)", path, ", ".join(missing), placement) + return { + "path": path_str, + "fields": missing, + "mode": "suggestable", + "placement": placement, + "snippet": snippet, + } + + placement = choose_suggestable_placement(content, pr_lines) + if placement is None: + logger.info( + "%s: missing %s but edit is outside the PR diff; fallback review only", + path, + ", ".join(missing), + ) + return { + "path": path_str, + "fields": missing, + "mode": "fallback", + "placement": "at_top", + "snippet": snippet, + } + + if placement == "append": + new_content, changed = inject_metadata_to_content(content, metadata) + else: + new_content, changed = inject_metadata_to_content( + content, + metadata, + new_block_placement=placement, + ) if not changed: - return False + return None path.write_text(new_content, encoding="utf-8") - logger.info("%s: added meta fields %s", path, ", ".join(missing)) - return True + logger.info( + "%s: added meta fields %s via %s (inline suggestion)", + path, + ", ".join(missing), + placement, + ) + return { + "path": path_str, + "fields": missing, + "mode": "suggestable", + "placement": placement, + "snippet": snippet, + } def _collect_rst_paths(paths: list[str]) -> list[Path]: @@ -117,6 +316,42 @@ def _collect_rst_paths(paths: list[str]) -> list[Path]: return rst_paths +def build_review_comment(results: list[dict[str, object]]) -> str: + """Build the pull request review / comment body for suggestable and fallback results.""" + suggestable = [r for r in results if r["mode"] == "suggestable"] + fallback = [r for r in results if r["mode"] == "fallback"] + + lines = [ + "This pull request is missing configured `product` / `distribution` " + "meta tags (defaults from `tools/meta_tags.yaml`).", + "", + ] + + if suggestable: + lines.append( + "Please **review and commit the inline suggestions**. They add the " + "missing fields in place so the documentation metadata stays " + "complete." + ) + lines.append("") + + if fallback: + lines.append( + "GitHub can only attach suggestions to lines already in the pull " + "request diff, so the following files could not get an inline " + "suggestion. Please add this block at the **top of each file**:" + ) + lines.append("") + for result in fallback: + lines.append(f"**`{result['path']}`**") + lines.append("```rst") + lines.append(str(result["snippet"]).rstrip()) + lines.append("```") + lines.append("") + + return "\n".join(lines).rstrip() + "\n" + + def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser( description=( @@ -135,6 +370,19 @@ def main(argv: list[str] | None = None) -> int: default=DEFAULT_CONFIG_PATH, help=f"YAML config file (default: {DEFAULT_CONFIG_PATH})", ) + parser.add_argument( + "--diff-base", + help=( + "Git commit SHA for the pull request base. When set, only writes " + "edits that overlap the PR diff; other files become review-comment " + "fallbacks." + ), + ) + parser.add_argument( + "--status-file", + type=Path, + help="Write suggestable=true|false, fallback=true|false, and the review comment for CI", + ) parser.add_argument( "-v", "--verbose", @@ -150,16 +398,42 @@ def main(argv: list[str] | None = None) -> int: meta_config = load_meta_config(args.config) rst_paths = _collect_rst_paths(args.paths) + results: list[dict[str, object]] = [] + if not rst_paths: logger.info("No RST files to process") - return 0 - - updated = 0 - for path in rst_paths: - if ensure_meta_tags_in_file(path, meta_config): - updated += 1 + else: + for path in rst_paths: + pr_lines: set[int] | None = None + if args.diff_base: + pr_lines = pr_diff_lines_for_file(args.diff_base, path) + result = ensure_meta_tags_in_file(path, meta_config, pr_lines=pr_lines) + if result is not None: + results.append(result) + + suggestable_count = sum(1 for r in results if r["mode"] == "suggestable") + fallback_count = sum(1 for r in results if r["mode"] == "fallback") + logger.info( + "Processed %d file(s): %d suggestable write(s), %d fallback(s)", + len(rst_paths), + suggestable_count, + fallback_count, + ) + + if args.status_file is not None: + has_suggestable = any(r["mode"] == "suggestable" for r in results) + has_fallback = any(r["mode"] == "fallback" for r in results) + with args.status_file.open("a", encoding="utf-8") as f: + f.write(f"suggestable={'true' if has_suggestable else 'false'}\n") + f.write(f"fallback={'true' if has_fallback else 'false'}\n") + if results: + review_body = build_review_comment(results) + f.write("comment< set[str]: return _extract_meta_names_from_block(inner) +def has_meta_block(content: str) -> bool: + """ + Return whether the document contains a ``.. meta::`` directive. + + Args: + content: The RST file content to search. + + Returns: + True if a ``.. meta::`` block exists, False otherwise. + """ + start, _marker_end, _block_end, _inner, _indent = _find_meta_block(content) + return start >= 0 + + def _normalise_meta_field_value(value: str) -> str: """ Collapse whitespace so the meta field body stays a single logical line. @@ -106,18 +120,30 @@ def _normalise_meta_field_value(value: str) -> str: return " ".join(value.split()) # Docutils treats the field body as one string; keep it one physical line -def inject_metadata_to_content(content: str, metadata: dict[str, str]) -> tuple[str, bool]: +def inject_metadata_to_content( + content: str, + metadata: dict[str, str], + *, + new_block_placement: str = "after_heading", +) -> tuple[str, bool]: """ Insert or append ``.. meta::`` field entries for the given name/value pairs. - Appends to an existing ``.. meta::`` block when present; otherwise prepends - a new block at the start of the document (leading whitespace is stripped so - the directive is the first element). Skips keys that already appear in the - block. + Appends to an existing ``.. meta::`` block when present. Otherwise inserts a + new block according to ``new_block_placement``: + + - ``after_heading`` — immediately after the first document heading, with a + blank line before and after (or at the start if no heading is found) + - ``at_top`` — at the start of the document + + Skips keys that already appear in the block. Returns: Updated source and whether any change was made. """ + if new_block_placement not in {"after_heading", "at_top"}: + raise ValueError(f"Unknown new_block_placement: {new_block_placement!r}") + start, marker_end, block_end, inner, indent = _find_meta_block(content) names = _extract_meta_names_from_block(inner) # Snapshot before we add keys from this same batch additions: list[str] = [] @@ -143,14 +169,57 @@ def inject_metadata_to_content(content: str, metadata: dict[str, str]) -> tuple[ # Normalise trailing whitespace: one blank line after the block remainder = content[block_end:].lstrip() new_content = content[:marker_end] + new_inner + "\n" + remainder - else: - # No ``.. meta::`` yet: insert at document start; strip leading whitespace so the block is truly first + elif new_block_placement == "at_top": remainder = content.lstrip() - new_content = ".. meta::\n" + "".join(additions) + "\n" + remainder # Blank line after block separates it from the body + new_content = ".. meta::\n" + "".join(additions) + "\n" + remainder + else: + # after_heading: blank line before and after the meta block + insert_at = _find_insertion_point_after_title(content) + remainder = content[insert_at:].lstrip() + block = f"\n.. meta::\n{''.join(additions)}\n" + new_content = content[:insert_at] + block + remainder return new_content, True +def _byte_offset_to_line_number(content: str, offset: int) -> int: + """Return the 1-based line number containing ``offset`` (or the next line at EOF).""" + if offset <= 0: + return 1 + if offset >= len(content): + return content.count("\n") + (0 if content.endswith("\n") else 1) + return content.count("\n", 0, offset) + 1 + + +def meta_block_line_span(content: str) -> tuple[int, int] | None: + """ + Return the inclusive 1-based line span of the first ``.. meta::`` block. + + Returns ``None`` if no meta block exists. + """ + start, _marker_end, block_end, _inner, _indent = _find_meta_block(content) + if start < 0: + return None + start_line = _byte_offset_to_line_number(content, start) + end_offset = block_end - 1 if block_end > start else start + end_line = _byte_offset_to_line_number(content, end_offset) + return start_line, end_line + + +def first_heading_line_span(content: str) -> tuple[int, int] | None: + """ + Return the inclusive 1-based line span of the first title (text + underline). + + Returns ``None`` if no heading is found. + """ + insert_at = _find_insertion_point_after_title(content) + if insert_at <= 0: + return None + end_line = _byte_offset_to_line_number(content, insert_at - 1) + start_line = max(1, end_line - 1) + return start_line, end_line + + def _find_short_description_block(content: str) -> tuple[int, int, int, str, str]: """ Locate the first ``.. short-description::`` directive in RST source. @@ -276,8 +345,9 @@ def _find_insertion_point_after_title(content: str) -> int: """ Return the index in ``content`` immediately after the first document title block. - A title block is a non-blank text line followed by a line of ``=``, ``-``, or ``~`` - underline characters (classic reStructuredText transition marker). + A title block is a non-blank text line followed by a line of repeating + underline characters (reStructuredText section markers such as ``=``, + ``-``, ``~``, and other Docutils-adornment characters). If no title is found, returns ``0``. Args: @@ -292,10 +362,11 @@ def _find_insertion_point_after_title(content: str) -> int: title_line = lines[i] underline_line = lines[i + 1] title_stripped = title_line.strip() - ul_match = re.match(r"^([=\-~]+)\s*$", underline_line.rstrip("\n")) + # Docutils section adornment characters + ul_match = re.match(r'^([!"#$%&\'()*+,\-./:;<=>?@\[\\\]^_`{|}~]+)\s*$', underline_line.rstrip("\n")) if title_stripped and ul_match is not None: ul = ul_match.group(1) - if len(ul) >= len(title_stripped): + if len(set(ul)) == 1 and len(ul) >= len(title_stripped): pos = 0 for j in range(i + 2): pos += len(lines[j]) From 10ce9bfdb711371b82f288df16653696bf417f16 Mon Sep 17 00:00:00 2001 From: Keith Kirkwood Date: Tue, 14 Jul 2026 14:59:50 +0100 Subject: [PATCH 03/23] OPENR-174: Review comment obsolete status, and only insert new meta block at top of the file --- .github/workflows/enhance.yml | 59 ++++++++++++++++++- tools/README.md | 21 ++++--- tools/ensure_meta_tags.py | 107 +++++++++++----------------------- tools/rst_utils.py | 33 +---------- 4 files changed, 107 insertions(+), 113 deletions(-) diff --git a/.github/workflows/enhance.yml b/.github/workflows/enhance.yml index 7aebbdc897a..fe94b7e02f6 100644 --- a/.github/workflows/enhance.yml +++ b/.github/workflows/enhance.yml @@ -44,6 +44,7 @@ jobs: BASE_SHA="${{ github.event.pull_request.base.sha }}" git fetch origin "$BASE_SHA" + # Collect changed RST files mapfile -t changed_rst < <( git diff --name-only --diff-filter=ACMR \ "${BASE_SHA}...${{ github.event.pull_request.head.sha }}" \ @@ -52,11 +53,13 @@ jobs: if [ "${#changed_rst[@]}" -eq 0 ]; then echo "No changed RST files in this pull request." + echo "meta_checked=false" >> "$GITHUB_OUTPUT" echo "suggestable=false" >> "$GITHUB_OUTPUT" echo "fallback=false" >> "$GITHUB_OUTPUT" exit 0 fi + # Check meta tags for changed RST files python3 .trusted-base/tools/ensure_meta_tags.py \ --config .trusted-base/tools/meta_tags.yaml \ --diff-base "$BASE_SHA" \ @@ -67,8 +70,59 @@ jobs: git status --short -- "${changed_rst[@]}" git diff -- "${changed_rst[@]}" + - name: Supersede stale meta-tag reviews + id: supersede + if: steps.ensure.outputs.meta_checked == 'true' + env: + GH_TOKEN: ${{ github.token }} + HAS_RESULTS: ${{ steps.ensure.outputs.has_results }} + run: | + set -euo pipefail + PR="${{ github.event.pull_request.number }}" + REPO="${{ github.repository }}" + MARKER="ros2-meta-tags-ensure" + + echo "Marking prior meta-tag reviews as outdated." + + # Find all previous PR reviews containing the meta-tag marker, + # and extract their node IDs + mapfile -t review_ids < <( + gh api "repos/${REPO}/pulls/${PR}/reviews" --paginate \ + --jq ".[] | select(.body != null and (.body | contains(\"${MARKER}\"))) | .node_id" + ) + + # Prepare a GraphQL mutation string to minimize comments (mark them as outdated) + minimize_query=' + mutation($subjectId: ID!) { + minimizeComment(input: { + subjectId: $subjectId + classifier: OUTDATED + }) { + minimizedComment { isMinimized } + } + } + ' + + # Run the GraphQL mutation for each eligible review + for node_id in "${review_ids[@]}"; do + [ -z "$node_id" ] && continue + gh api graphql \ + -f query="${minimize_query}" \ + -f subjectId="${node_id}" \ + || true + done + + if [ "$HAS_RESULTS" = "true" ]; then + echo "should_post=true" >> "$GITHUB_OUTPUT" + else + echo "All meta-tag issues resolved; not posting a new review." + echo "should_post=false" >> "$GITHUB_OUTPUT" + fi + - name: Suggest meta tag changes - if: steps.ensure.outputs.suggestable == 'true' + if: >- + steps.supersede.outputs.should_post == 'true' + && steps.ensure.outputs.suggestable == 'true' uses: parkerbxyz/suggest-changes@v3 with: comment: ${{ steps.ensure.outputs.comment }} @@ -76,7 +130,8 @@ jobs: - name: Comment fallback meta tag instructions if: >- - steps.ensure.outputs.fallback == 'true' + steps.supersede.outputs.should_post == 'true' + && steps.ensure.outputs.fallback == 'true' && steps.ensure.outputs.suggestable != 'true' env: GH_TOKEN: ${{ github.token }} diff --git a/tools/README.md b/tools/README.md index 478e5a3c6e8..81cdffbcb44 100644 --- a/tools/README.md +++ b/tools/README.md @@ -29,7 +29,7 @@ The `meta` map lists every field the script checks. Add a new key to extend cove Low-level utilities for locating and editing Sphinx directives in RST source: - **`get_meta_names_from_content`** — field names already present in the first `.. meta::` block -- **`inject_metadata_to_content`** — append missing `:name: value` lines to an existing block, or insert a new `.. meta::` block (`after_heading` or `at_top`); never overwrites existing fields +- **`inject_metadata_to_content`** — append missing `:name: value` lines to an existing block, or insert a new `.. meta::` block at the top of the file; never overwrites existing fields The module also contains helpers for `.. short-description::` directives for future use. @@ -55,7 +55,7 @@ Options: - `--config PATH` — YAML config file (default: `tools/meta_tags.yaml`) - `--diff-base SHA` — PR base commit; only write edits that overlap the PR diff (CI) -- `--status-file PATH` — write `suggestable=`, `fallback=`, and the review comment for CI +- `--status-file PATH` — write `meta_checked=`, `suggestable=`, `fallback=`, `has_results=`, and the review comment for CI - `-v` / `--verbose` — enable debug logging ### Example @@ -69,16 +69,16 @@ My Article Some content. ``` -After a local run (new `.. meta::` after the first heading by default): +After a local run (new `.. meta::` at the top of the file by default): ```rst -My Article -========== - .. meta:: :product: {PRODUCT} :distribution: {DISTRO} +My Article +========== + Some content. ``` @@ -102,7 +102,6 @@ GitHub only allows review suggestions on [lines already in the pull request diff | Situation | What happens | |-----------|----------------| | Missing fields; existing `.. meta::` overlaps the PR diff | Write append to the working tree → inline “Commit suggestion” via [`suggest-changes`](https://github.com/marketplace/actions/suggest-changes-action). Review text asks the submitter to accept it and why. | -| No `.. meta::`; first-heading region overlaps the PR diff | Insert after the first heading → inline suggestion + same review text. | | No `.. meta::`; top of file overlaps the PR diff | Insert at top of file → inline suggestion + same review text. | | Needed edit does **not** overlap the PR diff (new block or out-of-diff append) | **Do not** write an unsuggestable hunk. Review / comment only, with a copy-paste `.. meta::` block to place at the **top of the file**. | | All configured fields already present | No action | @@ -110,6 +109,14 @@ GitHub only allows review suggestions on [lines already in the pull request diff Mixed PRs are supported: suggestable files get working-tree edits + inline suggestions; fallback-only files appear only in the review body. If every file is fallback-only, a `COMMENT` review is posted without `suggest-changes`. +### Superseding outdated reviews + +Each bot review body includes a hidden marker (``). On every run that checks changed RST files: + +1. All prior stamped reviews on the pull request are minimized as **Outdated** via the GitHub API (no external state store). +2. A fresh review is posted only when work still remains. +3. When all issues are fixed, stale reviews are minimized and no new “all clear” comment is posted. + Typical examples: - PR edits near an existing `.. meta::` → inline suggestions for missing `product` / `distribution`. diff --git a/tools/ensure_meta_tags.py b/tools/ensure_meta_tags.py index 571bfaf2986..f405ad8b967 100644 --- a/tools/ensure_meta_tags.py +++ b/tools/ensure_meta_tags.py @@ -27,7 +27,6 @@ sys.path.insert(0, str(_TOOLS_DIR)) from rst_utils import ( - first_heading_line_span, get_meta_names_from_content, has_meta_block, inject_metadata_to_content, @@ -40,6 +39,9 @@ RST_EXTENSION = ".rst" _HUNK_HEADER = re.compile(r"^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@") +# Hidden marker in review bodies so CI can find and supersede prior bot reviews. +REVIEW_MARKER = "" + def load_meta_config(config_path: Path) -> dict[str, str]: """ @@ -168,36 +170,23 @@ def _span_overlaps(span: tuple[int, int] | None, pr_lines: set[int]) -> bool: return any(line in pr_lines for line in range(start, end + 1)) -def choose_suggestable_placement( - content: str, - pr_lines: set[int], -) -> str | None: +def can_suggest_inline(content: str, pr_lines: set[int]) -> bool: """ - Return how to apply an in-diff edit, or ``None`` if only fallback is possible. + Return whether a meta-tag edit can be anchored to the pull request diff. - Returns: - ``append``, ``after_heading``, ``at_top``, or ``None``. + Existing ``.. meta::`` blocks are suggestable when they overlap the diff + (wherever they already sit). New blocks are only suggestable when line 1 is + in the diff, since inserts always go at the top of the file. """ if has_meta_block(content): span = meta_block_line_span(content) if span is None: - return None + return False start, end = span # Include the line after the block where fields would be appended. - if _span_overlaps((start, end + 1), pr_lines): - return "append" - return None - - heading = first_heading_line_span(content) - if heading is not None: - start, end = heading - if _span_overlaps((start, end + 1), pr_lines): - return "after_heading" + return _span_overlaps((start, end + 1), pr_lines) - if _span_overlaps((1, 1), pr_lines): - return "at_top" - - return None + return _span_overlaps((1, 1), pr_lines) def ensure_meta_tags_in_file( @@ -213,13 +202,13 @@ def ensure_meta_tags_in_file( land on lines already in the PR diff (inline-suggestion path). Otherwise the result is a fallback entry without writing the file. - When ``pr_lines`` is ``None`` (local CLI use), behaviour is unconditional - write using append / after-heading placement. + When ``pr_lines`` is ``None`` (local CLI use), behaviour is an unconditional + write: append to an existing block, or insert a new block at the top. Returns: A result dict when action is needed, otherwise ``None``. Keys include ``path``, ``fields``, ``mode`` (``suggestable`` or - ``fallback``), ``placement``, and ``snippet``. + ``fallback``), and ``snippet``. """ content = path.read_text(encoding="utf-8") missing = _missing_meta_fields(content, meta_config) @@ -231,37 +220,7 @@ def ensure_meta_tags_in_file( snippet = format_meta_block(meta_config, missing) path_str = str(path).replace("\\", "/") - if pr_lines is None: - if has_meta_block(content): - placement = "append" - elif first_heading_line_span(content) is not None: - placement = "after_heading" - else: - placement = "at_top" - - if placement == "append": - new_content, changed = inject_metadata_to_content(content, metadata) - else: - new_content, changed = inject_metadata_to_content( - content, - metadata, - new_block_placement=placement, - ) - - if not changed: - return None - path.write_text(new_content, encoding="utf-8") - logger.info("%s: added meta fields %s (%s)", path, ", ".join(missing), placement) - return { - "path": path_str, - "fields": missing, - "mode": "suggestable", - "placement": placement, - "snippet": snippet, - } - - placement = choose_suggestable_placement(content, pr_lines) - if placement is None: + if pr_lines is not None and not can_suggest_inline(content, pr_lines): logger.info( "%s: missing %s but edit is outside the PR diff; fallback review only", path, @@ -271,33 +230,26 @@ def ensure_meta_tags_in_file( "path": path_str, "fields": missing, "mode": "fallback", - "placement": "at_top", "snippet": snippet, } - if placement == "append": - new_content, changed = inject_metadata_to_content(content, metadata) - else: - new_content, changed = inject_metadata_to_content( - content, - metadata, - new_block_placement=placement, - ) + new_content, changed = inject_metadata_to_content(content, metadata) if not changed: return None path.write_text(new_content, encoding="utf-8") - logger.info( - "%s: added meta fields %s via %s (inline suggestion)", - path, - ", ".join(missing), - placement, - ) + if pr_lines is None: + logger.info("%s: added meta fields %s", path, ", ".join(missing)) + else: + logger.info( + "%s: added meta fields %s (inline suggestion)", + path, + ", ".join(missing), + ) return { "path": path_str, "fields": missing, "mode": "suggestable", - "placement": placement, "snippet": snippet, } @@ -316,6 +268,11 @@ def _collect_rst_paths(paths: list[str]) -> list[Path]: return rst_paths +def stamp_review_comment(body: str) -> str: + """Append a hidden marker so CI can find and supersede this review later.""" + return body.rstrip() + f"\n\n{REVIEW_MARKER}\n" + + def build_review_comment(results: list[dict[str, object]]) -> str: """Build the pull request review / comment body for suggestable and fallback results.""" suggestable = [r for r in results if r["mode"] == "suggestable"] @@ -349,7 +306,8 @@ def build_review_comment(results: list[dict[str, object]]) -> str: lines.append("```") lines.append("") - return "\n".join(lines).rstrip() + "\n" + body = "\n".join(lines).rstrip() + "\n" + return stamp_review_comment(body) def main(argv: list[str] | None = None) -> int: @@ -423,9 +381,12 @@ def main(argv: list[str] | None = None) -> int: if args.status_file is not None: has_suggestable = any(r["mode"] == "suggestable" for r in results) has_fallback = any(r["mode"] == "fallback" for r in results) + has_results = bool(results) with args.status_file.open("a", encoding="utf-8") as f: + f.write("meta_checked=true\n") f.write(f"suggestable={'true' if has_suggestable else 'false'}\n") f.write(f"fallback={'true' if has_fallback else 'false'}\n") + f.write(f"has_results={'true' if has_results else 'false'}\n") if results: review_body = build_review_comment(results) f.write("comment< str: def inject_metadata_to_content( content: str, metadata: dict[str, str], - *, - new_block_placement: str = "after_heading", ) -> tuple[str, bool]: """ Insert or append ``.. meta::`` field entries for the given name/value pairs. Appends to an existing ``.. meta::`` block when present. Otherwise inserts a - new block according to ``new_block_placement``: - - - ``after_heading`` — immediately after the first document heading, with a - blank line before and after (or at the start if no heading is found) - - ``at_top`` — at the start of the document + new block at the start of the document. Skips keys that already appear in the block. Returns: Updated source and whether any change was made. """ - if new_block_placement not in {"after_heading", "at_top"}: - raise ValueError(f"Unknown new_block_placement: {new_block_placement!r}") - start, marker_end, block_end, inner, indent = _find_meta_block(content) names = _extract_meta_names_from_block(inner) # Snapshot before we add keys from this same batch additions: list[str] = [] @@ -169,15 +160,9 @@ def inject_metadata_to_content( # Normalise trailing whitespace: one blank line after the block remainder = content[block_end:].lstrip() new_content = content[:marker_end] + new_inner + "\n" + remainder - elif new_block_placement == "at_top": + else: remainder = content.lstrip() new_content = ".. meta::\n" + "".join(additions) + "\n" + remainder - else: - # after_heading: blank line before and after the meta block - insert_at = _find_insertion_point_after_title(content) - remainder = content[insert_at:].lstrip() - block = f"\n.. meta::\n{''.join(additions)}\n" - new_content = content[:insert_at] + block + remainder return new_content, True @@ -206,20 +191,6 @@ def meta_block_line_span(content: str) -> tuple[int, int] | None: return start_line, end_line -def first_heading_line_span(content: str) -> tuple[int, int] | None: - """ - Return the inclusive 1-based line span of the first title (text + underline). - - Returns ``None`` if no heading is found. - """ - insert_at = _find_insertion_point_after_title(content) - if insert_at <= 0: - return None - end_line = _byte_offset_to_line_number(content, insert_at - 1) - start_line = max(1, end_line - 1) - return start_line, end_line - - def _find_short_description_block(content: str) -> tuple[int, int, int, str, str]: """ Locate the first ``.. short-description::`` directive in RST source. From 551ea52a8442add30620956bedc824a07b2e77b0 Mon Sep 17 00:00:00 2001 From: Keith Kirkwood Date: Fri, 24 Jul 2026 14:13:06 +0100 Subject: [PATCH 04/23] OPENR-174: Add warning line to file for missing meta, and soft fail step in workflow --- .github/workflows/enhance.yml | 1 + tools/README.md | 28 ++++- tools/ensure_meta_tags.py | 201 +++++++++++++++++++++++++++++++++- 3 files changed, 222 insertions(+), 8 deletions(-) diff --git a/.github/workflows/enhance.yml b/.github/workflows/enhance.yml index fe94b7e02f6..ca81ba3518f 100644 --- a/.github/workflows/enhance.yml +++ b/.github/workflows/enhance.yml @@ -40,6 +40,7 @@ jobs: - name: Ensure product and distribution meta tags id: ensure + continue-on-error: true run: | BASE_SHA="${{ github.event.pull_request.base.sha }}" git fetch origin "$BASE_SHA" diff --git a/tools/README.md b/tools/README.md index 81cdffbcb44..a2225fd44c9 100644 --- a/tools/README.md +++ b/tools/README.md @@ -55,9 +55,11 @@ Options: - `--config PATH` — YAML config file (default: `tools/meta_tags.yaml`) - `--diff-base SHA` — PR base commit; only write edits that overlap the PR diff (CI) -- `--status-file PATH` — write `meta_checked=`, `suggestable=`, `fallback=`, `has_results=`, and the review comment for CI +- `--status-file PATH` — write `meta_checked=`, `suggestable=`, `fallback=`, `has_results=`, and the review comment for CI; when issues remain, the script also emits GitHub Actions warning annotations and exits with code `1` (the workflow soft-fails that step without failing the job) - `-v` / `--verbose` — enable debug logging +Local runs without `--status-file` still exit `0` after applying edits, even when fields were missing. + ### Example Before: @@ -91,10 +93,32 @@ The workflow [`.github/workflows/enhance.yml`](../.github/workflows/enhance.yml) 1. Checks out the PR’s `.rst` files as untrusted data 2. Checks out the base branch into `.trusted-base/` for the script and config 3. Runs the trusted `ensure_meta_tags.py` with `--diff-base` against changed RST files -4. Posts inline suggestions and/or a review comment +4. Emits per-file warning annotations and soft-fails the ensure step when metadata is still missing +5. Posts inline suggestions and/or a review comment Priority: **inline suggestions wherever GitHub allows them**. Copy-paste in the review body is only a fallback. +The ensure step uses `continue-on-error: true`, so the Enhance job still **succeeds** and does not block merge. Contributors see a warning on that step in Actions when work remains. + +### Warnings and annotations + +When metadata is missing, the script prints one [workflow warning](https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/workflow-commands-for-github-actions#setting-a-warning-message) per affected file: + +```text +::warning file=path/to/file.rst,line=N::Missing meta fields: product, distribution +``` + +`N` is the start line of an existing `.. meta::` block, or `1` when a new block would be inserted at the top of the file. + +### Contributor experience + +| Situation | Actions ensure step | File annotations | Pull request review | +|-----------|---------------------|------------------|---------------------| +| All configured fields present | Green | None | None (stale bot reviews cleared) | +| Missing fields; edit overlaps PR diff | Soft warning | Yes | Inline “Commit suggestion” | +| Missing fields; edit outside PR diff | Soft warning | Yes | Copy-paste `.. meta::` block | +| Mixed (some suggestable, some fallback) | Soft warning | Yes (all affected files) | Suggestions plus fallback blocks | + ### When inline suggestions appear GitHub only allows review suggestions on [lines already in the pull request diff](https://github.com/marketplace/actions/suggest-changes-action). The script compares each needed edit to that diff: diff --git a/tools/ensure_meta_tags.py b/tools/ensure_meta_tags.py index f405ad8b967..f26342159fa 100644 --- a/tools/ensure_meta_tags.py +++ b/tools/ensure_meta_tags.py @@ -8,6 +8,10 @@ When ``--diff-base`` is set, edits are only written to disk when they overlap the pull request diff (so GitHub can offer inline suggestions). Otherwise the review comment carries a top-of-file copy-paste fallback. + +In CI (``--status-file``), emits GitHub Actions ``::warning`` annotations per +affected file and exits with code ``1`` when issues remain so the workflow can +soft-fail the step without failing the job. """ from __future__ import annotations @@ -47,8 +51,11 @@ def load_meta_config(config_path: Path) -> dict[str, str]: """ Load the ``meta`` mapping from a YAML config file. + Args: + config_path: Path to the YAML configuration file. + Returns: - Field name to value mapping. + A mapping from each configured meta field name to its default value. Raises: SystemExit: If the file is missing, invalid, or has no usable ``meta`` map. @@ -91,7 +98,19 @@ def load_meta_config(config_path: Path) -> dict[str, str]: def format_meta_block(meta_config: dict[str, str], fields: list[str]) -> str: - """Return an RST ``.. meta::`` block for the given fields.""" + """ + Build an RST ``.. meta::`` block for the requested fields. + + Args: + meta_config: Mapping from meta field names to their default values. + fields: Field names to include in the block, in output order. + + Returns: + A formatted ``.. meta::`` directive ending with a blank line. + + Raises: + KeyError: If a requested field is absent from ``meta_config``. + """ lines = [".. meta::"] for field in fields: lines.append(f" :{field}: {meta_config[field]}") @@ -100,6 +119,16 @@ def format_meta_block(meta_config: dict[str, str], fields: list[str]) -> str: def _missing_meta_fields(content: str, meta_config: dict[str, str]) -> list[str]: + """ + Find configured meta fields that are absent from RST content. + + Args: + content: RST source to inspect. + meta_config: Mapping of meta fields that should be present. + + Returns: + Missing field names in configuration order. + """ present = get_meta_names_from_content(content) return [field for field in meta_config if field not in present] @@ -114,6 +143,12 @@ def parse_diff_new_side_lines(diff_text: str) -> set[int]: File headers (``---`` / ``+++``) are only skipped outside hunks. Inside a hunk those prefixes are ordinary ``-`` / ``+`` lines (e.g. RST table rows of ``+`` characters), and must advance ``new_line`` accordingly. + + Args: + diff_text: Unified diff text to parse. + + Returns: + One-based line numbers represented on the new side of diff hunks. """ lines: set[int] = set() new_line = 0 @@ -145,7 +180,16 @@ def parse_diff_new_side_lines(diff_text: str) -> set[int]: def pr_diff_lines_for_file(diff_base: str, path: Path) -> set[int]: - """Return PR-diff line numbers for ``path`` on the head side vs ``diff_base``.""" + """ + Find pull-request diff lines for a file on the head side. + + Args: + diff_base: Base commit SHA used for the three-dot comparison. + path: Repository-relative path to inspect. + + Returns: + One-based new-side line numbers, or an empty set if ``git diff`` fails. + """ result = subprocess.run( ["git", "diff", "-U3", f"{diff_base}...HEAD", "--", str(path)], check=False, @@ -164,12 +208,91 @@ def pr_diff_lines_for_file(diff_base: str, path: Path) -> set[int]: def _span_overlaps(span: tuple[int, int] | None, pr_lines: set[int]) -> bool: + """ + Test whether an inclusive line span overlaps pull-request lines. + + Args: + span: Inclusive one-based start and end lines, or ``None``. + pr_lines: One-based line numbers represented by the pull-request diff. + + Returns: + ``True`` when at least one line overlaps; otherwise ``False``. + """ if span is None or not pr_lines: return False start, end = span return any(line in pr_lines for line in range(start, end + 1)) +def _warning_line_for_content(content: str) -> int: + """ + Select a line for a GitHub annotation on RST content. + + Args: + content: RST source to inspect. + + Returns: + The first line of an existing meta block, or line 1 if none exists. + """ + span = meta_block_line_span(content) + if span is not None: + return span[0] + return 1 + + +def _escape_workflow_command_message(message: str) -> str: + """ + Escape a message for use in a GitHub Actions workflow command. + + Args: + message: Unescaped annotation message. + + Returns: + The message with workflow-command control characters escaped. + """ + return message.replace("%", "%25").replace("\r", "%0D").replace("\n", "%0A") + + +def emit_github_warning(path: str, fields: list[str], line: int) -> None: + """ + Print a GitHub Actions warning annotation for missing meta fields. + + Args: + path: Repository-relative path to annotate. + fields: Missing meta field names. + line: One-based source line to annotate. + + Returns: + None. + """ + field_list = ", ".join(fields) + message = _escape_workflow_command_message( + f"Missing meta fields: {field_list}", + ) + print(f"::warning file={path},line={line}::{message}") + + +def emit_github_error(path: str, fields: list[str], line: int) -> None: + """ + Print a GitHub Actions error annotation for missing meta fields. + + This helper is reserved for future checks that should be reported as errors. + + Args: + path: Repository-relative path to annotate. + fields: Missing meta field names. + line: One-based source line to annotate. + + Returns: + None. + """ + field_list = ", ".join(fields) + message = _escape_workflow_command_message( + f"Missing meta fields: {field_list}", + ) + print(f"::error file={path},line={line}::{message}") + + def can_suggest_inline(content: str, pr_lines: set[int]) -> bool: """ Return whether a meta-tag edit can be anchored to the pull request diff. @@ -177,6 +300,13 @@ def can_suggest_inline(content: str, pr_lines: set[int]) -> bool: Existing ``.. meta::`` blocks are suggestable when they overlap the diff (wherever they already sit). New blocks are only suggestable when line 1 is in the diff, since inserts always go at the top of the file. + + Args: + content: RST source before metadata is injected. + pr_lines: One-based lines represented by the pull-request diff. + + Returns: + ``True`` if GitHub can anchor the metadata edit to the diff. """ if has_meta_block(content): span = meta_block_line_span(content) @@ -205,10 +335,19 @@ def ensure_meta_tags_in_file( When ``pr_lines`` is ``None`` (local CLI use), behaviour is an unconditional write: append to an existing block, or insert a new block at the top. + Args: + path: RST file to inspect and, where permitted, update. + meta_config: Mapping from required field names to default values. + pr_lines: One-based pull-request diff lines, or ``None`` for local mode. + Returns: A result dict when action is needed, otherwise ``None``. Keys include ``path``, ``fields``, ``mode`` (``suggestable`` or - ``fallback``), and ``snippet``. + ``fallback``), ``snippet``, and ``line`` (for annotations). + + Raises: + OSError: If the RST file cannot be read or an eligible edit cannot be written. + UnicodeError: If the RST file cannot be decoded or encoded as UTF-8. """ content = path.read_text(encoding="utf-8") missing = _missing_meta_fields(content, meta_config) @@ -219,6 +358,7 @@ def ensure_meta_tags_in_file( metadata = {field: meta_config[field] for field in missing} snippet = format_meta_block(meta_config, missing) path_str = str(path).replace("\\", "/") + warning_line = _warning_line_for_content(content) if pr_lines is not None and not can_suggest_inline(content, pr_lines): logger.info( @@ -231,6 +371,7 @@ def ensure_meta_tags_in_file( "fields": missing, "mode": "fallback", "snippet": snippet, + "line": warning_line, } new_content, changed = inject_metadata_to_content(content, metadata) @@ -251,10 +392,20 @@ def ensure_meta_tags_in_file( "fields": missing, "mode": "suggestable", "snippet": snippet, + "line": warning_line, } def _collect_rst_paths(paths: list[str]) -> list[Path]: + """ + Collect existing RST files from command-line path strings. + + Args: + paths: Candidate filesystem paths. + + Returns: + Existing paths whose suffix is ``.rst`` (case-insensitive). + """ rst_paths: list[Path] = [] for raw in paths: path = Path(raw) @@ -269,12 +420,28 @@ def _collect_rst_paths(paths: list[str]) -> list[Path]: def stamp_review_comment(body: str) -> str: - """Append a hidden marker so CI can find and supersede this review later.""" + """ + Stamp a review comment so CI can supersede it later. + + Args: + body: Unstamped review comment body. + + Returns: + The body with the hidden review marker appended. + """ return body.rstrip() + f"\n\n{REVIEW_MARKER}\n" def build_review_comment(results: list[dict[str, object]]) -> str: - """Build the pull request review / comment body for suggestable and fallback results.""" + """ + Build a pull-request review body from metadata check results. + + Args: + results: Suggestable and fallback result dictionaries. + + Returns: + A stamped Markdown review body containing the relevant instructions. + """ suggestable = [r for r in results if r["mode"] == "suggestable"] fallback = [r for r in results if r["mode"] == "fallback"] @@ -311,6 +478,19 @@ def build_review_comment(results: list[dict[str, object]]) -> str: def main(argv: list[str] | None = None) -> int: + """ + Run the command-line metadata check. + + Args: + argv: Command-line arguments excluding the executable name, or ``None`` + to read them from ``sys.argv``. + + Returns: + Process exit code: ``1`` for outstanding CI results, otherwise ``0``. + + Raises: + SystemExit: If command-line arguments or metadata configuration are invalid. + """ parser = argparse.ArgumentParser( description=( "Ensure configured meta tags exist in RST files. " @@ -378,6 +558,13 @@ def main(argv: list[str] | None = None) -> int: fallback_count, ) + for result in results: + emit_github_warning( + str(result["path"]), + list(result["fields"]), + int(result["line"]), + ) + if args.status_file is not None: has_suggestable = any(r["mode"] == "suggestable" for r in results) has_fallback = any(r["mode"] == "fallback" for r in results) @@ -395,6 +582,8 @@ def main(argv: list[str] | None = None) -> int: f.write("\n") f.write("EOF_META_TAGS_COMMENT\n") + if args.status_file is not None and results: + return 1 return 0 From cff6651e4f1470a697b2987b298edde99881ebf0 Mon Sep 17 00:00:00 2001 From: Keith Kirkwood Date: Fri, 24 Jul 2026 16:36:23 +0100 Subject: [PATCH 05/23] OPENR-174: Allow for generic config of metas, including different severities and blank/prepopuldated values --- .github/workflows/enhance.yml | 10 +- tools/README.md | 113 +++++--- tools/ensure_meta_tags.py | 416 +++++++++++++++++---------- tools/meta_tags.yaml | 21 +- tools/rst_utils.py | 75 +++-- tools/tests/test_ensure_meta_tags.py | 197 +++++++++++++ 6 files changed, 616 insertions(+), 216 deletions(-) create mode 100644 tools/tests/test_ensure_meta_tags.py diff --git a/.github/workflows/enhance.yml b/.github/workflows/enhance.yml index ca81ba3518f..0a88c871128 100644 --- a/.github/workflows/enhance.yml +++ b/.github/workflows/enhance.yml @@ -38,7 +38,7 @@ jobs: - name: Install PyYAML run: pip install --no-warn-script-location pyyaml - - name: Ensure product and distribution meta tags + - name: Ensure documentation metadata id: ensure continue-on-error: true run: | @@ -57,6 +57,8 @@ jobs: echo "meta_checked=false" >> "$GITHUB_OUTPUT" echo "suggestable=false" >> "$GITHUB_OUTPUT" echo "fallback=false" >> "$GITHUB_OUTPUT" + echo "has_results=false" >> "$GITHUB_OUTPUT" + echo "has_errors=false" >> "$GITHUB_OUTPUT" exit 0 fi @@ -141,3 +143,9 @@ jobs: gh pr review "${{ github.event.pull_request.number }}" \ --comment \ --body "$REVIEW_COMMENT" + + - name: Enforce required metadata + if: always() && steps.ensure.outputs.has_errors == 'true' + run: | + echo "Required metadata (error severity) is still missing." + exit 1 diff --git a/tools/README.md b/tools/README.md index a2225fd44c9..6c1b9b5f81c 100644 --- a/tools/README.md +++ b/tools/README.md @@ -7,35 +7,60 @@ Helpers for ensuring reStructuredText (`.rst`) metadata on documentation pull re | File | Purpose | |------|---------| | [`rst_utils.py`](rst_utils.py) | Regex-based read/write of `.. meta::` and `.. short-description::` directives | -| [`meta_tags.yaml`](meta_tags.yaml) | Default values for missing meta fields | -| [`ensure_meta_tags.py`](ensure_meta_tags.py) | CLI that adds missing meta fields from the config | +| [`meta_tags.yaml`](meta_tags.yaml) | Metadata rules (severity and optional default values) | +| [`ensure_meta_tags.py`](ensure_meta_tags.py) | CLI that checks and fixes metadata from the config | +| [`tests/`](tests/) | Unit tests for the tools in this directory | ## Configuration -[`meta_tags.yaml`](meta_tags.yaml) defines which `.. meta::` fields to ensure and the value to inject when each is missing: +[`meta_tags.yaml`](meta_tags.yaml) defines every `.. meta::` field the script checks. Each entry has: + +- **`severity`**: `warning` (advisory; soft-fails the ensure step in CI) or `error` (fails the workflow after reviews are posted). +- **`value`**: default text to inject when the field is missing or blank. Leave empty when the contributor must supply a non-empty value. ```yaml meta: - product: "{PRODUCT}" - distribution: "{DISTRO}" + product: + severity: warning + value: "{PRODUCT}" + distribution: + severity: warning + value: "{DISTRO}" + area: + severity: error + value: + experience: + severity: warning + value: + content-type: + severity: warning + value: ``` -The `meta` map lists every field the script checks. Add a new key to extend coverage without changing Python code. +Add a new key under `meta` to extend coverage without changing Python code. The script applies every rule in the file. + +`{PRODUCT}` and `{DISTRO}` are Sphinx substitution macros expanded at build time from [`conf.py`](../conf.py). + +### Severity behaviour -`{PRODUCT}` and `{DISTRO}` are Sphinx substitution macros expanded at build time from [`conf.py`](../conf.py). Edit this file when you need different default values for suggested meta tags. +| Severity | Missing or blank field | CI ensure step | Workflow job | +|----------|------------------------|----------------|--------------| +| `warning` | Annotation + review | Soft warning (`continue-on-error`) | Succeeds | +| `error` | Error annotation + review | Soft warning (same step) | **Fails** on final enforce step | ## `rst_utils.py` Low-level utilities for locating and editing Sphinx directives in RST source: -- **`get_meta_names_from_content`** — field names already present in the first `.. meta::` block -- **`inject_metadata_to_content`** — append missing `:name: value` lines to an existing block, or insert a new `.. meta::` block at the top of the file; never overwrites existing fields +- **`get_meta_fields_from_content`** — field names and values in the first `.. meta::` block +- **`get_meta_names_from_content`** — field names only +- **`inject_metadata_to_content`** — add missing fields or fill blank values; never overwrites non-empty contributor values The module also contains helpers for `.. short-description::` directives for future use. ## `ensure_meta_tags.py` -Checks each given `.rst` file for the fields listed in `meta_tags.yaml`. When a field is missing, it is added with the configured value. Files that already have all configured fields are left unchanged. +Checks each given `.rst` file against `meta_tags.yaml`. Fields with a configured `value` are added or filled automatically when the edit can be suggested or applied locally. Fields with an empty `value` must be completed manually in the `.. meta::` block. ### Usage @@ -55,12 +80,12 @@ Options: - `--config PATH` — YAML config file (default: `tools/meta_tags.yaml`) - `--diff-base SHA` — PR base commit; only write edits that overlap the PR diff (CI) -- `--status-file PATH` — write `meta_checked=`, `suggestable=`, `fallback=`, `has_results=`, and the review comment for CI; when issues remain, the script also emits GitHub Actions warning annotations and exits with code `1` (the workflow soft-fails that step without failing the job) +- `--status-file PATH` — write `meta_checked`, `suggestable`, `fallback`, `has_results`, `has_errors`, and the review comment for CI; when issues remain, emits annotations and exits `1` (the ensure step uses `continue-on-error`) - `-v` / `--verbose` — enable debug logging -Local runs without `--status-file` still exit `0` after applying edits, even when fields were missing. +**Exit codes:** With `--status-file` (CI), exit `1` when any issues remain. Locally, exit `1` only when **error**-severity fields are still unresolved; warning-only issues exit `0` after applying automatic fixes. -### Example +### Example (configured values) Before: @@ -71,7 +96,7 @@ My Article Some content. ``` -After a local run (new `.. meta::` at the top of the file by default): +After a local run (new `.. meta::` at the top of the file): ```rst .. meta:: @@ -84,7 +109,7 @@ My Article Some content. ``` -If a `.. meta::` block already exists, missing fields are appended to it rather than creating a new block. +Fields such as `area` with an empty `value` in the config are listed in the review for manual completion; they are not given placeholder text. ## Continuous integration @@ -93,57 +118,63 @@ The workflow [`.github/workflows/enhance.yml`](../.github/workflows/enhance.yml) 1. Checks out the PR’s `.rst` files as untrusted data 2. Checks out the base branch into `.trusted-base/` for the script and config 3. Runs the trusted `ensure_meta_tags.py` with `--diff-base` against changed RST files -4. Emits per-file warning annotations and soft-fails the ensure step when metadata is still missing +4. Emits per-file warning/error annotations and soft-fails the ensure step when metadata is still missing 5. Posts inline suggestions and/or a review comment +6. **Fails the job** if any **error**-severity metadata remains (`Enforce required metadata`) -Priority: **inline suggestions wherever GitHub allows them**. Copy-paste in the review body is only a fallback. +Priority: **inline suggestions wherever GitHub allows them**. Copy-paste blocks and manual field lists in the review body are fallbacks. -The ensure step uses `continue-on-error: true`, so the Enhance job still **succeeds** and does not block merge. Contributors see a warning on that step in Actions when work remains. +The ensure step uses `continue-on-error: true`, so warning-only gaps do not fail the job. Error-severity gaps (for example `area`) still fail the workflow after contributors receive review feedback. -### Warnings and annotations +### Annotations -When metadata is missing, the script prints one [workflow warning](https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/workflow-commands-for-github-actions#setting-a-warning-message) per affected file: +When metadata is missing or blank: -```text -::warning file=path/to/file.rst,line=N::Missing meta fields: product, distribution -``` +- **Warning** severity → `::warning file=...,line=N::Missing meta fields: ...` +- **Error** severity → `::error file=...,line=N::Missing meta fields: ...` `N` is the start line of an existing `.. meta::` block, or `1` when a new block would be inserted at the top of the file. ### Contributor experience -| Situation | Actions ensure step | File annotations | Pull request review | -|-----------|---------------------|------------------|---------------------| -| All configured fields present | Green | None | None (stale bot reviews cleared) | -| Missing fields; edit overlaps PR diff | Soft warning | Yes | Inline “Commit suggestion” | -| Missing fields; edit outside PR diff | Soft warning | Yes | Copy-paste `.. meta::` block | -| Mixed (some suggestable, some fallback) | Soft warning | Yes (all affected files) | Suggestions plus fallback blocks | +| Situation | Ensure step | Annotations | Pull request review | Job result | +|-----------|-------------|-------------|---------------------|------------| +| All fields resolved | Green | None | None (stale bot reviews cleared) | Success | +| Warning-only gaps | Soft warning | Warnings | Suggestions and/or manual list | Success | +| Error gaps (e.g. missing `area`) | Soft warning | Errors (and warnings) | Suggestions and/or manual list | **Failure** after enforce step | +| Auto-fix in diff | Soft warning | As above | Inline “Commit suggestion” | As per severity | +| Auto-fix outside diff | Soft warning | As above | Copy-paste `.. meta::` for configured values | As per severity | +| Manual-only fields | Soft warning | As above | Field list with required/warning labels | As per severity | + +Reviews, annotations, and the soft-failed ensure step appear in different parts of the GitHub UI (Conversation, Files changed, Checks); only error-severity issues fail the overall workflow. ### When inline suggestions appear -GitHub only allows review suggestions on [lines already in the pull request diff](https://github.com/marketplace/actions/suggest-changes-action). The script compares each needed edit to that diff: +GitHub only allows review suggestions on [lines already in the pull request diff](https://github.com/marketplace/actions/suggest-changes-action). The script compares each automatic edit to that diff: | Situation | What happens | |-----------|----------------| -| Missing fields; existing `.. meta::` overlaps the PR diff | Write append to the working tree → inline “Commit suggestion” via [`suggest-changes`](https://github.com/marketplace/actions/suggest-changes-action). Review text asks the submitter to accept it and why. | -| No `.. meta::`; top of file overlaps the PR diff | Insert at top of file → inline suggestion + same review text. | -| Needed edit does **not** overlap the PR diff (new block or out-of-diff append) | **Do not** write an unsuggestable hunk. Review / comment only, with a copy-paste `.. meta::` block to place at the **top of the file**. | -| All configured fields already present | No action | +| Missing configured values; existing `.. meta::` overlaps the PR diff | Write append/fill to the working tree → inline suggestion via [`suggest-changes`](https://github.com/marketplace/actions/suggest-changes-action) | +| No `.. meta::`; top of file overlaps the PR diff | Insert at top → inline suggestion | +| Automatic edit does **not** overlap the PR diff | No unsuggestable write; review includes a copy-paste block for configured values | +| Only manual fields (empty `value` in config) | Review lists fields; no placeholder injection | +| All configured fields present and non-empty | No action | | No changed `.rst` files in the PR | Workflow exits early | -Mixed PRs are supported: suggestable files get working-tree edits + inline suggestions; fallback-only files appear only in the review body. If every file is fallback-only, a `COMMENT` review is posted without `suggest-changes`. - ### Superseding outdated reviews Each bot review body includes a hidden marker (``). On every run that checks changed RST files: -1. All prior stamped reviews on the pull request are minimized as **Outdated** via the GitHub API (no external state store). +1. All prior stamped reviews on the pull request are minimized as **Outdated** via the GitHub API. 2. A fresh review is posted only when work still remains. 3. When all issues are fixed, stale reviews are minimized and no new “all clear” comment is posted. -Typical examples: +The workflow uses [`pull_request_target`](https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request_target) so the default `GITHUB_TOKEN` can post review suggestions on fork PRs. The workflow definition itself is taken from the repository **default branch**; the trusted script and config come from the PR **base** branch. See [Mitigating the risks of untrusted code checkout](https://docs.github.com/en/actions/reference/security/secure-use#mitigating-the-risks-of-untrusted-code-checkout). + +## Tests -- PR edits near an existing `.. meta::` → inline suggestions for missing `product` / `distribution`. -- PR only changes a mid-file paragraph and the file has no meta → out of diff → copy-paste block at top of file in the review body. +Unit tests live in [`tests/`](tests/). From the repository root (with PyYAML installed): -The workflow uses [`pull_request_target`](https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request_target) so the default `GITHUB_TOKEN` can post review suggestions on fork PRs. To avoid running untrusted code with elevated permissions, only the base-branch copy of `ensure_meta_tags.py` and `meta_tags.yaml` is executed; PR content is treated as data to read and update. See [Mitigating the risks of untrusted code checkout](https://docs.github.com/en/actions/reference/security/secure-use#mitigating-the-risks-of-untrusted-code-checkout). +```bash +python3 -m unittest discover -s tools/tests -p 'test_*.py' +``` diff --git a/tools/ensure_meta_tags.py b/tools/ensure_meta_tags.py index f26342159fa..3a90dfad7c4 100644 --- a/tools/ensure_meta_tags.py +++ b/tools/ensure_meta_tags.py @@ -2,16 +2,17 @@ """ Ensure configured metadata fields exist in RST ``.. meta::`` blocks. -Missing fields are injected with values from ``meta_tags.yaml``. -Existing fields are never overwritten. +Rules are defined in ``meta_tags.yaml`` with severity and optional values. +Missing or blank fields are resolved automatically when a value is configured; +otherwise contributors must supply a non-empty value. When ``--diff-base`` is set, edits are only written to disk when they overlap the pull request diff (so GitHub can offer inline suggestions). Otherwise the -review comment carries a top-of-file copy-paste fallback. +review comment carries copy-paste or manual instructions. -In CI (``--status-file``), emits GitHub Actions ``::warning`` annotations per -affected file and exits with code ``1`` when issues remain so the workflow can -soft-fail the step without failing the job. +In CI (``--status-file``), emits GitHub Actions annotations per severity and +exits with code ``1`` when issues remain so the ensure step can soft-fail. +Error-severity issues also set ``has_errors`` for a final workflow gate. """ from __future__ import annotations @@ -21,20 +22,22 @@ import re import subprocess import sys +from dataclasses import dataclass from pathlib import Path +from typing import Literal import yaml # Allow ``python3 tools/ensure_meta_tags.py`` from the repository root. _TOOLS_DIR = Path(__file__).resolve().parent if str(_TOOLS_DIR) not in sys.path: - sys.path.insert(0, str(_TOOLS_DIR)) + sys.path.insert(0, str(_TOOLS_DIR)) from rst_utils import ( - get_meta_names_from_content, - has_meta_block, - inject_metadata_to_content, - meta_block_line_span, + get_meta_fields_from_content, + has_meta_block, + inject_metadata_to_content, + meta_block_line_span, ) logger = logging.getLogger(__name__) @@ -42,23 +45,37 @@ DEFAULT_CONFIG_PATH = _TOOLS_DIR / "meta_tags.yaml" RST_EXTENSION = ".rst" _HUNK_HEADER = re.compile(r"^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@") +Severity = Literal["warning", "error"] # Hidden marker in review bodies so CI can find and supersede prior bot reviews. REVIEW_MARKER = "" -def load_meta_config(config_path: Path) -> dict[str, str]: +@dataclass(frozen=True) +class MetaRule: + """A single metadata field rule from ``meta_tags.yaml``.""" + + severity: Severity + value: str + + @property + def has_configured_value(self) -> bool: + """Return whether the rule supplies a non-empty default value.""" + return bool(self.value.strip()) + + +def load_meta_config(config_path: Path) -> dict[str, MetaRule]: """ - Load the ``meta`` mapping from a YAML config file. + Load and validate metadata rules from a YAML config file. Args: - config_path: Path to the YAML configuration file. + config_path: Path to the YAML configuration file. Returns: - A mapping from each configured meta field name to its default value. + Mapping from meta field name to its rule. Raises: - SystemExit: If the file is missing, invalid, or has no usable ``meta`` map. + SystemExit: If the file is missing, invalid, or has unusable rules. """ if not config_path.is_file(): logger.error("Config file not found: %s", config_path) @@ -79,58 +96,86 @@ def load_meta_config(config_path: Path) -> dict[str, str]: logger.error("Config %s must contain a non-empty 'meta' mapping", config_path) raise SystemExit(1) - validated: dict[str, str] = {} - for key, value in meta.items(): + validated: dict[str, MetaRule] = {} + for key, entry in meta.items(): if not isinstance(key, str) or not key.strip(): logger.error("Config %s: meta keys must be non-empty strings", config_path) raise SystemExit(1) - if not isinstance(value, str): + if not isinstance(entry, dict): + logger.error( + "Config %s: meta entry for %r must be a mapping with severity and value", + config_path, + key, + ) + raise SystemExit(1) + severity = entry.get("severity") + if severity not in ("warning", "error"): + logger.error( + "Config %s: meta entry %r severity must be 'warning' or 'error', got %r", + config_path, + key, + severity, + ) + raise SystemExit(1) + if "value" not in entry: + logger.error("Config %s: meta entry %r must include a 'value' key", config_path, key) + raise SystemExit(1) + raw_value = entry.get("value") + if raw_value is None: + value = "" + elif isinstance(raw_value, str): + value = raw_value + else: logger.error( - "Config %s: meta value for %r must be a string, got %s", + "Config %s: meta value for %r must be a string or null, got %s", config_path, key, - type(value).__name__, + type(raw_value).__name__, ) raise SystemExit(1) - validated[key] = value + validated[key] = MetaRule(severity=severity, value=value) return validated -def format_meta_block(meta_config: dict[str, str], fields: list[str]) -> str: +def format_meta_block(rules: dict[str, MetaRule], fields: list[str]) -> str: """ - Build an RST ``.. meta::`` block for the requested fields. + Build an RST ``.. meta::`` block for auto-injectable fields. Args: - meta_config: Mapping from meta field names to their default values. - fields: Field names to include in the block, in output order. + rules: Configured metadata rules keyed by field name. + fields: Field names to include in the block, in output order. Returns: - A formatted ``.. meta::`` directive ending with a blank line. + A formatted ``.. meta::`` directive ending with a blank line. Raises: - KeyError: If a requested field is absent from ``meta_config``. + KeyError: If a requested field is absent from ``rules``. """ lines = [".. meta::"] for field in fields: - lines.append(f" :{field}: {meta_config[field]}") + lines.append(f" :{field}: {rules[field].value}") lines.append("") return "\n".join(lines) -def _missing_meta_fields(content: str, meta_config: dict[str, str]) -> list[str]: +def _unresolved_fields(content: str, rules: dict[str, MetaRule]) -> list[str]: """ - Find configured meta fields that are absent from RST content. + Find configured meta fields that are absent or blank in RST content. Args: - content: RST source to inspect. - meta_config: Mapping of meta fields that should be present. + content: RST source to inspect. + rules: Configured metadata rules. Returns: - Missing field names in configuration order. + Unresolved field names in configuration order. """ - present = get_meta_names_from_content(content) - return [field for field in meta_config if field not in present] + present = get_meta_fields_from_content(content) + unresolved: list[str] = [] + for name in rules: + if name not in present or not present[name].strip(): + unresolved.append(name) + return unresolved def parse_diff_new_side_lines(diff_text: str) -> set[int]: @@ -145,10 +190,10 @@ def parse_diff_new_side_lines(diff_text: str) -> set[int]: ``+`` characters), and must advance ``new_line`` accordingly. Args: - diff_text: Unified diff text to parse. + diff_text: Unified diff text to parse. Returns: - One-based line numbers represented on the new side of diff hunks. + One-based line numbers represented on the new side of diff hunks. """ lines: set[int] = set() new_line = 0 @@ -173,7 +218,6 @@ def parse_diff_new_side_lines(diff_text: str) -> set[int]: lines.add(new_line) new_line += 1 elif line.startswith("diff "): - # Next file in a multi-file diff; subsequent ---/+++ are headers again. in_hunk = False new_line = 0 return lines @@ -184,11 +228,11 @@ def pr_diff_lines_for_file(diff_base: str, path: Path) -> set[int]: Find pull-request diff lines for a file on the head side. Args: - diff_base: Base commit SHA used for the three-dot comparison. - path: Repository-relative path to inspect. + diff_base: Base commit SHA used for the three-dot comparison. + path: Repository-relative path to inspect. Returns: - One-based new-side line numbers, or an empty set if ``git diff`` fails. + One-based new-side line numbers, or an empty set if ``git diff`` fails. """ result = subprocess.run( ["git", "diff", "-U3", f"{diff_base}...HEAD", "--", str(path)], @@ -212,11 +256,11 @@ def _span_overlaps(span: tuple[int, int] | None, pr_lines: set[int]) -> bool: Test whether an inclusive line span overlaps pull-request lines. Args: - span: Inclusive one-based start and end lines, or ``None``. - pr_lines: One-based line numbers represented by the pull-request diff. + span: Inclusive one-based start and end lines, or ``None``. + pr_lines: One-based line numbers represented by the pull-request diff. Returns: - ``True`` when at least one line overlaps; otherwise ``False``. + ``True`` when at least one line overlaps; otherwise ``False``. """ if span is None or not pr_lines: return False @@ -224,15 +268,15 @@ def _span_overlaps(span: tuple[int, int] | None, pr_lines: set[int]) -> bool: return any(line in pr_lines for line in range(start, end + 1)) -def _warning_line_for_content(content: str) -> int: +def _annotation_line_for_content(content: str) -> int: """ Select a line for a GitHub annotation on RST content. Args: - content: RST source to inspect. + content: RST source to inspect. Returns: - The first line of an existing meta block, or line 1 if none exists. + The first line of an existing meta block, or line 1 if none exists. """ span = meta_block_line_span(content) if span is not None: @@ -245,10 +289,10 @@ def _escape_workflow_command_message(message: str) -> str: Escape a message for use in a GitHub Actions workflow command. Args: - message: Unescaped annotation message. + message: Unescaped annotation message. Returns: - The message with workflow-command control characters escaped. + The message with workflow-command control characters escaped. """ return message.replace("%", "%25").replace("\r", "%0D").replace("\n", "%0A") @@ -258,13 +302,15 @@ def emit_github_warning(path: str, fields: list[str], line: int) -> None: Print a GitHub Actions warning annotation for missing meta fields. Args: - path: Repository-relative path to annotate. - fields: Missing meta field names. - line: One-based source line to annotate. + path: Repository-relative path to annotate. + fields: Missing meta field names. + line: One-based source line to annotate. Returns: - None. + None. """ + if not fields: + return field_list = ", ".join(fields) message = _escape_workflow_command_message( f"Missing meta fields: {field_list}", @@ -276,16 +322,16 @@ def emit_github_error(path: str, fields: list[str], line: int) -> None: """ Print a GitHub Actions error annotation for missing meta fields. - This helper is reserved for future checks that should be reported as errors. - Args: - path: Repository-relative path to annotate. - fields: Missing meta field names. - line: One-based source line to annotate. + path: Repository-relative path to annotate. + fields: Missing meta field names. + line: One-based source line to annotate. Returns: - None. + None. """ + if not fields: + return field_list = ", ".join(fields) message = _escape_workflow_command_message( f"Missing meta fields: {field_list}", @@ -297,102 +343,119 @@ def can_suggest_inline(content: str, pr_lines: set[int]) -> bool: """ Return whether a meta-tag edit can be anchored to the pull request diff. - Existing ``.. meta::`` blocks are suggestable when they overlap the diff - (wherever they already sit). New blocks are only suggestable when line 1 is - in the diff, since inserts always go at the top of the file. + Existing ``.. meta::`` blocks are suggestable when the block's inclusive line + span overlaps the diff. New blocks are only suggestable when line 1 is in + the diff, since inserts always go at the top of the file. Args: - content: RST source before metadata is injected. - pr_lines: One-based lines represented by the pull-request diff. + content: RST source before metadata is injected. + pr_lines: One-based lines represented by the pull-request diff. Returns: - ``True`` if GitHub can anchor the metadata edit to the diff. + ``True`` if GitHub can anchor the metadata edit to the diff. """ if has_meta_block(content): span = meta_block_line_span(content) if span is None: return False - start, end = span - # Include the line after the block where fields would be appended. - return _span_overlaps((start, end + 1), pr_lines) + return _span_overlaps(span, pr_lines) return _span_overlaps((1, 1), pr_lines) +def _severity_fields( + field_names: list[str], + rules: dict[str, MetaRule], + severity: Severity, +) -> list[str]: + """Return ``field_names`` that use the given severity in ``rules``.""" + return [name for name in field_names if rules[name].severity == severity] + + def ensure_meta_tags_in_file( path: Path, - meta_config: dict[str, str], + rules: dict[str, MetaRule], *, pr_lines: set[int] | None = None, ) -> dict[str, object] | None: """ - Add missing configured meta fields, preferring pull-request-diff overlap. + Resolve missing metadata in one RST file where rules allow automatic fixes. - When ``pr_lines`` is provided, the file is only modified when the edit can - land on lines already in the PR diff (inline-suggestion path). Otherwise the - result is a fallback entry without writing the file. - - When ``pr_lines`` is ``None`` (local CLI use), behaviour is an unconditional - write: append to an existing block, or insert a new block at the top. + When ``pr_lines`` is provided, automatic edits are only written when they can + land on lines already in the PR diff. Fields without configured values always + require manual input. Args: - path: RST file to inspect and, where permitted, update. - meta_config: Mapping from required field names to default values. - pr_lines: One-based pull-request diff lines, or ``None`` for local mode. + path: RST file to inspect and, where permitted, update. + rules: Configured metadata rules keyed by field name. + pr_lines: One-based pull-request diff lines, or ``None`` for local mode. Returns: - A result dict when action is needed, otherwise ``None``. - Keys include ``path``, ``fields``, ``mode`` (``suggestable`` or - ``fallback``), ``snippet``, and ``line`` (for annotations). + A result dict when issues remain, otherwise ``None``. Raises: - OSError: If the RST file cannot be read or an eligible edit cannot be written. - UnicodeError: If the RST file cannot be decoded or encoded as UTF-8. + OSError: If the RST file cannot be read or an eligible edit cannot be written. + UnicodeError: If the RST file cannot be decoded or encoded as UTF-8. """ content = path.read_text(encoding="utf-8") - missing = _missing_meta_fields(content, meta_config) - if not missing: + unresolved = _unresolved_fields(content, rules) + if not unresolved: logger.info("%s: all configured meta fields present", path) return None - metadata = {field: meta_config[field] for field in missing} - snippet = format_meta_block(meta_config, missing) path_str = str(path).replace("\\", "/") - warning_line = _warning_line_for_content(content) - - if pr_lines is not None and not can_suggest_inline(content, pr_lines): - logger.info( - "%s: missing %s but edit is outside the PR diff; fallback review only", - path, - ", ".join(missing), - ) - return { - "path": path_str, - "fields": missing, - "mode": "fallback", - "snippet": snippet, - "line": warning_line, - } - - new_content, changed = inject_metadata_to_content(content, metadata) - if not changed: + annotation_line = _annotation_line_for_content(content) + + auto_fields = [name for name in unresolved if rules[name].has_configured_value] + auto_metadata = {name: rules[name].value for name in auto_fields} + + mode: str | None = None + snippet = "" + + if auto_metadata: + if pr_lines is not None and not can_suggest_inline(content, pr_lines): + mode = "fallback" + snippet = format_meta_block(rules, auto_fields) + logger.info( + "%s: missing %s but auto-fix is outside the PR diff; fallback review only", + path, + ", ".join(auto_fields), + ) + else: + new_content, changed = inject_metadata_to_content(content, auto_metadata) + if changed: + path.write_text(new_content, encoding="utf-8") + content = new_content + mode = "suggestable" + snippet = format_meta_block(rules, auto_fields) + if pr_lines is None: + logger.info("%s: added meta fields %s", path, ", ".join(auto_fields)) + else: + logger.info( + "%s: added meta fields %s (inline suggestion)", + path, + ", ".join(auto_fields), + ) + + still_unresolved = _unresolved_fields(content, rules) + if not still_unresolved: return None - path.write_text(new_content, encoding="utf-8") - if pr_lines is None: - logger.info("%s: added meta fields %s", path, ", ".join(missing)) - else: - logger.info( - "%s: added meta fields %s (inline suggestion)", - path, - ", ".join(missing), - ) + manual_fields = [name for name in still_unresolved if not rules[name].has_configured_value] + warning_fields = _severity_fields(still_unresolved, rules, "warning") + error_fields = _severity_fields(still_unresolved, rules, "error") + + if mode is None: + mode = "manual_only" + return { "path": path_str, - "fields": missing, - "mode": "suggestable", + "line": annotation_line, + "mode": mode, "snippet": snippet, - "line": warning_line, + "manual_fields": manual_fields, + "warning_fields": warning_fields, + "error_fields": error_fields, } @@ -401,10 +464,10 @@ def _collect_rst_paths(paths: list[str]) -> list[Path]: Collect existing RST files from command-line path strings. Args: - paths: Candidate filesystem paths. + paths: Candidate filesystem paths. Returns: - Existing paths whose suffix is ``.rst`` (case-insensitive). + Existing paths whose suffix is ``.rst`` (case-insensitive). """ rst_paths: list[Path] = [] for raw in paths: @@ -424,46 +487,60 @@ def stamp_review_comment(body: str) -> str: Stamp a review comment so CI can supersede it later. Args: - body: Unstamped review comment body. + body: Unstamped review comment body. Returns: - The body with the hidden review marker appended. + The body with the hidden review marker appended. """ return body.rstrip() + f"\n\n{REVIEW_MARKER}\n" -def build_review_comment(results: list[dict[str, object]]) -> str: +def _field_list_markdown(field_names: list[str], rules: dict[str, MetaRule]) -> str: + """Format field names with severity hints for review text.""" + parts: list[str] = [] + for name in field_names: + label = "required" if rules[name].severity == "error" else "warning" + parts.append(f"`{name}` ({label})") + return ", ".join(parts) + + +def build_review_comment( + results: list[dict[str, object]], + rules: dict[str, MetaRule], +) -> str: """ Build a pull-request review body from metadata check results. Args: - results: Suggestable and fallback result dictionaries. + results: Per-file result dictionaries from ``ensure_meta_tags_in_file``. + rules: Configured metadata rules. Returns: - A stamped Markdown review body containing the relevant instructions. + A stamped Markdown review body containing the relevant instructions. """ suggestable = [r for r in results if r["mode"] == "suggestable"] fallback = [r for r in results if r["mode"] == "fallback"] + manual_only = [r for r in results if r["mode"] == "manual_only"] lines = [ - "This pull request is missing configured `product` / `distribution` " - "meta tags (defaults from `tools/meta_tags.yaml`).", + "This pull request is missing configured documentation metadata " + "(see `tools/meta_tags.yaml`).", "", ] if suggestable: lines.append( - "Please **review and commit the inline suggestions**. They add the " - "missing fields in place so the documentation metadata stays " - "complete." + "Please **review and commit the inline suggestions**. They add configured " + "default values in place so metadata stays complete.", ) lines.append("") if fallback: lines.append( - "GitHub can only attach suggestions to lines already in the pull " - "request diff, so the following files could not get an inline " - "suggestion. Please add this block at the **top of each file**:" + "GitHub can only attach suggestions to lines already in the pull request " + "diff, so the following files could not get an inline suggestion for " + "auto-filled fields. Please add this block at the **top of each file** " + "(or append the listed fields to an existing `.. meta::` block):", ) lines.append("") for result in fallback: @@ -473,6 +550,26 @@ def build_review_comment(results: list[dict[str, object]]) -> str: lines.append("```") lines.append("") + manual_results = [ + r for r in results if list(r.get("manual_fields", [])) + ] + if manual_results: + lines.append( + "The following fields must be provided with **non-empty** values in each " + "file's `.. meta::` block:", + ) + lines.append("") + for result in manual_results: + manual = list(result["manual_fields"]) + lines.append(f"**`{result['path']}`**: {_field_list_markdown(manual, rules)}") + lines.append("") + + if manual_only and not suggestable and not fallback: + lines.append( + "Add or complete a `.. meta::` block at the top of each affected file.", + ) + lines.append("") + body = "\n".join(lines).rstrip() + "\n" return stamp_review_comment(body) @@ -482,19 +579,20 @@ def main(argv: list[str] | None = None) -> int: Run the command-line metadata check. Args: - argv: Command-line arguments excluding the executable name, or ``None`` - to read them from ``sys.argv``. + argv: Command-line arguments excluding the executable name, or ``None`` + to read them from ``sys.argv``. Returns: - Process exit code: ``1`` for outstanding CI results, otherwise ``0``. + Process exit code. In CI, ``1`` when any issues remain. Locally, ``1`` + only when error-severity fields remain unresolved. Raises: - SystemExit: If command-line arguments or metadata configuration are invalid. + SystemExit: If command-line arguments or metadata configuration are invalid. """ parser = argparse.ArgumentParser( description=( - "Ensure configured meta tags exist in RST files. " - "Missing fields are added with values from a YAML config file." + "Ensure configured meta tags exist in RST files using rules from a YAML " + "config file." ), ) parser.add_argument( @@ -511,15 +609,17 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument( "--diff-base", help=( - "Git commit SHA for the pull request base. When set, only writes " - "edits that overlap the PR diff; other files become review-comment " - "fallbacks." + "Git commit SHA for the pull request base. When set, only writes edits " + "that overlap the PR diff; other files become review-comment fallbacks." ), ) parser.add_argument( "--status-file", type=Path, - help="Write suggestable=true|false, fallback=true|false, and the review comment for CI", + help=( + "Write meta_checked, suggestable, fallback, has_results, has_errors, " + "and the review comment for CI" + ), ) parser.add_argument( "-v", @@ -534,7 +634,7 @@ def main(argv: list[str] | None = None) -> int: format="%(levelname)s: %(message)s", ) - meta_config = load_meta_config(args.config) + rules = load_meta_config(args.config) rst_paths = _collect_rst_paths(args.paths) results: list[dict[str, object]] = [] @@ -545,37 +645,43 @@ def main(argv: list[str] | None = None) -> int: pr_lines: set[int] | None = None if args.diff_base: pr_lines = pr_diff_lines_for_file(args.diff_base, path) - result = ensure_meta_tags_in_file(path, meta_config, pr_lines=pr_lines) + result = ensure_meta_tags_in_file(path, rules, pr_lines=pr_lines) if result is not None: results.append(result) suggestable_count = sum(1 for r in results if r["mode"] == "suggestable") fallback_count = sum(1 for r in results if r["mode"] == "fallback") + manual_count = sum(1 for r in results if r["mode"] == "manual_only") logger.info( - "Processed %d file(s): %d suggestable write(s), %d fallback(s)", + "Processed %d file(s): %d suggestable, %d fallback, %d manual-only", len(rst_paths), suggestable_count, fallback_count, + manual_count, ) for result in results: - emit_github_warning( - str(result["path"]), - list(result["fields"]), - int(result["line"]), - ) + path = str(result["path"]) + line = int(result["line"]) + emit_github_warning(path, list(result["warning_fields"]), line) + emit_github_error(path, list(result["error_fields"]), line) + + has_errors = any(result["error_fields"] for result in results) if args.status_file is not None: has_suggestable = any(r["mode"] == "suggestable" for r in results) - has_fallback = any(r["mode"] == "fallback" for r in results) + has_fallback = any( + r["mode"] in ("fallback", "manual_only") for r in results + ) has_results = bool(results) with args.status_file.open("a", encoding="utf-8") as f: f.write("meta_checked=true\n") f.write(f"suggestable={'true' if has_suggestable else 'false'}\n") f.write(f"fallback={'true' if has_fallback else 'false'}\n") f.write(f"has_results={'true' if has_results else 'false'}\n") + f.write(f"has_errors={'true' if has_errors else 'false'}\n") if results: - review_body = build_review_comment(results) + review_body = build_review_comment(results, rules) f.write("comment< int: if args.status_file is not None and results: return 1 + if args.status_file is None and has_errors: + return 1 return 0 diff --git a/tools/meta_tags.yaml b/tools/meta_tags.yaml index 4dca7064556..94db3eae9ab 100644 --- a/tools/meta_tags.yaml +++ b/tools/meta_tags.yaml @@ -1,5 +1,20 @@ -# Default values for missing .. meta:: fields injected by ensure_meta_tags.py. +# Metadata rules for ensure_meta_tags.py. +# severity: warning (soft-fail step) or error (workflow fails after review). +# value: injected when missing/blank; leave empty for contributor-provided values. # Sphinx expands {PRODUCT} and {DISTRO} at build time (see conf.py). meta: - product: "{PRODUCT}" - distribution: "{DISTRO}" + product: + severity: warning + value: "{PRODUCT}" + distribution: + severity: warning + value: "{DISTRO}" + area: + severity: error + value: + experience: + severity: warning + value: + content-type: + severity: warning + value: diff --git a/tools/rst_utils.py b/tools/rst_utils.py index 38a76cac791..63f4eebcb7a 100644 --- a/tools/rst_utils.py +++ b/tools/rst_utils.py @@ -70,11 +70,43 @@ def _extract_meta_names_from_block(meta_block_inner: str) -> set[str]: Returns: A set of field names found in the block. """ - names: set[str] = set() - # Field list lines only; group 1 is the name segment (includes ``attr=value`` forms before the final ``:``) - for field_match in re.finditer(r"^[ \t]+:([^:\n]+?):", meta_block_inner, re.MULTILINE): - names.add(field_match.group(1).strip()) - return names + return set(_extract_meta_fields_from_block(meta_block_inner)) + + +def _extract_meta_fields_from_block(meta_block_inner: str) -> dict[str, str]: + """ + Collect field names and values from the body of a ``.. meta::`` directive. + + Args: + meta_block_inner: The inner text of the meta block. + + Returns: + Mapping from field name to field body text (may be empty). + """ + fields: dict[str, str] = {} + for field_match in re.finditer( + r"^[ \t]+:([^:\n]+?):\s*(.*)$", + meta_block_inner, + re.MULTILINE, + ): + fields[field_match.group(1).strip()] = field_match.group(2) + return fields + + +def get_meta_fields_from_content(content: str) -> dict[str, str]: + """ + Return field names and values from the first ``.. meta::`` block. + + If no ``.. meta::`` directive exists, returns an empty mapping. + + Args: + content: The RST file content to search. + + Returns: + Mapping from meta field name to field body text. + """ + _start, _marker_end, _block_end, inner, _indent = _find_meta_block(content) + return _extract_meta_fields_from_block(inner) def get_meta_names_from_content(content: str) -> set[str]: @@ -130,30 +162,39 @@ def inject_metadata_to_content( Appends to an existing ``.. meta::`` block when present. Otherwise inserts a new block at the start of the document. - Skips keys that already appear in the block. + Skips keys that already have a non-empty value in the block. Fills keys that + are missing or present with a blank value. Returns: Updated source and whether any change was made. """ start, marker_end, block_end, inner, indent = _find_meta_block(content) - names = _extract_meta_names_from_block(inner) # Snapshot before we add keys from this same batch - additions: list[str] = [] + existing = _extract_meta_fields_from_block(inner) + merged: dict[str, str] = dict(existing) + changed = False for key, raw_value in metadata.items(): - if key in names: + value = _normalise_meta_field_value(raw_value) + if key not in merged: + merged[key] = value + changed = True + elif not merged[key].strip(): + merged[key] = value + changed = True + else: logger.warning( "Existing meta field %r in .. meta:: block; skipping", key, ) - continue - value = _normalise_meta_field_value(raw_value) - additions.append(f"{indent}:{key}: {value}\n") - names.add(key) # Prevent duplicate inserts if ``metadata`` repeats a key - if not additions: - return content, False # Nothing new to write; leave the file untouched + if not changed: + return content, False - new_inner = inner + "".join(additions) # Existing fields unchanged, then appended lines + ordered_keys: list[str] = list(existing.keys()) + for key in metadata: + if key not in ordered_keys: + ordered_keys.append(key) + new_inner = "".join(f"{indent}:{key}: {merged[key]}\n" for key in ordered_keys) if start >= 0: # Replace only the directive body slice; ``marker_end``/``block_end`` bracket the original inner @@ -162,7 +203,7 @@ def inject_metadata_to_content( new_content = content[:marker_end] + new_inner + "\n" + remainder else: remainder = content.lstrip() - new_content = ".. meta::\n" + "".join(additions) + "\n" + remainder + new_content = ".. meta::\n" + new_inner + "\n" + remainder return new_content, True diff --git a/tools/tests/test_ensure_meta_tags.py b/tools/tests/test_ensure_meta_tags.py new file mode 100644 index 00000000000..8d8efb4c1e3 --- /dev/null +++ b/tools/tests/test_ensure_meta_tags.py @@ -0,0 +1,197 @@ +# Copyright 2026 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import sys +import tempfile +import textwrap +import unittest +from pathlib import Path + +_TOOLS_DIR = Path(__file__).resolve().parent.parent +if str(_TOOLS_DIR) not in sys.path: + sys.path.insert(0, str(_TOOLS_DIR)) + +from ensure_meta_tags import ( # noqa: E402 + MetaRule, + _unresolved_fields, + build_review_comment, + can_suggest_inline, + ensure_meta_tags_in_file, + load_meta_config, + main, +) +from rst_utils import get_meta_fields_from_content, inject_metadata_to_content # noqa: E402 + +SAMPLE_CONFIG = textwrap.dedent( + """ + meta: + product: + severity: warning + value: "{PRODUCT}" + area: + severity: error + value: + experience: + severity: warning + value: + """ +).strip() + + +class TestMetaConfig(unittest.TestCase): + def test_load_meta_config_parses_rules(self) -> None: + with tempfile.NamedTemporaryFile("w", suffix=".yaml", delete=False) as handle: + handle.write(SAMPLE_CONFIG) + path = Path(handle.name) + try: + rules = load_meta_config(path) + finally: + path.unlink() + self.assertEqual(rules["product"].severity, "warning") + self.assertEqual(rules["product"].value, "{PRODUCT}") + self.assertTrue(rules["product"].has_configured_value) + self.assertFalse(rules["area"].has_configured_value) + self.assertEqual(rules["area"].severity, "error") + + +class TestCanSuggestInline(unittest.TestCase): + def test_meta_block_overlap_uses_inclusive_span_only(self) -> None: + content = textwrap.dedent( + """ + .. meta:: + :product: x + + Title + ===== + """ + ).lstrip() + # Meta block is lines 1-2; line 3 is blank after the block. + self.assertFalse(can_suggest_inline(content, {3})) + self.assertTrue(can_suggest_inline(content, {2})) + + +class TestUnresolvedFields(unittest.TestCase): + def test_missing_and_blank_count_as_unresolved(self) -> None: + rules = { + "product": MetaRule("warning", "{PRODUCT}"), + "area": MetaRule("error", ""), + } + content = textwrap.dedent( + """ + .. meta:: + :area: + + Title + ===== + """ + ) + self.assertEqual(_unresolved_fields(content, rules), ["product", "area"]) + + def test_inject_fills_blank_configured_value(self) -> None: + content = textwrap.dedent( + """ + .. meta:: + :product: + + Title + ===== + """ + ) + updated, changed = inject_metadata_to_content(content, {"product": "{PRODUCT}"}) + self.assertTrue(changed) + fields = get_meta_fields_from_content(updated) + self.assertEqual(fields["product"], "{PRODUCT}") + + +class TestEnsureMetaTagsInFile(unittest.TestCase): + def test_local_auto_inject_clears_configured_fields(self) -> None: + rules = { + "product": MetaRule("warning", "{PRODUCT}"), + "area": MetaRule("error", ""), + } + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "page.rst" + path.write_text("Title\n=====\n", encoding="utf-8") + result = ensure_meta_tags_in_file(path, rules) + self.assertIsNotNone(result) + self.assertEqual(result["mode"], "suggestable") + self.assertIn("area", result["manual_fields"]) + self.assertIn("area", result["error_fields"]) + fields = get_meta_fields_from_content(path.read_text(encoding="utf-8")) + self.assertEqual(fields["product"], "{PRODUCT}") + + +class TestReviewAndExit(unittest.TestCase): + def test_build_review_comment_lists_manual_fields(self) -> None: + rules = { + "area": MetaRule("error", ""), + "experience": MetaRule("warning", ""), + } + results = [ + { + "path": "source/Page.rst", + "mode": "manual_only", + "snippet": "", + "manual_fields": ["area", "experience"], + "warning_fields": ["experience"], + "error_fields": ["area"], + "line": 1, + }, + ] + body = build_review_comment(results, rules) + self.assertIn("area", body) + self.assertIn("required", body) + self.assertIn("experience", body) + + def test_local_exit_nonzero_only_for_error_severity(self) -> None: + warning_config = textwrap.dedent( + """ + meta: + product: + severity: warning + value: "{PRODUCT}" + """ + ).strip() + rules_path = Path(tempfile.mkdtemp()) / "meta.yaml" + rules_path.write_text(warning_config, encoding="utf-8") + with tempfile.TemporaryDirectory() as tmp: + warning_only = Path(tmp) / "warn.rst" + warning_only.write_text("Title\n=====\n", encoding="utf-8") + code = main( + [ + str(warning_only), + "--config", + str(rules_path), + ], + ) + self.assertEqual(code, 0) + + rules_path.write_text(SAMPLE_CONFIG, encoding="utf-8") + with tempfile.TemporaryDirectory() as tmp: + error_file = Path(tmp) / "err.rst" + error_file.write_text("Title\n=====\n", encoding="utf-8") + code = main( + [ + str(error_file), + "--config", + str(rules_path), + ], + ) + self.assertEqual(code, 1) + + +if __name__ == "__main__": + unittest.main() From cf3c70902b12a97411b7f4e9f4bdec140fc640a9 Mon Sep 17 00:00:00 2001 From: Keith Kirkwood Date: Fri, 24 Jul 2026 17:27:34 +0100 Subject: [PATCH 06/23] OPENR-174: Refactoring and cleanup --- tools/ensure_meta_tags.py | 53 +++++++++++++------------- tools/rst_utils.py | 78 +++++++++++++++++++-------------------- 2 files changed, 63 insertions(+), 68 deletions(-) diff --git a/tools/ensure_meta_tags.py b/tools/ensure_meta_tags.py index 3a90dfad7c4..f4f835136b6 100644 --- a/tools/ensure_meta_tags.py +++ b/tools/ensure_meta_tags.py @@ -171,11 +171,10 @@ def _unresolved_fields(content: str, rules: dict[str, MetaRule]) -> list[str]: Unresolved field names in configuration order. """ present = get_meta_fields_from_content(content) - unresolved: list[str] = [] - for name in rules: - if name not in present or not present[name].strip(): - unresolved.append(name) - return unresolved + return [ + name for name in rules + if name not in present or not present[name].strip() + ] def parse_diff_new_side_lines(diff_text: str) -> set[int]: @@ -297,6 +296,17 @@ def _escape_workflow_command_message(message: str) -> str: return message.replace("%", "%25").replace("\r", "%0D").replace("\n", "%0A") +def _emit_annotation(level: str, path: str, fields: list[str], line: int) -> None: + """Print a GitHub Actions workflow annotation for missing meta fields.""" + if not fields: + return + field_list = ", ".join(fields) + message = _escape_workflow_command_message( + f"Missing meta fields: {field_list}", + ) + print(f"::{level} file={path},line={line}::{message}") + + def emit_github_warning(path: str, fields: list[str], line: int) -> None: """ Print a GitHub Actions warning annotation for missing meta fields. @@ -309,13 +319,7 @@ def emit_github_warning(path: str, fields: list[str], line: int) -> None: Returns: None. """ - if not fields: - return - field_list = ", ".join(fields) - message = _escape_workflow_command_message( - f"Missing meta fields: {field_list}", - ) - print(f"::warning file={path},line={line}::{message}") + _emit_annotation("warning", path, fields, line) def emit_github_error(path: str, fields: list[str], line: int) -> None: @@ -330,13 +334,7 @@ def emit_github_error(path: str, fields: list[str], line: int) -> None: Returns: None. """ - if not fields: - return - field_list = ", ".join(fields) - message = _escape_workflow_command_message( - f"Missing meta fields: {field_list}", - ) - print(f"::error file={path},line={line}::{message}") + _emit_annotation("error", path, fields, line) def can_suggest_inline(content: str, pr_lines: set[int]) -> bool: @@ -550,9 +548,7 @@ def build_review_comment( lines.append("```") lines.append("") - manual_results = [ - r for r in results if list(r.get("manual_fields", [])) - ] + manual_results = [r for r in results if r.get("manual_fields")] if manual_results: lines.append( "The following fields must be provided with **non-empty** values in each " @@ -675,11 +671,14 @@ def main(argv: list[str] | None = None) -> int: ) has_results = bool(results) with args.status_file.open("a", encoding="utf-8") as f: - f.write("meta_checked=true\n") - f.write(f"suggestable={'true' if has_suggestable else 'false'}\n") - f.write(f"fallback={'true' if has_fallback else 'false'}\n") - f.write(f"has_results={'true' if has_results else 'false'}\n") - f.write(f"has_errors={'true' if has_errors else 'false'}\n") + for key, flag in ( + ("meta_checked", True), + ("suggestable", has_suggestable), + ("fallback", has_fallback), + ("has_results", has_results), + ("has_errors", has_errors), + ): + f.write(f"{key}={'true' if flag else 'false'}\n") if results: review_body = build_review_comment(results, rules) f.write("comment< tuple[int, int, int, str, str]: +def _find_directive_block(content: str, directive: str) -> tuple[int, int, int, str, str]: """ - Locate the first ``.. meta::`` directive in RST source. + Locate the first ``.. ::`` block in RST source. The directive block consists of the explicit marker line followed by contiguous indented lines; a blank line or a less-indented line ends the @@ -19,43 +19,65 @@ def _find_meta_block(content: str) -> tuple[int, int, int, str, str]: Args: content: The RST file content to search. + directive: Directive name without the ``..`` prefix (e.g. ``meta``). Returns: Tuple of ``(start, marker_end, block_end, inner, indent)``. If no directive is found, ``start``, ``marker_end``, and ``block_end`` are ``-1``, ``inner`` is ``''``, and ``indent`` defaults to three spaces. """ - # Explicit markup + directive name; block body starts on the following line only - match = re.search(r"^\.\.\s+meta::\s*\n", content, re.MULTILINE) + match = re.search( + rf"^\.\.\s+{re.escape(directive)}::\s*\n", + content, + re.MULTILINE, + ) if not match: return -1, -1, -1, "", " " - start = match.start() # Byte index of ``.. meta::`` (for whole-directive splice) - marker_end = match.end() # First character after the marker line's newline - indent = " " # Default field indent when the block is empty or we prepend a new block + start = match.start() + marker_end = match.end() + indent = " " inner_parts: list[str] = [] - consumed = 0 # Length of directive body in ``content`` (may omit final ``\n`` on last line) - remainder = content[marker_end:] # Scan forward only inside this file slice + consumed = 0 + remainder = content[marker_end:] for line in remainder.splitlines(keepends=True): if line.strip() == "": - break # Blank line terminates the directive block + break if not line.startswith((" ", "\t")): - break # Body element at column 0 ends the block + break if not inner_parts: ws_len = len(line) - len(line.lstrip(" \t")) - indent = line[:ws_len] # Reuse the author's indent for new ``:name:`` lines + indent = line[:ws_len] inner_parts.append(line) consumed += len(line) - block_end = marker_end + consumed # Exclusive end of the directive in ``content`` + block_end = marker_end + consumed inner = "".join(inner_parts) - # EOF without ``\n`` yields a last ``splitlines`` element with no newline—append one before new fields if inner and not inner.endswith("\n"): inner += "\n" return start, marker_end, block_end, inner, indent +def _find_meta_block(content: str) -> tuple[int, int, int, str, str]: + """ + Locate the first ``.. meta::`` directive in RST source. + + The directive block consists of the explicit marker line followed by + contiguous indented lines; a blank line or a less-indented line ends the + block (per reStructuredText directive block rules). + + Args: + content: The RST file content to search. + + Returns: + Tuple of ``(start, marker_end, block_end, inner, indent)``. + If no directive is found, ``start``, ``marker_end``, and ``block_end`` + are ``-1``, ``inner`` is ``''``, and ``indent`` defaults to three spaces. + """ + return _find_directive_block(content, "meta") + + def _extract_meta_names_from_block(meta_block_inner: str) -> set[str]: """ Collect field names from the body of a ``.. meta::`` directive. @@ -247,33 +269,7 @@ def _find_short_description_block(content: str) -> tuple[int, int, int, str, str If no directive is found, ``start``, ``marker_end``, and ``block_end`` are ``-1``, ``inner`` is ``''``, and ``indent`` defaults to three spaces. """ - match = re.search(r"^\.\.\s+short-description::\s*\n", content, re.MULTILINE) - if not match: - return -1, -1, -1, "", " " - - start = match.start() - marker_end = match.end() - indent = " " - inner_parts: list[str] = [] - consumed = 0 - remainder = content[marker_end:] - - for line in remainder.splitlines(keepends=True): - if line.strip() == "": - break - if not line.startswith((" ", "\t")): - break - if not inner_parts: - ws_len = len(line) - len(line.lstrip(" \t")) - indent = line[:ws_len] - inner_parts.append(line) - consumed += len(line) - - block_end = marker_end + consumed - inner = "".join(inner_parts) - if inner and not inner.endswith("\n"): - inner += "\n" - return start, marker_end, block_end, inner, indent + return _find_directive_block(content, "short-description") def _short_description_inner_has_content(inner: str) -> bool: From 1a1ff7dae5abfbdc0e976878913939b46bd60b5e Mon Sep 17 00:00:00 2001 From: Keith Kirkwood Date: Mon, 27 Jul 2026 13:59:40 +0100 Subject: [PATCH 07/23] OPENR-174: Some refactoring and tidy up to use Makefile --- .github/workflows/enhance.yml | 46 ++---- Makefile | 15 +- tools/README.md | 56 ++++++-- tools/ensure_meta_tags.py | 200 +++++++++++++++++++++------ tools/tests/test_ensure_meta_tags.py | 60 +++++++- 5 files changed, 286 insertions(+), 91 deletions(-) diff --git a/.github/workflows/enhance.yml b/.github/workflows/enhance.yml index 0a88c871128..67e244e3810 100644 --- a/.github/workflows/enhance.yml +++ b/.github/workflows/enhance.yml @@ -41,37 +41,15 @@ jobs: - name: Ensure documentation metadata id: ensure continue-on-error: true + env: + DIFF_BASE: ${{ github.event.pull_request.base.sha }} run: | - BASE_SHA="${{ github.event.pull_request.base.sha }}" - git fetch origin "$BASE_SHA" - - # Collect changed RST files - mapfile -t changed_rst < <( - git diff --name-only --diff-filter=ACMR \ - "${BASE_SHA}...${{ github.event.pull_request.head.sha }}" \ - -- '*.rst' - ) - - if [ "${#changed_rst[@]}" -eq 0 ]; then - echo "No changed RST files in this pull request." - echo "meta_checked=false" >> "$GITHUB_OUTPUT" - echo "suggestable=false" >> "$GITHUB_OUTPUT" - echo "fallback=false" >> "$GITHUB_OUTPUT" - echo "has_results=false" >> "$GITHUB_OUTPUT" - echo "has_errors=false" >> "$GITHUB_OUTPUT" - exit 0 - fi - - # Check meta tags for changed RST files - python3 .trusted-base/tools/ensure_meta_tags.py \ - --config .trusted-base/tools/meta_tags.yaml \ - --diff-base "$BASE_SHA" \ - --status-file "$GITHUB_OUTPUT" \ - "${changed_rst[@]}" - - echo "Working tree after ensure_meta_tags:" - git status --short -- "${changed_rst[@]}" - git diff -- "${changed_rst[@]}" + set -euo pipefail + git fetch origin "$DIFF_BASE" + make -f .trusted-base/Makefile ensure-meta-tags \ + TOOLS_DIR=.trusted-base/tools \ + DIFF_BASE="$DIFF_BASE" \ + STATUS_FILE="$GITHUB_OUTPUT" - name: Supersede stale meta-tag reviews id: supersede @@ -125,17 +103,17 @@ jobs: - name: Suggest meta tag changes if: >- steps.supersede.outputs.should_post == 'true' - && steps.ensure.outputs.suggestable == 'true' + && steps.ensure.outputs.inline_suggestions == 'true' uses: parkerbxyz/suggest-changes@v3 with: comment: ${{ steps.ensure.outputs.comment }} event: COMMENT - - name: Comment fallback meta tag instructions + - name: Post meta tag review comment if: >- steps.supersede.outputs.should_post == 'true' - && steps.ensure.outputs.fallback == 'true' - && steps.ensure.outputs.suggestable != 'true' + && steps.ensure.outputs.review_comment == 'true' + && steps.ensure.outputs.inline_suggestions != 'true' env: GH_TOKEN: ${{ github.token }} REVIEW_COMMENT: ${{ steps.ensure.outputs.comment }} diff --git a/Makefile b/Makefile index a33ff9ff627..fa82e1afa7a 100644 --- a/Makefile +++ b/Makefile @@ -12,6 +12,10 @@ OPTS =-c . -W # Treat warnings as errors LIVE_HOST ?= 0.0.0.0 LIVE_PORT ?= 2022 +TOOLS_DIR ?= tools +DIFF_BASE ?= +STATUS_FILE ?= + DICTIONARIES := codespell_dictionary.txt codespell_whitelist.txt help: @@ -38,6 +42,15 @@ test-tools: spellcheck: git ls-files '*.md' '*.rst' | xargs codespell --config codespell.cfg +ensure-meta-tags: +ifndef DIFF_BASE + $(error DIFF_BASE is required) +endif + $(PYTHON) $(TOOLS_DIR)/ensure_meta_tags.py \ + --config $(TOOLS_DIR)/meta_tags.yaml \ + --diff-base $(DIFF_BASE) \ + $(if $(STATUS_FILE),--status-file $(STATUS_FILE)) + check-dictionaries: @echo "Checking dictionaries..." @for dict in $(DICTIONARIES); do \ @@ -66,4 +79,4 @@ linkcheck: serve: sphinx-autobuild --host $(LIVE_HOST) --port $(LIVE_PORT) -c . $(SOURCE) $(OUT)/html -.PHONY: help Makefile multiversion test test-tools linkcheck serve lint spellcheck check-dictionaries sort-dictionaries +.PHONY: help Makefile multiversion test test-tools linkcheck serve lint spellcheck check-dictionaries sort-dictionaries ensure-meta-tags diff --git a/tools/README.md b/tools/README.md index 6c1b9b5f81c..b04ae416d4f 100644 --- a/tools/README.md +++ b/tools/README.md @@ -76,11 +76,20 @@ Multiple files: python3 tools/ensure_meta_tags.py source/Topic/A.rst source/Topic/B.rst ``` +Pull request scope (discover changed ``.rst`` files with a three-dot diff against a base commit): + +```bash +make ensure-meta-tags DIFF_BASE=origin/rolling +``` + +The repository [`Makefile`](../Makefile) target runs `ensure_meta_tags.py` with `--diff-base` and no explicit paths; changed ACMR ``*.rst`` files are discovered via ``git diff``. For CI-style output locally, pass ``STATUS_FILE=/path/to/file``. + Options: +- `paths` — optional; when omitted, `--diff-base` is required and changed ``.rst`` files are discovered automatically - `--config PATH` — YAML config file (default: `tools/meta_tags.yaml`) -- `--diff-base SHA` — PR base commit; only write edits that overlap the PR diff (CI) -- `--status-file PATH` — write `meta_checked`, `suggestable`, `fallback`, `has_results`, `has_errors`, and the review comment for CI; when issues remain, emits annotations and exits `1` (the ensure step uses `continue-on-error`) +- `--diff-base SHA` — PR base commit; limits on-disk writes to lines in the PR diff (inline suggestions); files that need a copy-paste or manual field list use a review comment instead +- `--status-file PATH` — write `meta_checked`, `inline_suggestions`, `review_comment`, `has_results`, `has_errors`, and the review comment body for CI; when issues remain, emits annotations and exits `1` (the ensure step uses `continue-on-error`) - `-v` / `--verbose` — enable debug logging **Exit codes:** With `--status-file` (CI), exit `1` when any issues remain. Locally, exit `1` only when **error**-severity fields are still unresolved; warning-only issues exit `0` after applying automatic fixes. @@ -116,13 +125,36 @@ Fields such as `area` with an empty `value` in the config are listed in the revi The workflow [`.github/workflows/enhance.yml`](../.github/workflows/enhance.yml) runs on every pull request (including from forks): 1. Checks out the PR’s `.rst` files as untrusted data -2. Checks out the base branch into `.trusted-base/` for the script and config -3. Runs the trusted `ensure_meta_tags.py` with `--diff-base` against changed RST files +2. Checks out the base branch into `.trusted-base/` for the trusted Makefile, script, and config +3. Runs `make -f .trusted-base/Makefile ensure-meta-tags` (with `TOOLS_DIR=.trusted-base/tools`, `DIFF_BASE`, and `STATUS_FILE=$GITHUB_OUTPUT`) so discovery and metadata checks use trusted code only 4. Emits per-file warning/error annotations and soft-fails the ensure step when metadata is still missing -5. Posts inline suggestions and/or a review comment +5. Posts inline suggestions (`suggest-changes`) and/or a review comment (`Post meta tag review comment`) 6. **Fails the job** if any **error**-severity metadata remains (`Enforce required metadata`) -Priority: **inline suggestions wherever GitHub allows them**. Copy-paste blocks and manual field lists in the review body are fallbacks. +Priority: **inline suggestions wherever GitHub allows them**. Copy-paste blocks and manual field lists are delivered via a pull request review comment when inline suggestions are not possible. + +### Per-file modes and CI outputs + +Each changed `.rst` file is classified with an internal **mode**: + +| Mode | Meaning | +|------|---------| +| `suggestable` | Configured values were written to the working tree for inline “Commit suggestion” | +| `snippet` | Configured values could not be written inline; the review includes a copy-paste `.. meta::` block | +| `manual_fields` | Only fields with empty `value` in the config are missing; the review lists field names | + +The script writes **CI outputs** (for example `$GITHUB_OUTPUT`) that describe which workflow steps to run: + +| Output | Meaning | +|--------|---------| +| `meta_checked` | At least one changed `.rst` was in scope (`false` when discovery finds no changed RST; supersede and review steps are skipped) | +| `inline_suggestions` | Run [`suggest-changes`](https://github.com/marketplace/actions/suggest-changes-action) | +| `review_comment` | Post the generated review body with `gh pr review` when inline suggestions are not used (covers `snippet` and `manual_fields` files) | +| `has_results` | Metadata issues remain (used to decide whether to post a new review after superseding stale ones) | +| `has_errors` | Unresolved **error**-severity fields (triggers the final enforce step) | +| `comment` | Full stamped review body (multiline) for suggest-changes or `gh pr review` | + +When `inline_suggestions` is `true`, the workflow runs suggest-changes even if `review_comment` is also `true` (mixed per-file modes). A separate review comment is posted only when `review_comment` is `true` and `inline_suggestions` is not. The ensure step uses `continue-on-error: true`, so warning-only gaps do not fail the job. Error-severity gaps (for example `area`) still fail the workflow after contributors receive review feedback. @@ -143,8 +175,8 @@ When metadata is missing or blank: | Warning-only gaps | Soft warning | Warnings | Suggestions and/or manual list | Success | | Error gaps (e.g. missing `area`) | Soft warning | Errors (and warnings) | Suggestions and/or manual list | **Failure** after enforce step | | Auto-fix in diff | Soft warning | As above | Inline “Commit suggestion” | As per severity | -| Auto-fix outside diff | Soft warning | As above | Copy-paste `.. meta::` for configured values | As per severity | -| Manual-only fields | Soft warning | As above | Field list with required/warning labels | As per severity | +| Auto-fix outside diff (`snippet`) | Soft warning | As above | Copy-paste `.. meta::` for configured values | As per severity | +| Manual fields only (`manual_fields`) | Soft warning | As above | Field list with required/warning labels | As per severity | Reviews, annotations, and the soft-failed ensure step appear in different parts of the GitHub UI (Conversation, Files changed, Checks); only error-severity issues fail the overall workflow. @@ -156,10 +188,10 @@ GitHub only allows review suggestions on [lines already in the pull request diff |-----------|----------------| | Missing configured values; existing `.. meta::` overlaps the PR diff | Write append/fill to the working tree → inline suggestion via [`suggest-changes`](https://github.com/marketplace/actions/suggest-changes-action) | | No `.. meta::`; top of file overlaps the PR diff | Insert at top → inline suggestion | -| Automatic edit does **not** overlap the PR diff | No unsuggestable write; review includes a copy-paste block for configured values | -| Only manual fields (empty `value` in config) | Review lists fields; no placeholder injection | +| Automatic edit does **not** overlap the PR diff (`snippet` mode) | No inline write; review includes a copy-paste block for configured values | +| Only manual fields (empty `value` in config, `manual_fields` mode) | Review lists fields; no placeholder injection | | All configured fields present and non-empty | No action | -| No changed `.rst` files in the PR | Workflow exits early | +| No changed `.rst` files in the PR | No check; `meta_checked=false`; supersede/review steps skipped | ### Superseding outdated reviews @@ -169,7 +201,7 @@ Each bot review body includes a hidden marker (``) 2. A fresh review is posted only when work still remains. 3. When all issues are fixed, stale reviews are minimized and no new “all clear” comment is posted. -The workflow uses [`pull_request_target`](https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request_target) so the default `GITHUB_TOKEN` can post review suggestions on fork PRs. The workflow definition itself is taken from the repository **default branch**; the trusted script and config come from the PR **base** branch. See [Mitigating the risks of untrusted code checkout](https://docs.github.com/en/actions/reference/security/secure-use#mitigating-the-risks-of-untrusted-code-checkout). +The workflow uses [`pull_request_target`](https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request_target) so the default `GITHUB_TOKEN` can post review suggestions on fork PRs. The workflow definition itself is taken from the repository **default branch**; the trusted Makefile, script, and config come from the PR **base** branch checkout at `.trusted-base/`. See [Mitigating the risks of untrusted code checkout](https://docs.github.com/en/actions/reference/security/secure-use#mitigating-the-risks-of-untrusted-code-checkout). ## Tests diff --git a/tools/ensure_meta_tags.py b/tools/ensure_meta_tags.py index f4f835136b6..062826cc6f7 100644 --- a/tools/ensure_meta_tags.py +++ b/tools/ensure_meta_tags.py @@ -250,6 +250,106 @@ def pr_diff_lines_for_file(diff_base: str, path: Path) -> set[int]: return parse_diff_new_side_lines(result.stdout) +def changed_rst_paths(diff_base: str) -> list[Path]: + """ + List ``.rst`` files changed between a pull request base and ``HEAD``. + + Args: + diff_base: Base commit SHA used for the three-dot comparison. + + Returns: + Repository-relative paths for added, copied, modified, or renamed + ``.rst`` files, or an empty list if ``git diff`` fails. + """ + result = subprocess.run( + [ + "git", + "diff", + "--name-only", + "--diff-filter=ACMR", + f"{diff_base}...HEAD", + "--", + "*.rst", + ], + check=False, + capture_output=True, + text=True, + ) + if result.returncode not in (0, 1): + logger.warning( + "git diff failed listing changed RST files (exit %s): %s", + result.returncode, + result.stderr.strip(), + ) + return [] + paths: list[Path] = [] + for line in result.stdout.splitlines(): + stripped = line.strip() + if stripped: + paths.append(Path(stripped)) + return paths + + +def _log_working_tree_summary(paths: list[Path]) -> None: + """Log ``git status`` and ``git diff`` for processed RST paths.""" + if not paths: + return + path_args = [str(p) for p in paths] + logger.info("Working tree after ensure_meta_tags:") + status = subprocess.run( + ["git", "status", "--short", "--", *path_args], + check=False, + capture_output=True, + text=True, + ) + if status.stdout.strip(): + for line in status.stdout.splitlines(): + logger.info("%s", line) + else: + logger.info("(no changes)") + diff = subprocess.run( + ["git", "diff", "--", *path_args], + check=False, + capture_output=True, + text=True, + ) + if diff.stdout.strip(): + for line in diff.stdout.splitlines(): + logger.info("%s", line) + + +def _write_ci_status_file( + status_file: Path, + *, + meta_checked: bool, + results: list[dict[str, object]], + rules: dict[str, MetaRule], + has_errors: bool, +) -> None: + """Append GitHub Actions output flags and optional review comment.""" + has_inline_suggestions = any(r["mode"] == "suggestable" for r in results) + has_review_comment = any( + r["mode"] in ("snippet", "manual_fields") for r in results + ) + has_results = bool(results) + with status_file.open("a", encoding="utf-8") as f: + for key, flag in ( + ("meta_checked", meta_checked), + ("inline_suggestions", has_inline_suggestions), + ("review_comment", has_review_comment), + ("has_results", has_results), + ("has_errors", has_errors), + ): + f.write(f"{key}={'true' if flag else 'false'}\n") + if results: + review_body = build_review_comment(results, rules) + f.write("comment< bool: """ Test whether an inclusive line span overlaps pull-request lines. @@ -412,10 +512,10 @@ def ensure_meta_tags_in_file( if auto_metadata: if pr_lines is not None and not can_suggest_inline(content, pr_lines): - mode = "fallback" + mode = "snippet" snippet = format_meta_block(rules, auto_fields) logger.info( - "%s: missing %s but auto-fix is outside the PR diff; fallback review only", + "%s: missing %s but auto-fix is outside the PR diff; snippet review only", path, ", ".join(auto_fields), ) @@ -444,7 +544,7 @@ def ensure_meta_tags_in_file( error_fields = _severity_fields(still_unresolved, rules, "error") if mode is None: - mode = "manual_only" + mode = "manual_fields" return { "path": path_str, @@ -516,9 +616,9 @@ def build_review_comment( Returns: A stamped Markdown review body containing the relevant instructions. """ - suggestable = [r for r in results if r["mode"] == "suggestable"] - fallback = [r for r in results if r["mode"] == "fallback"] - manual_only = [r for r in results if r["mode"] == "manual_only"] + inline_modes = [r for r in results if r["mode"] == "suggestable"] + snippet_modes = [r for r in results if r["mode"] == "snippet"] + manual_fields_modes = [r for r in results if r["mode"] == "manual_fields"] lines = [ "This pull request is missing configured documentation metadata " @@ -526,14 +626,14 @@ def build_review_comment( "", ] - if suggestable: + if inline_modes: lines.append( "Please **review and commit the inline suggestions**. They add configured " "default values in place so metadata stays complete.", ) lines.append("") - if fallback: + if snippet_modes: lines.append( "GitHub can only attach suggestions to lines already in the pull request " "diff, so the following files could not get an inline suggestion for " @@ -541,7 +641,7 @@ def build_review_comment( "(or append the listed fields to an existing `.. meta::` block):", ) lines.append("") - for result in fallback: + for result in snippet_modes: lines.append(f"**`{result['path']}`**") lines.append("```rst") lines.append(str(result["snippet"]).rstrip()) @@ -560,7 +660,7 @@ def build_review_comment( lines.append(f"**`{result['path']}`**: {_field_list_markdown(manual, rules)}") lines.append("") - if manual_only and not suggestable and not fallback: + if manual_fields_modes and not inline_modes and not snippet_modes: lines.append( "Add or complete a `.. meta::` block at the top of each affected file.", ) @@ -593,8 +693,11 @@ def main(argv: list[str] | None = None) -> int: ) parser.add_argument( "paths", - nargs="+", - help="One or more .rst file paths to check", + nargs="*", + help=( + "One or more .rst file paths to check; when omitted, " + "--diff-base must be set to discover changed files" + ), ) parser.add_argument( "--config", @@ -606,15 +709,15 @@ def main(argv: list[str] | None = None) -> int: "--diff-base", help=( "Git commit SHA for the pull request base. When set, only writes edits " - "that overlap the PR diff; other files become review-comment fallbacks." + "that overlap the PR diff; other files receive a review comment instead." ), ) parser.add_argument( "--status-file", type=Path, help=( - "Write meta_checked, suggestable, fallback, has_results, has_errors, " - "and the review comment for CI" + "Write meta_checked, inline_suggestions, review_comment, has_results, " + "has_errors, and the review comment body for CI" ), ) parser.add_argument( @@ -630,8 +733,32 @@ def main(argv: list[str] | None = None) -> int: format="%(levelname)s: %(message)s", ) + if not args.paths and not args.diff_base: + parser.error("provide at least one .rst path or set --diff-base to discover changes") + rules = load_meta_config(args.config) - rst_paths = _collect_rst_paths(args.paths) + checked_pull_request_rst = False + + if not args.paths: + assert args.diff_base is not None + discovered = changed_rst_paths(args.diff_base) + if not discovered: + logger.info("No changed RST files in this pull request.") + if args.status_file is not None: + _write_ci_status_file( + args.status_file, + meta_checked=False, + results=[], + rules=rules, + has_errors=False, + ) + return 0 + checked_pull_request_rst = True + rst_paths = _collect_rst_paths([str(p) for p in discovered]) + else: + checked_pull_request_rst = True + rst_paths = _collect_rst_paths(args.paths) + results: list[dict[str, object]] = [] if not rst_paths: @@ -645,16 +772,17 @@ def main(argv: list[str] | None = None) -> int: if result is not None: results.append(result) - suggestable_count = sum(1 for r in results if r["mode"] == "suggestable") - fallback_count = sum(1 for r in results if r["mode"] == "fallback") - manual_count = sum(1 for r in results if r["mode"] == "manual_only") + inline_count = sum(1 for r in results if r["mode"] == "suggestable") + snippet_count = sum(1 for r in results if r["mode"] == "snippet") + manual_fields_count = sum(1 for r in results if r["mode"] == "manual_fields") logger.info( - "Processed %d file(s): %d suggestable, %d fallback, %d manual-only", + "Processed %d file(s): %d inline, %d snippet, %d manual_fields", len(rst_paths), - suggestable_count, - fallback_count, - manual_count, + inline_count, + snippet_count, + manual_fields_count, ) + _log_working_tree_summary(rst_paths) for result in results: path = str(result["path"]) @@ -665,27 +793,13 @@ def main(argv: list[str] | None = None) -> int: has_errors = any(result["error_fields"] for result in results) if args.status_file is not None: - has_suggestable = any(r["mode"] == "suggestable" for r in results) - has_fallback = any( - r["mode"] in ("fallback", "manual_only") for r in results + _write_ci_status_file( + args.status_file, + meta_checked=checked_pull_request_rst, + results=results, + rules=rules, + has_errors=has_errors, ) - has_results = bool(results) - with args.status_file.open("a", encoding="utf-8") as f: - for key, flag in ( - ("meta_checked", True), - ("suggestable", has_suggestable), - ("fallback", has_fallback), - ("has_results", has_results), - ("has_errors", has_errors), - ): - f.write(f"{key}={'true' if flag else 'false'}\n") - if results: - review_body = build_review_comment(results, rules) - f.write("comment< None: results = [ { "path": "source/Page.rst", - "mode": "manual_only", + "mode": "manual_fields", "snippet": "", "manual_fields": ["area", "experience"], "warning_fields": ["experience"], @@ -193,5 +195,61 @@ def test_local_exit_nonzero_only_for_error_severity(self) -> None: self.assertEqual(code, 1) +class TestChangedRstPaths(unittest.TestCase): + def test_parses_git_diff_output(self) -> None: + completed = mock.Mock(returncode=0, stdout="source/A.rst\nsource/B.rst\n", stderr="") + with mock.patch("ensure_meta_tags.subprocess.run", return_value=completed) as run: + paths = changed_rst_paths("abc123") + self.assertEqual(paths, [Path("source/A.rst"), Path("source/B.rst")]) + run.assert_called_once() + call_args = run.call_args[0][0] + self.assertEqual(call_args[:4], ["git", "diff", "--name-only", "--diff-filter=ACMR"]) + self.assertIn("abc123...HEAD", call_args[4]) + + +class TestMainDiscovery(unittest.TestCase): + def test_requires_paths_or_diff_base(self) -> None: + with tempfile.NamedTemporaryFile("w", suffix=".yaml", delete=False) as handle: + handle.write(SAMPLE_CONFIG) + config_path = Path(handle.name) + try: + with self.assertRaises(SystemExit) as ctx: + main(["--config", str(config_path)]) + self.assertEqual(ctx.exception.code, 2) + finally: + config_path.unlink() + + def test_empty_discovery_writes_meta_checked_false(self) -> None: + with tempfile.NamedTemporaryFile("w", suffix=".yaml", delete=False) as handle: + handle.write(SAMPLE_CONFIG) + config_path = Path(handle.name) + try: + with tempfile.NamedTemporaryFile("w", delete=False) as status_handle: + status_path = Path(status_handle.name) + try: + with mock.patch( + "ensure_meta_tags.changed_rst_paths", + return_value=[], + ): + code = main( + [ + "--config", + str(config_path), + "--diff-base", + "base-sha", + "--status-file", + str(status_path), + ], + ) + self.assertEqual(code, 0) + status_text = status_path.read_text(encoding="utf-8") + self.assertIn("meta_checked=false", status_text) + self.assertIn("has_errors=false", status_text) + finally: + status_path.unlink() + finally: + config_path.unlink() + + if __name__ == "__main__": unittest.main() From ad3cbbc5362fb3e298fa0e9f9171fd5e1058340a Mon Sep 17 00:00:00 2001 From: Keith Kirkwood Date: Mon, 27 Jul 2026 14:40:03 +0100 Subject: [PATCH 08/23] OPENR-174: Move more into makefile and supporting sh script to keep workflow relatively clean --- .github/workflows/enhance.yml | 45 +----- Makefile | 29 +++- tools/README.md | 233 ++++++++++++++++++---------- tools/ensure_meta_tags.py | 97 ++++++++++-- tools/supersede_meta_tag_reviews.sh | 42 +++++ 5 files changed, 308 insertions(+), 138 deletions(-) create mode 100755 tools/supersede_meta_tag_reviews.sh diff --git a/.github/workflows/enhance.yml b/.github/workflows/enhance.yml index 67e244e3810..1984854165e 100644 --- a/.github/workflows/enhance.yml +++ b/.github/workflows/enhance.yml @@ -57,48 +57,13 @@ jobs: env: GH_TOKEN: ${{ github.token }} HAS_RESULTS: ${{ steps.ensure.outputs.has_results }} + PR_NUMBER: ${{ github.event.pull_request.number }} + REPOSITORY: ${{ github.repository }} run: | set -euo pipefail - PR="${{ github.event.pull_request.number }}" - REPO="${{ github.repository }}" - MARKER="ros2-meta-tags-ensure" - - echo "Marking prior meta-tag reviews as outdated." - - # Find all previous PR reviews containing the meta-tag marker, - # and extract their node IDs - mapfile -t review_ids < <( - gh api "repos/${REPO}/pulls/${PR}/reviews" --paginate \ - --jq ".[] | select(.body != null and (.body | contains(\"${MARKER}\"))) | .node_id" - ) - - # Prepare a GraphQL mutation string to minimize comments (mark them as outdated) - minimize_query=' - mutation($subjectId: ID!) { - minimizeComment(input: { - subjectId: $subjectId - classifier: OUTDATED - }) { - minimizedComment { isMinimized } - } - } - ' - - # Run the GraphQL mutation for each eligible review - for node_id in "${review_ids[@]}"; do - [ -z "$node_id" ] && continue - gh api graphql \ - -f query="${minimize_query}" \ - -f subjectId="${node_id}" \ - || true - done - - if [ "$HAS_RESULTS" = "true" ]; then - echo "should_post=true" >> "$GITHUB_OUTPUT" - else - echo "All meta-tag issues resolved; not posting a new review." - echo "should_post=false" >> "$GITHUB_OUTPUT" - fi + make -f .trusted-base/Makefile supersede-meta-tag-reviews \ + TOOLS_DIR=.trusted-base/tools \ + STATUS_FILE="$GITHUB_OUTPUT" - name: Suggest meta tag changes if: >- diff --git a/Makefile b/Makefile index fa82e1afa7a..caceb88580d 100644 --- a/Makefile +++ b/Makefile @@ -7,14 +7,18 @@ PYTHON := python3 ifeq ($(OS),Windows_NT) PYTHON := python endif +BASH := bash BUILD = $(PYTHON) -m sphinx OPTS =-c . -W # Treat warnings as errors LIVE_HOST ?= 0.0.0.0 LIVE_PORT ?= 2022 -TOOLS_DIR ?= tools -DIFF_BASE ?= -STATUS_FILE ?= +TOOLS_DIR ?= tools +DIFF_BASE ?= +STATUS_FILE ?= +PR_NUMBER ?= +REPOSITORY ?= +HAS_RESULTS ?= DICTIONARIES := codespell_dictionary.txt codespell_whitelist.txt @@ -45,11 +49,26 @@ spellcheck: ensure-meta-tags: ifndef DIFF_BASE $(error DIFF_BASE is required) +endif +ifndef STATUS_FILE + $(error STATUS_FILE is required) endif $(PYTHON) $(TOOLS_DIR)/ensure_meta_tags.py \ --config $(TOOLS_DIR)/meta_tags.yaml \ --diff-base $(DIFF_BASE) \ - $(if $(STATUS_FILE),--status-file $(STATUS_FILE)) + --status-file $(STATUS_FILE) + +supersede-meta-tag-reviews: +ifndef PR_NUMBER + $(error PR_NUMBER is required) +endif +ifndef REPOSITORY + $(error REPOSITORY is required) +endif +ifndef HAS_RESULTS + $(error HAS_RESULTS is required) +endif + $(BASH) $(TOOLS_DIR)/supersede_meta_tag_reviews.sh check-dictionaries: @echo "Checking dictionaries..." @@ -79,4 +98,4 @@ linkcheck: serve: sphinx-autobuild --host $(LIVE_HOST) --port $(LIVE_PORT) -c . $(SOURCE) $(OUT)/html -.PHONY: help Makefile multiversion test test-tools linkcheck serve lint spellcheck check-dictionaries sort-dictionaries ensure-meta-tags +.PHONY: help Makefile multiversion test test-tools linkcheck serve lint spellcheck check-dictionaries sort-dictionaries ensure-meta-tags supersede-meta-tag-reviews diff --git a/tools/README.md b/tools/README.md index b04ae416d4f..94971c570c8 100644 --- a/tools/README.md +++ b/tools/README.md @@ -2,18 +2,23 @@ Helpers for ensuring reStructuredText (`.rst`) metadata on documentation pull requests. -## Layout +--- -| File | Purpose | -|------|---------| -| [`rst_utils.py`](rst_utils.py) | Regex-based read/write of `.. meta::` and `.. short-description::` directives | -| [`meta_tags.yaml`](meta_tags.yaml) | Metadata rules (severity and optional default values) | -| [`ensure_meta_tags.py`](ensure_meta_tags.py) | CLI that checks and fixes metadata from the config | -| [`tests/`](tests/) | Unit tests for the tools in this directory | +## User guide + +Information for documentation contributors creating or updating `.rst` files. -## Configuration +### Prerequisites -[`meta_tags.yaml`](meta_tags.yaml) defines every `.. meta::` field the script checks. Each entry has: +| Component | Used by | +|-----------|---------| +| [PyYAML](https://pyyaml.org/) (`pip install pyyaml`) | [`ensure_meta_tags.py`](ensure_meta_tags.py) and unit tests in [`tests/`](tests/) | +| Git (with a usable `HEAD` and refs for your base commit) | PR-scope discovery and diff overlap checks (`--diff-base`) | +| [GitHub CLI](https://cli.github.com/) (`gh`) and `jq` | [`supersede_meta_tag_reviews.sh`](supersede_meta_tag_reviews.sh) only (Enhance workflow on `ubuntu-24.04`; optional for local supersede testing) | + +### Metadata configuration + +[`meta_tags.yaml`](meta_tags.yaml) defines every `.. meta::` field checked by the tooling. Each entry has: - **`severity`**: `warning` (advisory; soft-fails the ensure step in CI) or `error` (fails the workflow after reviews are posted). - **`value`**: default text to inject when the field is missing or blank. Leave empty when the contributor must supply a non-empty value. @@ -37,32 +42,20 @@ meta: value: ``` -Add a new key under `meta` to extend coverage without changing Python code. The script applies every rule in the file. - `{PRODUCT}` and `{DISTRO}` are Sphinx substitution macros expanded at build time from [`conf.py`](../conf.py). -### Severity behaviour +#### Severity behaviour | Severity | Missing or blank field | CI ensure step | Workflow job | |----------|------------------------|----------------|--------------| | `warning` | Annotation + review | Soft warning (`continue-on-error`) | Succeeds | | `error` | Error annotation + review | Soft warning (same step) | **Fails** on final enforce step | -## `rst_utils.py` - -Low-level utilities for locating and editing Sphinx directives in RST source: - -- **`get_meta_fields_from_content`** — field names and values in the first `.. meta::` block -- **`get_meta_names_from_content`** — field names only -- **`inject_metadata_to_content`** — add missing fields or fill blank values; never overwrites non-empty contributor values - -The module also contains helpers for `.. short-description::` directives for future use. +### Checking metadata locally -## `ensure_meta_tags.py` +`ensure_meta_tags.py` checks `.rst` files against `meta_tags.yaml`. Fields with a configured `value` are added or filled automatically when the edit can be suggested or applied locally. Fields with an empty `value` must be completed manually in the `.. meta::` block. -Checks each given `.rst` file against `meta_tags.yaml`. Fields with a configured `value` are added or filled automatically when the edit can be suggested or applied locally. Fields with an empty `value` must be completed manually in the `.. meta::` block. - -### Usage +#### Usage From the repository root: @@ -76,25 +69,26 @@ Multiple files: python3 tools/ensure_meta_tags.py source/Topic/A.rst source/Topic/B.rst ``` -Pull request scope (discover changed ``.rst`` files with a three-dot diff against a base commit): +Pull request scope (discovers changed `ACMR` `*.rst` files via `git diff`; requires Makefile variables `DIFF_BASE` and `STATUS_FILE`): ```bash -make ensure-meta-tags DIFF_BASE=origin/rolling +make ensure-meta-tags DIFF_BASE=origin/rolling STATUS_FILE=/tmp/meta-tags-out.txt ``` -The repository [`Makefile`](../Makefile) target runs `ensure_meta_tags.py` with `--diff-base` and no explicit paths; changed ACMR ``*.rst`` files are discovered via ``git diff``. For CI-style output locally, pass ``STATUS_FILE=/path/to/file``. +PR scope without Make (optional `--status-file`; exit codes follow local rules when omitted): + +```bash +python3 tools/ensure_meta_tags.py --diff-base origin/rolling +python3 tools/ensure_meta_tags.py --diff-base "$(git merge-base HEAD origin/rolling)" --status-file /tmp/out.txt +``` -Options: +For day-to-day editing of known files, pass paths explicitly and omit `--status-file` so the tool exits `1` only when **error**-severity fields remain. -- `paths` — optional; when omitted, `--diff-base` is required and changed ``.rst`` files are discovered automatically -- `--config PATH` — YAML config file (default: `tools/meta_tags.yaml`) -- `--diff-base SHA` — PR base commit; limits on-disk writes to lines in the PR diff (inline suggestions); files that need a copy-paste or manual field list use a review comment instead -- `--status-file PATH` — write `meta_checked`, `inline_suggestions`, `review_comment`, `has_results`, `has_errors`, and the review comment body for CI; when issues remain, emits annotations and exits `1` (the ensure step uses `continue-on-error`) -- `-v` / `--verbose` — enable debug logging +#### Exit codes -**Exit codes:** With `--status-file` (CI), exit `1` when any issues remain. Locally, exit `1` only when **error**-severity fields are still unresolved; warning-only issues exit `0` after applying automatic fixes. +With `--status-file` (CI), exit `1` when any issues remain. Locally, exit `1` only when **error**-severity fields are still unresolved; warning-only issues exit `0` after applying automatic fixes. -### Example (configured values) +#### Example (configured values) Before: @@ -105,7 +99,7 @@ My Article Some content. ``` -After a local run (new `.. meta::` at the top of the file): +After a local run (new `.. meta::` added at the top of the file): ```rst .. meta:: @@ -120,20 +114,109 @@ Some content. Fields such as `area` with an empty `value` in the config are listed in the review for manual completion; they are not given placeholder text. -## Continuous integration +### Contributor CI experience + +When you open or update a pull request, CI automatically checks metadata on all modified `.rst` files. + +#### Contributor experience overview + +| Situation | Ensure step | Annotations | Pull request review | Job result | +|-----------|-------------|-------------|---------------------|------------| +| All fields resolved | Green | None | None (stale bot reviews cleared) | Success | +| Warning-only gaps | Soft warning | Warnings | Suggestions and/or manual list | Success | +| Error gaps (e.g. missing `area`) | Soft warning | Errors (and warnings) | Suggestions and/or manual list | **Failure** after enforce step | +| Auto-fix in diff | Soft warning | As above | Inline “Commit suggestion” | As per severity | +| Auto-fix outside diff (`snippet`) | Soft warning | As above | Copy-paste `.. meta::` for configured values | As per severity | +| Manual fields only (`manual_fields`) | Soft warning | As above | Field list with required/warning labels | As per severity | + +Reviews, annotations, and the soft-failed ensure step appear in different parts of the GitHub UI (Conversation, Files changed, Checks); only error-severity issues fail the overall workflow. + +#### When inline suggestions appear + +GitHub only allows review suggestions on [lines already in the pull request diff](https://github.com/marketplace/actions/suggest-changes-action). The script compares each automatic edit to that diff: + +| Situation | What happens | +|-----------|----------------| +| Missing configured values; existing `.. meta::` overlaps the PR diff | Write append/fill to the working tree → inline suggestion via [`suggest-changes`](https://github.com/marketplace/actions/suggest-changes-action) | +| No `.. meta::`; top of file overlaps the PR diff | Insert at top → inline suggestion | +| Automatic edit does **not** overlap the PR diff (`snippet` mode) | No inline write; review includes a copy-paste block for configured values | +| Only manual fields (empty `value` in config, `manual_fields` mode) | Review lists fields; no placeholder injection | +| All configured fields present and non-empty | No action | +| No changed `.rst` files in the PR | No check; `meta_checked=false`; supersede/review steps skipped | + +--- + +## Developer guidance + +Information for maintainers and developers working on or extending the metadata tooling and CI workflows. + +### Repository layout + +| File | Purpose | +|------|---------| +| [`rst_utils.py`](rst_utils.py) | Regex-based read/write of `.. meta::` and `.. short-description::` directives | +| [`meta_tags.yaml`](meta_tags.yaml) | Metadata rules (severity and optional default values) | +| [`ensure_meta_tags.py`](ensure_meta_tags.py) | CLI that checks and fixes metadata from the config | +| [`supersede_meta_tag_reviews.sh`](supersede_meta_tag_reviews.sh) | Minimise outdated bot PR reviews and set `should_post` for CI | +| [`tests/`](tests/) | Unit tests for the tools in this directory | + +### Code modules + +#### `rst_utils.py` + +Low-level utilities for locating and editing Sphinx directives in RST source: + +- **`get_meta_fields_from_content`** — field names and values in the first `.. meta::` block +- **`get_meta_names_from_content`** — field names only +- **`inject_metadata_to_content`** — add missing fields or fill blank values; never overwrites non-empty contributor values + +The module also contains helpers for `.. short-description::` directives for future use. + +#### Extending configuration + +Add a new key under `meta` in [`meta_tags.yaml`](meta_tags.yaml) to extend coverage without changing Python code. The script applies every rule in the file. + +### CLI options & Makefile targets + +#### Options (`ensure_meta_tags.py`) + +- `paths` — optional; when omitted, `--diff-base` is required and changed `.rst` files are discovered automatically +- `--config PATH` — YAML config file (default: `tools/meta_tags.yaml`) +- `--diff-base SHA` — PR base commit; limits on-disk writes to lines in the PR diff (inline suggestions); files that need a copy-paste or manual field list use a review comment instead +- `--status-file PATH` — write `meta_checked`, `inline_suggestions`, `review_comment`, `has_results`, `has_errors`, and the review comment body for CI; when issues remain, emits annotations and exits `1` (the ensure step uses `continue-on-error`) +- `-v` / `--verbose` — enable debug logging + +#### Makefile targets (metadata CI) + +Both targets live in the repository root [`Makefile`](../Makefile). CI invokes them with `make -f .trusted-base/Makefile …` so recipes run from the PR **base** branch, not the PR head. + +| Target | Required variables | Purpose | +|--------|-------------------|---------| +| `ensure-meta-tags` | `DIFF_BASE`, `STATUS_FILE`; optional `TOOLS_DIR` (default `tools`) | Discover changed RST, run metadata check, append CI outputs | +| `supersede-meta-tag-reviews` | `PR_NUMBER`, `REPOSITORY`, `HAS_RESULTS`, `STATUS_FILE` (or `GITHUB_OUTPUT`); optional `TOOLS_DIR` | Minimise stamped bot reviews; append `should_post` | -The workflow [`.github/workflows/enhance.yml`](../.github/workflows/enhance.yml) runs on every pull request (including from forks): +Environment for `supersede-meta-tag-reviews` (set by the workflow or locally): `GH_TOKEN`, plus `PR_NUMBER`, `REPOSITORY`, and `HAS_RESULTS` from the ensure step’s `has_results` output. -1. Checks out the PR’s `.rst` files as untrusted data -2. Checks out the base branch into `.trusted-base/` for the trusted Makefile, script, and config -3. Runs `make -f .trusted-base/Makefile ensure-meta-tags` (with `TOOLS_DIR=.trusted-base/tools`, `DIFF_BASE`, and `STATUS_FILE=$GITHUB_OUTPUT`) so discovery and metadata checks use trusted code only -4. Emits per-file warning/error annotations and soft-fails the ensure step when metadata is still missing -5. Posts inline suggestions (`suggest-changes`) and/or a review comment (`Post meta tag review comment`) -6. **Fails the job** if any **error**-severity metadata remains (`Enforce required metadata`) +### Continuous integration architecture -Priority: **inline suggestions wherever GitHub allows them**. Copy-paste blocks and manual field lists are delivered via a pull request review comment when inline suggestions are not possible. +The workflow [`.github/workflows/enhance.yml`](../.github/workflows/enhance.yml) runs on **`pull_request_target`** when a pull request is **opened**, **synchronised**, or **reopened** (including from forks). That event type allows the default `GITHUB_TOKEN` to post review suggestions on fork PRs; the workflow file on the repository **default branch** defines the job, while trusted tooling comes from the PR **base** branch (see [Security](#security) below). -### Per-file modes and CI outputs +The Enhance workflow installs PyYAML in the job; it does not install the full documentation `requirements.txt` for metadata checks. + +#### Job flow (`ensure-meta-tags`) + +1. Check out the PR **head** (`.rst` content to inspect and, where allowed, modify for suggestions). +2. Check out the PR **base** into `.trusted-base/` (Makefile, `ensure_meta_tags.py`, `meta_tags.yaml`, `supersede_meta_tag_reviews.sh`). +3. Install Python 3.12 and PyYAML. +4. **Ensure documentation metadata** — `git fetch` the base SHA, then `make -f .trusted-base/Makefile ensure-meta-tags` with `TOOLS_DIR=.trusted-base/tools`, `DIFF_BASE`, and `STATUS_FILE=$GITHUB_OUTPUT`. Emits per-file annotations; the step uses `continue-on-error: true` so warning-only gaps do not fail the job immediately. +5. **Supersede stale meta-tag reviews** (only if `meta_checked=true`) — `make -f .trusted-base/Makefile supersede-meta-tag-reviews` with `HAS_RESULTS` from the ensure step; writes `should_post`. +6. **Suggest meta tag changes** — if `should_post` and `inline_suggestions`, run [`suggest-changes`](https://github.com/marketplace/actions/suggest-changes-action) with the multiline `comment` output. +7. **Post meta tag review comment** — if `should_post`, `review_comment`, and not `inline_suggestions`, post the same `comment` body via `gh pr review`. +8. **Enforce required metadata** — if `has_errors`, fail the job (runs `always()` so error gaps fail even when the ensure step soft-failed). + +Priority: **inline suggestions wherever GitHub allows them**. Copy-paste blocks and manual field lists are delivered via a pull request review comment when inline suggestions are not used for that run. + +#### Per-file modes and CI outputs Each changed `.rst` file is classified with an internal **mode**: @@ -143,6 +226,8 @@ Each changed `.rst` file is classified with an internal **mode**: | `snippet` | Configured values could not be written inline; the review includes a copy-paste `.. meta::` block | | `manual_fields` | Only fields with empty `value` in the config are missing; the review lists field names | +A single file can still list **manual** fields in the review when its mode is `suggestable` or `snippet` (auto-filled fields were handled; empty-config fields remain for the contributor). + The script writes **CI outputs** (for example `$GITHUB_OUTPUT`) that describe which workflow steps to run: | Output | Meaning | @@ -152,13 +237,15 @@ The script writes **CI outputs** (for example `$GITHUB_OUTPUT`) that describe wh | `review_comment` | Post the generated review body with `gh pr review` when inline suggestions are not used (covers `snippet` and `manual_fields` files) | | `has_results` | Metadata issues remain (used to decide whether to post a new review after superseding stale ones) | | `has_errors` | Unresolved **error**-severity fields (triggers the final enforce step) | -| `comment` | Full stamped review body (multiline) for suggest-changes or `gh pr review` | +| `comment` | Full stamped review body (multiline heredoc) for suggest-changes or `gh pr review` | + +**Supersede step output:** `should_post` — `true` when `has_results` is `true` (post a new review after minimising old ones); `false` when all metadata issues are resolved (minimise only, no new review). -When `inline_suggestions` is `true`, the workflow runs suggest-changes even if `review_comment` is also `true` (mixed per-file modes). A separate review comment is posted only when `review_comment` is `true` and `inline_suggestions` is not. +When `inline_suggestions` is `true`, the workflow runs suggest-changes even if `review_comment` is also `true` (mixed per-file modes in one PR). A separate `gh pr review` runs only when `review_comment` is `true` and `inline_suggestions` is not. -The ensure step uses `continue-on-error: true`, so warning-only gaps do not fail the job. Error-severity gaps (for example `area`) still fail the workflow after contributors receive review feedback. +The ensure step uses `continue-on-error: true`, so warning-only gaps do not fail the job. Error-severity gaps (for example `area`) still fail the workflow on the enforce step after contributors receive review feedback. -### Annotations +#### Annotations When metadata is missing or blank: @@ -167,46 +254,28 @@ When metadata is missing or blank: `N` is the start line of an existing `.. meta::` block, or `1` when a new block would be inserted at the top of the file. -### Contributor experience - -| Situation | Ensure step | Annotations | Pull request review | Job result | -|-----------|-------------|-------------|---------------------|------------| -| All fields resolved | Green | None | None (stale bot reviews cleared) | Success | -| Warning-only gaps | Soft warning | Warnings | Suggestions and/or manual list | Success | -| Error gaps (e.g. missing `area`) | Soft warning | Errors (and warnings) | Suggestions and/or manual list | **Failure** after enforce step | -| Auto-fix in diff | Soft warning | As above | Inline “Commit suggestion” | As per severity | -| Auto-fix outside diff (`snippet`) | Soft warning | As above | Copy-paste `.. meta::` for configured values | As per severity | -| Manual fields only (`manual_fields`) | Soft warning | As above | Field list with required/warning labels | As per severity | - -Reviews, annotations, and the soft-failed ensure step appear in different parts of the GitHub UI (Conversation, Files changed, Checks); only error-severity issues fail the overall workflow. +#### Superseding outdated reviews -### When inline suggestions appear +Each bot review body includes a hidden HTML marker (``). The marker id `ros2-meta-tags-ensure` (constant `REVIEW_MARKER_ID` in Python; override in the shell script with `META_TAG_REVIEW_MARKER_ID`) is what the supersede script searches for in review bodies. -GitHub only allows review suggestions on [lines already in the pull request diff](https://github.com/marketplace/actions/suggest-changes-action). The script compares each automatic edit to that diff: - -| Situation | What happens | -|-----------|----------------| -| Missing configured values; existing `.. meta::` overlaps the PR diff | Write append/fill to the working tree → inline suggestion via [`suggest-changes`](https://github.com/marketplace/actions/suggest-changes-action) | -| No `.. meta::`; top of file overlaps the PR diff | Insert at top → inline suggestion | -| Automatic edit does **not** overlap the PR diff (`snippet` mode) | No inline write; review includes a copy-paste block for configured values | -| Only manual fields (empty `value` in config, `manual_fields` mode) | Review lists fields; no placeholder injection | -| All configured fields present and non-empty | No action | -| No changed `.rst` files in the PR | No check; `meta_checked=false`; supersede/review steps skipped | +When `meta_checked=true`, the workflow runs `make -f .trusted-base/Makefile supersede-meta-tag-reviews` ([`supersede_meta_tag_reviews.sh`](supersede_meta_tag_reviews.sh)): -### Superseding outdated reviews +1. Lists pull request reviews whose body contains the marker id. +2. Minimises each as **Outdated** via the GitHub GraphQL API (`gh api graphql`; individual failures are ignored). +3. Appends `should_post=true` or `should_post=false` to `$GITHUB_OUTPUT` from `HAS_RESULTS` (`has_results` from the ensure step). -Each bot review body includes a hidden marker (``). On every run that checks changed RST files: +When all issues are fixed (`has_results=false`), stale reviews are minimised and no new “all clear” comment is posted. -1. All prior stamped reviews on the pull request are minimized as **Outdated** via the GitHub API. -2. A fresh review is posted only when work still remains. -3. When all issues are fixed, stale reviews are minimized and no new “all clear” comment is posted. +#### Security -The workflow uses [`pull_request_target`](https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request_target) so the default `GITHUB_TOKEN` can post review suggestions on fork PRs. The workflow definition itself is taken from the repository **default branch**; the trusted Makefile, script, and config come from the PR **base** branch checkout at `.trusted-base/`. See [Mitigating the risks of untrusted code checkout](https://docs.github.com/en/actions/reference/security/secure-use#mitigating-the-risks-of-untrusted-code-checkout). +The workflow uses [`pull_request_target`](https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request_target) so the default `GITHUB_TOKEN` can post review suggestions on fork PRs. **Do not** run `make` or scripts from the PR head checkout in that job; always use `-f .trusted-base/Makefile` and `TOOLS_DIR=.trusted-base/tools`. See [Mitigating the risks of untrusted code checkout](https://docs.github.com/en/actions/reference/security/secure-use#mitigating-the-risks-of-untrusted-code-checkout). -## Tests +### Tests -Unit tests live in [`tests/`](tests/). From the repository root (with PyYAML installed): +Unit tests for this directory live in [`tests/`](tests/). From the repository root (with PyYAML installed): ```bash python3 -m unittest discover -s tools/tests -p 'test_*.py' ``` + +The main documentation CI job [`test-tools`](../Makefile) runs `pytest` on the top-level [`test/`](../test/) tree; that is separate from `tools/tests`. diff --git a/tools/ensure_meta_tags.py b/tools/ensure_meta_tags.py index 062826cc6f7..6026c5b1ec5 100644 --- a/tools/ensure_meta_tags.py +++ b/tools/ensure_meta_tags.py @@ -48,19 +48,32 @@ Severity = Literal["warning", "error"] # Hidden marker in review bodies so CI can find and supersede prior bot reviews. -REVIEW_MARKER = "" +REVIEW_MARKER_ID = "ros2-meta-tags-ensure" +REVIEW_MARKER = f"" @dataclass(frozen=True) class MetaRule: - """A single metadata field rule from ``meta_tags.yaml``.""" + """ + A single metadata field rule from ``meta_tags.yaml``. + + Attributes: + severity: Advisory ``warning`` or blocking ``error`` in CI. + value: Default text to inject when missing or blank; empty when the + contributor must supply a non-empty value. + """ severity: Severity value: str @property def has_configured_value(self) -> bool: - """Return whether the rule supplies a non-empty default value.""" + """ + Return whether the rule supplies a non-empty default value. + + Returns: + ``True`` when ``value`` is non-empty after stripping whitespace. + """ return bool(self.value.strip()) @@ -291,7 +304,15 @@ def changed_rst_paths(diff_base: str) -> list[Path]: def _log_working_tree_summary(paths: list[Path]) -> None: - """Log ``git status`` and ``git diff`` for processed RST paths.""" + """ + Log ``git status`` and ``git diff`` for processed RST paths. + + Args: + paths: Repository-relative RST files that were checked or updated. + + Returns: + None. + """ if not paths: return path_args = [str(p) for p in paths] @@ -326,7 +347,23 @@ def _write_ci_status_file( rules: dict[str, MetaRule], has_errors: bool, ) -> None: - """Append GitHub Actions output flags and optional review comment.""" + """ + Append GitHub Actions output flags and optional review comment. + + Writes ``meta_checked``, ``inline_suggestions``, ``review_comment``, + ``has_results``, ``has_errors``, and a multiline ``comment`` block when + ``results`` is non-empty. + + Args: + status_file: Path to append to (for example ``$GITHUB_OUTPUT``). + meta_checked: Whether changed RST files were in scope for this run. + results: Per-file result dicts from ``ensure_meta_tags_in_file``. + rules: Configured metadata rules for building the review body. + has_errors: Whether any result has unresolved error-severity fields. + + Returns: + None. + """ has_inline_suggestions = any(r["mode"] == "suggestable" for r in results) has_review_comment = any( r["mode"] in ("snippet", "manual_fields") for r in results @@ -397,7 +434,18 @@ def _escape_workflow_command_message(message: str) -> str: def _emit_annotation(level: str, path: str, fields: list[str], line: int) -> None: - """Print a GitHub Actions workflow annotation for missing meta fields.""" + """ + Print a GitHub Actions workflow annotation for missing meta fields. + + Args: + level: Annotation level, typically ``warning`` or ``error``. + path: Repository-relative path to annotate. + fields: Missing meta field names. + line: One-based source line to annotate. + + Returns: + None. + """ if not fields: return field_list = ", ".join(fields) @@ -466,7 +514,17 @@ def _severity_fields( rules: dict[str, MetaRule], severity: Severity, ) -> list[str]: - """Return ``field_names`` that use the given severity in ``rules``.""" + """ + Return field names that use the given severity in ``rules``. + + Args: + field_names: Candidate meta field names. + rules: Configured metadata rules keyed by field name. + severity: Severity label to match. + + Returns: + Names from ``field_names`` whose rule uses ``severity``. + """ return [name for name in field_names if rules[name].severity == severity] @@ -489,7 +547,10 @@ def ensure_meta_tags_in_file( pr_lines: One-based pull-request diff lines, or ``None`` for local mode. Returns: - A result dict when issues remain, otherwise ``None``. + A result dict when issues remain, otherwise ``None``. Each result + includes ``path``, ``line``, ``mode`` (``suggestable``, ``snippet``, or + ``manual_fields``), ``snippet`` (RST for copy-paste when relevant), + ``manual_fields``, ``warning_fields``, and ``error_fields``. Raises: OSError: If the RST file cannot be read or an eligible edit cannot be written. @@ -594,7 +655,16 @@ def stamp_review_comment(body: str) -> str: def _field_list_markdown(field_names: list[str], rules: dict[str, MetaRule]) -> str: - """Format field names with severity hints for review text.""" + """ + Format field names with severity hints for review text. + + Args: + field_names: Meta field names to list. + rules: Configured metadata rules keyed by field name. + + Returns: + A comma-separated Markdown fragment such as `` `area` (required) ``. + """ parts: list[str] = [] for name in field_names: label = "required" if rules[name].severity == "error" else "warning" @@ -674,13 +744,18 @@ def main(argv: list[str] | None = None) -> int: """ Run the command-line metadata check. + When no ``paths`` are given, ``--diff-base`` must be set so changed ``.rst`` + files are discovered with ``git diff``. With ``--status-file``, writes CI + outputs and exits ``1`` when any per-file results remain; without it, exits + ``1`` only for unresolved error-severity fields. + Args: argv: Command-line arguments excluding the executable name, or ``None`` to read them from ``sys.argv``. Returns: - Process exit code. In CI, ``1`` when any issues remain. Locally, ``1`` - only when error-severity fields remain unresolved. + Process exit code (``0`` on success, ``1`` when issues remain per mode + above). Raises: SystemExit: If command-line arguments or metadata configuration are invalid. diff --git a/tools/supersede_meta_tag_reviews.sh b/tools/supersede_meta_tag_reviews.sh new file mode 100755 index 00000000000..0eafc9c3ff8 --- /dev/null +++ b/tools/supersede_meta_tag_reviews.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Minimise prior bot PR reviews stamped by ensure_meta_tags.py and set should_post. +set -euo pipefail + +: "${GH_TOKEN:?GH_TOKEN is required}" +: "${REPOSITORY:?REPOSITORY is required}" +: "${PR_NUMBER:?PR_NUMBER is required}" +: "${HAS_RESULTS:?HAS_RESULTS is required}" + +MARKER="${META_TAG_REVIEW_MARKER_ID:-ros2-meta-tags-ensure}" +STATUS_FILE="${STATUS_FILE:-${GITHUB_OUTPUT:-}}" + +if [ -z "${STATUS_FILE}" ]; then + echo "STATUS_FILE or GITHUB_OUTPUT is required" >&2 + exit 1 +fi + +echo "Marking prior meta-tag reviews as outdated." + +reviews_json="$(gh api "repos/${REPOSITORY}/pulls/${PR_NUMBER}/reviews" --paginate)" +mapfile -t review_ids < <( + echo "${reviews_json}" | jq -r \ + ".[] | select(.body != null and (.body | contains(\"${MARKER}\"))) | .node_id" +) + +minimize_query='mutation($subjectId: ID!) { + minimizeComment(input: { subjectId: $subjectId, classifier: OUTDATED }) { + minimizedComment { isMinimized } + } +}' + +for node_id in "${review_ids[@]:-}"; do + [ -z "${node_id}" ] && continue + gh api graphql -f query="${minimize_query}" -f subjectId="${node_id}" || true +done + +if [ "${HAS_RESULTS}" = "true" ]; then + echo "should_post=true" >>"${STATUS_FILE}" +else + echo "All meta-tag issues resolved; not posting a new review." + echo "should_post=false" >>"${STATUS_FILE}" +fi From 4361dceed4c7c6bbc5282fcbefadc47ede358ee4 Mon Sep 17 00:00:00 2001 From: Keith Kirkwood Date: Mon, 27 Jul 2026 17:20:22 +0100 Subject: [PATCH 09/23] OPENR-174: Update PHONY list to ensure catch-all doesn't try to remake --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index caceb88580d..90652f40f36 100644 --- a/Makefile +++ b/Makefile @@ -98,4 +98,4 @@ linkcheck: serve: sphinx-autobuild --host $(LIVE_HOST) --port $(LIVE_PORT) -c . $(SOURCE) $(OUT)/html -.PHONY: help Makefile multiversion test test-tools linkcheck serve lint spellcheck check-dictionaries sort-dictionaries ensure-meta-tags supersede-meta-tag-reviews +.PHONY: help Makefile multiversion test test-tools linkcheck serve lint spellcheck check-dictionaries sort-dictionaries ensure-meta-tags supersede-meta-tag-reviews $(MAKEFILE_LIST) From e3e3c5778c5684e35c6ef19705c0e4820fd3d38b Mon Sep 17 00:00:00 2001 From: Keith Kirkwood Date: Tue, 28 Jul 2026 15:29:47 +0100 Subject: [PATCH 10/23] OPENR-174: Improve annotations to include static fields --- tools/README.md | 2 ++ tools/ensure_meta_tags.py | 21 +++++++++------ tools/tests/test_ensure_meta_tags.py | 40 ++++++++++++++++++++++++++++ 3 files changed, 55 insertions(+), 8 deletions(-) diff --git a/tools/README.md b/tools/README.md index 94971c570c8..797298b6a0a 100644 --- a/tools/README.md +++ b/tools/README.md @@ -254,6 +254,8 @@ When metadata is missing or blank: `N` is the start line of an existing `.. meta::` block, or `1` when a new block would be inserted at the top of the file. +Annotations describe the pull request **as pushed**, so fields with a configured `value` are listed even when the same run offers them as an inline suggestion. Auto-injected values only exist in the CI working tree; they disappear from the annotations once the suggestion is committed and the workflow re-runs. + #### Superseding outdated reviews Each bot review body includes a hidden HTML marker (``). The marker id `ros2-meta-tags-ensure` (constant `REVIEW_MARKER_ID` in Python; override in the shell script with `META_TAG_REVIEW_MARKER_ID`) is what the supersede script searches for in review bodies. diff --git a/tools/ensure_meta_tags.py b/tools/ensure_meta_tags.py index 6026c5b1ec5..1b9ade60cc4 100644 --- a/tools/ensure_meta_tags.py +++ b/tools/ensure_meta_tags.py @@ -547,10 +547,12 @@ def ensure_meta_tags_in_file( pr_lines: One-based pull-request diff lines, or ``None`` for local mode. Returns: - A result dict when issues remain, otherwise ``None``. Each result - includes ``path``, ``line``, ``mode`` (``suggestable``, ``snippet``, or - ``manual_fields``), ``snippet`` (RST for copy-paste when relevant), - ``manual_fields``, ``warning_fields``, and ``error_fields``. + A result dict when fields were missing in the file as read, otherwise + ``None``. Each result includes ``path``, ``line``, ``mode`` + (``suggestable``, ``snippet``, or ``manual_fields``), ``snippet`` (RST + for copy-paste when relevant), ``manual_fields``, ``warning_fields``, + and ``error_fields``. The severity lists cover every field that was + missing or blank before any automatic injection. Raises: OSError: If the RST file cannot be read or an eligible edit cannot be written. @@ -597,12 +599,15 @@ def ensure_meta_tags_in_file( ) still_unresolved = _unresolved_fields(content, rules) - if not still_unresolved: + if not still_unresolved and mode is None: return None - manual_fields = [name for name in still_unresolved if not rules[name].has_configured_value] - warning_fields = _severity_fields(still_unresolved, rules, "warning") - error_fields = _severity_fields(still_unresolved, rules, "error") + # Annotations and severity describe the pull request as pushed. Auto-injected + # values only exist in the CI working tree until the suggestion is committed, + # so they are reported from ``unresolved`` rather than ``still_unresolved``. + manual_fields = [name for name in unresolved if not rules[name].has_configured_value] + warning_fields = _severity_fields(unresolved, rules, "warning") + error_fields = _severity_fields(unresolved, rules, "error") if mode is None: mode = "manual_fields" diff --git a/tools/tests/test_ensure_meta_tags.py b/tools/tests/test_ensure_meta_tags.py index 661895bed41..261a7ffdd42 100644 --- a/tools/tests/test_ensure_meta_tags.py +++ b/tools/tests/test_ensure_meta_tags.py @@ -135,6 +135,46 @@ def test_local_auto_inject_clears_configured_fields(self) -> None: fields = get_meta_fields_from_content(path.read_text(encoding="utf-8")) self.assertEqual(fields["product"], "{PRODUCT}") + def test_auto_injected_fields_are_still_annotated(self) -> None: + rules = { + "product": MetaRule("warning", "{PRODUCT}"), + "area": MetaRule("error", ""), + } + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "page.rst" + path.write_text("Title\n=====\n", encoding="utf-8") + result = ensure_meta_tags_in_file(path, rules) + self.assertIsNotNone(result) + self.assertIn("product", result["warning_fields"]) + self.assertNotIn("product", result["manual_fields"]) + + def test_result_returned_when_only_configured_fields_missing(self) -> None: + rules = {"product": MetaRule("warning", "{PRODUCT}")} + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "page.rst" + path.write_text("Title\n=====\n", encoding="utf-8") + result = ensure_meta_tags_in_file(path, rules) + self.assertIsNotNone(result) + self.assertEqual(result["mode"], "suggestable") + self.assertEqual(result["warning_fields"], ["product"]) + self.assertEqual(result["manual_fields"], []) + + def test_no_result_when_all_fields_present(self) -> None: + rules = {"product": MetaRule("warning", "{PRODUCT}")} + content = textwrap.dedent( + """ + .. meta:: + :product: ROS 2 + + Title + ===== + """ + ).lstrip() + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "page.rst" + path.write_text(content, encoding="utf-8") + self.assertIsNone(ensure_meta_tags_in_file(path, rules)) + class TestReviewAndExit(unittest.TestCase): def test_build_review_comment_lists_manual_fields(self) -> None: From 57a8515afc4c474c9258acc3c691b2e6f78d053f Mon Sep 17 00:00:00 2001 From: Keith Kirkwood Date: Tue, 28 Jul 2026 16:22:59 +0100 Subject: [PATCH 11/23] OPENR-174: Further clarification on review comment in PR for suggestions/manual --- tools/README.md | 12 ++++++++++++ tools/ensure_meta_tags.py | 16 +++++++++++----- tools/tests/test_ensure_meta_tags.py | 22 ++++++++++++++++++++++ 3 files changed, 45 insertions(+), 5 deletions(-) diff --git a/tools/README.md b/tools/README.md index 797298b6a0a..069cffc18b9 100644 --- a/tools/README.md +++ b/tools/README.md @@ -131,6 +131,18 @@ When you open or update a pull request, CI automatically checks metadata on all Reviews, annotations, and the soft-failed ensure step appear in different parts of the GitHub UI (Conversation, Files changed, Checks); only error-severity issues fail the overall workflow. +#### Review body sections + +The review comment names every affected file and splits the work by how it is fixed, so the fields listed for a file match that file's annotation: + +| Section | Files listed | Fields listed | +|---------|--------------|---------------| +| Inline suggestions | `suggestable` mode | Configured values added for you — commit the suggestion | +| Copy-paste blocks | `snippet` mode | Configured values as an RST block to paste yourself | +| Non-empty values required | Any file with manual fields | Fields with an empty `value`, labelled required or warning | + +A file can appear in two sections: the suggestion covers its configured values while the manual list covers the rest. + #### When inline suggestions appear GitHub only allows review suggestions on [lines already in the pull request diff](https://github.com/marketplace/actions/suggest-changes-action). The script compares each automatic edit to that diff: diff --git a/tools/ensure_meta_tags.py b/tools/ensure_meta_tags.py index 1b9ade60cc4..baa438f58ca 100644 --- a/tools/ensure_meta_tags.py +++ b/tools/ensure_meta_tags.py @@ -550,9 +550,10 @@ def ensure_meta_tags_in_file( A result dict when fields were missing in the file as read, otherwise ``None``. Each result includes ``path``, ``line``, ``mode`` (``suggestable``, ``snippet``, or ``manual_fields``), ``snippet`` (RST - for copy-paste when relevant), ``manual_fields``, ``warning_fields``, - and ``error_fields``. The severity lists cover every field that was - missing or blank before any automatic injection. + for copy-paste when relevant), ``auto_fields`` (configured values the + tool can fill), ``manual_fields``, ``warning_fields``, and + ``error_fields``. The severity lists cover every field that was missing + or blank before any automatic injection. Raises: OSError: If the RST file cannot be read or an eligible edit cannot be written. @@ -617,6 +618,7 @@ def ensure_meta_tags_in_file( "line": annotation_line, "mode": mode, "snippet": snippet, + "auto_fields": auto_fields, "manual_fields": manual_fields, "warning_fields": warning_fields, "error_fields": error_fields, @@ -703,10 +705,14 @@ def build_review_comment( if inline_modes: lines.append( - "Please **review and commit the inline suggestions**. They add configured " - "default values in place so metadata stays complete.", + "Please **review and commit the inline suggestions**. They add these " + "configured default values in place so metadata stays complete:", ) lines.append("") + for result in inline_modes: + auto = ", ".join(f"`{name}`" for name in result["auto_fields"]) + lines.append(f"**`{result['path']}`**: {auto}") + lines.append("") if snippet_modes: lines.append( diff --git a/tools/tests/test_ensure_meta_tags.py b/tools/tests/test_ensure_meta_tags.py index 261a7ffdd42..43eb6b791ff 100644 --- a/tools/tests/test_ensure_meta_tags.py +++ b/tools/tests/test_ensure_meta_tags.py @@ -187,6 +187,7 @@ def test_build_review_comment_lists_manual_fields(self) -> None: "path": "source/Page.rst", "mode": "manual_fields", "snippet": "", + "auto_fields": [], "manual_fields": ["area", "experience"], "warning_fields": ["experience"], "error_fields": ["area"], @@ -198,6 +199,27 @@ def test_build_review_comment_lists_manual_fields(self) -> None: self.assertIn("required", body) self.assertIn("experience", body) + def test_build_review_comment_lists_inline_suggestion_fields(self) -> None: + rules = { + "product": MetaRule("warning", "{PRODUCT}"), + "experience": MetaRule("warning", ""), + } + results = [ + { + "path": "source/Page.rst", + "mode": "suggestable", + "snippet": "", + "auto_fields": ["product"], + "manual_fields": ["experience"], + "warning_fields": ["product", "experience"], + "error_fields": [], + "line": 1, + }, + ] + body = build_review_comment(results, rules) + self.assertIn("**`source/Page.rst`**: `product`", body) + self.assertIn("`experience` (warning)", body) + def test_local_exit_nonzero_only_for_error_severity(self) -> None: warning_config = textwrap.dedent( """ From 70c144751b6919ddb79b5b6cd834468f17ce6ac5 Mon Sep 17 00:00:00 2001 From: Keith Kirkwood Date: Tue, 28 Jul 2026 17:03:49 +0100 Subject: [PATCH 12/23] OPENR-174: Fix workflow issues found in testing --- .github/workflows/enhance.yml | 33 ++++++++++----- Makefile | 4 -- tools/README.md | 36 ++++++++-------- tools/ensure_meta_tags.py | 56 ++++++++++++++++++------- tools/supersede_meta_tag_reviews.sh | 16 +------ tools/tests/test_ensure_meta_tags.py | 62 ++++++++++++++++++++++++++++ 6 files changed, 144 insertions(+), 63 deletions(-) diff --git a/.github/workflows/enhance.yml b/.github/workflows/enhance.yml index 1984854165e..2718cd95fc6 100644 --- a/.github/workflows/enhance.yml +++ b/.github/workflows/enhance.yml @@ -51,39 +51,50 @@ jobs: DIFF_BASE="$DIFF_BASE" \ STATUS_FILE="$GITHUB_OUTPUT" + # The ensure step soft-fails whenever metadata is missing, so an empty + # meta_checked is the only signal that the check itself never ran. + - name: Verify metadata check ran + if: steps.ensure.outputs.meta_checked == '' + run: | + echo "The metadata check produced no outputs; see the ensure step log." + exit 1 + - name: Supersede stale meta-tag reviews - id: supersede if: steps.ensure.outputs.meta_checked == 'true' env: GH_TOKEN: ${{ github.token }} - HAS_RESULTS: ${{ steps.ensure.outputs.has_results }} PR_NUMBER: ${{ github.event.pull_request.number }} REPOSITORY: ${{ github.repository }} run: | set -euo pipefail make -f .trusted-base/Makefile supersede-meta-tag-reviews \ - TOOLS_DIR=.trusted-base/tools \ - STATUS_FILE="$GITHUB_OUTPUT" + TOOLS_DIR=.trusted-base/tools + # Carries the file-level "Commit suggestion" comments. Its review body is a + # stamped pointer only, because the action posts nothing at all when every + # suggestion duplicates one from an earlier run. - name: Suggest meta tag changes if: >- - steps.supersede.outputs.should_post == 'true' - && steps.ensure.outputs.inline_suggestions == 'true' + ${{ !cancelled() + && steps.ensure.outputs.inline_suggestions == 'true' }} uses: parkerbxyz/suggest-changes@v3 with: - comment: ${{ steps.ensure.outputs.comment }} + comment: ${{ steps.ensure.outputs.suggestion_note }} event: COMMENT + # Always posts the summary so the Conversation view has a current review + # after the stale ones are minimised, whatever suggest-changes did. - name: Post meta tag review comment if: >- - steps.supersede.outputs.should_post == 'true' - && steps.ensure.outputs.review_comment == 'true' - && steps.ensure.outputs.inline_suggestions != 'true' + ${{ !cancelled() + && steps.ensure.outputs.has_results == 'true' }} env: GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} REVIEW_COMMENT: ${{ steps.ensure.outputs.comment }} run: | - gh pr review "${{ github.event.pull_request.number }}" \ + set -euo pipefail + gh pr review "$PR_NUMBER" \ --comment \ --body "$REVIEW_COMMENT" diff --git a/Makefile b/Makefile index 90652f40f36..7fb617a6bd3 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,6 @@ DIFF_BASE ?= STATUS_FILE ?= PR_NUMBER ?= REPOSITORY ?= -HAS_RESULTS ?= DICTIONARIES := codespell_dictionary.txt codespell_whitelist.txt @@ -64,9 +63,6 @@ ifndef PR_NUMBER endif ifndef REPOSITORY $(error REPOSITORY is required) -endif -ifndef HAS_RESULTS - $(error HAS_RESULTS is required) endif $(BASH) $(TOOLS_DIR)/supersede_meta_tag_reviews.sh diff --git a/tools/README.md b/tools/README.md index 069cffc18b9..eb7455c8038 100644 --- a/tools/README.md +++ b/tools/README.md @@ -169,7 +169,7 @@ Information for maintainers and developers working on or extending the metadata | [`rst_utils.py`](rst_utils.py) | Regex-based read/write of `.. meta::` and `.. short-description::` directives | | [`meta_tags.yaml`](meta_tags.yaml) | Metadata rules (severity and optional default values) | | [`ensure_meta_tags.py`](ensure_meta_tags.py) | CLI that checks and fixes metadata from the config | -| [`supersede_meta_tag_reviews.sh`](supersede_meta_tag_reviews.sh) | Minimise outdated bot PR reviews and set `should_post` for CI | +| [`supersede_meta_tag_reviews.sh`](supersede_meta_tag_reviews.sh) | Minimise outdated bot PR reviews | | [`tests/`](tests/) | Unit tests for the tools in this directory | ### Code modules @@ -195,7 +195,7 @@ Add a new key under `meta` in [`meta_tags.yaml`](meta_tags.yaml) to extend cover - `paths` — optional; when omitted, `--diff-base` is required and changed `.rst` files are discovered automatically - `--config PATH` — YAML config file (default: `tools/meta_tags.yaml`) - `--diff-base SHA` — PR base commit; limits on-disk writes to lines in the PR diff (inline suggestions); files that need a copy-paste or manual field list use a review comment instead -- `--status-file PATH` — write `meta_checked`, `inline_suggestions`, `review_comment`, `has_results`, `has_errors`, and the review comment body for CI; when issues remain, emits annotations and exits `1` (the ensure step uses `continue-on-error`) +- `--status-file PATH` — write `meta_checked`, `inline_suggestions`, `has_results`, `has_errors`, the review comment body, and the suggestion note for CI; when issues remain, emits annotations and exits `1` (the ensure step uses `continue-on-error`) - `-v` / `--verbose` — enable debug logging #### Makefile targets (metadata CI) @@ -205,9 +205,9 @@ Both targets live in the repository root [`Makefile`](../Makefile). CI invokes t | Target | Required variables | Purpose | |--------|-------------------|---------| | `ensure-meta-tags` | `DIFF_BASE`, `STATUS_FILE`; optional `TOOLS_DIR` (default `tools`) | Discover changed RST, run metadata check, append CI outputs | -| `supersede-meta-tag-reviews` | `PR_NUMBER`, `REPOSITORY`, `HAS_RESULTS`, `STATUS_FILE` (or `GITHUB_OUTPUT`); optional `TOOLS_DIR` | Minimise stamped bot reviews; append `should_post` | +| `supersede-meta-tag-reviews` | `PR_NUMBER`, `REPOSITORY`; optional `TOOLS_DIR` | Minimise stamped bot reviews | -Environment for `supersede-meta-tag-reviews` (set by the workflow or locally): `GH_TOKEN`, plus `PR_NUMBER`, `REPOSITORY`, and `HAS_RESULTS` from the ensure step’s `has_results` output. +Environment for `supersede-meta-tag-reviews` (set by the workflow or locally): `GH_TOKEN`, plus `PR_NUMBER` and `REPOSITORY`. It writes no CI outputs; the workflow decides what to post from the ensure step’s outputs. ### Continuous integration architecture @@ -221,10 +221,13 @@ The Enhance workflow installs PyYAML in the job; it does not install the full do 2. Check out the PR **base** into `.trusted-base/` (Makefile, `ensure_meta_tags.py`, `meta_tags.yaml`, `supersede_meta_tag_reviews.sh`). 3. Install Python 3.12 and PyYAML. 4. **Ensure documentation metadata** — `git fetch` the base SHA, then `make -f .trusted-base/Makefile ensure-meta-tags` with `TOOLS_DIR=.trusted-base/tools`, `DIFF_BASE`, and `STATUS_FILE=$GITHUB_OUTPUT`. Emits per-file annotations; the step uses `continue-on-error: true` so warning-only gaps do not fail the job immediately. -5. **Supersede stale meta-tag reviews** (only if `meta_checked=true`) — `make -f .trusted-base/Makefile supersede-meta-tag-reviews` with `HAS_RESULTS` from the ensure step; writes `should_post`. -6. **Suggest meta tag changes** — if `should_post` and `inline_suggestions`, run [`suggest-changes`](https://github.com/marketplace/actions/suggest-changes-action) with the multiline `comment` output. -7. **Post meta tag review comment** — if `should_post`, `review_comment`, and not `inline_suggestions`, post the same `comment` body via `gh pr review`. -8. **Enforce required metadata** — if `has_errors`, fail the job (runs `always()` so error gaps fail even when the ensure step soft-failed). +5. **Verify metadata check ran** — fail the job if `meta_checked` is empty. The ensure step soft-fails by design, so a missing output is the only way to tell a crashed check from a clean run. +6. **Supersede stale meta-tag reviews** (only if `meta_checked=true`) — `make -f .trusted-base/Makefile supersede-meta-tag-reviews`, which minimises stamped reviews and writes nothing back. +7. **Suggest meta tag changes** — if `inline_suggestions`, run [`suggest-changes`](https://github.com/marketplace/actions/suggest-changes-action) for file-level “Commit suggestion” comments, using `suggestion_note` as its stamped review body. +8. **Post meta tag review comment** — if `has_results`, post the stamped `comment` body via `gh pr review` (Conversation view). Independent of suggest-changes, which posts nothing when every suggestion duplicates one from an earlier run. +9. **Enforce required metadata** — if `has_errors`, fail the job (runs `always()` so error gaps fail even when the ensure step soft-failed). + +Steps 7 and 8 use `!cancelled()` rather than depending on the supersede step, so a transient GitHub API failure while minimising old reviews cannot stop contributors receiving feedback. Priority: **inline suggestions wherever GitHub allows them**. Copy-paste blocks and manual field lists are delivered via a pull request review comment when inline suggestions are not used for that run. @@ -244,16 +247,14 @@ The script writes **CI outputs** (for example `$GITHUB_OUTPUT`) that describe wh | Output | Meaning | |--------|---------| -| `meta_checked` | At least one changed `.rst` was in scope (`false` when discovery finds no changed RST; supersede and review steps are skipped) | -| `inline_suggestions` | Run [`suggest-changes`](https://github.com/marketplace/actions/suggest-changes-action) | -| `review_comment` | Post the generated review body with `gh pr review` when inline suggestions are not used (covers `snippet` and `manual_fields` files) | -| `has_results` | Metadata issues remain (used to decide whether to post a new review after superseding stale ones) | +| `meta_checked` | At least one changed `.rst` was in scope (`false` when discovery finds no changed RST; supersede is skipped). Empty only when the check never ran | +| `inline_suggestions` | Run [`suggest-changes`](https://github.com/marketplace/actions/suggest-changes-action) for file-level suggestions | +| `has_results` | Metadata issues remain; post the Conversation review | | `has_errors` | Unresolved **error**-severity fields (triggers the final enforce step) | -| `comment` | Full stamped review body (multiline heredoc) for suggest-changes or `gh pr review` | - -**Supersede step output:** `should_post` — `true` when `has_results` is `true` (post a new review after minimising old ones); `false` when all metadata issues are resolved (minimise only, no new review). +| `comment` | Full stamped review body (multiline heredoc) posted to Conversation via `gh pr review`; written only when `has_results` | +| `suggestion_note` | Short stamped body for the suggest-changes review; written only when `inline_suggestions` | -When `inline_suggestions` is `true`, the workflow runs suggest-changes even if `review_comment` is also `true` (mixed per-file modes in one PR). A separate `gh pr review` runs only when `review_comment` is `true` and `inline_suggestions` is not. +Both review bodies carry the marker, so the next run minimises the summary and the suggestion note together. The outputs are self-consistent by construction: `comment` exists whenever `has_results` is true, `suggestion_note` exists whenever `inline_suggestions` is true, and `has_errors` implies `has_results`. No step can therefore run with an empty review body, and the enforce step cannot fail the job without a review having been posted. The ensure step uses `continue-on-error: true`, so warning-only gaps do not fail the job. Error-severity gaps (for example `area`) still fail the workflow on the enforce step after contributors receive review feedback. @@ -276,9 +277,8 @@ When `meta_checked=true`, the workflow runs `make -f .trusted-base/Makefile supe 1. Lists pull request reviews whose body contains the marker id. 2. Minimises each as **Outdated** via the GitHub GraphQL API (`gh api graphql`; individual failures are ignored). -3. Appends `should_post=true` or `should_post=false` to `$GITHUB_OUTPUT` from `HAS_RESULTS` (`has_results` from the ensure step). -When all issues are fixed (`has_results=false`), stale reviews are minimised and no new “all clear” comment is posted. +When all issues are fixed (`has_results=false`), stale reviews are minimised and no new “all clear” comment is posted. Inline suggestion comments are separate from the review body and stay visible until they are committed or the file changes. #### Security diff --git a/tools/ensure_meta_tags.py b/tools/ensure_meta_tags.py index baa438f58ca..1e758ec20ff 100644 --- a/tools/ensure_meta_tags.py +++ b/tools/ensure_meta_tags.py @@ -24,7 +24,7 @@ import sys from dataclasses import dataclass from pathlib import Path -from typing import Literal +from typing import Literal, TextIO import yaml @@ -51,6 +51,13 @@ REVIEW_MARKER_ID = "ros2-meta-tags-ensure" REVIEW_MARKER = f"" +# Header for the separate review that carries the inline suggestions. It is stamped +# like the main body so both reviews are superseded together on the next run. +SUGGESTION_NOTE = ( + "Inline suggestions add configured documentation metadata defaults. " + "See the metadata review comment for the full list of fields to complete." +) + @dataclass(frozen=True) class MetaRule: @@ -339,6 +346,26 @@ def _log_working_tree_summary(paths: list[Path]) -> None: logger.info("%s", line) +def _write_multiline_output(handle: TextIO, key: str, value: str) -> None: + """ + Write a multiline GitHub Actions output using heredoc syntax. + + Args: + handle: Open text handle for the status file. + key: Output name. + value: Output value, which may span several lines. + + Returns: + None. + """ + delimiter = f"EOF_{key.upper()}" + handle.write(f"{key}<<{delimiter}\n") + handle.write(value) + if not value.endswith("\n"): + handle.write("\n") + handle.write(f"{delimiter}\n") + + def _write_ci_status_file( status_file: Path, *, @@ -350,9 +377,9 @@ def _write_ci_status_file( """ Append GitHub Actions output flags and optional review comment. - Writes ``meta_checked``, ``inline_suggestions``, ``review_comment``, - ``has_results``, ``has_errors``, and a multiline ``comment`` block when - ``results`` is non-empty. + Writes ``meta_checked``, ``inline_suggestions``, ``has_results``, and + ``has_errors``, plus a multiline ``comment`` block when ``results`` is + non-empty and ``suggestion_note`` when inline suggestions were written. Args: status_file: Path to append to (for example ``$GITHUB_OUTPUT``). @@ -365,26 +392,23 @@ def _write_ci_status_file( None. """ has_inline_suggestions = any(r["mode"] == "suggestable" for r in results) - has_review_comment = any( - r["mode"] in ("snippet", "manual_fields") for r in results - ) has_results = bool(results) with status_file.open("a", encoding="utf-8") as f: for key, flag in ( ("meta_checked", meta_checked), ("inline_suggestions", has_inline_suggestions), - ("review_comment", has_review_comment), ("has_results", has_results), ("has_errors", has_errors), ): f.write(f"{key}={'true' if flag else 'false'}\n") if results: - review_body = build_review_comment(results, rules) - f.write("comment< bool: @@ -802,8 +826,8 @@ def main(argv: list[str] | None = None) -> int: "--status-file", type=Path, help=( - "Write meta_checked, inline_suggestions, review_comment, has_results, " - "has_errors, and the review comment body for CI" + "Write meta_checked, inline_suggestions, has_results, has_errors, " + "the review comment body, and the suggestion note for CI" ), ) parser.add_argument( diff --git a/tools/supersede_meta_tag_reviews.sh b/tools/supersede_meta_tag_reviews.sh index 0eafc9c3ff8..0026696d4a8 100755 --- a/tools/supersede_meta_tag_reviews.sh +++ b/tools/supersede_meta_tag_reviews.sh @@ -1,19 +1,12 @@ #!/usr/bin/env bash -# Minimise prior bot PR reviews stamped by ensure_meta_tags.py and set should_post. +# Minimise prior bot PR reviews stamped by ensure_meta_tags.py. set -euo pipefail : "${GH_TOKEN:?GH_TOKEN is required}" : "${REPOSITORY:?REPOSITORY is required}" : "${PR_NUMBER:?PR_NUMBER is required}" -: "${HAS_RESULTS:?HAS_RESULTS is required}" MARKER="${META_TAG_REVIEW_MARKER_ID:-ros2-meta-tags-ensure}" -STATUS_FILE="${STATUS_FILE:-${GITHUB_OUTPUT:-}}" - -if [ -z "${STATUS_FILE}" ]; then - echo "STATUS_FILE or GITHUB_OUTPUT is required" >&2 - exit 1 -fi echo "Marking prior meta-tag reviews as outdated." @@ -34,9 +27,4 @@ for node_id in "${review_ids[@]:-}"; do gh api graphql -f query="${minimize_query}" -f subjectId="${node_id}" || true done -if [ "${HAS_RESULTS}" = "true" ]; then - echo "should_post=true" >>"${STATUS_FILE}" -else - echo "All meta-tag issues resolved; not posting a new review." - echo "should_post=false" >>"${STATUS_FILE}" -fi +echo "Superseded ${#review_ids[@]} prior meta-tag review(s)." diff --git a/tools/tests/test_ensure_meta_tags.py b/tools/tests/test_ensure_meta_tags.py index 43eb6b791ff..3b2ae2ca94d 100644 --- a/tools/tests/test_ensure_meta_tags.py +++ b/tools/tests/test_ensure_meta_tags.py @@ -26,6 +26,7 @@ sys.path.insert(0, str(_TOOLS_DIR)) from ensure_meta_tags import ( # noqa: E402 + REVIEW_MARKER, MetaRule, _unresolved_fields, build_review_comment, @@ -257,6 +258,67 @@ def test_local_exit_nonzero_only_for_error_severity(self) -> None: self.assertEqual(code, 1) +class TestCiStatusOutputs(unittest.TestCase): + """The workflow gates steps on these outputs, so keep them self-consistent.""" + + def _run_with_status_file(self, content: str) -> str: + rules_path = Path(tempfile.mkdtemp()) / "meta.yaml" + rules_path.write_text(SAMPLE_CONFIG, encoding="utf-8") + with tempfile.TemporaryDirectory() as tmp: + page = Path(tmp) / "page.rst" + page.write_text(content, encoding="utf-8") + status_path = Path(tmp) / "status.txt" + main( + [ + str(page), + "--config", + str(rules_path), + "--status-file", + str(status_path), + ], + ) + return status_path.read_text(encoding="utf-8") + + def test_suggestion_note_written_with_inline_suggestions(self) -> None: + status = self._run_with_status_file("Title\n=====\n") + self.assertIn("inline_suggestions=true", status) + self.assertIn("suggestion_note<<", status) + self.assertIn(REVIEW_MARKER, status) + + def test_no_suggestion_note_without_inline_suggestions(self) -> None: + content = textwrap.dedent( + """ + .. meta:: + :product: ROS 2 + + Title + ===== + """ + ).lstrip() + status = self._run_with_status_file(content) + self.assertIn("inline_suggestions=false", status) + self.assertNotIn("suggestion_note<<", status) + self.assertIn("comment<<", status) + + def test_clean_file_writes_no_review_bodies(self) -> None: + content = textwrap.dedent( + """ + .. meta:: + :product: ROS 2 + :area: docs + :experience: beginner + + Title + ===== + """ + ).lstrip() + status = self._run_with_status_file(content) + self.assertIn("has_results=false", status) + self.assertIn("has_errors=false", status) + self.assertNotIn("comment<<", status) + self.assertNotIn("suggestion_note<<", status) + + class TestChangedRstPaths(unittest.TestCase): def test_parses_git_diff_output(self) -> None: completed = mock.Mock(returncode=0, stdout="source/A.rst\nsource/B.rst\n", stderr="") From 07da8cb1a7458bbcba671b32096695cddfab45b9 Mon Sep 17 00:00:00 2001 From: Keith Kirkwood Date: Wed, 29 Jul 2026 15:18:28 +0100 Subject: [PATCH 13/23] OPENR-174: No longer supersede the suggestion reviews, only summaries. --- .github/workflows/enhance.yml | 6 +++--- tools/README.md | 10 +++++----- tools/ensure_meta_tags.py | 11 ++++------- tools/supersede_meta_tag_reviews.sh | 4 +++- tools/tests/test_ensure_meta_tags.py | 21 ++++++++++++++++++++- 5 files changed, 35 insertions(+), 17 deletions(-) diff --git a/.github/workflows/enhance.yml b/.github/workflows/enhance.yml index 2718cd95fc6..56fc17f1bff 100644 --- a/.github/workflows/enhance.yml +++ b/.github/workflows/enhance.yml @@ -70,9 +70,9 @@ jobs: make -f .trusted-base/Makefile supersede-meta-tag-reviews \ TOOLS_DIR=.trusted-base/tools - # Carries the file-level "Commit suggestion" comments. Its review body is a - # stamped pointer only, because the action posts nothing at all when every - # suggestion duplicates one from an earlier run. + # File-level "Commit suggestion" comments. suggestion_note is intentionally + # unstamped so supersede does not collapse this review; Conversation then + # keeps showing live suggestions until GitHub marks each comment outdated. - name: Suggest meta tag changes if: >- ${{ !cancelled() diff --git a/tools/README.md b/tools/README.md index eb7455c8038..05508c4babc 100644 --- a/tools/README.md +++ b/tools/README.md @@ -223,7 +223,7 @@ The Enhance workflow installs PyYAML in the job; it does not install the full do 4. **Ensure documentation metadata** — `git fetch` the base SHA, then `make -f .trusted-base/Makefile ensure-meta-tags` with `TOOLS_DIR=.trusted-base/tools`, `DIFF_BASE`, and `STATUS_FILE=$GITHUB_OUTPUT`. Emits per-file annotations; the step uses `continue-on-error: true` so warning-only gaps do not fail the job immediately. 5. **Verify metadata check ran** — fail the job if `meta_checked` is empty. The ensure step soft-fails by design, so a missing output is the only way to tell a crashed check from a clean run. 6. **Supersede stale meta-tag reviews** (only if `meta_checked=true`) — `make -f .trusted-base/Makefile supersede-meta-tag-reviews`, which minimises stamped reviews and writes nothing back. -7. **Suggest meta tag changes** — if `inline_suggestions`, run [`suggest-changes`](https://github.com/marketplace/actions/suggest-changes-action) for file-level “Commit suggestion” comments, using `suggestion_note` as its stamped review body. +7. **Suggest meta tag changes** — if `inline_suggestions`, run [`suggest-changes`](https://github.com/marketplace/actions/suggest-changes-action) for file-level “Commit suggestion” comments, using `suggestion_note` as its review body (unstamped so supersede does not hide live suggestions in Conversation). 8. **Post meta tag review comment** — if `has_results`, post the stamped `comment` body via `gh pr review` (Conversation view). Independent of suggest-changes, which posts nothing when every suggestion duplicates one from an earlier run. 9. **Enforce required metadata** — if `has_errors`, fail the job (runs `always()` so error gaps fail even when the ensure step soft-failed). @@ -252,9 +252,9 @@ The script writes **CI outputs** (for example `$GITHUB_OUTPUT`) that describe wh | `has_results` | Metadata issues remain; post the Conversation review | | `has_errors` | Unresolved **error**-severity fields (triggers the final enforce step) | | `comment` | Full stamped review body (multiline heredoc) posted to Conversation via `gh pr review`; written only when `has_results` | -| `suggestion_note` | Short stamped body for the suggest-changes review; written only when `inline_suggestions` | +| `suggestion_note` | Short unstamped body for the suggest-changes review; written only when `inline_suggestions` | -Both review bodies carry the marker, so the next run minimises the summary and the suggestion note together. The outputs are self-consistent by construction: `comment` exists whenever `has_results` is true, `suggestion_note` exists whenever `inline_suggestions` is true, and `has_errors` implies `has_results`. No step can therefore run with an empty review body, and the enforce step cannot fail the job without a review having been posted. +Only the summary `comment` carries the hidden marker, so supersede replaces that review each run while suggestion-carrying reviews stay expanded. The outputs are self-consistent by construction: `comment` exists whenever `has_results` is true, `suggestion_note` exists whenever `inline_suggestions` is true, and `has_errors` implies `has_results`. No step can therefore run with an empty review body, and the enforce step cannot fail the job without a review having been posted. The ensure step uses `continue-on-error: true`, so warning-only gaps do not fail the job. Error-severity gaps (for example `area`) still fail the workflow on the enforce step after contributors receive review feedback. @@ -271,14 +271,14 @@ Annotations describe the pull request **as pushed**, so fields with a configured #### Superseding outdated reviews -Each bot review body includes a hidden HTML marker (``). The marker id `ros2-meta-tags-ensure` (constant `REVIEW_MARKER_ID` in Python; override in the shell script with `META_TAG_REVIEW_MARKER_ID`) is what the supersede script searches for in review bodies. +Only the **summary** review body includes a hidden HTML marker (``). The marker id `ros2-meta-tags-ensure` (constant `REVIEW_MARKER_ID` in Python; override in the shell script with `META_TAG_REVIEW_MARKER_ID`) is what the supersede script searches for in review bodies. The suggest-changes review uses an unstamped `suggestion_note` so it is not minimised: that card is the only place inline suggestions render in Conversation, and GitHub marks individual suggestion comments outdated once they are committed or their anchor leaves the diff. When `meta_checked=true`, the workflow runs `make -f .trusted-base/Makefile supersede-meta-tag-reviews` ([`supersede_meta_tag_reviews.sh`](supersede_meta_tag_reviews.sh)): 1. Lists pull request reviews whose body contains the marker id. 2. Minimises each as **Outdated** via the GitHub GraphQL API (`gh api graphql`; individual failures are ignored). -When all issues are fixed (`has_results=false`), stale reviews are minimised and no new “all clear” comment is posted. Inline suggestion comments are separate from the review body and stay visible until they are committed or the file changes. +When all issues are fixed (`has_results=false`), stale summary reviews are minimised and no new “all clear” comment is posted. Inline suggestion comments remain on the pull request until they are actioned; suggest-changes skips re-posting duplicates, so earlier suggestion reviews stay the Conversation surface for pending commits. #### Security diff --git a/tools/ensure_meta_tags.py b/tools/ensure_meta_tags.py index 1e758ec20ff..86fd23dc83a 100644 --- a/tools/ensure_meta_tags.py +++ b/tools/ensure_meta_tags.py @@ -51,8 +51,9 @@ REVIEW_MARKER_ID = "ros2-meta-tags-ensure" REVIEW_MARKER = f"" -# Header for the separate review that carries the inline suggestions. It is stamped -# like the main body so both reviews are superseded together on the next run. +# Short body for the suggest-changes review. Deliberately unstamped: that review is +# the only Conversation surface that shows live inline suggestions, so it must not be +# minimised with the summary. GitHub marks individual comments outdated when actioned. SUGGESTION_NOTE = ( "Inline suggestions add configured documentation metadata defaults. " "See the metadata review comment for the full list of fields to complete." @@ -404,11 +405,7 @@ def _write_ci_status_file( if results: _write_multiline_output(f, "comment", build_review_comment(results, rules)) if has_inline_suggestions: - _write_multiline_output( - f, - "suggestion_note", - stamp_review_comment(SUGGESTION_NOTE), - ) + _write_multiline_output(f, "suggestion_note", SUGGESTION_NOTE) def _span_overlaps(span: tuple[int, int] | None, pr_lines: set[int]) -> bool: diff --git a/tools/supersede_meta_tag_reviews.sh b/tools/supersede_meta_tag_reviews.sh index 0026696d4a8..66d3563c9c2 100755 --- a/tools/supersede_meta_tag_reviews.sh +++ b/tools/supersede_meta_tag_reviews.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash -# Minimise prior bot PR reviews stamped by ensure_meta_tags.py. +# Minimise prior stamped summary reviews from ensure_meta_tags.py. Suggestion-carrying +# reviews (unstamped suggest-changes bodies) are left visible so Conversation keeps +# live inline suggestions until they are actioned. set -euo pipefail : "${GH_TOKEN:?GH_TOKEN is required}" diff --git a/tools/tests/test_ensure_meta_tags.py b/tools/tests/test_ensure_meta_tags.py index 3b2ae2ca94d..4cdb5b0ca62 100644 --- a/tools/tests/test_ensure_meta_tags.py +++ b/tools/tests/test_ensure_meta_tags.py @@ -258,6 +258,20 @@ def test_local_exit_nonzero_only_for_error_severity(self) -> None: self.assertEqual(code, 1) +def _extract_multiline_output(status: str, key: str) -> str | None: + """Return the body of a GitHub Actions heredoc output block, or None if absent.""" + delimiter = f"EOF_{key.upper()}" + header = f"{key}<<{delimiter}\n" + start = status.find(header) + if start < 0: + return None + start += len(header) + end = status.find(f"\n{delimiter}\n", start) + if end < 0: + return None + return status[start:end] + + class TestCiStatusOutputs(unittest.TestCase): """The workflow gates steps on these outputs, so keep them self-consistent.""" @@ -283,7 +297,12 @@ def test_suggestion_note_written_with_inline_suggestions(self) -> None: status = self._run_with_status_file("Title\n=====\n") self.assertIn("inline_suggestions=true", status) self.assertIn("suggestion_note<<", status) - self.assertIn(REVIEW_MARKER, status) + suggestion_note = _extract_multiline_output(status, "suggestion_note") + comment = _extract_multiline_output(status, "comment") + self.assertIsNotNone(suggestion_note) + self.assertIsNotNone(comment) + self.assertNotIn(REVIEW_MARKER, suggestion_note or "") + self.assertIn(REVIEW_MARKER, comment or "") def test_no_suggestion_note_without_inline_suggestions(self) -> None: content = textwrap.dedent( From 1fd8f757c1e9f70a7b9f0496a52b6cff74296f50 Mon Sep 17 00:00:00 2001 From: Keith Kirkwood Date: Wed, 29 Jul 2026 16:44:32 +0100 Subject: [PATCH 14/23] OPENR-174: Improve review comments with headers and for clarity --- tools/README.md | 10 ++--- tools/ensure_meta_tags.py | 66 ++++++++++++++++++++-------- tools/supersede_meta_tag_reviews.sh | 2 +- tools/tests/test_ensure_meta_tags.py | 14 +++++- 4 files changed, 65 insertions(+), 27 deletions(-) diff --git a/tools/README.md b/tools/README.md index 05508c4babc..bb41495e2f3 100644 --- a/tools/README.md +++ b/tools/README.md @@ -133,13 +133,13 @@ Reviews, annotations, and the soft-failed ensure step appear in different parts #### Review body sections -The review comment names every affected file and splits the work by how it is fixed, so the fields listed for a file match that file's annotation: +The **Documentation metadata** summary review (`## Documentation metadata`) names every affected file and splits the work by how it is fixed, so the fields listed for a file match that file's annotation. The **Inline metadata suggestions** review (`## Inline metadata suggestions`) is a short pointer to the commit suggestions; details stay in the summary. -| Section | Files listed | Fields listed | +| Heading | Files listed | Fields listed | |---------|--------------|---------------| -| Inline suggestions | `suggestable` mode | Configured values added for you — commit the suggestion | -| Copy-paste blocks | `snippet` mode | Configured values as an RST block to paste yourself | -| Non-empty values required | Any file with manual fields | Fields with an empty `value`, labelled required or warning | +| `### Commit inline suggestions` | `suggestable` mode | Configured values added for you — commit the suggestion | +| `### Copy-paste \`.. meta::\` blocks` | `snippet` mode | Configured values as an RST block to paste yourself | +| `### Provide non-empty values` | Any file with manual fields | Fields with an empty `value`, labelled required or optional | A file can appear in two sections: the suggestion covers its configured values while the manual list covers the rest. diff --git a/tools/ensure_meta_tags.py b/tools/ensure_meta_tags.py index 86fd23dc83a..51a7a9a64eb 100644 --- a/tools/ensure_meta_tags.py +++ b/tools/ensure_meta_tags.py @@ -51,12 +51,24 @@ REVIEW_MARKER_ID = "ros2-meta-tags-ensure" REVIEW_MARKER = f"" +# Titles and section headings for pull request review bodies (GitHub Markdown). +SUMMARY_REVIEW_TITLE = "## Documentation metadata" +SUGGESTION_REVIEW_TITLE = "## Inline metadata suggestions" +SECTION_INLINE_SUGGESTIONS = "### Commit inline suggestions" +SECTION_COPY_PASTE_BLOCKS = "### Copy-paste `.. meta::` blocks" +SECTION_NON_EMPTY_VALUES = "### Provide non-empty values" + # Short body for the suggest-changes review. Deliberately unstamped: that review is # the only Conversation surface that shows live inline suggestions, so it must not be # minimised with the summary. GitHub marks individual comments outdated when actioned. SUGGESTION_NOTE = ( - "Inline suggestions add configured documentation metadata defaults. " - "See the metadata review comment for the full list of fields to complete." + f"{SUGGESTION_REVIEW_TITLE}\n" + "\n" + "Each **Commit suggestion** below adds configured documentation metadata " + "defaults from `tools/meta_tags.yaml`.\n" + "\n" + "For copy-paste blocks, required fields, and the full per-file breakdown, " + "see the **Documentation metadata** review comment." ) @@ -695,7 +707,7 @@ def _field_list_markdown(field_names: list[str], rules: dict[str, MetaRule]) -> """ parts: list[str] = [] for name in field_names: - label = "required" if rules[name].severity == "error" else "warning" + label = "required" if rules[name].severity == "error" else "optional" parts.append(f"`{name}` ({label})") return ", ".join(parts) @@ -719,30 +731,40 @@ def build_review_comment( manual_fields_modes = [r for r in results if r["mode"] == "manual_fields"] lines = [ + SUMMARY_REVIEW_TITLE, + "", "This pull request is missing configured documentation metadata " "(see `tools/meta_tags.yaml`).", "", ] if inline_modes: - lines.append( - "Please **review and commit the inline suggestions**. They add these " - "configured default values in place so metadata stays complete:", + lines.extend( + [ + SECTION_INLINE_SUGGESTIONS, + "", + "Please **review and commit the inline suggestions** on the " + "**Files changed** tab (or use the separate *Inline metadata " + "suggestions* review). They add these configured default values:", + "", + ] ) - lines.append("") for result in inline_modes: auto = ", ".join(f"`{name}`" for name in result["auto_fields"]) - lines.append(f"**`{result['path']}`**: {auto}") + lines.append(f"- **`{result['path']}`**: {auto}") lines.append("") if snippet_modes: - lines.append( - "GitHub can only attach suggestions to lines already in the pull request " - "diff, so the following files could not get an inline suggestion for " - "auto-filled fields. Please add this block at the **top of each file** " - "(or append the listed fields to an existing `.. meta::` block):", + lines.extend( + [ + SECTION_COPY_PASTE_BLOCKS, + "", + "These files could not receive inline suggestions because the edits are " + "outside the pull request diff. Add this block at the **top of each " + "file** (or append the listed fields to an existing `.. meta::` block):", + "", + ] ) - lines.append("") for result in snippet_modes: lines.append(f"**`{result['path']}`**") lines.append("```rst") @@ -752,14 +774,20 @@ def build_review_comment( manual_results = [r for r in results if r.get("manual_fields")] if manual_results: - lines.append( - "The following fields must be provided with **non-empty** values in each " - "file's `.. meta::` block:", + lines.extend( + [ + SECTION_NON_EMPTY_VALUES, + "", + "These fields must have **non-empty** values in each file's " + "`.. meta::` block:", + "", + ] ) - lines.append("") for result in manual_results: manual = list(result["manual_fields"]) - lines.append(f"**`{result['path']}`**: {_field_list_markdown(manual, rules)}") + lines.append( + f"- **`{result['path']}`**: {_field_list_markdown(manual, rules)}" + ) lines.append("") if manual_fields_modes and not inline_modes and not snippet_modes: diff --git a/tools/supersede_meta_tag_reviews.sh b/tools/supersede_meta_tag_reviews.sh index 66d3563c9c2..315f18172ad 100755 --- a/tools/supersede_meta_tag_reviews.sh +++ b/tools/supersede_meta_tag_reviews.sh @@ -29,4 +29,4 @@ for node_id in "${review_ids[@]:-}"; do gh api graphql -f query="${minimize_query}" -f subjectId="${node_id}" || true done -echo "Superseded ${#review_ids[@]} prior meta-tag review(s)." +echo "Superseded ${#review_ids[@]} prior review comment(s)." diff --git a/tools/tests/test_ensure_meta_tags.py b/tools/tests/test_ensure_meta_tags.py index 4cdb5b0ca62..e35e0ecd431 100644 --- a/tools/tests/test_ensure_meta_tags.py +++ b/tools/tests/test_ensure_meta_tags.py @@ -27,6 +27,9 @@ from ensure_meta_tags import ( # noqa: E402 REVIEW_MARKER, + SECTION_INLINE_SUGGESTIONS, + SECTION_NON_EMPTY_VALUES, + SUMMARY_REVIEW_TITLE, MetaRule, _unresolved_fields, build_review_comment, @@ -196,6 +199,8 @@ def test_build_review_comment_lists_manual_fields(self) -> None: }, ] body = build_review_comment(results, rules) + self.assertIn(SUMMARY_REVIEW_TITLE, body) + self.assertIn(SECTION_NON_EMPTY_VALUES, body) self.assertIn("area", body) self.assertIn("required", body) self.assertIn("experience", body) @@ -218,8 +223,11 @@ def test_build_review_comment_lists_inline_suggestion_fields(self) -> None: }, ] body = build_review_comment(results, rules) - self.assertIn("**`source/Page.rst`**: `product`", body) - self.assertIn("`experience` (warning)", body) + self.assertIn(SUMMARY_REVIEW_TITLE, body) + self.assertIn(SECTION_INLINE_SUGGESTIONS, body) + self.assertIn(SECTION_NON_EMPTY_VALUES, body) + self.assertIn("- **`source/Page.rst`**: `product`", body) + self.assertIn("`experience` (optional)", body) def test_local_exit_nonzero_only_for_error_severity(self) -> None: warning_config = textwrap.dedent( @@ -301,7 +309,9 @@ def test_suggestion_note_written_with_inline_suggestions(self) -> None: comment = _extract_multiline_output(status, "comment") self.assertIsNotNone(suggestion_note) self.assertIsNotNone(comment) + self.assertIn("## Inline metadata suggestions", suggestion_note or "") self.assertNotIn(REVIEW_MARKER, suggestion_note or "") + self.assertIn(SUMMARY_REVIEW_TITLE, comment or "") self.assertIn(REVIEW_MARKER, comment or "") def test_no_suggestion_note_without_inline_suggestions(self) -> None: From 6420fa4f09912a2456142aeeeb40942dc3b80358 Mon Sep 17 00:00:00 2001 From: Keith Kirkwood Date: Thu, 30 Jul 2026 15:12:46 +0100 Subject: [PATCH 15/23] OPENR-174: Add short description and showmeta directives --- conf.py | 2 + plugins/meta_util.py | 70 ++++++++++++++++++++ plugins/short_description.py | 32 ++++++++++ plugins/showmeta.py | 120 +++++++++++++++++++++++++++++++++++ source/_static/custom.css | 7 ++ 5 files changed, 231 insertions(+) create mode 100644 plugins/meta_util.py create mode 100644 plugins/short_description.py create mode 100644 plugins/showmeta.py diff --git a/conf.py b/conf.py index de1ab02d9ce..1a2d5768ee4 100644 --- a/conf.py +++ b/conf.py @@ -90,6 +90,8 @@ 'sphinxcontrib.googleanalytics', 'sphinxcontrib.mermaid', 'sphinxext.opengraph', + 'short_description', + 'showmeta' ] # Intersphinx mapping diff --git a/plugins/meta_util.py b/plugins/meta_util.py new file mode 100644 index 00000000000..6242ef196b0 --- /dev/null +++ b/plugins/meta_util.py @@ -0,0 +1,70 @@ +# Copyright 2026 Open Robotics — shared helpers for ``.. meta::`` / Pagefind +""" +Collect every ``.. meta::`` field from the doctree, sanitize keys, and expand +``{MACRO}`` placeholders using the Sphinx ``macros`` config (longest keys first). + +Sphinx / the HTML theme may also emit plain ```` tags for the same fields. +The Pagefind extension emits additional tags with ``data-pagefind-filter`` and may +split comma-separated values into multiple tags for faceted search. +""" + +from __future__ import annotations + +import re +from typing import Dict, List, Optional + +from docutils import nodes + +# HTML ```` names should be conservative; allow common patterns. +_META_NAME_RE = re.compile(r'^[A-Za-z0-9_.:-]+$') + + +def sanitize_meta_key(raw: str) -> Optional[str]: + s = str(raw).strip() + if not s or not _META_NAME_RE.match(s): + return None + return s + + +def all_doctree_meta(doctree: Optional[nodes.document]) -> Dict[str, str]: + """Return last-wins mapping of every ``nodes.meta`` ``name``/``property`` → ``content``.""" + if doctree is None: + return {} + + out: Dict[str, str] = {} + for meta in doctree.findall(nodes.meta): + if meta.get('http-equiv'): + continue + content = meta.get('content') + if not content: + continue + key: Optional[str] = None + name = meta.get('name') + if name: + key = sanitize_meta_key(str(name)) + else: + prop = meta.get('property') + if prop: + key = sanitize_meta_key(str(prop)) + if not key: + continue + out[key] = str(content).strip() + return out + + +def expand_meta_macros(text: str, macros: Dict[str, str]) -> str: + """Expand ``{KEY}`` placeholders; longer macro names first to avoid partial matches.""" + result = text + for key, value in sorted(macros.items(), key=lambda kv: len(kv[0]), reverse=True): + result = result.replace(f'{{{key}}}', value) + return result + + +def expand_all_meta_values(meta: Dict[str, str], macros: Dict[str, str]) -> Dict[str, str]: + """Apply ``expand_meta_macros`` to every meta value.""" + return {k: expand_meta_macros(v, macros) for k, v in meta.items()} + + +def split_meta_values(value: str) -> List[str]: + """Return comma-separated metadata values as individual Pagefind values.""" + return [part.strip() for part in value.split(',') if part.strip()] \ No newline at end of file diff --git a/plugins/short_description.py b/plugins/short_description.py new file mode 100644 index 00000000000..9b275626162 --- /dev/null +++ b/plugins/short_description.py @@ -0,0 +1,32 @@ +from __future__ import annotations + +from docutils import nodes +from sphinx.util.docutils import SphinxDirective + + +class ShortDescriptionDirective(SphinxDirective): + """Directive to render the short description of an article.""" + + has_content = True + required_arguments = 0 + optional_arguments = 0 + option_spec = {} + + def run(self) -> list[nodes.Node]: + # Create a container node to hold the parsed content + node = nodes.container() + node['classes'].append('short-description') + + # Parse the directive content into the container node + self.state.nested_parse(self.content, self.content_offset, node) + + return [node] + + +def setup(app): + app.add_directive('short-description', ShortDescriptionDirective) + return { + 'parallel_read_safe': True, + 'parallel_write_safe': True, + 'version': '0.1.0', + } \ No newline at end of file diff --git a/plugins/showmeta.py b/plugins/showmeta.py new file mode 100644 index 00000000000..f844b2db15c --- /dev/null +++ b/plugins/showmeta.py @@ -0,0 +1,120 @@ +# Copyright 2026 Open Robotics — explicit in-body ``.. showmeta::`` summary +""" +Render selected ``.. meta::`` fields in the document body with author-controlled +order and labels. Place ``.. showmeta::`` where the summary should appear (HTML only). +""" + +from __future__ import annotations + +import html as html_module +import re +from typing import List + +from docutils import nodes +from docutils.parsers.rst import directives +from sphinx.util.docutils import SphinxDirective + +from meta_util import all_doctree_meta, expand_all_meta_values + + +def _macros_flat(app) -> dict[str, str]: + return {str(k): str(v) for k, v in (getattr(app.config, 'macros', {}) or {}).items()} + + +def _default_showmeta_label(key: str) -> str: + spaced = re.sub(r'([a-z])([A-Z])', r'\1 \2', key) + return spaced.replace('_', ' ').replace('-', ' ').strip().title() + + +class showmeta_node(nodes.General, nodes.Element): + """Placeholder replaced on ``doctree-resolved`` (HTML builds only).""" + + +class ShowMetaDirective(SphinxDirective): + """Insert a visible metadata line built from ``.. meta::`` on this page.""" + + has_content = False + option_spec = { + 'order': directives.unchanged, + 'labels': directives.unchanged, + } + + def run(self) -> List[nodes.Node]: + node = showmeta_node() + node['order'] = self.options.get('order', '') + node['labels'] = self.options.get('labels', '') + self.set_source_info(node) + return [node] + + +def visit_skip_showmeta(self, node: showmeta_node) -> None: + raise nodes.SkipNode + + +def depart_showmeta_noop(self, node: showmeta_node) -> None: + pass + + +def _parse_labels(raw: str) -> dict[str, str]: + out: dict[str, str] = {} + for part in [p.strip() for p in raw.split(',') if p.strip() and '=' in p]: + key, _, value = part.partition('=') + key, value = key.strip(), value.strip() + if key: + out[key] = value + return out + + +def replace_showmeta_nodes(app, doctree: nodes.document, docname: str) -> None: + if app.builder.format != 'html': + for node in list(doctree.findall(showmeta_node)): + node.parent.remove(node) + return + + macros = _macros_flat(app) + meta = expand_all_meta_values(all_doctree_meta(doctree), macros) + + for node in list(doctree.findall(showmeta_node)): + order = [x.strip() for x in node.get('order', '').split(',') if x.strip()] + labels_map = _parse_labels(node.get('labels', '')) + if not order: + node.parent.remove(node) + continue + + parts: List[str] = [] + for key in order: + val = meta.get(key, '').strip() + if not val: + continue + label_base = labels_map.get(key) or _default_showmeta_label(key) + label_display = label_base if label_base.rstrip().endswith(':') else f'{label_base}:' + parts.append( + f'{html_module.escape(label_display)} ' + f'{html_module.escape(val)}' + ) + + if not parts: + node.parent.remove(node) + else: + inner = ' | '.join(parts) + raw = nodes.raw( + '', + f'

{inner}

', + format='html', + ) + node.replace_self(raw) + + +def setup(app): + app.add_node( + showmeta_node, + html=(visit_skip_showmeta, depart_showmeta_noop), + latex=(visit_skip_showmeta, depart_showmeta_noop), + ) + app.add_directive('showmeta', ShowMetaDirective) + app.connect('doctree-resolved', replace_showmeta_nodes) + return { + 'version': '1.0.0', + 'parallel_read_safe': True, + 'parallel_write_safe': True, + } \ No newline at end of file diff --git a/source/_static/custom.css b/source/_static/custom.css index 4252f921bb8..99f8209fdbb 100644 --- a/source/_static/custom.css +++ b/source/_static/custom.css @@ -1,3 +1,10 @@ .wy-nav-content { max-width: 64rem; } + +.short-description p{ + font-size: 1.25rem; + line-height: 1.5; + color: #777777; + margin-bottom: 1.5rem; +} From c2d4a7e4d0d7c409191ddbcd2366f60be3c0613f Mon Sep 17 00:00:00 2001 From: Keith Kirkwood Date: Thu, 30 Jul 2026 16:30:25 +0100 Subject: [PATCH 16/23] OPENR-174: First pass at short-description and showmeta suggestions or comments --- Makefile | 2 +- source/First-Steps.rst | 20 +- tools/README.md | 36 +- tools/enhance.yaml | 32 ++ tools/ensure_meta_tags.py | 594 +++++++++++++++++++++++---- tools/meta_tags.yaml | 20 - tools/rst_utils.py | 372 ++++++++++++++++- tools/tests/test_ensure_meta_tags.py | 164 +++++++- tools/tests/test_rst_utils.py | 206 ++++++++++ 9 files changed, 1321 insertions(+), 125 deletions(-) create mode 100644 tools/enhance.yaml delete mode 100644 tools/meta_tags.yaml create mode 100644 tools/tests/test_rst_utils.py diff --git a/Makefile b/Makefile index 7fb617a6bd3..4b75bbe97b9 100644 --- a/Makefile +++ b/Makefile @@ -53,7 +53,7 @@ ifndef STATUS_FILE $(error STATUS_FILE is required) endif $(PYTHON) $(TOOLS_DIR)/ensure_meta_tags.py \ - --config $(TOOLS_DIR)/meta_tags.yaml \ + --config $(TOOLS_DIR)/enhance.yaml \ --diff-base $(DIFF_BASE) \ --status-file $(STATUS_FILE) diff --git a/source/First-Steps.rst b/source/First-Steps.rst index 98cbcb04732..ebf955cd364 100644 --- a/source/First-Steps.rst +++ b/source/First-Steps.rst @@ -1,13 +1,25 @@ +.. meta:: + :description: The ROS framework is the “plumbing” which makes communication between different parts of a robot possible. + :keywords: ROS, framework, communication, robotics, learning path + :area: framework + :contentType: learning-path + :experience: beginner + :product: {PRODUCT} + :distribution: {DISTRO} + .. _First-steps-with-ROS-learning-path: First steps with ROS - learning path ==================================== -ROS (Robot Operating System) is an open-source ecosystem that provides framework, tools, and libraries for building, deploying, running, and maintaining robotic applications. -This page presents a set of articles and hands-on activities to introduce the main concepts behind the ROS framework. -Working through these will give you the essential knowledge needed to start developing applications with ROS. +.. short-description:: + ROS (Robot Operating System) is an open-source ecosystem that provides framework, tools, and libraries for building, deploying, running, and maintaining robotic applications. + This page presents a set of articles and hands-on activities to introduce the main concepts behind the ROS framework. + Working through these will give you the essential knowledge needed to start developing applications with ROS. -**Area: ROS-framework | Content-type: learning-path | Experience: beginner** +.. showmeta:: + :order: area, contentType, experience + :labels: area=Area, contentType=Content type, experience=Level .. contents:: Contents :depth: 2 diff --git a/tools/README.md b/tools/README.md index bb41495e2f3..516bd202166 100644 --- a/tools/README.md +++ b/tools/README.md @@ -18,7 +18,7 @@ Information for documentation contributors creating or updating `.rst` files. ### Metadata configuration -[`meta_tags.yaml`](meta_tags.yaml) defines every `.. meta::` field checked by the tooling. Each entry has: +[`enhance.yaml`](enhance.yaml) defines documentation enhancement rules. The `meta` section lists every `.. meta::` field checked by the tooling. Each entry has: - **`severity`**: `warning` (advisory; soft-fails the ensure step in CI) or `error` (fails the workflow after reviews are posted). - **`value`**: default text to inject when the field is missing or blank. Leave empty when the contributor must supply a non-empty value. @@ -44,6 +44,24 @@ meta: `{PRODUCT}` and `{DISTRO}` are Sphinx substitution macros expanded at build time from [`conf.py`](../conf.py). +The `after_title` section lists directives inserted after the first document title, in order: + +```yaml +after_title: + - directive: short-description + severity: warning + content: first_paragraph + + - directive: showmeta + severity: warning + options: + order: "area, contentType, experience" + required_options: + - order +``` + +For `short-description`, the tool wraps the first prose paragraph after the title into the directive (removing it from the body). For `showmeta`, it inserts the directive with the configured `:order:` option when missing. + #### Severity behaviour | Severity | Missing or blank field | CI ensure step | Workflow job | @@ -53,7 +71,7 @@ meta: ### Checking metadata locally -`ensure_meta_tags.py` checks `.rst` files against `meta_tags.yaml`. Fields with a configured `value` are added or filled automatically when the edit can be suggested or applied locally. Fields with an empty `value` must be completed manually in the `.. meta::` block. +`ensure_meta_tags.py` checks `.rst` files against `enhance.yaml`. Fields with a configured `value` are added or filled automatically when the edit can be suggested or applied locally. Fields with an empty `value` must be completed manually in the `.. meta::` block. After-title directives are added using the rules above. #### Usage @@ -166,8 +184,8 @@ Information for maintainers and developers working on or extending the metadata | File | Purpose | |------|---------| -| [`rst_utils.py`](rst_utils.py) | Regex-based read/write of `.. meta::` and `.. short-description::` directives | -| [`meta_tags.yaml`](meta_tags.yaml) | Metadata rules (severity and optional default values) | +| [`rst_utils.py`](rst_utils.py) | Regex-based read/write of `.. meta::`, `.. short-description::`, and `.. showmeta::` directives | +| [`enhance.yaml`](enhance.yaml) | Enhancement rules (`meta` fields and `after_title` directives) | | [`ensure_meta_tags.py`](ensure_meta_tags.py) | CLI that checks and fixes metadata from the config | | [`supersede_meta_tag_reviews.sh`](supersede_meta_tag_reviews.sh) | Minimise outdated bot PR reviews | | [`tests/`](tests/) | Unit tests for the tools in this directory | @@ -181,19 +199,19 @@ Low-level utilities for locating and editing Sphinx directives in RST source: - **`get_meta_fields_from_content`** — field names and values in the first `.. meta::` block - **`get_meta_names_from_content`** — field names only - **`inject_metadata_to_content`** — add missing fields or fill blank values; never overwrites non-empty contributor values - -The module also contains helpers for `.. short-description::` directives for future use. +- **`wrap_first_paragraph_as_short_description`** — wrap the first prose paragraph after the title +- **`inject_showmeta_to_content`** — insert or fill `.. showmeta::` with configured options #### Extending configuration -Add a new key under `meta` in [`meta_tags.yaml`](meta_tags.yaml) to extend coverage without changing Python code. The script applies every rule in the file. +Add a new key under `meta` in [`enhance.yaml`](enhance.yaml) to extend metadata coverage without changing Python code. Add entries to `after_title` for additional post-heading directives supported by the tooling. ### CLI options & Makefile targets #### Options (`ensure_meta_tags.py`) - `paths` — optional; when omitted, `--diff-base` is required and changed `.rst` files are discovered automatically -- `--config PATH` — YAML config file (default: `tools/meta_tags.yaml`) +- `--config PATH` — YAML config file (default: `tools/enhance.yaml`) - `--diff-base SHA` — PR base commit; limits on-disk writes to lines in the PR diff (inline suggestions); files that need a copy-paste or manual field list use a review comment instead - `--status-file PATH` — write `meta_checked`, `inline_suggestions`, `has_results`, `has_errors`, the review comment body, and the suggestion note for CI; when issues remain, emits annotations and exits `1` (the ensure step uses `continue-on-error`) - `-v` / `--verbose` — enable debug logging @@ -218,7 +236,7 @@ The Enhance workflow installs PyYAML in the job; it does not install the full do #### Job flow (`ensure-meta-tags`) 1. Check out the PR **head** (`.rst` content to inspect and, where allowed, modify for suggestions). -2. Check out the PR **base** into `.trusted-base/` (Makefile, `ensure_meta_tags.py`, `meta_tags.yaml`, `supersede_meta_tag_reviews.sh`). +2. Check out the PR **base** into `.trusted-base/` (Makefile, `ensure_meta_tags.py`, `enhance.yaml`, `supersede_meta_tag_reviews.sh`). 3. Install Python 3.12 and PyYAML. 4. **Ensure documentation metadata** — `git fetch` the base SHA, then `make -f .trusted-base/Makefile ensure-meta-tags` with `TOOLS_DIR=.trusted-base/tools`, `DIFF_BASE`, and `STATUS_FILE=$GITHUB_OUTPUT`. Emits per-file annotations; the step uses `continue-on-error: true` so warning-only gaps do not fail the job immediately. 5. **Verify metadata check ran** — fail the job if `meta_checked` is empty. The ensure step soft-fails by design, so a missing output is the only way to tell a crashed check from a clean run. diff --git a/tools/enhance.yaml b/tools/enhance.yaml new file mode 100644 index 00000000000..a37849affe0 --- /dev/null +++ b/tools/enhance.yaml @@ -0,0 +1,32 @@ +# Enhancement rules for ensure_meta_tags.py. +# meta: .. meta:: field rules (severity and optional default values). +# after_title: directives inserted after the first document title. +# Sphinx expands {PRODUCT} and {DISTRO} at build time (see conf.py). +meta: + product: + severity: warning + value: "{PRODUCT}" + distribution: + severity: warning + value: "{DISTRO}" + area: + severity: error + value: + experience: + severity: warning + value: + content-type: + severity: warning + value: + +after_title: + - directive: short-description + severity: warning + content: first_paragraph + + - directive: showmeta + severity: warning + options: + order: "area, contentType, experience" + required_options: + - order diff --git a/tools/ensure_meta_tags.py b/tools/ensure_meta_tags.py index 51a7a9a64eb..a76869de198 100644 --- a/tools/ensure_meta_tags.py +++ b/tools/ensure_meta_tags.py @@ -1,10 +1,10 @@ #!/usr/bin/env python3 """ -Ensure configured metadata fields exist in RST ``.. meta::`` blocks. +Ensure configured documentation enhancements exist in RST source files. -Rules are defined in ``meta_tags.yaml`` with severity and optional values. -Missing or blank fields are resolved automatically when a value is configured; -otherwise contributors must supply a non-empty value. +Rules are defined in ``enhance.yaml``: ``meta`` field rules for ``.. meta::`` +blocks and ``after_title`` rules for post-heading directives such as +``.. short-description::`` and ``.. showmeta::``. When ``--diff-base`` is set, edits are only written to disk when they overlap the pull request diff (so GitHub can offer inline suggestions). Otherwise the @@ -34,15 +34,22 @@ sys.path.insert(0, str(_TOOLS_DIR)) from rst_utils import ( + after_title_directives_line_span, + extract_first_paragraph_after_title, + format_showmeta_block, get_meta_fields_from_content, has_meta_block, + has_short_description_content, + has_showmeta_with_order, inject_metadata_to_content, + inject_showmeta_to_content, meta_block_line_span, + wrap_first_paragraph_as_short_description, ) logger = logging.getLogger(__name__) -DEFAULT_CONFIG_PATH = _TOOLS_DIR / "meta_tags.yaml" +DEFAULT_CONFIG_PATH = _TOOLS_DIR / "enhance.yaml" RST_EXTENSION = ".rst" _HUNK_HEADER = re.compile(r"^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@") Severity = Literal["warning", "error"] @@ -52,11 +59,13 @@ REVIEW_MARKER = f"" # Titles and section headings for pull request review bodies (GitHub Markdown). -SUMMARY_REVIEW_TITLE = "## Documentation metadata" -SUGGESTION_REVIEW_TITLE = "## Inline metadata suggestions" +SUMMARY_REVIEW_TITLE = "## Documentation enhancements" +SUGGESTION_REVIEW_TITLE = "## Inline documentation suggestions" SECTION_INLINE_SUGGESTIONS = "### Commit inline suggestions" SECTION_COPY_PASTE_BLOCKS = "### Copy-paste `.. meta::` blocks" +SECTION_COPY_PASTE_AFTER_TITLE = "### Copy-paste after-title directives" SECTION_NON_EMPTY_VALUES = "### Provide non-empty values" +SECTION_AFTER_TITLE_MANUAL = "### Add after-title directives" # Short body for the suggest-changes review. Deliberately unstamped: that review is # the only Conversation surface that shows live inline suggestions, so it must not be @@ -64,18 +73,46 @@ SUGGESTION_NOTE = ( f"{SUGGESTION_REVIEW_TITLE}\n" "\n" - "Each **Commit suggestion** below adds configured documentation metadata " - "defaults from `tools/meta_tags.yaml`.\n" + "Each **Commit suggestion** below applies configured documentation " + "enhancements from `tools/enhance.yaml`.\n" "\n" "For copy-paste blocks, required fields, and the full per-file breakdown, " - "see the **Documentation metadata** review comment." + "see the **Documentation enhancements** review comment." ) +@dataclass(frozen=True) +class AfterTitleRule: + """ + A single after-title directive rule from ``enhance.yaml``. + + Attributes: + directive: Directive name (e.g. ``short-description``, ``showmeta``). + severity: Advisory ``warning`` or blocking ``error`` in CI. + content: Content source for body directives (e.g. ``first_paragraph``). + options: Option name/value pairs for option-only directives. + required_options: Option names that must be non-empty when present. + """ + + directive: str + severity: Severity + content: str | None = None + options: dict[str, str] | None = None + required_options: tuple[str, ...] = () + + +@dataclass(frozen=True) +class EnhanceConfig: + """Full enhancement configuration loaded from ``enhance.yaml``.""" + + meta: dict[str, MetaRule] + after_title: tuple[AfterTitleRule, ...] + + @dataclass(frozen=True) class MetaRule: """ - A single metadata field rule from ``meta_tags.yaml``. + A single metadata field rule from ``enhance.yaml``. Attributes: severity: Advisory ``warning`` or blocking ``error`` in CI. @@ -97,38 +134,8 @@ def has_configured_value(self) -> bool: return bool(self.value.strip()) -def load_meta_config(config_path: Path) -> dict[str, MetaRule]: - """ - Load and validate metadata rules from a YAML config file. - - Args: - config_path: Path to the YAML configuration file. - - Returns: - Mapping from meta field name to its rule. - - Raises: - SystemExit: If the file is missing, invalid, or has unusable rules. - """ - if not config_path.is_file(): - logger.error("Config file not found: %s", config_path) - raise SystemExit(1) - - try: - raw = yaml.safe_load(config_path.read_text(encoding="utf-8")) - except yaml.YAMLError as exc: - logger.error("Invalid YAML in %s: %s", config_path, exc) - raise SystemExit(1) from exc - - if not isinstance(raw, dict): - logger.error("Config %s must be a YAML mapping", config_path) - raise SystemExit(1) - - meta = raw.get("meta") - if not isinstance(meta, dict) or not meta: - logger.error("Config %s must contain a non-empty 'meta' mapping", config_path) - raise SystemExit(1) - +def _parse_meta_rules(meta: dict, config_path: Path) -> dict[str, MetaRule]: + """Validate and parse the ``meta`` mapping from config YAML.""" validated: dict[str, MetaRule] = {} for key, entry in meta.items(): if not isinstance(key, str) or not key.strip(): @@ -167,10 +174,178 @@ def load_meta_config(config_path: Path) -> dict[str, MetaRule]: ) raise SystemExit(1) validated[key] = MetaRule(severity=severity, value=value) - return validated +def _parse_after_title_rules( + raw_list: object, + config_path: Path, +) -> tuple[AfterTitleRule, ...]: + """Validate and parse the ``after_title`` list from config YAML.""" + if raw_list is None: + return () + if not isinstance(raw_list, list): + logger.error("Config %s: 'after_title' must be a list", config_path) + raise SystemExit(1) + + supported_directives = {"short-description", "showmeta"} + validated: list[AfterTitleRule] = [] + for index, entry in enumerate(raw_list): + if not isinstance(entry, dict): + logger.error( + "Config %s: after_title[%d] must be a mapping", + config_path, + index, + ) + raise SystemExit(1) + directive = entry.get("directive") + if not isinstance(directive, str) or not directive.strip(): + logger.error( + "Config %s: after_title[%d] must include a non-empty 'directive'", + config_path, + index, + ) + raise SystemExit(1) + if directive not in supported_directives: + logger.error( + "Config %s: after_title[%d] directive %r is not supported", + config_path, + index, + directive, + ) + raise SystemExit(1) + severity = entry.get("severity") + if severity not in ("warning", "error"): + logger.error( + "Config %s: after_title[%d] severity must be 'warning' or 'error', got %r", + config_path, + index, + severity, + ) + raise SystemExit(1) + + content = entry.get("content") + if content is not None and not isinstance(content, str): + logger.error( + "Config %s: after_title[%d] content must be a string", + config_path, + index, + ) + raise SystemExit(1) + + raw_options = entry.get("options") + options: dict[str, str] | None = None + if raw_options is not None: + if not isinstance(raw_options, dict): + logger.error( + "Config %s: after_title[%d] options must be a mapping", + config_path, + index, + ) + raise SystemExit(1) + options = {} + for opt_key, opt_value in raw_options.items(): + if not isinstance(opt_key, str) or not isinstance(opt_value, str): + logger.error( + "Config %s: after_title[%d] option keys and values must be strings", + config_path, + index, + ) + raise SystemExit(1) + options[opt_key] = opt_value + + raw_required = entry.get("required_options") + required_options: tuple[str, ...] = () + if raw_required is not None: + if not isinstance(raw_required, list): + logger.error( + "Config %s: after_title[%d] required_options must be a list", + config_path, + index, + ) + raise SystemExit(1) + required_options = tuple(str(item) for item in raw_required) + + if directive == "short-description": + if content != "first_paragraph": + logger.error( + "Config %s: short-description rule must use content: first_paragraph", + config_path, + ) + raise SystemExit(1) + elif directive == "showmeta": + if not options or not options.get("order", "").strip(): + logger.error( + "Config %s: showmeta rule must include options.order", + config_path, + ) + raise SystemExit(1) + if "order" not in required_options: + logger.error( + "Config %s: showmeta rule must list order in required_options", + config_path, + ) + raise SystemExit(1) + + validated.append( + AfterTitleRule( + directive=directive, + severity=severity, + content=content, + options=options, + required_options=required_options, + ), + ) + return tuple(validated) + + +def load_enhance_config(config_path: Path) -> EnhanceConfig: + """ + Load and validate enhancement rules from a YAML config file. + + Args: + config_path: Path to the YAML configuration file. + + Returns: + Parsed enhancement configuration. + + Raises: + SystemExit: If the file is missing, invalid, or has unusable rules. + """ + if not config_path.is_file(): + logger.error("Config file not found: %s", config_path) + raise SystemExit(1) + + try: + raw = yaml.safe_load(config_path.read_text(encoding="utf-8")) + except yaml.YAMLError as exc: + logger.error("Invalid YAML in %s: %s", config_path, exc) + raise SystemExit(1) from exc + + if not isinstance(raw, dict): + logger.error("Config %s must be a YAML mapping", config_path) + raise SystemExit(1) + + meta = raw.get("meta") + if not isinstance(meta, dict) or not meta: + logger.error("Config %s must contain a non-empty 'meta' mapping", config_path) + raise SystemExit(1) + + return EnhanceConfig( + meta=_parse_meta_rules(meta, config_path), + after_title=_parse_after_title_rules(raw.get("after_title"), config_path), + ) + + +def load_meta_config(config_path: Path) -> dict[str, MetaRule]: + """ + Load metadata rules from a YAML config file. + + Deprecated alias for ``load_enhance_config(...).meta``. + """ + return load_enhance_config(config_path).meta + + def format_meta_block(rules: dict[str, MetaRule], fields: list[str]) -> str: """ Build an RST ``.. meta::`` block for auto-injectable fields. @@ -445,14 +620,33 @@ def _annotation_line_for_content(content: str) -> int: content: RST source to inspect. Returns: - The first line of an existing meta block, or line 1 if none exists. + The first line of an existing meta block, the after-title area, or line 1. """ span = meta_block_line_span(content) if span is not None: return span[0] + after_title = after_title_directives_line_span(content) + if after_title is not None: + return after_title[0] return 1 +def _emit_after_title_annotation( + level: str, + path: str, + directives: list[str], + line: int, +) -> None: + """Print a GitHub Actions annotation for missing after-title directives.""" + if not directives: + return + directive_list = ", ".join(directives) + message = _escape_workflow_command_message( + f"Missing after-title directives: {directive_list}", + ) + print(f"::{level} file={path},line={line}::{message}") + + def _escape_workflow_command_message(message: str) -> str: """ Escape a message for use in a GitHub Actions workflow command. @@ -542,6 +736,148 @@ def can_suggest_inline(content: str, pr_lines: set[int]) -> bool: return _span_overlaps((1, 1), pr_lines) +def can_suggest_after_title_inline( + content: str, + pr_lines: set[int], + *, + paragraph_span: tuple[int, int] | None = None, +) -> bool: + """ + Return whether after-title directive edits can anchor to the PR diff. + + Args: + content: RST source before after-title injections. + pr_lines: One-based lines represented by the pull-request diff. + paragraph_span: Optional line span of the paragraph to wrap. + + Returns: + ``True`` when the after-title area or paragraph span overlaps the diff. + """ + if paragraph_span is not None and _span_overlaps(paragraph_span, pr_lines): + return True + return _span_overlaps(after_title_directives_line_span(content), pr_lines) + + +def _after_title_rule_satisfied(content: str, rule: AfterTitleRule) -> bool: + """Return whether an after-title rule is already satisfied in ``content``.""" + if rule.directive == "short-description": + return has_short_description_content(content) + if rule.directive == "showmeta": + return has_showmeta_with_order(content) + return True + + +def _format_short_description_snippet(paragraph: str) -> str: + """Build a copy-paste ``.. short-description::`` block for a paragraph.""" + lines = [".. short-description::"] + for chunk in paragraph.split("\n\n"): + for line in chunk.split("\n"): + stripped = line.strip() + if stripped: + lines.append(f" {stripped}") + lines.append("") + return "\n".join(lines) + + +def _process_after_title_rules( + path: Path, + content: str, + after_title_rules: tuple[AfterTitleRule, ...], + *, + pr_lines: set[int] | None, +) -> tuple[str, dict[str, object]]: + """ + Apply configured after-title directive rules to RST content. + + Returns: + Updated content and a dict of after-title result fields. + """ + unresolved_rules = [rule for rule in after_title_rules if not _after_title_rule_satisfied(content, rule)] + if not unresolved_rules: + return content, {} + + after_title_auto: list[str] = [] + after_title_manual: list[str] = [] + after_title_warning: list[str] = [] + after_title_error: list[str] = [] + after_title_snippets: list[dict[str, str]] = [] + suggestable = False + snippet_only = False + + for rule in unresolved_rules: + paragraph_span: tuple[int, int] | None = None + if rule.directive == "short-description": + paragraph, paragraph_span = extract_first_paragraph_after_title(content) + if paragraph is None: + after_title_manual.append(rule.directive) + if rule.severity == "error": + after_title_error.append(rule.directive) + else: + after_title_warning.append(rule.directive) + continue + + can_suggest = pr_lines is None or can_suggest_after_title_inline( + content, + pr_lines, + paragraph_span=paragraph_span, + ) + snippet_text = _format_short_description_snippet(paragraph) + + if can_suggest: + new_content, changed = wrap_first_paragraph_as_short_description(content) + if changed: + content = new_content + after_title_auto.append(rule.directive) + suggestable = True + if rule.severity == "error": + after_title_error.append(rule.directive) + else: + after_title_warning.append(rule.directive) + else: + after_title_snippets.append({"directive": rule.directive, "snippet": snippet_text}) + snippet_only = True + if rule.severity == "error": + after_title_error.append(rule.directive) + else: + after_title_warning.append(rule.directive) + + elif rule.directive == "showmeta": + assert rule.options is not None + can_suggest = pr_lines is None or can_suggest_after_title_inline(content, pr_lines) + snippet_text = format_showmeta_block(rule.options) + + if can_suggest: + new_content, changed = inject_showmeta_to_content(content, rule.options) + if changed: + content = new_content + after_title_auto.append(rule.directive) + suggestable = True + if rule.severity == "error": + after_title_error.append(rule.directive) + else: + after_title_warning.append(rule.directive) + else: + after_title_snippets.append({"directive": rule.directive, "snippet": snippet_text}) + snippet_only = True + if rule.severity == "error": + after_title_error.append(rule.directive) + else: + after_title_warning.append(rule.directive) + + if not (after_title_auto or after_title_manual or after_title_snippets): + return content, {} + + mode = "suggestable" if suggestable else ("snippet" if snippet_only else "manual_fields") + return content, { + "after_title_auto": after_title_auto, + "after_title_manual": after_title_manual, + "after_title_warning": after_title_warning, + "after_title_error": after_title_error, + "after_title_snippets": after_title_snippets, + "after_title_mode": mode, + } + + def _severity_fields( field_names: list[str], rules: dict[str, MetaRule], @@ -563,12 +899,12 @@ def _severity_fields( def ensure_meta_tags_in_file( path: Path, - rules: dict[str, MetaRule], + config: EnhanceConfig | dict[str, MetaRule], *, pr_lines: set[int] | None = None, ) -> dict[str, object] | None: """ - Resolve missing metadata in one RST file where rules allow automatic fixes. + Resolve missing documentation enhancements in one RST file. When ``pr_lines`` is provided, automatic edits are only written when they can land on lines already in the PR diff. Fields without configured values always @@ -576,31 +912,38 @@ def ensure_meta_tags_in_file( Args: path: RST file to inspect and, where permitted, update. - rules: Configured metadata rules keyed by field name. + config: Enhancement configuration, or a legacy meta-rules mapping. pr_lines: One-based pull-request diff lines, or ``None`` for local mode. Returns: - A result dict when fields were missing in the file as read, otherwise - ``None``. Each result includes ``path``, ``line``, ``mode`` - (``suggestable``, ``snippet``, or ``manual_fields``), ``snippet`` (RST - for copy-paste when relevant), ``auto_fields`` (configured values the - tool can fill), ``manual_fields``, ``warning_fields``, and - ``error_fields``. The severity lists cover every field that was missing - or blank before any automatic injection. + A result dict when issues were found in the file as read, otherwise + ``None``. Raises: OSError: If the RST file cannot be read or an eligible edit cannot be written. UnicodeError: If the RST file cannot be decoded or encoded as UTF-8. """ + if isinstance(config, dict): + enhance_config = EnhanceConfig(meta=config, after_title=()) + else: + enhance_config = config + rules = enhance_config.meta + content = path.read_text(encoding="utf-8") + path_str = str(path).replace("\\", "/") + unresolved = _unresolved_fields(content, rules) - if not unresolved: - logger.info("%s: all configured meta fields present", path) + after_title_unresolved = [ + rule + for rule in enhance_config.after_title + if not _after_title_rule_satisfied(content, rule) + ] + + if not unresolved and not after_title_unresolved: + logger.info("%s: all configured enhancements present", path) return None - path_str = str(path).replace("\\", "/") annotation_line = _annotation_line_for_content(content) - auto_fields = [name for name in unresolved if rules[name].has_configured_value] auto_metadata = {name: rules[name].value for name in auto_fields} @@ -632,20 +975,33 @@ def ensure_meta_tags_in_file( ", ".join(auto_fields), ) - still_unresolved = _unresolved_fields(content, rules) - if not still_unresolved and mode is None: - return None + after_title_result: dict[str, object] = {} + if enhance_config.after_title: + content_before_after_title = content + content, after_title_result = _process_after_title_rules( + path, + content, + enhance_config.after_title, + pr_lines=pr_lines, + ) + if after_title_result and content != content_before_after_title: + path.write_text(content, encoding="utf-8") - # Annotations and severity describe the pull request as pushed. Auto-injected - # values only exist in the CI working tree until the suggestion is committed, - # so they are reported from ``unresolved`` rather than ``still_unresolved``. manual_fields = [name for name in unresolved if not rules[name].has_configured_value] warning_fields = _severity_fields(unresolved, rules, "warning") error_fields = _severity_fields(unresolved, rules, "error") - if mode is None: + after_title_mode = after_title_result.get("after_title_mode") + if after_title_mode == "suggestable": + mode = "suggestable" + elif after_title_mode == "snippet" and mode != "suggestable": + mode = "snippet" + elif mode is None and (unresolved or after_title_result): mode = "manual_fields" + if not unresolved and not after_title_result and mode is None: + return None + return { "path": path_str, "line": annotation_line, @@ -655,6 +1011,11 @@ def ensure_meta_tags_in_file( "manual_fields": manual_fields, "warning_fields": warning_fields, "error_fields": error_fields, + "after_title_auto": after_title_result.get("after_title_auto", []), + "after_title_manual": after_title_result.get("after_title_manual", []), + "after_title_warning": after_title_result.get("after_title_warning", []), + "after_title_error": after_title_result.get("after_title_error", []), + "after_title_snippets": after_title_result.get("after_title_snippets", []), } @@ -717,7 +1078,7 @@ def build_review_comment( rules: dict[str, MetaRule], ) -> str: """ - Build a pull-request review body from metadata check results. + Build a pull-request review body from enhancement check results. Args: results: Per-file result dictionaries from ``ensure_meta_tags_in_file``. @@ -729,12 +1090,18 @@ def build_review_comment( inline_modes = [r for r in results if r["mode"] == "suggestable"] snippet_modes = [r for r in results if r["mode"] == "snippet"] manual_fields_modes = [r for r in results if r["mode"] == "manual_fields"] + after_title_snippet_results = [ + r for r in results if r.get("after_title_snippets") + ] + after_title_manual_results = [ + r for r in results if r.get("after_title_manual") + ] lines = [ SUMMARY_REVIEW_TITLE, "", - "This pull request is missing configured documentation metadata " - "(see `tools/meta_tags.yaml`).", + "This pull request is missing configured documentation enhancements " + "(see `tools/enhance.yaml`).", "", ] @@ -744,32 +1111,57 @@ def build_review_comment( SECTION_INLINE_SUGGESTIONS, "", "Please **review and commit the inline suggestions** on the " - "**Files changed** tab (or use the separate *Inline metadata " - "suggestions* review). They add these configured default values:", + "**Files changed** tab (or use the separate *Inline documentation " + "suggestions* review). They apply these configured defaults:", "", ] ) for result in inline_modes: - auto = ", ".join(f"`{name}`" for name in result["auto_fields"]) - lines.append(f"- **`{result['path']}`**: {auto}") + parts: list[str] = [] + auto = result.get("auto_fields") or [] + if auto: + parts.append(", ".join(f"`{name}`" for name in auto)) + after_auto = result.get("after_title_auto") or [] + if after_auto: + parts.append(", ".join(f"`{name}`" for name in after_auto)) + lines.append(f"- **`{result['path']}`**: {', '.join(parts)}") lines.append("") if snippet_modes: + meta_snippet_modes = [r for r in snippet_modes if r.get("snippet")] + if meta_snippet_modes: + lines.extend( + [ + SECTION_COPY_PASTE_BLOCKS, + "", + "These files could not receive inline suggestions because the edits are " + "outside the pull request diff. Add this block at the **top of each " + "file** (or append the listed fields to an existing `.. meta::` block):", + "", + ] + ) + for result in meta_snippet_modes: + lines.append(f"**`{result['path']}`**") + lines.append("```rst") + lines.append(str(result["snippet"]).rstrip()) + lines.append("```") + lines.append("") + + if after_title_snippet_results: lines.extend( [ - SECTION_COPY_PASTE_BLOCKS, + SECTION_COPY_PASTE_AFTER_TITLE, "", - "These files could not receive inline suggestions because the edits are " - "outside the pull request diff. Add this block at the **top of each " - "file** (or append the listed fields to an existing `.. meta::` block):", + "Add these directives **after the first document title** in each file:", "", ] ) - for result in snippet_modes: + for result in after_title_snippet_results: lines.append(f"**`{result['path']}`**") - lines.append("```rst") - lines.append(str(result["snippet"]).rstrip()) - lines.append("```") + for entry in result.get("after_title_snippets") or []: + lines.append("```rst") + lines.append(str(entry["snippet"]).rstrip()) + lines.append("```") lines.append("") manual_results = [r for r in results if r.get("manual_fields")] @@ -790,7 +1182,22 @@ def build_review_comment( ) lines.append("") - if manual_fields_modes and not inline_modes and not snippet_modes: + if after_title_manual_results: + lines.extend( + [ + SECTION_AFTER_TITLE_MANUAL, + "", + "These files need after-title directives that could not be added " + "automatically (for example, no prose paragraph to wrap):", + "", + ] + ) + for result in after_title_manual_results: + manual = ", ".join(f"`{name}`" for name in result.get("after_title_manual") or []) + lines.append(f"- **`{result['path']}`**: {manual}") + lines.append("") + + if manual_fields_modes and not inline_modes and not snippet_modes and not after_title_snippet_results: lines.append( "Add or complete a `.. meta::` block at the top of each affected file.", ) @@ -871,7 +1278,7 @@ def main(argv: list[str] | None = None) -> int: if not args.paths and not args.diff_base: parser.error("provide at least one .rst path or set --diff-base to discover changes") - rules = load_meta_config(args.config) + rules = load_enhance_config(args.config) checked_pull_request_rst = False if not args.paths: @@ -884,7 +1291,7 @@ def main(argv: list[str] | None = None) -> int: args.status_file, meta_checked=False, results=[], - rules=rules, + rules=rules.meta, has_errors=False, ) return 0 @@ -924,15 +1331,30 @@ def main(argv: list[str] | None = None) -> int: line = int(result["line"]) emit_github_warning(path, list(result["warning_fields"]), line) emit_github_error(path, list(result["error_fields"]), line) + _emit_after_title_annotation( + "warning", + path, + list(result.get("after_title_warning") or []), + line, + ) + _emit_after_title_annotation( + "error", + path, + list(result.get("after_title_error") or []), + line, + ) - has_errors = any(result["error_fields"] for result in results) + has_errors = any( + result["error_fields"] or result.get("after_title_error") + for result in results + ) if args.status_file is not None: _write_ci_status_file( args.status_file, meta_checked=checked_pull_request_rst, results=results, - rules=rules, + rules=rules.meta, has_errors=has_errors, ) diff --git a/tools/meta_tags.yaml b/tools/meta_tags.yaml deleted file mode 100644 index 94db3eae9ab..00000000000 --- a/tools/meta_tags.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# Metadata rules for ensure_meta_tags.py. -# severity: warning (soft-fail step) or error (workflow fails after review). -# value: injected when missing/blank; leave empty for contributor-provided values. -# Sphinx expands {PRODUCT} and {DISTRO} at build time (see conf.py). -meta: - product: - severity: warning - value: "{PRODUCT}" - distribution: - severity: warning - value: "{DISTRO}" - area: - severity: error - value: - experience: - severity: warning - value: - content-type: - severity: warning - value: diff --git a/tools/rst_utils.py b/tools/rst_utils.py index 6d7183f4ad6..c23c1ae88ff 100644 --- a/tools/rst_utils.py +++ b/tools/rst_utils.py @@ -1,6 +1,6 @@ """ -Utilities for editing reStructuredText source, in particular ``.. meta::`` and -``.. short-description::`` directives. +Utilities for editing reStructuredText source, in particular ``.. meta::``, +``.. short-description::``, and ``.. showmeta::`` directives. """ import logging @@ -416,3 +416,371 @@ def inject_short_description_to_content(content: str, text: str) -> tuple[str, b new_content = content[:insert_at] + block + remainder return new_content, True + +_SECTION_ADORNMENT_RE = re.compile( + r'^([!"#$%&\'()*+,\-./:;<=>?@\[\\\]^_`{|}~]+)\s*$', +) + + +def _is_section_title_at(lines: list[str], index: int) -> bool: + """ + Return whether ``lines[index]`` is an RST section title with an underline. + + Args: + lines: Document lines (with or without trailing newlines). + index: Zero-based line index of the candidate title line. + + Returns: + True when the line is followed by a valid adornment underline. + """ + if index + 1 >= len(lines): + return False + title_stripped = lines[index].strip() + if not title_stripped: + return False + ul_match = _SECTION_ADORNMENT_RE.match(lines[index + 1].rstrip("\n")) + if ul_match is None: + return False + ul = ul_match.group(1) + return len(set(ul)) == 1 and len(ul) >= len(title_stripped) + + +def _line_starts_directive(line: str) -> bool: + """Return whether ``line`` begins an explicit RST directive marker.""" + return bool(re.match(r"^\.\.\s+\S+::", line)) + + +def _skip_past_directive_block(lines: list[str], directive_index: int) -> int: + """ + Return the index of the first line after a directive block. + + Args: + lines: Document lines (with or without trailing newlines). + directive_index: Zero-based index of the ``.. directive::`` line. + + Returns: + Index of the first line following the directive block. + """ + i = directive_index + 1 + while i < len(lines): + line = lines[i] + if line.strip() == "": + i += 1 + continue + if not line.startswith((" ", "\t")): + return i + i += 1 + return i + + +def _title_line_span(content: str) -> tuple[int, int] | None: + """ + Return the inclusive 1-based line span of the first document title block. + + Returns ``None`` when no title is found. + """ + lines = content.splitlines() + for i in range(len(lines) - 1): + if _is_section_title_at(lines, i): + return i + 1, i + 2 + return None + + +def extract_first_paragraph_after_title( + content: str, +) -> tuple[str | None, tuple[int, int] | None]: + """ + Find the first prose paragraph after the first document title. + + Skips blank lines, directives, indented non-prose lines (such as toctree + entries), and section titles. Collects contiguous prose until the next + blank line, directive, or section title. + + Args: + content: RST source to search. + + Returns: + Normalised paragraph text and its inclusive 1-based line span, or + ``(None, None)`` when no prose paragraph is found. + """ + lines = content.splitlines(keepends=True) + stripped_lines = [line.rstrip("\n") for line in lines] + insert_at = _find_insertion_point_after_title(content) + if insert_at <= 0: + start_index = 0 + else: + start_index = content[:insert_at].count("\n") + + prose_lines: list[str] = [] + prose_start: int | None = None + i = start_index + while i < len(lines): + line = lines[i] + stripped = line.strip() + if not stripped: + if prose_lines: + break + i += 1 + continue + if _line_starts_directive(stripped): + if prose_lines: + break + i = _skip_past_directive_block(lines, i) + continue + if _is_section_title_at(stripped_lines, i): + if prose_lines: + break + i += 2 + continue + if not prose_lines and line.startswith((" ", "\t")): + i += 1 + continue + if prose_lines and line.startswith((" ", "\t")): + prose_lines.append(stripped) + i += 1 + continue + if line.startswith((" ", "\t")): + i += 1 + continue + if prose_start is None: + prose_start = i + prose_lines.append(stripped) + i += 1 + + if not prose_lines or prose_start is None: + return None, None + + start_line = prose_start + 1 + end_line = prose_start + len(prose_lines) + return " ".join(prose_lines), (start_line, end_line) + + +def wrap_first_paragraph_as_short_description(content: str) -> tuple[str, bool]: + """ + Wrap the first prose paragraph after the title into ``.. short-description::``. + + If a non-empty short-description already exists, returns unchanged. If the + directive exists with an empty body, fills it from the first paragraph. + Otherwise inserts a new directive after the title and removes the paragraph + from the body. + + Returns: + Updated source and whether any change was made. + """ + if has_short_description_content(content): + return content, False + + paragraph, span = extract_first_paragraph_after_title(content) + if paragraph is None or span is None: + return content, False + + without_para = _remove_line_span(content, span) + start, marker_end, block_end, _inner, indent = _find_short_description_block(without_para) + new_inner = _format_short_description_inner(paragraph, indent) + + if start >= 0: + remainder = without_para[block_end:].lstrip() + new_content = without_para[:marker_end] + new_inner + "\n" + remainder + return new_content, True + + insert_at = _find_insertion_point_after_title(without_para) + remainder = without_para[insert_at:].lstrip() + block = f"\n.. short-description::\n{new_inner}\n" + new_content = without_para[:insert_at] + block + remainder + return new_content, True + + +def _remove_line_span(content: str, span: tuple[int, int]) -> str: + """ + Remove an inclusive 1-based line span from RST source. + + Args: + content: RST source. + span: Inclusive start and end line numbers (1-based). + + Returns: + Source with the span removed and adjacent blank lines collapsed. + """ + start_line, end_line = span + lines = content.splitlines(keepends=True) + kept = lines[: start_line - 1] + lines[end_line:] + result = "".join(kept) + while "\n\n\n" in result: + result = result.replace("\n\n\n", "\n\n") + return result + + +def _find_showmeta_block(content: str) -> tuple[int, int, int, str, str]: + """Locate the first ``.. showmeta::`` directive in RST source.""" + return _find_directive_block(content, "showmeta") + + +def _extract_directive_options_from_block(block_inner: str) -> dict[str, str]: + """ + Collect option names and values from a directive body. + + Each line of the form ``:name: value`` contributes ``name``. + """ + options: dict[str, str] = {} + for field_match in re.finditer( + r"^[ \t]+:([^:\n]+?):\s*(.*)$", + block_inner, + re.MULTILINE, + ): + options[field_match.group(1).strip()] = field_match.group(2) + return options + + +def has_showmeta_with_order(content: str) -> bool: + """ + Return whether the first ``.. showmeta::`` block has a non-empty ``:order:``. + + Args: + content: RST source to search. + + Returns: + True when showmeta exists with a non-blank order option. + """ + _s, _m, _b, inner, _i = _find_showmeta_block(content) + if not inner.strip(): + return False + options = _extract_directive_options_from_block(inner) + return bool(options.get("order", "").strip()) + + +def showmeta_line_span(content: str) -> tuple[int, int] | None: + """ + Return the inclusive 1-based line span of the first ``.. showmeta::`` block. + + Returns ``None`` if no showmeta block exists. + """ + start, _marker_end, block_end, _inner, _indent = _find_showmeta_block(content) + if start < 0: + return None + start_line = _byte_offset_to_line_number(content, start) + end_offset = block_end - 1 if block_end > start else start + end_line = _byte_offset_to_line_number(content, end_offset) + return start_line, end_line + + +def short_description_line_span(content: str) -> tuple[int, int] | None: + """ + Return the inclusive 1-based line span of the first ``.. short-description::`` block. + + Returns ``None`` if no short-description block exists. + """ + start, _marker_end, block_end, _inner, _indent = _find_short_description_block(content) + if start < 0: + return None + start_line = _byte_offset_to_line_number(content, start) + end_offset = block_end - 1 if block_end > start else start + end_line = _byte_offset_to_line_number(content, end_offset) + return start_line, end_line + + +def _insertion_point_after_short_description(content: str) -> int: + """ + Return the byte index immediately after the first short-description block. + + Falls back to after the title when no short-description exists. + """ + _s, _m, block_end, _inner, _indent = _find_short_description_block(content) + if block_end >= 0: + return block_end + return _find_insertion_point_after_title(content) + + +def format_showmeta_block(options: dict[str, str], indent: str = " ") -> str: + """ + Build an RST ``.. showmeta::`` block for the given options. + + Args: + options: Option name to value mapping. + indent: Indentation for option lines. + + Returns: + A formatted ``.. showmeta::`` directive ending with a blank line. + """ + lines = [".. showmeta::"] + for key, value in options.items(): + lines.append(f"{indent}:{key}: {value}") + lines.append("") + return "\n".join(lines) + + +def inject_showmeta_to_content( + content: str, + options: dict[str, str], +) -> tuple[str, bool]: + """ + Insert or fill a ``.. showmeta::`` directive with the given options. + + Appends to an existing block when present. Otherwise inserts after the first + short-description block, or after the title when none exists. Skips options + that already have non-empty values. + + Returns: + Updated source and whether any change was made. + """ + start, marker_end, block_end, inner, indent = _find_showmeta_block(content) + existing = _extract_directive_options_from_block(inner) + merged: dict[str, str] = dict(existing) + changed = False + + for key, raw_value in options.items(): + value = _normalise_meta_field_value(raw_value) + if key not in merged: + merged[key] = value + changed = True + elif not merged[key].strip(): + merged[key] = value + changed = True + else: + logger.warning( + "Existing showmeta option %r; skipping", + key, + ) + + if not changed: + return content, False + + ordered_keys: list[str] = list(existing.keys()) + for key in options: + if key not in ordered_keys: + ordered_keys.append(key) + new_inner = "".join(f"{indent}:{key}: {merged[key]}\n" for key in ordered_keys) + + if start >= 0: + remainder = content[block_end:].lstrip() + new_content = content[:marker_end] + new_inner + "\n" + remainder + return new_content, True + + insert_at = _insertion_point_after_short_description(content) + remainder = content[insert_at:].lstrip() + block = f"\n.. showmeta::\n{new_inner}\n" + new_content = content[:insert_at] + block + remainder + return new_content, True + + +def after_title_directives_line_span(content: str) -> tuple[int, int] | None: + """ + Return the inclusive 1-based line span of the post-title directive area. + + Covers short-description and/or showmeta blocks. When neither exists, returns + the line immediately after the title (or line 1 when no title is found). + """ + spans: list[tuple[int, int]] = [] + for span_fn in (short_description_line_span, showmeta_line_span): + span = span_fn(content) + if span is not None: + spans.append(span) + + if spans: + return min(s[0] for s in spans), max(s[1] for s in spans) + + title_span = _title_line_span(content) + if title_span is not None: + return title_span[1] + 1, title_span[1] + 1 + + return 1, 1 + diff --git a/tools/tests/test_ensure_meta_tags.py b/tools/tests/test_ensure_meta_tags.py index e35e0ecd431..9bfb685f93d 100644 --- a/tools/tests/test_ensure_meta_tags.py +++ b/tools/tests/test_ensure_meta_tags.py @@ -27,21 +27,65 @@ from ensure_meta_tags import ( # noqa: E402 REVIEW_MARKER, + SECTION_COPY_PASTE_AFTER_TITLE, SECTION_INLINE_SUGGESTIONS, SECTION_NON_EMPTY_VALUES, SUMMARY_REVIEW_TITLE, + AfterTitleRule, + EnhanceConfig, MetaRule, _unresolved_fields, build_review_comment, + can_suggest_after_title_inline, can_suggest_inline, changed_rst_paths, ensure_meta_tags_in_file, + load_enhance_config, load_meta_config, main, ) from rst_utils import get_meta_fields_from_content, inject_metadata_to_content # noqa: E402 SAMPLE_CONFIG = textwrap.dedent( + """ + meta: + product: + severity: warning + value: "{PRODUCT}" + area: + severity: error + value: + experience: + severity: warning + value: + after_title: + - directive: short-description + severity: warning + content: first_paragraph + - directive: showmeta + severity: warning + options: + order: "area, contentType, experience" + required_options: + - order + """ +).strip() + +AFTER_TITLE_RULES = ( + AfterTitleRule( + directive="short-description", + severity="warning", + content="first_paragraph", + ), + AfterTitleRule( + directive="showmeta", + severity="warning", + options={"order": "area, contentType, experience"}, + required_options=("order",), + ), +) + +META_ONLY_CONFIG = textwrap.dedent( """ meta: product: @@ -72,6 +116,18 @@ def test_load_meta_config_parses_rules(self) -> None: self.assertFalse(rules["area"].has_configured_value) self.assertEqual(rules["area"].severity, "error") + def test_load_enhance_config_parses_after_title(self) -> None: + with tempfile.NamedTemporaryFile("w", suffix=".yaml", delete=False) as handle: + handle.write(SAMPLE_CONFIG) + path = Path(handle.name) + try: + config = load_enhance_config(path) + finally: + path.unlink() + self.assertEqual(len(config.after_title), 2) + self.assertEqual(config.after_title[0].directive, "short-description") + self.assertEqual(config.after_title[1].directive, "showmeta") + class TestCanSuggestInline(unittest.TestCase): def test_meta_block_overlap_uses_inclusive_span_only(self) -> None: @@ -88,6 +144,18 @@ def test_meta_block_overlap_uses_inclusive_span_only(self) -> None: self.assertFalse(can_suggest_inline(content, {3})) self.assertTrue(can_suggest_inline(content, {2})) + def test_after_title_overlap_uses_paragraph_span(self) -> None: + content = textwrap.dedent( + """ + Title + ===== + + Opening paragraph. + """ + ).lstrip() + self.assertTrue(can_suggest_after_title_inline(content, {4}, paragraph_span=(4, 4))) + self.assertFalse(can_suggest_after_title_inline(content, {8}, paragraph_span=(4, 4))) + class TestUnresolvedFields(unittest.TestCase): def test_missing_and_blank_count_as_unresolved(self) -> None: @@ -180,6 +248,90 @@ def test_no_result_when_all_fields_present(self) -> None: self.assertIsNone(ensure_meta_tags_in_file(path, rules)) +class TestAfterTitleEnhancements(unittest.TestCase): + def test_inserts_short_description_and_showmeta(self) -> None: + config = EnhanceConfig( + meta={"product": MetaRule("warning", "{PRODUCT}")}, + after_title=AFTER_TITLE_RULES, + ) + content = textwrap.dedent( + """ + Title + ===== + + Opening paragraph for the page. + + More content. + """ + ).lstrip() + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "page.rst" + path.write_text(content, encoding="utf-8") + result = ensure_meta_tags_in_file(path, config) + self.assertIsNotNone(result) + updated = path.read_text(encoding="utf-8") + self.assertIn(".. short-description::", updated) + self.assertIn(".. showmeta::", updated) + self.assertIn(":order: area, contentType, experience", updated) + self.assertIn("Opening paragraph for the page.", updated) + self.assertIn("More content.", updated) + + def test_toctree_before_paragraph_wraps_correct_paragraph(self) -> None: + config = EnhanceConfig(meta={}, after_title=AFTER_TITLE_RULES) + content = textwrap.dedent( + """ + Title + ===== + + .. toctree:: + Page + + First paragraph here. + + Second paragraph. + """ + ).lstrip() + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "page.rst" + path.write_text(content, encoding="utf-8") + result = ensure_meta_tags_in_file(path, config) + self.assertIsNotNone(result) + updated = path.read_text(encoding="utf-8") + self.assertIn("First paragraph here.", updated) + self.assertIn(".. showmeta::", updated) + toctree_pos = updated.index(".. toctree::") + showmeta_pos = updated.index(".. showmeta::") + self.assertLess(showmeta_pos, toctree_pos) + + def test_build_review_comment_includes_after_title_snippets(self) -> None: + rules = {"area": MetaRule("error", "")} + results = [ + { + "path": "source/Page.rst", + "mode": "snippet", + "snippet": "", + "auto_fields": [], + "manual_fields": [], + "warning_fields": ["short-description"], + "error_fields": [], + "line": 1, + "after_title_auto": [], + "after_title_manual": [], + "after_title_warning": ["short-description"], + "after_title_error": [], + "after_title_snippets": [ + { + "directive": "showmeta", + "snippet": ".. showmeta::\n :order: area\n", + }, + ], + }, + ] + body = build_review_comment(results, rules) + self.assertIn(SECTION_COPY_PASTE_AFTER_TITLE, body) + self.assertIn(".. showmeta::", body) + + class TestReviewAndExit(unittest.TestCase): def test_build_review_comment_lists_manual_fields(self) -> None: rules = { @@ -283,9 +435,9 @@ def _extract_multiline_output(status: str, key: str) -> str | None: class TestCiStatusOutputs(unittest.TestCase): """The workflow gates steps on these outputs, so keep them self-consistent.""" - def _run_with_status_file(self, content: str) -> str: + def _run_with_status_file(self, content: str, *, config: str = META_ONLY_CONFIG) -> str: rules_path = Path(tempfile.mkdtemp()) / "meta.yaml" - rules_path.write_text(SAMPLE_CONFIG, encoding="utf-8") + rules_path.write_text(config, encoding="utf-8") with tempfile.TemporaryDirectory() as tmp: page = Path(tmp) / "page.rst" page.write_text(content, encoding="utf-8") @@ -309,7 +461,7 @@ def test_suggestion_note_written_with_inline_suggestions(self) -> None: comment = _extract_multiline_output(status, "comment") self.assertIsNotNone(suggestion_note) self.assertIsNotNone(comment) - self.assertIn("## Inline metadata suggestions", suggestion_note or "") + self.assertIn("## Inline documentation suggestions", suggestion_note or "") self.assertNotIn(REVIEW_MARKER, suggestion_note or "") self.assertIn(SUMMARY_REVIEW_TITLE, comment or "") self.assertIn(REVIEW_MARKER, comment or "") @@ -339,6 +491,12 @@ def test_clean_file_writes_no_review_bodies(self) -> None: Title ===== + + .. short-description:: + Summary for the page. + + .. showmeta:: + :order: area, contentType, experience """ ).lstrip() status = self._run_with_status_file(content) diff --git a/tools/tests/test_rst_utils.py b/tools/tests/test_rst_utils.py new file mode 100644 index 00000000000..9fc34a810c8 --- /dev/null +++ b/tools/tests/test_rst_utils.py @@ -0,0 +1,206 @@ +# Copyright 2026 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import sys +import textwrap +import unittest +from pathlib import Path + +_TOOLS_DIR = Path(__file__).resolve().parent.parent +if str(_TOOLS_DIR) not in sys.path: + sys.path.insert(0, str(_TOOLS_DIR)) + +from rst_utils import ( # noqa: E402 + extract_first_paragraph_after_title, + format_showmeta_block, + has_short_description_content, + has_showmeta_with_order, + inject_showmeta_to_content, + wrap_first_paragraph_as_short_description, +) + + +class TestExtractFirstParagraph(unittest.TestCase): + def test_skips_directives_before_prose(self) -> None: + content = textwrap.dedent( + """ + Title + ===== + + .. toctree:: + :maxdepth: 1 + + Page + + First paragraph here. + + Second paragraph. + """ + ).lstrip() + paragraph, span = extract_first_paragraph_after_title(content) + self.assertEqual(paragraph, "First paragraph here.") + self.assertEqual(span, (9, 9)) + + def test_finds_paragraph_after_dash_title(self) -> None: + content = textwrap.dedent( + """ + Summary + ------- + + Opening prose for the page. + """ + ).lstrip() + paragraph, span = extract_first_paragraph_after_title(content) + self.assertEqual(paragraph, "Opening prose for the page.") + self.assertEqual(span, (4, 4)) + + def test_returns_none_when_no_prose(self) -> None: + content = textwrap.dedent( + """ + Title + ===== + + .. toctree:: + Page + """ + ).lstrip() + paragraph, span = extract_first_paragraph_after_title(content) + self.assertIsNone(paragraph) + self.assertIsNone(span) + + +class TestWrapShortDescription(unittest.TestCase): + def test_wraps_first_paragraph_after_equals_title(self) -> None: + content = textwrap.dedent( + """ + Title + ===== + + Opening paragraph for the article. + + More content. + """ + ).lstrip() + updated, changed = wrap_first_paragraph_as_short_description(content) + self.assertTrue(changed) + self.assertTrue(has_short_description_content(updated)) + self.assertIn(".. short-description::", updated) + self.assertIn("Opening paragraph for the article.", updated) + self.assertIn("More content.", updated) + body_after_directive = updated.split(".. short-description::", 1)[1] + self.assertNotIn("Opening paragraph for the article.", body_after_directive.split("More content.", 1)[1]) + + def test_does_not_replace_existing_short_description(self) -> None: + content = textwrap.dedent( + """ + Title + ===== + + .. short-description:: + Existing summary. + + Body paragraph. + """ + ).lstrip() + updated, changed = wrap_first_paragraph_as_short_description(content) + self.assertFalse(changed) + self.assertEqual(updated, content) + + def test_fills_empty_short_description_from_first_paragraph(self) -> None: + content = textwrap.dedent( + """ + Title + ===== + + .. short-description:: + + Body paragraph here. + """ + ).lstrip() + updated, changed = wrap_first_paragraph_as_short_description(content) + self.assertTrue(changed) + self.assertTrue(has_short_description_content(updated)) + self.assertEqual(updated.count("Body paragraph here."), 1) + + +class TestShowmetaHelpers(unittest.TestCase): + def test_inject_showmeta_after_short_description(self) -> None: + content = textwrap.dedent( + """ + Title + ===== + + .. short-description:: + Summary text. + + Body content. + """ + ).lstrip() + updated, changed = inject_showmeta_to_content( + content, + {"order": "area, contentType, experience"}, + ) + self.assertTrue(changed) + self.assertTrue(has_showmeta_with_order(updated)) + short_desc_pos = updated.index(".. short-description::") + showmeta_pos = updated.index(".. showmeta::") + body_pos = updated.index("Body content.") + self.assertLess(short_desc_pos, showmeta_pos) + self.assertLess(showmeta_pos, body_pos) + + def test_fills_missing_order_on_existing_showmeta(self) -> None: + content = textwrap.dedent( + """ + Title + ===== + + .. showmeta:: + :order: + + Body content. + """ + ).lstrip() + updated, changed = inject_showmeta_to_content(content, {"order": "area"}) + self.assertTrue(changed) + self.assertIn(":order: area", updated) + + def test_does_not_overwrite_existing_order(self) -> None: + content = textwrap.dedent( + """ + Title + ===== + + .. showmeta:: + :order: area, experience + + Body content. + """ + ).lstrip() + updated, changed = inject_showmeta_to_content( + content, + {"order": "area, contentType, experience"}, + ) + self.assertFalse(changed) + self.assertEqual(updated, content) + + def test_format_showmeta_block(self) -> None: + block = format_showmeta_block({"order": "area, contentType, experience"}) + self.assertIn(".. showmeta::", block) + self.assertIn(":order: area, contentType, experience", block) + + +if __name__ == "__main__": + unittest.main() From 6fd03d0247c9f043846bdb9296c69f793a656493 Mon Sep 17 00:00:00 2001 From: Keith Kirkwood Date: Thu, 30 Jul 2026 16:38:57 +0100 Subject: [PATCH 17/23] OPENR-174: Revert test changes to First steps article. --- source/First-Steps.rst | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/source/First-Steps.rst b/source/First-Steps.rst index ebf955cd364..98cbcb04732 100644 --- a/source/First-Steps.rst +++ b/source/First-Steps.rst @@ -1,25 +1,13 @@ -.. meta:: - :description: The ROS framework is the “plumbing” which makes communication between different parts of a robot possible. - :keywords: ROS, framework, communication, robotics, learning path - :area: framework - :contentType: learning-path - :experience: beginner - :product: {PRODUCT} - :distribution: {DISTRO} - .. _First-steps-with-ROS-learning-path: First steps with ROS - learning path ==================================== -.. short-description:: - ROS (Robot Operating System) is an open-source ecosystem that provides framework, tools, and libraries for building, deploying, running, and maintaining robotic applications. - This page presents a set of articles and hands-on activities to introduce the main concepts behind the ROS framework. - Working through these will give you the essential knowledge needed to start developing applications with ROS. +ROS (Robot Operating System) is an open-source ecosystem that provides framework, tools, and libraries for building, deploying, running, and maintaining robotic applications. +This page presents a set of articles and hands-on activities to introduce the main concepts behind the ROS framework. +Working through these will give you the essential knowledge needed to start developing applications with ROS. -.. showmeta:: - :order: area, contentType, experience - :labels: area=Area, contentType=Content type, experience=Level +**Area: ROS-framework | Content-type: learning-path | Experience: beginner** .. contents:: Contents :depth: 2 From 8bc34874eba279e94ccb2c920f907053591abc7c Mon Sep 17 00:00:00 2001 From: Keith Kirkwood Date: Fri, 31 Jul 2026 12:24:18 +0100 Subject: [PATCH 18/23] OPENR-174: Renaming throughout to reflect enhancement instead of just metadata --- .github/workflows/enhance.yml | 30 ++--- Makefile | 10 +- tools/README.md | 119 +++++++++--------- tools/enhance.yaml | 2 +- ...re_meta_tags.py => ensure_enhancements.py} | 47 +++---- ...ws.sh => supersede_enhancement_reviews.sh} | 7 +- ...ta_tags.py => test_ensure_enhancements.py} | 34 ++--- 7 files changed, 122 insertions(+), 127 deletions(-) rename tools/{ensure_meta_tags.py => ensure_enhancements.py} (97%) rename tools/{supersede_meta_tag_reviews.sh => supersede_enhancement_reviews.sh} (81%) rename tools/tests/{test_ensure_meta_tags.py => test_ensure_enhancements.py} (94%) diff --git a/.github/workflows/enhance.yml b/.github/workflows/enhance.yml index 56fc17f1bff..ee15b0e788a 100644 --- a/.github/workflows/enhance.yml +++ b/.github/workflows/enhance.yml @@ -14,7 +14,7 @@ permissions: pull-requests: write jobs: - ensure-meta-tags: + ensure-enhancements: runs-on: ubuntu-24.04 steps: - name: Checkout PR code @@ -38,7 +38,7 @@ jobs: - name: Install PyYAML run: pip install --no-warn-script-location pyyaml - - name: Ensure documentation metadata + - name: Ensure documentation enhancements id: ensure continue-on-error: true env: @@ -46,34 +46,34 @@ jobs: run: | set -euo pipefail git fetch origin "$DIFF_BASE" - make -f .trusted-base/Makefile ensure-meta-tags \ + make -f .trusted-base/Makefile ensure-enhancements \ TOOLS_DIR=.trusted-base/tools \ DIFF_BASE="$DIFF_BASE" \ STATUS_FILE="$GITHUB_OUTPUT" - # The ensure step soft-fails whenever metadata is missing, so an empty - # meta_checked is the only signal that the check itself never ran. - - name: Verify metadata check ran - if: steps.ensure.outputs.meta_checked == '' + # The ensure step soft-fails whenever enhancements are missing, so an empty + # enhancements_checked is the only signal that the check itself never ran. + - name: Verify enhancement check ran + if: steps.ensure.outputs.enhancements_checked == '' run: | - echo "The metadata check produced no outputs; see the ensure step log." + echo "The enhancement check produced no outputs; see the ensure step log." exit 1 - - name: Supersede stale meta-tag reviews - if: steps.ensure.outputs.meta_checked == 'true' + - name: Supersede stale enhancement reviews + if: steps.ensure.outputs.enhancements_checked == 'true' env: GH_TOKEN: ${{ github.token }} PR_NUMBER: ${{ github.event.pull_request.number }} REPOSITORY: ${{ github.repository }} run: | set -euo pipefail - make -f .trusted-base/Makefile supersede-meta-tag-reviews \ + make -f .trusted-base/Makefile supersede-enhancement-reviews \ TOOLS_DIR=.trusted-base/tools # File-level "Commit suggestion" comments. suggestion_note is intentionally # unstamped so supersede does not collapse this review; Conversation then # keeps showing live suggestions until GitHub marks each comment outdated. - - name: Suggest meta tag changes + - name: Suggest documentation enhancements if: >- ${{ !cancelled() && steps.ensure.outputs.inline_suggestions == 'true' }} @@ -84,7 +84,7 @@ jobs: # Always posts the summary so the Conversation view has a current review # after the stale ones are minimised, whatever suggest-changes did. - - name: Post meta tag review comment + - name: Post enhancement review comment if: >- ${{ !cancelled() && steps.ensure.outputs.has_results == 'true' }} @@ -98,8 +98,8 @@ jobs: --comment \ --body "$REVIEW_COMMENT" - - name: Enforce required metadata + - name: Enforce required enhancements if: always() && steps.ensure.outputs.has_errors == 'true' run: | - echo "Required metadata (error severity) is still missing." + echo "Required enhancements (error level) are still missing." exit 1 diff --git a/Makefile b/Makefile index 4b75bbe97b9..b3f2d674c6d 100644 --- a/Makefile +++ b/Makefile @@ -45,26 +45,26 @@ test-tools: spellcheck: git ls-files '*.md' '*.rst' | xargs codespell --config codespell.cfg -ensure-meta-tags: +ensure-enhancements: ifndef DIFF_BASE $(error DIFF_BASE is required) endif ifndef STATUS_FILE $(error STATUS_FILE is required) endif - $(PYTHON) $(TOOLS_DIR)/ensure_meta_tags.py \ + $(PYTHON) $(TOOLS_DIR)/ensure_enhancements.py \ --config $(TOOLS_DIR)/enhance.yaml \ --diff-base $(DIFF_BASE) \ --status-file $(STATUS_FILE) -supersede-meta-tag-reviews: +supersede-enhancement-reviews: ifndef PR_NUMBER $(error PR_NUMBER is required) endif ifndef REPOSITORY $(error REPOSITORY is required) endif - $(BASH) $(TOOLS_DIR)/supersede_meta_tag_reviews.sh + $(BASH) $(TOOLS_DIR)/supersede_enhancement_reviews.sh check-dictionaries: @echo "Checking dictionaries..." @@ -94,4 +94,4 @@ linkcheck: serve: sphinx-autobuild --host $(LIVE_HOST) --port $(LIVE_PORT) -c . $(SOURCE) $(OUT)/html -.PHONY: help Makefile multiversion test test-tools linkcheck serve lint spellcheck check-dictionaries sort-dictionaries ensure-meta-tags supersede-meta-tag-reviews $(MAKEFILE_LIST) +.PHONY: help Makefile multiversion test test-tools linkcheck serve lint spellcheck check-dictionaries sort-dictionaries ensure-enhancements supersede-enhancement-reviews $(MAKEFILE_LIST) diff --git a/tools/README.md b/tools/README.md index 516bd202166..e6d5391bec8 100644 --- a/tools/README.md +++ b/tools/README.md @@ -1,6 +1,6 @@ # Documentation tools -Helpers for ensuring reStructuredText (`.rst`) metadata on documentation pull requests. +Helpers for ensuring reStructuredText (`.rst`) documentation enhancements on pull requests. --- @@ -12,11 +12,11 @@ Information for documentation contributors creating or updating `.rst` files. | Component | Used by | |-----------|---------| -| [PyYAML](https://pyyaml.org/) (`pip install pyyaml`) | [`ensure_meta_tags.py`](ensure_meta_tags.py) and unit tests in [`tests/`](tests/) | +| [PyYAML](https://pyyaml.org/) (`pip install pyyaml`) | [`ensure_enhancements.py`](ensure_enhancements.py) and unit tests in [`tests/`](tests/) | | Git (with a usable `HEAD` and refs for your base commit) | PR-scope discovery and diff overlap checks (`--diff-base`) | -| [GitHub CLI](https://cli.github.com/) (`gh`) and `jq` | [`supersede_meta_tag_reviews.sh`](supersede_meta_tag_reviews.sh) only (Enhance workflow on `ubuntu-24.04`; optional for local supersede testing) | +| [GitHub CLI](https://cli.github.com/) (`gh`) and `jq` | [`supersede_enhancement_reviews.sh`](supersede_enhancement_reviews.sh) only (Enhance workflow on `ubuntu-24.04`; optional for local supersede testing) | -### Metadata configuration +### Enhancement configuration [`enhance.yaml`](enhance.yaml) defines documentation enhancement rules. The `meta` section lists every `.. meta::` field checked by the tooling. Each entry has: @@ -69,42 +69,42 @@ For `short-description`, the tool wraps the first prose paragraph after the titl | `warning` | Annotation + review | Soft warning (`continue-on-error`) | Succeeds | | `error` | Error annotation + review | Soft warning (same step) | **Fails** on final enforce step | -### Checking metadata locally +### Checking enhancements locally -`ensure_meta_tags.py` checks `.rst` files against `enhance.yaml`. Fields with a configured `value` are added or filled automatically when the edit can be suggested or applied locally. Fields with an empty `value` must be completed manually in the `.. meta::` block. After-title directives are added using the rules above. +[`ensure_enhancements.py`](ensure_enhancements.py) checks `.rst` files against [`enhance.yaml`](enhance.yaml). Fields with a configured `value` are added or filled automatically when the edit can be suggested or applied locally. Fields with an empty `value` must be completed manually in the `.. meta::` block. After-title directives are added using the rules above. #### Usage From the repository root: ```bash -python3 tools/ensure_meta_tags.py path/to/article.rst +python3 tools/ensure_enhancements.py path/to/article.rst ``` Multiple files: ```bash -python3 tools/ensure_meta_tags.py source/Topic/A.rst source/Topic/B.rst +python3 tools/ensure_enhancements.py source/Topic/A.rst source/Topic/B.rst ``` Pull request scope (discovers changed `ACMR` `*.rst` files via `git diff`; requires Makefile variables `DIFF_BASE` and `STATUS_FILE`): ```bash -make ensure-meta-tags DIFF_BASE=origin/rolling STATUS_FILE=/tmp/meta-tags-out.txt +make ensure-enhancements DIFF_BASE=origin/rolling STATUS_FILE=/tmp/enhance-out.txt ``` PR scope without Make (optional `--status-file`; exit codes follow local rules when omitted): ```bash -python3 tools/ensure_meta_tags.py --diff-base origin/rolling -python3 tools/ensure_meta_tags.py --diff-base "$(git merge-base HEAD origin/rolling)" --status-file /tmp/out.txt +python3 tools/ensure_enhancements.py --diff-base origin/rolling +python3 tools/ensure_enhancements.py --diff-base "$(git merge-base HEAD origin/rolling)" --status-file /tmp/out.txt ``` -For day-to-day editing of known files, pass paths explicitly and omit `--status-file` so the tool exits `1` only when **error**-severity fields remain. +For day-to-day editing of known files, pass paths explicitly and omit `--status-file` so the tool exits `1` only when **error**-severity issues remain. #### Exit codes -With `--status-file` (CI), exit `1` when any issues remain. Locally, exit `1` only when **error**-severity fields are still unresolved; warning-only issues exit `0` after applying automatic fixes. +With `--status-file` (CI), exit `1` when any issues remain. Locally, exit `1` only when **error**-severity issues are still unresolved; warning-only issues exit `0` after applying automatic fixes. #### Example (configured values) @@ -134,32 +134,34 @@ Fields such as `area` with an empty `value` in the config are listed in the revi ### Contributor CI experience -When you open or update a pull request, CI automatically checks metadata on all modified `.rst` files. +When you open or update a pull request, CI automatically checks enhancements on all modified `.rst` files. #### Contributor experience overview | Situation | Ensure step | Annotations | Pull request review | Job result | |-----------|-------------|-------------|---------------------|------------| -| All fields resolved | Green | None | None (stale bot reviews cleared) | Success | +| All enhancements resolved | Green | None | None (stale bot reviews cleared) | Success | | Warning-only gaps | Soft warning | Warnings | Suggestions and/or manual list | Success | | Error gaps (e.g. missing `area`) | Soft warning | Errors (and warnings) | Suggestions and/or manual list | **Failure** after enforce step | | Auto-fix in diff | Soft warning | As above | Inline “Commit suggestion” | As per severity | -| Auto-fix outside diff (`snippet`) | Soft warning | As above | Copy-paste `.. meta::` for configured values | As per severity | -| Manual fields only (`manual_fields`) | Soft warning | As above | Field list with required/warning labels | As per severity | +| Auto-fix outside diff (`snippet`) | Soft warning | As above | Copy-paste blocks for configured values | As per severity | +| Manual fields only (`manual_fields`) | Soft warning | As above | Field/directive list with required/warning labels | As per severity | Reviews, annotations, and the soft-failed ensure step appear in different parts of the GitHub UI (Conversation, Files changed, Checks); only error-severity issues fail the overall workflow. #### Review body sections -The **Documentation metadata** summary review (`## Documentation metadata`) names every affected file and splits the work by how it is fixed, so the fields listed for a file match that file's annotation. The **Inline metadata suggestions** review (`## Inline metadata suggestions`) is a short pointer to the commit suggestions; details stay in the summary. +The **Documentation enhancements** summary review (`## Documentation enhancements`) names every affected file and splits the work by how it is fixed. The **Inline documentation suggestions** review (`## Inline documentation suggestions`) is a short pointer to the commit suggestions; details stay in the summary. -| Heading | Files listed | Fields listed | -|---------|--------------|---------------| -| `### Commit inline suggestions` | `suggestable` mode | Configured values added for you — commit the suggestion | -| `### Copy-paste \`.. meta::\` blocks` | `snippet` mode | Configured values as an RST block to paste yourself | -| `### Provide non-empty values` | Any file with manual fields | Fields with an empty `value`, labelled required or optional | +| Heading | Files listed | What is listed | +|---------|--------------|----------------| +| `### Commit inline suggestions` | `suggestable` mode | Configured values and directives added for you — commit the suggestion | +| `### Copy-paste \`.. meta::\` blocks` | `snippet` mode (meta) | Configured meta values as an RST block to paste yourself | +| `### Copy-paste after-title directives` | `snippet` mode (after-title) | `.. short-description::` / `.. showmeta::` blocks to paste after the title | +| `### Provide non-empty values` | Any file with manual meta fields | Fields with an empty `value`, labelled required or optional | +| `### Add after-title directives` | Manual after-title gaps | Directives that could not be added automatically | -A file can appear in two sections: the suggestion covers its configured values while the manual list covers the rest. +A file can appear in multiple sections: the suggestion covers auto-filled items while the manual lists cover the rest. #### When inline suggestions appear @@ -169,16 +171,17 @@ GitHub only allows review suggestions on [lines already in the pull request diff |-----------|----------------| | Missing configured values; existing `.. meta::` overlaps the PR diff | Write append/fill to the working tree → inline suggestion via [`suggest-changes`](https://github.com/marketplace/actions/suggest-changes-action) | | No `.. meta::`; top of file overlaps the PR diff | Insert at top → inline suggestion | -| Automatic edit does **not** overlap the PR diff (`snippet` mode) | No inline write; review includes a copy-paste block for configured values | +| After-title edit overlaps the PR diff (title area or paragraph being wrapped) | Insert/wrap in working tree → inline suggestion | +| Automatic edit does **not** overlap the PR diff (`snippet` mode) | No inline write; review includes a copy-paste block | | Only manual fields (empty `value` in config, `manual_fields` mode) | Review lists fields; no placeholder injection | -| All configured fields present and non-empty | No action | -| No changed `.rst` files in the PR | No check; `meta_checked=false`; supersede/review steps skipped | +| All configured enhancements present | No action | +| No changed `.rst` files in the PR | No check; `enhancements_checked=false`; supersede/review steps skipped | --- ## Developer guidance -Information for maintainers and developers working on or extending the metadata tooling and CI workflows. +Information for maintainers and developers working on or extending the enhancement tooling and CI workflows. ### Repository layout @@ -186,8 +189,8 @@ Information for maintainers and developers working on or extending the metadata |------|---------| | [`rst_utils.py`](rst_utils.py) | Regex-based read/write of `.. meta::`, `.. short-description::`, and `.. showmeta::` directives | | [`enhance.yaml`](enhance.yaml) | Enhancement rules (`meta` fields and `after_title` directives) | -| [`ensure_meta_tags.py`](ensure_meta_tags.py) | CLI that checks and fixes metadata from the config | -| [`supersede_meta_tag_reviews.sh`](supersede_meta_tag_reviews.sh) | Minimise outdated bot PR reviews | +| [`ensure_enhancements.py`](ensure_enhancements.py) | CLI that checks and applies enhancements from the config | +| [`supersede_enhancement_reviews.sh`](supersede_enhancement_reviews.sh) | Minimise outdated bot PR reviews | | [`tests/`](tests/) | Unit tests for the tools in this directory | ### Code modules @@ -208,42 +211,42 @@ Add a new key under `meta` in [`enhance.yaml`](enhance.yaml) to extend metadata ### CLI options & Makefile targets -#### Options (`ensure_meta_tags.py`) +#### Options (`ensure_enhancements.py`) - `paths` — optional; when omitted, `--diff-base` is required and changed `.rst` files are discovered automatically - `--config PATH` — YAML config file (default: `tools/enhance.yaml`) - `--diff-base SHA` — PR base commit; limits on-disk writes to lines in the PR diff (inline suggestions); files that need a copy-paste or manual field list use a review comment instead -- `--status-file PATH` — write `meta_checked`, `inline_suggestions`, `has_results`, `has_errors`, the review comment body, and the suggestion note for CI; when issues remain, emits annotations and exits `1` (the ensure step uses `continue-on-error`) +- `--status-file PATH` — write `enhancements_checked`, `inline_suggestions`, `has_results`, `has_errors`, the review comment body, and the suggestion note for CI; when issues remain, emits annotations and exits `1` (the ensure step uses `continue-on-error`) - `-v` / `--verbose` — enable debug logging -#### Makefile targets (metadata CI) +#### Makefile targets (enhancement CI) Both targets live in the repository root [`Makefile`](../Makefile). CI invokes them with `make -f .trusted-base/Makefile …` so recipes run from the PR **base** branch, not the PR head. | Target | Required variables | Purpose | |--------|-------------------|---------| -| `ensure-meta-tags` | `DIFF_BASE`, `STATUS_FILE`; optional `TOOLS_DIR` (default `tools`) | Discover changed RST, run metadata check, append CI outputs | -| `supersede-meta-tag-reviews` | `PR_NUMBER`, `REPOSITORY`; optional `TOOLS_DIR` | Minimise stamped bot reviews | +| `ensure-enhancements` | `DIFF_BASE`, `STATUS_FILE`; optional `TOOLS_DIR` (default `tools`) | Discover changed RST, run enhancement check, append CI outputs | +| `supersede-enhancement-reviews` | `PR_NUMBER`, `REPOSITORY`; optional `TOOLS_DIR` | Minimise stamped bot reviews | -Environment for `supersede-meta-tag-reviews` (set by the workflow or locally): `GH_TOKEN`, plus `PR_NUMBER` and `REPOSITORY`. It writes no CI outputs; the workflow decides what to post from the ensure step’s outputs. +Environment for `supersede-enhancement-reviews` (set by the workflow or locally): `GH_TOKEN`, plus `PR_NUMBER` and `REPOSITORY`. It writes no CI outputs; the workflow decides what to post from the ensure step’s outputs. ### Continuous integration architecture The workflow [`.github/workflows/enhance.yml`](../.github/workflows/enhance.yml) runs on **`pull_request_target`** when a pull request is **opened**, **synchronised**, or **reopened** (including from forks). That event type allows the default `GITHUB_TOKEN` to post review suggestions on fork PRs; the workflow file on the repository **default branch** defines the job, while trusted tooling comes from the PR **base** branch (see [Security](#security) below). -The Enhance workflow installs PyYAML in the job; it does not install the full documentation `requirements.txt` for metadata checks. +The Enhance workflow installs PyYAML in the job; it does not install the full documentation `requirements.txt` for enhancement checks. -#### Job flow (`ensure-meta-tags`) +#### Job flow (`ensure-enhancements`) 1. Check out the PR **head** (`.rst` content to inspect and, where allowed, modify for suggestions). -2. Check out the PR **base** into `.trusted-base/` (Makefile, `ensure_meta_tags.py`, `enhance.yaml`, `supersede_meta_tag_reviews.sh`). +2. Check out the PR **base** into `.trusted-base/` (Makefile, `ensure_enhancements.py`, `enhance.yaml`, `supersede_enhancement_reviews.sh`). 3. Install Python 3.12 and PyYAML. -4. **Ensure documentation metadata** — `git fetch` the base SHA, then `make -f .trusted-base/Makefile ensure-meta-tags` with `TOOLS_DIR=.trusted-base/tools`, `DIFF_BASE`, and `STATUS_FILE=$GITHUB_OUTPUT`. Emits per-file annotations; the step uses `continue-on-error: true` so warning-only gaps do not fail the job immediately. -5. **Verify metadata check ran** — fail the job if `meta_checked` is empty. The ensure step soft-fails by design, so a missing output is the only way to tell a crashed check from a clean run. -6. **Supersede stale meta-tag reviews** (only if `meta_checked=true`) — `make -f .trusted-base/Makefile supersede-meta-tag-reviews`, which minimises stamped reviews and writes nothing back. -7. **Suggest meta tag changes** — if `inline_suggestions`, run [`suggest-changes`](https://github.com/marketplace/actions/suggest-changes-action) for file-level “Commit suggestion” comments, using `suggestion_note` as its review body (unstamped so supersede does not hide live suggestions in Conversation). -8. **Post meta tag review comment** — if `has_results`, post the stamped `comment` body via `gh pr review` (Conversation view). Independent of suggest-changes, which posts nothing when every suggestion duplicates one from an earlier run. -9. **Enforce required metadata** — if `has_errors`, fail the job (runs `always()` so error gaps fail even when the ensure step soft-failed). +4. **Ensure documentation enhancements** — `git fetch` the base SHA, then `make -f .trusted-base/Makefile ensure-enhancements` with `TOOLS_DIR=.trusted-base/tools`, `DIFF_BASE`, and `STATUS_FILE=$GITHUB_OUTPUT`. Emits per-file annotations; the step uses `continue-on-error: true` so warning-only gaps do not fail the job immediately. +5. **Verify enhancement check ran** — fail the job if `enhancements_checked` is empty. The ensure step soft-fails by design, so a missing output is the only way to tell a crashed check from a clean run. +6. **Supersede stale enhancement reviews** (only if `enhancements_checked=true`) — `make -f .trusted-base/Makefile supersede-enhancement-reviews`, which minimises stamped reviews and writes nothing back. +7. **Suggest documentation enhancements** — if `inline_suggestions`, run [`suggest-changes`](https://github.com/marketplace/actions/suggest-changes-action) for file-level “Commit suggestion” comments, using `suggestion_note` as its review body (unstamped so supersede does not hide live suggestions in Conversation). +8. **Post enhancement review comment** — if `has_results`, post the stamped `comment` body via `gh pr review` (Conversation view). Independent of suggest-changes, which posts nothing when every suggestion duplicates one from an earlier run. +9. **Enforce required enhancements** — if `has_errors`, fail the job (runs `always()` so error gaps fail even when the ensure step soft-failed). Steps 7 and 8 use `!cancelled()` rather than depending on the supersede step, so a transient GitHub API failure while minimising old reviews cannot stop contributors receiving feedback. @@ -255,20 +258,20 @@ Each changed `.rst` file is classified with an internal **mode**: | Mode | Meaning | |------|---------| -| `suggestable` | Configured values were written to the working tree for inline “Commit suggestion” | -| `snippet` | Configured values could not be written inline; the review includes a copy-paste `.. meta::` block | -| `manual_fields` | Only fields with empty `value` in the config are missing; the review lists field names | +| `suggestable` | Configured enhancements were written to the working tree for inline “Commit suggestion” | +| `snippet` | Enhancements could not be written inline; the review includes copy-paste RST blocks | +| `manual_fields` | Only manual items remain (empty meta `value`, or after-title gaps with no auto-fix) | -A single file can still list **manual** fields in the review when its mode is `suggestable` or `snippet` (auto-filled fields were handled; empty-config fields remain for the contributor). +A single file can still list **manual** fields in the review when its mode is `suggestable` or `snippet` (auto-filled items were handled; manual items remain for the contributor). The script writes **CI outputs** (for example `$GITHUB_OUTPUT`) that describe which workflow steps to run: | Output | Meaning | |--------|---------| -| `meta_checked` | At least one changed `.rst` was in scope (`false` when discovery finds no changed RST; supersede is skipped). Empty only when the check never ran | +| `enhancements_checked` | At least one changed `.rst` was in scope (`false` when discovery finds no changed RST; supersede is skipped). Empty only when the check never ran | | `inline_suggestions` | Run [`suggest-changes`](https://github.com/marketplace/actions/suggest-changes-action) for file-level suggestions | -| `has_results` | Metadata issues remain; post the Conversation review | -| `has_errors` | Unresolved **error**-severity fields (triggers the final enforce step) | +| `has_results` | Enhancement issues remain; post the Conversation review | +| `has_errors` | Unresolved **error**-severity issues (triggers the final enforce step) | | `comment` | Full stamped review body (multiline heredoc) posted to Conversation via `gh pr review`; written only when `has_results` | | `suggestion_note` | Short unstamped body for the suggest-changes review; written only when `inline_suggestions` | @@ -278,20 +281,20 @@ The ensure step uses `continue-on-error: true`, so warning-only gaps do not fail #### Annotations -When metadata is missing or blank: +When enhancements are missing: -- **Warning** severity → `::warning file=...,line=N::Missing meta fields: ...` -- **Error** severity → `::error file=...,line=N::Missing meta fields: ...` +- **Warning** severity → `::warning file=...,line=N::Missing meta fields: ...` or `Missing after-title directives: ...` +- **Error** severity → `::error file=...,line=N::Missing meta fields: ...` or `Missing after-title directives: ...` -`N` is the start line of an existing `.. meta::` block, or `1` when a new block would be inserted at the top of the file. +`N` is the start line of an existing `.. meta::` block, the after-title area, or `1` when a new block would be inserted at the top of the file. Annotations describe the pull request **as pushed**, so fields with a configured `value` are listed even when the same run offers them as an inline suggestion. Auto-injected values only exist in the CI working tree; they disappear from the annotations once the suggestion is committed and the workflow re-runs. #### Superseding outdated reviews -Only the **summary** review body includes a hidden HTML marker (``). The marker id `ros2-meta-tags-ensure` (constant `REVIEW_MARKER_ID` in Python; override in the shell script with `META_TAG_REVIEW_MARKER_ID`) is what the supersede script searches for in review bodies. The suggest-changes review uses an unstamped `suggestion_note` so it is not minimised: that card is the only place inline suggestions render in Conversation, and GitHub marks individual suggestion comments outdated once they are committed or their anchor leaves the diff. +Only the **summary** review body includes a hidden HTML marker (``). The marker id `ros2-doc-enhance-ensure` (constant `REVIEW_MARKER_ID` in Python; override in the shell script with `ENHANCEMENT_REVIEW_MARKER_ID`) is what the supersede script searches for in review bodies. The suggest-changes review uses an unstamped `suggestion_note` so it is not minimised: that card is the only place inline suggestions render in Conversation, and GitHub marks individual suggestion comments outdated once they are committed or their anchor leaves the diff. -When `meta_checked=true`, the workflow runs `make -f .trusted-base/Makefile supersede-meta-tag-reviews` ([`supersede_meta_tag_reviews.sh`](supersede_meta_tag_reviews.sh)): +When `enhancements_checked=true`, the workflow runs `make -f .trusted-base/Makefile supersede-enhancement-reviews` ([`supersede_enhancement_reviews.sh`](supersede_enhancement_reviews.sh)): 1. Lists pull request reviews whose body contains the marker id. 2. Minimises each as **Outdated** via the GitHub GraphQL API (`gh api graphql`; individual failures are ignored). diff --git a/tools/enhance.yaml b/tools/enhance.yaml index a37849affe0..dc7a59ca316 100644 --- a/tools/enhance.yaml +++ b/tools/enhance.yaml @@ -1,4 +1,4 @@ -# Enhancement rules for ensure_meta_tags.py. +# Enhancement rules for ensure_enhancements.py. # meta: .. meta:: field rules (severity and optional default values). # after_title: directives inserted after the first document title. # Sphinx expands {PRODUCT} and {DISTRO} at build time (see conf.py). diff --git a/tools/ensure_meta_tags.py b/tools/ensure_enhancements.py similarity index 97% rename from tools/ensure_meta_tags.py rename to tools/ensure_enhancements.py index a76869de198..cb8766316d2 100644 --- a/tools/ensure_meta_tags.py +++ b/tools/ensure_enhancements.py @@ -28,7 +28,7 @@ import yaml -# Allow ``python3 tools/ensure_meta_tags.py`` from the repository root. +# Allow ``python3 tools/ensure_enhancements.py`` from the repository root. _TOOLS_DIR = Path(__file__).resolve().parent if str(_TOOLS_DIR) not in sys.path: sys.path.insert(0, str(_TOOLS_DIR)) @@ -55,7 +55,7 @@ Severity = Literal["warning", "error"] # Hidden marker in review bodies so CI can find and supersede prior bot reviews. -REVIEW_MARKER_ID = "ros2-meta-tags-ensure" +REVIEW_MARKER_ID = "ros2-doc-enhance-ensure" REVIEW_MARKER = f"" # Titles and section headings for pull request review bodies (GitHub Markdown). @@ -337,15 +337,6 @@ def load_enhance_config(config_path: Path) -> EnhanceConfig: ) -def load_meta_config(config_path: Path) -> dict[str, MetaRule]: - """ - Load metadata rules from a YAML config file. - - Deprecated alias for ``load_enhance_config(...).meta``. - """ - return load_enhance_config(config_path).meta - - def format_meta_block(rules: dict[str, MetaRule], fields: list[str]) -> str: """ Build an RST ``.. meta::`` block for auto-injectable fields. @@ -511,7 +502,7 @@ def _log_working_tree_summary(paths: list[Path]) -> None: if not paths: return path_args = [str(p) for p in paths] - logger.info("Working tree after ensure_meta_tags:") + logger.info("Working tree after ensure_enhancements:") status = subprocess.run( ["git", "status", "--short", "--", *path_args], check=False, @@ -557,7 +548,7 @@ def _write_multiline_output(handle: TextIO, key: str, value: str) -> None: def _write_ci_status_file( status_file: Path, *, - meta_checked: bool, + enhancements_checked: bool, results: list[dict[str, object]], rules: dict[str, MetaRule], has_errors: bool, @@ -565,14 +556,14 @@ def _write_ci_status_file( """ Append GitHub Actions output flags and optional review comment. - Writes ``meta_checked``, ``inline_suggestions``, ``has_results``, and + Writes ``enhancements_checked``, ``inline_suggestions``, ``has_results``, and ``has_errors``, plus a multiline ``comment`` block when ``results`` is non-empty and ``suggestion_note`` when inline suggestions were written. Args: status_file: Path to append to (for example ``$GITHUB_OUTPUT``). - meta_checked: Whether changed RST files were in scope for this run. - results: Per-file result dicts from ``ensure_meta_tags_in_file``. + enhancements_checked: Whether changed RST files were in scope for this run. + results: Per-file result dicts from ``ensure_enhancements_in_file``. rules: Configured metadata rules for building the review body. has_errors: Whether any result has unresolved error-severity fields. @@ -583,7 +574,7 @@ def _write_ci_status_file( has_results = bool(results) with status_file.open("a", encoding="utf-8") as f: for key, flag in ( - ("meta_checked", meta_checked), + ("enhancements_checked", enhancements_checked), ("inline_suggestions", has_inline_suggestions), ("has_results", has_results), ("has_errors", has_errors), @@ -714,7 +705,7 @@ def emit_github_error(path: str, fields: list[str], line: int) -> None: def can_suggest_inline(content: str, pr_lines: set[int]) -> bool: """ - Return whether a meta-tag edit can be anchored to the pull request diff. + Return whether a ``.. meta::`` edit can be anchored to the pull request diff. Existing ``.. meta::`` blocks are suggestable when the block's inclusive line span overlaps the diff. New blocks are only suggestable when line 1 is in @@ -897,7 +888,7 @@ def _severity_fields( return [name for name in field_names if rules[name].severity == severity] -def ensure_meta_tags_in_file( +def ensure_enhancements_in_file( path: Path, config: EnhanceConfig | dict[str, MetaRule], *, @@ -1081,7 +1072,7 @@ def build_review_comment( Build a pull-request review body from enhancement check results. Args: - results: Per-file result dictionaries from ``ensure_meta_tags_in_file``. + results: Per-file result dictionaries from ``ensure_enhancements_in_file``. rules: Configured metadata rules. Returns: @@ -1209,7 +1200,7 @@ def build_review_comment( def main(argv: list[str] | None = None) -> int: """ - Run the command-line metadata check. + Run the command-line documentation enhancement check. When no ``paths`` are given, ``--diff-base`` must be set so changed ``.rst`` files are discovered with ``git diff``. With ``--status-file``, writes CI @@ -1225,12 +1216,12 @@ def main(argv: list[str] | None = None) -> int: above). Raises: - SystemExit: If command-line arguments or metadata configuration are invalid. + SystemExit: If command-line arguments or enhancement configuration are invalid. """ parser = argparse.ArgumentParser( description=( - "Ensure configured meta tags exist in RST files using rules from a YAML " - "config file." + "Ensure configured documentation enhancements exist in RST files " + "using rules from a YAML config file." ), ) parser.add_argument( @@ -1258,7 +1249,7 @@ def main(argv: list[str] | None = None) -> int: "--status-file", type=Path, help=( - "Write meta_checked, inline_suggestions, has_results, has_errors, " + "Write enhancements_checked, inline_suggestions, has_results, has_errors, " "the review comment body, and the suggestion note for CI" ), ) @@ -1289,7 +1280,7 @@ def main(argv: list[str] | None = None) -> int: if args.status_file is not None: _write_ci_status_file( args.status_file, - meta_checked=False, + enhancements_checked=False, results=[], rules=rules.meta, has_errors=False, @@ -1310,7 +1301,7 @@ def main(argv: list[str] | None = None) -> int: pr_lines: set[int] | None = None if args.diff_base: pr_lines = pr_diff_lines_for_file(args.diff_base, path) - result = ensure_meta_tags_in_file(path, rules, pr_lines=pr_lines) + result = ensure_enhancements_in_file(path, rules, pr_lines=pr_lines) if result is not None: results.append(result) @@ -1352,7 +1343,7 @@ def main(argv: list[str] | None = None) -> int: if args.status_file is not None: _write_ci_status_file( args.status_file, - meta_checked=checked_pull_request_rst, + enhancements_checked=checked_pull_request_rst, results=results, rules=rules.meta, has_errors=has_errors, diff --git a/tools/supersede_meta_tag_reviews.sh b/tools/supersede_enhancement_reviews.sh similarity index 81% rename from tools/supersede_meta_tag_reviews.sh rename to tools/supersede_enhancement_reviews.sh index 315f18172ad..8737cedd5a4 100755 --- a/tools/supersede_meta_tag_reviews.sh +++ b/tools/supersede_enhancement_reviews.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Minimise prior stamped summary reviews from ensure_meta_tags.py. Suggestion-carrying +# Minimise prior stamped summary reviews from ensure_enhancements.py. Suggestion-carrying # reviews (unstamped suggest-changes bodies) are left visible so Conversation keeps # live inline suggestions until they are actioned. set -euo pipefail @@ -8,9 +8,9 @@ set -euo pipefail : "${REPOSITORY:?REPOSITORY is required}" : "${PR_NUMBER:?PR_NUMBER is required}" -MARKER="${META_TAG_REVIEW_MARKER_ID:-ros2-meta-tags-ensure}" +MARKER="${ENHANCEMENT_REVIEW_MARKER_ID:-ros2-doc-enhance-ensure}" -echo "Marking prior meta-tag reviews as outdated." +echo "Marking prior enhancement reviews as outdated." reviews_json="$(gh api "repos/${REPOSITORY}/pulls/${PR_NUMBER}/reviews" --paginate)" mapfile -t review_ids < <( @@ -30,3 +30,4 @@ for node_id in "${review_ids[@]:-}"; do done echo "Superseded ${#review_ids[@]} prior review comment(s)." + diff --git a/tools/tests/test_ensure_meta_tags.py b/tools/tests/test_ensure_enhancements.py similarity index 94% rename from tools/tests/test_ensure_meta_tags.py rename to tools/tests/test_ensure_enhancements.py index 9bfb685f93d..287138825b0 100644 --- a/tools/tests/test_ensure_meta_tags.py +++ b/tools/tests/test_ensure_enhancements.py @@ -25,7 +25,7 @@ if str(_TOOLS_DIR) not in sys.path: sys.path.insert(0, str(_TOOLS_DIR)) -from ensure_meta_tags import ( # noqa: E402 +from ensure_enhancements import ( # noqa: E402 REVIEW_MARKER, SECTION_COPY_PASTE_AFTER_TITLE, SECTION_INLINE_SUGGESTIONS, @@ -39,9 +39,8 @@ can_suggest_after_title_inline, can_suggest_inline, changed_rst_paths, - ensure_meta_tags_in_file, + ensure_enhancements_in_file, load_enhance_config, - load_meta_config, main, ) from rst_utils import get_meta_fields_from_content, inject_metadata_to_content # noqa: E402 @@ -101,15 +100,16 @@ ).strip() -class TestMetaConfig(unittest.TestCase): - def test_load_meta_config_parses_rules(self) -> None: +class TestEnhanceConfig(unittest.TestCase): + def test_load_enhance_config_parses_meta_rules(self) -> None: with tempfile.NamedTemporaryFile("w", suffix=".yaml", delete=False) as handle: handle.write(SAMPLE_CONFIG) path = Path(handle.name) try: - rules = load_meta_config(path) + config = load_enhance_config(path) finally: path.unlink() + rules = config.meta self.assertEqual(rules["product"].severity, "warning") self.assertEqual(rules["product"].value, "{PRODUCT}") self.assertTrue(rules["product"].has_configured_value) @@ -190,7 +190,7 @@ def test_inject_fills_blank_configured_value(self) -> None: self.assertEqual(fields["product"], "{PRODUCT}") -class TestEnsureMetaTagsInFile(unittest.TestCase): +class TestEnsureEnhancementsInFile(unittest.TestCase): def test_local_auto_inject_clears_configured_fields(self) -> None: rules = { "product": MetaRule("warning", "{PRODUCT}"), @@ -199,7 +199,7 @@ def test_local_auto_inject_clears_configured_fields(self) -> None: with tempfile.TemporaryDirectory() as tmp: path = Path(tmp) / "page.rst" path.write_text("Title\n=====\n", encoding="utf-8") - result = ensure_meta_tags_in_file(path, rules) + result = ensure_enhancements_in_file(path, rules) self.assertIsNotNone(result) self.assertEqual(result["mode"], "suggestable") self.assertIn("area", result["manual_fields"]) @@ -215,7 +215,7 @@ def test_auto_injected_fields_are_still_annotated(self) -> None: with tempfile.TemporaryDirectory() as tmp: path = Path(tmp) / "page.rst" path.write_text("Title\n=====\n", encoding="utf-8") - result = ensure_meta_tags_in_file(path, rules) + result = ensure_enhancements_in_file(path, rules) self.assertIsNotNone(result) self.assertIn("product", result["warning_fields"]) self.assertNotIn("product", result["manual_fields"]) @@ -225,7 +225,7 @@ def test_result_returned_when_only_configured_fields_missing(self) -> None: with tempfile.TemporaryDirectory() as tmp: path = Path(tmp) / "page.rst" path.write_text("Title\n=====\n", encoding="utf-8") - result = ensure_meta_tags_in_file(path, rules) + result = ensure_enhancements_in_file(path, rules) self.assertIsNotNone(result) self.assertEqual(result["mode"], "suggestable") self.assertEqual(result["warning_fields"], ["product"]) @@ -245,7 +245,7 @@ def test_no_result_when_all_fields_present(self) -> None: with tempfile.TemporaryDirectory() as tmp: path = Path(tmp) / "page.rst" path.write_text(content, encoding="utf-8") - self.assertIsNone(ensure_meta_tags_in_file(path, rules)) + self.assertIsNone(ensure_enhancements_in_file(path, rules)) class TestAfterTitleEnhancements(unittest.TestCase): @@ -267,7 +267,7 @@ def test_inserts_short_description_and_showmeta(self) -> None: with tempfile.TemporaryDirectory() as tmp: path = Path(tmp) / "page.rst" path.write_text(content, encoding="utf-8") - result = ensure_meta_tags_in_file(path, config) + result = ensure_enhancements_in_file(path, config) self.assertIsNotNone(result) updated = path.read_text(encoding="utf-8") self.assertIn(".. short-description::", updated) @@ -294,7 +294,7 @@ def test_toctree_before_paragraph_wraps_correct_paragraph(self) -> None: with tempfile.TemporaryDirectory() as tmp: path = Path(tmp) / "page.rst" path.write_text(content, encoding="utf-8") - result = ensure_meta_tags_in_file(path, config) + result = ensure_enhancements_in_file(path, config) self.assertIsNotNone(result) updated = path.read_text(encoding="utf-8") self.assertIn("First paragraph here.", updated) @@ -509,7 +509,7 @@ def test_clean_file_writes_no_review_bodies(self) -> None: class TestChangedRstPaths(unittest.TestCase): def test_parses_git_diff_output(self) -> None: completed = mock.Mock(returncode=0, stdout="source/A.rst\nsource/B.rst\n", stderr="") - with mock.patch("ensure_meta_tags.subprocess.run", return_value=completed) as run: + with mock.patch("ensure_enhancements.subprocess.run", return_value=completed) as run: paths = changed_rst_paths("abc123") self.assertEqual(paths, [Path("source/A.rst"), Path("source/B.rst")]) run.assert_called_once() @@ -530,7 +530,7 @@ def test_requires_paths_or_diff_base(self) -> None: finally: config_path.unlink() - def test_empty_discovery_writes_meta_checked_false(self) -> None: + def test_empty_discovery_writes_enhancements_checked_false(self) -> None: with tempfile.NamedTemporaryFile("w", suffix=".yaml", delete=False) as handle: handle.write(SAMPLE_CONFIG) config_path = Path(handle.name) @@ -539,7 +539,7 @@ def test_empty_discovery_writes_meta_checked_false(self) -> None: status_path = Path(status_handle.name) try: with mock.patch( - "ensure_meta_tags.changed_rst_paths", + "ensure_enhancements.changed_rst_paths", return_value=[], ): code = main( @@ -554,7 +554,7 @@ def test_empty_discovery_writes_meta_checked_false(self) -> None: ) self.assertEqual(code, 0) status_text = status_path.read_text(encoding="utf-8") - self.assertIn("meta_checked=false", status_text) + self.assertIn("enhancements_checked=false", status_text) self.assertIn("has_errors=false", status_text) finally: status_path.unlink() From 69436c5e4aedd0474f01ec14c3b29a8e91cf21a2 Mon Sep 17 00:00:00 2001 From: Keith Kirkwood Date: Fri, 31 Jul 2026 16:57:40 +0100 Subject: [PATCH 19/23] OPENR-174: Make enhance config file more consistent --- tools/README.md | 20 +++- tools/enhance.yaml | 9 +- tools/ensure_enhancements.py | 141 ++++++++++++------------ tools/tests/test_ensure_enhancements.py | 30 ++--- tools/tests/test_rst_utils.py | 8 +- 5 files changed, 109 insertions(+), 99 deletions(-) diff --git a/tools/README.md b/tools/README.md index e6d5391bec8..0e426377f40 100644 --- a/tools/README.md +++ b/tools/README.md @@ -44,22 +44,23 @@ meta: `{PRODUCT}` and `{DISTRO}` are Sphinx substitution macros expanded at build time from [`conf.py`](../conf.py). -The `after_title` section lists directives inserted after the first document title, in order: +The `after_title` section maps directive names to rules, in the order they are inserted after the first document title (same shape as `meta`): ```yaml after_title: - - directive: short-description + short-description: severity: warning content: first_paragraph - - - directive: showmeta + showmeta: severity: warning options: - order: "area, contentType, experience" + order: area, content-type, experience required_options: - order ``` +The `:order:` value lists `.. meta::` field names and must match the `meta` section (e.g. `content-type`, not `contentType`). + For `short-description`, the tool wraps the first prose paragraph after the title into the directive (removing it from the body). For `showmeta`, it inserts the directive with the configured `:order:` option when missing. #### Severity behaviour @@ -207,7 +208,14 @@ Low-level utilities for locating and editing Sphinx directives in RST source: #### Extending configuration -Add a new key under `meta` in [`enhance.yaml`](enhance.yaml) to extend metadata coverage without changing Python code. Add entries to `after_title` for additional post-heading directives supported by the tooling. +Add a new key under `meta` in [`enhance.yaml`](enhance.yaml) to extend metadata coverage without changing Python code. `_parse_meta_rules` accepts any key with `severity` and `value`, and the rest of the pipeline (unresolved-field detection, auto-injection, annotations, review sections) is driven entirely by that mapping. + +`after_title` is only partly config-driven. Its mapping shape lets you reorder or retune the two existing directives (`short-description`, `showmeta`) from YAML alone — but `_parse_after_title_rules` whitelists `supported_directives = {"short-description", "showmeta"}`, so adding a *new* after-title directive needs Python changes: + +1. Add the name to `supported_directives` and its directive-specific validation in `_parse_after_title_rules`. +2. Extend `_after_title_rule_satisfied` with an "already present" check for the new directive. +3. Extend `_process_after_title_rules` to inject content or produce a copy-paste snippet for it. +4. Add matching read/write/format helpers to [`rst_utils.py`](rst_utils.py) — see `has_showmeta_with_order`, `inject_showmeta_to_content`, and `format_showmeta_block` for the `showmeta` example. ### CLI options & Makefile targets diff --git a/tools/enhance.yaml b/tools/enhance.yaml index dc7a59ca316..c11bf53cc2c 100644 --- a/tools/enhance.yaml +++ b/tools/enhance.yaml @@ -1,6 +1,6 @@ # Enhancement rules for ensure_enhancements.py. # meta: .. meta:: field rules (severity and optional default values). -# after_title: directives inserted after the first document title. +# after_title: mapping of directive name → rules for post-title insertion (YAML key order). # Sphinx expands {PRODUCT} and {DISTRO} at build time (see conf.py). meta: product: @@ -20,13 +20,12 @@ meta: value: after_title: - - directive: short-description + short-description: severity: warning content: first_paragraph - - - directive: showmeta + showmeta: severity: warning options: - order: "area, contentType, experience" + order: area, content-type, experience required_options: - order diff --git a/tools/ensure_enhancements.py b/tools/ensure_enhancements.py index cb8766316d2..24bfd789e2e 100644 --- a/tools/ensure_enhancements.py +++ b/tools/ensure_enhancements.py @@ -2,8 +2,8 @@ """ Ensure configured documentation enhancements exist in RST source files. -Rules are defined in ``enhance.yaml``: ``meta`` field rules for ``.. meta::`` -blocks and ``after_title`` rules for post-heading directives such as +Rules are defined in ``enhance.yaml``: ``meta`` and ``after_title`` mappings +for ``.. meta::`` fields and post-heading directives such as ``.. short-description::`` and ``.. showmeta::``. When ``--diff-base`` is set, edits are only written to disk when they overlap @@ -86,15 +86,15 @@ class AfterTitleRule: """ A single after-title directive rule from ``enhance.yaml``. + The directive name is the key in the ``after_title`` mapping (like ``meta``). + Attributes: - directive: Directive name (e.g. ``short-description``, ``showmeta``). severity: Advisory ``warning`` or blocking ``error`` in CI. content: Content source for body directives (e.g. ``first_paragraph``). options: Option name/value pairs for option-only directives. required_options: Option names that must be non-empty when present. """ - directive: str severity: Severity content: str | None = None options: dict[str, str] | None = None @@ -106,7 +106,7 @@ class EnhanceConfig: """Full enhancement configuration loaded from ``enhance.yaml``.""" meta: dict[str, MetaRule] - after_title: tuple[AfterTitleRule, ...] + after_title: dict[str, AfterTitleRule] @dataclass(frozen=True) @@ -178,48 +178,48 @@ def _parse_meta_rules(meta: dict, config_path: Path) -> dict[str, MetaRule]: def _parse_after_title_rules( - raw_list: object, + raw: object, config_path: Path, -) -> tuple[AfterTitleRule, ...]: - """Validate and parse the ``after_title`` list from config YAML.""" - if raw_list is None: - return () - if not isinstance(raw_list, list): - logger.error("Config %s: 'after_title' must be a list", config_path) +) -> dict[str, AfterTitleRule]: + """Validate and parse the ``after_title`` mapping from config YAML.""" + if raw is None: + return {} + if not isinstance(raw, dict): + logger.error( + "Config %s: 'after_title' must be a mapping keyed by directive name", + config_path, + ) raise SystemExit(1) supported_directives = {"short-description", "showmeta"} - validated: list[AfterTitleRule] = [] - for index, entry in enumerate(raw_list): - if not isinstance(entry, dict): + validated: dict[str, AfterTitleRule] = {} + for directive, entry in raw.items(): + if not isinstance(directive, str) or not directive.strip(): logger.error( - "Config %s: after_title[%d] must be a mapping", + "Config %s: after_title keys must be non-empty directive names", config_path, - index, ) raise SystemExit(1) - directive = entry.get("directive") - if not isinstance(directive, str) or not directive.strip(): + if directive not in supported_directives: logger.error( - "Config %s: after_title[%d] must include a non-empty 'directive'", + "Config %s: after_title directive %r is not supported", config_path, - index, + directive, ) raise SystemExit(1) - if directive not in supported_directives: + if not isinstance(entry, dict): logger.error( - "Config %s: after_title[%d] directive %r is not supported", + "Config %s: after_title entry for %r must be a mapping", config_path, - index, directive, ) raise SystemExit(1) severity = entry.get("severity") if severity not in ("warning", "error"): logger.error( - "Config %s: after_title[%d] severity must be 'warning' or 'error', got %r", + "Config %s: after_title entry %r severity must be 'warning' or 'error', got %r", config_path, - index, + directive, severity, ) raise SystemExit(1) @@ -227,9 +227,9 @@ def _parse_after_title_rules( content = entry.get("content") if content is not None and not isinstance(content, str): logger.error( - "Config %s: after_title[%d] content must be a string", + "Config %s: after_title entry %r content must be a string", config_path, - index, + directive, ) raise SystemExit(1) @@ -238,18 +238,18 @@ def _parse_after_title_rules( if raw_options is not None: if not isinstance(raw_options, dict): logger.error( - "Config %s: after_title[%d] options must be a mapping", + "Config %s: after_title entry %r options must be a mapping", config_path, - index, + directive, ) raise SystemExit(1) options = {} for opt_key, opt_value in raw_options.items(): if not isinstance(opt_key, str) or not isinstance(opt_value, str): logger.error( - "Config %s: after_title[%d] option keys and values must be strings", + "Config %s: after_title entry %r option keys and values must be strings", config_path, - index, + directive, ) raise SystemExit(1) options[opt_key] = opt_value @@ -259,9 +259,9 @@ def _parse_after_title_rules( if raw_required is not None: if not isinstance(raw_required, list): logger.error( - "Config %s: after_title[%d] required_options must be a list", + "Config %s: after_title entry %r required_options must be a list", config_path, - index, + directive, ) raise SystemExit(1) required_options = tuple(str(item) for item in raw_required) @@ -287,16 +287,13 @@ def _parse_after_title_rules( ) raise SystemExit(1) - validated.append( - AfterTitleRule( - directive=directive, - severity=severity, - content=content, - options=options, - required_options=required_options, - ), + validated[directive] = AfterTitleRule( + severity=severity, + content=content, + options=options, + required_options=required_options, ) - return tuple(validated) + return validated def load_enhance_config(config_path: Path) -> EnhanceConfig: @@ -749,11 +746,11 @@ def can_suggest_after_title_inline( return _span_overlaps(after_title_directives_line_span(content), pr_lines) -def _after_title_rule_satisfied(content: str, rule: AfterTitleRule) -> bool: +def _after_title_rule_satisfied(content: str, directive: str, rule: AfterTitleRule) -> bool: """Return whether an after-title rule is already satisfied in ``content``.""" - if rule.directive == "short-description": + if directive == "short-description": return has_short_description_content(content) - if rule.directive == "showmeta": + if directive == "showmeta": return has_showmeta_with_order(content) return True @@ -773,7 +770,7 @@ def _format_short_description_snippet(paragraph: str) -> str: def _process_after_title_rules( path: Path, content: str, - after_title_rules: tuple[AfterTitleRule, ...], + after_title_rules: dict[str, AfterTitleRule], *, pr_lines: set[int] | None, ) -> tuple[str, dict[str, object]]: @@ -783,7 +780,11 @@ def _process_after_title_rules( Returns: Updated content and a dict of after-title result fields. """ - unresolved_rules = [rule for rule in after_title_rules if not _after_title_rule_satisfied(content, rule)] + unresolved_rules = [ + (directive, rule) + for directive, rule in after_title_rules.items() + if not _after_title_rule_satisfied(content, directive, rule) + ] if not unresolved_rules: return content, {} @@ -795,16 +796,16 @@ def _process_after_title_rules( suggestable = False snippet_only = False - for rule in unresolved_rules: + for directive, rule in unresolved_rules: paragraph_span: tuple[int, int] | None = None - if rule.directive == "short-description": + if directive == "short-description": paragraph, paragraph_span = extract_first_paragraph_after_title(content) if paragraph is None: - after_title_manual.append(rule.directive) + after_title_manual.append(directive) if rule.severity == "error": - after_title_error.append(rule.directive) + after_title_error.append(directive) else: - after_title_warning.append(rule.directive) + after_title_warning.append(directive) continue can_suggest = pr_lines is None or can_suggest_after_title_inline( @@ -818,21 +819,21 @@ def _process_after_title_rules( new_content, changed = wrap_first_paragraph_as_short_description(content) if changed: content = new_content - after_title_auto.append(rule.directive) + after_title_auto.append(directive) suggestable = True if rule.severity == "error": - after_title_error.append(rule.directive) + after_title_error.append(directive) else: - after_title_warning.append(rule.directive) + after_title_warning.append(directive) else: - after_title_snippets.append({"directive": rule.directive, "snippet": snippet_text}) + after_title_snippets.append({"directive": directive, "snippet": snippet_text}) snippet_only = True if rule.severity == "error": - after_title_error.append(rule.directive) + after_title_error.append(directive) else: - after_title_warning.append(rule.directive) + after_title_warning.append(directive) - elif rule.directive == "showmeta": + elif directive == "showmeta": assert rule.options is not None can_suggest = pr_lines is None or can_suggest_after_title_inline(content, pr_lines) snippet_text = format_showmeta_block(rule.options) @@ -841,19 +842,19 @@ def _process_after_title_rules( new_content, changed = inject_showmeta_to_content(content, rule.options) if changed: content = new_content - after_title_auto.append(rule.directive) + after_title_auto.append(directive) suggestable = True if rule.severity == "error": - after_title_error.append(rule.directive) + after_title_error.append(directive) else: - after_title_warning.append(rule.directive) + after_title_warning.append(directive) else: - after_title_snippets.append({"directive": rule.directive, "snippet": snippet_text}) + after_title_snippets.append({"directive": directive, "snippet": snippet_text}) snippet_only = True if rule.severity == "error": - after_title_error.append(rule.directive) + after_title_error.append(directive) else: - after_title_warning.append(rule.directive) + after_title_warning.append(directive) if not (after_title_auto or after_title_manual or after_title_snippets): return content, {} @@ -915,7 +916,7 @@ def ensure_enhancements_in_file( UnicodeError: If the RST file cannot be decoded or encoded as UTF-8. """ if isinstance(config, dict): - enhance_config = EnhanceConfig(meta=config, after_title=()) + enhance_config = EnhanceConfig(meta=config, after_title={}) else: enhance_config = config rules = enhance_config.meta @@ -925,9 +926,9 @@ def ensure_enhancements_in_file( unresolved = _unresolved_fields(content, rules) after_title_unresolved = [ - rule - for rule in enhance_config.after_title - if not _after_title_rule_satisfied(content, rule) + directive + for directive, rule in enhance_config.after_title.items() + if not _after_title_rule_satisfied(content, directive, rule) ] if not unresolved and not after_title_unresolved: diff --git a/tools/tests/test_ensure_enhancements.py b/tools/tests/test_ensure_enhancements.py index 287138825b0..1e9d3da9091 100644 --- a/tools/tests/test_ensure_enhancements.py +++ b/tools/tests/test_ensure_enhancements.py @@ -58,31 +58,29 @@ severity: warning value: after_title: - - directive: short-description + short-description: severity: warning content: first_paragraph - - directive: showmeta + showmeta: severity: warning options: - order: "area, contentType, experience" + order: area, content-type, experience required_options: - order """ ).strip() -AFTER_TITLE_RULES = ( - AfterTitleRule( - directive="short-description", +AFTER_TITLE_RULES = { + "short-description": AfterTitleRule( severity="warning", content="first_paragraph", ), - AfterTitleRule( - directive="showmeta", + "showmeta": AfterTitleRule( severity="warning", - options={"order": "area, contentType, experience"}, + options={"order": "area, content-type, experience"}, required_options=("order",), ), -) +} META_ONLY_CONFIG = textwrap.dedent( """ @@ -125,8 +123,12 @@ def test_load_enhance_config_parses_after_title(self) -> None: finally: path.unlink() self.assertEqual(len(config.after_title), 2) - self.assertEqual(config.after_title[0].directive, "short-description") - self.assertEqual(config.after_title[1].directive, "showmeta") + self.assertEqual(list(config.after_title.keys()), ["short-description", "showmeta"]) + self.assertEqual(config.after_title["short-description"].content, "first_paragraph") + self.assertEqual( + config.after_title["showmeta"].options, + {"order": "area, content-type, experience"}, + ) class TestCanSuggestInline(unittest.TestCase): @@ -272,7 +274,7 @@ def test_inserts_short_description_and_showmeta(self) -> None: updated = path.read_text(encoding="utf-8") self.assertIn(".. short-description::", updated) self.assertIn(".. showmeta::", updated) - self.assertIn(":order: area, contentType, experience", updated) + self.assertIn(":order: area, content-type, experience", updated) self.assertIn("Opening paragraph for the page.", updated) self.assertIn("More content.", updated) @@ -496,7 +498,7 @@ def test_clean_file_writes_no_review_bodies(self) -> None: Summary for the page. .. showmeta:: - :order: area, contentType, experience + :order: area, content-type, experience """ ).lstrip() status = self._run_with_status_file(content) diff --git a/tools/tests/test_rst_utils.py b/tools/tests/test_rst_utils.py index 9fc34a810c8..5ee8eba0157 100644 --- a/tools/tests/test_rst_utils.py +++ b/tools/tests/test_rst_utils.py @@ -151,7 +151,7 @@ def test_inject_showmeta_after_short_description(self) -> None: ).lstrip() updated, changed = inject_showmeta_to_content( content, - {"order": "area, contentType, experience"}, + {"order": "area, content-type, experience"}, ) self.assertTrue(changed) self.assertTrue(has_showmeta_with_order(updated)) @@ -191,15 +191,15 @@ def test_does_not_overwrite_existing_order(self) -> None: ).lstrip() updated, changed = inject_showmeta_to_content( content, - {"order": "area, contentType, experience"}, + {"order": "area, content-type, experience"}, ) self.assertFalse(changed) self.assertEqual(updated, content) def test_format_showmeta_block(self) -> None: - block = format_showmeta_block({"order": "area, contentType, experience"}) + block = format_showmeta_block({"order": "area, content-type, experience"}) self.assertIn(".. showmeta::", block) - self.assertIn(":order: area, contentType, experience", block) + self.assertIn(":order: area, content-type, experience", block) if __name__ == "__main__": From 4dd1148bf40afb4ac401918976aab0cbbda6aecb Mon Sep 17 00:00:00 2001 From: Keith Kirkwood Date: Mon, 3 Aug 2026 12:16:39 +0100 Subject: [PATCH 20/23] OPENR-174: Fix annotation and short desc formatting bugs --- tools/ensure_enhancements.py | 40 ++++++++++++++++------- tools/rst_utils.py | 7 ++-- tools/tests/test_ensure_enhancements.py | 43 +++++++++++++++++++++++++ tools/tests/test_rst_utils.py | 29 +++++++++++++++++ 4 files changed, 104 insertions(+), 15 deletions(-) diff --git a/tools/ensure_enhancements.py b/tools/ensure_enhancements.py index 24bfd789e2e..5dea34510cf 100644 --- a/tools/ensure_enhancements.py +++ b/tools/ensure_enhancements.py @@ -600,17 +600,30 @@ def _span_overlaps(span: tuple[int, int] | None, pr_lines: set[int]) -> bool: return any(line in pr_lines for line in range(start, end + 1)) -def _annotation_line_for_content(content: str) -> int: +def _annotation_line_for_meta(content: str) -> int: """ - Select a line for a GitHub annotation on RST content. - - Args: - content: RST source to inspect. + Select a line for a GitHub annotation on ``.. meta::`` issues. Returns: - The first line of an existing meta block, the after-title area, or line 1. + The first line of an existing meta block, or line 1 if none exists. """ span = meta_block_line_span(content) + if span is not None: + return span[0] + return 1 + + +def _annotation_line_for_after_title(content: str) -> int: + """ + Select a line for a GitHub annotation on after-title directive issues. + + Prefers the first prose paragraph after the title when present, otherwise + the post-title directive area or the line after the document title. + + Returns: + A 1-based source line appropriate for after-title annotations. + """ + _paragraph, span = extract_first_paragraph_after_title(content) if span is not None: return span[0] after_title = after_title_directives_line_span(content) @@ -935,7 +948,8 @@ def ensure_enhancements_in_file( logger.info("%s: all configured enhancements present", path) return None - annotation_line = _annotation_line_for_content(content) + annotation_line = _annotation_line_for_meta(content) + after_title_line = _annotation_line_for_after_title(content) auto_fields = [name for name in unresolved if rules[name].has_configured_value] auto_metadata = {name: rules[name].value for name in auto_fields} @@ -997,6 +1011,7 @@ def ensure_enhancements_in_file( return { "path": path_str, "line": annotation_line, + "after_title_line": after_title_line, "mode": mode, "snippet": snippet, "auto_fields": auto_fields, @@ -1320,20 +1335,21 @@ def main(argv: list[str] | None = None) -> int: for result in results: path = str(result["path"]) - line = int(result["line"]) - emit_github_warning(path, list(result["warning_fields"]), line) - emit_github_error(path, list(result["error_fields"]), line) + meta_line = int(result["line"]) + after_title_line = int(result.get("after_title_line") or meta_line) + emit_github_warning(path, list(result["warning_fields"]), meta_line) + emit_github_error(path, list(result["error_fields"]), meta_line) _emit_after_title_annotation( "warning", path, list(result.get("after_title_warning") or []), - line, + after_title_line, ) _emit_after_title_annotation( "error", path, list(result.get("after_title_error") or []), - line, + after_title_line, ) has_errors = any( diff --git a/tools/rst_utils.py b/tools/rst_utils.py index c23c1ae88ff..6fcb2436f10 100644 --- a/tools/rst_utils.py +++ b/tools/rst_utils.py @@ -500,8 +500,9 @@ def extract_first_paragraph_after_title( content: RST source to search. Returns: - Normalised paragraph text and its inclusive 1-based line span, or - ``(None, None)`` when no prose paragraph is found. + Paragraph text with each source prose line separated by a newline, and + its inclusive 1-based line span, or ``(None, None)`` when no prose + paragraph is found. """ lines = content.splitlines(keepends=True) stripped_lines = [line.rstrip("\n") for line in lines] @@ -552,7 +553,7 @@ def extract_first_paragraph_after_title( start_line = prose_start + 1 end_line = prose_start + len(prose_lines) - return " ".join(prose_lines), (start_line, end_line) + return "\n".join(prose_lines), (start_line, end_line) def wrap_first_paragraph_as_short_description(content: str) -> tuple[str, bool]: diff --git a/tools/tests/test_ensure_enhancements.py b/tools/tests/test_ensure_enhancements.py index 1e9d3da9091..2c60c9d3bff 100644 --- a/tools/tests/test_ensure_enhancements.py +++ b/tools/tests/test_ensure_enhancements.py @@ -42,6 +42,8 @@ ensure_enhancements_in_file, load_enhance_config, main, + _annotation_line_for_after_title, + _annotation_line_for_meta, ) from rst_utils import get_meta_fields_from_content, inject_metadata_to_content # noqa: E402 @@ -159,6 +161,47 @@ def test_after_title_overlap_uses_paragraph_span(self) -> None: self.assertFalse(can_suggest_after_title_inline(content, {8}, paragraph_span=(4, 4))) +class TestAnnotationLines(unittest.TestCase): + def test_after_title_line_uses_paragraph_when_meta_is_at_top(self) -> None: + content = textwrap.dedent( + """ + .. meta:: + :product: x + + Title + ===== + + Opening paragraph beneath the title. + + More body. + """ + ).lstrip() + self.assertEqual(_annotation_line_for_meta(content), 1) + self.assertEqual(_annotation_line_for_after_title(content), 7) + + def test_result_includes_separate_after_title_line(self) -> None: + config = EnhanceConfig(meta={}, after_title=AFTER_TITLE_RULES) + content = textwrap.dedent( + """ + .. meta:: + :product: x + + Title + ===== + + Opening paragraph beneath the title. + """ + ).lstrip() + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "page.rst" + path.write_text(content, encoding="utf-8") + result = ensure_enhancements_in_file(path, config) + self.assertIsNotNone(result) + assert result is not None + self.assertEqual(result["line"], 1) + self.assertEqual(result["after_title_line"], 7) + + class TestUnresolvedFields(unittest.TestCase): def test_missing_and_blank_count_as_unresolved(self) -> None: rules = { diff --git a/tools/tests/test_rst_utils.py b/tools/tests/test_rst_utils.py index 5ee8eba0157..0ccb737a0fd 100644 --- a/tools/tests/test_rst_utils.py +++ b/tools/tests/test_rst_utils.py @@ -83,6 +83,35 @@ def test_returns_none_when_no_prose(self) -> None: class TestWrapShortDescription(unittest.TestCase): + def test_preserves_one_sentence_per_line(self) -> None: + content = textwrap.dedent( + """ + First steps with ROS - learning path + ==================================== + + ROS (Robot Operating System) is an open-source ecosystem that provides framework, tools, and libraries for building, deploying, running, and maintaining robotic applications. + This page presents a set of articles and hands-on activities to introduce the main concepts behind the ROS framework. + Working through these will give you the essential knowledge needed to start developing applications with ROS. + + More content. + """ + ).lstrip() + paragraph, span = extract_first_paragraph_after_title(content) + self.assertEqual(span, (4, 6)) + self.assertIn("\n", paragraph or "") + updated, changed = wrap_first_paragraph_as_short_description(content) + self.assertTrue(changed) + expected_block = textwrap.dedent( + """ + .. short-description:: + ROS (Robot Operating System) is an open-source ecosystem that provides framework, tools, and libraries for building, deploying, running, and maintaining robotic applications. + This page presents a set of articles and hands-on activities to introduce the main concepts behind the ROS framework. + Working through these will give you the essential knowledge needed to start developing applications with ROS. + """ + ).strip() + self.assertIn(expected_block, updated) + self.assertIn("More content.", updated) + def test_wraps_first_paragraph_after_equals_title(self) -> None: content = textwrap.dedent( """ From 3dc62eadce0c69f601b33f1da8909ebdee561a6a Mon Sep 17 00:00:00 2001 From: Keith Kirkwood Date: Mon, 3 Aug 2026 17:06:34 +0100 Subject: [PATCH 21/23] OPENR-174: Updates to README --- tools/README.md | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tools/README.md b/tools/README.md index 0e426377f40..024ee80c4dd 100644 --- a/tools/README.md +++ b/tools/README.md @@ -61,7 +61,7 @@ after_title: The `:order:` value lists `.. meta::` field names and must match the `meta` section (e.g. `content-type`, not `contentType`). -For `short-description`, the tool wraps the first prose paragraph after the title into the directive (removing it from the body). For `showmeta`, it inserts the directive with the configured `:order:` option when missing. +For `short-description`, the tool wraps the first prose paragraph after the title into the directive (removing it from the body). Each source line in that paragraph is preserved as its own indented body line (one sentence per line is kept when the source uses that layout). For `showmeta`, it inserts the directive with the configured `:order:` option when missing. #### Severity behaviour @@ -294,7 +294,14 @@ When enhancements are missing: - **Warning** severity → `::warning file=...,line=N::Missing meta fields: ...` or `Missing after-title directives: ...` - **Error** severity → `::error file=...,line=N::Missing meta fields: ...` or `Missing after-title directives: ...` -`N` is the start line of an existing `.. meta::` block, the after-title area, or `1` when a new block would be inserted at the top of the file. +Line anchors differ by issue type: + +| Issue | Line `N` | +|-------|----------| +| Missing `.. meta::` fields | Start of the existing `.. meta::` block, or `1` when a new block would be inserted at the top of the file | +| Missing after-title directives | Start of the first prose paragraph after the title (the text being wrapped into `.. short-description::`), or the post-title directive area when no paragraph is available | + +Meta and after-title annotations therefore appear on different lines when a file has `.. meta::` at the top and prose beneath the heading. Annotations describe the pull request **as pushed**, so fields with a configured `value` are listed even when the same run offers them as an inline suggestion. Auto-injected values only exist in the CI working tree; they disappear from the annotations once the suggestion is committed and the workflow re-runs. From f4977aac716d9f8b4701000db4487b9769d75f93 Mon Sep 17 00:00:00 2001 From: Keith Kirkwood Date: Wed, 12 Aug 2026 17:10:55 +0100 Subject: [PATCH 22/23] OPENR-174: Remove suggestions and annotations from enhance --- .github/workflows/enhance.yml | 16 +- Makefile | 2 +- plugins/showmeta.py | 2 +- tools/README.md | 188 ++--- tools/ensure_enhancements.py | 918 ++++-------------------- tools/rst_utils.py | 712 +----------------- tools/supersede_enhancement_reviews.sh | 5 +- tools/tests/test_ensure_enhancements.py | 315 ++------ tools/tests/test_rst_utils.py | 177 +---- 9 files changed, 298 insertions(+), 2037 deletions(-) diff --git a/.github/workflows/enhance.yml b/.github/workflows/enhance.yml index ee15b0e788a..d17c30318b7 100644 --- a/.github/workflows/enhance.yml +++ b/.github/workflows/enhance.yml @@ -70,20 +70,8 @@ jobs: make -f .trusted-base/Makefile supersede-enhancement-reviews \ TOOLS_DIR=.trusted-base/tools - # File-level "Commit suggestion" comments. suggestion_note is intentionally - # unstamped so supersede does not collapse this review; Conversation then - # keeps showing live suggestions until GitHub marks each comment outdated. - - name: Suggest documentation enhancements - if: >- - ${{ !cancelled() - && steps.ensure.outputs.inline_suggestions == 'true' }} - uses: parkerbxyz/suggest-changes@v3 - with: - comment: ${{ steps.ensure.outputs.suggestion_note }} - event: COMMENT - - # Always posts the summary so the Conversation view has a current review - # after the stale ones are minimised, whatever suggest-changes did. + # Posts the summary so the Conversation view has a current review after + # the stale ones are minimised. - name: Post enhancement review comment if: >- ${{ !cancelled() diff --git a/Makefile b/Makefile index c70e126d81b..0cc3db7bbb5 100644 --- a/Makefile +++ b/Makefile @@ -43,7 +43,7 @@ test: doc8 --ignore D001 --ignore-path $(OUT) -- $(SOURCE) test-tools: - $(PYTHON) -m pytest test/ + $(PYTHON) -m pytest test/ tools/tests/ spellcheck: git ls-files '*.md' '*.rst' | xargs codespell --config codespell.cfg diff --git a/plugins/showmeta.py b/plugins/showmeta.py index f844b2db15c..41ad69ca28a 100644 --- a/plugins/showmeta.py +++ b/plugins/showmeta.py @@ -14,7 +14,7 @@ from docutils.parsers.rst import directives from sphinx.util.docutils import SphinxDirective -from meta_util import all_doctree_meta, expand_all_meta_values +from .meta_util import all_doctree_meta, expand_all_meta_values def _macros_flat(app) -> dict[str, str]: diff --git a/tools/README.md b/tools/README.md index 024ee80c4dd..f60ed41261d 100644 --- a/tools/README.md +++ b/tools/README.md @@ -13,15 +13,15 @@ Information for documentation contributors creating or updating `.rst` files. | Component | Used by | |-----------|---------| | [PyYAML](https://pyyaml.org/) (`pip install pyyaml`) | [`ensure_enhancements.py`](ensure_enhancements.py) and unit tests in [`tests/`](tests/) | -| Git (with a usable `HEAD` and refs for your base commit) | PR-scope discovery and diff overlap checks (`--diff-base`) | +| Git (with a usable `HEAD` and refs for your base commit) | PR-scope discovery (`--diff-base`) | | [GitHub CLI](https://cli.github.com/) (`gh`) and `jq` | [`supersede_enhancement_reviews.sh`](supersede_enhancement_reviews.sh) only (Enhance workflow on `ubuntu-24.04`; optional for local supersede testing) | ### Enhancement configuration [`enhance.yaml`](enhance.yaml) defines documentation enhancement rules. The `meta` section lists every `.. meta::` field checked by the tooling. Each entry has: -- **`severity`**: `warning` (advisory; soft-fails the ensure step in CI) or `error` (fails the workflow after reviews are posted). -- **`value`**: default text to inject when the field is missing or blank. Leave empty when the contributor must supply a non-empty value. +- **`severity`**: `warning` (advisory; soft-fails the ensure step in CI) or `error` (fails the workflow after the review is posted). +- **`value`**: suggested default text when the field is missing or blank. Leave empty when the contributor must supply a non-empty value. ```yaml meta: @@ -44,7 +44,7 @@ meta: `{PRODUCT}` and `{DISTRO}` are Sphinx substitution macros expanded at build time from [`conf.py`](../conf.py). -The `after_title` section maps directive names to rules, in the order they are inserted after the first document title (same shape as `meta`): +The `after_title` section maps directive names to rules, in the order they should appear after the first document title: ```yaml after_title: @@ -61,18 +61,18 @@ after_title: The `:order:` value lists `.. meta::` field names and must match the `meta` section (e.g. `content-type`, not `contentType`). -For `short-description`, the tool wraps the first prose paragraph after the title into the directive (removing it from the body). Each source line in that paragraph is preserved as its own indented body line (one sentence per line is kept when the source uses that layout). For `showmeta`, it inserts the directive with the configured `:order:` option when missing. +For `short-description`, the contributor should wrap the first prose paragraph after the title into the directive. For `showmeta`, the contributor should add the directive with the configured `:order:` option when missing. #### Severity behaviour | Severity | Missing or blank field | CI ensure step | Workflow job | |----------|------------------------|----------------|--------------| -| `warning` | Annotation + review | Soft warning (`continue-on-error`) | Succeeds | -| `error` | Error annotation + review | Soft warning (same step) | **Fails** on final enforce step | +| `warning` | Listed in review | Soft warning (`continue-on-error`) | Succeeds | +| `error` | Listed in review | Soft warning (same step) | **Fails** on final enforce step | ### Checking enhancements locally -[`ensure_enhancements.py`](ensure_enhancements.py) checks `.rst` files against [`enhance.yaml`](enhance.yaml). Fields with a configured `value` are added or filled automatically when the edit can be suggested or applied locally. Fields with an empty `value` must be completed manually in the `.. meta::` block. After-title directives are added using the rules above. +[`ensure_enhancements.py`](ensure_enhancements.py) checks `.rst` files against [`enhance.yaml`](enhance.yaml). It reports missing meta fields and after-title directives; it does not modify files. #### Usage @@ -88,95 +88,38 @@ Multiple files: python3 tools/ensure_enhancements.py source/Topic/A.rst source/Topic/B.rst ``` -Pull request scope (discovers changed `ACMR` `*.rst` files via `git diff`; requires Makefile variables `DIFF_BASE` and `STATUS_FILE`): - -```bash -make ensure-enhancements DIFF_BASE=origin/rolling STATUS_FILE=/tmp/enhance-out.txt -``` - -PR scope without Make (optional `--status-file`; exit codes follow local rules when omitted): +Pull request scope (discovers changed `ACMR` `*.rst` files via `git diff`): ```bash python3 tools/ensure_enhancements.py --diff-base origin/rolling -python3 tools/ensure_enhancements.py --diff-base "$(git merge-base HEAD origin/rolling)" --status-file /tmp/out.txt ``` -For day-to-day editing of known files, pass paths explicitly and omit `--status-file` so the tool exits `1` only when **error**-severity issues remain. - -#### Exit codes - -With `--status-file` (CI), exit `1` when any issues remain. Locally, exit `1` only when **error**-severity issues are still unresolved; warning-only issues exit `0` after applying automatic fixes. - -#### Example (configured values) +For day-to-day editing, pass paths explicitly so the tool exits `1` only when **error**-severity issues remain. -Before: +To simulate the CI ensure step locally (writes status outputs and uses CI exit codes): -```rst -My Article -========== - -Some content. +```bash +make ensure-enhancements DIFF_BASE=origin/rolling STATUS_FILE=/tmp/enhance-out.txt ``` -After a local run (new `.. meta::` added at the top of the file): - -```rst -.. meta:: - :product: {PRODUCT} - :distribution: {DISTRO} - -My Article -========== - -Some content. -``` +#### Exit codes -Fields such as `area` with an empty `value` in the config are listed in the review for manual completion; they are not given placeholder text. +With `--status-file` (CI), exit `1` when any issues remain. Locally, exit `1` only when **error**-severity issues are still unresolved; warning-only issues exit `0`. ### Contributor CI experience When you open or update a pull request, CI automatically checks enhancements on all modified `.rst` files. -#### Contributor experience overview - -| Situation | Ensure step | Annotations | Pull request review | Job result | -|-----------|-------------|-------------|---------------------|------------| -| All enhancements resolved | Green | None | None (stale bot reviews cleared) | Success | -| Warning-only gaps | Soft warning | Warnings | Suggestions and/or manual list | Success | -| Error gaps (e.g. missing `area`) | Soft warning | Errors (and warnings) | Suggestions and/or manual list | **Failure** after enforce step | -| Auto-fix in diff | Soft warning | As above | Inline “Commit suggestion” | As per severity | -| Auto-fix outside diff (`snippet`) | Soft warning | As above | Copy-paste blocks for configured values | As per severity | -| Manual fields only (`manual_fields`) | Soft warning | As above | Field/directive list with required/warning labels | As per severity | +| Situation | Ensure step | Pull request review | Job result | +|-----------|-------------|---------------------|------------| +| All enhancements resolved | Green | None (stale bot reviews cleared) | Success | +| Warning-only gaps | Soft warning | Summary review comment | Success | +| Error gaps (e.g. missing `area`) | Soft warning | Summary review comment | **Failure** after enforce step | +| No changed `.rst` files in the PR | No check; `enhancements_checked=false` | Supersede/review steps skipped | Success | -Reviews, annotations, and the soft-failed ensure step appear in different parts of the GitHub UI (Conversation, Files changed, Checks); only error-severity issues fail the overall workflow. +The **Documentation enhancements** review comment (`## Documentation enhancements`) lists every affected file and what is missing. Each file section names missing `.. meta::` fields (with required/optional labels and suggested values where configured) and missing after-title directives (with brief guidance). -#### Review body sections - -The **Documentation enhancements** summary review (`## Documentation enhancements`) names every affected file and splits the work by how it is fixed. The **Inline documentation suggestions** review (`## Inline documentation suggestions`) is a short pointer to the commit suggestions; details stay in the summary. - -| Heading | Files listed | What is listed | -|---------|--------------|----------------| -| `### Commit inline suggestions` | `suggestable` mode | Configured values and directives added for you — commit the suggestion | -| `### Copy-paste \`.. meta::\` blocks` | `snippet` mode (meta) | Configured meta values as an RST block to paste yourself | -| `### Copy-paste after-title directives` | `snippet` mode (after-title) | `.. short-description::` / `.. showmeta::` blocks to paste after the title | -| `### Provide non-empty values` | Any file with manual meta fields | Fields with an empty `value`, labelled required or optional | -| `### Add after-title directives` | Manual after-title gaps | Directives that could not be added automatically | - -A file can appear in multiple sections: the suggestion covers auto-filled items while the manual lists cover the rest. - -#### When inline suggestions appear - -GitHub only allows review suggestions on [lines already in the pull request diff](https://github.com/marketplace/actions/suggest-changes-action). The script compares each automatic edit to that diff: - -| Situation | What happens | -|-----------|----------------| -| Missing configured values; existing `.. meta::` overlaps the PR diff | Write append/fill to the working tree → inline suggestion via [`suggest-changes`](https://github.com/marketplace/actions/suggest-changes-action) | -| No `.. meta::`; top of file overlaps the PR diff | Insert at top → inline suggestion | -| After-title edit overlaps the PR diff (title area or paragraph being wrapped) | Insert/wrap in working tree → inline suggestion | -| Automatic edit does **not** overlap the PR diff (`snippet` mode) | No inline write; review includes a copy-paste block | -| Only manual fields (empty `value` in config, `manual_fields` mode) | Review lists fields; no placeholder injection | -| All configured enhancements present | No action | -| No changed `.rst` files in the PR | No check; `enhancements_checked=false`; supersede/review steps skipped | +When you push new commits, the workflow minimises the previous summary review as outdated and posts a fresh one reflecting the current state. --- @@ -188,9 +131,9 @@ Information for maintainers and developers working on or extending the enhanceme | File | Purpose | |------|---------| -| [`rst_utils.py`](rst_utils.py) | Regex-based read/write of `.. meta::`, `.. short-description::`, and `.. showmeta::` directives | +| [`rst_utils.py`](rst_utils.py) | Read-only detection of `.. meta::`, `.. short-description::`, and `.. showmeta::` directives | | [`enhance.yaml`](enhance.yaml) | Enhancement rules (`meta` fields and `after_title` directives) | -| [`ensure_enhancements.py`](ensure_enhancements.py) | CLI that checks and applies enhancements from the config | +| [`ensure_enhancements.py`](ensure_enhancements.py) | CLI that checks enhancements from the config and builds the review comment | | [`supersede_enhancement_reviews.sh`](supersede_enhancement_reviews.sh) | Minimise outdated bot PR reviews | | [`tests/`](tests/) | Unit tests for the tools in this directory | @@ -198,24 +141,22 @@ Information for maintainers and developers working on or extending the enhanceme #### `rst_utils.py` -Low-level utilities for locating and editing Sphinx directives in RST source: +Read-only utilities for locating Sphinx directives in RST source: - **`get_meta_fields_from_content`** — field names and values in the first `.. meta::` block -- **`get_meta_names_from_content`** — field names only -- **`inject_metadata_to_content`** — add missing fields or fill blank values; never overwrites non-empty contributor values -- **`wrap_first_paragraph_as_short_description`** — wrap the first prose paragraph after the title -- **`inject_showmeta_to_content`** — insert or fill `.. showmeta::` with configured options +- **`has_short_description_content`** — whether a non-empty `.. short-description::` body exists +- **`has_showmeta_with_order`** — whether `.. showmeta::` exists with a non-empty `:order:` option #### Extending configuration -Add a new key under `meta` in [`enhance.yaml`](enhance.yaml) to extend metadata coverage without changing Python code. `_parse_meta_rules` accepts any key with `severity` and `value`, and the rest of the pipeline (unresolved-field detection, auto-injection, annotations, review sections) is driven entirely by that mapping. +Add a new key under `meta` in [`enhance.yaml`](enhance.yaml) to extend metadata coverage without changing Python code. `_parse_meta_rules` accepts any key with `severity` and `value`, and the rest of the pipeline (unresolved-field detection and review hints) is driven entirely by that mapping. `after_title` is only partly config-driven. Its mapping shape lets you reorder or retune the two existing directives (`short-description`, `showmeta`) from YAML alone — but `_parse_after_title_rules` whitelists `supported_directives = {"short-description", "showmeta"}`, so adding a *new* after-title directive needs Python changes: 1. Add the name to `supported_directives` and its directive-specific validation in `_parse_after_title_rules`. 2. Extend `_after_title_rule_satisfied` with an "already present" check for the new directive. -3. Extend `_process_after_title_rules` to inject content or produce a copy-paste snippet for it. -4. Add matching read/write/format helpers to [`rst_utils.py`](rst_utils.py) — see `has_showmeta_with_order`, `inject_showmeta_to_content`, and `format_showmeta_block` for the `showmeta` example. +3. Extend `_after_title_hint` with review text for the new directive. +4. Add matching read-only detection helpers to [`rst_utils.py`](rst_utils.py) — see `has_showmeta_with_order` for the `showmeta` example. ### CLI options & Makefile targets @@ -223,13 +164,13 @@ Add a new key under `meta` in [`enhance.yaml`](enhance.yaml) to extend metadata - `paths` — optional; when omitted, `--diff-base` is required and changed `.rst` files are discovered automatically - `--config PATH` — YAML config file (default: `tools/enhance.yaml`) -- `--diff-base SHA` — PR base commit; limits on-disk writes to lines in the PR diff (inline suggestions); files that need a copy-paste or manual field list use a review comment instead -- `--status-file PATH` — write `enhancements_checked`, `inline_suggestions`, `has_results`, `has_errors`, the review comment body, and the suggestion note for CI; when issues remain, emits annotations and exits `1` (the ensure step uses `continue-on-error`) +- `--diff-base SHA` — PR base commit; discovers changed `.rst` files for the check +- `--status-file PATH` — write `enhancements_checked`, `has_results`, `has_errors`, and the review comment body for CI; when issues remain, exits `1` (the ensure step uses `continue-on-error`) - `-v` / `--verbose` — enable debug logging -#### Makefile targets (enhancement CI) +#### Makefile targets -Both targets live in the repository root [`Makefile`](../Makefile). CI invokes them with `make -f .trusted-base/Makefile …` so recipes run from the PR **base** branch, not the PR head. +Enhancement Make targets live in the repository root [`Makefile`](../Makefile). CI invokes them with `make -f .trusted-base/Makefile …` so those recipes run from the PR **base** branch, not the PR head. | Target | Required variables | Purpose | |--------|-------------------|---------| @@ -240,92 +181,59 @@ Environment for `supersede-enhancement-reviews` (set by the workflow or locally) ### Continuous integration architecture -The workflow [`.github/workflows/enhance.yml`](../.github/workflows/enhance.yml) runs on **`pull_request_target`** when a pull request is **opened**, **synchronised**, or **reopened** (including from forks). That event type allows the default `GITHUB_TOKEN` to post review suggestions on fork PRs; the workflow file on the repository **default branch** defines the job, while trusted tooling comes from the PR **base** branch (see [Security](#security) below). +The workflow [`.github/workflows/enhance.yml`](../.github/workflows/enhance.yml) runs on **`pull_request_target`** when a pull request is **opened**, **synchronised**, or **reopened** (including from forks). That event type allows the default `GITHUB_TOKEN` to post review comments on fork PRs; the workflow file on the repository **default branch** defines the job, while trusted tooling comes from the PR **base** branch (see [Security](#security) below). The Enhance workflow installs PyYAML in the job; it does not install the full documentation `requirements.txt` for enhancement checks. #### Job flow (`ensure-enhancements`) -1. Check out the PR **head** (`.rst` content to inspect and, where allowed, modify for suggestions). +1. Check out the PR **head** (`.rst` content to inspect). 2. Check out the PR **base** into `.trusted-base/` (Makefile, `ensure_enhancements.py`, `enhance.yaml`, `supersede_enhancement_reviews.sh`). 3. Install Python 3.12 and PyYAML. -4. **Ensure documentation enhancements** — `git fetch` the base SHA, then `make -f .trusted-base/Makefile ensure-enhancements` with `TOOLS_DIR=.trusted-base/tools`, `DIFF_BASE`, and `STATUS_FILE=$GITHUB_OUTPUT`. Emits per-file annotations; the step uses `continue-on-error: true` so warning-only gaps do not fail the job immediately. +4. **Ensure documentation enhancements** — `git fetch` the base SHA, then `make -f .trusted-base/Makefile ensure-enhancements` with `TOOLS_DIR=.trusted-base/tools`, `DIFF_BASE`, and `STATUS_FILE=$GITHUB_OUTPUT`. The step uses `continue-on-error: true` so warning-only gaps do not fail the job immediately. 5. **Verify enhancement check ran** — fail the job if `enhancements_checked` is empty. The ensure step soft-fails by design, so a missing output is the only way to tell a crashed check from a clean run. 6. **Supersede stale enhancement reviews** (only if `enhancements_checked=true`) — `make -f .trusted-base/Makefile supersede-enhancement-reviews`, which minimises stamped reviews and writes nothing back. -7. **Suggest documentation enhancements** — if `inline_suggestions`, run [`suggest-changes`](https://github.com/marketplace/actions/suggest-changes-action) for file-level “Commit suggestion” comments, using `suggestion_note` as its review body (unstamped so supersede does not hide live suggestions in Conversation). -8. **Post enhancement review comment** — if `has_results`, post the stamped `comment` body via `gh pr review` (Conversation view). Independent of suggest-changes, which posts nothing when every suggestion duplicates one from an earlier run. -9. **Enforce required enhancements** — if `has_errors`, fail the job (runs `always()` so error gaps fail even when the ensure step soft-failed). - -Steps 7 and 8 use `!cancelled()` rather than depending on the supersede step, so a transient GitHub API failure while minimising old reviews cannot stop contributors receiving feedback. - -Priority: **inline suggestions wherever GitHub allows them**. Copy-paste blocks and manual field lists are delivered via a pull request review comment when inline suggestions are not used for that run. - -#### Per-file modes and CI outputs +7. **Post enhancement review comment** — if `has_results`, post the stamped `comment` body via `gh pr review` (Conversation view). +8. **Enforce required enhancements** — if `has_errors`, fail the job (runs `always()` so error gaps fail even when the ensure step soft-failed). -Each changed `.rst` file is classified with an internal **mode**: +Step 7 uses `!cancelled()` rather than depending on the supersede step, so a transient GitHub API failure while minimising old reviews cannot stop contributors receiving feedback. -| Mode | Meaning | -|------|---------| -| `suggestable` | Configured enhancements were written to the working tree for inline “Commit suggestion” | -| `snippet` | Enhancements could not be written inline; the review includes copy-paste RST blocks | -| `manual_fields` | Only manual items remain (empty meta `value`, or after-title gaps with no auto-fix) | - -A single file can still list **manual** fields in the review when its mode is `suggestable` or `snippet` (auto-filled items were handled; manual items remain for the contributor). +#### CI outputs The script writes **CI outputs** (for example `$GITHUB_OUTPUT`) that describe which workflow steps to run: | Output | Meaning | |--------|---------| | `enhancements_checked` | At least one changed `.rst` was in scope (`false` when discovery finds no changed RST; supersede is skipped). Empty only when the check never ran | -| `inline_suggestions` | Run [`suggest-changes`](https://github.com/marketplace/actions/suggest-changes-action) for file-level suggestions | | `has_results` | Enhancement issues remain; post the Conversation review | | `has_errors` | Unresolved **error**-severity issues (triggers the final enforce step) | | `comment` | Full stamped review body (multiline heredoc) posted to Conversation via `gh pr review`; written only when `has_results` | -| `suggestion_note` | Short unstamped body for the suggest-changes review; written only when `inline_suggestions` | -Only the summary `comment` carries the hidden marker, so supersede replaces that review each run while suggestion-carrying reviews stay expanded. The outputs are self-consistent by construction: `comment` exists whenever `has_results` is true, `suggestion_note` exists whenever `inline_suggestions` is true, and `has_errors` implies `has_results`. No step can therefore run with an empty review body, and the enforce step cannot fail the job without a review having been posted. +The outputs are self-consistent by construction: `comment` exists whenever `has_results` is true, and `has_errors` implies `has_results`. The enforce step cannot fail the job without a review having been posted. The ensure step uses `continue-on-error: true`, so warning-only gaps do not fail the job. Error-severity gaps (for example `area`) still fail the workflow on the enforce step after contributors receive review feedback. -#### Annotations - -When enhancements are missing: - -- **Warning** severity → `::warning file=...,line=N::Missing meta fields: ...` or `Missing after-title directives: ...` -- **Error** severity → `::error file=...,line=N::Missing meta fields: ...` or `Missing after-title directives: ...` - -Line anchors differ by issue type: - -| Issue | Line `N` | -|-------|----------| -| Missing `.. meta::` fields | Start of the existing `.. meta::` block, or `1` when a new block would be inserted at the top of the file | -| Missing after-title directives | Start of the first prose paragraph after the title (the text being wrapped into `.. short-description::`), or the post-title directive area when no paragraph is available | - -Meta and after-title annotations therefore appear on different lines when a file has `.. meta::` at the top and prose beneath the heading. - -Annotations describe the pull request **as pushed**, so fields with a configured `value` are listed even when the same run offers them as an inline suggestion. Auto-injected values only exist in the CI working tree; they disappear from the annotations once the suggestion is committed and the workflow re-runs. - #### Superseding outdated reviews -Only the **summary** review body includes a hidden HTML marker (``). The marker id `ros2-doc-enhance-ensure` (constant `REVIEW_MARKER_ID` in Python; override in the shell script with `ENHANCEMENT_REVIEW_MARKER_ID`) is what the supersede script searches for in review bodies. The suggest-changes review uses an unstamped `suggestion_note` so it is not minimised: that card is the only place inline suggestions render in Conversation, and GitHub marks individual suggestion comments outdated once they are committed or their anchor leaves the diff. +The summary review body includes a hidden HTML marker (``). The marker id `ros2-doc-enhance-ensure` (constant `REVIEW_MARKER_ID` in Python; override in the shell script with `ENHANCEMENT_REVIEW_MARKER_ID`) is what the supersede script searches for in review bodies. When `enhancements_checked=true`, the workflow runs `make -f .trusted-base/Makefile supersede-enhancement-reviews` ([`supersede_enhancement_reviews.sh`](supersede_enhancement_reviews.sh)): 1. Lists pull request reviews whose body contains the marker id. 2. Minimises each as **Outdated** via the GitHub GraphQL API (`gh api graphql`; individual failures are ignored). -When all issues are fixed (`has_results=false`), stale summary reviews are minimised and no new “all clear” comment is posted. Inline suggestion comments remain on the pull request until they are actioned; suggest-changes skips re-posting duplicates, so earlier suggestion reviews stay the Conversation surface for pending commits. +When all issues are fixed (`has_results=false`), stale summary reviews are minimised and no new "all clear" comment is posted. #### Security -The workflow uses [`pull_request_target`](https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request_target) so the default `GITHUB_TOKEN` can post review suggestions on fork PRs. **Do not** run `make` or scripts from the PR head checkout in that job; always use `-f .trusted-base/Makefile` and `TOOLS_DIR=.trusted-base/tools`. See [Mitigating the risks of untrusted code checkout](https://docs.github.com/en/actions/reference/security/secure-use#mitigating-the-risks-of-untrusted-code-checkout). +The workflow uses [`pull_request_target`](https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request_target) so the default `GITHUB_TOKEN` can post review comments on fork PRs. **Do not** run `make` or scripts from the PR head checkout in that job; always use `-f .trusted-base/Makefile` and `TOOLS_DIR=.trusted-base/tools`. See [Mitigating the risks of untrusted code checkout](https://docs.github.com/en/actions/reference/security/secure-use#mitigating-the-risks-of-untrusted-code-checkout). ### Tests Unit tests for this directory live in [`tests/`](tests/). From the repository root (with PyYAML installed): ```bash -python3 -m unittest discover -s tools/tests -p 'test_*.py' +python3 -m pytest tools/tests/ ``` -The main documentation CI job [`test-tools`](../Makefile) runs `pytest` on the top-level [`test/`](../test/) tree; that is separate from `tools/tests`. +The [`test-tools`](../Makefile) target (run in [`.github/workflows/test.yml`](../.github/workflows/test.yml)) runs `pytest` on the top-level [`test/`](../test/) tree and `tools/tests/`. diff --git a/tools/ensure_enhancements.py b/tools/ensure_enhancements.py index 5dea34510cf..2af59f5681f 100644 --- a/tools/ensure_enhancements.py +++ b/tools/ensure_enhancements.py @@ -6,20 +6,15 @@ for ``.. meta::`` fields and post-heading directives such as ``.. short-description::`` and ``.. showmeta::``. -When ``--diff-base`` is set, edits are only written to disk when they overlap -the pull request diff (so GitHub can offer inline suggestions). Otherwise the -review comment carries copy-paste or manual instructions. - -In CI (``--status-file``), emits GitHub Actions annotations per severity and -exits with code ``1`` when issues remain so the ensure step can soft-fail. -Error-severity issues also set ``has_errors`` for a final workflow gate. +In CI (``--status-file``), writes GitHub Actions outputs and exits with code +``1`` when issues remain so the ensure step can soft-fail. Error-severity +issues also set ``has_errors`` for a final workflow gate. """ from __future__ import annotations import argparse import logging -import re import subprocess import sys from dataclasses import dataclass @@ -31,54 +26,26 @@ # Allow ``python3 tools/ensure_enhancements.py`` from the repository root. _TOOLS_DIR = Path(__file__).resolve().parent if str(_TOOLS_DIR) not in sys.path: - sys.path.insert(0, str(_TOOLS_DIR)) - -from rst_utils import ( - after_title_directives_line_span, - extract_first_paragraph_after_title, - format_showmeta_block, - get_meta_fields_from_content, - has_meta_block, - has_short_description_content, - has_showmeta_with_order, - inject_metadata_to_content, - inject_showmeta_to_content, - meta_block_line_span, - wrap_first_paragraph_as_short_description, + sys.path.insert(0, str(_TOOLS_DIR)) + +from rst_utils import ( # noqa: E402 + get_meta_fields_from_content, + has_short_description_content, + has_showmeta_with_order, ) logger = logging.getLogger(__name__) DEFAULT_CONFIG_PATH = _TOOLS_DIR / "enhance.yaml" RST_EXTENSION = ".rst" -_HUNK_HEADER = re.compile(r"^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@") Severity = Literal["warning", "error"] # Hidden marker in review bodies so CI can find and supersede prior bot reviews. REVIEW_MARKER_ID = "ros2-doc-enhance-ensure" REVIEW_MARKER = f"" -# Titles and section headings for pull request review bodies (GitHub Markdown). +# Title for pull request review bodies (GitHub Markdown). SUMMARY_REVIEW_TITLE = "## Documentation enhancements" -SUGGESTION_REVIEW_TITLE = "## Inline documentation suggestions" -SECTION_INLINE_SUGGESTIONS = "### Commit inline suggestions" -SECTION_COPY_PASTE_BLOCKS = "### Copy-paste `.. meta::` blocks" -SECTION_COPY_PASTE_AFTER_TITLE = "### Copy-paste after-title directives" -SECTION_NON_EMPTY_VALUES = "### Provide non-empty values" -SECTION_AFTER_TITLE_MANUAL = "### Add after-title directives" - -# Short body for the suggest-changes review. Deliberately unstamped: that review is -# the only Conversation surface that shows live inline suggestions, so it must not be -# minimised with the summary. GitHub marks individual comments outdated when actioned. -SUGGESTION_NOTE = ( - f"{SUGGESTION_REVIEW_TITLE}\n" - "\n" - "Each **Commit suggestion** below applies configured documentation " - "enhancements from `tools/enhance.yaml`.\n" - "\n" - "For copy-paste blocks, required fields, and the full per-file breakdown, " - "see the **Documentation enhancements** review comment." -) @dataclass(frozen=True) @@ -89,10 +56,10 @@ class AfterTitleRule: The directive name is the key in the ``after_title`` mapping (like ``meta``). Attributes: - severity: Advisory ``warning`` or blocking ``error`` in CI. - content: Content source for body directives (e.g. ``first_paragraph``). - options: Option name/value pairs for option-only directives. - required_options: Option names that must be non-empty when present. + severity: Advisory ``warning`` or blocking ``error`` in CI. + content: Content source for body directives (e.g. ``first_paragraph``). + options: Option name/value pairs for option-only directives. + required_options: Option names that must be non-empty when present. """ severity: Severity @@ -115,9 +82,9 @@ class MetaRule: A single metadata field rule from ``enhance.yaml``. Attributes: - severity: Advisory ``warning`` or blocking ``error`` in CI. - value: Default text to inject when missing or blank; empty when the - contributor must supply a non-empty value. + severity: Advisory ``warning`` or blocking ``error`` in CI. + value: Default text to inject when missing or blank; empty when the + contributor must supply a non-empty value. """ severity: Severity @@ -129,7 +96,7 @@ def has_configured_value(self) -> bool: Return whether the rule supplies a non-empty default value. Returns: - ``True`` when ``value`` is non-empty after stripping whitespace. + ``True`` when ``value`` is non-empty after stripping whitespace. """ return bool(self.value.strip()) @@ -301,13 +268,13 @@ def load_enhance_config(config_path: Path) -> EnhanceConfig: Load and validate enhancement rules from a YAML config file. Args: - config_path: Path to the YAML configuration file. + config_path: Path to the YAML configuration file. Returns: - Parsed enhancement configuration. + Parsed enhancement configuration. Raises: - SystemExit: If the file is missing, invalid, or has unusable rules. + SystemExit: If the file is missing, invalid, or has unusable rules. """ if not config_path.is_file(): logger.error("Config file not found: %s", config_path) @@ -334,37 +301,16 @@ def load_enhance_config(config_path: Path) -> EnhanceConfig: ) -def format_meta_block(rules: dict[str, MetaRule], fields: list[str]) -> str: - """ - Build an RST ``.. meta::`` block for auto-injectable fields. - - Args: - rules: Configured metadata rules keyed by field name. - fields: Field names to include in the block, in output order. - - Returns: - A formatted ``.. meta::`` directive ending with a blank line. - - Raises: - KeyError: If a requested field is absent from ``rules``. - """ - lines = [".. meta::"] - for field in fields: - lines.append(f" :{field}: {rules[field].value}") - lines.append("") - return "\n".join(lines) - - def _unresolved_fields(content: str, rules: dict[str, MetaRule]) -> list[str]: """ Find configured meta fields that are absent or blank in RST content. Args: - content: RST source to inspect. - rules: Configured metadata rules. + content: RST source to inspect. + rules: Configured metadata rules. Returns: - Unresolved field names in configuration order. + Unresolved field names in configuration order. """ present = get_meta_fields_from_content(content) return [ @@ -373,89 +319,16 @@ def _unresolved_fields(content: str, rules: dict[str, MetaRule]) -> list[str]: ] -def parse_diff_new_side_lines(diff_text: str) -> set[int]: - """ - Parse a unified diff and return 1-based line numbers on the new (``+``) side. - - Both added and context lines within hunks are included so nearby suggestion - anchors count as overlapping the pull request diff. - - File headers (``---`` / ``+++``) are only skipped outside hunks. Inside a - hunk those prefixes are ordinary ``-`` / ``+`` lines (e.g. RST table rows of - ``+`` characters), and must advance ``new_line`` accordingly. - - Args: - diff_text: Unified diff text to parse. - - Returns: - One-based line numbers represented on the new side of diff hunks. - """ - lines: set[int] = set() - new_line = 0 - in_hunk = False - for line in diff_text.splitlines(): - match = _HUNK_HEADER.match(line) - if match: - in_hunk = True - new_line = int(match.group(1)) - continue - if not in_hunk and (line.startswith("---") or line.startswith("+++")): - continue - if line.startswith("\\"): - continue - if line.startswith("+"): - lines.add(new_line) - new_line += 1 - elif line.startswith("-"): - continue - elif line.startswith(" ") or line == "": - if new_line > 0: - lines.add(new_line) - new_line += 1 - elif line.startswith("diff "): - in_hunk = False - new_line = 0 - return lines - - -def pr_diff_lines_for_file(diff_base: str, path: Path) -> set[int]: - """ - Find pull-request diff lines for a file on the head side. - - Args: - diff_base: Base commit SHA used for the three-dot comparison. - path: Repository-relative path to inspect. - - Returns: - One-based new-side line numbers, or an empty set if ``git diff`` fails. - """ - result = subprocess.run( - ["git", "diff", "-U3", f"{diff_base}...HEAD", "--", str(path)], - check=False, - capture_output=True, - text=True, - ) - if result.returncode not in (0, 1): - logger.warning( - "git diff failed for %s (exit %s): %s", - path, - result.returncode, - result.stderr.strip(), - ) - return set() - return parse_diff_new_side_lines(result.stdout) - - def changed_rst_paths(diff_base: str) -> list[Path]: """ List ``.rst`` files changed between a pull request base and ``HEAD``. Args: - diff_base: Base commit SHA used for the three-dot comparison. + diff_base: Base commit SHA used for the three-dot comparison. Returns: - Repository-relative paths for added, copied, modified, or renamed - ``.rst`` files, or an empty list if ``git diff`` fails. + Repository-relative paths for added, copied, modified, or renamed + ``.rst`` files, or an empty list if ``git diff`` fails. """ result = subprocess.run( [ @@ -486,53 +359,17 @@ def changed_rst_paths(diff_base: str) -> list[Path]: return paths -def _log_working_tree_summary(paths: list[Path]) -> None: - """ - Log ``git status`` and ``git diff`` for processed RST paths. - - Args: - paths: Repository-relative RST files that were checked or updated. - - Returns: - None. - """ - if not paths: - return - path_args = [str(p) for p in paths] - logger.info("Working tree after ensure_enhancements:") - status = subprocess.run( - ["git", "status", "--short", "--", *path_args], - check=False, - capture_output=True, - text=True, - ) - if status.stdout.strip(): - for line in status.stdout.splitlines(): - logger.info("%s", line) - else: - logger.info("(no changes)") - diff = subprocess.run( - ["git", "diff", "--", *path_args], - check=False, - capture_output=True, - text=True, - ) - if diff.stdout.strip(): - for line in diff.stdout.splitlines(): - logger.info("%s", line) - - def _write_multiline_output(handle: TextIO, key: str, value: str) -> None: """ Write a multiline GitHub Actions output using heredoc syntax. Args: - handle: Open text handle for the status file. - key: Output name. - value: Output value, which may span several lines. + handle: Open text handle for the status file. + key: Output name. + value: Output value, which may span several lines. Returns: - None. + None. """ delimiter = f"EOF_{key.upper()}" handle.write(f"{key}<<{delimiter}\n") @@ -547,219 +384,42 @@ def _write_ci_status_file( *, enhancements_checked: bool, results: list[dict[str, object]], - rules: dict[str, MetaRule], + config: EnhanceConfig, has_errors: bool, ) -> None: """ Append GitHub Actions output flags and optional review comment. - Writes ``enhancements_checked``, ``inline_suggestions``, ``has_results``, and - ``has_errors``, plus a multiline ``comment`` block when ``results`` is - non-empty and ``suggestion_note`` when inline suggestions were written. + Writes ``enhancements_checked``, ``has_results``, and ``has_errors``, plus + a multiline ``comment`` block when ``results`` is non-empty. Args: - status_file: Path to append to (for example ``$GITHUB_OUTPUT``). - enhancements_checked: Whether changed RST files were in scope for this run. - results: Per-file result dicts from ``ensure_enhancements_in_file``. - rules: Configured metadata rules for building the review body. - has_errors: Whether any result has unresolved error-severity fields. + status_file: Path to append to (for example ``$GITHUB_OUTPUT``). + enhancements_checked: Whether changed RST files were in scope for this run. + results: Per-file result dicts from ``ensure_enhancements_in_file``. + config: Enhancement configuration used to build the review comment. + has_errors: Whether any result has unresolved error-severity fields. Returns: - None. + None. """ - has_inline_suggestions = any(r["mode"] == "suggestable" for r in results) has_results = bool(results) with status_file.open("a", encoding="utf-8") as f: for key, flag in ( ("enhancements_checked", enhancements_checked), - ("inline_suggestions", has_inline_suggestions), ("has_results", has_results), ("has_errors", has_errors), ): f.write(f"{key}={'true' if flag else 'false'}\n") if results: - _write_multiline_output(f, "comment", build_review_comment(results, rules)) - if has_inline_suggestions: - _write_multiline_output(f, "suggestion_note", SUGGESTION_NOTE) - - -def _span_overlaps(span: tuple[int, int] | None, pr_lines: set[int]) -> bool: - """ - Test whether an inclusive line span overlaps pull-request lines. - - Args: - span: Inclusive one-based start and end lines, or ``None``. - pr_lines: One-based line numbers represented by the pull-request diff. - - Returns: - ``True`` when at least one line overlaps; otherwise ``False``. - """ - if span is None or not pr_lines: - return False - start, end = span - return any(line in pr_lines for line in range(start, end + 1)) - - -def _annotation_line_for_meta(content: str) -> int: - """ - Select a line for a GitHub annotation on ``.. meta::`` issues. - - Returns: - The first line of an existing meta block, or line 1 if none exists. - """ - span = meta_block_line_span(content) - if span is not None: - return span[0] - return 1 - - -def _annotation_line_for_after_title(content: str) -> int: - """ - Select a line for a GitHub annotation on after-title directive issues. - - Prefers the first prose paragraph after the title when present, otherwise - the post-title directive area or the line after the document title. - - Returns: - A 1-based source line appropriate for after-title annotations. - """ - _paragraph, span = extract_first_paragraph_after_title(content) - if span is not None: - return span[0] - after_title = after_title_directives_line_span(content) - if after_title is not None: - return after_title[0] - return 1 - - -def _emit_after_title_annotation( - level: str, - path: str, - directives: list[str], - line: int, -) -> None: - """Print a GitHub Actions annotation for missing after-title directives.""" - if not directives: - return - directive_list = ", ".join(directives) - message = _escape_workflow_command_message( - f"Missing after-title directives: {directive_list}", - ) - print(f"::{level} file={path},line={line}::{message}") - - -def _escape_workflow_command_message(message: str) -> str: - """ - Escape a message for use in a GitHub Actions workflow command. - - Args: - message: Unescaped annotation message. - - Returns: - The message with workflow-command control characters escaped. - """ - return message.replace("%", "%25").replace("\r", "%0D").replace("\n", "%0A") - - -def _emit_annotation(level: str, path: str, fields: list[str], line: int) -> None: - """ - Print a GitHub Actions workflow annotation for missing meta fields. - - Args: - level: Annotation level, typically ``warning`` or ``error``. - path: Repository-relative path to annotate. - fields: Missing meta field names. - line: One-based source line to annotate. - - Returns: - None. - """ - if not fields: - return - field_list = ", ".join(fields) - message = _escape_workflow_command_message( - f"Missing meta fields: {field_list}", - ) - print(f"::{level} file={path},line={line}::{message}") - - -def emit_github_warning(path: str, fields: list[str], line: int) -> None: - """ - Print a GitHub Actions warning annotation for missing meta fields. - - Args: - path: Repository-relative path to annotate. - fields: Missing meta field names. - line: One-based source line to annotate. - - Returns: - None. - """ - _emit_annotation("warning", path, fields, line) - - -def emit_github_error(path: str, fields: list[str], line: int) -> None: - """ - Print a GitHub Actions error annotation for missing meta fields. - - Args: - path: Repository-relative path to annotate. - fields: Missing meta field names. - line: One-based source line to annotate. - - Returns: - None. - """ - _emit_annotation("error", path, fields, line) - - -def can_suggest_inline(content: str, pr_lines: set[int]) -> bool: - """ - Return whether a ``.. meta::`` edit can be anchored to the pull request diff. - - Existing ``.. meta::`` blocks are suggestable when the block's inclusive line - span overlaps the diff. New blocks are only suggestable when line 1 is in - the diff, since inserts always go at the top of the file. - - Args: - content: RST source before metadata is injected. - pr_lines: One-based lines represented by the pull-request diff. - - Returns: - ``True`` if GitHub can anchor the metadata edit to the diff. - """ - if has_meta_block(content): - span = meta_block_line_span(content) - if span is None: - return False - return _span_overlaps(span, pr_lines) - - return _span_overlaps((1, 1), pr_lines) - - -def can_suggest_after_title_inline( - content: str, - pr_lines: set[int], - *, - paragraph_span: tuple[int, int] | None = None, -) -> bool: - """ - Return whether after-title directive edits can anchor to the PR diff. - - Args: - content: RST source before after-title injections. - pr_lines: One-based lines represented by the pull-request diff. - paragraph_span: Optional line span of the paragraph to wrap. - - Returns: - ``True`` when the after-title area or paragraph span overlaps the diff. - """ - if paragraph_span is not None and _span_overlaps(paragraph_span, pr_lines): - return True - return _span_overlaps(after_title_directives_line_span(content), pr_lines) + _write_multiline_output( + f, + "comment", + build_review_comment(results, config=config), + ) -def _after_title_rule_satisfied(content: str, directive: str, rule: AfterTitleRule) -> bool: +def _after_title_rule_satisfied(content: str, directive: str) -> bool: """Return whether an after-title rule is already satisfied in ``content``.""" if directive == "short-description": return has_short_description_content(content) @@ -768,261 +428,71 @@ def _after_title_rule_satisfied(content: str, directive: str, rule: AfterTitleRu return True -def _format_short_description_snippet(paragraph: str) -> str: - """Build a copy-paste ``.. short-description::`` block for a paragraph.""" - lines = [".. short-description::"] - for chunk in paragraph.split("\n\n"): - for line in chunk.split("\n"): - stripped = line.strip() - if stripped: - lines.append(f" {stripped}") - lines.append("") - return "\n".join(lines) - - -def _process_after_title_rules( - path: Path, - content: str, - after_title_rules: dict[str, AfterTitleRule], - *, - pr_lines: set[int] | None, -) -> tuple[str, dict[str, object]]: - """ - Apply configured after-title directive rules to RST content. - - Returns: - Updated content and a dict of after-title result fields. - """ - unresolved_rules = [ - (directive, rule) - for directive, rule in after_title_rules.items() - if not _after_title_rule_satisfied(content, directive, rule) - ] - if not unresolved_rules: - return content, {} - - after_title_auto: list[str] = [] - after_title_manual: list[str] = [] - after_title_warning: list[str] = [] - after_title_error: list[str] = [] - after_title_snippets: list[dict[str, str]] = [] - suggestable = False - snippet_only = False - - for directive, rule in unresolved_rules: - paragraph_span: tuple[int, int] | None = None - if directive == "short-description": - paragraph, paragraph_span = extract_first_paragraph_after_title(content) - if paragraph is None: - after_title_manual.append(directive) - if rule.severity == "error": - after_title_error.append(directive) - else: - after_title_warning.append(directive) - continue - - can_suggest = pr_lines is None or can_suggest_after_title_inline( - content, - pr_lines, - paragraph_span=paragraph_span, - ) - snippet_text = _format_short_description_snippet(paragraph) - - if can_suggest: - new_content, changed = wrap_first_paragraph_as_short_description(content) - if changed: - content = new_content - after_title_auto.append(directive) - suggestable = True - if rule.severity == "error": - after_title_error.append(directive) - else: - after_title_warning.append(directive) - else: - after_title_snippets.append({"directive": directive, "snippet": snippet_text}) - snippet_only = True - if rule.severity == "error": - after_title_error.append(directive) - else: - after_title_warning.append(directive) - - elif directive == "showmeta": - assert rule.options is not None - can_suggest = pr_lines is None or can_suggest_after_title_inline(content, pr_lines) - snippet_text = format_showmeta_block(rule.options) - - if can_suggest: - new_content, changed = inject_showmeta_to_content(content, rule.options) - if changed: - content = new_content - after_title_auto.append(directive) - suggestable = True - if rule.severity == "error": - after_title_error.append(directive) - else: - after_title_warning.append(directive) - else: - after_title_snippets.append({"directive": directive, "snippet": snippet_text}) - snippet_only = True - if rule.severity == "error": - after_title_error.append(directive) - else: - after_title_warning.append(directive) - - if not (after_title_auto or after_title_manual or after_title_snippets): - return content, {} - - mode = "suggestable" if suggestable else ("snippet" if snippet_only else "manual_fields") - return content, { - "after_title_auto": after_title_auto, - "after_title_manual": after_title_manual, - "after_title_warning": after_title_warning, - "after_title_error": after_title_error, - "after_title_snippets": after_title_snippets, - "after_title_mode": mode, - } - - def _severity_fields( field_names: list[str], - rules: dict[str, MetaRule], + rules: dict[str, MetaRule] | dict[str, AfterTitleRule], severity: Severity, ) -> list[str]: """ - Return field names that use the given severity in ``rules``. + Return field or directive names that use the given severity in ``rules``. Args: - field_names: Candidate meta field names. - rules: Configured metadata rules keyed by field name. - severity: Severity label to match. + field_names: Candidate meta field or directive names. + rules: Configured metadata or after-title rules keyed by name. + severity: Severity label to match. Returns: - Names from ``field_names`` whose rule uses ``severity``. + Names from ``field_names`` whose rule uses ``severity``. """ return [name for name in field_names if rules[name].severity == severity] def ensure_enhancements_in_file( path: Path, - config: EnhanceConfig | dict[str, MetaRule], - *, - pr_lines: set[int] | None = None, + config: EnhanceConfig, ) -> dict[str, object] | None: """ - Resolve missing documentation enhancements in one RST file. - - When ``pr_lines`` is provided, automatic edits are only written when they can - land on lines already in the PR diff. Fields without configured values always - require manual input. + Check one RST file for missing documentation enhancements. Args: - path: RST file to inspect and, where permitted, update. - config: Enhancement configuration, or a legacy meta-rules mapping. - pr_lines: One-based pull-request diff lines, or ``None`` for local mode. + path: RST file to inspect. + config: Enhancement configuration. Returns: - A result dict when issues were found in the file as read, otherwise - ``None``. + A result dict when issues were found, otherwise ``None``. Raises: - OSError: If the RST file cannot be read or an eligible edit cannot be written. - UnicodeError: If the RST file cannot be decoded or encoded as UTF-8. + OSError: If the RST file cannot be read. + UnicodeError: If the RST file cannot be decoded as UTF-8. """ - if isinstance(config, dict): - enhance_config = EnhanceConfig(meta=config, after_title={}) - else: - enhance_config = config - rules = enhance_config.meta - content = path.read_text(encoding="utf-8") path_str = str(path).replace("\\", "/") - unresolved = _unresolved_fields(content, rules) + unresolved = _unresolved_fields(content, config.meta) after_title_unresolved = [ directive - for directive, rule in enhance_config.after_title.items() - if not _after_title_rule_satisfied(content, directive, rule) + for directive in config.after_title + if not _after_title_rule_satisfied(content, directive) ] if not unresolved and not after_title_unresolved: logger.info("%s: all configured enhancements present", path) return None - annotation_line = _annotation_line_for_meta(content) - after_title_line = _annotation_line_for_after_title(content) - auto_fields = [name for name in unresolved if rules[name].has_configured_value] - auto_metadata = {name: rules[name].value for name in auto_fields} - - mode: str | None = None - snippet = "" - - if auto_metadata: - if pr_lines is not None and not can_suggest_inline(content, pr_lines): - mode = "snippet" - snippet = format_meta_block(rules, auto_fields) - logger.info( - "%s: missing %s but auto-fix is outside the PR diff; snippet review only", - path, - ", ".join(auto_fields), - ) - else: - new_content, changed = inject_metadata_to_content(content, auto_metadata) - if changed: - path.write_text(new_content, encoding="utf-8") - content = new_content - mode = "suggestable" - snippet = format_meta_block(rules, auto_fields) - if pr_lines is None: - logger.info("%s: added meta fields %s", path, ", ".join(auto_fields)) - else: - logger.info( - "%s: added meta fields %s (inline suggestion)", - path, - ", ".join(auto_fields), - ) - - after_title_result: dict[str, object] = {} - if enhance_config.after_title: - content_before_after_title = content - content, after_title_result = _process_after_title_rules( - path, - content, - enhance_config.after_title, - pr_lines=pr_lines, - ) - if after_title_result and content != content_before_after_title: - path.write_text(content, encoding="utf-8") - - manual_fields = [name for name in unresolved if not rules[name].has_configured_value] - warning_fields = _severity_fields(unresolved, rules, "warning") - error_fields = _severity_fields(unresolved, rules, "error") - - after_title_mode = after_title_result.get("after_title_mode") - if after_title_mode == "suggestable": - mode = "suggestable" - elif after_title_mode == "snippet" and mode != "suggestable": - mode = "snippet" - elif mode is None and (unresolved or after_title_result): - mode = "manual_fields" - - if not unresolved and not after_title_result and mode is None: - return None - return { "path": path_str, - "line": annotation_line, - "after_title_line": after_title_line, - "mode": mode, - "snippet": snippet, - "auto_fields": auto_fields, - "manual_fields": manual_fields, - "warning_fields": warning_fields, - "error_fields": error_fields, - "after_title_auto": after_title_result.get("after_title_auto", []), - "after_title_manual": after_title_result.get("after_title_manual", []), - "after_title_warning": after_title_result.get("after_title_warning", []), - "after_title_error": after_title_result.get("after_title_error", []), - "after_title_snippets": after_title_result.get("after_title_snippets", []), + "meta_required": _severity_fields(unresolved, config.meta, "error"), + "meta_optional": _severity_fields(unresolved, config.meta, "warning"), + "after_title_required": _severity_fields( + after_title_unresolved, + config.after_title, + "error", + ), + "after_title_optional": _severity_fields( + after_title_unresolved, + config.after_title, + "warning", + ), } @@ -1031,10 +501,10 @@ def _collect_rst_paths(paths: list[str]) -> list[Path]: Collect existing RST files from command-line path strings. Args: - paths: Candidate filesystem paths. + paths: Candidate filesystem paths. Returns: - Existing paths whose suffix is ``.rst`` (case-insensitive). + Existing paths whose suffix is ``.rst`` (case-insensitive). """ rst_paths: list[Path] = [] for raw in paths: @@ -1049,61 +519,40 @@ def _collect_rst_paths(paths: list[str]) -> list[Path]: return rst_paths -def stamp_review_comment(body: str) -> str: - """ - Stamp a review comment so CI can supersede it later. +def _meta_field_hint(name: str, rule: MetaRule) -> str: + """Format one missing meta field for the review comment.""" + severity = "required" if rule.severity == "error" else "optional" + if rule.has_configured_value: + return f"`{name}` ({severity}, suggested value `{rule.value}`)" + return f"`{name}` ({severity})" - Args: - body: Unstamped review comment body. - Returns: - The body with the hidden review marker appended. - """ - return body.rstrip() + f"\n\n{REVIEW_MARKER}\n" - - -def _field_list_markdown(field_names: list[str], rules: dict[str, MetaRule]) -> str: - """ - Format field names with severity hints for review text. - - Args: - field_names: Meta field names to list. - rules: Configured metadata rules keyed by field name. - - Returns: - A comma-separated Markdown fragment such as `` `area` (required) ``. - """ - parts: list[str] = [] - for name in field_names: - label = "required" if rules[name].severity == "error" else "optional" - parts.append(f"`{name}` ({label})") - return ", ".join(parts) +def _after_title_hint(name: str, rule: AfterTitleRule) -> str: + """Format one missing after-title directive for the review comment.""" + severity = "required" if rule.severity == "error" else "optional" + if name == "short-description": + return f"`{name}` ({severity}, wrap the opening paragraph)" + if name == "showmeta" and rule.options: + order = rule.options.get("order", "") + return f"`{name}` ({severity}, add with `:order: {order}`)" + return f"`{name}` ({severity})" def build_review_comment( results: list[dict[str, object]], - rules: dict[str, MetaRule], + *, + config: EnhanceConfig, ) -> str: """ Build a pull-request review body from enhancement check results. Args: - results: Per-file result dictionaries from ``ensure_enhancements_in_file``. - rules: Configured metadata rules. + results: Per-file result dictionaries from ``ensure_enhancements_in_file``. + config: Enhancement configuration used to format hints. Returns: - A stamped Markdown review body containing the relevant instructions. + A stamped Markdown review body listing missing items per file. """ - inline_modes = [r for r in results if r["mode"] == "suggestable"] - snippet_modes = [r for r in results if r["mode"] == "snippet"] - manual_fields_modes = [r for r in results if r["mode"] == "manual_fields"] - after_title_snippet_results = [ - r for r in results if r.get("after_title_snippets") - ] - after_title_manual_results = [ - r for r in results if r.get("after_title_manual") - ] - lines = [ SUMMARY_REVIEW_TITLE, "", @@ -1112,106 +561,31 @@ def build_review_comment( "", ] - if inline_modes: - lines.extend( - [ - SECTION_INLINE_SUGGESTIONS, - "", - "Please **review and commit the inline suggestions** on the " - "**Files changed** tab (or use the separate *Inline documentation " - "suggestions* review). They apply these configured defaults:", - "", - ] - ) - for result in inline_modes: - parts: list[str] = [] - auto = result.get("auto_fields") or [] - if auto: - parts.append(", ".join(f"`{name}`" for name in auto)) - after_auto = result.get("after_title_auto") or [] - if after_auto: - parts.append(", ".join(f"`{name}`" for name in after_auto)) - lines.append(f"- **`{result['path']}`**: {', '.join(parts)}") - lines.append("") - - if snippet_modes: - meta_snippet_modes = [r for r in snippet_modes if r.get("snippet")] - if meta_snippet_modes: - lines.extend( - [ - SECTION_COPY_PASTE_BLOCKS, - "", - "These files could not receive inline suggestions because the edits are " - "outside the pull request diff. Add this block at the **top of each " - "file** (or append the listed fields to an existing `.. meta::` block):", - "", - ] - ) - for result in meta_snippet_modes: - lines.append(f"**`{result['path']}`**") - lines.append("```rst") - lines.append(str(result["snippet"]).rstrip()) - lines.append("```") - lines.append("") - - if after_title_snippet_results: - lines.extend( - [ - SECTION_COPY_PASTE_AFTER_TITLE, - "", - "Add these directives **after the first document title** in each file:", - "", - ] - ) - for result in after_title_snippet_results: - lines.append(f"**`{result['path']}`**") - for entry in result.get("after_title_snippets") or []: - lines.append("```rst") - lines.append(str(entry["snippet"]).rstrip()) - lines.append("```") - lines.append("") - - manual_results = [r for r in results if r.get("manual_fields")] - if manual_results: - lines.extend( - [ - SECTION_NON_EMPTY_VALUES, - "", - "These fields must have **non-empty** values in each file's " - "`.. meta::` block:", - "", + for result in results: + path = str(result["path"]) + lines.append(f"### `{path}`") + + meta_required = list(result.get("meta_required") or []) + meta_optional = list(result.get("meta_optional") or []) + if meta_required or meta_optional: + hints = [ + _meta_field_hint(name, config.meta[name]) + for name in meta_required + meta_optional ] - ) - for result in manual_results: - manual = list(result["manual_fields"]) - lines.append( - f"- **`{result['path']}`**: {_field_list_markdown(manual, rules)}" - ) - lines.append("") - - if after_title_manual_results: - lines.extend( - [ - SECTION_AFTER_TITLE_MANUAL, - "", - "These files need after-title directives that could not be added " - "automatically (for example, no prose paragraph to wrap):", - "", + lines.append(f"- Missing `.. meta::` fields: {', '.join(hints)}") + + after_title_required = list(result.get("after_title_required") or []) + after_title_optional = list(result.get("after_title_optional") or []) + if after_title_required or after_title_optional: + hints = [ + _after_title_hint(name, config.after_title[name]) + for name in after_title_required + after_title_optional ] - ) - for result in after_title_manual_results: - manual = ", ".join(f"`{name}`" for name in result.get("after_title_manual") or []) - lines.append(f"- **`{result['path']}`**: {manual}") - lines.append("") + lines.append(f"- Missing after-title directives: {', '.join(hints)}") - if manual_fields_modes and not inline_modes and not snippet_modes and not after_title_snippet_results: - lines.append( - "Add or complete a `.. meta::` block at the top of each affected file.", - ) lines.append("") - body = "\n".join(lines).rstrip() + "\n" - return stamp_review_comment(body) + return "\n".join(lines).rstrip() + f"\n\n{REVIEW_MARKER}\n" def main(argv: list[str] | None = None) -> int: @@ -1224,15 +598,15 @@ def main(argv: list[str] | None = None) -> int: ``1`` only for unresolved error-severity fields. Args: - argv: Command-line arguments excluding the executable name, or ``None`` - to read them from ``sys.argv``. + argv: Command-line arguments excluding the executable name, or ``None`` + to read them from ``sys.argv``. Returns: - Process exit code (``0`` on success, ``1`` when issues remain per mode - above). + Process exit code (``0`` on success, ``1`` when issues remain per mode + above). Raises: - SystemExit: If command-line arguments or enhancement configuration are invalid. + SystemExit: If command-line arguments or enhancement configuration are invalid. """ parser = argparse.ArgumentParser( description=( @@ -1257,16 +631,16 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument( "--diff-base", help=( - "Git commit SHA for the pull request base. When set, only writes edits " - "that overlap the PR diff; other files receive a review comment instead." + "Git commit SHA for the pull request base. When set, changed " + "`.rst` files are discovered automatically." ), ) parser.add_argument( "--status-file", type=Path, help=( - "Write enhancements_checked, inline_suggestions, has_results, has_errors, " - "the review comment body, and the suggestion note for CI" + "Write enhancements_checked, has_results, has_errors, and the " + "review comment body for CI" ), ) parser.add_argument( @@ -1285,7 +659,7 @@ def main(argv: list[str] | None = None) -> int: if not args.paths and not args.diff_base: parser.error("provide at least one .rst path or set --diff-base to discover changes") - rules = load_enhance_config(args.config) + config = load_enhance_config(args.config) checked_pull_request_rst = False if not args.paths: @@ -1298,7 +672,7 @@ def main(argv: list[str] | None = None) -> int: args.status_file, enhancements_checked=False, results=[], - rules=rules.meta, + config=config, has_errors=False, ) return 0 @@ -1314,46 +688,18 @@ def main(argv: list[str] | None = None) -> int: logger.info("No RST files to process") else: for path in rst_paths: - pr_lines: set[int] | None = None - if args.diff_base: - pr_lines = pr_diff_lines_for_file(args.diff_base, path) - result = ensure_enhancements_in_file(path, rules, pr_lines=pr_lines) + result = ensure_enhancements_in_file(path, config) if result is not None: results.append(result) - inline_count = sum(1 for r in results if r["mode"] == "suggestable") - snippet_count = sum(1 for r in results if r["mode"] == "snippet") - manual_fields_count = sum(1 for r in results if r["mode"] == "manual_fields") logger.info( - "Processed %d file(s): %d inline, %d snippet, %d manual_fields", + "Processed %d file(s): %d with missing enhancements", len(rst_paths), - inline_count, - snippet_count, - manual_fields_count, - ) - _log_working_tree_summary(rst_paths) - - for result in results: - path = str(result["path"]) - meta_line = int(result["line"]) - after_title_line = int(result.get("after_title_line") or meta_line) - emit_github_warning(path, list(result["warning_fields"]), meta_line) - emit_github_error(path, list(result["error_fields"]), meta_line) - _emit_after_title_annotation( - "warning", - path, - list(result.get("after_title_warning") or []), - after_title_line, - ) - _emit_after_title_annotation( - "error", - path, - list(result.get("after_title_error") or []), - after_title_line, + len(results), ) has_errors = any( - result["error_fields"] or result.get("after_title_error") + result["meta_required"] or result.get("after_title_required") for result in results ) @@ -1362,7 +708,7 @@ def main(argv: list[str] | None = None) -> int: args.status_file, enhancements_checked=checked_pull_request_rst, results=results, - rules=rules.meta, + config=config, has_errors=has_errors, ) diff --git a/tools/rst_utils.py b/tools/rst_utils.py index 6fcb2436f10..3c172a1c55b 100644 --- a/tools/rst_utils.py +++ b/tools/rst_utils.py @@ -1,17 +1,15 @@ """ -Utilities for editing reStructuredText source, in particular ``.. meta::``, -``.. short-description::``, and ``.. showmeta::`` directives. +Read-only utilities for detecting Sphinx directives in reStructuredText source. + +Supports ``.. meta::``, ``.. short-description::``, and ``.. showmeta::``. """ -import logging import re -logger = logging.getLogger(__name__) - -def _find_directive_block(content: str, directive: str) -> tuple[int, int, int, str, str]: +def _find_directive_block(content: str, directive: str) -> str | None: """ - Locate the first ``.. ::`` block in RST source. + Locate the inner body of the first ``.. ::`` block in RST source. The directive block consists of the explicit marker line followed by contiguous indented lines; a blank line or a less-indented line ends the @@ -22,9 +20,8 @@ def _find_directive_block(content: str, directive: str) -> tuple[int, int, int, directive: Directive name without the ``..`` prefix (e.g. ``meta``). Returns: - Tuple of ``(start, marker_end, block_end, inner, indent)``. - If no directive is found, ``start``, ``marker_end``, and ``block_end`` - are ``-1``, ``inner`` is ``''``, and ``indent`` defaults to three spaces. + The inner body text of the directive block, or ``None`` when no block + is found. """ match = re.search( rf"^\.\.\s+{re.escape(directive)}::\s*\n", @@ -32,13 +29,10 @@ def _find_directive_block(content: str, directive: str) -> tuple[int, int, int, re.MULTILINE, ) if not match: - return -1, -1, -1, "", " " + return None - start = match.start() marker_end = match.end() - indent = " " inner_parts: list[str] = [] - consumed = 0 remainder = content[marker_end:] for line in remainder.splitlines(keepends=True): @@ -46,69 +40,32 @@ def _find_directive_block(content: str, directive: str) -> tuple[int, int, int, break if not line.startswith((" ", "\t")): break - if not inner_parts: - ws_len = len(line) - len(line.lstrip(" \t")) - indent = line[:ws_len] inner_parts.append(line) - consumed += len(line) - block_end = marker_end + consumed inner = "".join(inner_parts) if inner and not inner.endswith("\n"): inner += "\n" - return start, marker_end, block_end, inner, indent - - -def _find_meta_block(content: str) -> tuple[int, int, int, str, str]: - """ - Locate the first ``.. meta::`` directive in RST source. - - The directive block consists of the explicit marker line followed by - contiguous indented lines; a blank line or a less-indented line ends the - block (per reStructuredText directive block rules). - - Args: - content: The RST file content to search. - - Returns: - Tuple of ``(start, marker_end, block_end, inner, indent)``. - If no directive is found, ``start``, ``marker_end``, and ``block_end`` - are ``-1``, ``inner`` is ``''``, and ``indent`` defaults to three spaces. - """ - return _find_directive_block(content, "meta") + return inner if inner.strip() else None -def _extract_meta_names_from_block(meta_block_inner: str) -> set[str]: +def _extract_field_values(block_inner: str) -> dict[str, str]: """ - Collect field names from the body of a ``.. meta::`` directive. + Collect field or option names and values from a directive body. Each line of the form ``:name: value`` contributes ``name`` (Docutils also allows forms such as ``:name attr=value:``; the captured segment matches that usage). Args: - meta_block_inner: The inner text of the meta block. + block_inner: The inner text of a directive block. Returns: - A set of field names found in the block. - """ - return set(_extract_meta_fields_from_block(meta_block_inner)) - - -def _extract_meta_fields_from_block(meta_block_inner: str) -> dict[str, str]: - """ - Collect field names and values from the body of a ``.. meta::`` directive. - - Args: - meta_block_inner: The inner text of the meta block. - - Returns: - Mapping from field name to field body text (may be empty). + Mapping from field or option name to body text (may be empty). """ fields: dict[str, str] = {} for field_match in re.finditer( r"^[ \t]+:([^:\n]+?):\s*(.*)$", - meta_block_inner, + block_inner, re.MULTILINE, ): fields[field_match.group(1).strip()] = field_match.group(2) @@ -127,165 +84,10 @@ def get_meta_fields_from_content(content: str) -> dict[str, str]: Returns: Mapping from meta field name to field body text. """ - _start, _marker_end, _block_end, inner, _indent = _find_meta_block(content) - return _extract_meta_fields_from_block(inner) - - -def get_meta_names_from_content(content: str) -> set[str]: - """ - Return the set of field names already present in the first ``.. meta::`` block. - - If no ``.. meta::`` directive exists, returns an empty set. - - Args: - content: The RST file content to search. - - Returns: - A set of field names present in the meta block. - """ - _start, _marker_end, _block_end, inner, _indent = _find_meta_block(content) - return _extract_meta_names_from_block(inner) - - -def has_meta_block(content: str) -> bool: - """ - Return whether the document contains a ``.. meta::`` directive. - - Args: - content: The RST file content to search. - - Returns: - True if a ``.. meta::`` block exists, False otherwise. - """ - start, _marker_end, _block_end, _inner, _indent = _find_meta_block(content) - return start >= 0 - - -def _normalise_meta_field_value(value: str) -> str: - """ - Collapse whitespace so the meta field body stays a single logical line. - - Args: - value: The raw field value. - - Returns: - The normalised field value. - """ - return " ".join(value.split()) # Docutils treats the field body as one string; keep it one physical line - - -def inject_metadata_to_content( - content: str, - metadata: dict[str, str], -) -> tuple[str, bool]: - """ - Insert or append ``.. meta::`` field entries for the given name/value pairs. - - Appends to an existing ``.. meta::`` block when present. Otherwise inserts a - new block at the start of the document. - - Skips keys that already have a non-empty value in the block. Fills keys that - are missing or present with a blank value. - - Returns: - Updated source and whether any change was made. - """ - start, marker_end, block_end, inner, indent = _find_meta_block(content) - existing = _extract_meta_fields_from_block(inner) - merged: dict[str, str] = dict(existing) - changed = False - - for key, raw_value in metadata.items(): - value = _normalise_meta_field_value(raw_value) - if key not in merged: - merged[key] = value - changed = True - elif not merged[key].strip(): - merged[key] = value - changed = True - else: - logger.warning( - "Existing meta field %r in .. meta:: block; skipping", - key, - ) - - if not changed: - return content, False - - ordered_keys: list[str] = list(existing.keys()) - for key in metadata: - if key not in ordered_keys: - ordered_keys.append(key) - new_inner = "".join(f"{indent}:{key}: {merged[key]}\n" for key in ordered_keys) - - if start >= 0: - # Replace only the directive body slice; ``marker_end``/``block_end`` bracket the original inner - # Normalise trailing whitespace: one blank line after the block - remainder = content[block_end:].lstrip() - new_content = content[:marker_end] + new_inner + "\n" + remainder - else: - remainder = content.lstrip() - new_content = ".. meta::\n" + new_inner + "\n" + remainder - - return new_content, True - - -def _byte_offset_to_line_number(content: str, offset: int) -> int: - """Return the 1-based line number containing ``offset`` (or the next line at EOF).""" - if offset <= 0: - return 1 - if offset >= len(content): - return content.count("\n") + (0 if content.endswith("\n") else 1) - return content.count("\n", 0, offset) + 1 - - -def meta_block_line_span(content: str) -> tuple[int, int] | None: - """ - Return the inclusive 1-based line span of the first ``.. meta::`` block. - - Returns ``None`` if no meta block exists. - """ - start, _marker_end, block_end, _inner, _indent = _find_meta_block(content) - if start < 0: - return None - start_line = _byte_offset_to_line_number(content, start) - end_offset = block_end - 1 if block_end > start else start - end_line = _byte_offset_to_line_number(content, end_offset) - return start_line, end_line - - -def _find_short_description_block(content: str) -> tuple[int, int, int, str, str]: - """ - Locate the first ``.. short-description::`` directive in RST source. - - Uses the same block-boundary rules as ``_find_meta_block``: the body is - contiguous indented lines until a blank line or a line starting at column 0. - - Args: - content: The RST file content to search. - - Returns: - Tuple of ``(start, marker_end, block_end, inner, indent)``. - If no directive is found, ``start``, ``marker_end``, and ``block_end`` - are ``-1``, ``inner`` is ``''``, and ``indent`` defaults to three spaces. - """ - return _find_directive_block(content, "short-description") - - -def _short_description_inner_has_content(inner: str) -> bool: - """ - True when the directive body contains non-whitespace text. - - Args: - inner: The inner text of the short-description block. - - Returns: - True if the body has content, False otherwise. - """ - for line in inner.splitlines(): - if line.strip(): - return True - return False + inner = _find_directive_block(content, "meta") + if not inner: + return {} + return _extract_field_values(inner) def has_short_description_content(content: str) -> bool: @@ -298,338 +100,8 @@ def has_short_description_content(content: str) -> bool: Returns: True if a non-empty short-description block exists, False otherwise. """ - _s, _m, _b, inner, _i = _find_short_description_block(content) - return _short_description_inner_has_content(inner) - - -def get_short_description_body(content: str) -> str | None: - """ - Return the normalised inner body text of the first ``.. short-description::`` block. - - Returns ``None`` if the directive is missing or the body is empty. - """ - _s, _m, _b, inner, _i = _find_short_description_block(content) - if not _short_description_inner_has_content(inner): - return None - paragraphs: list[str] = [] - current: list[str] = [] - for line in inner.splitlines(): - stripped = line.strip() - if not stripped: - if current: - paragraphs.append(" ".join(current)) - current = [] - continue - current.append(stripped) - if current: - paragraphs.append(" ".join(current)) - return "\n\n".join(paragraphs) if paragraphs else None - - -def _format_short_description_inner(text: str, indent: str) -> str: - """ - Turn model output into RST directive body lines (indented paragraphs). - - Args: - text: The model-generated prose. - indent: The indentation string to use. - - Returns: - The formatted and indented inner text for the directive. - """ - chunks = [p.strip() for p in text.split("\n\n") if p.strip()] - lines_out: list[str] = [] - for i, para in enumerate(chunks): - for line in para.split("\n"): - s = line.strip() - if s: - lines_out.append(f"{indent}{s}\n") - if i < len(chunks) - 1: - lines_out.append(f"{indent}\n") - return "".join(lines_out) - - -def _find_insertion_point_after_title(content: str) -> int: - """ - Return the index in ``content`` immediately after the first document title block. - - A title block is a non-blank text line followed by a line of repeating - underline characters (reStructuredText section markers such as ``=``, - ``-``, ``~``, and other Docutils-adornment characters). - If no title is found, returns ``0``. - - Args: - content: The RST file content to search. - - Returns: - The byte index where the title block ends. - """ - lines = content.splitlines(keepends=True) - i = 0 - while i + 1 < len(lines): - title_line = lines[i] - underline_line = lines[i + 1] - title_stripped = title_line.strip() - # Docutils section adornment characters - ul_match = re.match(r'^([!"#$%&\'()*+,\-./:;<=>?@\[\\\]^_`{|}~]+)\s*$', underline_line.rstrip("\n")) - if title_stripped and ul_match is not None: - ul = ul_match.group(1) - if len(set(ul)) == 1 and len(ul) >= len(title_stripped): - pos = 0 - for j in range(i + 2): - pos += len(lines[j]) - return pos - i += 1 - return 0 - - -def inject_short_description_to_content(content: str, text: str) -> tuple[str, bool]: - """ - Insert or fill the first ``.. short-description::`` directive with the given prose. - - If the directive exists and already has body text, logs a warning and returns - the original content unchanged. If the directive exists with an empty body, - fills the body. If the directive is missing, inserts a new block after the - first detected document title (or at the start of the file if none). - - Returns: - Updated source and whether any change was made. - """ - start, marker_end, block_end, inner, indent = _find_short_description_block(content) - new_inner = _format_short_description_inner(text, indent) - - if start >= 0: - if _short_description_inner_has_content(inner): - logger.warning( - "Existing .. short-description:: body has content; skipping replacement", - ) - return content, False - # Normalise trailing whitespace: one blank line after the block - remainder = content[block_end:].lstrip() - new_content = content[:marker_end] + new_inner + "\n" + remainder - return new_content, True - - insert_at = _find_insertion_point_after_title(content) - # Normalise trailing whitespace: one blank line before and after the block - remainder = content[insert_at:].lstrip() - block = f"\n.. short-description::\n{new_inner}\n" - new_content = content[:insert_at] + block + remainder - return new_content, True - - -_SECTION_ADORNMENT_RE = re.compile( - r'^([!"#$%&\'()*+,\-./:;<=>?@\[\\\]^_`{|}~]+)\s*$', -) - - -def _is_section_title_at(lines: list[str], index: int) -> bool: - """ - Return whether ``lines[index]`` is an RST section title with an underline. - - Args: - lines: Document lines (with or without trailing newlines). - index: Zero-based line index of the candidate title line. - - Returns: - True when the line is followed by a valid adornment underline. - """ - if index + 1 >= len(lines): - return False - title_stripped = lines[index].strip() - if not title_stripped: - return False - ul_match = _SECTION_ADORNMENT_RE.match(lines[index + 1].rstrip("\n")) - if ul_match is None: - return False - ul = ul_match.group(1) - return len(set(ul)) == 1 and len(ul) >= len(title_stripped) - - -def _line_starts_directive(line: str) -> bool: - """Return whether ``line`` begins an explicit RST directive marker.""" - return bool(re.match(r"^\.\.\s+\S+::", line)) - - -def _skip_past_directive_block(lines: list[str], directive_index: int) -> int: - """ - Return the index of the first line after a directive block. - - Args: - lines: Document lines (with or without trailing newlines). - directive_index: Zero-based index of the ``.. directive::`` line. - - Returns: - Index of the first line following the directive block. - """ - i = directive_index + 1 - while i < len(lines): - line = lines[i] - if line.strip() == "": - i += 1 - continue - if not line.startswith((" ", "\t")): - return i - i += 1 - return i - - -def _title_line_span(content: str) -> tuple[int, int] | None: - """ - Return the inclusive 1-based line span of the first document title block. - - Returns ``None`` when no title is found. - """ - lines = content.splitlines() - for i in range(len(lines) - 1): - if _is_section_title_at(lines, i): - return i + 1, i + 2 - return None - - -def extract_first_paragraph_after_title( - content: str, -) -> tuple[str | None, tuple[int, int] | None]: - """ - Find the first prose paragraph after the first document title. - - Skips blank lines, directives, indented non-prose lines (such as toctree - entries), and section titles. Collects contiguous prose until the next - blank line, directive, or section title. - - Args: - content: RST source to search. - - Returns: - Paragraph text with each source prose line separated by a newline, and - its inclusive 1-based line span, or ``(None, None)`` when no prose - paragraph is found. - """ - lines = content.splitlines(keepends=True) - stripped_lines = [line.rstrip("\n") for line in lines] - insert_at = _find_insertion_point_after_title(content) - if insert_at <= 0: - start_index = 0 - else: - start_index = content[:insert_at].count("\n") - - prose_lines: list[str] = [] - prose_start: int | None = None - i = start_index - while i < len(lines): - line = lines[i] - stripped = line.strip() - if not stripped: - if prose_lines: - break - i += 1 - continue - if _line_starts_directive(stripped): - if prose_lines: - break - i = _skip_past_directive_block(lines, i) - continue - if _is_section_title_at(stripped_lines, i): - if prose_lines: - break - i += 2 - continue - if not prose_lines and line.startswith((" ", "\t")): - i += 1 - continue - if prose_lines and line.startswith((" ", "\t")): - prose_lines.append(stripped) - i += 1 - continue - if line.startswith((" ", "\t")): - i += 1 - continue - if prose_start is None: - prose_start = i - prose_lines.append(stripped) - i += 1 - - if not prose_lines or prose_start is None: - return None, None - - start_line = prose_start + 1 - end_line = prose_start + len(prose_lines) - return "\n".join(prose_lines), (start_line, end_line) - - -def wrap_first_paragraph_as_short_description(content: str) -> tuple[str, bool]: - """ - Wrap the first prose paragraph after the title into ``.. short-description::``. - - If a non-empty short-description already exists, returns unchanged. If the - directive exists with an empty body, fills it from the first paragraph. - Otherwise inserts a new directive after the title and removes the paragraph - from the body. - - Returns: - Updated source and whether any change was made. - """ - if has_short_description_content(content): - return content, False - - paragraph, span = extract_first_paragraph_after_title(content) - if paragraph is None or span is None: - return content, False - - without_para = _remove_line_span(content, span) - start, marker_end, block_end, _inner, indent = _find_short_description_block(without_para) - new_inner = _format_short_description_inner(paragraph, indent) - - if start >= 0: - remainder = without_para[block_end:].lstrip() - new_content = without_para[:marker_end] + new_inner + "\n" + remainder - return new_content, True - - insert_at = _find_insertion_point_after_title(without_para) - remainder = without_para[insert_at:].lstrip() - block = f"\n.. short-description::\n{new_inner}\n" - new_content = without_para[:insert_at] + block + remainder - return new_content, True - - -def _remove_line_span(content: str, span: tuple[int, int]) -> str: - """ - Remove an inclusive 1-based line span from RST source. - - Args: - content: RST source. - span: Inclusive start and end line numbers (1-based). - - Returns: - Source with the span removed and adjacent blank lines collapsed. - """ - start_line, end_line = span - lines = content.splitlines(keepends=True) - kept = lines[: start_line - 1] + lines[end_line:] - result = "".join(kept) - while "\n\n\n" in result: - result = result.replace("\n\n\n", "\n\n") - return result - - -def _find_showmeta_block(content: str) -> tuple[int, int, int, str, str]: - """Locate the first ``.. showmeta::`` directive in RST source.""" - return _find_directive_block(content, "showmeta") - - -def _extract_directive_options_from_block(block_inner: str) -> dict[str, str]: - """ - Collect option names and values from a directive body. - - Each line of the form ``:name: value`` contributes ``name``. - """ - options: dict[str, str] = {} - for field_match in re.finditer( - r"^[ \t]+:([^:\n]+?):\s*(.*)$", - block_inner, - re.MULTILINE, - ): - options[field_match.group(1).strip()] = field_match.group(2) - return options + inner = _find_directive_block(content, "short-description") + return bool(inner and inner.strip()) def has_showmeta_with_order(content: str) -> bool: @@ -642,146 +114,8 @@ def has_showmeta_with_order(content: str) -> bool: Returns: True when showmeta exists with a non-blank order option. """ - _s, _m, _b, inner, _i = _find_showmeta_block(content) - if not inner.strip(): + inner = _find_directive_block(content, "showmeta") + if not inner: return False - options = _extract_directive_options_from_block(inner) + options = _extract_field_values(inner) return bool(options.get("order", "").strip()) - - -def showmeta_line_span(content: str) -> tuple[int, int] | None: - """ - Return the inclusive 1-based line span of the first ``.. showmeta::`` block. - - Returns ``None`` if no showmeta block exists. - """ - start, _marker_end, block_end, _inner, _indent = _find_showmeta_block(content) - if start < 0: - return None - start_line = _byte_offset_to_line_number(content, start) - end_offset = block_end - 1 if block_end > start else start - end_line = _byte_offset_to_line_number(content, end_offset) - return start_line, end_line - - -def short_description_line_span(content: str) -> tuple[int, int] | None: - """ - Return the inclusive 1-based line span of the first ``.. short-description::`` block. - - Returns ``None`` if no short-description block exists. - """ - start, _marker_end, block_end, _inner, _indent = _find_short_description_block(content) - if start < 0: - return None - start_line = _byte_offset_to_line_number(content, start) - end_offset = block_end - 1 if block_end > start else start - end_line = _byte_offset_to_line_number(content, end_offset) - return start_line, end_line - - -def _insertion_point_after_short_description(content: str) -> int: - """ - Return the byte index immediately after the first short-description block. - - Falls back to after the title when no short-description exists. - """ - _s, _m, block_end, _inner, _indent = _find_short_description_block(content) - if block_end >= 0: - return block_end - return _find_insertion_point_after_title(content) - - -def format_showmeta_block(options: dict[str, str], indent: str = " ") -> str: - """ - Build an RST ``.. showmeta::`` block for the given options. - - Args: - options: Option name to value mapping. - indent: Indentation for option lines. - - Returns: - A formatted ``.. showmeta::`` directive ending with a blank line. - """ - lines = [".. showmeta::"] - for key, value in options.items(): - lines.append(f"{indent}:{key}: {value}") - lines.append("") - return "\n".join(lines) - - -def inject_showmeta_to_content( - content: str, - options: dict[str, str], -) -> tuple[str, bool]: - """ - Insert or fill a ``.. showmeta::`` directive with the given options. - - Appends to an existing block when present. Otherwise inserts after the first - short-description block, or after the title when none exists. Skips options - that already have non-empty values. - - Returns: - Updated source and whether any change was made. - """ - start, marker_end, block_end, inner, indent = _find_showmeta_block(content) - existing = _extract_directive_options_from_block(inner) - merged: dict[str, str] = dict(existing) - changed = False - - for key, raw_value in options.items(): - value = _normalise_meta_field_value(raw_value) - if key not in merged: - merged[key] = value - changed = True - elif not merged[key].strip(): - merged[key] = value - changed = True - else: - logger.warning( - "Existing showmeta option %r; skipping", - key, - ) - - if not changed: - return content, False - - ordered_keys: list[str] = list(existing.keys()) - for key in options: - if key not in ordered_keys: - ordered_keys.append(key) - new_inner = "".join(f"{indent}:{key}: {merged[key]}\n" for key in ordered_keys) - - if start >= 0: - remainder = content[block_end:].lstrip() - new_content = content[:marker_end] + new_inner + "\n" + remainder - return new_content, True - - insert_at = _insertion_point_after_short_description(content) - remainder = content[insert_at:].lstrip() - block = f"\n.. showmeta::\n{new_inner}\n" - new_content = content[:insert_at] + block + remainder - return new_content, True - - -def after_title_directives_line_span(content: str) -> tuple[int, int] | None: - """ - Return the inclusive 1-based line span of the post-title directive area. - - Covers short-description and/or showmeta blocks. When neither exists, returns - the line immediately after the title (or line 1 when no title is found). - """ - spans: list[tuple[int, int]] = [] - for span_fn in (short_description_line_span, showmeta_line_span): - span = span_fn(content) - if span is not None: - spans.append(span) - - if spans: - return min(s[0] for s in spans), max(s[1] for s in spans) - - title_span = _title_line_span(content) - if title_span is not None: - return title_span[1] + 1, title_span[1] + 1 - - return 1, 1 - diff --git a/tools/supersede_enhancement_reviews.sh b/tools/supersede_enhancement_reviews.sh index 8737cedd5a4..8d685feee9c 100755 --- a/tools/supersede_enhancement_reviews.sh +++ b/tools/supersede_enhancement_reviews.sh @@ -1,7 +1,6 @@ #!/usr/bin/env bash -# Minimise prior stamped summary reviews from ensure_enhancements.py. Suggestion-carrying -# reviews (unstamped suggest-changes bodies) are left visible so Conversation keeps -# live inline suggestions until they are actioned. +# Minimise prior stamped summary reviews from ensure_enhancements.py so each +# workflow run replaces the last Documentation enhancements review comment. set -euo pipefail : "${GH_TOKEN:?GH_TOKEN is required}" diff --git a/tools/tests/test_ensure_enhancements.py b/tools/tests/test_ensure_enhancements.py index 2c60c9d3bff..90a2bdf9aac 100644 --- a/tools/tests/test_ensure_enhancements.py +++ b/tools/tests/test_ensure_enhancements.py @@ -27,25 +27,17 @@ from ensure_enhancements import ( # noqa: E402 REVIEW_MARKER, - SECTION_COPY_PASTE_AFTER_TITLE, - SECTION_INLINE_SUGGESTIONS, - SECTION_NON_EMPTY_VALUES, SUMMARY_REVIEW_TITLE, AfterTitleRule, EnhanceConfig, MetaRule, _unresolved_fields, build_review_comment, - can_suggest_after_title_inline, - can_suggest_inline, changed_rst_paths, ensure_enhancements_in_file, load_enhance_config, main, - _annotation_line_for_after_title, - _annotation_line_for_meta, ) -from rst_utils import get_meta_fields_from_content, inject_metadata_to_content # noqa: E402 SAMPLE_CONFIG = textwrap.dedent( """ @@ -133,75 +125,6 @@ def test_load_enhance_config_parses_after_title(self) -> None: ) -class TestCanSuggestInline(unittest.TestCase): - def test_meta_block_overlap_uses_inclusive_span_only(self) -> None: - content = textwrap.dedent( - """ - .. meta:: - :product: x - - Title - ===== - """ - ).lstrip() - # Meta block is lines 1-2; line 3 is blank after the block. - self.assertFalse(can_suggest_inline(content, {3})) - self.assertTrue(can_suggest_inline(content, {2})) - - def test_after_title_overlap_uses_paragraph_span(self) -> None: - content = textwrap.dedent( - """ - Title - ===== - - Opening paragraph. - """ - ).lstrip() - self.assertTrue(can_suggest_after_title_inline(content, {4}, paragraph_span=(4, 4))) - self.assertFalse(can_suggest_after_title_inline(content, {8}, paragraph_span=(4, 4))) - - -class TestAnnotationLines(unittest.TestCase): - def test_after_title_line_uses_paragraph_when_meta_is_at_top(self) -> None: - content = textwrap.dedent( - """ - .. meta:: - :product: x - - Title - ===== - - Opening paragraph beneath the title. - - More body. - """ - ).lstrip() - self.assertEqual(_annotation_line_for_meta(content), 1) - self.assertEqual(_annotation_line_for_after_title(content), 7) - - def test_result_includes_separate_after_title_line(self) -> None: - config = EnhanceConfig(meta={}, after_title=AFTER_TITLE_RULES) - content = textwrap.dedent( - """ - .. meta:: - :product: x - - Title - ===== - - Opening paragraph beneath the title. - """ - ).lstrip() - with tempfile.TemporaryDirectory() as tmp: - path = Path(tmp) / "page.rst" - path.write_text(content, encoding="utf-8") - result = ensure_enhancements_in_file(path, config) - self.assertIsNotNone(result) - assert result is not None - self.assertEqual(result["line"], 1) - self.assertEqual(result["after_title_line"], 7) - - class TestUnresolvedFields(unittest.TestCase): def test_missing_and_blank_count_as_unresolved(self) -> None: rules = { @@ -219,65 +142,32 @@ def test_missing_and_blank_count_as_unresolved(self) -> None: ) self.assertEqual(_unresolved_fields(content, rules), ["product", "area"]) - def test_inject_fills_blank_configured_value(self) -> None: - content = textwrap.dedent( - """ - .. meta:: - :product: - - Title - ===== - """ - ) - updated, changed = inject_metadata_to_content(content, {"product": "{PRODUCT}"}) - self.assertTrue(changed) - fields = get_meta_fields_from_content(updated) - self.assertEqual(fields["product"], "{PRODUCT}") - class TestEnsureEnhancementsInFile(unittest.TestCase): - def test_local_auto_inject_clears_configured_fields(self) -> None: - rules = { - "product": MetaRule("warning", "{PRODUCT}"), - "area": MetaRule("error", ""), - } - with tempfile.TemporaryDirectory() as tmp: - path = Path(tmp) / "page.rst" - path.write_text("Title\n=====\n", encoding="utf-8") - result = ensure_enhancements_in_file(path, rules) - self.assertIsNotNone(result) - self.assertEqual(result["mode"], "suggestable") - self.assertIn("area", result["manual_fields"]) - self.assertIn("area", result["error_fields"]) - fields = get_meta_fields_from_content(path.read_text(encoding="utf-8")) - self.assertEqual(fields["product"], "{PRODUCT}") - - def test_auto_injected_fields_are_still_annotated(self) -> None: - rules = { - "product": MetaRule("warning", "{PRODUCT}"), - "area": MetaRule("error", ""), - } - with tempfile.TemporaryDirectory() as tmp: - path = Path(tmp) / "page.rst" - path.write_text("Title\n=====\n", encoding="utf-8") - result = ensure_enhancements_in_file(path, rules) - self.assertIsNotNone(result) - self.assertIn("product", result["warning_fields"]) - self.assertNotIn("product", result["manual_fields"]) - - def test_result_returned_when_only_configured_fields_missing(self) -> None: - rules = {"product": MetaRule("warning", "{PRODUCT}")} + def test_reports_missing_meta_fields(self) -> None: + config = EnhanceConfig( + meta={ + "product": MetaRule("warning", "{PRODUCT}"), + "area": MetaRule("error", ""), + }, + after_title={}, + ) with tempfile.TemporaryDirectory() as tmp: path = Path(tmp) / "page.rst" path.write_text("Title\n=====\n", encoding="utf-8") - result = ensure_enhancements_in_file(path, rules) + result = ensure_enhancements_in_file(path, config) self.assertIsNotNone(result) - self.assertEqual(result["mode"], "suggestable") - self.assertEqual(result["warning_fields"], ["product"]) - self.assertEqual(result["manual_fields"], []) + assert result is not None + self.assertIn("product", result["meta_optional"]) + self.assertIn("area", result["meta_required"]) + self.assertEqual(result["after_title_optional"], []) + self.assertEqual(result["after_title_required"], []) def test_no_result_when_all_fields_present(self) -> None: - rules = {"product": MetaRule("warning", "{PRODUCT}")} + config = EnhanceConfig( + meta={"product": MetaRule("warning", "{PRODUCT}")}, + after_title={}, + ) content = textwrap.dedent( """ .. meta:: @@ -290,50 +180,30 @@ def test_no_result_when_all_fields_present(self) -> None: with tempfile.TemporaryDirectory() as tmp: path = Path(tmp) / "page.rst" path.write_text(content, encoding="utf-8") - self.assertIsNone(ensure_enhancements_in_file(path, rules)) - + self.assertIsNone(ensure_enhancements_in_file(path, config)) -class TestAfterTitleEnhancements(unittest.TestCase): - def test_inserts_short_description_and_showmeta(self) -> None: + def test_does_not_modify_files(self) -> None: config = EnhanceConfig( meta={"product": MetaRule("warning", "{PRODUCT}")}, - after_title=AFTER_TITLE_RULES, + after_title={}, ) - content = textwrap.dedent( - """ - Title - ===== - - Opening paragraph for the page. - - More content. - """ - ).lstrip() + original = "Title\n=====\n" with tempfile.TemporaryDirectory() as tmp: path = Path(tmp) / "page.rst" - path.write_text(content, encoding="utf-8") - result = ensure_enhancements_in_file(path, config) - self.assertIsNotNone(result) - updated = path.read_text(encoding="utf-8") - self.assertIn(".. short-description::", updated) - self.assertIn(".. showmeta::", updated) - self.assertIn(":order: area, content-type, experience", updated) - self.assertIn("Opening paragraph for the page.", updated) - self.assertIn("More content.", updated) - - def test_toctree_before_paragraph_wraps_correct_paragraph(self) -> None: + path.write_text(original, encoding="utf-8") + ensure_enhancements_in_file(path, config) + self.assertEqual(path.read_text(encoding="utf-8"), original) + + +class TestAfterTitleEnhancements(unittest.TestCase): + def test_reports_missing_after_title_directives(self) -> None: config = EnhanceConfig(meta={}, after_title=AFTER_TITLE_RULES) content = textwrap.dedent( """ Title ===== - .. toctree:: - Page - - First paragraph here. - - Second paragraph. + Opening paragraph for the page. """ ).lstrip() with tempfile.TemporaryDirectory() as tmp: @@ -341,90 +211,39 @@ def test_toctree_before_paragraph_wraps_correct_paragraph(self) -> None: path.write_text(content, encoding="utf-8") result = ensure_enhancements_in_file(path, config) self.assertIsNotNone(result) - updated = path.read_text(encoding="utf-8") - self.assertIn("First paragraph here.", updated) - self.assertIn(".. showmeta::", updated) - toctree_pos = updated.index(".. toctree::") - showmeta_pos = updated.index(".. showmeta::") - self.assertLess(showmeta_pos, toctree_pos) - - def test_build_review_comment_includes_after_title_snippets(self) -> None: - rules = {"area": MetaRule("error", "")} - results = [ - { - "path": "source/Page.rst", - "mode": "snippet", - "snippet": "", - "auto_fields": [], - "manual_fields": [], - "warning_fields": ["short-description"], - "error_fields": [], - "line": 1, - "after_title_auto": [], - "after_title_manual": [], - "after_title_warning": ["short-description"], - "after_title_error": [], - "after_title_snippets": [ - { - "directive": "showmeta", - "snippet": ".. showmeta::\n :order: area\n", - }, - ], - }, - ] - body = build_review_comment(results, rules) - self.assertIn(SECTION_COPY_PASTE_AFTER_TITLE, body) - self.assertIn(".. showmeta::", body) + assert result is not None + self.assertIn("short-description", result["after_title_optional"]) + self.assertIn("showmeta", result["after_title_optional"]) class TestReviewAndExit(unittest.TestCase): - def test_build_review_comment_lists_manual_fields(self) -> None: - rules = { - "area": MetaRule("error", ""), - "experience": MetaRule("warning", ""), - } - results = [ - { - "path": "source/Page.rst", - "mode": "manual_fields", - "snippet": "", - "auto_fields": [], - "manual_fields": ["area", "experience"], - "warning_fields": ["experience"], - "error_fields": ["area"], - "line": 1, + def test_build_review_comment_lists_missing_items(self) -> None: + config = EnhanceConfig( + meta={ + "product": MetaRule("warning", "{PRODUCT}"), + "area": MetaRule("error", ""), + "experience": MetaRule("warning", ""), }, - ] - body = build_review_comment(results, rules) - self.assertIn(SUMMARY_REVIEW_TITLE, body) - self.assertIn(SECTION_NON_EMPTY_VALUES, body) - self.assertIn("area", body) - self.assertIn("required", body) - self.assertIn("experience", body) - - def test_build_review_comment_lists_inline_suggestion_fields(self) -> None: - rules = { - "product": MetaRule("warning", "{PRODUCT}"), - "experience": MetaRule("warning", ""), - } + after_title=AFTER_TITLE_RULES, + ) results = [ { "path": "source/Page.rst", - "mode": "suggestable", - "snippet": "", - "auto_fields": ["product"], - "manual_fields": ["experience"], - "warning_fields": ["product", "experience"], - "error_fields": [], - "line": 1, + "meta_required": ["area"], + "meta_optional": ["product", "experience"], + "after_title_required": [], + "after_title_optional": ["short-description", "showmeta"], }, ] - body = build_review_comment(results, rules) + body = build_review_comment(results, config=config) self.assertIn(SUMMARY_REVIEW_TITLE, body) - self.assertIn(SECTION_INLINE_SUGGESTIONS, body) - self.assertIn(SECTION_NON_EMPTY_VALUES, body) - self.assertIn("- **`source/Page.rst`**: `product`", body) - self.assertIn("`experience` (optional)", body) + self.assertIn("source/Page.rst", body) + self.assertIn("Missing `.. meta::` fields", body) + self.assertIn("Missing after-title directives", body) + self.assertIn("required", body) + self.assertIn("{PRODUCT}", body) + self.assertIn(":order: area, content-type, experience", body) + self.assertIn(REVIEW_MARKER, body) def test_local_exit_nonzero_only_for_error_severity(self) -> None: warning_config = textwrap.dedent( @@ -498,35 +317,16 @@ def _run_with_status_file(self, content: str, *, config: str = META_ONLY_CONFIG) ) return status_path.read_text(encoding="utf-8") - def test_suggestion_note_written_with_inline_suggestions(self) -> None: + def test_status_file_writes_comment_when_issues_remain(self) -> None: status = self._run_with_status_file("Title\n=====\n") - self.assertIn("inline_suggestions=true", status) - self.assertIn("suggestion_note<<", status) - suggestion_note = _extract_multiline_output(status, "suggestion_note") + self.assertIn("has_results=true", status) + self.assertIn("has_errors=true", status) comment = _extract_multiline_output(status, "comment") - self.assertIsNotNone(suggestion_note) self.assertIsNotNone(comment) - self.assertIn("## Inline documentation suggestions", suggestion_note or "") - self.assertNotIn(REVIEW_MARKER, suggestion_note or "") self.assertIn(SUMMARY_REVIEW_TITLE, comment or "") self.assertIn(REVIEW_MARKER, comment or "") - def test_no_suggestion_note_without_inline_suggestions(self) -> None: - content = textwrap.dedent( - """ - .. meta:: - :product: ROS 2 - - Title - ===== - """ - ).lstrip() - status = self._run_with_status_file(content) - self.assertIn("inline_suggestions=false", status) - self.assertNotIn("suggestion_note<<", status) - self.assertIn("comment<<", status) - - def test_clean_file_writes_no_review_bodies(self) -> None: + def test_clean_file_writes_no_review_body(self) -> None: content = textwrap.dedent( """ .. meta:: @@ -548,7 +348,6 @@ def test_clean_file_writes_no_review_bodies(self) -> None: self.assertIn("has_results=false", status) self.assertIn("has_errors=false", status) self.assertNotIn("comment<<", status) - self.assertNotIn("suggestion_note<<", status) class TestChangedRstPaths(unittest.TestCase): diff --git a/tools/tests/test_rst_utils.py b/tools/tests/test_rst_utils.py index 0ccb737a0fd..7e0577b6321 100644 --- a/tools/tests/test_rst_utils.py +++ b/tools/tests/test_rst_utils.py @@ -24,115 +24,35 @@ sys.path.insert(0, str(_TOOLS_DIR)) from rst_utils import ( # noqa: E402 - extract_first_paragraph_after_title, - format_showmeta_block, + get_meta_fields_from_content, has_short_description_content, has_showmeta_with_order, - inject_showmeta_to_content, - wrap_first_paragraph_as_short_description, ) -class TestExtractFirstParagraph(unittest.TestCase): - def test_skips_directives_before_prose(self) -> None: +class TestMetaFields(unittest.TestCase): + def test_get_meta_fields_from_content(self) -> None: content = textwrap.dedent( """ - Title - ===== - - .. toctree:: - :maxdepth: 1 - - Page - - First paragraph here. - - Second paragraph. - """ - ).lstrip() - paragraph, span = extract_first_paragraph_after_title(content) - self.assertEqual(paragraph, "First paragraph here.") - self.assertEqual(span, (9, 9)) - - def test_finds_paragraph_after_dash_title(self) -> None: - content = textwrap.dedent( - """ - Summary - ------- - - Opening prose for the page. - """ - ).lstrip() - paragraph, span = extract_first_paragraph_after_title(content) - self.assertEqual(paragraph, "Opening prose for the page.") - self.assertEqual(span, (4, 4)) + .. meta:: + :product: ROS 2 + :area: docs - def test_returns_none_when_no_prose(self) -> None: - content = textwrap.dedent( - """ Title ===== - - .. toctree:: - Page """ ).lstrip() - paragraph, span = extract_first_paragraph_after_title(content) - self.assertIsNone(paragraph) - self.assertIsNone(span) - + fields = get_meta_fields_from_content(content) + self.assertEqual(fields["product"], "ROS 2") + self.assertEqual(fields["area"], "docs") -class TestWrapShortDescription(unittest.TestCase): - def test_preserves_one_sentence_per_line(self) -> None: - content = textwrap.dedent( - """ - First steps with ROS - learning path - ==================================== + def test_returns_empty_when_no_meta_block(self) -> None: + content = "Title\n=====\n" + self.assertEqual(get_meta_fields_from_content(content), {}) - ROS (Robot Operating System) is an open-source ecosystem that provides framework, tools, and libraries for building, deploying, running, and maintaining robotic applications. - This page presents a set of articles and hands-on activities to introduce the main concepts behind the ROS framework. - Working through these will give you the essential knowledge needed to start developing applications with ROS. - More content. - """ - ).lstrip() - paragraph, span = extract_first_paragraph_after_title(content) - self.assertEqual(span, (4, 6)) - self.assertIn("\n", paragraph or "") - updated, changed = wrap_first_paragraph_as_short_description(content) - self.assertTrue(changed) - expected_block = textwrap.dedent( - """ - .. short-description:: - ROS (Robot Operating System) is an open-source ecosystem that provides framework, tools, and libraries for building, deploying, running, and maintaining robotic applications. - This page presents a set of articles and hands-on activities to introduce the main concepts behind the ROS framework. - Working through these will give you the essential knowledge needed to start developing applications with ROS. - """ - ).strip() - self.assertIn(expected_block, updated) - self.assertIn("More content.", updated) - - def test_wraps_first_paragraph_after_equals_title(self) -> None: - content = textwrap.dedent( - """ - Title - ===== - - Opening paragraph for the article. - - More content. - """ - ).lstrip() - updated, changed = wrap_first_paragraph_as_short_description(content) - self.assertTrue(changed) - self.assertTrue(has_short_description_content(updated)) - self.assertIn(".. short-description::", updated) - self.assertIn("Opening paragraph for the article.", updated) - self.assertIn("More content.", updated) - body_after_directive = updated.split(".. short-description::", 1)[1] - self.assertNotIn("Opening paragraph for the article.", body_after_directive.split("More content.", 1)[1]) - - def test_does_not_replace_existing_short_description(self) -> None: +class TestShortDescription(unittest.TestCase): + def test_has_short_description_content(self) -> None: content = textwrap.dedent( """ Title @@ -144,11 +64,9 @@ def test_does_not_replace_existing_short_description(self) -> None: Body paragraph. """ ).lstrip() - updated, changed = wrap_first_paragraph_as_short_description(content) - self.assertFalse(changed) - self.assertEqual(updated, content) + self.assertTrue(has_short_description_content(content)) - def test_fills_empty_short_description_from_first_paragraph(self) -> None: + def test_empty_short_description_is_not_present(self) -> None: content = textwrap.dedent( """ Title @@ -156,79 +74,48 @@ def test_fills_empty_short_description_from_first_paragraph(self) -> None: .. short-description:: - Body paragraph here. + Body paragraph. """ ).lstrip() - updated, changed = wrap_first_paragraph_as_short_description(content) - self.assertTrue(changed) - self.assertTrue(has_short_description_content(updated)) - self.assertEqual(updated.count("Body paragraph here."), 1) + self.assertFalse(has_short_description_content(content)) + def test_missing_short_description(self) -> None: + content = "Title\n=====\n\nBody paragraph.\n" + self.assertFalse(has_short_description_content(content)) -class TestShowmetaHelpers(unittest.TestCase): - def test_inject_showmeta_after_short_description(self) -> None: - content = textwrap.dedent( - """ - Title - ===== - .. short-description:: - Summary text. - - Body content. - """ - ).lstrip() - updated, changed = inject_showmeta_to_content( - content, - {"order": "area, content-type, experience"}, - ) - self.assertTrue(changed) - self.assertTrue(has_showmeta_with_order(updated)) - short_desc_pos = updated.index(".. short-description::") - showmeta_pos = updated.index(".. showmeta::") - body_pos = updated.index("Body content.") - self.assertLess(short_desc_pos, showmeta_pos) - self.assertLess(showmeta_pos, body_pos) - - def test_fills_missing_order_on_existing_showmeta(self) -> None: +class TestShowmeta(unittest.TestCase): + def test_has_showmeta_with_order(self) -> None: content = textwrap.dedent( """ Title ===== .. showmeta:: - :order: + :order: area, content-type, experience Body content. """ ).lstrip() - updated, changed = inject_showmeta_to_content(content, {"order": "area"}) - self.assertTrue(changed) - self.assertIn(":order: area", updated) + self.assertTrue(has_showmeta_with_order(content)) - def test_does_not_overwrite_existing_order(self) -> None: + def test_blank_order_is_not_present(self) -> None: content = textwrap.dedent( """ Title ===== .. showmeta:: - :order: area, experience + :order: Body content. """ ).lstrip() - updated, changed = inject_showmeta_to_content( - content, - {"order": "area, content-type, experience"}, - ) - self.assertFalse(changed) - self.assertEqual(updated, content) - - def test_format_showmeta_block(self) -> None: - block = format_showmeta_block({"order": "area, content-type, experience"}) - self.assertIn(".. showmeta::", block) - self.assertIn(":order: area, content-type, experience", block) + self.assertFalse(has_showmeta_with_order(content)) + + def test_missing_showmeta(self) -> None: + content = "Title\n=====\n\nBody content.\n" + self.assertFalse(has_showmeta_with_order(content)) if __name__ == "__main__": From 911737cbf8216e43211ffc6b4362afda2d46bb3e Mon Sep 17 00:00:00 2001 From: Keith Kirkwood Date: Thu, 13 Aug 2026 14:16:53 +0100 Subject: [PATCH 23/23] OPENR-174: Clean up python modules --- Makefile | 3 +- tools/README.md | 22 +- tools/enhance_config.py | 275 +++++++++++++++++++++++ tools/ensure_enhancements.py | 278 +----------------------- tools/tests/test_ensure_enhancements.py | 15 +- tools/tests/test_rst_utils.py | 8 +- 6 files changed, 310 insertions(+), 291 deletions(-) create mode 100644 tools/enhance_config.py diff --git a/Makefile b/Makefile index 0cc3db7bbb5..01678c0f5a3 100644 --- a/Makefile +++ b/Makefile @@ -43,7 +43,8 @@ test: doc8 --ignore D001 --ignore-path $(OUT) -- $(SOURCE) test-tools: - $(PYTHON) -m pytest test/ tools/tests/ + $(PYTHON) -m pytest test/ + PYTHONPATH=$(TOOLS_DIR) $(PYTHON) -m pytest $(TOOLS_DIR)/tests/ spellcheck: git ls-files '*.md' '*.rst' | xargs codespell --config codespell.cfg diff --git a/tools/README.md b/tools/README.md index f60ed41261d..3dc36522ba0 100644 --- a/tools/README.md +++ b/tools/README.md @@ -132,13 +132,21 @@ Information for maintainers and developers working on or extending the enhanceme | File | Purpose | |------|---------| | [`rst_utils.py`](rst_utils.py) | Read-only detection of `.. meta::`, `.. short-description::`, and `.. showmeta::` directives | +| [`enhance_config.py`](enhance_config.py) | Load and validate rules from [`enhance.yaml`](enhance.yaml) | | [`enhance.yaml`](enhance.yaml) | Enhancement rules (`meta` fields and `after_title` directives) | -| [`ensure_enhancements.py`](ensure_enhancements.py) | CLI that checks enhancements from the config and builds the review comment | +| [`ensure_enhancements.py`](ensure_enhancements.py) | CLI, per-file checks, review comments, and CI outputs | | [`supersede_enhancement_reviews.sh`](supersede_enhancement_reviews.sh) | Minimise outdated bot PR reviews | | [`tests/`](tests/) | Unit tests for the tools in this directory | ### Code modules +#### `enhance_config.py` + +Loads and validates [`enhance.yaml`](enhance.yaml): + +- **`MetaRule`**, **`AfterTitleRule`**, **`EnhanceConfig`** — configuration schema +- **`load_enhance_config`** — parse and validate a config file + #### `rst_utils.py` Read-only utilities for locating Sphinx directives in RST source: @@ -149,13 +157,13 @@ Read-only utilities for locating Sphinx directives in RST source: #### Extending configuration -Add a new key under `meta` in [`enhance.yaml`](enhance.yaml) to extend metadata coverage without changing Python code. `_parse_meta_rules` accepts any key with `severity` and `value`, and the rest of the pipeline (unresolved-field detection and review hints) is driven entirely by that mapping. +Add a new key under `meta` in [`enhance.yaml`](enhance.yaml) to extend metadata coverage without changing Python code. `_parse_meta_rules` in [`enhance_config.py`](enhance_config.py) accepts any key with `severity` and `value`, and the rest of the pipeline (unresolved-field detection and review hints) is driven entirely by that mapping. -`after_title` is only partly config-driven. Its mapping shape lets you reorder or retune the two existing directives (`short-description`, `showmeta`) from YAML alone — but `_parse_after_title_rules` whitelists `supported_directives = {"short-description", "showmeta"}`, so adding a *new* after-title directive needs Python changes: +`after_title` is only partly config-driven. Its mapping shape lets you reorder or retune the two existing directives (`short-description`, `showmeta`) from YAML alone — but `_parse_after_title_rules` in [`enhance_config.py`](enhance_config.py) whitelists `supported_directives = {"short-description", "showmeta"}`, so adding a *new* after-title directive needs Python changes: -1. Add the name to `supported_directives` and its directive-specific validation in `_parse_after_title_rules`. -2. Extend `_after_title_rule_satisfied` with an "already present" check for the new directive. -3. Extend `_after_title_hint` with review text for the new directive. +1. Add the name to `supported_directives` and its directive-specific validation in `_parse_after_title_rules` ([`enhance_config.py`](enhance_config.py)). +2. Extend `_after_title_rule_satisfied` in [`ensure_enhancements.py`](ensure_enhancements.py) with an "already present" check for the new directive. +3. Extend `_after_title_hint` in [`ensure_enhancements.py`](ensure_enhancements.py) with review text for the new directive. 4. Add matching read-only detection helpers to [`rst_utils.py`](rst_utils.py) — see `has_showmeta_with_order` for the `showmeta` example. ### CLI options & Makefile targets @@ -233,7 +241,7 @@ The workflow uses [`pull_request_target`](https://docs.github.com/en/actions/usi Unit tests for this directory live in [`tests/`](tests/). From the repository root (with PyYAML installed): ```bash -python3 -m pytest tools/tests/ +PYTHONPATH=tools python3 -m pytest tools/tests/ ``` The [`test-tools`](../Makefile) target (run in [`.github/workflows/test.yml`](../.github/workflows/test.yml)) runs `pytest` on the top-level [`test/`](../test/) tree and `tools/tests/`. diff --git a/tools/enhance_config.py b/tools/enhance_config.py new file mode 100644 index 00000000000..201e66ce4d9 --- /dev/null +++ b/tools/enhance_config.py @@ -0,0 +1,275 @@ +""" +Load and validate documentation enhancement rules from ``enhance.yaml``. + +Defines the configuration schema (``meta`` and ``after_title`` mappings) used by +``ensure_enhancements.py``. +""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass +from pathlib import Path +from typing import Literal + +import yaml + +_TOOLS_DIR = Path(__file__).resolve().parent +DEFAULT_CONFIG_PATH = _TOOLS_DIR / "enhance.yaml" + +logger = logging.getLogger(__name__) + +Severity = Literal["warning", "error"] + + +@dataclass(frozen=True) +class MetaRule: + """ + A single metadata field rule from ``enhance.yaml``. + + Attributes: + severity: Advisory ``warning`` or blocking ``error`` in CI. + value: Suggested default text when missing or blank; empty when the + contributor must supply a non-empty value. + """ + + severity: Severity + value: str + + @property + def has_configured_value(self) -> bool: + """ + Return whether the rule supplies a non-empty default value. + + Returns: + ``True`` when ``value`` is non-empty after stripping whitespace. + """ + return bool(self.value.strip()) + + +@dataclass(frozen=True) +class AfterTitleRule: + """ + A single after-title directive rule from ``enhance.yaml``. + + The directive name is the key in the ``after_title`` mapping (like ``meta``). + + Attributes: + severity: Advisory ``warning`` or blocking ``error`` in CI. + content: Content source for body directives (e.g. ``first_paragraph``). + options: Option name/value pairs for option-only directives. + required_options: Option names that must be non-empty when present. + """ + + severity: Severity + content: str | None = None + options: dict[str, str] | None = None + required_options: tuple[str, ...] = () + + +@dataclass(frozen=True) +class EnhanceConfig: + """Full enhancement configuration loaded from ``enhance.yaml``.""" + + meta: dict[str, MetaRule] + after_title: dict[str, AfterTitleRule] + + +def _parse_meta_rules(meta: dict, config_path: Path) -> dict[str, MetaRule]: + """Validate and parse the ``meta`` mapping from config YAML.""" + validated: dict[str, MetaRule] = {} + for key, entry in meta.items(): + if not isinstance(key, str) or not key.strip(): + logger.error("Config %s: meta keys must be non-empty strings", config_path) + raise SystemExit(1) + if not isinstance(entry, dict): + logger.error( + "Config %s: meta entry for %r must be a mapping with severity and value", + config_path, + key, + ) + raise SystemExit(1) + severity = entry.get("severity") + if severity not in ("warning", "error"): + logger.error( + "Config %s: meta entry %r severity must be 'warning' or 'error', got %r", + config_path, + key, + severity, + ) + raise SystemExit(1) + if "value" not in entry: + logger.error("Config %s: meta entry %r must include a 'value' key", config_path, key) + raise SystemExit(1) + raw_value = entry.get("value") + if raw_value is None: + value = "" + elif isinstance(raw_value, str): + value = raw_value + else: + logger.error( + "Config %s: meta value for %r must be a string or null, got %s", + config_path, + key, + type(raw_value).__name__, + ) + raise SystemExit(1) + validated[key] = MetaRule(severity=severity, value=value) + return validated + + +def _parse_after_title_rules( + raw: object, + config_path: Path, +) -> dict[str, AfterTitleRule]: + """Validate and parse the ``after_title`` mapping from config YAML.""" + if raw is None: + return {} + if not isinstance(raw, dict): + logger.error( + "Config %s: 'after_title' must be a mapping keyed by directive name", + config_path, + ) + raise SystemExit(1) + + supported_directives = {"short-description", "showmeta"} + validated: dict[str, AfterTitleRule] = {} + for directive, entry in raw.items(): + if not isinstance(directive, str) or not directive.strip(): + logger.error( + "Config %s: after_title keys must be non-empty directive names", + config_path, + ) + raise SystemExit(1) + if directive not in supported_directives: + logger.error( + "Config %s: after_title directive %r is not supported", + config_path, + directive, + ) + raise SystemExit(1) + if not isinstance(entry, dict): + logger.error( + "Config %s: after_title entry for %r must be a mapping", + config_path, + directive, + ) + raise SystemExit(1) + severity = entry.get("severity") + if severity not in ("warning", "error"): + logger.error( + "Config %s: after_title entry %r severity must be 'warning' or 'error', got %r", + config_path, + directive, + severity, + ) + raise SystemExit(1) + + content = entry.get("content") + if content is not None and not isinstance(content, str): + logger.error( + "Config %s: after_title entry %r content must be a string", + config_path, + directive, + ) + raise SystemExit(1) + + raw_options = entry.get("options") + options: dict[str, str] | None = None + if raw_options is not None: + if not isinstance(raw_options, dict): + logger.error( + "Config %s: after_title entry %r options must be a mapping", + config_path, + directive, + ) + raise SystemExit(1) + options = {} + for opt_key, opt_value in raw_options.items(): + if not isinstance(opt_key, str) or not isinstance(opt_value, str): + logger.error( + "Config %s: after_title entry %r option keys and values must be strings", + config_path, + directive, + ) + raise SystemExit(1) + options[opt_key] = opt_value + + raw_required = entry.get("required_options") + required_options: tuple[str, ...] = () + if raw_required is not None: + if not isinstance(raw_required, list): + logger.error( + "Config %s: after_title entry %r required_options must be a list", + config_path, + directive, + ) + raise SystemExit(1) + required_options = tuple(str(item) for item in raw_required) + + if directive == "short-description": + if content != "first_paragraph": + logger.error( + "Config %s: short-description rule must use content: first_paragraph", + config_path, + ) + raise SystemExit(1) + elif directive == "showmeta": + if not options or not options.get("order", "").strip(): + logger.error( + "Config %s: showmeta rule must include options.order", + config_path, + ) + raise SystemExit(1) + if "order" not in required_options: + logger.error( + "Config %s: showmeta rule must list order in required_options", + config_path, + ) + raise SystemExit(1) + + validated[directive] = AfterTitleRule( + severity=severity, + content=content, + options=options, + required_options=required_options, + ) + return validated + + +def load_enhance_config(config_path: Path) -> EnhanceConfig: + """ + Load and validate enhancement rules from a YAML config file. + + Args: + config_path: Path to the YAML configuration file. + + Returns: + Parsed enhancement configuration. + + Raises: + SystemExit: If the file is missing, invalid, or has unusable rules. + """ + if not config_path.is_file(): + logger.error("Config file not found: %s", config_path) + raise SystemExit(1) + + try: + raw = yaml.safe_load(config_path.read_text(encoding="utf-8")) + except yaml.YAMLError as exc: + logger.error("Invalid YAML in %s: %s", config_path, exc) + raise SystemExit(1) from exc + + if not isinstance(raw, dict): + logger.error("Config %s must be a YAML mapping", config_path) + raise SystemExit(1) + + meta = raw.get("meta") + if not isinstance(meta, dict) or not meta: + logger.error("Config %s must contain a non-empty 'meta' mapping", config_path) + raise SystemExit(1) + + return EnhanceConfig( + meta=_parse_meta_rules(meta, config_path), + after_title=_parse_after_title_rules(raw.get("after_title"), config_path), + ) diff --git a/tools/ensure_enhancements.py b/tools/ensure_enhancements.py index 2af59f5681f..e24380279dc 100644 --- a/tools/ensure_enhancements.py +++ b/tools/ensure_enhancements.py @@ -16,19 +16,18 @@ import argparse import logging import subprocess -import sys -from dataclasses import dataclass from pathlib import Path -from typing import Literal, TextIO - -import yaml - -# Allow ``python3 tools/ensure_enhancements.py`` from the repository root. -_TOOLS_DIR = Path(__file__).resolve().parent -if str(_TOOLS_DIR) not in sys.path: - sys.path.insert(0, str(_TOOLS_DIR)) - -from rst_utils import ( # noqa: E402 +from typing import TextIO + +from enhance_config import ( + DEFAULT_CONFIG_PATH, + AfterTitleRule, + EnhanceConfig, + MetaRule, + Severity, + load_enhance_config, +) +from rst_utils import ( get_meta_fields_from_content, has_short_description_content, has_showmeta_with_order, @@ -36,9 +35,7 @@ logger = logging.getLogger(__name__) -DEFAULT_CONFIG_PATH = _TOOLS_DIR / "enhance.yaml" RST_EXTENSION = ".rst" -Severity = Literal["warning", "error"] # Hidden marker in review bodies so CI can find and supersede prior bot reviews. REVIEW_MARKER_ID = "ros2-doc-enhance-ensure" @@ -48,259 +45,6 @@ SUMMARY_REVIEW_TITLE = "## Documentation enhancements" -@dataclass(frozen=True) -class AfterTitleRule: - """ - A single after-title directive rule from ``enhance.yaml``. - - The directive name is the key in the ``after_title`` mapping (like ``meta``). - - Attributes: - severity: Advisory ``warning`` or blocking ``error`` in CI. - content: Content source for body directives (e.g. ``first_paragraph``). - options: Option name/value pairs for option-only directives. - required_options: Option names that must be non-empty when present. - """ - - severity: Severity - content: str | None = None - options: dict[str, str] | None = None - required_options: tuple[str, ...] = () - - -@dataclass(frozen=True) -class EnhanceConfig: - """Full enhancement configuration loaded from ``enhance.yaml``.""" - - meta: dict[str, MetaRule] - after_title: dict[str, AfterTitleRule] - - -@dataclass(frozen=True) -class MetaRule: - """ - A single metadata field rule from ``enhance.yaml``. - - Attributes: - severity: Advisory ``warning`` or blocking ``error`` in CI. - value: Default text to inject when missing or blank; empty when the - contributor must supply a non-empty value. - """ - - severity: Severity - value: str - - @property - def has_configured_value(self) -> bool: - """ - Return whether the rule supplies a non-empty default value. - - Returns: - ``True`` when ``value`` is non-empty after stripping whitespace. - """ - return bool(self.value.strip()) - - -def _parse_meta_rules(meta: dict, config_path: Path) -> dict[str, MetaRule]: - """Validate and parse the ``meta`` mapping from config YAML.""" - validated: dict[str, MetaRule] = {} - for key, entry in meta.items(): - if not isinstance(key, str) or not key.strip(): - logger.error("Config %s: meta keys must be non-empty strings", config_path) - raise SystemExit(1) - if not isinstance(entry, dict): - logger.error( - "Config %s: meta entry for %r must be a mapping with severity and value", - config_path, - key, - ) - raise SystemExit(1) - severity = entry.get("severity") - if severity not in ("warning", "error"): - logger.error( - "Config %s: meta entry %r severity must be 'warning' or 'error', got %r", - config_path, - key, - severity, - ) - raise SystemExit(1) - if "value" not in entry: - logger.error("Config %s: meta entry %r must include a 'value' key", config_path, key) - raise SystemExit(1) - raw_value = entry.get("value") - if raw_value is None: - value = "" - elif isinstance(raw_value, str): - value = raw_value - else: - logger.error( - "Config %s: meta value for %r must be a string or null, got %s", - config_path, - key, - type(raw_value).__name__, - ) - raise SystemExit(1) - validated[key] = MetaRule(severity=severity, value=value) - return validated - - -def _parse_after_title_rules( - raw: object, - config_path: Path, -) -> dict[str, AfterTitleRule]: - """Validate and parse the ``after_title`` mapping from config YAML.""" - if raw is None: - return {} - if not isinstance(raw, dict): - logger.error( - "Config %s: 'after_title' must be a mapping keyed by directive name", - config_path, - ) - raise SystemExit(1) - - supported_directives = {"short-description", "showmeta"} - validated: dict[str, AfterTitleRule] = {} - for directive, entry in raw.items(): - if not isinstance(directive, str) or not directive.strip(): - logger.error( - "Config %s: after_title keys must be non-empty directive names", - config_path, - ) - raise SystemExit(1) - if directive not in supported_directives: - logger.error( - "Config %s: after_title directive %r is not supported", - config_path, - directive, - ) - raise SystemExit(1) - if not isinstance(entry, dict): - logger.error( - "Config %s: after_title entry for %r must be a mapping", - config_path, - directive, - ) - raise SystemExit(1) - severity = entry.get("severity") - if severity not in ("warning", "error"): - logger.error( - "Config %s: after_title entry %r severity must be 'warning' or 'error', got %r", - config_path, - directive, - severity, - ) - raise SystemExit(1) - - content = entry.get("content") - if content is not None and not isinstance(content, str): - logger.error( - "Config %s: after_title entry %r content must be a string", - config_path, - directive, - ) - raise SystemExit(1) - - raw_options = entry.get("options") - options: dict[str, str] | None = None - if raw_options is not None: - if not isinstance(raw_options, dict): - logger.error( - "Config %s: after_title entry %r options must be a mapping", - config_path, - directive, - ) - raise SystemExit(1) - options = {} - for opt_key, opt_value in raw_options.items(): - if not isinstance(opt_key, str) or not isinstance(opt_value, str): - logger.error( - "Config %s: after_title entry %r option keys and values must be strings", - config_path, - directive, - ) - raise SystemExit(1) - options[opt_key] = opt_value - - raw_required = entry.get("required_options") - required_options: tuple[str, ...] = () - if raw_required is not None: - if not isinstance(raw_required, list): - logger.error( - "Config %s: after_title entry %r required_options must be a list", - config_path, - directive, - ) - raise SystemExit(1) - required_options = tuple(str(item) for item in raw_required) - - if directive == "short-description": - if content != "first_paragraph": - logger.error( - "Config %s: short-description rule must use content: first_paragraph", - config_path, - ) - raise SystemExit(1) - elif directive == "showmeta": - if not options or not options.get("order", "").strip(): - logger.error( - "Config %s: showmeta rule must include options.order", - config_path, - ) - raise SystemExit(1) - if "order" not in required_options: - logger.error( - "Config %s: showmeta rule must list order in required_options", - config_path, - ) - raise SystemExit(1) - - validated[directive] = AfterTitleRule( - severity=severity, - content=content, - options=options, - required_options=required_options, - ) - return validated - - -def load_enhance_config(config_path: Path) -> EnhanceConfig: - """ - Load and validate enhancement rules from a YAML config file. - - Args: - config_path: Path to the YAML configuration file. - - Returns: - Parsed enhancement configuration. - - Raises: - SystemExit: If the file is missing, invalid, or has unusable rules. - """ - if not config_path.is_file(): - logger.error("Config file not found: %s", config_path) - raise SystemExit(1) - - try: - raw = yaml.safe_load(config_path.read_text(encoding="utf-8")) - except yaml.YAMLError as exc: - logger.error("Invalid YAML in %s: %s", config_path, exc) - raise SystemExit(1) from exc - - if not isinstance(raw, dict): - logger.error("Config %s must be a YAML mapping", config_path) - raise SystemExit(1) - - meta = raw.get("meta") - if not isinstance(meta, dict) or not meta: - logger.error("Config %s must contain a non-empty 'meta' mapping", config_path) - raise SystemExit(1) - - return EnhanceConfig( - meta=_parse_meta_rules(meta, config_path), - after_title=_parse_after_title_rules(raw.get("after_title"), config_path), - ) - - def _unresolved_fields(content: str, rules: dict[str, MetaRule]) -> list[str]: """ Find configured meta fields that are absent or blank in RST content. diff --git a/tools/tests/test_ensure_enhancements.py b/tools/tests/test_ensure_enhancements.py index 90a2bdf9aac..f6680058ab1 100644 --- a/tools/tests/test_ensure_enhancements.py +++ b/tools/tests/test_ensure_enhancements.py @@ -14,28 +14,25 @@ from __future__ import annotations -import sys import tempfile import textwrap import unittest from pathlib import Path from unittest import mock -_TOOLS_DIR = Path(__file__).resolve().parent.parent -if str(_TOOLS_DIR) not in sys.path: - sys.path.insert(0, str(_TOOLS_DIR)) - -from ensure_enhancements import ( # noqa: E402 - REVIEW_MARKER, - SUMMARY_REVIEW_TITLE, +from enhance_config import ( AfterTitleRule, EnhanceConfig, MetaRule, + load_enhance_config, +) +from ensure_enhancements import ( + REVIEW_MARKER, + SUMMARY_REVIEW_TITLE, _unresolved_fields, build_review_comment, changed_rst_paths, ensure_enhancements_in_file, - load_enhance_config, main, ) diff --git a/tools/tests/test_rst_utils.py b/tools/tests/test_rst_utils.py index 7e0577b6321..b810004221b 100644 --- a/tools/tests/test_rst_utils.py +++ b/tools/tests/test_rst_utils.py @@ -14,16 +14,10 @@ from __future__ import annotations -import sys import textwrap import unittest -from pathlib import Path -_TOOLS_DIR = Path(__file__).resolve().parent.parent -if str(_TOOLS_DIR) not in sys.path: - sys.path.insert(0, str(_TOOLS_DIR)) - -from rst_utils import ( # noqa: E402 +from rst_utils import ( get_meta_fields_from_content, has_short_description_content, has_showmeta_with_order,