feat(coverage): gap detector + Wikipedia sources for CPU/GPU/smartphone

Implements the framework called for in #1: pull upstream catalogs, slugify,
diff against the curated TechAPI dataset, emit a Markdown report.

Modules
- app/coverage/normalize.py — vendor name → kebab-case slug with manufacturer
  prefix stripping (Intel/AMD/NVIDIA/Samsung/Apple/...).
- app/coverage/curated.py   — walks TECHAPI_DATA_DIR for curated slugs.
- app/coverage/report.py    — diffs upstream vs curated and renders Markdown
  grouped by category × manufacturer, with per-row source links and a top-N cap.
- app/coverage/__main__.py  — CLI entry. `python -m app.coverage --output …`.

Sources (3, all Wikipedia REST API + BeautifulSoup)
- wikipedia_cpu.py        — Intel Core/Xeon/Atom/Pentium/Celeron + AMD Ryzen/
                            EPYC/Threadripper/Opteron list pages.
- wikipedia_gpu.py        — NVIDIA/AMD/Intel GPU list pages.
- wikipedia_smartphone.py — Samsung/Apple/Google/OnePlus/Xiaomi flagship lists.

CI
- .github/workflows/coverage-report.yml — Mondays 06:23 UTC. Builds the report,
  uploads it as an artifact, and syncs a sticky issue. Posts to TechAPI when
  the TECHAPI_PR_TOKEN secret is set; otherwise opens on this repo.

Tests (21 passing locally)
- normalize, curated loader, report builder (with synthetic points), Wikipedia
  table parser (with vendored HTML — no network).

Refs #1

Author: Seungpyo1007

.github/workflows/coverage-report.yml

@@ -0,0 +1,61 @@
+name: coverage-report
+
+# Weekly: pull upstream catalogs, diff vs the curated TechAPI dataset, post
+# the gap list as a sticky issue (auto-updates the existing one each run).
+on:
+  schedule:
+    - cron: "23 6 * * 1"   # Mondays 06:23 UTC, after refresh-data (06:17)
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  issues: write
+
+jobs:
+  report:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/checkout@v4
+        with:
+          repository: GetTechAPI/TechAPI
+          path: TechAPI
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+          cache: pip
+
+      - name: Install
+        run: pip install -e .
+
+      - name: Build coverage report
+        env:
+          TECHAPI_DATA_DIR: ${{ github.workspace }}/TechAPI/data
+        run: python -m app.coverage --output coverage-report.md
+
+      - name: Upload report artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-report
+          path: coverage-report.md
+
+      # Sticky issue: search for an open issue with the well-known title and
+      # update it; create one if missing. Defaults to this repo; if a PAT
+      # scoped to TechAPI is provided as TECHAPI_PR_TOKEN, posts there instead.
+      - name: Sync sticky coverage issue
+        env:
+          GH_TOKEN: ${{ secrets.TECHAPI_PR_TOKEN || secrets.GITHUB_TOKEN }}
+          TARGET_REPO: ${{ secrets.TECHAPI_PR_TOKEN && 'GetTechAPI/TechAPI' || github.repository }}
+        run: |
+          set -euo pipefail
+          TITLE="Coverage gaps (auto-generated)"
+          BODY="$(cat coverage-report.md)"
+          NUMBER=$(gh issue list --repo "$TARGET_REPO" --state open \
+            --search "in:title \"$TITLE\"" --json number --jq '.[0].number // empty')
+          if [ -z "${NUMBER:-}" ]; then
+            gh issue create --repo "$TARGET_REPO" --title "$TITLE" --body "$BODY"
+          else
+            gh issue edit "$NUMBER" --repo "$TARGET_REPO" --body "$BODY"
+          fi

app/coverage/__init__.py

@@ -0,0 +1,7 @@
+"""Coverage gap detector.
+
+Diffs the curated TechAPI dataset against upstream catalogs (Wikipedia, vendor
+product pages) and surfaces SKUs that are present upstream but missing locally.
+
+Entry point: ``python -m app.coverage`` — writes ``coverage-report.md``.
+"""

app/coverage/__main__.py

@@ -0,0 +1,59 @@
+"""Coverage CLI entry point.
+
+::
+
+    python -m app.coverage [--output coverage-report.md]
+
+Fetches every wired source, diffs the union against the curated dataset
+(found via ``TECHAPI_DATA_DIR``), and writes a Markdown report.
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from collections.abc import Iterator
+from pathlib import Path
+
+from .report import build_report
+from .sources.base import CoveragePoint, Source
+from .sources.wikipedia_cpu import WikipediaCpu
+from .sources.wikipedia_gpu import WikipediaGpu
+from .sources.wikipedia_smartphone import WikipediaSmartphone
+
+DEFAULT_SOURCES: list[Source] = [
+    WikipediaCpu(),
+    WikipediaGpu(),
+    WikipediaSmartphone(),
+]
+
+
+def collect(sources: list[Source]) -> Iterator[CoveragePoint]:
+    for source in sources:
+        yield from source.fetch()
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(prog="app.coverage")
+    parser.add_argument(
+        "--output",
+        type=Path,
+        default=Path("coverage-report.md"),
+        help="Markdown report destination (default: coverage-report.md).",
+    )
+    parser.add_argument(
+        "--top",
+        type=int,
+        default=30,
+        help="Max entries per category × manufacturer in the report.",
+    )
+    args = parser.parse_args(argv)
+
+    points = list(collect(DEFAULT_SOURCES))
+    args.output.write_text(build_report(points, top_n=args.top), encoding="utf-8")
+    print(f"wrote {args.output} ({len(points)} upstream points)")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))

app/coverage/curated.py

@@ -0,0 +1,36 @@
+"""Read curated TechAPI slugs from disk.
+
+Resolves the dataset location the same way as ``app.validate`` / ``app.seed``:
+``TECHAPI_DATA_DIR`` env var, falling back to ``../TechAPI/data`` next to this
+repo.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+
+
+def data_dir() -> Path:
+    default = Path(__file__).resolve().parent.parent.parent.parent / "TechAPI" / "data"
+    return Path(os.environ.get("TECHAPI_DATA_DIR", default))
+
+
+def curated_slugs(category: str, manufacturer: str | None = None) -> set[str]:
+    """All slugs found under ``data/<category>[/<manufacturer>]/**/*.json``."""
+    root = data_dir() / category
+    if manufacturer:
+        root = root / manufacturer
+    if not root.exists():
+        return set()
+    slugs: set[str] = set()
+    for path in root.rglob("*.json"):
+        try:
+            record = json.loads(path.read_text(encoding="utf-8"))
+        except (json.JSONDecodeError, OSError):
+            continue
+        slug = record.get("slug")
+        if isinstance(slug, str):
+            slugs.add(slug)
+    return slugs

app/coverage/normalize.py

@@ -0,0 +1,60 @@
+"""Vendor name → TechAPI kebab-case slug.
+
+The same normalization runs over both upstream catalog entries and (when needed)
+curated names, so equivalent SKUs collapse to the same slug regardless of the
+source's punctuation/casing/manufacturer prefix.
+"""
+
+from __future__ import annotations
+
+import re
+import unicodedata
+
+# Known prefixes per manufacturer slug. Compared lowercased, longest first so
+# "Advanced Micro Devices" wins over "AMD" when both could match.
+_MANUFACTURER_PREFIXES: dict[str, list[str]] = {
+    "intel": ["intel"],
+    "amd": ["advanced micro devices", "amd"],
+    "nvidia": ["nvidia corporation", "nvidia"],
+    "samsung": ["samsung electronics", "samsung"],
+    "apple": ["apple inc.", "apple"],
+    "qualcomm": ["qualcomm technologies", "qualcomm"],
+    "mediatek": ["mediatek"],
+    "ibm": ["ibm"],
+    "motorola": ["motorola"],
+    "google": ["google"],
+    "huawei": ["huawei"],
+    "xiaomi": ["xiaomi"],
+    "oppo": ["oppo"],
+    "vivo": ["vivo"],
+    "oneplus": ["oneplus"],
+    "lg": ["lg electronics", "lg"],
+    "sony": ["sony"],
+    "asus": ["asustek", "asus"],
+    "msi": ["msi"],
+    "gigabyte": ["gigabyte"],
+}
+
+_SEPARATOR_RE = re.compile(r"[^a-z0-9]+")
+_COLLAPSE_RE = re.compile(r"-+")
+
+
+def slugify(name: str, manufacturer: str | None = None) -> str:
+    """Normalize a vendor-style name to a kebab-case slug.
+
+    Strips a known manufacturer prefix when ``manufacturer`` is given so that
+    "Intel Core i9-14900K" matches the existing TechAPI slug "core-i9-14900k".
+    """
+    text = unicodedata.normalize("NFKD", name).encode("ascii", "ignore").decode("ascii")
+    text = text.strip().lower()
+    if manufacturer:
+        for prefix in _MANUFACTURER_PREFIXES.get(manufacturer, [manufacturer]):
+            if text.startswith(prefix + " "):
+                text = text[len(prefix) + 1 :]
+                break
+            if text == prefix:
+                text = ""
+                break
+    text = _SEPARATOR_RE.sub("-", text)
+    text = _COLLAPSE_RE.sub("-", text).strip("-")
+    return text

app/coverage/report.py

@@ -0,0 +1,51 @@
+"""Build the coverage-gap Markdown report from collected ``CoveragePoint`` s."""
+
+from __future__ import annotations
+
+from collections import defaultdict
+from collections.abc import Iterable
+
+from .curated import curated_slugs
+from .sources.base import CoveragePoint
+
+
+def build_report(points: Iterable[CoveragePoint], *, top_n: int = 30) -> str:
+    """Render the report. ``top_n`` caps lines per category × manufacturer."""
+    # Aggregate upstream slugs and remember the first (name, url) seen per slug.
+    upstream: dict[tuple[str, str], set[str]] = defaultdict(set)
+    meta: dict[tuple[str, str, str], tuple[str, str]] = {}
+    for point in points:
+        upstream[(point.category, point.manufacturer)].add(point.slug)
+        meta.setdefault((point.category, point.manufacturer, point.slug), (point.name, point.url))
+
+    out: list[str] = ["# TechAPI coverage gaps", ""]
+    out.append("_Auto-generated by TechEngine. Top entries shown per category × manufacturer._")
+    out.append("")
+
+    total_missing = 0
+    section_lines: list[str] = []
+    for (category, manufacturer) in sorted(upstream.keys()):
+        upstream_slugs = upstream[(category, manufacturer)]
+        curated = curated_slugs(category, manufacturer)
+        missing = sorted(upstream_slugs - curated)
+        total_missing += len(missing)
+        section_lines.append(
+            f"## {category} / {manufacturer} — {len(missing)} missing "
+            f"(upstream {len(upstream_slugs)}, curated {len(curated)})"
+        )
+        if not missing:
+            section_lines.append("_(none)_")
+            section_lines.append("")
+            continue
+        for slug in missing[:top_n]:
+            name, url = meta.get((category, manufacturer, slug), (slug, ""))
+            link = f" ([source]({url}))" if url else ""
+            section_lines.append(f"- `{slug}` — {name}{link}")
+        if len(missing) > top_n:
+            section_lines.append(f"_… and {len(missing) - top_n} more._")
+        section_lines.append("")
+
+    out.append(f"**Total missing:** {total_missing}")
+    out.append("")
+    out.extend(section_lines)
+    return "\n".join(out).rstrip() + "\n"

app/coverage/sources/__init__.py

@@ -0,0 +1,6 @@
+"""Per-source upstream catalog adapters.
+
+Each module exports one or more classes implementing the ``Source`` protocol
+from ``base.py``: they fetch a remote catalog and yield ``CoveragePoint``
+records that ``app.coverage.report`` then diffs against the curated dataset.
+"""

app/coverage/sources/base.py

@@ -0,0 +1,34 @@
+"""Shared types for coverage sources."""
+
+from __future__ import annotations
+
+from collections.abc import Iterator
+from dataclasses import dataclass
+from typing import Protocol
+
+
+@dataclass(frozen=True)
+class CoveragePoint:
+    """One SKU surfaced by an upstream catalog.
+
+    Sources should set ``slug`` via :func:`app.coverage.normalize.slugify` so
+    that comparisons with curated slugs are apples-to-apples.
+    """
+
+    category: str  # "cpu" | "gpu" | "smartphone" | "soc"
+    manufacturer: str  # brand slug, e.g. "intel"
+    name: str  # raw display name, e.g. "Intel Core i9-14900K"
+    slug: str  # normalized, e.g. "core-i9-14900k"
+    source: str  # short ID, e.g. "wikipedia:List_of_Intel_Core_processors"
+    url: str  # link back to the source page
+
+
+class Source(Protocol):
+    """Coverage source contract."""
+
+    name: str
+    description: str
+
+    def fetch(self) -> Iterator[CoveragePoint]:
+        """Yield every SKU the source surfaces. May silently skip failed pages."""
+        ...

app/coverage/sources/wikipedia.py

@@ -0,0 +1,43 @@
+"""Wikipedia fetch + parse helpers shared across category-specific sources.
+
+Uses the public REST API (``en.wikipedia.org/api/rest_v1/page/html/<title>``)
+which returns prerendered HTML — easier to parse than wikitext and stable.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterator
+
+import httpx
+from bs4 import BeautifulSoup
+
+WIKI_REST_HTML = "https://en.wikipedia.org/api/rest_v1/page/html/{title}"
+USER_AGENT = "TechEngine-Coverage/0.1 (+https://github.com/GetTechAPI/TechEngine)"
+
+
+def fetch_wikipedia_html(page_title: str, *, timeout: float = 30.0) -> str:
+    """Download the parsed HTML for a Wikipedia page."""
+    url = WIKI_REST_HTML.format(title=page_title)
+    headers = {"User-Agent": USER_AGENT}
+    with httpx.Client(headers=headers, timeout=timeout, follow_redirects=True) as client:
+        response = client.get(url)
+        response.raise_for_status()
+        return response.text
+
+
+def wikitable_first_cells(html: str) -> Iterator[str]:
+    """Yield the text of the first cell of every row in every ``table.wikitable``.
+
+    Most ``List_of_*_processors`` and ``List_of_*_graphics_processing_units``
+    pages put the model name in column 1. Header rows whose first cell is a
+    ``<th>`` are still emitted; the slug normalizer filters obvious non-models.
+    """
+    soup = BeautifulSoup(html, "html.parser")
+    for table in soup.select("table.wikitable"):
+        for row in table.select("tr"):
+            cell = row.find(["th", "td"])
+            if not cell:
+                continue
+            text = cell.get_text(" ", strip=True)
+            if text:
+                yield text