Fix CI: switch to data/models.json with paid & free multipliers

scripts/update-model-multipliers.py

@@ -1,115 +1,117 @@
 #!/usr/bin/env python3
 """
-Regenerate `src/lib/model-multipliers.generated.ts` from the latest release of
-rajbos/github-copilot-model-notifier.
+Regenerate `src/lib/model-multipliers.generated.ts` from the live model data
+published by rajbos/github-copilot-model-notifier.
 
-The source repo publishes a markdown table of current models in the release body
-under a `### Current Models` heading. This script parses that table and fully
-overwrites the generated TypeScript file. The companion file
-`model-multipliers.legacy.ts` contains hand-maintained backward-compat entries
-and is never touched here.
+Model data is read from `data/models.json` in that repo (always at `main`),
+which contains structured fields including `multiplier_paid` and
+`multiplier_free`.  The latest GitHub release is still fetched for its
+`tag_name` so we can record a meaningful provenance comment.
+
+The companion file `model-multipliers.legacy.ts` contains hand-maintained
+backward-compat entries and is never touched here.
 
 Usage:
     python scripts/update-model-multipliers.py
 
 Env:
-    GITHUB_TOKEN  Optional. Used to authenticate the GitHub API request.
+    GITHUB_TOKEN  Optional. Used to authenticate GitHub API requests.
 """
 
 from __future__ import annotations
 
 import json
 import os
-import re
 import sys
 import urllib.error
 import urllib.request
 from pathlib import Path
 
+MODELS_JSON_URL = (
+    "https://raw.githubusercontent.com/rajbos/"
+    "github-copilot-model-notifier/main/data/models.json"
+)
 RELEASE_URL = (
     "https://api.github.com/repos/rajbos/"
     "github-copilot-model-notifier/releases/latest"
 )
 REPO_ROOT = Path(__file__).resolve().parent.parent
 GENERATED_PATH = REPO_ROOT / "src" / "lib" / "model-multipliers.generated.ts"
 
+# Sentinel: model not available on this plan
+NOT_APPLICABLE: float = -1.0
+
 
-def fetch_latest_release() -> dict:
-    """Fetch the latest release JSON from the source repo."""
-    headers = {
-        "Accept": "application/vnd.github+json",
+def _fetch(url: str, *, github_api: bool = False) -> bytes:
+    """Fetch *url* and return the raw response body."""
+    headers: dict[str, str] = {
         "User-Agent": "github-copilot-premium-reqs-usage-updater",
     }
-    token = os.environ.get("GITHUB_TOKEN")
-    if token:
-        headers["Authorization"] = f"Bearer {token}"
+    if github_api:
+        headers["Accept"] = "application/vnd.github+json"
+        token = os.environ.get("GITHUB_TOKEN")
+        if token:
+            headers["Authorization"] = f"Bearer {token}"
 
-    req = urllib.request.Request(RELEASE_URL, headers=headers)
+    req = urllib.request.Request(url, headers=headers)
     try:
         with urllib.request.urlopen(req, timeout=30) as resp:
-            return json.loads(resp.read().decode("utf-8"))
+            return resp.read()
     except urllib.error.HTTPError as e:
-        sys.stderr.write(
-            f"HTTP error {e.code} fetching latest release: {e.reason}\n"
-        )
+        sys.stderr.write(f"HTTP error {e.code} fetching {url}: {e.reason}\n")
         raise
 
 
-def parse_models_table(body: str) -> dict[str, float]:
-    """Parse the `### Current Models` markdown table from the release body.
+def fetch_tag_name() -> str:
+    """Return the tag_name of the latest release (used for provenance only)."""
+    data = json.loads(_fetch(RELEASE_URL, github_api=True).decode("utf-8"))
+    return data.get("tag_name", "<unknown>")
 
-    Returns a mapping of model name -> paid multiplier (float).
-    Multiplier 'Not applicable' is mapped to 0.
-    """
-    if not body:
-        raise ValueError("Release body is empty; cannot parse models table.")
 
-    m = re.search(
-        r"###\s+Current Models\s*\n(.+?)(?=\n#{1,6}\s|\Z)",
-        body,
-        re.DOTALL,
-    )
-    if not m:
-        raise ValueError(
-            "Could not find '### Current Models' section in release body."
+def parse_multiplier(raw: str, model_name: str, field: str) -> float:
+    """Convert a raw multiplier string to a float.
+
+    Rules:
+      ""                  -> 1.0   (no data yet; treat as 1 premium request)
+      "Not applicable"    -> -1.0  (model not available on this plan)
+      "0", "0.33", …      -> the numeric value
+    """
+    raw = raw.strip()
+    if raw == "":
+        return 1.0
+    if raw.lower() == "not applicable":
+        return NOT_APPLICABLE
+    try:
+        return float(raw)
+    except ValueError:
+        sys.stderr.write(
+            f"Warning: {model_name!r} {field}={raw!r} unparseable, defaulting to 1\n"
         )
-    section = m.group(1)
+        return 1.0
 
-    models: dict[str, float] = {}
-    for line in section.splitlines():
-        line = line.strip()
-        if not line.startswith("|") or not line.endswith("|"):
-            continue
-        cells = [c.strip() for c in line.strip("|").split("|")]
-        if len(cells) < 3:
-            continue
-        if cells[0].lower() == "model":
-            continue
-        if set(cells[0]) <= set("-: "):
-            continue
-
-        name = cells[0]
-        raw_mult = cells[2]
-        if raw_mult.lower() == "not applicable":
-            mult: float = 0.0
-        else:
-            try:
-                mult = float(raw_mult)
-            except ValueError:
-                sys.stderr.write(
-                    f"Warning: skipping {name!r} - "
-                    f"unparseable multiplier {raw_mult!r}\n"
-                )
-                continue
-        models[name] = mult
+
+def fetch_models() -> dict[str, tuple[float, float]]:
+    """Fetch data/models.json and return {name: (paid, free)} mappings."""
+    raw = _fetch(MODELS_JSON_URL).decode("utf-8")
+    data: dict = json.loads(raw)
+
+    models: dict[str, tuple[float, float]] = {}
+    for name, info in data.items():
+        paid = parse_multiplier(
+            info.get("multiplier_paid", ""), name, "multiplier_paid"
+        )
+        free = parse_multiplier(
+            info.get("multiplier_free", ""), name, "multiplier_free"
+        )
+        models[name] = (paid, free)
 
     if not models:
-        raise ValueError("Parsed zero models from release body.")
+        raise ValueError("Fetched zero models from data/models.json.")
     return models
 
 
 def format_multiplier(value: float) -> str:
-    """Render a multiplier as JS/TS literal: drop .0 for whole numbers."""
+    """Render a multiplier as a JS/TS literal."""
     if value == int(value):
         return str(int(value))
     return repr(value)
@@ -121,31 +123,66 @@ def js_string_literal(s: str) -> str:
     return f"'{escaped}'"
 
 
-def render_generated_file(models: dict[str, float], tag_name: str) -> str:
+def render_generated_file(
+    models: dict[str, tuple[float, float]], tag_name: str
+) -> str:
     sorted_names = sorted(models.keys(), key=str.lower)
-    default_names = [n for n in sorted_names if models[n] == 0]
+    # Default models: paid multiplier == 0 (included in subscription)
+    default_names = [n for n in sorted_names if models[n][0] == 0]
 
     lines = [
         "// AUTO-GENERATED FILE — DO NOT EDIT BY HAND.",
         "//",
-        "// Source: https://github.com/rajbos/github-copilot-model-notifier (latest release)",
-        "// Updated by: scripts/update-model-multipliers.py (run daily via GitHub Actions)",
+        "// Source: https://github.com/rajbos/github-copilot-model-notifier"
+        " (data/models.json)",
+        "// Updated by: scripts/update-model-multipliers.py"
+        " (run daily via GitHub Actions)",
         "//",
         "// To make manual changes, edit `model-multipliers.legacy.ts` instead.",
         "",
-        f"export const CURRENT_MODELS_SOURCE_RELEASE = {js_string_literal(tag_name)};",
+        f"export const CURRENT_MODELS_SOURCE_RELEASE ="
+        f" {js_string_literal(tag_name)};",
         "",
-        "export const CURRENT_MODEL_MULTIPLIERS: Record<string, number> = {",
+        "// Multipliers for paid plans (Business / Enterprise / Pro / Pro+).",
+        "// -1 = not available on this plan; 0 = included (free); >0 = premium.",
+        "export const CURRENT_MODEL_MULTIPLIERS_PAID:"
+        " Record<string, number> = {",
     ]
     for name in sorted_names:
         lines.append(
-            f"  {js_string_literal(name)}: {format_multiplier(models[name])},"
+            f"  {js_string_literal(name)}:"
+            f" {format_multiplier(models[name][0])},"
+        )
+    lines.append("};")
+    lines.append("")
+    lines.append(
+        "// Multipliers for Copilot Free plan."
+    )
+    lines.append(
+        "// -1 = not available on free plan; 0 = included; >0 = premium."
+    )
+    lines.append(
+        "export const CURRENT_MODEL_MULTIPLIERS_FREE:"
+        " Record<string, number> = {"
+    )
+    for name in sorted_names:
+        lines.append(
+            f"  {js_string_literal(name)}:"
+            f" {format_multiplier(models[name][1])},"
         )
     lines.append("};")
     lines.append("")
     lines.append(
-        "// Models with a 0x multiplier (free) are treated as \"Default\" "
-        "and grouped together."
+        "// Backward-compat alias — defaults to paid-plan multipliers."
+    )
+    lines.append(
+        "export const CURRENT_MODEL_MULTIPLIERS ="
+        " CURRENT_MODEL_MULTIPLIERS_PAID;"
+    )
+    lines.append("")
+    lines.append(
+        "// Models with a 0x paid multiplier are included in the subscription"
+        ' and grouped as "Default".'
     )
     lines.append("export const CURRENT_DEFAULT_MODELS: string[] = [")
     for name in default_names:
@@ -155,18 +192,26 @@ def render_generated_file(models: dict[str, float], tag_name: str) -> str:
     return "\n".join(lines)
 
 
-def parse_existing_models(content: str) -> dict[str, float]:
-    """Parse the existing CURRENT_MODEL_MULTIPLIERS object for a diff summary."""
+def parse_existing_paid(content: str) -> dict[str, float]:
+    """Parse the existing CURRENT_MODEL_MULTIPLIERS_PAID block for a diff."""
+    import re
     m = re.search(
-        r"CURRENT_MODEL_MULTIPLIERS[^=]*=\s*\{(.*?)\};",
+        r"CURRENT_MODEL_MULTIPLIERS_PAID[^=]*=\s*\{(.*?)\};",
         content,
         re.DOTALL,
     )
+    if not m:
+        # Fall back to the old single-block format
+        m = re.search(
+            r"CURRENT_MODEL_MULTIPLIERS[^=P][^=]*=\s*\{(.*?)\};",
+            content,
+            re.DOTALL,
+        )
     if not m:
         return {}
     body = m.group(1)
     models: dict[str, float] = {}
-    entry_re = re.compile(r"'((?:\\'|[^'])*)'\s*:\s*([0-9.]+)")
+    entry_re = re.compile(r"'((?:\\'|[^'])*)'\s*:\s*(-?[0-9.]+)")
     for line in body.splitlines():
         line = line.split("//", 1)[0]
         em = entry_re.search(line)
@@ -180,53 +225,53 @@ def parse_existing_models(content: str) -> dict[str, float]:
 
 
 def print_diff_summary(
-    old: dict[str, float], new: dict[str, float], tag_name: str
+    old: dict[str, float], new: dict[str, tuple[float, float]], tag_name: str
 ) -> None:
-    added = sorted(set(new) - set(old), key=str.lower)
-    removed = sorted(set(old) - set(new), key=str.lower)
+    new_paid = {n: v[0] for n, v in new.items()}
+    added = sorted(set(new_paid) - set(old), key=str.lower)
+    removed = sorted(set(old) - set(new_paid), key=str.lower)
     changed = sorted(
-        (n for n in set(new) & set(old) if old[n] != new[n]), key=str.lower
+        (n for n in set(new_paid) & set(old) if old[n] != new_paid[n]),
+        key=str.lower,
     )
 
-    print(f"Source release: {tag_name}")
+    print(f"Source: {tag_name}")
     if not (added or removed or changed):
         print("No model changes detected.")
         return
     if added:
         print("Added:")
         for n in added:
-            print(f"  + {n} = {format_multiplier(new[n])}")
+            print(f"  + {n} (paid={format_multiplier(new_paid[n])},"
+                  f" free={format_multiplier(new[n][1])})")
     if removed:
         print("Removed:")
         for n in removed:
-            print(f"  - {n} (was {format_multiplier(old[n])})")
+            print(f"  - {n} (was paid={format_multiplier(old[n])})")
     if changed:
         print("Changed:")
         for n in changed:
             print(
-                f"  ~ {n}: {format_multiplier(old[n])} -> "
-                f"{format_multiplier(new[n])}"
+                f"  ~ {n}: paid {format_multiplier(old[n])}"
+                f" -> {format_multiplier(new_paid[n])}"
             )
 
 
 def main() -> int:
-    release = fetch_latest_release()
-    tag_name = release.get("tag_name", "<unknown>")
-    body = release.get("body") or ""
-
-    new_models = parse_models_table(body)
+    tag_name = fetch_tag_name()
+    new_models = fetch_models()
 
     old_content = (
         GENERATED_PATH.read_text(encoding="utf-8")
         if GENERATED_PATH.exists()
         else ""
     )
-    old_models = parse_existing_models(old_content)
+    old_models = parse_existing_paid(old_content)
 
     new_content = render_generated_file(new_models, tag_name)
 
     if new_content == old_content:
-        print(f"Source release: {tag_name}")
+        print(f"Source: {tag_name}")
         print(
             f"{GENERATED_PATH.relative_to(REPO_ROOT)} is already up to date."
         )