Extension manifests, CI smoke tests, and README rewrite

.github/workflows/extension-tests.yml

@@ -0,0 +1,41 @@
+name: Extension Smoke Test
+
+on:
+  push:
+    branches: [main, dev]
+  pull_request:
+    branches: [main, dev]
+
+permissions:
+  contents: read
+
+jobs:
+  smoke:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.11", "3.12"]
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: pip
+
+      - name: Install all extension requirements
+        run: |
+          python -m pip install --upgrade pip
+          # The base `publoader` package isn't on PyPI; the smoke test only
+          # imports each extension's <name>.py and the manifest validator,
+          # neither of which import publoader.* at module level by the time
+          # the new publoader.api shim has landed.
+          find ./src -mindepth 2 -maxdepth 2 -name requirements.txt \
+            -exec pip install --no-cache-dir -r {} \;
+
+      - name: Validate manifests
+        run: python tools/validate_manifests.py
+
+      - name: Smoke-load each extension
+        run: python tools/smoke_load.py

src/mangaplus/mangaplus.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import asyncio
 import logging
 import re
@@ -7,47 +9,71 @@
 from copy import deepcopy
 from datetime import datetime, time, timezone, timedelta
 from pathlib import Path
-from typing import List, Optional, Union
+from typing import TYPE_CHECKING, List, Optional, Union
 
 import aiohttp
 import itertools
 import math
 import requests
-from publoader.models.dataclasses import Chapter, Manga
-from publoader.utils.logs import setup_extension_logs
-from publoader.utils.misc import create_new_event_loop, find_key_from_list_value
-from publoader.utils.utils import (
-    chapter_number_regex,
-    open_manga_id_map,
-    open_title_regex,
-)
-from publoader.webhook import PubloaderWebhook
+
+if TYPE_CHECKING:
+    from publoader.models.dataclasses import Chapter, Manga
 
 DEFAULT_TIMESTAMP = 1
 
 __version__ = "0.2.04"
 
-setup_extension_logs(
-    logger_name="mangaplus",
-    logger_filename="mangaplus",
-)
-
 logger = logging.getLogger("mangaplus")
 
 
+def _load_publoader_api() -> None:
+    """Bind publoader.* symbols to module globals.
+
+    Deferred so smoke tests (which don't install publoader) can import this
+    module without resolving the runtime deps. Called from the entrypoint
+    that actually does work.
+    """
+    global Chapter, Manga, create_new_event_loop, find_key_from_list_value
+    global chapter_number_regex, open_manga_id_map, open_title_regex, PubloaderWebhook
+    from publoader.models.dataclasses import Chapter, Manga
+    from publoader.utils.misc import create_new_event_loop, find_key_from_list_value
+    from publoader.utils.utils import (
+        chapter_number_regex,
+        open_manga_id_map,
+        open_title_regex,
+    )
+    from publoader.webhook import PubloaderWebhook
+
+
 class Extension:
     def __init__(self, extension_dirpath: Path, **kwargs):
+        try:
+            from publoader.utils.logs import setup_extension_logs
+        except ModuleNotFoundError:
+            pass
+        else:
+            setup_extension_logs(
+                logger_name="mangaplus",
+                logger_filename="mangaplus",
+            )
+
         self.name = "mangaplus"
         self.mangadex_group_id = "4f1de6a2-f0c5-4ac5-bce5-02c7dbb67deb"
         self.manga_id_map_filename = "manga_id_map.json"
         self.override_options_filename = "override_options.json"
         self.extension_dirpath = extension_dirpath
 
         self.fetch_all_chapters = False
-        self._posted_chapters_ids = []
-        self._updated_chapters: List[Chapter] = []
-        self._all_mplus_chapters: List[Chapter] = []
-        self._untracked_manga: List[Manga] = []
+        self._posted_chapters_ids: List[str] = []
+        self._updated_chapters: list = []
+        self._all_mplus_chapters: list = []
+        self._untracked_manga: list = []
+        self.override_options: dict = {}
+        self.tracked_mangadex_ids: list = []
+        self.tracked_manga: list = []
+        self.manga_no_chapters: list = []
+        self._manga_id_map: dict = {}
+        self._num2words: Optional[str] = None
         self._mplus_base_api_url = "https://jumpg-webapi.tokyo-cdn.com/api/"
         self._chapter_url_format = "https://mangaplus.shueisha.co.jp/viewer/{}"
         self._manga_url_format = "https://mangaplus.shueisha.co.jp/titles/{}"
@@ -87,6 +113,7 @@ def get_updated_manga(self) -> List[Manga]:
     def update_external_data(
         self, posted_chapter_ids: List[str], fetch_all_chapters: bool, **kwargs
     ) -> None:
+        _load_publoader_api()
         self._posted_chapters_ids = posted_chapter_ids
         self.fetch_all_chapters = fetch_all_chapters
 

src/mangaplus/manifest.json

@@ -0,0 +1,30 @@
+{
+  "name": "mangaplus",
+  "version": "0.2.04",
+  "publoader_api": "^1.0.0",
+  "entrypoint": "mangaplus.py",
+  "class_name": "Extension",
+  "mangadex_group_id": "4f1de6a2-f0c5-4ac5-bce5-02c7dbb67deb",
+  "languages": ["en", "es", "fr", "id", "pt-br", "ru", "th", "de", "vi"],
+  "allowed_hosts": [
+    "jumpg-webapi.tokyo-cdn.com",
+    "mangaplus.shueisha.co.jp"
+  ],
+  "permissions": {
+    "network": true,
+    "filesystem_read": ["manga_id_map.json", "override_options.json"],
+    "filesystem_write": [],
+    "subprocess": false
+  },
+  "schedule": {
+    "hour": 15,
+    "minute": 5,
+    "timezone": "UTC"
+  },
+  "data_files": {
+    "manga_id_map": "manga_id_map.json",
+    "override_options": "override_options.json"
+  },
+  "maintainers": ["publoader"],
+  "homepage": "https://mangaplus.shueisha.co.jp/"
+}

sync_extensions.py

@@ -1,18 +1,121 @@
+"""Sync extension source trees into the shared runtime volume.
+
+Only `src/<extension>/` subtrees are copied. The Dockerfile, LICENSE, README,
+.github, sync_extensions.py itself, .git, and top-level configs are excluded
+so the runtime volume contains exactly what the base loader expects.
+
+Destination layout:
+    <TARGET_DIR>/<extension_name>/<extension_name>.py
+    <TARGET_DIR>/<extension_name>/manifest.json
+    <TARGET_DIR>/<extension_name>/manga_id_map.json
+    <TARGET_DIR>/<extension_name>/...
+    <TARGET_DIR>/schedule.json   (copied from repo root)
+"""
+from __future__ import annotations
+
+import json
+import logging
 import os
 import shutil
+import sys
+import tempfile
+from pathlib import Path
+
+SOURCE_ROOT = Path(os.environ.get("PUBLOADER_SOURCE", "/extensions"))
+SOURCE_SRC = SOURCE_ROOT / "src"
+TARGET_DIR = Path(
+    os.environ.get("PUBLOADER_TARGET", "/shared/publoader/extensions")
+)
+SCHEDULE_FILE = SOURCE_ROOT / "schedule.json"
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s %(levelname)s sync_extensions: %(message)s",
+)
+log = logging.getLogger("sync_extensions")
+
+
+def _is_valid_extension_name(name: str) -> bool:
+    return bool(name) and all(c.islower() or c.isdigit() or c == "_" for c in name)
+
+
+def _atomic_replace_tree(src: Path, dst: Path) -> None:
+    """Replace `dst` with `src` atomically via rename (same filesystem)."""
+    dst.parent.mkdir(parents=True, exist_ok=True)
+    staging = Path(tempfile.mkdtemp(prefix=f".{dst.name}.", dir=dst.parent))
+    try:
+        shutil.copytree(src, staging / dst.name, dirs_exist_ok=False)
+        backup = None
+        if dst.exists():
+            backup = dst.with_suffix(dst.suffix + ".old")
+            if backup.exists():
+                shutil.rmtree(backup)
+            dst.rename(backup)
+        (staging / dst.name).rename(dst)
+        if backup is not None:
+            shutil.rmtree(backup, ignore_errors=True)
+    finally:
+        shutil.rmtree(staging, ignore_errors=True)
+
+
+def _validate_extension(ext_dir: Path) -> bool:
+    name = ext_dir.name
+    if not _is_valid_extension_name(name):
+        log.error("skip %s: invalid extension name", name)
+        return False
+    if not (ext_dir / f"{name}.py").is_file():
+        log.error("skip %s: missing %s.py", name, name)
+        return False
+    manifest_path = ext_dir / "manifest.json"
+    if not manifest_path.is_file():
+        log.error("skip %s: missing manifest.json", name)
+        return False
+    try:
+        manifest = json.loads(manifest_path.read_text())
+    except (OSError, ValueError) as exc:
+        log.error("skip %s: manifest.json invalid (%s)", name, exc)
+        return False
+    if manifest.get("name") != name:
+        log.error(
+            "skip %s: manifest.name=%r doesn't match directory",
+            name,
+            manifest.get("name"),
+        )
+        return False
+    return True
+
+
+def main() -> int:
+    if not SOURCE_SRC.is_dir():
+        log.error("source missing: %s", SOURCE_SRC)
+        return 2
+
+    TARGET_DIR.mkdir(parents=True, exist_ok=True)
+    synced: list = []
+    skipped: list = []
 
-SOURCE_DIR = "/extensions"
-TARGET_DIR = "/shared/publoader/extensions"
+    for child in sorted(SOURCE_SRC.iterdir()):
+        if not child.is_dir() or child.name.startswith((".", "__")):
+            continue
+        if not _validate_extension(child):
+            skipped.append(child.name)
+            continue
+        try:
+            _atomic_replace_tree(child, TARGET_DIR / child.name)
+            synced.append(child.name)
+        except OSError as exc:
+            log.exception("failed syncing %s: %s", child.name, exc)
+            skipped.append(child.name)
 
-os.makedirs(TARGET_DIR, exist_ok=True)
+    if SCHEDULE_FILE.is_file():
+        try:
+            shutil.copy2(SCHEDULE_FILE, TARGET_DIR / SCHEDULE_FILE.name)
+        except OSError as exc:
+            log.exception("failed copying schedule.json: %s", exc)
 
-for item in os.listdir(SOURCE_DIR):
-    source_path = os.path.join(SOURCE_DIR, item)
-    target_path = os.path.join(TARGET_DIR, item)
+    log.info("synced=%s skipped=%s target=%s", synced, skipped, TARGET_DIR)
+    return 0 if not skipped else 1
 
-    if os.path.isdir(source_path):
-        shutil.copytree(source_path, target_path, dirs_exist_ok=True)
-    else:
-        shutil.copy2(source_path, target_path)
 
-print(f"Extensions synced to {TARGET_DIR}")
+if __name__ == "__main__":
+    sys.exit(main())