diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 4bea9c5..839a4a8 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -38,6 +38,8 @@ results to Tk's main loop. - `profile_manager.py` owns Content-tree discovery, STFS ownership inspection, profile/save indexing, verified snapshots, and conflict-safe restore. - `knowledge_sync.py` exposes complete source-import workflows to CLI and GUI. +- `offline_knowledge.py` safely imports browser-saved pages and renders the + self-contained local reading library. - `collection_intelligence.py` coordinates snapshots, exact MediaID matching, health, preservation matching, repair previews, and offline exports. - `console_sync.py` owns durable transfer jobs, resumable FTP, remote @@ -72,7 +74,7 @@ Main schema groups: - Library: `titleids`, `title_updates`, `covers`, `download_history` - Knowledge: sources, documents, revisions, entities, names, identifiers, facts, citations, relationships, import runs, conflicts, source priorities, - conflict decisions, and scheduled sync state + conflict decisions, scheduled sync state, and offline archive/import runs - Backups: targets, scans, inventory, and operations - Profiles: scan runs, profiles, saves, snapshots, snapshot files, GPD inventories, achievements, comparisons, Xenia migration runs, and auditable @@ -91,12 +93,13 @@ data and be covered by tests. ```text source discovery -> rate-limited fetch - -> raw cache + -> raw cache or validated browser-saved page -> source adapter -> normalized records -> citations and conflicts -> preferred fact selection -> fill unknown library metadata + -> script-free offline HTML archive ``` Raw documents and retrieval metadata remain attached to their source. A parser diff --git a/CHANGELOG.md b/CHANGELOG.md index b986b38..d366c29 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,13 @@ Notable changes to UnityScraper are documented here. The project follows ### Added +- Additive schema migration 11 for offline knowledge archive builds, rendered + document state, and browser-saved wiki import history. +- A self-contained offline Knowledge library with local search, source and + license attribution, stale-cache labels, and script-free article pages. +- Safe `.html`, `.htm`, folder, and ZIP imports for pages saved from + ConsoleMods, XenonLibrary, or Free60 in a normal browser. + - Additive schema migration 10 for metadata snapshot runs, library intelligence, preservation reports, correction packages, hardware records, and package extractions. - Visual Studio 2010-inspired dark desktop theme shared by the modern and legacy @@ -107,6 +114,16 @@ Notable changes to UnityScraper are documented here. The project follows - Tagged-release packaging with SHA-256 checksums. - Contributor, security, community, and architecture documentation. +### Fixed + +- ConsoleMods and XenonLibrary Cloudflare responses now produce actionable + browser-verification diagnostics instead of opaque HTTP failures. +- Wiki refreshes retain known seed and cache URLs, preserve the original fetch + timestamp when falling back offline, and no longer discard successful pages + because another page failed. +- Knowledge enrichment now recognizes `Unknown game` and `Unknown title` as + replaceable placeholders when a source provides a real title. + ### Changed - Version advanced to `1.2.0-beta.1`. diff --git a/DOCS_INDEX.md b/DOCS_INDEX.md index a19f4ee..6e79eac 100644 --- a/DOCS_INDEX.md +++ b/DOCS_INDEX.md @@ -13,8 +13,8 @@ XEX identity, Title Update compatibility, preservation, and repair previews - [Console Sync](CONSOLE_SYNC.md) - persistent transfers, resume, snapshots, comparison, and verification -- [Knowledge Sources](KNOWLEDGE_SOURCES.md) - imports, provenance, caching, and - source licensing +- [Knowledge Sources](KNOWLEDGE_SOURCES.md) - imports, provenance, caching, + browser-verification recovery, offline viewing, and source licensing - [Advanced Features](ADVANCED_FEATURES.md) - rate limits, resume, diagnostics, portable mode, API, and conversion - [REST API](API.md) - authentication, endpoints, configuration, and safety diff --git a/KNOWLEDGE_SOURCES.md b/KNOWLEDGE_SOURCES.md index ef562fa..70d8e82 100644 --- a/KNOWLEDGE_SOURCES.md +++ b/KNOWLEDGE_SOURCES.md @@ -33,7 +33,8 @@ python main.py --sync-wikis ConsoleMods and XenonLibrary pages are discovered through MediaWiki's paginated all-pages API, with XML sitemap discovery as an additional path. Free60 uses -its XML sitemap. Seed pages are used only when discovery is unavailable. +its XML sitemap. Seed pages and URLs from successful earlier cache entries are +always retained, so a temporary discovery failure does not hide known pages. Every fetched article is cached locally and imported as a searchable knowledge entity with its source URL, revision snapshot, summary, and visible article @@ -42,6 +43,45 @@ receive a freshness warning. Use `--wiki-limit N` to restrict each source during testing or a first sync. +## Browser Verification And Offline Use + +ConsoleMods and XenonLibrary currently may place Cloudflare browser +verification in front of wiki and API requests. A normal browser can work while +the same URL returns HTTP 403 to UnityScraper. The application deliberately +does not imitate a browser session, solve challenges, or bypass source access +controls. + +When this happens UnityScraper: + +- reports that browser verification blocked the refresh; +- uses a prior cached page when one exists without changing its original fetch + timestamp; +- records the stale-cache state and refresh error with the source document; +- continues importing other available sources; +- rebuilds the offline library from every usable cached page. + +In the desktop **Knowledge** workspace, choose a source and use **Import Saved +Wiki Pages**. You can select individual `.html`/`.htm` files, a folder, or a ZIP +containing saved pages. For command-line use: + +```powershell +python main.py --import-saved-wiki "C:\Saved Wikis" --saved-wiki-source consolemods-wiki +python main.py --import-saved-wiki "C:\Saved Wikis\xenon.zip" --saved-wiki-source xenonlibrary +python main.py --build-offline-knowledge +``` + +The importer accepts at most 5,000 pages, 10 MB per page, and 250 MB total per +operation. It never executes imported HTML. Canonical URLs are accepted only +when they match the selected source. A saved ConsoleMods TitleID or Multi-ID +list also runs through the structured game metadata parser and still enriches +only unknown local names or publishers. + +The generated `offline_knowledge/index.html` is a private, self-contained, +dark-theme reading library. It contains readable article text, cache status, +source links, timestamps, and license attribution. Remote scripts, styles, +trackers, and images are not copied into rendered pages. Raw source snapshots +remain in the application cache for provenance and future reprocessing. + ## Preservation DAT Import Download DATs directly from their source and import them locally: @@ -71,6 +111,9 @@ The database migration adds these normalized tables: - `entity_relationships` - `knowledge_import_runs` - `knowledge_conflicts` +- `offline_archive_runs` +- `offline_archive_documents` +- `offline_page_import_runs` Facts are source-attributed claims. If two sources disagree, both claims can exist and the disagreement is recorded in `knowledge_conflicts`. @@ -110,6 +153,7 @@ The **Knowledge** page includes: - details with source names and citation URLs; - source license, document count, fact count, and latest import status; - ConsoleMods ID sync and whole-wiki sync; +- saved-page import, offline-library rebuild, and local-browser access; - Redump and No-Intro file import; - per-property source priorities, where lower numbers are preferred for display; @@ -120,7 +164,8 @@ The **Knowledge** page includes: ## Remaining Boundaries - Source availability, access controls, and licenses can change. Failed syncs - are isolated per source and previously cached pages remain available. + are isolated per source and previously cached or manually imported pages + remain available. - Redump and No-Intro DATs are not bundled. Users obtain them from the source. - Wiki content is reference material, not automatically trusted repair advice. - Commercial game images, firmware, keys, and leaked SDK files are never diff --git a/PROJECT_STATUS.md b/PROJECT_STATUS.md index b7aa4c8..9ced133 100644 --- a/PROJECT_STATUS.md +++ b/PROJECT_STATUS.md @@ -16,6 +16,8 @@ backup-management, and source-attributed knowledge application. - Searchable ConsoleMods, XenonLibrary, and Free60 wiki article ingestion using MediaWiki API or sitemap discovery, local caching, attribution, and per-source failure isolation. +- Cloudflare-aware diagnostics, stale-cache recovery, browser-saved page + import, and a self-contained offline Knowledge reading library. - User-selected Redump and No-Intro XML DAT import for release identity, serial, size, status, and checksum metadata. - Desktop knowledge browser with source status, licenses, citations, import @@ -65,6 +67,8 @@ backup-management, and source-attributed knowledge application. duplicate actions, plus selectable restore controls. - Additive schema migration 10 for metadata snapshots, library audits, preservation reports, correction exports, hardware records, and package extraction. +- Additive schema migration 11 for offline archive builds, rendered document + state, and saved-page import history. - Visual Studio 2010-inspired shared desktop theme, classic menus, scoped API tokens, out-of-process plugin execution, and bounded community language packs. - Read-only consecutive STFS extraction, direct Xenia launch controls, and @@ -78,8 +82,8 @@ backup-management, and source-attributed knowledge application. - Offline tests cover scraper configuration, database behavior, ConsoleMods parsing, DAT parsing, wiki parsing, knowledge search/provenance, downloads, queueing, - STFS/XBE inspection, safe archives, backup scanning, atomic copies, and an - end-to-end local workflow. + blocked-source recovery, saved-page rendering, STFS/XBE inspection, safe + archives, backup scanning, atomic copies, and an end-to-end local workflow. - Network-backed source syncs remain dependent on each source's availability and access policy. Cached copies are used when available. - Windows, Linux, and macOS artifacts are generated by CI and releases diff --git a/README.md b/README.md index 75aaba7..357d2fd 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,8 @@ service exposes. UnityScraper does not silently substitute HTTPS URLs. - Imports TitleID, publisher, region, and Multi-ID references from ConsoleMods. - Caches and indexes Xbox 360 articles from ConsoleMods, XenonLibrary, and Free60. +- Builds a searchable, self-contained offline library from cached or + browser-saved wiki pages. - Imports user-supplied Redump and No-Intro XML DAT files. - Stores entities, identifiers, facts, citations, revisions, import runs, and conflicts with provenance. @@ -299,10 +301,25 @@ python main.py --sync-wikis # Limit a first test sync per source python main.py --sync-wikis --wiki-limit 25 +# Build or refresh the local offline reading library +python main.py --build-offline-knowledge + +# Import a page, folder, or ZIP saved in a browser when a wiki blocks automation +python main.py --import-saved-wiki "C:\Saved Wikis" --saved-wiki-source xenonlibrary + # Import a user-downloaded preservation DAT python main.py --import-dat "D:\DATs\xbox360.dat" --dat-source redump ``` +ConsoleMods and XenonLibrary may require Cloudflare browser verification and +return HTTP 403 to command-line clients. UnityScraper does not bypass that +protection. It continues using the last successful cache and identifies the +blocked source clearly. In the **Knowledge** workspace, use **Import Saved Wiki +Pages** for pages saved through a normal browser, then **Open Offline Library** +to browse the local copy. The offline renderer keeps article text, source +links, timestamps, and license attribution while excluding remote scripts and +trackers. + ### Backup Manager ```powershell diff --git a/app_paths.py b/app_paths.py index d40bd09..bc698c6 100644 --- a/app_paths.py +++ b/app_paths.py @@ -148,6 +148,7 @@ def xdg_path(variable: str, fallback: Path) -> Path: PROFILE_BACKUPS_DIR = DATA_DIR / "profile_backups" PLUGINS_DIR = DATA_DIR / "plugins" LANGUAGE_PACKS_DIR = DATA_DIR / "languages" +OFFLINE_KNOWLEDGE_DIR = DATA_DIR / "offline_knowledge" DATABASE_PATH = DATA_DIR / "unityscraper.db" CONFIG_PATH = CONFIG_DIR / "config.json" @@ -176,6 +177,7 @@ def ensure_app_dirs() -> None: PROFILE_BACKUPS_DIR, PLUGINS_DIR, LANGUAGE_PACKS_DIR, + OFFLINE_KNOWLEDGE_DIR, ): path.mkdir(parents=True, exist_ok=True) diff --git a/database_migrations.py b/database_migrations.py index d953636..35650e1 100644 --- a/database_migrations.py +++ b/database_migrations.py @@ -8,7 +8,7 @@ from pathlib import Path -SCHEMA_VERSION = 10 +SCHEMA_VERSION = 11 def _now() -> str: @@ -71,6 +71,7 @@ def ensure_application_schema(connection: sqlite3.Connection) -> int: (8, "community roadmap workspaces", _migration_community_roadmap), (9, "hardening and plugin runtime", _migration_hardening), (10, "release readiness workspaces", _migration_release_readiness), + (11, "offline knowledge archive", _migration_offline_knowledge), ) for version, name, migration in migrations: if version in applied: @@ -805,3 +806,39 @@ def _migration_release_readiness(connection: sqlite3.Connection) -> None: ); """ ) + + +def _migration_offline_knowledge(connection: sqlite3.Connection) -> None: + """Track generated offline pages and user-provided wiki imports.""" + connection.executescript( + """ + CREATE TABLE IF NOT EXISTS offline_archive_runs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + started_at TEXT NOT NULL, + finished_at TEXT, + status TEXT NOT NULL, + documents_written INTEGER NOT NULL DEFAULT 0, + index_path TEXT, + errors TEXT + ); + CREATE TABLE IF NOT EXISTS offline_archive_documents ( + document_id INTEGER PRIMARY KEY, + archive_path TEXT NOT NULL, + rendered_at TEXT NOT NULL, + content_sha256 TEXT NOT NULL, + stale INTEGER NOT NULL DEFAULT 0, + FOREIGN KEY(document_id) REFERENCES source_documents(id) + ); + CREATE TABLE IF NOT EXISTS offline_page_import_runs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + source_slug TEXT NOT NULL, + source_path TEXT NOT NULL, + started_at TEXT NOT NULL, + finished_at TEXT, + status TEXT NOT NULL, + files_seen INTEGER NOT NULL DEFAULT 0, + files_imported INTEGER NOT NULL DEFAULT 0, + errors TEXT + ); + """ + ) diff --git a/knowledge_base.py b/knowledge_base.py index e9fdbff..1fd958c 100644 --- a/knowledge_base.py +++ b/knowledge_base.py @@ -17,7 +17,16 @@ logger = logging.getLogger(__name__) -UNKNOWN_VALUES = {"", "unknown", "unknown publisher", "n/a", "none", "null"} +UNKNOWN_VALUES = { + "", + "unknown", + "unknown game", + "unknown title", + "unknown publisher", + "n/a", + "none", + "null", +} def utc_now() -> str: diff --git a/knowledge_gui.py b/knowledge_gui.py index 7fcf060..ab4ec31 100644 --- a/knowledge_gui.py +++ b/knowledge_gui.py @@ -10,6 +10,8 @@ from knowledge_service import KnowledgeService from knowledge_scheduler import KnowledgeScheduler +from offline_knowledge import OfflineKnowledgeArchive +from platform_support import open_path from ui_theme import PALETTE TEXT = PALETTE.text @@ -150,7 +152,7 @@ def _build_browse(self, parent: ttk.Frame) -> None: def _build_sources(self, parent: ttk.Frame) -> None: parent.columnconfigure(0, weight=1) - parent.rowconfigure(1, weight=1) + parent.rowconfigure(2, weight=1) controls = ttk.Frame(parent) controls.grid(row=0, column=0, sticky="ew", pady=(0, 8)) buttons = ( @@ -172,6 +174,33 @@ def _build_sources(self, parent: ttk.Frame) -> None: style="Subheader.TLabel", ).pack(side=tk.RIGHT) + archive_controls = ttk.Frame(parent) + archive_controls.grid(row=1, column=0, sticky="ew", pady=(0, 8)) + self.saved_page_source_var = tk.StringVar(value="ConsoleMods") + ttk.Combobox( + archive_controls, + textvariable=self.saved_page_source_var, + values=("ConsoleMods", "XenonLibrary", "Free60"), + state="readonly", + width=17, + ).pack(side=tk.LEFT, padx=(0, 6)) + ttk.Button( + archive_controls, + text="Import Saved Wiki Pages", + command=self._select_saved_pages, + ).pack(side=tk.LEFT, padx=(0, 6)) + ttk.Button( + archive_controls, + text="Rebuild Offline Library", + command=self._rebuild_offline_library, + ).pack(side=tk.LEFT, padx=(0, 6)) + ttk.Button( + archive_controls, + text="Open Offline Library", + command=self._open_offline_library, + style="Accent.TButton", + ).pack(side=tk.LEFT) + self.source_tree = ttk.Treeview( parent, columns=("license", "documents", "facts", "status", "last_sync"), @@ -188,7 +217,7 @@ def _build_sources(self, parent: ttk.Frame) -> None: for column, label, width in columns: self.source_tree.heading(column, text=label) self.source_tree.column(column, width=width, minwidth=55) - self.source_tree.grid(row=1, column=0, sticky="nsew") + self.source_tree.grid(row=2, column=0, sticky="nsew") def _build_conflicts(self, parent: ttk.Frame) -> None: parent.columnconfigure(0, weight=1) @@ -501,6 +530,63 @@ def _select_dat(self, source_kind: str) -> None: lambda: self._import_dat(Path(path), source_kind), ) + def _select_saved_pages(self) -> None: + choose_folder = messagebox.askyesnocancel( + "Import saved wiki pages", + "Choose Yes to import a folder, or No to select page and ZIP files.", + parent=self.root, + ) + if choose_folder is None: + return + if choose_folder: + folder = filedialog.askdirectory( + parent=self.root, + title="Select folder containing saved wiki pages", + ) + paths = (folder,) if folder else () + else: + paths = filedialog.askopenfilenames( + parent=self.root, + title="Select browser-saved wiki pages or ZIP archives", + filetypes=( + ("Saved wiki pages", "*.html *.htm *.zip"), + ("All files", "*.*"), + ), + ) + if not paths: + return + source_slugs = { + "ConsoleMods": "consolemods-wiki", + "XenonLibrary": "xenonlibrary", + "Free60": "free60", + } + source_slug = source_slugs[self.saved_page_source_var.get()] + self._run_job( + "Importing saved wiki pages...", + lambda: OfflineKnowledgeArchive( + database_path=self.service.database_path + ).import_saved_pages(paths, source_slug), + ) + + def _rebuild_offline_library(self) -> None: + self._run_job( + "Rebuilding offline library...", + lambda: OfflineKnowledgeArchive( + database_path=self.service.database_path + ).rebuild(), + ) + + def _open_offline_library(self) -> None: + archive = OfflineKnowledgeArchive(database_path=self.service.database_path) + try: + if not archive.index_path.exists(): + archive.rebuild() + open_path(archive.index_path) + except Exception as exc: + messagebox.showerror( + "Offline library unavailable", str(exc), parent=self.root + ) + @staticmethod def _import_dat(path: Path, source_kind: str) -> dict[str, Any]: from knowledge_sync import import_dat_knowledge diff --git a/knowledge_sources.py b/knowledge_sources.py index e646ad0..a0b7693 100644 --- a/knowledge_sources.py +++ b/knowledge_sources.py @@ -3,12 +3,13 @@ from __future__ import annotations import hashlib +import json import logging import time from dataclasses import dataclass from pathlib import Path from typing import Iterable, Protocol -from urllib.parse import quote +from urllib.parse import quote, urlparse import requests @@ -42,6 +43,23 @@ class SourceDocument: content_sha256: str cache_path: Path http_status: int = 200 + from_cache: bool = False + fetch_error: str = "" + final_url: str = "" + + +class SourceAccessBlockedError(requests.RequestException): + """Raised when a site requires interactive browser verification.""" + + def __init__(self, url: str, status_code: int = 403) -> None: + self.url = url + self.status_code = status_code + super().__init__( + f"{url} denied automated access (HTTP {status_code}). " + "The site appears to require browser verification. Existing offline " + "content remains available; open the page in a browser and use " + "Import Saved Wiki Pages to add or refresh it." + ) @dataclass(frozen=True) @@ -86,14 +104,13 @@ def __init__( self.session = session or requests.Session() self.session.headers.setdefault( "User-Agent", - ( - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " - "AppleWebKit/537.36 (KHTML, like Gecko) " - "UnityScraperKnowledge/1.0 Safari/537.36" - ), + "UnityScraper/1.2 (+https://github.com/TrapEmAll/UnityScraper)", + ) + self.session.headers.setdefault( + "Accept", "text/html,application/xhtml+xml,application/json" ) - self.session.headers.setdefault("Accept", "text/html,application/xhtml+xml") self._last_request = 0.0 + self._blocked_hosts: dict[str, SourceAccessBlockedError] = {} def get_text( self, @@ -105,16 +122,40 @@ def get_text( """Fetch a URL, falling back to a cached copy if the network fails.""" cache_path = self._cache_path(url) urls = (url, *tuple(fallback_urls)) + metadata_path = self._metadata_path(url) last_error: requests.RequestException | None = None for fetch_url in urls: + host = urlparse(fetch_url).netloc.casefold() + if host in self._blocked_hosts: + last_error = self._blocked_hosts[host] + continue self._wait() try: response = self.session.get(fetch_url, timeout=self.timeout) + if self._is_browser_challenge(response): + raise SourceAccessBlockedError(fetch_url, response.status_code) response.raise_for_status() text = response.text + self._validate_content(text, document_type, fetch_url) cache_path.write_text(text, encoding="utf-8") fetched_at = utc_now() + metadata = { + "url": url, + "fetch_url": fetch_url, + "final_url": str(getattr(response, "url", fetch_url)), + "title": title, + "document_type": document_type, + "fetched_at": fetched_at, + "http_status": response.status_code, + "content_sha256": self._sha256_text(text), + "content_type": response.headers.get("Content-Type", ""), + "etag": response.headers.get("ETag", ""), + "last_modified": response.headers.get("Last-Modified", ""), + } + metadata_path.write_text( + json.dumps(metadata, indent=2, sort_keys=True), encoding="utf-8" + ) return SourceDocument( url=url, title=title, @@ -124,28 +165,100 @@ def get_text( content_sha256=self._sha256_text(text), cache_path=cache_path, http_status=response.status_code, + final_url=str(metadata["final_url"]), ) except requests.RequestException as exc: last_error = exc + if isinstance(exc, SourceAccessBlockedError): + self._blocked_hosts[host] = exc if cache_path.exists(): logger.warning("Using cached source document for %s", url) text = cache_path.read_text(encoding="utf-8") + metadata = self._read_metadata(metadata_path) return SourceDocument( url=url, title=title, document_type=document_type, text=text, - fetched_at=utc_now(), + fetched_at=str(metadata.get("fetched_at") or utc_now()), content_sha256=self._sha256_text(text), cache_path=cache_path, http_status=0, + from_cache=True, + fetch_error=str(last_error or "Network unavailable"), + final_url=str(metadata.get("final_url") or url), ) if last_error: raise last_error raise requests.RequestException(f"No fetch URLs available for {url}") + def store_text( + self, + url: str, + title: str, + document_type: str, + text: str, + *, + fetched_at: str | None = None, + imported_from: str = "", + ) -> SourceDocument: + """Store a user-provided page in the standard cache with provenance.""" + if not text.strip(): + raise ValueError("The saved page is empty") + self._validate_content(text, document_type, url) + cache_path = self._cache_path(url) + metadata_path = self._metadata_path(url) + timestamp = fetched_at or utc_now() + digest = self._sha256_text(text) + cache_path.write_text(text, encoding="utf-8") + metadata_path.write_text( + json.dumps( + { + "url": url, + "fetch_url": url, + "final_url": url, + "title": title, + "document_type": document_type, + "fetched_at": timestamp, + "http_status": 0, + "content_sha256": digest, + "imported_from": imported_from, + }, + indent=2, + sort_keys=True, + ), + encoding="utf-8", + ) + return SourceDocument( + url=url, + title=title, + document_type=document_type, + text=text, + fetched_at=timestamp, + content_sha256=digest, + cache_path=cache_path, + http_status=0, + from_cache=True, + final_url=url, + ) + + def cached_urls(self, hosts: Iterable[str] = ()) -> list[str]: + """Return canonical URLs recorded by cache metadata sidecars.""" + allowed = {host.casefold() for host in hosts} + urls: set[str] = set() + for path in self.cache_dir.glob("*.json"): + metadata = self._read_metadata(path) + url = str(metadata.get("url") or "") + if not url: + continue + if allowed and urlparse(url).netloc.casefold() not in allowed: + continue + if self._cache_path(url).exists(): + urls.add(url) + return sorted(urls) + def _wait(self) -> None: elapsed = time.time() - self._last_request if elapsed < self.rate_limit_seconds: @@ -156,6 +269,55 @@ def _cache_path(self, url: str) -> Path: digest = hashlib.sha256(url.encode("utf-8")).hexdigest() return self.cache_dir / f"{digest}.html" + def _metadata_path(self, url: str) -> Path: + return self._cache_path(url).with_suffix(".json") + + @staticmethod + def _read_metadata(path: Path) -> dict[str, object]: + try: + value = json.loads(path.read_text(encoding="utf-8")) + return value if isinstance(value, dict) else {} + except (OSError, ValueError): + return {} + + @staticmethod + def _is_browser_challenge(response: requests.Response) -> bool: + markers = " ".join( + ( + response.headers.get("Server", ""), + response.headers.get("cf-ray", ""), + response.text[:4096], + ) + ).casefold() + strong_challenge = any( + marker in markers + for marker in ("just a moment", "challenge-platform", "browser verification") + ) + if response.status_code == 200: + return strong_challenge and "cloudflare" in markers + if response.status_code not in {403, 429, 503}: + return False + return any( + marker in markers + for marker in ( + "cloudflare", + "cf-ray", + "just a moment", + "challenge-platform", + "browser verification", + ) + ) + + @staticmethod + def _validate_content(text: str, document_type: str, url: str) -> None: + stripped = text.lstrip() + if not stripped: + raise requests.RequestException(f"Empty response from {url}") + if document_type == "mediawiki_api" and not stripped.startswith(("{", "[")): + raise requests.RequestException( + f"Expected MediaWiki JSON but received a web page from {url}" + ) + @staticmethod def _sha256_text(text: str) -> str: return hashlib.sha256(text.encode("utf-8")).hexdigest() @@ -189,11 +351,16 @@ def run_adapter(self, adapter: SourceAdapter) -> dict[str, int | str]: run_id = self.repository.begin_import_run(source.slug, adapter.adapter_name) records_seen = 0 records_imported = 0 + cached_documents = 0 errors: list[str] = [] status = "success" try: for document in adapter.fetch_documents(): + if document.fetch_error: + cached_documents += 1 + if document.fetch_error not in errors: + errors.append(document.fetch_error) document_id, revision_id = self.repository.upsert_document( source_id, document.url, @@ -204,6 +371,11 @@ def run_adapter(self, adapter: SourceAdapter) -> dict[str, int | str]: str(document.cache_path), http_status=document.http_status, license_name=source.license_name, + metadata={ + "from_cache": document.from_cache, + "fetch_error": document.fetch_error, + "final_url": document.final_url, + }, ) parsed = adapter.parse_document(document) records_seen += len(parsed.records) @@ -215,11 +387,16 @@ def run_adapter(self, adapter: SourceAdapter) -> dict[str, int | str]: revision_id=revision_id, ) records_imported += 1 + for warning in getattr(adapter, "errors", ()): + if warning not in errors: + errors.append(warning) except Exception as exc: errors.append(str(exc)) status = "partial" if records_imported else "failed" if status == "success" and records_seen == 0: status = "empty" + elif status == "success" and errors: + status = "partial" self.repository.finish_import_run( run_id, @@ -234,5 +411,6 @@ def run_adapter(self, adapter: SourceAdapter) -> dict[str, int | str]: "status": status, "records_seen": records_seen, "records_imported": records_imported, + "cached_documents": cached_documents, "error": "; ".join(errors), } diff --git a/knowledge_sync.py b/knowledge_sync.py index dce5302..a4e6523 100644 --- a/knowledge_sync.py +++ b/knowledge_sync.py @@ -61,10 +61,37 @@ def sync_reference_wikis( """Import searchable Xbox 360 wiki articles with per-source isolation.""" db = db or DatabaseManager() client = CachedHttpClient(cache_dir=cache_dir) + with db.get_connection() as connection: + known_by_source = { + slug: tuple( + row[0] + for row in connection.execute( + """ + SELECT d.url FROM source_documents d + JOIN knowledge_sources s ON s.id=d.source_id + WHERE s.slug=? AND d.document_type='wiki_article' + """, + (slug,), + ).fetchall() + ) + for slug in ("consolemods-wiki", "xenonlibrary", "free60") + } adapters = ( - ConsoleModsWikiAdapter(client, max_documents=max_documents_per_source), - XenonLibraryWikiAdapter(client, max_documents=max_documents_per_source), - Free60WikiAdapter(client, max_documents=max_documents_per_source), + ConsoleModsWikiAdapter( + client, + max_documents=max_documents_per_source, + known_urls=known_by_source["consolemods-wiki"], + ), + XenonLibraryWikiAdapter( + client, + max_documents=max_documents_per_source, + known_urls=known_by_source["xenonlibrary"], + ), + Free60WikiAdapter( + client, + max_documents=max_documents_per_source, + known_urls=known_by_source["free60"], + ), ) summaries: list[dict[str, Any]] = [] @@ -86,7 +113,17 @@ def sync_reference_wikis( "records_imported": 0, } ) - return {"adapters": summaries} + try: + from offline_knowledge import OfflineKnowledgeArchive + + archive = OfflineKnowledgeArchive( + database_path=db.db_path, + cache_dir=cache_dir, + ).rebuild() + except Exception as exc: + logger.exception("Offline knowledge archive rebuild failed") + archive = {"status": "failed", "error": str(exc)} + return {"adapters": summaries, "offline_archive": archive} def import_dat_knowledge( diff --git a/main.py b/main.py index 4a0a407..517a15e 100644 --- a/main.py +++ b/main.py @@ -935,6 +935,21 @@ def main(): default=None, help='Optional maximum article count per wiki source' ) + parser.add_argument( + '--build-offline-knowledge', + action='store_true', + help='Build a private offline HTML library from cached wiki pages' + ) + parser.add_argument( + '--import-saved-wiki', + type=str, + help='Import a browser-saved HTML page, folder, or ZIP archive' + ) + parser.add_argument( + '--saved-wiki-source', + choices=['consolemods-wiki', 'xenonlibrary', 'free60'], + help='Source attribution required by --import-saved-wiki' + ) parser.add_argument( '--import-dat', type=str, @@ -1152,6 +1167,33 @@ def main(): logger.error(f"Wiki sync failed: {e}") sys.exit(1) + if args.build_offline_knowledge: + try: + from offline_knowledge import OfflineKnowledgeArchive + + summary = OfflineKnowledgeArchive().rebuild() + logger.info("Offline knowledge library built: %s", summary) + sys.exit(0) + except Exception as e: + logger.error("Offline knowledge build failed: %s", e) + sys.exit(1) + + if args.import_saved_wiki: + if not args.saved_wiki_source: + parser.error("--saved-wiki-source is required with --import-saved-wiki") + try: + from offline_knowledge import OfflineKnowledgeArchive + + summary = OfflineKnowledgeArchive().import_saved_pages( + args.import_saved_wiki, + args.saved_wiki_source, + ) + logger.info("Saved wiki import completed: %s", summary) + sys.exit(0) + except Exception as e: + logger.error("Saved wiki import failed: %s", e) + sys.exit(1) + if args.import_dat: if not args.dat_source: parser.error("--dat-source is required with --import-dat") diff --git a/offline_knowledge.py b/offline_knowledge.py new file mode 100644 index 0000000..1ef4943 --- /dev/null +++ b/offline_knowledge.py @@ -0,0 +1,418 @@ +"""Safe, self-contained offline archive for cached community wiki pages.""" + +from __future__ import annotations + +import hashlib +import html +import json +import re +import sqlite3 +import zipfile +from dataclasses import dataclass +from contextlib import closing +from html.parser import HTMLParser +from pathlib import Path +from typing import Iterable +from urllib.parse import urlparse + +from app_paths import DATABASE_PATH, OFFLINE_KNOWLEDGE_DIR +from consolemods_adapters import ( + ConsoleModsMultiIdAdapter, + ConsoleModsTitleIdAdapter, +) +from database import DatabaseManager +from knowledge_base import KnowledgeRepository, utc_now +from knowledge_sources import CachedHttpClient, KnowledgeImportService, SourceDocument +from wiki_adapters import ( + CONSOLEMODS_WIKI_SOURCE, + FREE60_SOURCE, + XENONLIBRARY_SOURCE, + ConsoleModsWikiAdapter, + Free60WikiAdapter, + XenonLibraryWikiAdapter, + extract_article_text, +) + +MAX_FILES = 5000 +MAX_FILE_BYTES = 10 * 1024 * 1024 +MAX_TOTAL_BYTES = 250 * 1024 * 1024 + +SOURCE_ADAPTERS = { + "consolemods-wiki": ConsoleModsWikiAdapter, + "xenonlibrary": XenonLibraryWikiAdapter, + "free60": Free60WikiAdapter, +} +SOURCE_INFO = { + source.slug: source + for source in (CONSOLEMODS_WIKI_SOURCE, XENONLIBRARY_SOURCE, FREE60_SOURCE) +} +SOURCE_HOSTS = { + "consolemods-wiki": {"consolemods.org", "www.consolemods.org"}, + "xenonlibrary": {"xenonlibrary.com", "www.xenonlibrary.com"}, + "free60": {"free60.org", "www.free60.org"}, +} + + +class OfflineArchiveError(ValueError): + """A saved-page import or offline archive could not be completed safely.""" + + +class _PageMetadataParser(HTMLParser): + def __init__(self) -> None: + super().__init__(convert_charrefs=True) + self.canonical_url = "" + self.title = "" + self._in_title = False + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + values = {key.casefold(): value or "" for key, value in attrs} + if tag.casefold() == "title": + self._in_title = True + if tag.casefold() == "link" and "canonical" in values.get("rel", "").casefold(): + self.canonical_url = values.get("href", "") + if tag.casefold() == "meta" and values.get("property", "").casefold() == "og:url": + self.canonical_url = self.canonical_url or values.get("content", "") + + def handle_endtag(self, tag: str) -> None: + if tag.casefold() == "title": + self._in_title = False + + def handle_data(self, data: str) -> None: + if self._in_title: + self.title += data + + +@dataclass(frozen=True) +class _SavedPage: + name: str + text: str + + +class _SavedWikiAdapter: + def __init__( + self, + source_slug: str, + documents: Iterable[SourceDocument], + client: CachedHttpClient, + ) -> None: + adapter_class = SOURCE_ADAPTERS[source_slug] + self.source = SOURCE_INFO[source_slug] + self.adapter_name = f"{source_slug}_saved_pages" + self._parser = adapter_class(client=client, max_documents=0) + self._documents = tuple(documents) + + def fetch_documents(self) -> Iterable[SourceDocument]: + return iter(self._documents) + + def parse_document(self, document: SourceDocument): + if self.source.slug == "consolemods-wiki": + lowered_url = document.url.casefold() + if "list_of_every_xbox_360_title_id" in lowered_url: + return ConsoleModsTitleIdAdapter(self._parser.client).parse_document(document) + if "list_of_multi-id_games" in lowered_url: + return ConsoleModsMultiIdAdapter(self._parser.client).parse_document(document) + return self._parser.parse_document(document) + + +class OfflineKnowledgeArchive: + """Render cached source documents into a private, script-free local library.""" + + def __init__( + self, + database_path: Path | str = DATABASE_PATH, + output_dir: Path | str = OFFLINE_KNOWLEDGE_DIR, + cache_dir: Path | str | None = None, + ) -> None: + self.database_path = Path(database_path) + self.output_dir = Path(output_dir) + self.client = CachedHttpClient(cache_dir=cache_dir, rate_limit_seconds=0) + DatabaseManager(str(self.database_path)) + + @property + def index_path(self) -> Path: + return self.output_dir / "index.html" + + def rebuild(self) -> dict[str, object]: + self.output_dir.mkdir(parents=True, exist_ok=True) + started_at = utc_now() + errors: list[str] = [] + written: list[dict[str, object]] = [] + with closing(sqlite3.connect(self.database_path)) as connection: + connection.row_factory = sqlite3.Row + cursor = connection.execute( + "INSERT INTO offline_archive_runs(started_at, status) VALUES (?, 'running')", + (started_at,), + ) + if cursor.lastrowid is None: + raise RuntimeError("Offline archive run was created without an identifier") + run_id = int(cursor.lastrowid) + rows = connection.execute( + """ + SELECT d.id, d.url, d.title, d.fetched_at, d.content_sha256, + d.cache_path, d.metadata, s.slug source_slug, s.name source_name, + s.homepage_url, s.license_name, s.license_url + FROM source_documents d + JOIN knowledge_sources s ON s.id=d.source_id + WHERE d.document_type='wiki_article' + ORDER BY s.name, d.title COLLATE NOCASE + """ + ).fetchall() + for row in rows: + try: + cache_path = Path(row["cache_path"] or "") + raw = cache_path.read_text(encoding="utf-8") + title, body = extract_article_text(raw, row["title"] or row["url"]) + if not body.strip(): + raise OfflineArchiveError("cached page contains no readable text") + slug = _slug(title) + relative = Path("pages") / str(row["source_slug"]) / f"{row['id']}-{slug}.html" + destination = self.output_dir / relative + destination.parent.mkdir(parents=True, exist_ok=True) + metadata = _json_object(row["metadata"]) + stale = bool(metadata.get("from_cache")) + page = self._article_html(dict(row), title, body, stale) + _atomic_write(destination, page) + connection.execute( + """ + INSERT INTO offline_archive_documents( + document_id, archive_path, rendered_at, content_sha256, stale + ) VALUES (?, ?, ?, ?, ?) + ON CONFLICT(document_id) DO UPDATE SET + archive_path=excluded.archive_path, + rendered_at=excluded.rendered_at, + content_sha256=excluded.content_sha256, + stale=excluded.stale + """, + (row["id"], str(destination), utc_now(), row["content_sha256"], int(stale)), + ) + written.append( + { + "title": title, + "source": row["source_name"], + "source_slug": row["source_slug"], + "path": relative.as_posix(), + "stale": stale, + "fetched_at": row["fetched_at"] or "Unknown", + } + ) + except (OSError, ValueError) as exc: + errors.append(f"{row['url']}: {exc}") + + _atomic_write(self.index_path, self._index_html(written)) + manifest = { + "format": "UnityScraper Offline Knowledge Archive", + "generated_at": utc_now(), + "documents": written, + "errors": errors, + } + _atomic_write( + self.output_dir / "manifest.json", + json.dumps(manifest, indent=2, sort_keys=True), + ) + status = "success" if not errors else ("partial" if written else "failed") + connection.execute( + """ + UPDATE offline_archive_runs SET finished_at=?, status=?, + documents_written=?, index_path=?, errors=? WHERE id=? + """, + (utc_now(), status, len(written), str(self.index_path), json.dumps(errors), run_id), + ) + connection.commit() + return { + "status": status, + "documents_written": len(written), + "errors": len(errors), + "index_path": str(self.index_path), + } + + def import_saved_pages( + self, + paths: Path | str | Iterable[Path | str], + source_slug: str, + ) -> dict[str, object]: + if source_slug not in SOURCE_ADAPTERS: + raise OfflineArchiveError(f"Unsupported wiki source: {source_slug}") + inputs = [paths] if isinstance(paths, (str, Path)) else list(paths) + pages = list(_read_saved_pages(Path(item) for item in inputs)) + documents: list[SourceDocument] = [] + for page in pages: + metadata = _PageMetadataParser() + metadata.feed(page.text) + canonical = metadata.canonical_url.strip() + if canonical and not _allowed_source_url(canonical, source_slug): + canonical = "" + if not canonical: + digest = hashlib.sha256(page.text.encode("utf-8")).hexdigest() + canonical = f"offline-import://{source_slug}/{digest}" + title, _body = extract_article_text(page.text, metadata.title.strip() or page.name) + documents.append( + self.client.store_text( + canonical, + title, + "wiki_article", + page.text, + imported_from=page.name, + ) + ) + + db = DatabaseManager(str(self.database_path)) + with db.get_connection() as connection: + repository = KnowledgeRepository(connection) + summary = KnowledgeImportService(repository).run_adapter( + _SavedWikiAdapter(source_slug, documents, self.client) + ) + connection.execute( + """ + INSERT INTO offline_page_import_runs( + source_slug, source_path, started_at, finished_at, status, + files_seen, files_imported, errors + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + source_slug, + "; ".join(str(Path(item)) for item in inputs), + utc_now(), + utc_now(), + summary["status"], + len(pages), + summary["records_imported"], + json.dumps([summary.get("error", "")] if summary.get("error") else []), + ), + ) + enriched = db.enrich_existing_titleids_from_knowledge() + archive = self.rebuild() + return { + **summary, + "files_seen": len(pages), + "titleids_enriched": enriched, + "archive": archive, + } + + @staticmethod + def _article_html(row: dict[str, object], title: str, body: str, stale: bool) -> str: + status = "Offline copy used after a refresh failure" if stale else "Cached copy" + source_link = _safe_web_url(str(row["url"])) or _safe_web_url( + str(row["homepage_url"] or "") + ) + source_footer = ( + f'

Open original source

' + if source_link + else "" + ) + paragraphs = "\n".join( + f"

{html.escape(line)}

" for line in body.splitlines() if line.strip() + ) + return _shell( + title, + f""" + +

{html.escape(str(row['source_name']))}

+

{html.escape(title)}

+
{html.escape(status)}Saved {html.escape(str(row['fetched_at'] or 'Unknown'))}
+
{paragraphs}
+
+""", + ) + + @staticmethod + def _index_html(documents: list[dict[str, object]]) -> str: + items = "\n".join( + f"""
  • +{html.escape(str(item['title']))} +{html.escape(str(item['source']))}{' | offline fallback' if item['stale'] else ''}
    • """ + for item in documents + ) or "
  • No cached wiki articles are available yet.
    • " + return _shell( + "Offline Knowledge Library", + f""" +

    UnityScraper

    Offline Knowledge Library

    +

    A private, readable archive built from pages you synchronized or imported.

    + +
    + +""", + ) + + +def _read_saved_pages(paths: Iterable[Path]) -> Iterable[_SavedPage]: + count = 0 + total = 0 + for path in paths: + candidates = sorted(path.rglob("*")) if path.is_dir() else [path] + for candidate in candidates: + if candidate.is_symlink(): + continue + if candidate.suffix.casefold() == ".zip": + with zipfile.ZipFile(candidate) as archive: + for info in archive.infolist(): + if info.is_dir() or Path(info.filename).suffix.casefold() not in {".html", ".htm"}: + continue + _validate_size(info.file_size, total, count) + data = archive.read(info) + count += 1 + total += len(data) + yield _SavedPage(f"{candidate}!{info.filename}", _decode_html(data)) + continue + if candidate.suffix.casefold() not in {".html", ".htm"}: + continue + size = candidate.stat().st_size + _validate_size(size, total, count) + data = candidate.read_bytes() + count += 1 + total += len(data) + yield _SavedPage(str(candidate), _decode_html(data)) + if count == 0: + raise OfflineArchiveError("No HTML pages were found in the selected files") + + +def _validate_size(size: int, total: int, count: int) -> None: + if count >= MAX_FILES: + raise OfflineArchiveError(f"Saved-page import is limited to {MAX_FILES} files") + if size > MAX_FILE_BYTES: + raise OfflineArchiveError("A saved page exceeds the 10 MB safety limit") + if total + size > MAX_TOTAL_BYTES: + raise OfflineArchiveError("Saved-page import exceeds the 250 MB safety limit") + + +def _decode_html(data: bytes) -> str: + for encoding in ("utf-8-sig", "utf-8", "windows-1252"): + try: + return data.decode(encoding) + except UnicodeDecodeError: + continue + return data.decode("utf-8", errors="replace") + + +def _allowed_source_url(url: str, source_slug: str) -> bool: + parsed = urlparse(url) + return parsed.scheme in {"http", "https"} and parsed.netloc.casefold() in SOURCE_HOSTS[source_slug] + + +def _safe_web_url(url: str) -> str: + parsed = urlparse(url) + return url if parsed.scheme in {"http", "https"} and parsed.netloc else "" + + +def _slug(value: str) -> str: + return re.sub(r"[^a-z0-9]+", "-", value.casefold()).strip("-")[:80] or "article" + + +def _json_object(value: object) -> dict[str, object]: + try: + parsed = json.loads(str(value or "{}")) + return parsed if isinstance(parsed, dict) else {} + except ValueError: + return {} + + +def _atomic_write(path: Path, text: str) -> None: + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text(text, encoding="utf-8") + temporary.replace(path) + + +def _shell(title: str, content: str) -> str: + return f""" +{html.escape(title)} +{content}""" diff --git a/tests.py b/tests.py index c2c290e..6ac8ca9 100644 --- a/tests.py +++ b/tests.py @@ -35,7 +35,13 @@ ) from external_tools_gui import bundled_xextool_path from knowledge_service import KnowledgeService -from knowledge_sources import KnowledgeImportService, SourceInfo +from knowledge_sources import ( + CachedHttpClient, + KnowledgeImportService, + SourceAccessBlockedError, + SourceInfo, +) +from offline_knowledge import OfflineKnowledgeArchive from library_service import LibraryService from modern_gui import LE_FLUFFIE_CREATOR, XEXTOOL_CREATOR, navigation_shortcut from title_catalog import XboxUnityTitleCatalog @@ -913,6 +919,115 @@ def parse_document(document): run = service.list_import_runs()[0] self.assertEqual(run["status"], "failed") self.assertIn("source unavailable", run["errors"]) + + +class TestOfflineKnowledgeArchive(unittest.TestCase): + """Test blocked-source fallback, saved-page import, and local rendering.""" + + def setUp(self): + self.temp_dir = Path(tempfile.mkdtemp()) + self.db_path = self.temp_dir / "knowledge.db" + self.cache_dir = self.temp_dir / "cache" + self.output_dir = self.temp_dir / "offline" + + def tearDown(self): + shutil.rmtree(self.temp_dir) + + @staticmethod + def _response(status: int, text: str, url: str): + response = Mock() + response.status_code = status + response.text = text + response.url = url + response.headers = {"Server": "cloudflare"} if status != 200 else {} + if status == 200: + response.raise_for_status.return_value = None + return response + + def test_browser_challenge_uses_cache_and_preserves_fetch_time(self): + url = "https://xenonlibrary.com/wiki/Motherboard" + session = Mock() + session.get.side_effect = ( + self._response(200, "

    Motherboard

    Known data.

    ", url), + self._response(403, "Just a moment", url), + ) + client = CachedHttpClient( + cache_dir=self.cache_dir, + rate_limit_seconds=0, + session=session, + ) + fresh = client.get_text(url, "Motherboard", "wiki_article") + cached = client.get_text(url, "Motherboard", "wiki_article") + cached_again = client.get_text(url, "Motherboard", "wiki_article") + self.assertTrue(cached.from_cache) + self.assertTrue(cached_again.from_cache) + self.assertEqual(cached.fetched_at, fresh.fetched_at) + self.assertIn("browser verification", cached.fetch_error) + self.assertEqual(session.get.call_count, 2) + self.assertEqual(client.cached_urls(("xenonlibrary.com",)), [url]) + + def test_browser_challenge_without_cache_has_actionable_error(self): + url = "https://consolemods.org/wiki/Xbox_360:Main_Page" + session = Mock() + session.get.return_value = self._response( + 403, "Just a moment", url + ) + client = CachedHttpClient( + cache_dir=self.cache_dir, + rate_limit_seconds=0, + session=session, + ) + with self.assertRaisesRegex(SourceAccessBlockedError, "Import Saved Wiki Pages"): + client.get_text(url, "Xbox 360", "wiki_article") + + def test_saved_page_import_builds_script_free_offline_article(self): + saved = self.temp_dir / "motherboard.html" + saved.write_text( + """Motherboard - XenonLibrary + + +

    Motherboard

    Board revision reference.

    """, + encoding="utf-8", + ) + archive = OfflineKnowledgeArchive( + database_path=self.db_path, + output_dir=self.output_dir, + cache_dir=self.cache_dir, + ) + summary = archive.import_saved_pages(saved, "xenonlibrary") + self.assertEqual(summary["records_imported"], 1) + self.assertTrue(archive.index_path.exists()) + index = archive.index_path.read_text(encoding="utf-8") + self.assertIn("Motherboard", index) + article_path = next((self.output_dir / "pages" / "xenonlibrary").glob("*.html")) + article = article_path.read_text(encoding="utf-8") + self.assertIn("Board revision reference.", article) + self.assertNotIn("tracker.invalid", article) + self.assertIn("https://xenonlibrary.com/wiki/Motherboard", article) + + def test_saved_consolemods_title_list_enriches_unknown_game(self): + saved = self.temp_dir / "titleids.html" + saved.write_text( + """List of Every Xbox 360 Title ID + +

    SQ (5351) --> Square Enix

    +

    SQ-2052 (53510804)

    Hitman: Absolution

    """, + encoding="utf-8", + ) + database = DatabaseManager(str(self.db_path)) + database.add_titleid("53510804", "Unknown game", "Unknown Publisher") + archive = OfflineKnowledgeArchive( + database_path=self.db_path, + output_dir=self.output_dir, + cache_dir=self.cache_dir, + ) + summary = archive.import_saved_pages(saved, "consolemods-wiki") + self.assertEqual(summary["titleids_enriched"], 1) + row = database.get_titleid_info("53510804") + self.assertIsNotNone(row) + assert row is not None + self.assertEqual(row["name"], "Hitman: Absolution") + self.assertEqual(row["publisher"], "Square Enix") class TestDownloadProgress(unittest.TestCase): @@ -1755,7 +1870,10 @@ def test_versioned_migrations_create_all_foundation_tables(self): versions = connection.execute( "SELECT version FROM app_schema_migrations ORDER BY version" ).fetchall() - self.assertEqual([row[0] for row in versions], [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) + self.assertEqual( + [row[0] for row in versions], + [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], + ) self.assertIn("collection_snapshots", tables) self.assertIn("preservation_matches", tables) self.assertIn("console_transfer_jobs", tables) @@ -1776,6 +1894,9 @@ def test_versioned_migrations_create_all_foundation_tables(self): self.assertIn("library_intelligence_runs", tables) self.assertIn("preservation_report_runs", tables) self.assertIn("hardware_inventory_records", tables) + self.assertIn("offline_archive_runs", tables) + self.assertIn("offline_archive_documents", tables) + self.assertIn("offline_page_import_runs", tables) def test_release_readiness_toolkit_exports_portable_nonpersonal_metadata(self): import sqlite3 diff --git a/wiki_adapters.py b/wiki_adapters.py index bff9cc0..483d583 100644 --- a/wiki_adapters.py +++ b/wiki_adapters.py @@ -105,26 +105,33 @@ def __init__( self, client: CachedHttpClient | None = None, max_documents: int | None = None, + known_urls: Iterable[str] = (), ) -> None: self.client = client or CachedHttpClient(rate_limit_seconds=1.0) self.max_documents = max_documents + self.known_urls = tuple(known_urls) + self.errors: list[str] = [] def fetch_documents(self) -> Iterable[SourceDocument]: urls = self._discover_urls() if self.max_documents is not None: urls = urls[: max(self.max_documents, 0)] errors: list[str] = [] + self.errors = errors + yielded = 0 for url in urls: try: - yield self.client.get_text( + document = self.client.get_text( url, title=_title_from_url(url), document_type="wiki_article", ) + yielded += 1 + yield document except Exception as exc: errors.append(f"{url}: {exc}") continue - if errors: + if errors and not yielded: preview = "; ".join(errors[:5]) if len(errors) > 5: preview += f"; and {len(errors) - 5} more" @@ -168,7 +175,15 @@ def parse_document(self, document: SourceDocument) -> ParsedDocument: return ParsedDocument(document, (record,)) def _discover_urls(self) -> list[str]: - discovered: set[str] = set() + discovered: set[str] = { + url for url in self.seed_urls if self._allowed(url) + } + discovered.update(url for url in self.known_urls if self._allowed(url)) + discovered.update( + url + for url in self.client.cached_urls(self.allowed_hosts) + if self._allowed(url) + ) discovered.update(self._discover_mediawiki_urls()) pending = list(self.sitemap_urls) visited: set[str] = set() @@ -190,8 +205,6 @@ def _discover_urls(self) -> list[str]: pending.extend(url for url in child_sitemaps if url not in visited) discovered.update(url for url in page_urls if self._allowed(url)) - if not discovered: - discovered.update(url for url in self.seed_urls if self._allowed(url)) return sorted(discovered) def _discover_mediawiki_urls(self) -> list[str]: