diff --git a/CHANGELOG.md b/CHANGELOG.md index 736be6f..d2fb68d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,8 @@ Notable changes to UnityScraper are documented here. The project follows ### Changed +- Cached XboxUnity titles now resolve immediately in library lists and details, + enrich matching rows page by page, and recover after interrupted refreshes. - Library rows now show `Unknown game` instead of duplicating the TitleID when no real game name is known. - Cached XboxUnity names enrich only blank, unknown, or TitleID-shaped values diff --git a/database.py b/database.py index 96f8f60..28674f0 100644 --- a/database.py +++ b/database.py @@ -146,6 +146,7 @@ def init_database(self): KnowledgeRepository(conn).ensure_schema() ensure_backup_schema(conn) ensure_application_schema(conn) + self._enrich_existing_titleids_from_catalog_connection(conn) logger.info(f"Database initialized at {self.db_path}") @@ -193,6 +194,35 @@ def enrich_existing_titleids_from_knowledge(self) -> int: logger.error(f"Failed to enrich TitleIDs from knowledge data: {e}") return 0 + def enrich_existing_titleids_from_catalog(self) -> int: + """Fill unknown names from XboxUnity titles already present in the cache.""" + try: + with self.get_connection() as conn: + return self._enrich_existing_titleids_from_catalog_connection(conn) + except Exception as e: + logger.error(f"Failed to enrich TitleIDs from XboxUnity catalog: {e}") + return 0 + + def _enrich_existing_titleids_from_catalog_connection(self, conn) -> int: + rows = conn.execute( + """ + SELECT t.titleid + FROM titleids AS t + JOIN xboxunity_title_catalog AS c ON c.titleid = t.titleid + WHERE t.name IS NULL + OR TRIM(t.name) = '' + OR UPPER(TRIM(t.name)) = UPPER(t.titleid) + OR LOWER(TRIM(t.name)) IN ( + 'unknown', 'unknown game', 'unknown title', + 'n/a', 'none', 'null' + ) + """ + ).fetchall() + return sum( + self._enrich_unknown_titleid_from_catalog(conn, row["titleid"]) + for row in rows + ) + def _enrich_unknown_titleid_metadata(self, conn, titleid: str) -> int: """Apply preferred knowledge facts only where local values are unknown.""" repository = KnowledgeRepository(conn) diff --git a/library_service.py b/library_service.py index 5fbbf67..0a178c3 100644 --- a/library_service.py +++ b/library_service.py @@ -56,13 +56,17 @@ def list_games(self, search: str = "") -> list[GameSummary]: CASE WHEN t.name IS NULL OR TRIM(t.name) = '' OR UPPER(TRIM(t.name)) = UPPER(t.titleid) - THEN 'Unknown game' + OR LOWER(TRIM(t.name)) IN ( + 'unknown', 'unknown game', 'unknown title', + 'n/a', 'none', 'null' + ) + THEN COALESCE(NULLIF(TRIM(xc.name), ''), 'Unknown game') ELSE t.name END AS name, COALESCE(t.publisher, '') AS publisher, COALESCE(t.last_scraped, '') AS last_scraped, - COUNT(DISTINCT c.id) AS covers_total, - COUNT(DISTINCT CASE WHEN c.status = 'downloaded' THEN c.id END) + COUNT(DISTINCT cv.id) AS covers_total, + COUNT(DISTINCT CASE WHEN cv.status = 'downloaded' THEN cv.id END) AS covers_downloaded, COUNT(DISTINCT u.id) AS updates_total, COUNT(DISTINCT CASE WHEN u.status = 'downloaded' THEN u.id END) @@ -70,7 +74,8 @@ def list_games(self, search: str = "") -> list[GameSummary]: COUNT(DISTINCT CASE WHEN u.status = 'failed' THEN u.id END) AS updates_failed FROM titleids AS t - LEFT JOIN covers AS c ON c.titleid = t.titleid + LEFT JOIN xboxunity_title_catalog AS xc ON xc.titleid = t.titleid + LEFT JOIN covers AS cv ON cv.titleid = t.titleid LEFT JOIN title_updates AS u ON u.titleid = t.titleid """ parameters: list[Any] = [] @@ -79,13 +84,14 @@ def list_games(self, search: str = "") -> list[GameSummary]: query += """ WHERE LOWER(t.titleid) LIKE ? OR LOWER(COALESCE(t.name, '')) LIKE ? + OR LOWER(COALESCE(xc.name, '')) LIKE ? OR LOWER(COALESCE(t.publisher, '')) LIKE ? """ value = f"%{search.strip().lower()}%" - parameters.extend([value, value, value]) + parameters.extend([value, value, value, value]) query += """ - GROUP BY t.titleid, t.name, t.publisher, t.last_scraped + GROUP BY t.titleid, t.name, t.publisher, t.last_scraped, xc.name ORDER BY name COLLATE NOCASE, t.titleid """ @@ -114,7 +120,12 @@ def get_game_details(self, titleid: str) -> dict[str, Any]: with closing(self._connect()) as connection: title = connection.execute( - "SELECT * FROM titleids WHERE titleid = ?", + """ + SELECT t.*, xc.name AS catalog_name + FROM titleids AS t + LEFT JOIN xboxunity_title_catalog AS xc ON xc.titleid = t.titleid + WHERE t.titleid = ? + """, (titleid,), ).fetchone() @@ -147,8 +158,27 @@ def get_game_details(self, titleid: str) -> dict[str, Any]: (titleid,), ).fetchall() + title_record = dict(title) + current_name = title_record.get("name") + unknown_names = { + "", + "unknown", + "unknown game", + "unknown title", + "n/a", + "none", + "null", + } + if ( + current_name is None + or str(current_name).strip().casefold() in unknown_names + or str(current_name).strip().upper() == titleid.upper() + ): + title_record["name"] = title_record.get("catalog_name") + title_record.pop("catalog_name", None) + return { - "title": dict(title), + "title": title_record, "covers": [dict(row) for row in covers], "updates": [dict(row) for row in updates], } diff --git a/tests.py b/tests.py index 1b81da9..59dab41 100644 --- a/tests.py +++ b/tests.py @@ -542,6 +542,76 @@ def test_library_does_not_display_titleid_as_the_game_name(self): self.assertEqual(games[0].name, "Unknown game") + def test_library_uses_cached_catalog_name_before_sync_finishes(self): + self.database.add_titleid("53510804") + catalog = XboxUnityTitleCatalog(self.db_path) + catalog._store_page( + [{"TitleID": "53510804", "Name": "Hitman: Absolution", "TitleType": "360"}], + "http://xboxunity.net/Resources/Lib/TitleList.php?page=47", + ) + + library = LibraryService(self.db_path) + games = library.list_games("Hitman") + details = library.get_game_details("53510804") + + self.assertEqual(games[0].name, "Hitman: Absolution") + self.assertEqual(details["title"]["name"], "Hitman: Absolution") + + def test_failed_sync_keeps_page_progress_and_enriches_downloaded_names(self): + self.database.add_titleid("53510804") + session = Mock() + session.get.side_effect = [ + self._response( + [{"TitleID": "53510804", "Name": "Hitman: Absolution"}], + pages=2, + page=0, + ), + requests.ConnectionError("connection lost"), + ] + catalog = XboxUnityTitleCatalog( + self.db_path, + session=session, + request_interval=0, + ) + + with self.assertRaises(requests.ConnectionError): + catalog.sync() + + self.assertEqual( + self.database.get_titleid_info("53510804")["name"], + "Hitman: Absolution", + ) + with self.database.get_connection() as connection: + run = connection.execute( + """ + SELECT status, pages_expected, pages_fetched, items_upserted + FROM xboxunity_catalog_sync_runs + ORDER BY id DESC + LIMIT 1 + """ + ).fetchone() + self.assertEqual(dict(run), { + "status": "failed", + "pages_expected": 2, + "pages_fetched": 1, + "items_upserted": 1, + }) + + def test_database_startup_repairs_names_from_an_interrupted_cache(self): + self.database.add_titleid("53510804") + catalog = XboxUnityTitleCatalog(self.db_path) + catalog._store_page( + [{"TitleID": "53510804", "Name": "Hitman: Absolution"}], + "http://xboxunity.net/Resources/Lib/TitleList.php?page=47", + ) + + reopened = DatabaseManager(self.db_path) + + self.assertEqual( + reopened.get_titleid_info("53510804")["name"], + "Hitman: Absolution", + ) + def test_non_http_xboxunity_base_url_is_rejected(self): with self.assertRaises(ValueError): XboxUnityTitleCatalog(self.db_path, base_url="https://xboxunity.net") diff --git a/title_catalog.py b/title_catalog.py index 390f2d8..6d8541c 100644 --- a/title_catalog.py +++ b/title_catalog.py @@ -172,6 +172,18 @@ def sync( """Refresh every title page, preserving the last usable cache on failure.""" started_at = _now() with self._connection() as connection: + connection.execute( + """ + UPDATE xboxunity_catalog_sync_runs + SET completed_at = ?, status = 'interrupted', + error_message = COALESCE( + error_message, + 'Application exited before catalog refresh completed' + ) + WHERE status = 'running' + """, + (started_at,), + ) cursor = connection.execute( """ INSERT INTO xboxunity_catalog_sync_runs(started_at, status) @@ -186,6 +198,7 @@ def sync( pages_expected = 0 pages_fetched = 0 items_upserted = 0 + library_names_enriched = 0 try: page = 0 while page == 0 or page < pages_expected: @@ -196,6 +209,16 @@ def sync( raise ValueError("XboxUnity title list returned invalid Items data") items_upserted += self._store_page(items, source_url) pages_fetched += 1 + library_names_enriched += self.enrich_library_names() + with self._connection() as connection: + connection.execute( + """ + UPDATE xboxunity_catalog_sync_runs + SET pages_expected = ?, pages_fetched = ?, items_upserted = ? + WHERE id = ? + """, + (pages_expected, pages_fetched, items_upserted, run_id), + ) if progress: progress(pages_fetched, pages_expected, items_upserted) page += 1 @@ -207,7 +230,7 @@ def sync( "DELETE FROM xboxunity_title_catalog WHERE fetched_at < ?", (started_at,), ) - enriched = self.enrich_library_names() + library_names_enriched += self.enrich_library_names() with self._connection() as connection: connection.execute( """ @@ -224,7 +247,11 @@ def sync( run_id, ), ) - return CatalogSyncResult(pages_fetched, items_upserted, enriched) + return CatalogSyncResult( + pages_fetched, + items_upserted, + library_names_enriched, + ) except Exception as exc: with self._connection() as connection: connection.execute(