Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,8 @@ Notable changes to UnityScraper are documented here. The project follows

### Changed

- Cached XboxUnity titles now resolve immediately in library lists and details,
enrich matching rows page by page, and recover after interrupted refreshes.
- Library rows now show `Unknown game` instead of duplicating the TitleID when
no real game name is known.
- Cached XboxUnity names enrich only blank, unknown, or TitleID-shaped values
Expand Down
30 changes: 30 additions & 0 deletions database.py
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,7 @@ def init_database(self):
KnowledgeRepository(conn).ensure_schema()
ensure_backup_schema(conn)
ensure_application_schema(conn)
self._enrich_existing_titleids_from_catalog_connection(conn)

logger.info(f"Database initialized at {self.db_path}")

Expand Down Expand Up @@ -193,6 +194,35 @@ def enrich_existing_titleids_from_knowledge(self) -> int:
logger.error(f"Failed to enrich TitleIDs from knowledge data: {e}")
return 0

def enrich_existing_titleids_from_catalog(self) -> int:
"""Fill unknown names from XboxUnity titles already present in the cache."""
try:
with self.get_connection() as conn:
return self._enrich_existing_titleids_from_catalog_connection(conn)
except Exception as e:
logger.error(f"Failed to enrich TitleIDs from XboxUnity catalog: {e}")
return 0

def _enrich_existing_titleids_from_catalog_connection(self, conn) -> int:
rows = conn.execute(
"""
SELECT t.titleid
FROM titleids AS t
JOIN xboxunity_title_catalog AS c ON c.titleid = t.titleid
WHERE t.name IS NULL
OR TRIM(t.name) = ''
OR UPPER(TRIM(t.name)) = UPPER(t.titleid)
OR LOWER(TRIM(t.name)) IN (
'unknown', 'unknown game', 'unknown title',
'n/a', 'none', 'null'
)
"""
).fetchall()
return sum(
self._enrich_unknown_titleid_from_catalog(conn, row["titleid"])
for row in rows
)

def _enrich_unknown_titleid_metadata(self, conn, titleid: str) -> int:
"""Apply preferred knowledge facts only where local values are unknown."""
repository = KnowledgeRepository(conn)
Expand Down
46 changes: 38 additions & 8 deletions library_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -56,21 +56,26 @@ def list_games(self, search: str = "") -> list[GameSummary]:
CASE
WHEN t.name IS NULL OR TRIM(t.name) = ''
OR UPPER(TRIM(t.name)) = UPPER(t.titleid)
THEN 'Unknown game'
OR LOWER(TRIM(t.name)) IN (
'unknown', 'unknown game', 'unknown title',
'n/a', 'none', 'null'
)
THEN COALESCE(NULLIF(TRIM(xc.name), ''), 'Unknown game')
ELSE t.name
END AS name,
COALESCE(t.publisher, '') AS publisher,
COALESCE(t.last_scraped, '') AS last_scraped,
COUNT(DISTINCT c.id) AS covers_total,
COUNT(DISTINCT CASE WHEN c.status = 'downloaded' THEN c.id END)
COUNT(DISTINCT cv.id) AS covers_total,
COUNT(DISTINCT CASE WHEN cv.status = 'downloaded' THEN cv.id END)
AS covers_downloaded,
COUNT(DISTINCT u.id) AS updates_total,
COUNT(DISTINCT CASE WHEN u.status = 'downloaded' THEN u.id END)
AS updates_downloaded,
COUNT(DISTINCT CASE WHEN u.status = 'failed' THEN u.id END)
AS updates_failed
FROM titleids AS t
LEFT JOIN covers AS c ON c.titleid = t.titleid
LEFT JOIN xboxunity_title_catalog AS xc ON xc.titleid = t.titleid
LEFT JOIN covers AS cv ON cv.titleid = t.titleid
LEFT JOIN title_updates AS u ON u.titleid = t.titleid
"""
parameters: list[Any] = []
Expand All @@ -79,13 +84,14 @@ def list_games(self, search: str = "") -> list[GameSummary]:
query += """
WHERE LOWER(t.titleid) LIKE ?
OR LOWER(COALESCE(t.name, '')) LIKE ?
OR LOWER(COALESCE(xc.name, '')) LIKE ?
OR LOWER(COALESCE(t.publisher, '')) LIKE ?
"""
value = f"%{search.strip().lower()}%"
parameters.extend([value, value, value])
parameters.extend([value, value, value, value])

query += """
GROUP BY t.titleid, t.name, t.publisher, t.last_scraped
GROUP BY t.titleid, t.name, t.publisher, t.last_scraped, xc.name
ORDER BY name COLLATE NOCASE, t.titleid
"""

Expand Down Expand Up @@ -114,7 +120,12 @@ def get_game_details(self, titleid: str) -> dict[str, Any]:

with closing(self._connect()) as connection:
title = connection.execute(
"SELECT * FROM titleids WHERE titleid = ?",
"""
SELECT t.*, xc.name AS catalog_name
FROM titleids AS t
LEFT JOIN xboxunity_title_catalog AS xc ON xc.titleid = t.titleid
WHERE t.titleid = ?
""",
(titleid,),
).fetchone()

Expand Down Expand Up @@ -147,8 +158,27 @@ def get_game_details(self, titleid: str) -> dict[str, Any]:
(titleid,),
).fetchall()

title_record = dict(title)
current_name = title_record.get("name")
unknown_names = {
"",
"unknown",
"unknown game",
"unknown title",
"n/a",
"none",
"null",
}
if (
current_name is None
or str(current_name).strip().casefold() in unknown_names
or str(current_name).strip().upper() == titleid.upper()
):
title_record["name"] = title_record.get("catalog_name")
title_record.pop("catalog_name", None)

return {
"title": dict(title),
"title": title_record,
"covers": [dict(row) for row in covers],
"updates": [dict(row) for row in updates],
}
Expand Down
70 changes: 70 additions & 0 deletions tests.py
Original file line number Diff line number Diff line change
Expand Up @@ -542,6 +542,76 @@ def test_library_does_not_display_titleid_as_the_game_name(self):

self.assertEqual(games[0].name, "Unknown game")

def test_library_uses_cached_catalog_name_before_sync_finishes(self):
self.database.add_titleid("53510804")
catalog = XboxUnityTitleCatalog(self.db_path)
catalog._store_page(
[{"TitleID": "53510804", "Name": "Hitman: Absolution", "TitleType": "360"}],
"http://xboxunity.net/Resources/Lib/TitleList.php?page=47",
)

library = LibraryService(self.db_path)
games = library.list_games("Hitman")
details = library.get_game_details("53510804")

self.assertEqual(games[0].name, "Hitman: Absolution")
self.assertEqual(details["title"]["name"], "Hitman: Absolution")

def test_failed_sync_keeps_page_progress_and_enriches_downloaded_names(self):
self.database.add_titleid("53510804")
session = Mock()
session.get.side_effect = [
self._response(
[{"TitleID": "53510804", "Name": "Hitman: Absolution"}],
pages=2,
page=0,
),
requests.ConnectionError("connection lost"),
]
catalog = XboxUnityTitleCatalog(
self.db_path,
session=session,
request_interval=0,
)

with self.assertRaises(requests.ConnectionError):
catalog.sync()

self.assertEqual(
self.database.get_titleid_info("53510804")["name"],
"Hitman: Absolution",
)
with self.database.get_connection() as connection:
run = connection.execute(
"""
SELECT status, pages_expected, pages_fetched, items_upserted
FROM xboxunity_catalog_sync_runs
ORDER BY id DESC
LIMIT 1
"""
).fetchone()
self.assertEqual(dict(run), {
"status": "failed",
"pages_expected": 2,
"pages_fetched": 1,
"items_upserted": 1,
})

def test_database_startup_repairs_names_from_an_interrupted_cache(self):
self.database.add_titleid("53510804")
catalog = XboxUnityTitleCatalog(self.db_path)
catalog._store_page(
[{"TitleID": "53510804", "Name": "Hitman: Absolution"}],
"http://xboxunity.net/Resources/Lib/TitleList.php?page=47",
)

reopened = DatabaseManager(self.db_path)

self.assertEqual(
reopened.get_titleid_info("53510804")["name"],
"Hitman: Absolution",
)

def test_non_http_xboxunity_base_url_is_rejected(self):
with self.assertRaises(ValueError):
XboxUnityTitleCatalog(self.db_path, base_url="https://xboxunity.net")
Expand Down
31 changes: 29 additions & 2 deletions title_catalog.py
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,18 @@ def sync(
"""Refresh every title page, preserving the last usable cache on failure."""
started_at = _now()
with self._connection() as connection:
connection.execute(
"""
UPDATE xboxunity_catalog_sync_runs
SET completed_at = ?, status = 'interrupted',
error_message = COALESCE(
error_message,
'Application exited before catalog refresh completed'
)
WHERE status = 'running'
""",
(started_at,),
)
cursor = connection.execute(
"""
INSERT INTO xboxunity_catalog_sync_runs(started_at, status)
Expand All @@ -186,6 +198,7 @@ def sync(
pages_expected = 0
pages_fetched = 0
items_upserted = 0
library_names_enriched = 0
try:
page = 0
while page == 0 or page < pages_expected:
Expand All @@ -196,6 +209,16 @@ def sync(
raise ValueError("XboxUnity title list returned invalid Items data")
items_upserted += self._store_page(items, source_url)
pages_fetched += 1
library_names_enriched += self.enrich_library_names()
with self._connection() as connection:
connection.execute(
"""
UPDATE xboxunity_catalog_sync_runs
SET pages_expected = ?, pages_fetched = ?, items_upserted = ?
WHERE id = ?
""",
(pages_expected, pages_fetched, items_upserted, run_id),
)
if progress:
progress(pages_fetched, pages_expected, items_upserted)
page += 1
Expand All @@ -207,7 +230,7 @@ def sync(
"DELETE FROM xboxunity_title_catalog WHERE fetched_at < ?",
(started_at,),
)
enriched = self.enrich_library_names()
library_names_enriched += self.enrich_library_names()
with self._connection() as connection:
connection.execute(
"""
Expand All @@ -224,7 +247,11 @@ def sync(
run_id,
),
)
return CatalogSyncResult(pages_fetched, items_upserted, enriched)
return CatalogSyncResult(
pages_fetched,
items_upserted,
library_names_enriched,
)
except Exception as exc:
with self._connection() as connection:
connection.execute(
Expand Down
Loading