From e1366f6e7f663aaed2e3afddf652a9412cfcb7f1 Mon Sep 17 00:00:00 2001 From: Sthornberry9 <46094434+Sthornberry9@users.noreply.github.com> Date: Fri, 24 Jul 2026 04:18:31 -0400 Subject: [PATCH] feat: complete profile intelligence roadmap --- ARCHITECTURE.md | 24 ++- CHANGELOG.md | 12 ++ CONSOLE_SYNC.md | 7 + DOCS_INDEX.md | 2 + KNOWLEDGE_SOURCES.md | 6 +- PROFILES_AND_SAVES.md | 11 ++ PROFILE_INTELLIGENCE.md | 80 ++++++++ PROJECT_STATUS.md | 18 +- README.md | 13 +- SECURITY.md | 6 + UnityScraper.spec | 4 + backup_gui.py | 7 + console_sync.py | 36 +++- database_migrations.py | 115 ++++++++++- gpd_parser.py | 293 ++++++++++++++++++++++++++++ knowledge_base.py | 48 ++++- knowledge_gui.py | 175 +++++++++++++++++ knowledge_scheduler.py | 143 ++++++++++++++ knowledge_service.py | 108 ++++++++++- modern_gui.py | 20 ++ profile_gui.py | 415 ++++++++++++++++++++++++++++++++++++++++ profile_intelligence.py | 405 +++++++++++++++++++++++++++++++++++++++ tests.py | 188 +++++++++++++++++- xenia_bridge.py | 239 +++++++++++++++++++++++ 24 files changed, 2353 insertions(+), 22 deletions(-) create mode 100644 PROFILE_INTELLIGENCE.md create mode 100644 gpd_parser.py create mode 100644 knowledge_scheduler.py create mode 100644 profile_intelligence.py create mode 100644 xenia_bridge.py diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 9f40a2c..a388899 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -63,10 +63,12 @@ Main schema groups: - Library: `titleids`, `title_updates`, `covers`, `download_history` - Knowledge: sources, documents, revisions, entities, names, identifiers, - facts, citations, relationships, import runs, and conflicts + facts, citations, relationships, import runs, conflicts, source priorities, + conflict decisions, and scheduled sync state - Backups: targets, scans, inventory, and operations -- Profiles: scan runs, profiles, saves, snapshots, snapshot files, and - auditable operations +- Profiles: scan runs, profiles, saves, snapshots, snapshot files, GPD + inventories, achievements, comparisons, Xenia migration runs, and auditable + operations Schema initialization is idempotent. New migrations should preserve existing data and be covered by tests. @@ -105,6 +107,22 @@ user-selected file or ZIP Game payloads are not stored in SQLite. Inventory records contain paths, identifiers, sizes, statuses, and notes. +### Profile Intelligence + +```text +standalone/extracted XDBF file + -> bounded table and offset validation + -> read-only achievement/setting parsing + -> local inventory and profile comparison + +indexed saves + Xenia content root + -> non-mutating migration preview + -> verified automatic snapshot + -> .partial copy and SHA-256 verification + -> skip identical / retain conflicts + -> migration audit record +``` + ### Profile Snapshot ```text diff --git a/CHANGELOG.md b/CHANGELOG.md index 9fe5750..4328463 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,18 @@ Notable changes to UnityScraper are documented here. The project follows ### Added +- Read-only XDBF/GPD inspection with bounded entry parsing, achievement state, + gamerscore summaries, extracted-folder discovery, and local inventory. +- Profile comparison for save hashes and imported achievement state. +- Snapshot-first Xenia migration previews with atomic verified copies and + non-overwriting conflict handling. +- Additive schema migration 7 for GPDs, achievements, comparisons, Xenia + migration runs, source priorities, conflict decisions, sync schedules, and + opt-in remote hash verification. +- Field-specific knowledge source priorities, auditable conflict resolution, + and opt-in app-start knowledge refresh scheduling. +- Feature-detected remote SHA-256 verification for compatible console FTP + dashboards. - Profiles & Saves workspace with read-only Content-tree discovery, masked profile inventory, STFS ownership metadata, save search, duplicate and mismatch reporting, verified snapshots, manifests, and conflict-safe restore. diff --git a/CONSOLE_SYNC.md b/CONSOLE_SYNC.md index 788098e..cb29a37 100644 --- a/CONSOLE_SYNC.md +++ b/CONSOLE_SYNC.md @@ -11,6 +11,8 @@ whose FTP server the user explicitly configures. - FTP `REST` is used when the server supports ranged transfer. - Uploads are published by renaming the completed partial file. - Final sizes are verified; downloads can also require a SHA-256. +- Uploads can optionally require a remote SHA-256 when the dashboard exposes a + compatible read-only hash command. - Each job can have a bytes-per-second bandwidth limit. - Passwords remain in memory and are never stored. @@ -22,6 +24,11 @@ files. A snapshot can be compared with a local directory to find files only on the PC, only on the console, different-sized files, and matching files. Discovery has a default 100,000-entry safety limit. +Remote hash verification probes `XSHA256`, standardized `HASH`, and compatible +`SITE SHA256` commands. When the option is enabled, a server without one of +those commands fails verification instead of silently falling back to size +only. + ```powershell python main.py --ftp-host 192.168.1.50 --ftp-user xbox --ftp-snapshot /Hdd1 diff --git a/DOCS_INDEX.md b/DOCS_INDEX.md index 631eec0..45cf20c 100644 --- a/DOCS_INDEX.md +++ b/DOCS_INDEX.md @@ -7,6 +7,8 @@ verification, and external conversion - [Profiles and Saves](PROFILES_AND_SAVES.md) - profile inventory, save snapshots, privacy, restore behavior, and Le Fluffie attribution +- [Profile Intelligence and Xenia](PROFILE_INTELLIGENCE.md) - read-only GPD + achievements, profile comparison, and snapshot-first Xenia migration - [Collection Intelligence](COLLECTION_INTELLIGENCE.md) - storage discovery, XEX identity, Title Update compatibility, preservation, and repair previews - [Console Sync](CONSOLE_SYNC.md) - persistent transfers, resume, snapshots, diff --git a/KNOWLEDGE_SOURCES.md b/KNOWLEDGE_SOURCES.md index 82aeb9e..ef562fa 100644 --- a/KNOWLEDGE_SOURCES.md +++ b/KNOWLEDGE_SOURCES.md @@ -111,7 +111,11 @@ The **Knowledge** page includes: - source license, document count, fact count, and latest import status; - ConsoleMods ID sync and whole-wiki sync; - Redump and No-Intro file import; -- conflicting-claim review. +- per-property source priorities, where lower numbers are preferred for + display; +- conflicting-claim review with recorded prefer-existing, prefer-incoming, + and dismiss decisions; +- an opt-in app-start refresh schedule with a minimum six-hour interval. ## Remaining Boundaries diff --git a/PROFILES_AND_SAVES.md b/PROFILES_AND_SAVES.md index 5b8c64c..dc31a31 100644 --- a/PROFILES_AND_SAVES.md +++ b/PROFILES_AND_SAVES.md @@ -95,6 +95,17 @@ incorrectly. Future editing and migration support should only ship with complete package verification, automatic pre-change snapshots, and well-tested cross-platform signing support. +## Profile Intelligence and Xenia + +The workspace also imports standalone or already-extracted XDBF/GPD files for +read-only achievement views, compares two indexed profiles, and previews Xenia +save mappings. Xenia migration always creates a verified save snapshot first, +copies through `.partial` staging, verifies SHA-256, skips identical files, and +never overwrites a different destination. + +See [PROFILE_INTELLIGENCE.md](PROFILE_INTELLIGENCE.md) for the parser bounds, +comparison fields, Xenia paths, and migration safety model. + ## Le Fluffie Attribution The profile/STFS field model is informed by Dalavin, also known as diff --git a/PROFILE_INTELLIGENCE.md b/PROFILE_INTELLIGENCE.md new file mode 100644 index 0000000..e90d7fa --- /dev/null +++ b/PROFILE_INTELLIGENCE.md @@ -0,0 +1,80 @@ +# Profile Intelligence and Xenia + +The **Profiles & Saves** workspace includes read-only profile intelligence and +a snapshot-first bridge for Xenia save folders. + +## GPD and Achievements + +UnityScraper reads standalone or already-extracted Xbox 360 XDBF/GPD files. +Choose **Import GPD** for one file or **Scan Extracted Folder** to find files +whose first four bytes are the `XDBF` signature. + +The bounded parser validates: + +- XDBF magic, version, table capacity, and active counts; +- every entry offset and size before reading it; +- achievement record minimum sizes; +- variable-length setting sizes; +- a 512 MiB per-file safety limit. + +For game GPDs, the application displays achievement ID, title, gamerscore, +locked/unlocked state, and a valid online unlock timestamp when present. It +also records totals for unlocked achievements and earned/possible gamerscore. + +The parser never writes to the source file. It does not unlock achievements, +alter sync records, extract images, edit account settings, or repair malformed +databases. + +UnityScraper currently reads standalone or extracted GPD files. It does not +silently unpack or rewrite the profile's STFS container. + +## Profile Comparison + +Choose two indexed profiles on the **Compare** tab. The report identifies: + +- save TitleIDs present on only one profile; +- TitleIDs whose indexed save hashes differ; +- TitleIDs with identical indexed save hashes; +- imported achievements unlocked by only one profile; +- achievements unlocked by both profiles. + +Comparison history is stored locally. Profile identifiers remain masked in the +normal inventory interface and no profile information is uploaded. + +## Xenia Migration + +Xenia normally keeps saves in a `content` directory. Common locations are +suggested on Windows and Linux, and any Xenia folder or content root can be +selected manually. + +The migration workflow is: + +1. Select an indexed source profile. +2. Choose the Xenia folder and target profile ID. +3. Preview every destination and conflict. +4. Create an automatic verified snapshot. +5. Copy only new files through `.partial` staging. +6. Verify each copied file with SHA-256. + +Identical destination files are skipped. Different files and non-file +destinations are conflicts and are never overwritten. Migration runs, counts, +plans, and their pre-change snapshot IDs are recorded in SQLite. + +Xenia's folder guidance is based on the official +[Xenia Canary FAQ](https://github.com/xenia-canary/xenia-canary/wiki/FAQ) and +[Quickstart](https://github.com/xenia-canary/xenia-canary/wiki/Quickstart). +UnityScraper does not bundle or modify the emulator. + +## Deliberate Boundary + +This release still does not: + +- edit GPD achievements, settings, gamertags, or account blocks; +- rewrite ownership identifiers; +- rehash or resign a modified CON package; +- store signing material, CPU keys, passwords, or Xbox Live credentials; +- write raw FATX devices. + +Those operations require complete package extraction, mutation, rehashing, +signature verification, and recovery testing across real profiles before they +can be offered responsibly. diff --git a/PROJECT_STATUS.md b/PROJECT_STATUS.md index 2cd2eca..f545b1f 100644 --- a/PROJECT_STATUS.md +++ b/PROJECT_STATUS.md @@ -42,6 +42,12 @@ backup-management, and source-attributed knowledge application. reporting, verified snapshots, manifests, and conflict-safe restore. - Credited GPLv3 technical lineage from Dalavin / DJ SkunkieButt's X360 library and Le Fluffie without bundling its updater, keys, or executable. +- Bounded read-only XDBF/GPD achievement inspection, profile comparison, and + snapshot-first Xenia migration with non-overwriting verified copies. +- Per-property knowledge source priorities, recorded conflict decisions, and + opt-in scheduled app-start refreshes. +- Feature-detected, opt-in remote SHA-256 verification for console FTP servers + that expose a compatible read-only command. - Local-by-default REST API with token-required remote binding, restricted browser origins, validated settings, and current version reporting. - Cross-platform CI, Windows packaging checks, tagged release archives, @@ -83,9 +89,9 @@ backup-management, and source-attributed knowledge application. ## Future Work -- Add field-specific source-priority controls and conflict resolution actions. -- Add optional scheduled knowledge refreshes. -- Validate console resume behavior against a broader matrix of dashboard FTP - servers and add opt-in remote hash verification where servers expose it. -- Add read-only GPD/achievement views, Xenia save mapping, and audited - profile-migration previews before considering package mutation. +- Validate console resume and optional hash behavior against a broader matrix + of real dashboard FTP servers. +- Expand read-only GPD coverage with dashboard title-history and safe image + previews after adding a decompression and image validation boundary. +- Consider package mutation only after complete STFS extraction, rehashing, + signing, verification, and automatic recovery have independent test vectors. diff --git a/README.md b/README.md index 60fb5bb..b12d39c 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,9 @@ service exposes. UnityScraper does not silently substitute HTTPS URLs. - Imports user-supplied Redump and No-Intro XML DAT files. - Stores entities, identifiers, facts, citations, revisions, import runs, and conflicts with provenance. +- Supports per-property source priorities and records explicit conflict + decisions without deleting competing claims. +- Offers an opt-in app-start refresh schedule with a minimum six-hour interval. - Fills blank or unknown local metadata without replacing better known values. ### Backup Management @@ -53,6 +56,8 @@ service exposes. UnityScraper does not silently substitute HTTPS URLs. manifest. - Queues resumable uploads and downloads to a configured Aurora-style FTP server. - Captures read-only console inventories and compares PC and console content. +- Can opt into remote SHA-256 verification when the selected FTP dashboard + advertises a compatible read-only hash command. - Runs a user-selected external ISO converter without bundling converter code. ### Profiles and Save Data @@ -67,10 +72,14 @@ service exposes. UnityScraper does not silently substitute HTTPS URLs. verified atomic copies. - Restores snapshots without overwriting different existing files. - Exports portable JSON preservation manifests. +- Reads extracted XDBF/GPD achievement databases without modifying them. +- Compares save hashes and imported achievement state across two profiles. +- Previews Xenia save mappings and creates a verified snapshot before migration. The profile/package model is informed by Dalavin, also known as DJ SkunkieButt, and the GPLv3 X360 library and Le Fluffie source. See -[PROFILES_AND_SAVES.md](PROFILES_AND_SAVES.md) and +[PROFILES_AND_SAVES.md](PROFILES_AND_SAVES.md), +[PROFILE_INTELLIGENCE.md](PROFILE_INTELLIGENCE.md), and [THIRD_PARTY_NOTICES.md](THIRD_PARTY_NOTICES.md). ### External Tools @@ -167,7 +176,7 @@ Linux source setup: | Add Games | Search cached game names, select TitleIDs, or import lists | | Downloads | Review and manage download activity | | Backup Manager | Scan, install, verify, export, convert, and transfer owned content | -| Profiles & Saves | Inventory profiles, back up saves, and restore snapshots | +| Profiles & Saves | Inventory profiles, inspect achievements, compare, snapshot, restore, and migrate to Xenia | | External Tools | Run XeXTool and other user-supplied command-line utilities | | Collections | Identify storage, compare Title Updates, verify preservation data, and preview repairs | | Knowledge | Search sources, facts, citations, imports, and conflicts | diff --git a/SECURITY.md b/SECURITY.md index 85dd9e3..afba2ba 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -37,6 +37,12 @@ filesystem contents in a report. - Profile and save identifiers are masked in the GUI by default. - Profile scans are read-only. Snapshot restores preserve different existing files and write the restored copy alongside them. +- GPD parsing validates bounded entry tables and file offsets and never writes + achievement, setting, sync, or image records. +- Xenia migrations require a preview and verified snapshot, publish through + partial files, and never overwrite different destination data. +- Optional remote hashes use read-only FTP commands and fail closed when the + selected dashboard does not expose a supported SHA-256 response. - Profiles, saves, gamertags, XUIDs, console IDs, and device IDs are not sent to metadata sources. - External converters run only when explicitly configured by the user. diff --git a/UnityScraper.spec b/UnityScraper.spec index 650b396..385c7fb 100644 --- a/UnityScraper.spec +++ b/UnityScraper.spec @@ -27,12 +27,16 @@ a = Analysis( 'dat_adapters', 'knowledge_gui', 'knowledge_service', + 'knowledge_scheduler', 'knowledge_sync', 'plugins', + 'gpd_parser', 'profile_gui', + 'profile_intelligence', 'profile_manager', 'updater', 'wiki_adapters', + 'xenia_bridge', ], hookspath=[], hooksconfig={}, diff --git a/backup_gui.py b/backup_gui.py index 1222b4b..a3d4a1a 100644 --- a/backup_gui.py +++ b/backup_gui.py @@ -184,6 +184,12 @@ def _build_transfer(self) -> None: ttk.Spinbox( limit_row, from_=0, to=102400, textvariable=self.ftp_limit_var, width=10 ).pack(side=tk.LEFT, padx=(8, 0)) + self.ftp_remote_hash_var = tk.BooleanVar(value=False) + ttk.Checkbutton( + limit_row, + text="Require remote SHA-256 when supported", + variable=self.ftp_remote_hash_var, + ).pack(side=tk.LEFT, padx=(18, 0)) self.queue_var = tk.StringVar(value="Persistent queue: empty") ttk.Label(tab, textvariable=self.queue_var, style="Subheader.TLabel").grid( row=9, column=0, columnspan=2, sticky=tk.W, pady=(8, 0) @@ -458,6 +464,7 @@ def queue_upload(self) -> None: source, remote, bandwidth_limit=max(0, int(self.ftp_limit_var.get() or "0")) * 1024, + verify_remote_hash=self.ftp_remote_hash_var.get(), ) except Exception as exc: self._failed(exc) diff --git a/console_sync.py b/console_sync.py index 133561f..8731da4 100644 --- a/console_sync.py +++ b/console_sync.py @@ -6,6 +6,7 @@ import hashlib import json import posixpath +import re import sqlite3 import threading import time @@ -88,6 +89,7 @@ def enqueue( priority: int = 100, bandwidth_limit: int = 0, expected_sha256: str = "", + verify_remote_hash: bool = False, ) -> int: if direction not in {"upload", "download"}: raise ValueError("direction must be upload or download") @@ -100,8 +102,8 @@ def enqueue( INSERT INTO console_transfer_jobs (target_id, direction, local_path, remote_path, total_bytes, status, priority, bandwidth_limit, expected_sha256, - created_at, updated_at) - VALUES (?, ?, ?, ?, ?, 'queued', ?, ?, ?, ?, ?) + verify_remote_hash, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, 'queued', ?, ?, ?, ?, ?, ?) """, ( target_id, @@ -112,6 +114,7 @@ def enqueue( priority, max(0, bandwidth_limit), expected_sha256.lower(), + int(verify_remote_hash), now, now, ), @@ -314,7 +317,13 @@ def _checkpoint(self, job_id, current, total, started, job, progress) -> None: def _verify(self, job: dict, target: FtpTarget, total: int) -> None: if job["direction"] == "upload": with _ftp(target) as ftp: - actual = _remote_size(ftp, _remote_path(job["remote_path"])) + remote_path = _remote_path(job["remote_path"]) + actual = _remote_size(ftp, remote_path) + if bool(job.get("verify_remote_hash")): + remote_digest = _remote_sha256(ftp, remote_path) + local_digest = _sha256(Path(job["local_path"])) + if remote_digest != local_digest: + raise IOError("Remote SHA-256 verification failed") if actual != total: raise IOError(f"Remote verification failed: {actual} != {total}") else: @@ -499,3 +508,24 @@ def _sha256(path: Path) -> str: for chunk in iter(lambda: handle.read(1024 * 1024), b""): digest.update(chunk) return digest.hexdigest() + + +def _remote_sha256(ftp: ftplib.FTP, path: str) -> str: + """Use read-only FTP hash extensions when a dashboard exposes one.""" + attempts = ( + f"XSHA256 {path}", + "OPTS HASH SHA-256", + f"HASH {path}", + f"SITE SHA256 {path}", + ) + for command in attempts: + try: + response = ftp.sendcmd(command) + except ftplib.all_errors: + continue + match = re.search(r"(?i)\b[0-9a-f]{64}\b", response) + if match: + return match.group(0).lower() + raise IOError( + "The console FTP server does not expose a supported SHA-256 command" + ) diff --git a/database_migrations.py b/database_migrations.py index 902aaf6..947a439 100644 --- a/database_migrations.py +++ b/database_migrations.py @@ -8,7 +8,7 @@ from pathlib import Path -SCHEMA_VERSION = 6 +SCHEMA_VERSION = 7 def _now() -> str: @@ -67,6 +67,7 @@ def ensure_application_schema(connection: sqlite3.Connection) -> int: (4, "user overrides and recovery", _migration_reliability), (5, "XboxUnity title catalog", _migration_xboxunity_catalog), (6, "profile and save management", _migration_profiles_and_saves), + (7, "profile intelligence and knowledge controls", _migration_roadmap), ) for version, name, migration in migrations: if version in applied: @@ -386,3 +387,115 @@ def _migration_profiles_and_saves(connection: sqlite3.Connection) -> None: ); """ ) + + +def _migration_roadmap(connection: sqlite3.Connection) -> None: + connection.executescript( + """ + CREATE TABLE IF NOT EXISTS profile_gpd_files ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + profile_id TEXT, + titleid TEXT, + source_path TEXT NOT NULL UNIQUE, + sha256 TEXT NOT NULL, + size INTEGER NOT NULL, + version INTEGER NOT NULL, + entry_count INTEGER NOT NULL, + achievement_count INTEGER NOT NULL DEFAULT 0, + unlocked_count INTEGER NOT NULL DEFAULT 0, + gamerscore_earned INTEGER NOT NULL DEFAULT 0, + gamerscore_possible INTEGER NOT NULL DEFAULT 0, + parsed_at TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'parsed', + warnings_json TEXT + ); + CREATE INDEX IF NOT EXISTS idx_profile_gpd_owner + ON profile_gpd_files(profile_id, titleid); + + CREATE TABLE IF NOT EXISTS profile_achievements ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + gpd_file_id INTEGER NOT NULL, + achievement_id INTEGER NOT NULL, + title TEXT, + locked_description TEXT, + unlocked_description TEXT, + gamerscore INTEGER NOT NULL DEFAULT 0, + unlock_state TEXT NOT NULL, + unlocked_at TEXT, + image_id INTEGER, + entry_id INTEGER, + UNIQUE(gpd_file_id, achievement_id), + FOREIGN KEY(gpd_file_id) REFERENCES profile_gpd_files(id) + ); + CREATE INDEX IF NOT EXISTS idx_profile_achievements_state + ON profile_achievements(gpd_file_id, unlock_state); + + CREATE TABLE IF NOT EXISTS profile_comparisons ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + left_profile_id TEXT NOT NULL, + right_profile_id TEXT NOT NULL, + created_at TEXT NOT NULL, + summary_json TEXT NOT NULL + ); + + CREATE TABLE IF NOT EXISTS xenia_migration_runs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + source_profile_id TEXT NOT NULL, + target_profile_id TEXT NOT NULL, + destination_root TEXT NOT NULL, + snapshot_id INTEGER, + created_at TEXT NOT NULL, + completed_at TEXT, + status TEXT NOT NULL, + copied_count INTEGER NOT NULL DEFAULT 0, + skipped_count INTEGER NOT NULL DEFAULT 0, + conflict_count INTEGER NOT NULL DEFAULT 0, + plan_json TEXT NOT NULL, + error_message TEXT, + FOREIGN KEY(snapshot_id) REFERENCES save_snapshots(id) + ); + + CREATE TABLE IF NOT EXISTS knowledge_source_priorities ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + property TEXT NOT NULL, + source_id INTEGER NOT NULL, + priority INTEGER NOT NULL DEFAULT 100, + updated_at TEXT NOT NULL, + UNIQUE(property, source_id), + FOREIGN KEY(source_id) REFERENCES knowledge_sources(id) + ); + + CREATE TABLE IF NOT EXISTS knowledge_conflict_resolutions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + conflict_id INTEGER NOT NULL, + resolution TEXT NOT NULL, + preferred_value TEXT, + preferred_source_id INTEGER, + notes TEXT, + resolved_at TEXT NOT NULL, + FOREIGN KEY(conflict_id) REFERENCES knowledge_conflicts(id), + FOREIGN KEY(preferred_source_id) REFERENCES knowledge_sources(id) + ); + + CREATE TABLE IF NOT EXISTS scheduled_sync_state ( + task_name TEXT PRIMARY KEY, + enabled INTEGER NOT NULL DEFAULT 0, + interval_hours INTEGER NOT NULL DEFAULT 168, + last_started_at TEXT, + last_completed_at TEXT, + last_status TEXT, + last_error TEXT, + updated_at TEXT NOT NULL + ); + """ + ) + transfer_columns = { + row[1] for row in connection.execute("PRAGMA table_info(console_transfer_jobs)") + } + if "verify_remote_hash" not in transfer_columns: + connection.execute( + """ + ALTER TABLE console_transfer_jobs + ADD COLUMN verify_remote_hash INTEGER NOT NULL DEFAULT 0 + """ + ) diff --git a/gpd_parser.py b/gpd_parser.py new file mode 100644 index 0000000..f08176a --- /dev/null +++ b/gpd_parser.py @@ -0,0 +1,293 @@ +"""Bounded, read-only parser for Xbox 360 XDBF/GPD databases.""" + +from __future__ import annotations + +import hashlib +import re +import struct +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any + + +XDBF_MAGIC = b"XDBF" +XDBF_HEADER = struct.Struct(">4sIIIII") +XDBF_ENTRY = struct.Struct(">Hqii") +XDBF_FREE_ENTRY_SIZE = 8 +MAX_TABLE_ENTRIES = 100_000 +MAX_FILE_SIZE = 512 * 1024 * 1024 +TITLE_ID_RE = re.compile(r"^[0-9A-Fa-f]{8}$") + +NAMESPACE_NAMES = { + 0: "nothing", + 1: "achievement", + 2: "image", + 3: "setting", + 4: "title", + 5: "string", +} + +SETTING_TYPES = { + 0: "context", + 1: "uint32", + 2: "int64", + 3: "double", + 4: "unicode", + 5: "float", + 6: "binary", + 7: "datetime", + 0xFF: "null", +} + + +class GpdError(ValueError): + """Raised when a GPD file is malformed or exceeds a safety bound.""" + + +@dataclass(frozen=True) +class XdbfEntry: + namespace: int + entry_id: int + offset: int + size: int + + @property + def namespace_name(self) -> str: + return NAMESPACE_NAMES.get(self.namespace, f"unknown-{self.namespace}") + + +@dataclass(frozen=True) +class GpdAchievement: + entry_id: int + achievement_id: int + image_id: int + gamerscore: int + state: str + unlocked_at: str + title: str + locked_description: str + unlocked_description: str + + @property + def unlocked(self) -> bool: + return self.state in {"unlocked-offline", "unlocked-online"} + + +@dataclass(frozen=True) +class GpdSetting: + entry_id: int + setting_id: int + value_type: str + value: Any + + +@dataclass(frozen=True) +class GpdReport: + path: Path + title_id: str + version: int + sha256: str + size: int + entry_count: int + achievements: tuple[GpdAchievement, ...] + settings: tuple[GpdSetting, ...] + namespace_counts: dict[str, int] + warnings: tuple[str, ...] + + @property + def unlocked_count(self) -> int: + return sum(item.unlocked for item in self.achievements) + + @property + def gamerscore_earned(self) -> int: + return sum(item.gamerscore for item in self.achievements if item.unlocked) + + @property + def gamerscore_possible(self) -> int: + return sum(item.gamerscore for item in self.achievements) + + +def parse_gpd(path: str | Path, title_id: str = "") -> GpdReport: + """Parse one standalone GPD without changing it.""" + source = Path(path).expanduser().resolve() + if not source.is_file(): + raise FileNotFoundError(source) + size = source.stat().st_size + if size > MAX_FILE_SIZE: + raise GpdError(f"GPD exceeds the {MAX_FILE_SIZE}-byte safety limit") + data = source.read_bytes() + return parse_gpd_bytes(data, path=source, title_id=title_id) + + +def parse_gpd_bytes( + data: bytes, + *, + path: str | Path = "memory.gpd", + title_id: str = "", +) -> GpdReport: + """Parse bytes using XDBF offsets documented by the X360/Le Fluffie lineage.""" + if len(data) < XDBF_HEADER.size: + raise GpdError("GPD is smaller than the XDBF header") + magic, version, entry_max, entry_count, free_max, free_count = XDBF_HEADER.unpack_from( + data + ) + if magic != XDBF_MAGIC: + raise GpdError("File is not an XDBF/GPD database") + for label, value in ( + ("entry capacity", entry_max), + ("entry count", entry_count), + ("free-entry capacity", free_max), + ("free-entry count", free_count), + ): + if value > MAX_TABLE_ENTRIES: + raise GpdError(f"XDBF {label} exceeds the safety limit") + if entry_count > entry_max: + raise GpdError("XDBF entry count exceeds its table capacity") + if free_count > free_max: + raise GpdError("XDBF free-entry count exceeds its table capacity") + + header_size = ( + XDBF_HEADER.size + + (entry_max * XDBF_ENTRY.size) + + (free_max * XDBF_FREE_ENTRY_SIZE) + ) + if header_size > len(data): + raise GpdError("XDBF table extends beyond the end of the file") + + entries: list[XdbfEntry] = [] + namespace_counts: dict[str, int] = {} + for index in range(entry_count): + table_offset = XDBF_HEADER.size + (index * XDBF_ENTRY.size) + namespace, entry_id, offset, entry_size = XDBF_ENTRY.unpack_from( + data, table_offset + ) + if offset < 0 or entry_size < 0: + raise GpdError(f"XDBF entry {index} has a negative offset or size") + absolute = header_size + offset + if absolute > len(data) or entry_size > len(data) - absolute: + raise GpdError(f"XDBF entry {index} extends beyond the end of the file") + entry = XdbfEntry(namespace, entry_id, offset, entry_size) + entries.append(entry) + name = entry.namespace_name + namespace_counts[name] = namespace_counts.get(name, 0) + 1 + + achievements: list[GpdAchievement] = [] + settings: list[GpdSetting] = [] + warnings: list[str] = [] + for entry in entries: + payload = data[ + header_size + entry.offset : header_size + entry.offset + entry.size + ] + if entry.namespace == 1 and entry.entry_id not in {-1, -2}: + try: + achievements.append(_parse_achievement(entry, payload)) + except GpdError as exc: + warnings.append(f"Achievement {entry.entry_id}: {exc}") + elif entry.namespace == 3 and entry.entry_id not in {-1, -2}: + try: + settings.append(_parse_setting(entry, payload)) + except GpdError as exc: + warnings.append(f"Setting {entry.entry_id}: {exc}") + + source = Path(path) + inferred = source.stem.upper() if TITLE_ID_RE.fullmatch(source.stem) else "" + normalized_title_id = title_id.strip().upper() or inferred + if normalized_title_id and not TITLE_ID_RE.fullmatch(normalized_title_id): + raise GpdError(f"Invalid TitleID: {normalized_title_id}") + return GpdReport( + source, + normalized_title_id, + version, + hashlib.sha256(data).hexdigest().upper(), + len(data), + entry_count, + tuple(sorted(achievements, key=lambda item: item.achievement_id)), + tuple(sorted(settings, key=lambda item: item.setting_id)), + namespace_counts, + tuple(warnings), + ) + + +def _parse_achievement(entry: XdbfEntry, payload: bytes) -> GpdAchievement: + if len(payload) < 0x1C: + raise GpdError("record is smaller than the achievement header") + achievement_id, image_id, gamerscore = struct.unpack_from(">iiI", payload, 4) + flags = payload[16:20] + filetime = struct.unpack_from(">q", payload, 20)[0] + state = { + 0x12: "unlocked-offline", + 0x13: "unlocked-online", + }.get(flags[1], "locked") + strings = _split_utf16be_strings(payload[0x1C:], limit=3) + while len(strings) < 3: + strings.append("") + return GpdAchievement( + entry.entry_id, + achievement_id, + image_id, + gamerscore, + state, + _filetime_iso(filetime) if state == "unlocked-online" else "", + strings[0], + strings[1], + strings[2], + ) + + +def _parse_setting(entry: XdbfEntry, payload: bytes) -> GpdSetting: + if len(payload) < 0x18: + raise GpdError("record is smaller than the setting header") + setting_id = struct.unpack_from(">i", payload, 0)[0] + type_id = payload[8] + type_name = SETTING_TYPES.get(type_id, f"unknown-{type_id}") + value_area = payload[16:24] + if type_id in {0, 1}: + value: Any = struct.unpack_from(">I", value_area)[0] + elif type_id in {2, 7}: + raw = struct.unpack_from(">q", value_area)[0] + value = _filetime_iso(raw) if type_id == 7 else raw + elif type_id == 3: + value = struct.unpack_from(">d", value_area)[0] + elif type_id == 5: + value = struct.unpack_from(">f", value_area)[0] + elif type_id in {4, 6}: + length = struct.unpack_from(">i", value_area)[0] + if length < 0 or length > len(payload) - 0x18: + raise GpdError("variable-length setting has an invalid length") + raw = payload[0x18 : 0x18 + length] + value = ( + raw.decode("utf-16-be", errors="replace").rstrip("\0") + if type_id == 4 + else raw.hex().upper() + ) + else: + value = value_area.hex().upper() + return GpdSetting(entry.entry_id, setting_id, type_name, value) + + +def _split_utf16be_strings(data: bytes, limit: int) -> list[str]: + result: list[str] = [] + current = bytearray() + for index in range(0, len(data) - 1, 2): + pair = data[index : index + 2] + if pair == b"\0\0": + result.append(current.decode("utf-16-be", errors="replace")) + current.clear() + if len(result) >= limit: + break + else: + current.extend(pair) + if current and len(result) < limit: + result.append(current.decode("utf-16-be", errors="replace")) + return result + + +def _filetime_iso(value: int) -> str: + if value <= 0: + return "" + try: + origin = datetime(1601, 1, 1, tzinfo=timezone.utc) + return (origin + timedelta(microseconds=value / 10)).isoformat() + except (OverflowError, ValueError): + return "" diff --git a/knowledge_base.py b/knowledge_base.py index 9bb7cee..e9fdbff 100644 --- a/knowledge_base.py +++ b/knowledge_base.py @@ -390,7 +390,9 @@ def begin_import_run(self, source_slug: str, adapter_name: str) -> int: """, (source_slug, adapter_name, utc_now()), ) - return int(cursor.lastrowid) + if cursor.lastrowid is None: + raise RuntimeError("Import run was created without an identifier") + return cursor.lastrowid def finish_import_run( self, @@ -615,17 +617,57 @@ def get_preferred_facts( placeholders = ",".join("?" for _ in wanted) rows = self.connection.execute( f""" + WITH latest_resolution AS ( + SELECT + c.entity_id, + c.property, + r.preferred_value, + r.preferred_source_id + FROM knowledge_conflict_resolutions AS r + JOIN knowledge_conflicts AS c ON c.id = r.conflict_id + WHERE r.resolution IN ('prefer_existing', 'prefer_incoming') + AND r.id = ( + SELECT MAX(newer.id) + FROM knowledge_conflict_resolutions AS newer + JOIN knowledge_conflicts AS newer_conflict + ON newer_conflict.id = newer.conflict_id + WHERE newer_conflict.entity_id = c.entity_id + AND newer_conflict.property = c.property + AND newer.resolution IN ( + 'prefer_existing', 'prefer_incoming' + ) + ) + ) SELECT f.property, f.value, f.normalized_value, f.confidence, s.slug AS source_slug, - s.name AS source_name + s.name AS source_name, + COALESCE(p.priority, 100) AS source_priority FROM knowledge_facts AS f JOIN knowledge_sources AS s ON s.id = f.source_id + LEFT JOIN knowledge_source_priorities AS p + ON p.source_id = f.source_id AND p.property = f.property + LEFT JOIN latest_resolution AS resolution + ON resolution.entity_id = f.entity_id + AND resolution.property = f.property WHERE f.entity_id = ? AND f.property IN ({placeholders}) - ORDER BY f.property, f.confidence DESC, f.imported_at DESC + ORDER BY + f.property, + CASE + WHEN resolution.preferred_value = f.value + AND ( + resolution.preferred_source_id IS NULL + OR resolution.preferred_source_id = f.source_id + ) + THEN 0 + ELSE 1 + END, + source_priority, + f.confidence DESC, + f.imported_at DESC """, (entity_id, *wanted), ).fetchall() diff --git a/knowledge_gui.py b/knowledge_gui.py index 2d23ce4..6004d9d 100644 --- a/knowledge_gui.py +++ b/knowledge_gui.py @@ -9,6 +9,7 @@ from typing import Any from knowledge_service import KnowledgeService +from knowledge_scheduler import KnowledgeScheduler TEXT = "#f2f5f2" ACCENT = "#72e000" @@ -27,6 +28,7 @@ def __init__( self.root = root self.parent = parent self.service = service + self.scheduler = KnowledgeScheduler(service.database_path) self.status_var = tk.StringVar(value="Ready") self.search_var = tk.StringVar() self._build() @@ -65,12 +67,15 @@ def _build(self) -> None: browse = ttk.Frame(notebook, padding=10) sources = ttk.Frame(notebook, padding=10) conflicts = ttk.Frame(notebook, padding=10) + priorities = ttk.Frame(notebook, padding=10) notebook.add(browse, text="Browse") notebook.add(sources, text="Sources & Imports") notebook.add(conflicts, text="Conflicts") + notebook.add(priorities, text="Priorities & Schedule") self._build_browse(browse) self._build_sources(sources) self._build_conflicts(conflicts) + self._build_priorities(priorities) self.refresh() def _build_browse(self, parent: ttk.Frame) -> None: @@ -203,6 +208,105 @@ def _build_conflicts(self, parent: ttk.Frame) -> None: self.conflict_tree.heading(column, text=label) self.conflict_tree.column(column, width=width, minwidth=60) self.conflict_tree.grid(row=0, column=0, sticky="nsew") + actions = ttk.Frame(parent) + actions.grid(row=1, column=0, sticky="ew", pady=(8, 0)) + ttk.Button( + actions, + text="Prefer Existing", + command=lambda: self._resolve_selected_conflict("prefer_existing"), + ).pack(side=tk.LEFT) + ttk.Button( + actions, + text="Prefer Incoming", + command=lambda: self._resolve_selected_conflict("prefer_incoming"), + ).pack(side=tk.LEFT, padx=(8, 0)) + ttk.Button( + actions, + text="Dismiss", + command=lambda: self._resolve_selected_conflict("dismiss"), + ).pack(side=tk.LEFT, padx=(8, 0)) + + def _build_priorities(self, parent: ttk.Frame) -> None: + parent.columnconfigure(0, weight=1) + parent.rowconfigure(2, weight=1) + priority_controls = ttk.LabelFrame( + parent, text="Field-specific source priority", padding=10 + ) + priority_controls.grid(row=0, column=0, sticky="ew") + priority_controls.columnconfigure(1, weight=1) + self.priority_property_var = tk.StringVar(value="publisher") + self.priority_source_var = tk.StringVar() + self.priority_value_var = tk.IntVar(value=100) + ttk.Label(priority_controls, text="Fact property").grid( + row=0, column=0, sticky=tk.W + ) + ttk.Entry( + priority_controls, textvariable=self.priority_property_var + ).grid(row=0, column=1, sticky="ew", padx=8) + ttk.Label(priority_controls, text="Source").grid( + row=1, column=0, sticky=tk.W, pady=(8, 0) + ) + self.priority_source_combo = ttk.Combobox( + priority_controls, + textvariable=self.priority_source_var, + state="readonly", + ) + self.priority_source_combo.grid( + row=1, column=1, sticky="ew", padx=8, pady=(8, 0) + ) + ttk.Label(priority_controls, text="Priority").grid( + row=2, column=0, sticky=tk.W, pady=(8, 0) + ) + ttk.Spinbox( + priority_controls, + from_=1, + to=1000, + textvariable=self.priority_value_var, + width=8, + ).grid(row=2, column=1, sticky=tk.W, padx=8, pady=(8, 0)) + ttk.Button( + priority_controls, + text="Save Priority", + command=self._save_priority, + style="Accent.TButton", + ).grid(row=2, column=2, pady=(8, 0)) + + schedule = ttk.LabelFrame(parent, text="Automatic refresh", padding=10) + schedule.grid(row=1, column=0, sticky="ew", pady=10) + self.schedule_enabled_var = tk.BooleanVar() + self.schedule_hours_var = tk.IntVar(value=168) + ttk.Checkbutton( + schedule, + text="Refresh knowledge when the application starts and the interval is due", + variable=self.schedule_enabled_var, + ).pack(side=tk.LEFT) + ttk.Label(schedule, text="Hours").pack(side=tk.LEFT, padx=(18, 6)) + ttk.Spinbox( + schedule, + from_=6, + to=8760, + textvariable=self.schedule_hours_var, + width=7, + ).pack(side=tk.LEFT) + ttk.Button( + schedule, + text="Save Schedule", + command=self._save_schedule, + ).pack(side=tk.LEFT, padx=(8, 0)) + + self.priority_tree = ttk.Treeview( + parent, + columns=("property", "source", "priority"), + show="headings", + ) + for column, label, width in ( + ("property", "Property", 180), + ("source", "Source", 260), + ("priority", "Priority", 90), + ): + self.priority_tree.heading(column, text=label) + self.priority_tree.column(column, width=width, anchor=tk.W) + self.priority_tree.grid(row=2, column=0, sticky="nsew") def refresh(self) -> None: counts = self.service.counts() @@ -211,6 +315,7 @@ def refresh(self) -> None: self.refresh_results() self.refresh_sources() self.refresh_conflicts() + self.refresh_priorities() def refresh_results(self) -> None: self._clear_tree(self.result_tree) @@ -253,6 +358,7 @@ def refresh_conflicts(self) -> None: self.conflict_tree.insert( "", tk.END, + iid=str(row["id"]), text=row["canonical_name"], values=( row["property"], @@ -262,6 +368,75 @@ def refresh_conflicts(self) -> None: ), ) + def refresh_priorities(self) -> None: + sources = self.service.list_sources() + self.priority_sources = { + str(row["name"]): int(row["id"]) for row in sources + } + names = list(self.priority_sources) + self.priority_source_combo.configure(values=names) + if names and self.priority_source_var.get() not in self.priority_sources: + self.priority_source_var.set(names[0]) + self.priority_tree.delete(*self.priority_tree.get_children()) + for row in self.service.list_priorities(): + if not row.get("property"): + continue + self.priority_tree.insert( + "", + tk.END, + values=( + row["property"], + row["source_name"], + row["priority"], + ), + ) + state = self.scheduler.status() + self.schedule_enabled_var.set(bool(state.get("enabled"))) + self.schedule_hours_var.set(int(state.get("interval_hours") or 168)) + + def _save_priority(self) -> None: + source_id = self.priority_sources.get(self.priority_source_var.get()) + if source_id is None: + return + try: + self.service.set_source_priority( + source_id, + self.priority_property_var.get(), + int(self.priority_value_var.get()), + ) + except Exception as exc: + messagebox.showerror("Priority not saved", str(exc), parent=self.root) + return + self.status_var.set("Source priority saved") + self.refresh_priorities() + + def _save_schedule(self) -> None: + try: + self.scheduler.configure( + self.schedule_enabled_var.get(), + int(self.schedule_hours_var.get()), + ) + except Exception as exc: + messagebox.showerror("Schedule not saved", str(exc), parent=self.root) + return + self.status_var.set("Knowledge refresh schedule saved") + self.refresh_priorities() + + def _resolve_selected_conflict(self, resolution: str) -> None: + selection = self.conflict_tree.selection() + if not selection: + messagebox.showinfo( + "Resolve conflict", "Select a conflict first.", parent=self.root + ) + return + try: + self.service.resolve_conflict(int(selection[0]), resolution) + except Exception as exc: + messagebox.showerror("Conflict not resolved", str(exc), parent=self.root) + return + self.status_var.set("Conflict decision recorded") + self.refresh() + def _show_details(self, _event: tk.Event[Any] | None = None) -> None: selection = self.result_tree.selection() if not selection: diff --git a/knowledge_scheduler.py b/knowledge_scheduler.py new file mode 100644 index 0000000..1a1c378 --- /dev/null +++ b/knowledge_scheduler.py @@ -0,0 +1,143 @@ +"""App-start knowledge refresh scheduling with persistent, opt-in state.""" + +from __future__ import annotations + +import sqlite3 +from contextlib import contextmanager +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any, Callable + +from app_paths import DATABASE_PATH +from database_migrations import ensure_application_schema + + +TASK_NAME = "knowledge-refresh" +MIN_INTERVAL_HOURS = 6 + + +def utc_now() -> datetime: + return datetime.now(timezone.utc) + + +class KnowledgeScheduler: + """Run cached/rate-limited imports when an enabled schedule becomes due.""" + + def __init__(self, database_path: str | Path = DATABASE_PATH) -> None: + self.database_path = Path(database_path) + with self._connect() as connection: + ensure_application_schema(connection) + connection.execute( + """ + INSERT INTO scheduled_sync_state( + task_name, enabled, interval_hours, updated_at + ) VALUES (?, 0, 168, ?) + ON CONFLICT(task_name) DO NOTHING + """, + (TASK_NAME, utc_now().isoformat()), + ) + + @contextmanager + def _connect(self): + connection = sqlite3.connect(self.database_path) + connection.row_factory = sqlite3.Row + try: + yield connection + connection.commit() + except Exception: + connection.rollback() + raise + finally: + connection.close() + + def status(self) -> dict[str, Any]: + with self._connect() as connection: + row = connection.execute( + "SELECT * FROM scheduled_sync_state WHERE task_name=?", (TASK_NAME,) + ).fetchone() + return dict(row) if row else {} + + def configure(self, enabled: bool, interval_hours: int) -> None: + interval = max(MIN_INTERVAL_HOURS, min(int(interval_hours), 24 * 365)) + with self._connect() as connection: + connection.execute( + """ + UPDATE scheduled_sync_state + SET enabled=?, interval_hours=?, updated_at=? + WHERE task_name=? + """, + (int(enabled), interval, utc_now().isoformat(), TASK_NAME), + ) + + def is_due(self, now: datetime | None = None) -> bool: + state = self.status() + if not state or not bool(state["enabled"]): + return False + current = now or utc_now() + completed = _parse_time(state.get("last_completed_at")) + if completed is None: + return True + return current >= completed + timedelta(hours=int(state["interval_hours"])) + + def run_if_due( + self, + sync: Callable[[], Any] | None = None, + ) -> dict[str, Any] | None: + if not self.is_due(): + return None + operation = sync or _sync_all + started = utc_now().isoformat() + with self._connect() as connection: + connection.execute( + """ + UPDATE scheduled_sync_state + SET last_started_at=?, last_status='running', last_error=NULL, + updated_at=? WHERE task_name=? + """, + (started, started, TASK_NAME), + ) + try: + result = operation() + except Exception as exc: + with self._connect() as connection: + connection.execute( + """ + UPDATE scheduled_sync_state + SET last_status='failed', last_error=?, updated_at=? + WHERE task_name=? + """, + (str(exc), utc_now().isoformat(), TASK_NAME), + ) + raise + completed = utc_now().isoformat() + with self._connect() as connection: + connection.execute( + """ + UPDATE scheduled_sync_state + SET last_completed_at=?, last_status='completed', + last_error=NULL, updated_at=? WHERE task_name=? + """, + (completed, completed, TASK_NAME), + ) + return {"completed_at": completed, "result": result} + + +def _sync_all() -> dict[str, Any]: + from knowledge_sync import sync_consolemods_knowledge, sync_reference_wikis + + return { + "consolemods": sync_consolemods_knowledge(), + "wikis": sync_reference_wikis(), + } + + +def _parse_time(value: Any) -> datetime | None: + if not value: + return None + try: + parsed = datetime.fromisoformat(str(value)) + except ValueError: + return None + if parsed.tzinfo is None: + return parsed.replace(tzinfo=timezone.utc) + return parsed.astimezone(timezone.utc) diff --git a/knowledge_service.py b/knowledge_service.py index d22b375..20a2187 100644 --- a/knowledge_service.py +++ b/knowledge_service.py @@ -4,6 +4,7 @@ import sqlite3 from contextlib import contextmanager +from datetime import datetime, timezone from pathlib import Path from typing import Any @@ -144,12 +145,15 @@ def entity_details(self, entity_id: int) -> dict[str, Any]: SELECT f.id, f.property, f.value, f.confidence, s.name source_name, s.homepage_url, - c.source_url, c.source_title + c.source_url, c.source_title, + COALESCE(p.priority, 100) source_priority FROM knowledge_facts f JOIN knowledge_sources s ON s.id = f.source_id LEFT JOIN fact_citations c ON c.fact_id = f.id + LEFT JOIN knowledge_source_priorities p + ON p.source_id=f.source_id AND p.property=f.property WHERE f.entity_id = ? - ORDER BY f.property, f.confidence DESC, s.name + ORDER BY f.property, source_priority, f.confidence DESC, s.name """, (entity_id,), ).fetchall() @@ -202,6 +206,7 @@ def list_conflicts(self, status: str = "open") -> list[dict[str, Any]]: SELECT c.id, c.property, c.existing_value, c.incoming_value, c.detected_at, c.status, e.canonical_name, + c.existing_source_id, c.incoming_source_id, old.name existing_source, new.name incoming_source FROM knowledge_conflicts c JOIN knowledge_entities e ON e.id = c.entity_id @@ -213,3 +218,102 @@ def list_conflicts(self, status: str = "open") -> list[dict[str, Any]]: (status, status), ).fetchall() return [dict(row) for row in rows] + + def list_priorities(self, property_name: str = "") -> list[dict[str, Any]]: + with self._connect() as connection: + rows = connection.execute( + """ + SELECT s.id source_id, s.name source_name, p.property, + COALESCE(p.priority, 100) priority + FROM knowledge_sources s + LEFT JOIN knowledge_source_priorities p ON p.source_id=s.id + WHERE (?='' OR p.property=?) + ORDER BY COALESCE(p.property, ''), priority, s.name + """, + (property_name, property_name), + ).fetchall() + return [dict(row) for row in rows] + + def set_source_priority( + self, source_id: int, property_name: str, priority: int + ) -> None: + property_name = property_name.strip().casefold().replace(" ", "_") + if not property_name: + raise ValueError("A fact property is required") + if not 1 <= priority <= 1000: + raise ValueError("Priority must be between 1 and 1000") + with self._connect() as connection: + source = connection.execute( + "SELECT id FROM knowledge_sources WHERE id=?", (source_id,) + ).fetchone() + if source is None: + raise KeyError(source_id) + connection.execute( + """ + INSERT INTO knowledge_source_priorities( + property, source_id, priority, updated_at + ) VALUES (?, ?, ?, ?) + ON CONFLICT(property, source_id) DO UPDATE SET + priority=excluded.priority, updated_at=excluded.updated_at + """, + ( + property_name, + source_id, + priority, + datetime.now(timezone.utc).isoformat(), + ), + ) + connection.commit() + + def resolve_conflict( + self, + conflict_id: int, + resolution: str, + *, + notes: str = "", + ) -> dict[str, Any]: + allowed = {"prefer_existing", "prefer_incoming", "dismiss"} + if resolution not in allowed: + raise ValueError(f"Resolution must be one of: {', '.join(sorted(allowed))}") + with self._connect() as connection: + row = connection.execute( + "SELECT * FROM knowledge_conflicts WHERE id=?", (conflict_id,) + ).fetchone() + if row is None: + raise KeyError(conflict_id) + preferred_value = None + preferred_source_id = None + if resolution == "prefer_existing": + preferred_value = row["existing_value"] + preferred_source_id = row["existing_source_id"] + elif resolution == "prefer_incoming": + preferred_value = row["incoming_value"] + preferred_source_id = row["incoming_source_id"] + resolved_at = datetime.now(timezone.utc).isoformat() + connection.execute( + """ + INSERT INTO knowledge_conflict_resolutions( + conflict_id, resolution, preferred_value, + preferred_source_id, notes, resolved_at + ) VALUES (?, ?, ?, ?, ?, ?) + """, + ( + conflict_id, + resolution, + preferred_value, + preferred_source_id, + notes.strip(), + resolved_at, + ), + ) + connection.execute( + "UPDATE knowledge_conflicts SET status=? WHERE id=?", + ("dismissed" if resolution == "dismiss" else "resolved", conflict_id), + ) + connection.commit() + return { + "conflict_id": conflict_id, + "resolution": resolution, + "preferred_value": preferred_value, + "resolved_at": resolved_at, + } diff --git a/modern_gui.py b/modern_gui.py index 91e7d40..da51916 100644 --- a/modern_gui.py +++ b/modern_gui.py @@ -42,6 +42,7 @@ from diagnostics import create_diagnostics_bundle from external_tools_gui import ExternalToolsPage from knowledge_service import KnowledgeService +from knowledge_scheduler import KnowledgeScheduler from knowledge_gui import KnowledgePage from library_service import GameSummary, LibraryService from platform_support import desktop_font_family, open_path @@ -326,6 +327,7 @@ def __init__(self, root: tk.Tk) -> None: self.root = root self.library = LibraryService() self.knowledge = KnowledgeService() + self.knowledge_scheduler = KnowledgeScheduler() self.backups = BackupService() self.profiles = ProfileSaveManager() self.collections = CollectionIntelligenceService() @@ -351,6 +353,7 @@ def __init__(self, root: tk.Tk) -> None: if run_first_run_wizard(self.root): self.refresh_library() self.root.after(750, self._start_catalog_sync_if_stale) + self.root.after(1500, self._start_scheduled_knowledge_refresh) else: self.root.after(0, self.root.destroy) @@ -934,6 +937,23 @@ def _set_catalog_status(self, text: str) -> None: if hasattr(self, "catalog_status_var"): self.catalog_status_var.set(text) + def _start_scheduled_knowledge_refresh(self) -> None: + if not self.knowledge_scheduler.is_due(): + return + + def worker() -> None: + try: + self.knowledge_scheduler.run_if_due() + except Exception: + # The scheduler records the source error for the Knowledge page. + return + + threading.Thread( + target=worker, + name="scheduled-knowledge-refresh", + daemon=True, + ).start() + def _import_titleid_file(self) -> None: selected = filedialog.askopenfilename( parent=self.root, diff --git a/profile_gui.py b/profile_gui.py index fcf2e5a..611854c 100644 --- a/profile_gui.py +++ b/profile_gui.py @@ -11,7 +11,9 @@ from typing import Any, Callable from platform_support import open_path +from profile_intelligence import ProfileIntelligenceService from profile_manager import ProfileSaveManager, mask_identifier +from xenia_bridge import MigrationPlan, candidate_xenia_content_roots def _size(value: int) -> str: @@ -37,12 +39,16 @@ def __init__( self.root = root self.parent = parent self.manager = manager + self.intelligence = ProfileIntelligenceService(manager.db_path, manager) self.config_path = config_path self.events: queue.Queue[tuple[str, Any]] = queue.Queue() self.running = False self.profiles: dict[str, dict[str, Any]] = {} self.saves: dict[str, dict[str, Any]] = {} self.snapshots: dict[str, dict[str, Any]] = {} + self.gpd_files: dict[str, dict[str, Any]] = {} + self.profile_choices: dict[str, str] = {} + self.migration_plan: MigrationPlan | None = None page_header( "Profiles & Saves", @@ -89,10 +95,19 @@ def _build(self) -> None: inventory = ttk.Frame(notebook, padding=10) snapshots = ttk.Frame(notebook, padding=10) + achievements = ttk.Frame(notebook, padding=10) + compare = ttk.Frame(notebook, padding=10) + xenia = ttk.Frame(notebook, padding=10) notebook.add(inventory, text="Inventory") notebook.add(snapshots, text="Snapshots") + notebook.add(achievements, text="Achievements") + notebook.add(compare, text="Compare") + notebook.add(xenia, text="Xenia") self._build_inventory(inventory) self._build_snapshots(snapshots) + self._build_achievements(achievements) + self._build_compare(compare) + self._build_xenia(xenia) self.status_var = tk.StringVar(value="Choose a Content folder to begin.") ttk.Label(body, textvariable=self.status_var, style="Subheader.TLabel").grid( @@ -254,6 +269,178 @@ def _build_snapshots(self, parent: ttk.Frame) -> None: ).pack(side=tk.LEFT, padx=(8, 0)) ttk.Button(actions, text="Refresh", command=self.refresh).pack(side=tk.RIGHT) + def _build_achievements(self, parent: ttk.Frame) -> None: + parent.columnconfigure(0, weight=1) + parent.rowconfigure(1, weight=2) + parent.rowconfigure(3, weight=3) + controls = ttk.Frame(parent) + controls.grid(row=0, column=0, sticky="ew", pady=(0, 8)) + ttk.Button( + controls, + text="Import GPD", + command=self.import_gpd, + style="Accent.TButton", + ).pack(side=tk.LEFT) + ttk.Button( + controls, + text="Scan Extracted Folder", + command=self.scan_gpd_folder, + ).pack(side=tk.LEFT, padx=(8, 0)) + self.achievement_search_var = tk.StringVar() + ttk.Label(controls, text="Find").pack(side=tk.LEFT, padx=(20, 6)) + search = ttk.Entry( + controls, textvariable=self.achievement_search_var, width=28 + ) + search.pack(side=tk.LEFT) + search.bind("", lambda _event: self._refresh_achievements()) + + self.gpd_tree = ttk.Treeview( + parent, + columns=("titleid", "earned", "score", "status", "path"), + show="headings", + selectmode="browse", + height=6, + ) + for column, label, width in ( + ("titleid", "TitleID", 90), + ("earned", "Unlocked", 90), + ("score", "Gamerscore", 110), + ("status", "Status", 85), + ("path", "Extracted GPD", 430), + ): + self.gpd_tree.heading(column, text=label) + self.gpd_tree.column(column, width=width, anchor=tk.W) + self.gpd_tree.grid(row=1, column=0, sticky="nsew") + self.gpd_tree.bind( + "<>", lambda _event: self._refresh_achievements() + ) + + ttk.Label( + parent, + text=( + "Read-only achievement records from standalone or extracted XDBF/GPD " + "files. UnityScraper never edits these databases." + ), + style="Subheader.TLabel", + ).grid(row=2, column=0, sticky="ew", pady=(8, 6)) + self.achievement_tree = ttk.Treeview( + parent, + columns=("id", "title", "score", "state", "unlocked"), + show="headings", + ) + for column, label, width in ( + ("id", "ID", 65), + ("title", "Achievement", 330), + ("score", "G", 55), + ("state", "State", 125), + ("unlocked", "Unlocked", 180), + ): + self.achievement_tree.heading(column, text=label) + self.achievement_tree.column(column, width=width, anchor=tk.W) + self.achievement_tree.grid(row=3, column=0, sticky="nsew") + + def _build_compare(self, parent: ttk.Frame) -> None: + parent.columnconfigure(1, weight=1) + parent.rowconfigure(3, weight=1) + self.compare_left_var = tk.StringVar() + self.compare_right_var = tk.StringVar() + ttk.Label(parent, text="First profile").grid(row=0, column=0, sticky=tk.W) + self.compare_left = ttk.Combobox( + parent, textvariable=self.compare_left_var, state="readonly" + ) + self.compare_left.grid(row=0, column=1, sticky="ew", padx=(8, 0)) + ttk.Label(parent, text="Second profile").grid( + row=1, column=0, sticky=tk.W, pady=(8, 0) + ) + self.compare_right = ttk.Combobox( + parent, textvariable=self.compare_right_var, state="readonly" + ) + self.compare_right.grid( + row=1, column=1, sticky="ew", padx=(8, 0), pady=(8, 0) + ) + ttk.Button( + parent, + text="Compare Profiles", + command=self.compare_profiles, + style="Accent.TButton", + ).grid(row=2, column=1, sticky=tk.W, pady=10) + self.compare_text = tk.Text( + parent, + wrap=tk.WORD, + background="#070b08", + foreground="#f2f5f2", + insertbackground="#72e000", + relief=tk.FLAT, + padx=12, + pady=10, + ) + self.compare_text.grid(row=3, column=0, columnspan=2, sticky="nsew") + self.compare_text.insert( + tk.END, + "Choose two indexed profiles to compare saves and imported achievements.", + ) + self.compare_text.configure(state=tk.DISABLED) + + def _build_xenia(self, parent: ttk.Frame) -> None: + parent.columnconfigure(1, weight=1) + parent.rowconfigure(3, weight=1) + candidates = [str(path) for path in candidate_xenia_content_roots()] + self.xenia_root_var = tk.StringVar(value=candidates[0] if candidates else "") + self.xenia_target_var = tk.StringVar() + ttk.Label(parent, text="Xenia folder or content root").grid( + row=0, column=0, sticky=tk.W + ) + ttk.Entry(parent, textvariable=self.xenia_root_var).grid( + row=0, column=1, sticky="ew", padx=8 + ) + ttk.Button(parent, text="Browse", command=self.choose_xenia_root).grid( + row=0, column=2 + ) + ttk.Label(parent, text="Target profile ID").grid( + row=1, column=0, sticky=tk.W, pady=(8, 0) + ) + ttk.Entry(parent, textvariable=self.xenia_target_var).grid( + row=1, column=1, sticky="ew", padx=8, pady=(8, 0) + ) + controls = ttk.Frame(parent) + controls.grid(row=2, column=0, columnspan=3, sticky="ew", pady=10) + ttk.Button( + controls, + text="Preview Migration", + command=self.preview_xenia_migration, + style="Accent.TButton", + ).pack(side=tk.LEFT) + self.xenia_execute_button = ttk.Button( + controls, + text="Create Snapshot and Migrate", + command=self.execute_xenia_migration, + state=tk.DISABLED, + ) + self.xenia_execute_button.pack(side=tk.LEFT, padx=(8, 0)) + self.migration_tree = ttk.Treeview( + parent, + columns=("titleid", "file", "action", "reason"), + show="headings", + ) + for column, label, width in ( + ("titleid", "TitleID", 90), + ("file", "Save", 300), + ("action", "Action", 80), + ("reason", "Reason", 320), + ): + self.migration_tree.heading(column, text=label) + self.migration_tree.column(column, width=width, anchor=tk.W) + self.migration_tree.grid(row=3, column=0, columnspan=3, sticky="nsew") + ttk.Label( + parent, + text=( + "Migration is previewed first. A verified save snapshot is created " + "before any copy, and different destination files are never overwritten." + ), + style="Subheader.TLabel", + wraplength=900, + ).grid(row=4, column=0, columnspan=3, sticky="ew", pady=(8, 0)) + def choose_source(self) -> None: selected = filedialog.askdirectory( parent=self.root, @@ -279,9 +466,138 @@ def scan(self) -> None: "scan-complete", ) + def import_gpd(self) -> None: + path = filedialog.askopenfilename( + parent=self.root, + title="Choose an extracted Xbox 360 GPD", + filetypes=(("GPD databases", "*.gpd"), ("All files", "*.*")), + ) + if not path: + return + profile_id = self._selected_profile_id() + self._run( + "Reading the GPD database...", + lambda: self.intelligence.import_gpd(path, profile_id=profile_id), + "gpd-complete", + ) + + def scan_gpd_folder(self) -> None: + path = filedialog.askdirectory( + parent=self.root, + title="Choose a folder containing extracted GPD files", + ) + if not path: + return + profile_id = self._selected_profile_id() + self._run( + "Finding and reading extracted GPD databases...", + lambda: self.intelligence.scan_gpd_directory( + path, profile_id=profile_id + ), + "gpd-scan-complete", + ) + + def compare_profiles(self) -> None: + left = self.profile_choices.get(self.compare_left_var.get(), "") + right = self.profile_choices.get(self.compare_right_var.get(), "") + if not left or not right: + messagebox.showinfo( + "Compare profiles", + "Choose two indexed profiles first.", + parent=self.root, + ) + return + try: + result = self.intelligence.compare_profiles(left, right) + except Exception as exc: + messagebox.showerror("Comparison failed", str(exc), parent=self.root) + return + lines = [ + "PROFILE COMPARISON", + "", + f"Identical save titles: {len(result['save_titles_identical'])}", + f"Different save titles: {len(result['save_titles_different'])}", + f"Only in first profile: {', '.join(result['save_titles_only_left']) or 'None'}", + f"Only in second profile: {', '.join(result['save_titles_only_right']) or 'None'}", + "", + f"Shared unlocked achievements: {result['achievements_shared']}", + f"Unlocked only in first: {len(result['achievements_only_left'])}", + f"Unlocked only in second: {len(result['achievements_only_right'])}", + "", + "This is a read-only comparison. No profile or save was changed.", + ] + self.compare_text.configure(state=tk.NORMAL) + self.compare_text.delete("1.0", tk.END) + self.compare_text.insert(tk.END, "\n".join(lines)) + self.compare_text.configure(state=tk.DISABLED) + + def choose_xenia_root(self) -> None: + path = filedialog.askdirectory( + parent=self.root, + title="Choose Xenia folder or content directory", + ) + if path: + self.xenia_root_var.set(path) + + def preview_xenia_migration(self) -> None: + profile_id = self._selected_profile_id() + if not profile_id: + messagebox.showinfo( + "Xenia migration", "Select a source profile first.", parent=self.root + ) + return + target_id = self.xenia_target_var.get().strip() or profile_id + try: + self.migration_plan = self.intelligence.preview_xenia_migration( + profile_id, + self.xenia_root_var.get(), + target_profile_id=target_id, + ) + except Exception as exc: + messagebox.showerror("Migration preview failed", str(exc), parent=self.root) + return + self.migration_tree.delete(*self.migration_tree.get_children()) + for index, item in enumerate(self.migration_plan.items): + self.migration_tree.insert( + "", + tk.END, + iid=f"migration-{index}", + values=( + item.title_id, + item.relative_path.name, + item.action.title(), + item.reason, + ), + ) + self.xenia_execute_button.configure( + state=tk.NORMAL if self.migration_plan.copy_count else tk.DISABLED + ) + self.status_var.set( + f"Migration preview: {self.migration_plan.copy_count} copies, " + f"{self.migration_plan.conflict_count} conflicts." + ) + + def execute_xenia_migration(self) -> None: + plan = self.migration_plan + if plan is None: + return + if not messagebox.askyesno( + "Migrate saves to Xenia", + "Create a verified snapshot, then copy every non-conflicting save " + "shown in the preview?", + parent=self.root, + ): + return + self._run( + "Creating a snapshot and migrating saves to Xenia...", + lambda: self.intelligence.execute_xenia_migration(plan), + "xenia-complete", + ) + def refresh(self) -> None: self._refresh_profiles() self._refresh_snapshots() + self._refresh_gpd_files() def _refresh_profiles(self) -> None: selected = self._selected_profile_id() @@ -315,8 +631,82 @@ def _refresh_profiles(self) -> None: first = self.profile_tree.get_children()[0] self.profile_tree.selection_set(first) self.profile_tree.focus(first) + self._refresh_profile_choices() self._refresh_saves() + def _refresh_profile_choices(self) -> None: + current_left = self.compare_left_var.get() + current_right = self.compare_right_var.get() + self.profile_choices.clear() + for row in self.profiles.values(): + profile_id = str(row["profile_id"]) + name = str(row.get("gamertag") or "Profile") + label = f"{name} ({mask_identifier(profile_id)})" + self.profile_choices[label] = profile_id + choices = list(self.profile_choices) + self.compare_left.configure(values=choices) + self.compare_right.configure(values=choices) + if current_left in self.profile_choices: + self.compare_left_var.set(current_left) + elif choices: + self.compare_left_var.set(choices[0]) + if current_right in self.profile_choices: + self.compare_right_var.set(current_right) + elif len(choices) > 1: + self.compare_right_var.set(choices[1]) + + def _refresh_gpd_files(self) -> None: + selected = self.gpd_tree.selection() if hasattr(self, "gpd_tree") else () + selected_id = selected[0] if selected else "" + self.gpd_files.clear() + self.gpd_tree.delete(*self.gpd_tree.get_children()) + for row in self.intelligence.list_gpd_files(): + item_id = f"gpd-{row['id']}" + self.gpd_files[item_id] = row + self.gpd_tree.insert( + "", + tk.END, + iid=item_id, + values=( + row.get("titleid") or "Unknown", + f"{row['unlocked_count']} / {row['achievement_count']}", + f"{row['gamerscore_earned']} / {row['gamerscore_possible']}", + row["status"], + row["source_path"], + ), + ) + if selected_id in self.gpd_files: + self.gpd_tree.selection_set(selected_id) + elif self.gpd_tree.get_children(): + self.gpd_tree.selection_set(self.gpd_tree.get_children()[0]) + self._refresh_achievements() + + def _refresh_achievements(self) -> None: + if not hasattr(self, "achievement_tree"): + return + self.achievement_tree.delete(*self.achievement_tree.get_children()) + selection = self.gpd_tree.selection() + if not selection: + return + row = self.gpd_files.get(selection[0]) + if not row: + return + for achievement in self.intelligence.list_achievements( + int(row["id"]), + search=self.achievement_search_var.get(), + ): + self.achievement_tree.insert( + "", + tk.END, + values=( + achievement["achievement_id"], + achievement.get("title") or "Unnamed achievement", + achievement["gamerscore"], + str(achievement["unlock_state"]).replace("-", " ").title(), + str(achievement.get("unlocked_at") or "").replace("T", " ")[:19], + ), + ) + def _refresh_saves(self) -> None: profile_id = self._selected_profile_id() self.saves.clear() @@ -567,6 +957,31 @@ def _poll(self) -> None: f"Conflicts preserved: {value.conflicts}", parent=self.root, ) + elif event == "gpd-complete": + self.status_var.set(f"GPD inventory record {value} imported.") + self._refresh_gpd_files() + elif event == "gpd-scan-complete": + self.status_var.set( + f"Imported {value['imported']} GPD databases; " + f"{len(value['errors'])} could not be read." + ) + self._refresh_gpd_files() + elif event == "xenia-complete": + self.status_var.set( + f"Xenia migration copied {value['copied']}; " + f"skipped {value['skipped']}; conflicts {value['conflicts']}." + ) + self.migration_plan = None + self.xenia_execute_button.configure(state=tk.DISABLED) + self._refresh_snapshots() + messagebox.showinfo( + "Xenia migration complete", + f"Snapshot: {value['snapshot_id']}\n" + f"Copied: {value['copied']}\n" + f"Skipped: {value['skipped']}\n" + f"Conflicts: {value['conflicts']}", + parent=self.root, + ) if self.running: self.root.after(100, self._poll) diff --git a/profile_intelligence.py b/profile_intelligence.py new file mode 100644 index 0000000..bb9fb32 --- /dev/null +++ b/profile_intelligence.py @@ -0,0 +1,405 @@ +"""GPD inventory, profile comparison, and snapshot-first Xenia migration.""" + +from __future__ import annotations + +import json +import sqlite3 +from contextlib import contextmanager +from dataclasses import asdict +from pathlib import Path +from typing import Any, Iterable + +from app_paths import DATABASE_PATH +from database_migrations import ensure_application_schema +from gpd_parser import GpdReport, parse_gpd +from profile_manager import PROFILE_ID_RE, ProfileSaveManager, utc_now +from xenia_bridge import ( + MigrationPlan, + build_migration_plan, + execute_migration_plan, + find_xenia_content_root, +) + + +class ProfileIntelligenceError(RuntimeError): + """Raised when an intelligence or migration operation is invalid.""" + + +class ProfileIntelligenceService: + """Persist read-only GPD facts and orchestrate safe migration previews.""" + + def __init__( + self, + db_path: str | Path = DATABASE_PATH, + profile_manager: ProfileSaveManager | None = None, + ) -> None: + self.db_path = Path(db_path) + backup_root = self.db_path.parent / "profile_backups" + self.profile_manager = profile_manager or ProfileSaveManager( + db_path=db_path, + backup_root=backup_root, + ) + with self._connect() as connection: + ensure_application_schema(connection) + + @contextmanager + def _connect(self): + connection = sqlite3.connect(self.db_path) + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA foreign_keys = ON") + try: + yield connection + connection.commit() + except Exception: + connection.rollback() + raise + finally: + connection.close() + + def import_gpd( + self, + path: str | Path, + *, + profile_id: str = "", + title_id: str = "", + ) -> int: + normalized_profile = profile_id.strip().upper() + if normalized_profile and not PROFILE_ID_RE.fullmatch(normalized_profile): + raise ProfileIntelligenceError(f"Invalid profile ID: {profile_id}") + report = parse_gpd(path, title_id=title_id) + with self._connect() as connection: + cursor = connection.execute( + """ + INSERT INTO profile_gpd_files( + profile_id, titleid, source_path, sha256, size, version, + entry_count, achievement_count, unlocked_count, + gamerscore_earned, gamerscore_possible, parsed_at, status, + warnings_json + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'parsed', ?) + ON CONFLICT(source_path) DO UPDATE SET + profile_id=excluded.profile_id, + titleid=excluded.titleid, + sha256=excluded.sha256, + size=excluded.size, + version=excluded.version, + entry_count=excluded.entry_count, + achievement_count=excluded.achievement_count, + unlocked_count=excluded.unlocked_count, + gamerscore_earned=excluded.gamerscore_earned, + gamerscore_possible=excluded.gamerscore_possible, + parsed_at=excluded.parsed_at, + status=excluded.status, + warnings_json=excluded.warnings_json + """, + ( + normalized_profile, + report.title_id, + str(report.path.resolve()), + report.sha256, + report.size, + report.version, + report.entry_count, + len(report.achievements), + report.unlocked_count, + report.gamerscore_earned, + report.gamerscore_possible, + utc_now(), + json.dumps(report.warnings), + ), + ) + row = connection.execute( + "SELECT id FROM profile_gpd_files WHERE source_path=?", + (str(report.path.resolve()),), + ).fetchone() + if row is None: + if cursor.lastrowid is None: + raise ProfileIntelligenceError("GPD import was not recorded") + gpd_id = int(cursor.lastrowid) + else: + gpd_id = int(row["id"]) + connection.execute( + "DELETE FROM profile_achievements WHERE gpd_file_id=?", (gpd_id,) + ) + connection.executemany( + """ + INSERT INTO profile_achievements( + gpd_file_id, achievement_id, title, locked_description, + unlocked_description, gamerscore, unlock_state, unlocked_at, + image_id, entry_id + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + ( + gpd_id, + item.achievement_id, + item.title, + item.locked_description, + item.unlocked_description, + item.gamerscore, + item.state, + item.unlocked_at, + item.image_id, + item.entry_id, + ) + for item in report.achievements + ), + ) + return gpd_id + + def scan_gpd_directory( + self, root: str | Path, *, profile_id: str = "" + ) -> dict[str, Any]: + directory = Path(root).expanduser().resolve() + if not directory.is_dir(): + raise FileNotFoundError(directory) + imported: list[int] = [] + errors: list[str] = [] + for candidate in directory.rglob("*"): + if not candidate.is_file(): + continue + try: + with candidate.open("rb") as handle: + if handle.read(4) != b"XDBF": + continue + imported.append(self.import_gpd(candidate, profile_id=profile_id)) + except (OSError, ValueError) as exc: + errors.append(f"{candidate}: {exc}") + return {"imported": len(imported), "errors": errors} + + def list_gpd_files(self, profile_id: str = "") -> list[dict[str, Any]]: + with self._connect() as connection: + if profile_id: + rows = connection.execute( + """ + SELECT * FROM profile_gpd_files + WHERE profile_id=? ORDER BY titleid, source_path + """, + (profile_id,), + ).fetchall() + else: + rows = connection.execute( + "SELECT * FROM profile_gpd_files ORDER BY parsed_at DESC" + ).fetchall() + return [dict(row) for row in rows] + + def list_achievements( + self, + gpd_file_id: int, + *, + state: str = "", + search: str = "", + ) -> list[dict[str, Any]]: + clauses = ["gpd_file_id=?"] + values: list[Any] = [gpd_file_id] + if state: + clauses.append("unlock_state=?") + values.append(state) + if search.strip(): + clauses.append( + "(title LIKE ? OR locked_description LIKE ? " + "OR unlocked_description LIKE ?)" + ) + term = f"%{search.strip()}%" + values.extend((term, term, term)) + with self._connect() as connection: + rows = connection.execute( + f""" + SELECT * FROM profile_achievements + WHERE {' AND '.join(clauses)} + ORDER BY achievement_id + """, + values, + ).fetchall() + return [dict(row) for row in rows] + + def compare_profiles(self, left_profile_id: str, right_profile_id: str) -> dict[str, Any]: + left = left_profile_id.strip().upper() + right = right_profile_id.strip().upper() + if left == right: + raise ProfileIntelligenceError("Choose two different profiles") + if not PROFILE_ID_RE.fullmatch(left) or not PROFILE_ID_RE.fullmatch(right): + raise ProfileIntelligenceError("Both profile IDs must be 16 hexadecimal digits") + with self._connect() as connection: + saves = connection.execute( + """ + SELECT profile_id, titleid, name, sha256, size + FROM profile_saves WHERE profile_id IN (?, ?) + """, + (left, right), + ).fetchall() + achievements = connection.execute( + """ + SELECT g.profile_id, g.titleid, a.achievement_id, a.title, + a.gamerscore, a.unlock_state + FROM profile_achievements a + JOIN profile_gpd_files g ON g.id=a.gpd_file_id + WHERE g.profile_id IN (?, ?) + """, + (left, right), + ).fetchall() + summary = _comparison_summary(left, right, saves, achievements) + with self._connect() as connection: + cursor = connection.execute( + """ + INSERT INTO profile_comparisons( + left_profile_id, right_profile_id, created_at, summary_json + ) VALUES (?, ?, ?, ?) + """, + (left, right, utc_now(), json.dumps(summary, sort_keys=True)), + ) + summary["comparison_id"] = int(cursor.lastrowid or 0) + return summary + + def preview_xenia_migration( + self, + profile_id: str, + destination: str | Path, + *, + target_profile_id: str = "", + save_ids: Iterable[int] | None = None, + ) -> MigrationPlan: + saves = self._migration_saves(profile_id, save_ids) + content = find_xenia_content_root(destination) + if content is None: + selected = Path(destination).expanduser().resolve() + if selected.name.casefold() == "content": + content = selected + else: + content = selected / "content" + return build_migration_plan( + ((row["source_path"], row["titleid"]) for row in saves), + content, + source_profile_id=profile_id, + target_profile_id=target_profile_id or profile_id, + ) + + def execute_xenia_migration( + self, + plan: MigrationPlan, + *, + save_ids: Iterable[int] | None = None, + ) -> dict[str, Any]: + selected_ids = list(save_ids) if save_ids is not None else [ + int(row["id"]) for row in self._migration_saves(plan.source_profile_id, None) + ] + snapshot_id = self.profile_manager.create_snapshot( + plan.source_profile_id, + save_ids=selected_ids, + label="Automatic snapshot before Xenia migration", + ) + plan_json = json.dumps( + { + "items": [ + { + **asdict(item), + "source": str(item.source), + "destination": str(item.destination), + "relative_path": item.relative_path.as_posix(), + } + for item in plan.items + ] + }, + sort_keys=True, + ) + with self._connect() as connection: + cursor = connection.execute( + """ + INSERT INTO xenia_migration_runs( + source_profile_id, target_profile_id, destination_root, + snapshot_id, created_at, status, plan_json + ) VALUES (?, ?, ?, ?, ?, 'running', ?) + """, + ( + plan.source_profile_id, + plan.target_profile_id, + str(plan.destination_content), + snapshot_id, + utc_now(), + plan_json, + ), + ) + run_id = int(cursor.lastrowid or 0) + try: + copied, skipped, conflicts = execute_migration_plan(plan) + with self._connect() as connection: + connection.execute( + """ + UPDATE xenia_migration_runs + SET completed_at=?, status='completed', copied_count=?, + skipped_count=?, conflict_count=? WHERE id=? + """, + (utc_now(), copied, skipped, conflicts, run_id), + ) + return { + "run_id": run_id, + "snapshot_id": snapshot_id, + "copied": copied, + "skipped": skipped, + "conflicts": conflicts, + } + except Exception as exc: + with self._connect() as connection: + connection.execute( + """ + UPDATE xenia_migration_runs + SET completed_at=?, status='failed', error_message=? WHERE id=? + """, + (utc_now(), str(exc), run_id), + ) + raise + + def _migration_saves( + self, profile_id: str, save_ids: Iterable[int] | None + ) -> list[dict[str, Any]]: + saves = self.profile_manager.list_saves(profile_id) + if save_ids is None: + selected = saves + else: + wanted = {int(value) for value in save_ids} + selected = [row for row in saves if int(row["id"]) in wanted] + if not selected: + raise ProfileIntelligenceError("No indexed saves were selected") + return selected + + +def _comparison_summary( + left: str, + right: str, + saves: Iterable[sqlite3.Row], + achievements: Iterable[sqlite3.Row], +) -> dict[str, Any]: + save_maps: dict[str, dict[str, set[str]]] = { + left: {}, + right: {}, + } + for row in saves: + save_maps[str(row["profile_id"])].setdefault(str(row["titleid"]), set()).add( + str(row["sha256"]) + ) + left_titles = set(save_maps[left]) + right_titles = set(save_maps[right]) + different = sorted( + title + for title in left_titles & right_titles + if save_maps[left][title] != save_maps[right][title] + ) + + unlocked: dict[str, set[tuple[str, int]]] = {left: set(), right: set()} + for row in achievements: + if str(row["unlock_state"]).startswith("unlocked"): + unlocked[str(row["profile_id"])].add( + (str(row["titleid"]), int(row["achievement_id"])) + ) + return { + "left_profile_id": left, + "right_profile_id": right, + "save_titles_only_left": sorted(left_titles - right_titles), + "save_titles_only_right": sorted(right_titles - left_titles), + "save_titles_different": different, + "save_titles_identical": sorted( + (left_titles & right_titles) - set(different) + ), + "achievements_only_left": sorted(unlocked[left] - unlocked[right]), + "achievements_only_right": sorted(unlocked[right] - unlocked[left]), + "achievements_shared": len(unlocked[left] & unlocked[right]), + } diff --git a/tests.py b/tests.py index c38ebce..4bfbc22 100644 --- a/tests.py +++ b/tests.py @@ -1428,6 +1428,186 @@ def test_profile_snapshot_contains_manifest_and_every_profile_file(self): self.assertIn("DJ SkunkieButt", payload["attribution"]) +class TestRoadmapFeatures(unittest.TestCase): + """Read-only GPD, Xenia, knowledge, and verification roadmap coverage.""" + + PROFILE_ID = "E000012345678BD2" + TITLE_ID = "53510804" + + def setUp(self): + self.temp_dir = Path(tempfile.mkdtemp()) + self.db_path = self.temp_dir / "roadmap.db" + DatabaseManager(str(self.db_path)) + + def tearDown(self): + shutil.rmtree(self.temp_dir) + + @staticmethod + def _gpd_bytes(unlocked=True): + import struct + + strings = b"".join( + value.encode("utf-16-be") + b"\0\0" + for value in ("First Steps", "Locked text", "Unlocked text") + ) + payload = bytearray(0x1C) + payload[:4] = (0x1C).to_bytes(4, "big") + payload[4:8] = (7).to_bytes(4, "big", signed=True) + payload[8:12] = (42).to_bytes(4, "big", signed=True) + payload[12:16] = (25).to_bytes(4, "big") + payload[16:20] = bytes((0, 0x12 if unlocked else 0, 0, 0)) + entry_payload = bytes(payload) + strings + header = struct.pack(">4sIIIII", b"XDBF", 1, 1, 1, 0, 0) + entry = struct.pack(">Hqii", 1, 100, 0, len(entry_payload)) + return header + entry + entry_payload + + def test_gpd_parser_reads_achievement_without_modifying_file(self): + from gpd_parser import parse_gpd + + path = self.temp_dir / f"{self.TITLE_ID}.gpd" + original = self._gpd_bytes() + path.write_bytes(original) + report = parse_gpd(path) + + self.assertEqual(report.title_id, self.TITLE_ID) + self.assertEqual(report.unlocked_count, 1) + self.assertEqual(report.gamerscore_earned, 25) + self.assertEqual(report.achievements[0].title, "First Steps") + self.assertEqual(path.read_bytes(), original) + + def test_profile_intelligence_imports_and_compares_achievements(self): + from profile_intelligence import ProfileIntelligenceService + + left = self.temp_dir / f"{self.TITLE_ID}.gpd" + right = self.temp_dir / "right" / f"{self.TITLE_ID}.gpd" + right.parent.mkdir() + left.write_bytes(self._gpd_bytes(unlocked=True)) + right.write_bytes(self._gpd_bytes(unlocked=False)) + service = ProfileIntelligenceService(self.db_path) + left_id = service.import_gpd(left, profile_id=self.PROFILE_ID) + service.import_gpd(right, profile_id="E000000000000002") + + self.assertEqual(len(service.list_achievements(left_id)), 1) + comparison = service.compare_profiles( + self.PROFILE_ID, "E000000000000002" + ) + self.assertEqual(len(comparison["achievements_only_left"]), 1) + self.assertEqual(len(comparison["achievements_only_right"]), 0) + + def test_xenia_plan_copies_then_skips_identical_save(self): + from xenia_bridge import build_migration_plan, execute_migration_plan + + source = ( + self.temp_dir + / self.PROFILE_ID + / self.TITLE_ID + / "00000001" + / "save.bin" + ) + source.parent.mkdir(parents=True) + source.write_bytes(b"user-owned-save") + destination = self.temp_dir / "xenia" / "content" + plan = build_migration_plan( + [(source, self.TITLE_ID)], + destination, + source_profile_id=self.PROFILE_ID, + target_profile_id=self.PROFILE_ID, + ) + self.assertEqual(plan.copy_count, 1) + self.assertEqual(execute_migration_plan(plan), (1, 0, 0)) + second = build_migration_plan( + [(source, self.TITLE_ID)], + destination, + source_profile_id=self.PROFILE_ID, + target_profile_id=self.PROFILE_ID, + ) + self.assertEqual(second.items[0].action, "skip") + + def test_knowledge_priority_and_conflict_resolution_are_persistent(self): + import sqlite3 + from contextlib import closing + + from knowledge_base import KnowledgeRepository + + service = KnowledgeService(self.db_path) + sources = service.list_sources() + source = sources[0] + other_source = sources[1] + service.set_source_priority(int(source["id"]), "publisher", 10) + service.set_source_priority(int(other_source["id"]), "publisher", 20) + priority = [ + row + for row in service.list_priorities() + if row.get("property") == "publisher" + ][0] + self.assertEqual(priority["priority"], 10) + with closing(sqlite3.connect(self.db_path)) as connection: + connection.row_factory = sqlite3.Row + entity = connection.execute( + """ + INSERT INTO knowledge_entities( + entity_type, canonical_name, normalized_name + ) VALUES ('game', 'Example', 'example') + """ + ) + entity_id = int(entity.lastrowid) + conflict = connection.execute( + """ + INSERT INTO knowledge_conflicts( + entity_id, property, existing_value, incoming_value, + existing_source_id, incoming_source_id, detected_at + ) VALUES (?, 'publisher', 'A', 'B', ?, ?, 'now') + """, + (entity_id, source["id"], other_source["id"]), + ) + connection.executemany( + """ + INSERT INTO knowledge_facts( + entity_id, property, value, normalized_value, + source_id, confidence, imported_at + ) VALUES (?, 'publisher', ?, ?, ?, ?, 'now') + """, + ( + (entity_id, "A", "a", source["id"], 0.70), + (entity_id, "B", "b", other_source["id"], 0.99), + ), + ) + connection.commit() + conflict_id = int(conflict.lastrowid) + preferred = KnowledgeRepository(connection).get_preferred_facts( + entity_id, ("publisher",) + ) + self.assertEqual(preferred["publisher"]["value"], "A") + result = service.resolve_conflict(conflict_id, "prefer_incoming") + self.assertEqual(result["preferred_value"], "B") + self.assertEqual(service.list_conflicts(), []) + with closing(sqlite3.connect(self.db_path)) as connection: + connection.row_factory = sqlite3.Row + preferred = KnowledgeRepository(connection).get_preferred_facts( + entity_id, ("publisher",) + ) + self.assertEqual(preferred["publisher"]["value"], "B") + + def test_scheduler_runs_only_when_enabled_and_due(self): + from knowledge_scheduler import KnowledgeScheduler + + scheduler = KnowledgeScheduler(self.db_path) + self.assertIsNone(scheduler.run_if_due(lambda: "unused")) + scheduler.configure(True, 24) + calls = [] + result = scheduler.run_if_due(lambda: calls.append("run") or "done") + self.assertEqual(calls, ["run"]) + self.assertEqual(result["result"], "done") + self.assertIsNone(scheduler.run_if_due(lambda: calls.append("again"))) + + def test_remote_sha256_detects_supported_read_only_command(self): + from console_sync import _remote_sha256 + + ftp = Mock() + ftp.sendcmd.return_value = "213 " + ("AB" * 32) + self.assertEqual(_remote_sha256(ftp, "/Hdd1/save"), ("ab" * 32)) + + class TestUnifiedV1Foundation(unittest.TestCase): def setUp(self): self.temp_dir = tempfile.mkdtemp() @@ -1451,7 +1631,7 @@ def test_versioned_migrations_create_all_foundation_tables(self): versions = connection.execute( "SELECT version FROM app_schema_migrations ORDER BY version" ).fetchall() - self.assertEqual([row[0] for row in versions], [1, 2, 3, 4, 5, 6]) + self.assertEqual([row[0] for row in versions], [1, 2, 3, 4, 5, 6, 7]) self.assertIn("collection_snapshots", tables) self.assertIn("preservation_matches", tables) self.assertIn("console_transfer_jobs", tables) @@ -1461,6 +1641,11 @@ def test_versioned_migrations_create_all_foundation_tables(self): self.assertIn("profile_saves", tables) self.assertIn("save_snapshots", tables) self.assertIn("profile_save_operations", tables) + self.assertIn("profile_gpd_files", tables) + self.assertIn("profile_achievements", tables) + self.assertIn("xenia_migration_runs", tables) + self.assertIn("knowledge_source_priorities", tables) + self.assertIn("scheduled_sync_state", tables) def test_xex_execution_info_is_parsed(self): from backup_manager import inspect_xex @@ -1575,6 +1760,7 @@ def run_tests(): suite.addTests(loader.loadTestsFromTestCase(TestBackupManager)) suite.addTests(loader.loadTestsFromTestCase(TestRestAPI)) suite.addTests(loader.loadTestsFromTestCase(TestProfileSaveManager)) + suite.addTests(loader.loadTestsFromTestCase(TestRoadmapFeatures)) suite.addTests(loader.loadTestsFromTestCase(TestUnifiedV1Foundation)) # Run tests diff --git a/xenia_bridge.py b/xenia_bridge.py new file mode 100644 index 0000000..af6457c --- /dev/null +++ b/xenia_bridge.py @@ -0,0 +1,239 @@ +"""Read-first Xenia content discovery and verified save migration plans.""" + +from __future__ import annotations + +import hashlib +import os +import re +import shutil +from dataclasses import dataclass +from pathlib import Path +from typing import Iterable + + +PROFILE_ID_RE = re.compile(r"^[0-9A-Fa-f]{16}$") +TITLE_ID_RE = re.compile(r"^[0-9A-Fa-f]{8}$") +SAVE_DIRECTORY = "00000001" +COPY_CHUNK = 1024 * 1024 + + +class XeniaBridgeError(RuntimeError): + """Raised when a Xenia root or migration plan is unsafe.""" + + +@dataclass(frozen=True) +class XeniaSave: + profile_id: str + title_id: str + path: Path + relative_path: Path + size: int + sha256: str + + +@dataclass(frozen=True) +class MigrationItem: + source: Path + destination: Path + relative_path: Path + title_id: str + size: int + sha256: str + action: str + reason: str + + +@dataclass(frozen=True) +class MigrationPlan: + source_profile_id: str + target_profile_id: str + destination_content: Path + items: tuple[MigrationItem, ...] + + @property + def copy_count(self) -> int: + return sum(item.action == "copy" for item in self.items) + + @property + def conflict_count(self) -> int: + return sum(item.action == "conflict" for item in self.items) + + +def candidate_xenia_content_roots() -> tuple[Path, ...]: + """Return conventional Xenia content locations without creating them.""" + home = Path.home() + documents = Path(os.environ.get("USERPROFILE", home)) / "Documents" + candidates = ( + documents / "xenia" / "content", + documents / "Xenia" / "content", + home / "Documents" / "xenia" / "content", + home / ".local" / "share" / "xenia" / "content", + home / ".config" / "xenia" / "content", + ) + unique: list[Path] = [] + seen: set[str] = set() + for candidate in candidates: + key = os.path.normcase(str(candidate)) + if key not in seen: + seen.add(key) + unique.append(candidate) + return tuple(unique) + + +def find_xenia_content_root(path: str | Path | None = None) -> Path | None: + """Locate a user-selected or conventional Xenia content directory.""" + candidates = (Path(path).expanduser(),) if path else candidate_xenia_content_roots() + for candidate in candidates: + root = candidate.resolve() + if root.is_dir() and root.name.casefold() == "content": + return root + nested = root / "content" + if nested.is_dir(): + return nested.resolve() + return None + + +def scan_xenia_saves(content_root: str | Path) -> tuple[XeniaSave, ...]: + """Index visible save packages under a Xenia content tree.""" + root = Path(content_root).expanduser().resolve() + if not root.is_dir(): + raise FileNotFoundError(root) + results: list[XeniaSave] = [] + for profile in root.iterdir(): + profile_id = profile.name.upper() + if not profile.is_dir() or not PROFILE_ID_RE.fullmatch(profile_id): + continue + for title in profile.iterdir(): + title_id = title.name.upper() + if not title.is_dir() or not TITLE_ID_RE.fullmatch(title_id): + continue + save_dir = _child_named(title, SAVE_DIRECTORY) + if not save_dir: + continue + for package in save_dir.rglob("*"): + if not package.is_file(): + continue + results.append( + XeniaSave( + profile_id, + title_id, + package, + package.relative_to(profile), + package.stat().st_size, + sha256_file(package), + ) + ) + return tuple(sorted(results, key=lambda item: str(item.path).casefold())) + + +def build_migration_plan( + sources: Iterable[tuple[str | Path, str]], + destination_content: str | Path, + *, + source_profile_id: str, + target_profile_id: str, +) -> MigrationPlan: + """Preview copies to Xenia without changing either side.""" + source_id = _profile_id(source_profile_id) + target_id = _profile_id(target_profile_id) + destination_root = Path(destination_content).expanduser().resolve() + items: list[MigrationItem] = [] + for source_value, title_value in sources: + source = Path(source_value).expanduser().resolve() + title_id = title_value.strip().upper() + if not source.is_file(): + raise FileNotFoundError(source) + if not TITLE_ID_RE.fullmatch(title_id): + raise XeniaBridgeError(f"Invalid TitleID: {title_id}") + relative = _save_relative_path(source, source_id, title_id) + destination = destination_root / target_id / relative + digest = sha256_file(source) + action, reason = "copy", "New file" + if destination.exists(): + if not destination.is_file(): + action, reason = "conflict", "Destination is not a file" + elif sha256_file(destination) == digest: + action, reason = "skip", "Identical file already exists" + else: + action, reason = "conflict", "Different file already exists" + items.append( + MigrationItem( + source, + destination, + relative, + title_id, + source.stat().st_size, + digest, + action, + reason, + ) + ) + return MigrationPlan(source_id, target_id, destination_root, tuple(items)) + + +def execute_migration_plan(plan: MigrationPlan) -> tuple[int, int, int]: + """Execute only non-conflicting plan items through verified atomic copies.""" + copied = skipped = conflicts = 0 + for item in plan.items: + if item.action == "skip": + skipped += 1 + continue + if item.action != "copy": + conflicts += 1 + continue + item.destination.parent.mkdir(parents=True, exist_ok=True) + partial = item.destination.with_name(item.destination.name + ".partial") + try: + with item.source.open("rb") as source, partial.open("wb") as output: + shutil.copyfileobj(source, output, COPY_CHUNK) + if sha256_file(partial) != item.sha256: + raise XeniaBridgeError(f"Copy verification failed: {item.relative_path}") + if item.destination.exists(): + conflicts += 1 + partial.unlink(missing_ok=True) + continue + partial.replace(item.destination) + copied += 1 + finally: + partial.unlink(missing_ok=True) + return copied, skipped, conflicts + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(COPY_CHUNK), b""): + digest.update(chunk) + return digest.hexdigest().upper() + + +def _save_relative_path(source: Path, profile_id: str, title_id: str) -> Path: + parts = source.parts + folded = [part.casefold() for part in parts] + try: + profile_index = folded.index(profile_id.casefold()) + relative = Path(*parts[profile_index + 1 :]) + if ( + len(relative.parts) >= 3 + and relative.parts[0].casefold() == title_id.casefold() + and relative.parts[1].casefold() == SAVE_DIRECTORY.casefold() + ): + return relative + except ValueError: + pass + return Path(title_id) / SAVE_DIRECTORY / source.name + + +def _profile_id(value: str) -> str: + normalized = value.strip().upper() + if not PROFILE_ID_RE.fullmatch(normalized): + raise XeniaBridgeError(f"Invalid profile ID: {value}") + return normalized + + +def _child_named(parent: Path, name: str) -> Path | None: + expected = name.casefold() + try: + return next(child for child in parent.iterdir() if child.name.casefold() == expected) + except (OSError, StopIteration): + return None