From 1da6e06b1be1bb89216a3ba9d0751fab41742bd4 Mon Sep 17 00:00:00 2001 From: Sthornberry9 <46094434+Sthornberry9@users.noreply.github.com> Date: Sat, 1 Aug 2026 04:39:11 -0400 Subject: [PATCH 1/2] feat: add VS2010 release toolkit foundation --- .github/workflows/release.yml | 2 +- API.md | 13 + BACKUP_MANAGER.md | 14 + CHANGELOG.md | 18 +- COMMUNITY_HUB.md | 14 +- DOCS_INDEX.md | 2 + GUI.py | 11 +- PLUGIN_API.md | 6 +- PROJECT_STATUS.md | 12 +- README.md | 34 +- RELEASE_TOOLKIT.md | 51 +++ UnityScraper.spec | 3 + VERSION | 14 +- api.py | 150 ++++++- app_paths.py | 2 + app_version.py | 4 +- backup_manager.py | 109 +++++ build_linux.sh | 2 +- collection_intelligence.py | 6 +- community_gui.py | 128 +++++- community_services.py | 32 +- database_migrations.py | 69 +++- desktop_app.py | 5 + external_tools_gui.py | 15 +- i18n.py | 54 ++- knowledge_gui.py | 11 +- main.py | 56 ++- modern_gui.py | 269 ++++++------- ...github.trapemall.UnityScraper.metainfo.xml | 2 +- packaging/macos/Info.plist | 4 +- plugin_worker.py | 80 ++++ plugins.py | 82 +++- profile_gui.py | 70 +++- pyproject.toml | 2 +- roadmap_services.py | 379 ++++++++++++++++++ setup_wizard.py | 32 +- tests.py | 146 ++++++- ui_theme.py | 207 ++++++++++ xenia_bridge.py | 49 +++ 39 files changed, 1941 insertions(+), 218 deletions(-) create mode 100644 RELEASE_TOOLKIT.md create mode 100644 plugin_worker.py create mode 100644 roadmap_services.py create mode 100644 ui_theme.py diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6ddea01..d51de1f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -62,7 +62,7 @@ jobs: shell: pwsh run: | New-Item -ItemType Directory -Path package | Out-Null - Copy-Item dist\UnityScraper.exe, README.md, CHANGELOG.md, LICENSE, DOCS_INDEX.md, BACKUP_MANAGER.md, COLLECTION_INTELLIGENCE.md, COMMUNITY_HUB.md, CONSOLE_SYNC.md, KNOWLEDGE_SOURCES.md, LINUX.md, PLUGIN_API.md, PROFILE_INTELLIGENCE.md, PROFILES_AND_SAVES.md, SECURITY.md, THIRD_PARTY_NOTICES.md package\ + Copy-Item dist\UnityScraper.exe, README.md, CHANGELOG.md, LICENSE, DOCS_INDEX.md, BACKUP_MANAGER.md, COLLECTION_INTELLIGENCE.md, COMMUNITY_HUB.md, CONSOLE_SYNC.md, KNOWLEDGE_SOURCES.md, LINUX.md, PLUGIN_API.md, PROFILE_INTELLIGENCE.md, PROFILES_AND_SAVES.md, RELEASE_TOOLKIT.md, SECURITY.md, THIRD_PARTY_NOTICES.md package\ Compress-Archive -Path package\* -DestinationPath UnityScraper-Windows-x64.zip $hash = (Get-FileHash UnityScraper-Windows-x64.zip -Algorithm SHA256).Hash.ToLower() "$hash *UnityScraper-Windows-x64.zip" | Set-Content UnityScraper-Windows-x64.zip.sha256 diff --git a/API.md b/API.md index 06a7164..0a5fb43 100644 --- a/API.md +++ b/API.md @@ -37,6 +37,10 @@ X-API-Key: The health endpoint does not require authentication. All other endpoints do when a token is configured. +For separate automation clients, `UNITYSCRAPER_API_TOKENS` accepts a JSON object +mapping tokens to `read`, `write`, or `transfer` scopes. A legacy single token +has all scopes. Requests are limited per client to 120 per minute by default. + ## Endpoints | Method | Path | Purpose | @@ -51,6 +55,13 @@ when a token is configured. | `POST` | `/api/preservation/dedup//apply` | Apply `quarantine` or `hardlink` mode | | `POST` | `/api/preservation/dedup//restore` | Revalidate and restore a quarantined original | | `GET` | `/api/plugins` | List managed plugins and checksum trust state | +| `GET` | `/api/library/audit` | Find incomplete names, publishers, covers, updates, and MediaIDs | +| `POST` | `/api/metadata-snapshots/export` | Export a non-personal `.usmeta` snapshot | +| `POST` | `/api/metadata-snapshots/import` | Merge a validated `.usmeta` snapshot | +| `POST` | `/api/packages/extract` | Extract supported STFS files read-only | +| `POST` | `/api/reports/preservation` | Create a privacy-conscious HTML report | +| `GET` | `/api/hardware` | List local console hardware notes | +| `POST` | `/api/hardware` | Add a local console hardware record | | `POST` | `/api/metadata/` | Collect metadata | | `POST` | `/api/download/` | Process downloads | | `GET` | `/api/statistics` | Library statistics | @@ -106,3 +117,5 @@ Invoke-RestMethod ` - Browser CORS is restricted to localhost origins unless the API is embedded programmatically with an explicit origin list. - Responses disable caching and include basic content and frame protections. +- Tokens can be separated by read, write, and transfer scope, and each client + has a bounded request window. diff --git a/BACKUP_MANAGER.md b/BACKUP_MANAGER.md index 68040ed..51b38e4 100644 --- a/BACKUP_MANAGER.md +++ b/BACKUP_MANAGER.md @@ -40,6 +40,17 @@ and atomically renamed. Existing files are skipped by default. **Import ZIP** applies the same checks to packages in a user-supplied ZIP. Absolute paths, parent traversal, and archive symlinks are rejected. +## Read-only STFS extraction + +The Community Hub package workspace can inventory and extract files stored in +consecutive STFS blocks. Extraction rejects unsafe paths, limits total output, +never replaces existing files, stages each output atomically, and creates a +manifest with source and extracted-file SHA-256 values. Fragmented files are +reported and skipped rather than reconstructed without verified block-chain and +hash-tree support. + +The parser follows the public [Free60 STFS format reference](https://free60.org/System-Software/Formats/STFS/). + ## Export and verification **Export Selected** copies an inventoried title to a separate archive folder @@ -103,6 +114,9 @@ python main.py --verify-backups E:\ --backup-report health.json # Upload a package to a console FTP server python main.py --ftp-upload game.live --ftp-host 192.168.1.50 + +# Read-only extraction of supported STFS files +python main.py --extract-stfs save.con --extract-destination extracted ``` ## Inspiration and licensing diff --git a/CHANGELOG.md b/CHANGELOG.md index 8651f81..b986b38 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,22 @@ Notable changes to UnityScraper are documented here. The project follows ### Added +- Additive schema migration 10 for metadata snapshot runs, library intelligence, + preservation reports, correction packages, hardware records, and package extractions. +- Visual Studio 2010-inspired dark desktop theme shared by the modern and legacy + interfaces, including compact tool chrome, classic menus, dense tabs, and blue focus states. +- Read-only extraction of consecutive STFS files with traversal protection, + bounded output, atomic publication, hashes, and an extraction manifest. +- Portable `.usmeta` metadata snapshots that exclude profile and filesystem data + and merge source-attributed facts without replacing better library metadata. +- Library attention audits, privacy-conscious HTML preservation reports, + community correction exports, and console hardware inventory records. +- Xenia/Xenia Canary discovery and direct argument-vector game launching. +- Scoped API tokens, fixed-window request limits, and API/CLI parity for the new toolkit. +- Bounded JSON community language packs with English fallback. +- Timeout-controlled child-process execution for enabled metadata plugins, with + POSIX CPU, memory, and output limits where the operating system supports them. + - Additive schema migration 9 for plugin collection audits and reversible duplicate recovery records. - Runtime plugin loading with approved-checksum enforcement, bounded results, @@ -93,7 +109,7 @@ Notable changes to UnityScraper are documented here. The project follows ### Changed -- Version advanced to `1.1.0-beta.1`. +- Version advanced to `1.2.0-beta.1`. - CI now runs the Python suite on Windows, Linux, and macOS. - Cached XboxUnity titles now resolve immediately in library lists and details, enrich matching rows page by page, and recover after interrupted refreshes. diff --git a/COMMUNITY_HUB.md b/COMMUNITY_HUB.md index c9abda0..e0fe545 100644 --- a/COMMUNITY_HUB.md +++ b/COMMUNITY_HUB.md @@ -4,7 +4,7 @@ The Community Hub brings the wider Xbox and preservation workflows into one source-attributed, offline-capable workspace. Open it from the desktop sidebar or press `Ctrl+K` to focus unified search. -## Twenty Integrated Capabilities +## Integrated Capabilities 1. Unified local search across games, identifiers, wiki knowledge, profiles, saves, achievements, files, and structured reference records. @@ -40,6 +40,13 @@ or press `Ctrl+K` to focus unified search. 20. Cross-platform accessibility and packaging: scalable text, high contrast, reduced-motion preferences, keyboard hints, and Windows, Linux, and macOS build paths. +21. Read-only extraction of supported consecutive STFS files with hashes and a + portable extraction manifest. +22. Non-personal `.usmeta` snapshot export/import for offline title and knowledge data. +23. Library attention audits and privacy-conscious preservation reports. +24. Reviewed correction-package exports and local hardware inventory records. +25. Direct Xenia/Xenia Canary discovery and game launch controls alongside the + existing snapshot-first save migration workflow. ## Safety Rules @@ -71,6 +78,11 @@ plugins run during normal metadata collection only while their approved entrypoi checksum still matches. Long Community Hub operations run in a background worker; search results can be opened with Enter or a double-click. +Migration 10 records metadata snapshot operations, library audits, preservation +reports, correction exports, hardware notes, and package extractions. Enabled +plugins execute in a timeout-controlled child process; this improves crash and +resource isolation but is not an operating-system permission sandbox. + Imported ConsoleMods, XenonLibrary, Free60, Redump, and No-Intro information continues to retain source, revision, citation, licensing, and conflict data. Redump and No-Intro DAT files remain user-supplied; copyrighted game content is diff --git a/DOCS_INDEX.md b/DOCS_INDEX.md index bb50305..a19f4ee 100644 --- a/DOCS_INDEX.md +++ b/DOCS_INDEX.md @@ -24,6 +24,8 @@ and signing limitations - [Community Hub](COMMUNITY_HUB.md) - unified search, profile/package tools, guided console plans, preservation, plugins, recovery, and safety boundaries +- [Release Toolkit](RELEASE_TOOLKIT.md) - portable metadata snapshots, library + audits, preservation reports, corrections, and hardware inventory - [Project Status](PROJECT_STATUS.md) - completed work, boundaries, and roadmap - [Changelog](CHANGELOG.md) - release history diff --git a/GUI.py b/GUI.py index b01ff8f..d8959eb 100644 --- a/GUI.py +++ b/GUI.py @@ -27,6 +27,7 @@ from main import UnityScraper, Config from i18n import init_translator, get_translator, t from updater import VersionChecker + from ui_theme import apply_vs2010_theme from queue_manager import DownloadQueue except ImportError as e: print(f"Error: Missing required module: {e}") @@ -89,15 +90,7 @@ def set_window_icon(self): def setup_styles(self): """Configure ttk styles""" - style = ttk.Style() - style.theme_use('clam') - - # Configure colors - style.configure('TFrame', background='#f0f0f0') - style.configure('Title.TLabel', font=('Arial', 16, 'bold'), background='#f0f0f0') - style.configure('Subtitle.TLabel', font=('Arial', 10), background='#f0f0f0') - style.configure('Success.TLabel', foreground='green', background='#f0f0f0') - style.configure('Error.TLabel', foreground='red', background='#f0f0f0') + apply_vs2010_theme(self.root) def create_widgets(self): """Create all GUI widgets""" diff --git a/PLUGIN_API.md b/PLUGIN_API.md index 43234ba..56a0c06 100644 --- a/PLUGIN_API.md +++ b/PLUGIN_API.md @@ -29,8 +29,12 @@ to 2 MiB, cover/update counts are bounded, failures are isolated, and every run is audited in SQLite. Known title and publisher values are never replaced by a plugin fallback. +Enabled collectors run in a child process with a 30-second timeout and a bounded +result file. POSIX builds also request CPU, address-space, and output-file limits. +A timeout or worker crash does not terminate the desktop collection job. + Requested access is disclosure metadata, not an operating-system sandbox. -Plugin code executes with the user's account permissions, so only enable source +Plugin code still executes with the user's account permissions, so only enable source and publishers you trust. Editing an enabled entrypoint automatically prevents it from loading until it is reviewed and enabled again. diff --git a/PROJECT_STATUS.md b/PROJECT_STATUS.md index a1a93e4..b7aa4c8 100644 --- a/PROJECT_STATUS.md +++ b/PROJECT_STATUS.md @@ -63,6 +63,12 @@ backup-management, and source-attributed knowledge application. - Additive schema migration 8 and local audit history for every new workspace. - Additive schema migration 9 for audited plugin collection and reversible duplicate actions, plus selectable restore controls. +- Additive schema migration 10 for metadata snapshots, library audits, + preservation reports, correction exports, hardware records, and package extraction. +- Visual Studio 2010-inspired shared desktop theme, classic menus, scoped API + tokens, out-of-process plugin execution, and bounded community language packs. +- Read-only consecutive STFS extraction, direct Xenia launch controls, and + portable non-personal metadata distribution. - Background Community Hub jobs, actionable unified-search navigation, CLI/API parity for search and preservation, FATX geometry reports, and bounded STFS file-table inventory. @@ -107,5 +113,7 @@ backup-management, and source-attributed knowledge application. of real dashboard FTP servers. - Add notarization and universal binaries after macOS signing infrastructure is available. -- Consider package mutation only after complete STFS extraction, rehashing, - signing, verification, and automatic recovery have independent test vectors. +- Complete fragmented STFS block-chain traversal and hash-tree verification + against independent real-package test vectors before considering mutation. +- Consider package mutation only after complete extraction, rehashing, signing, + verification, and automatic recovery have independent test vectors. diff --git a/README.md b/README.md index 85c8696..75aaba7 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,8 @@ service exposes. UnityScraper does not silently substitute HTTPS URLs. - Reads extracted XDBF/GPD achievement databases without modifying them. - Compares save hashes and imported achievement state across two profiles. - Previews Xenia save mappings and creates a verified snapshot before migration. +- Discovers Xenia or Xenia Canary beside a selected content folder and launches + user-selected games directly without constructing a shell command. The profile/package model is informed by Dalavin, also known as DJ SkunkieButt, and the GPLv3 X360 library and Le Fluffie source. See @@ -125,7 +127,19 @@ checksums, platform notes, and safety guidance. search results in their native workspace, and restores quarantined duplicates. - Inventories STFS file tables in read-only package workspaces without claiming unsupported package rebuilding or signing. +- Extracts supported consecutive STFS files read-only with path validation, + output limits, atomic files, hashes, and a manifest. - Stores high-contrast, large-text, reduced-motion, and keyboard-hint settings. +- Exports and imports non-personal metadata snapshots, audits incomplete library + metadata, creates preservation reports and correction packages, and records + local hardware notes. + +### Desktop Style + +The desktop uses a Visual Studio 2010-inspired dark tool aesthetic: compact +square controls, charcoal chrome, blue selection and focus states, classic menu +commands, dense tables, and status bars. The primary and Advanced Tools windows +share the same theme, with high-contrast and large-text alternatives retained. See [COMMUNITY_HUB.md](COMMUNITY_HUB.md) for all twenty capabilities and their safety boundaries. @@ -207,7 +221,7 @@ Linux source setup: | External Tools | Run XeXTool and other user-supplied command-line utilities | | Collections | Identify storage, compare Title Updates, verify preservation data, and preview repairs | | Knowledge | Search sources, facts, citations, imports, and conflicts | -| Community Hub | Unified search, console plans, profiles, preservation, plugins, recovery, and compatibility | +| Community Hub | Unified search, console plans, profiles, preservation, plugins, recovery, compatibility, and release toolkit | | Archive Health | Find missing or inconsistent downloaded files | | Settings | Configure storage and scraper behavior | | Help & About | Version, diagnostics, storage, and advanced tools | @@ -338,6 +352,17 @@ python main.py --audit-storage E:\drive.img python main.py --dedup-preview D:\XboxArchive python main.py --dedup-apply 42 --dedup-mode quarantine python main.py --dedup-restore 42 + +# Share or consume a non-personal offline metadata snapshot +python main.py --metadata-snapshot-export xbox360.usmeta +python main.py --metadata-snapshot-import xbox360.usmeta + +# Audit the library and produce a privacy-conscious report +python main.py --library-audit +python main.py --preservation-report preservation.html + +# Extract supported files without changing the STFS source package +python main.py --extract-stfs save.con --extract-destination extracted ``` ## Optional REST API @@ -356,9 +381,16 @@ python main.py --api-mode --api-host 0.0.0.0 ``` Clients send the token as `Authorization: Bearer ` or `X-API-Key`. +Multiple scoped tokens can be provided through `UNITYSCRAPER_API_TOKENS` as a +JSON object whose values contain `read`, `write`, or `transfer`. Requests are +rate-limited per client. Remote HTTP is not encrypted; place it behind a trusted local reverse proxy or use it only on an isolated network. See [API.md](API.md). +See [RELEASE_TOOLKIT.md](RELEASE_TOOLKIT.md) for metadata snapshot privacy, +library intelligence, reports, corrections, hardware records, and STFS +extraction boundaries. + ## Build and Test Install development dependencies: diff --git a/RELEASE_TOOLKIT.md b/RELEASE_TOOLKIT.md new file mode 100644 index 0000000..ed65b16 --- /dev/null +++ b/RELEASE_TOOLKIT.md @@ -0,0 +1,51 @@ +# Release Toolkit + +The **Community Hub > Toolkit** page groups portable metadata, collection +attention, reporting, correction, and hardware workflows. + +## Metadata snapshots + +`.usmeta` files are compressed JSON snapshots containing the XboxUnity title +catalog and normalized knowledge entities, identifiers, facts, citations, source +names, and licenses. They explicitly exclude profiles, saves, download history, +credentials, local paths, console identifiers, and game content. + +Imports validate the archive shape, schema, expanded size, TitleIDs, and fact +records. Existing source-attributed claims can coexist; the normal preference and +conflict rules choose display values. + +## Library intelligence + +The audit highlights unknown game names or publishers, available but unarchived +covers and updates, and updates whose MediaID compatibility still needs evidence. +It records only a summary in SQLite and does not automatically download content. + +## Preservation reports + +The HTML report summarizes library attention and knowledge-source provenance. +Personal profile identifiers and filesystem paths are deliberately excluded. + +## Correction packages + +Correction exports contain reviewed local metadata overrides. They are suitable +for manual community review and do not publish or upload anything automatically. + +## Hardware records + +Users can keep local notes for motherboard, DVD drive, NAND, dashboard, and +console type. Serial numbers and keys are not requested. + +## Command line + +```powershell +python main.py --metadata-snapshot-export xbox360.usmeta +python main.py --metadata-snapshot-import xbox360.usmeta +python main.py --library-audit +python main.py --preservation-report preservation.html +python main.py --corrections-export corrections.json +python main.py --extract-stfs save.con --extract-destination extracted +``` + +STFS extraction currently supports files stored in consecutive blocks. A +fragmented file is reported and skipped until block-chain traversal and hash-tree +verification have independent real-package test vectors. diff --git a/UnityScraper.spec b/UnityScraper.spec index 5de0ef5..f1eac0a 100644 --- a/UnityScraper.spec +++ b/UnityScraper.spec @@ -43,6 +43,9 @@ a = Analysis( 'structured_knowledge', 'unified_search', 'plugins', + 'plugin_worker', + 'roadmap_services', + 'ui_theme', 'gpd_parser', 'profile_gui', 'profile_intelligence', diff --git a/VERSION b/VERSION index bc85f78..f42c8f2 100644 --- a/VERSION +++ b/VERSION @@ -1,5 +1,5 @@ { - "version": "1.1.0b1", + "version": "1.2.0b1", "name": "Unified Xbox 360 Collection and Preservation Manager", "changes": [ "Unified Xbox 360 knowledge browser", @@ -20,8 +20,16 @@ "Unified Community Hub across games, knowledge, profiles, preservation, and recovery", "Guided console sync plans and recoverable duplicate cleanup", "Played-title history and validated embedded GPD image export", - "Windows, Linux, and unsigned Apple Silicon macOS build paths" + "Windows, Linux, and unsigned Apple Silicon macOS build paths", + "Visual Studio 2010-inspired desktop theme and classic tool chrome", + "Read-only consecutive STFS extraction with manifests", + "Portable source-attributed metadata snapshots", + "Library intelligence, preservation reports, and correction packages", + "Out-of-process metadata plugin execution with timeouts", + "Scoped and rate-limited local REST API", + "Xenia discovery and direct game launching", + "Community language-pack loading and hardware inventory" ], "download_url": "https://github.com/TrapEmAll/UnityScraper/releases", - "release_date": "2026-07-23" + "release_date": "2026-08-01" } diff --git a/api.py b/api.py index de72db2..2592cdf 100644 --- a/api.py +++ b/api.py @@ -3,9 +3,12 @@ from __future__ import annotations import logging +import json import os import secrets import threading +import time +from collections import defaultdict, deque from datetime import datetime, timezone from pathlib import Path from typing import TYPE_CHECKING, Any, Callable, Optional @@ -49,16 +52,36 @@ def __init__( port: int = 8000, host: str = "127.0.0.1", token: Optional[str] = None, + token_scopes: Optional[dict[str, list[str]]] = None, cors_origins: Optional[list[str]] = None, + requests_per_minute: int = 120, ): self.scraper = scraper self.port = port self.host = host self.token = token or os.environ.get("UNITYSCRAPER_API_TOKEN", "").strip() - if host not in LOOPBACK_HOSTS and not self.token: + self.tokens: dict[str, frozenset[str]] = { + key: frozenset(values) for key, values in (token_scopes or {}).items() if key + } + if self.token: + self.tokens[self.token] = frozenset({"*"}) + raw_tokens = os.environ.get("UNITYSCRAPER_API_TOKENS", "").strip() + if raw_tokens: + try: + configured = json.loads(raw_tokens) + if isinstance(configured, dict): + for key, values in configured.items(): + if isinstance(key, str) and key and isinstance(values, list): + self.tokens[key] = frozenset(str(value) for value in values) + except json.JSONDecodeError: + logger.warning("UNITYSCRAPER_API_TOKENS is not valid JSON") + if host not in LOOPBACK_HOSTS and not self.tokens: raise ValueError( "A token is required when the API is bound beyond localhost" ) + self.requests_per_minute = max(10, min(int(requests_per_minute), 10_000)) + self._request_windows: dict[str, deque[float]] = defaultdict(deque) + self._security_lock = threading.Lock() self.app = Flask(__name__) self.app.config["MAX_CONTENT_LENGTH"] = 64 * 1024 @@ -77,14 +100,31 @@ def __init__( def _register_security(self) -> None: @self.app.before_request def require_token(): - if not self.token or request.path == "/api/health": + client = request.remote_addr or "unknown" + now = time.monotonic() + with self._security_lock: + window = self._request_windows[client] + while window and now - window[0] >= 60: + window.popleft() + if len(window) >= self.requests_per_minute: + return jsonify({"error": "Request limit exceeded"}), 429 + window.append(now) + if not self.tokens or request.path == "/api/health": return None supplied = request.headers.get("X-API-Key", "") authorization = request.headers.get("Authorization", "") if authorization.startswith("Bearer "): supplied = authorization[7:] - if not secrets.compare_digest(supplied, self.token): + matched_scopes: frozenset[str] | None = None + for configured_token, scopes in self.tokens.items(): + if secrets.compare_digest(supplied, configured_token): + matched_scopes = scopes + break + if matched_scopes is None: return jsonify({"error": "Authentication required"}), 401 + required = self._required_scope(request.method, request.path) + if "*" not in matched_scopes and required not in matched_scopes: + return jsonify({"error": f"Token lacks the {required} scope"}), 403 return None @self.app.after_request @@ -102,7 +142,8 @@ def health(): "status": "healthy", "version": DISPLAY_VERSION, "scraper_loaded": self.scraper is not None, - "authentication_required": bool(self.token), + "authentication_required": bool(self.tokens), + "rate_limit_per_minute": self.requests_per_minute, } ) @@ -209,6 +250,99 @@ def plugins(): "plugins": PluginControlService(self._database_path()).discover(PLUGINS_DIR) }) + @self.app.get("/api/library/audit") + def library_audit(): + from roadmap_services import LibraryIntelligenceService + + return self._execute( + lambda: LibraryIntelligenceService(self._database_path()).audit() + ) + + @self.app.post("/api/metadata-snapshots/export") + def metadata_snapshot_export(): + from roadmap_services import MetadataSnapshotService + + payload = request.get_json(silent=True) + if not isinstance(payload, dict) or not isinstance(payload.get("destination"), str): + return jsonify({"error": "A destination path is required"}), 400 + return self._execute( + lambda: MetadataSnapshotService(self._database_path()).export( + payload["destination"] + ) + ) + + @self.app.post("/api/metadata-snapshots/import") + def metadata_snapshot_import(): + from roadmap_services import MetadataSnapshotService + + payload = request.get_json(silent=True) + if not isinstance(payload, dict) or not isinstance(payload.get("source"), str): + return jsonify({"error": "A source path is required"}), 400 + return self._execute( + lambda: MetadataSnapshotService(self._database_path()).import_snapshot( + payload["source"] + ) + ) + + @self.app.post("/api/packages/extract") + def package_extract(): + from community_services import PackageWorkspaceService + + payload = request.get_json(silent=True) + if not isinstance(payload, dict): + return jsonify({"error": "A JSON object is required"}), 400 + source = payload.get("source") + destination = payload.get("destination") + selected = payload.get("selected_paths") + if not isinstance(source, str) or not isinstance(destination, str): + return jsonify({"error": "source and destination paths are required"}), 400 + if selected is not None and ( + not isinstance(selected, list) or not all(isinstance(item, str) for item in selected) + ): + return jsonify({"error": "selected_paths must be a string array"}), 400 + return self._execute( + lambda: PackageWorkspaceService(self._database_path()).extract_read_only( + source, destination, selected + ) + ) + + @self.app.post("/api/reports/preservation") + def preservation_report(): + from roadmap_services import PreservationReportService + + payload = request.get_json(silent=True) + if not isinstance(payload, dict) or not isinstance(payload.get("destination"), str): + return jsonify({"error": "A destination path is required"}), 400 + return self._execute( + lambda: PreservationReportService(self._database_path()).export_html( + payload["destination"] + ) + ) + + @self.app.get("/api/hardware") + def hardware_list(): + from roadmap_services import HardwareInventoryService + + return self._execute( + lambda: {"records": HardwareInventoryService(self._database_path()).list()} + ) + + @self.app.post("/api/hardware") + def hardware_save(): + from roadmap_services import HardwareInventoryService + + payload = request.get_json(silent=True) + if not isinstance(payload, dict) or not isinstance(payload.get("label"), str): + return jsonify({"error": "A record label is required"}), 400 + values = {key: value for key, value in payload.items() if key != "label"} + if not all(isinstance(value, str) for value in values.values()): + return jsonify({"error": "Hardware fields must be strings"}), 400 + return self._execute( + lambda: HardwareInventoryService(self._database_path()).save( + payload["label"], **values + ) + ) + @self.app.post("/api/metadata/") def collect_metadata(titleid: str): normalized = self._titleid_or_error(titleid) @@ -352,6 +486,14 @@ def _require_scraper(self) -> "UnityScraper": raise RuntimeError("Scraper not initialized") return self.scraper + @staticmethod + def _required_scope(method: str, path: str) -> str: + if method in {"GET", "HEAD", "OPTIONS"}: + return "read" + if path.startswith("/api/download") or path.startswith("/api/retry-failed"): + return "transfer" + return "write" + def _database_path(self) -> Path: if self.scraper is None: return DATABASE_PATH diff --git a/app_paths.py b/app_paths.py index d1c646d..d40bd09 100644 --- a/app_paths.py +++ b/app_paths.py @@ -147,6 +147,7 @@ def xdg_path(variable: str, fallback: Path) -> Path: DIAGNOSTICS_DIR = _PATHS.diagnostics PROFILE_BACKUPS_DIR = DATA_DIR / "profile_backups" PLUGINS_DIR = DATA_DIR / "plugins" +LANGUAGE_PACKS_DIR = DATA_DIR / "languages" DATABASE_PATH = DATA_DIR / "unityscraper.db" CONFIG_PATH = CONFIG_DIR / "config.json" @@ -174,6 +175,7 @@ def ensure_app_dirs() -> None: DIAGNOSTICS_DIR, PROFILE_BACKUPS_DIR, PLUGINS_DIR, + LANGUAGE_PACKS_DIR, ): path.mkdir(parents=True, exist_ok=True) diff --git a/app_version.py b/app_version.py index 43919b1..d48c2cd 100644 --- a/app_version.py +++ b/app_version.py @@ -1,4 +1,4 @@ """Single source of truth for UnityScraper version information.""" -APP_VERSION = "1.1.0b1" -DISPLAY_VERSION = "1.1.0-beta.1" +APP_VERSION = "1.2.0b1" +DISPLAY_VERSION = "1.2.0-beta.1" diff --git a/backup_manager.py b/backup_manager.py index 8f1af2c..46ef53e 100644 --- a/backup_manager.py +++ b/backup_manager.py @@ -312,6 +312,115 @@ def list_stfs_entries(path: str | Path, max_entries: int = 100_000) -> list[Stfs return entries +def extract_stfs_files( + path: str | Path, + destination: str | Path, + selected_paths: Iterable[str] | None = None, + *, + max_output_size: int = 32 * 1024 * 1024 * 1024, +) -> dict[str, Any]: + """Extract consecutive STFS files read-only with path and size validation. + + Fragmented files are reported instead of guessed. This keeps extraction useful + while making the unsupported block-chain case explicit and non-destructive. + """ + package = Path(path).expanduser().resolve() + target = Path(destination).expanduser().resolve() + if package == target or target.is_relative_to(package): + raise InvalidPackageError("Extraction destination must be outside the package") + requested = {item.replace("\\", "/") for item in selected_paths or ()} + entries = list_stfs_entries(package) + files = [ + entry for entry in entries + if not entry.is_directory and (not requested or entry.path in requested) + ] + if requested - {entry.path for entry in files}: + missing = sorted(requested - {entry.path for entry in files}) + raise InvalidPackageError(f"STFS entries were not found: {', '.join(missing[:5])}") + total_size = sum(entry.size for entry in files) + if total_size > max_output_size: + raise InvalidPackageError("Selected STFS output exceeds the extraction safety limit") + + with package.open("rb") as handle: + header = handle.read(0x3AD) + if len(header) < 0x3AD or header[:4] not in STFS_MAGICS: + raise InvalidPackageError("Not a supported STFS package") + header_size = int.from_bytes(header[0x340:0x344], "big") + descriptor = header[0x379:0x39D] + block_separation = descriptor[2] + allocated = int.from_bytes(descriptor[0x1C:0x20], "big") + aligned_header = (header_size + 0xFFF) & 0xFFFFF000 + target.mkdir(parents=True, exist_ok=True) + extracted: list[dict[str, Any]] = [] + skipped: list[dict[str, str]] = [] + for entry in files: + if not entry.consecutive and entry.allocated_blocks > 1: + skipped.append({"path": entry.path, "reason": "fragmented block chain"}) + continue + required_blocks = (entry.size + 0xFFF) // 0x1000 + if required_blocks > entry.allocated_blocks or entry.starting_block + required_blocks > allocated: + skipped.append({"path": entry.path, "reason": "invalid block allocation"}) + continue + relative = _safe_archive_member(entry.path) + output = (target / relative).resolve() + if not output.is_relative_to(target): + raise UnsafeArchiveError(f"STFS path escapes destination: {entry.path}") + if output.exists(): + skipped.append({"path": entry.path, "reason": "destination exists"}) + continue + output.parent.mkdir(parents=True, exist_ok=True) + partial = output.with_name(output.name + ".partial") + digest = hashlib.sha256() + remaining = entry.size + try: + with partial.open("xb") as destination_handle: + for block in range(entry.starting_block, entry.starting_block + required_blocks): + physical = _stfs_data_block_number( + block, header[:4], header_size, block_separation + ) + offset = aligned_header + physical * 0x1000 + if offset + min(remaining, 0x1000) > package.stat().st_size: + raise InvalidPackageError( + f"STFS data points outside the package: {entry.path}" + ) + handle.seek(offset) + chunk = handle.read(min(remaining, 0x1000)) + if len(chunk) != min(remaining, 0x1000): + raise InvalidPackageError(f"STFS data is truncated: {entry.path}") + destination_handle.write(chunk) + digest.update(chunk) + remaining -= len(chunk) + partial.replace(output) + finally: + partial.unlink(missing_ok=True) + extracted.append({ + "path": entry.path, + "output": str(output), + "size": entry.size, + "sha256": digest.hexdigest(), + }) + manifest = target / "unityscraper-stfs-extraction.json" + manifest.write_text(json.dumps({ + "schema": 1, + "source": str(package), + "source_sha256": sha256_file(package), + "read_only": True, + "extracted": extracted, + "skipped": skipped, + }, indent=2), encoding="utf-8") + return {"manifest": str(manifest), "extracted": extracted, "skipped": skipped} + + +def _safe_archive_member(value: str) -> Path: + normalized = value.replace("\\", "/") + pure = PurePosixPath(normalized) + if pure.is_absolute() or not pure.parts or any(part in {"", ".", ".."} for part in pure.parts): + raise UnsafeArchiveError(f"Unsafe package path: {value}") + if ":" in pure.parts[0]: + raise UnsafeArchiveError(f"Unsafe package path: {value}") + return Path(*pure.parts) + + def inspect_xbe(path: str | Path) -> XbePackage: """Read TitleID and title from an original Xbox executable certificate.""" package_path = Path(path) diff --git a/build_linux.sh b/build_linux.sh index fb6f905..5266bbb 100755 --- a/build_linux.sh +++ b/build_linux.sh @@ -29,7 +29,7 @@ install -m 0644 README.md CHANGELOG.md LICENSE "$STAGE/" install -m 0644 DOCS_INDEX.md BACKUP_MANAGER.md COLLECTION_INTELLIGENCE.md \ COMMUNITY_HUB.md CONSOLE_SYNC.md KNOWLEDGE_SOURCES.md LINUX.md \ PLUGIN_API.md PROFILE_INTELLIGENCE.md PROFILES_AND_SAVES.md \ - SECURITY.md THIRD_PARTY_NOTICES.md "$STAGE/" + RELEASE_TOOLKIT.md SECURITY.md THIRD_PARTY_NOTICES.md "$STAGE/" tar -C dist -czf "$ARCHIVE" "UnityScraper-Linux-${ARCH}" sha256sum "$ARCHIVE" > "$ARCHIVE.sha256" diff --git a/collection_intelligence.py b/collection_intelligence.py index 897e4d7..1229fc5 100644 --- a/collection_intelligence.py +++ b/collection_intelligence.py @@ -615,9 +615,9 @@ def export_html( ) document = f""" UnityScraper Collection - +

Xbox 360 Collection Report

Health score: {analysis.health_score}/100

Generated {html.escape(utc_now())}; source {html.escape(str(analysis.result.root))}

diff --git a/community_gui.py b/community_gui.py index f901ae0..70cb15d 100644 --- a/community_gui.py +++ b/community_gui.py @@ -25,9 +25,17 @@ StorageAndXboxService, ) from profile_intelligence import ProfileIntelligenceService +from roadmap_services import ( + CorrectionPackageService, + HardwareInventoryService, + LibraryIntelligenceService, + MetadataSnapshotService, + PreservationReportService, +) from platform_support import open_path from structured_knowledge import StructuredKnowledgeService from unified_search import UnifiedSearchService +from ui_theme import PALETTE class CommunityHubPage: @@ -54,6 +62,11 @@ def __init__( self.recovery = RecoveryService() self.compatibility = DashboardCompatibilityService() self.accessibility = AccessibilityService() + self.metadata_snapshots = MetadataSnapshotService() + self.library_intelligence = LibraryIntelligenceService() + self.preservation_reports = PreservationReportService() + self.corrections = CorrectionPackageService() + self.hardware = HardwareInventoryService() self.navigate = navigate self.search_rows: dict[str, dict[str, Any]] = {} self.task_events: queue.Queue[tuple[str, Future, Callable | None]] = queue.Queue() @@ -77,6 +90,7 @@ def _build(self) -> None: ("Plugins", self._build_plugins), ("Recovery", self._build_recovery), ("Compatibility", self._build_compatibility), + ("Toolkit", self._build_toolkit), ("Accessibility", self._build_accessibility), ): frame = ttk.Frame(self.notebook, padding=12) @@ -286,6 +300,9 @@ def _build_profiles(self, frame: ttk.Frame) -> None: ttk.Button(buttons, text="Create Read-only Workspace", command=self._package_workspace).pack( side=tk.LEFT, padx=8 ) + ttk.Button(buttons, text="Extract Supported Files", command=self._extract_package).pack( + side=tk.LEFT, padx=(0, 8) + ) ttk.Button(buttons, text="Ownership Migration Preview", command=self._ownership_preview).pack( side=tk.LEFT ) @@ -316,6 +333,18 @@ def _package_workspace(self) -> None: "manifest": str(self.packages.create_workspace( package_path, destination))}) + def _extract_package(self) -> None: + destination = filedialog.askdirectory( + parent=self.root, title="Choose read-only extraction folder" + ) + if destination: + package_path = self.package_path.get() + self._submit_output( + "Package extraction", + self.profile_output, + lambda: self.packages.extract_read_only(package_path, destination), + ) + def _ownership_preview(self) -> None: profile_id = self.profile_id.get() package_path = self.package_path.get() @@ -614,6 +643,97 @@ def _probe(self) -> None: self._submit_output("Dashboard probe", self.compat_output, lambda: self.compatibility.probe(dashboard, target)) + def _build_toolkit(self, frame: ttk.Frame) -> None: + frame.columnconfigure(1, weight=1) + frame.rowconfigure(8, weight=1) + metadata = ttk.LabelFrame(frame, text="Portable metadata", padding=8) + metadata.grid(row=0, column=0, columnspan=3, sticky="ew", pady=(0, 8)) + ttk.Button(metadata, text="Export Snapshot", command=self._export_metadata_snapshot).pack( + side=tk.LEFT + ) + ttk.Button(metadata, text="Import Snapshot", command=self._import_metadata_snapshot).pack( + side=tk.LEFT, padx=6 + ) + ttk.Button(metadata, text="Audit Library", command=self._audit_library).pack(side=tk.LEFT) + ttk.Button(metadata, text="Preservation Report", command=self._preservation_report).pack( + side=tk.LEFT, padx=6 + ) + ttk.Button(metadata, text="Export Corrections", command=self._export_corrections).pack( + side=tk.LEFT + ) + + hardware = ttk.LabelFrame(frame, text="Console hardware record", padding=8) + hardware.grid(row=1, column=0, columnspan=3, sticky="ew", pady=(0, 8)) + hardware.columnconfigure(1, weight=1) + self.hardware_vars = { + key: tk.StringVar() for key in ( + "label", "motherboard", "dvd_drive", "nand_type", + "dashboard_version", "console_type", "notes", + ) + } + fields = ( + ("Record label", "label"), ("Motherboard", "motherboard"), + ("DVD drive", "dvd_drive"), ("NAND", "nand_type"), + ("Dashboard", "dashboard_version"), ("Console type", "console_type"), + ("Notes", "notes"), + ) + for row, (label, key) in enumerate(fields): + ttk.Label(hardware, text=label).grid(row=row // 2, column=(row % 2) * 2, + sticky="w", padx=(0, 5), pady=3) + ttk.Entry(hardware, textvariable=self.hardware_vars[key], width=28).grid( + row=row // 2, column=(row % 2) * 2 + 1, sticky="ew", padx=(0, 12), pady=3 + ) + ttk.Button(hardware, text="Save Hardware Record", command=self._save_hardware).grid( + row=4, column=0, columnspan=4, sticky="w", pady=(6, 0) + ) + self.toolkit_output = self._output(frame, 8, 3) + + def _export_metadata_snapshot(self) -> None: + destination = filedialog.asksaveasfilename( + parent=self.root, title="Export metadata snapshot", + defaultextension=".usmeta", filetypes=(("UnityScraper metadata", "*.usmeta"),), + ) + if destination: + self._submit_output("Metadata snapshot", self.toolkit_output, + lambda: self.metadata_snapshots.export(destination)) + + def _import_metadata_snapshot(self) -> None: + source = filedialog.askopenfilename( + parent=self.root, title="Import metadata snapshot", + filetypes=(("UnityScraper metadata", "*.usmeta"), ("All files", "*.*")), + ) + if source: + self._submit_output("Metadata import", self.toolkit_output, + lambda: self.metadata_snapshots.import_snapshot(source)) + + def _audit_library(self) -> None: + self._submit_output("Library intelligence", self.toolkit_output, + self.library_intelligence.audit) + + def _preservation_report(self) -> None: + destination = filedialog.asksaveasfilename( + parent=self.root, title="Export preservation report", + defaultextension=".html", filetypes=(("HTML report", "*.html"),), + ) + if destination: + self._submit_output("Preservation report", self.toolkit_output, + lambda: self.preservation_reports.export_html(destination)) + + def _export_corrections(self) -> None: + destination = filedialog.asksaveasfilename( + parent=self.root, title="Export community corrections", + defaultextension=".json", filetypes=(("JSON package", "*.json"),), + ) + if destination: + self._submit_output("Correction package", self.toolkit_output, + lambda: self.corrections.export(destination)) + + def _save_hardware(self) -> None: + values = {key: variable.get() for key, variable in self.hardware_vars.items()} + label = values.pop("label") + self._submit_output("Hardware record", self.toolkit_output, + lambda: self.hardware.save(label, **values)) + def _build_accessibility(self, frame: ttk.Frame) -> None: values = self.accessibility.get() self.access_vars: dict[str, tk.BooleanVar] = {} @@ -649,8 +769,12 @@ def choose() -> None: @staticmethod def _output(frame: ttk.Frame, row: int, columnspan: int) -> tk.Text: frame.rowconfigure(row, weight=1) - output = tk.Text(frame, wrap=tk.WORD, height=12, background="#070b08", - foreground="#eef4ef", insertbackground="#75d34b") + output = tk.Text( + frame, wrap=tk.WORD, height=12, background=PALETTE.field, + foreground=PALETTE.text, insertbackground=PALETTE.accent_hot, + selectbackground=PALETTE.selection, relief=tk.FLAT, + highlightthickness=1, highlightbackground=PALETTE.border, + ) output.grid(row=row, column=0, columnspan=columnspan, sticky="nsew", pady=(8, 0)) output.configure(state=tk.DISABLED) return output diff --git a/community_services.py b/community_services.py index aa92215..c731ded 100644 --- a/community_services.py +++ b/community_services.py @@ -18,7 +18,13 @@ from PIL import Image from app_paths import DATABASE_PATH -from backup_manager import FtpTarget, inspect_stfs, inspect_xbe, list_stfs_entries +from backup_manager import ( + FtpTarget, + extract_stfs_files, + inspect_stfs, + inspect_xbe, + list_stfs_entries, +) from console_sync import ConsoleSyncService from database_migrations import ensure_application_schema from plugins import PluginManifest @@ -291,6 +297,26 @@ def create_workspace(self, package_path: str | Path, destination: str | Path) -> ) return manifest + def extract_read_only( + self, + package_path: str | Path, + destination: str | Path, + selected_paths: Iterable[str] | None = None, + ) -> dict[str, Any]: + """Extract supported files without changing or replacing package data.""" + result = extract_stfs_files(package_path, destination, selected_paths) + with self.connect() as connection: + cursor = connection.execute( + """INSERT INTO package_extraction_runs( + source_path,destination,created_at,extracted_count,skipped_count, + manifest_path,status) VALUES (?,?,?,?,?,?,'completed')""", + (str(Path(package_path).expanduser().resolve()), + str(Path(destination).expanduser().resolve()), utc_now(), + len(result["extracted"]), len(result["skipped"]), result["manifest"]), + ) + result["run_id"] = int(cursor.lastrowid or 0) + return result + class ArtworkService(CommunityRepository): PRESETS = { @@ -796,6 +822,10 @@ def scan(self, roots: Iterable[str | Path]) -> list[dict[str, Any]]: ("failed_transfer", "SELECT id, local_path source, error_message details FROM console_transfer_jobs WHERE status='failed'"), ("incomplete_snapshot", "SELECT id, snapshot_path source, status details FROM save_snapshots WHERE status<>'complete'"), ("failed_operation", "SELECT id, source, error_message details FROM backup_operations WHERE status='failed'"), + ("failed_profile_operation", "SELECT id, target_path source, error_message details FROM profile_save_operations WHERE status='failed'"), + ("failed_catalog_sync", "SELECT id, 'XboxUnity title catalog' source, error_message details FROM xboxunity_catalog_sync_runs WHERE status IN ('failed','interrupted')"), + ("failed_plugin_run", "SELECT id, plugin_id || ':' || titleid source, error_message details FROM plugin_collection_runs WHERE status='failed'"), + ("failed_knowledge_sync", "SELECT id, source_slug || ':' || adapter_name source, errors details FROM knowledge_import_runs WHERE status IN ('failed','partial')"), ) for event_type, sql in queries: for row in connection.execute(sql).fetchall(): diff --git a/database_migrations.py b/database_migrations.py index 7abd6f6..d953636 100644 --- a/database_migrations.py +++ b/database_migrations.py @@ -8,7 +8,7 @@ from pathlib import Path -SCHEMA_VERSION = 9 +SCHEMA_VERSION = 10 def _now() -> str: @@ -70,6 +70,7 @@ def ensure_application_schema(connection: sqlite3.Connection) -> int: (7, "profile intelligence and knowledge controls", _migration_roadmap), (8, "community roadmap workspaces", _migration_community_roadmap), (9, "hardening and plugin runtime", _migration_hardening), + (10, "release readiness workspaces", _migration_release_readiness), ) for version, name, migration in migrations: if version in applied: @@ -738,3 +739,69 @@ def _migration_hardening(connection: sqlite3.Connection) -> None: ); """ ) + + +def _migration_release_readiness(connection: sqlite3.Connection) -> None: + """Add audit history for portable metadata, reports, and hardware records.""" + connection.executescript( + """ + CREATE TABLE IF NOT EXISTS metadata_snapshot_runs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + operation TEXT NOT NULL, + snapshot_path TEXT NOT NULL, + created_at TEXT NOT NULL, + completed_at TEXT, + catalog_count INTEGER NOT NULL DEFAULT 0, + fact_count INTEGER NOT NULL DEFAULT 0, + status TEXT NOT NULL, + sha256 TEXT, + error_message TEXT + ); + CREATE TABLE IF NOT EXISTS library_intelligence_runs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + created_at TEXT NOT NULL, + title_count INTEGER NOT NULL DEFAULT 0, + issue_count INTEGER NOT NULL DEFAULT 0, + summary_json TEXT NOT NULL + ); + CREATE TABLE IF NOT EXISTS preservation_report_runs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + destination TEXT NOT NULL, + created_at TEXT NOT NULL, + report_format TEXT NOT NULL, + status TEXT NOT NULL, + sha256 TEXT + ); + CREATE TABLE IF NOT EXISTS correction_packages ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + operation TEXT NOT NULL, + package_path TEXT NOT NULL, + created_at TEXT NOT NULL, + correction_count INTEGER NOT NULL DEFAULT 0, + status TEXT NOT NULL, + sha256 TEXT + ); + CREATE TABLE IF NOT EXISTS hardware_inventory_records ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + label TEXT NOT NULL, + motherboard TEXT, + dvd_drive TEXT, + nand_type TEXT, + dashboard_version TEXT, + console_type TEXT, + notes TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL + ); + CREATE TABLE IF NOT EXISTS package_extraction_runs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + source_path TEXT NOT NULL, + destination TEXT NOT NULL, + created_at TEXT NOT NULL, + extracted_count INTEGER NOT NULL DEFAULT 0, + skipped_count INTEGER NOT NULL DEFAULT 0, + manifest_path TEXT, + status TEXT NOT NULL + ); + """ + ) diff --git a/desktop_app.py b/desktop_app.py index 131d43c..684bd47 100644 --- a/desktop_app.py +++ b/desktop_app.py @@ -19,6 +19,11 @@ def _write_startup_report(exc: BaseException) -> str: def main() -> int: """Initialize writable storage and launch the desktop application.""" + if len(sys.argv) > 1 and sys.argv[1] == "--plugin-worker": + from plugin_worker import main as plugin_worker_main + + return plugin_worker_main(sys.argv[2:]) + ensure_app_dirs() ensure_user_titleids_file() diff --git a/external_tools_gui.py b/external_tools_gui.py index 0942932..c7e31e4 100644 --- a/external_tools_gui.py +++ b/external_tools_gui.py @@ -19,6 +19,7 @@ format_command, split_arguments, ) +from ui_theme import PALETTE XEXTOOL_PRESETS = { @@ -165,15 +166,15 @@ def _build(self) -> None: self.output_text = tk.Text( output_panel, wrap=tk.WORD, - background="#070b08", - foreground="#f2f5f2", - insertbackground="#72e000", - selectbackground="#315f12", - selectforeground="#f2f5f2", + background=PALETTE.field, + foreground=PALETTE.text, + insertbackground=PALETTE.accent_hot, + selectbackground=PALETTE.selection, + selectforeground=PALETTE.text, relief=tk.FLAT, highlightthickness=1, - highlightbackground="#26352a", - highlightcolor="#72e000", + highlightbackground=PALETTE.border, + highlightcolor=PALETTE.accent, ) self.output_text.grid(row=0, column=0, sticky="nsew") scrollbar = ttk.Scrollbar( diff --git a/i18n.py b/i18n.py index 100a289..18926f9 100644 --- a/i18n.py +++ b/i18n.py @@ -5,6 +5,7 @@ import json import logging +import re from pathlib import Path from typing import Dict, Optional @@ -12,6 +13,7 @@ # Supported languages SUPPORTED_LANGUAGES = ['en', 'es', 'fr', 'de', 'it', 'pt', 'ja'] +LANGUAGE_CODE_RE = re.compile(r"^[a-z]{2,3}(?:-[A-Z]{2})?$") # Translation strings TRANSLATIONS = { @@ -68,6 +70,18 @@ 'pending': 'Pending', 'downloaded': 'Downloaded', 'failed': 'Failed', + 'nav_library': 'LIBRARY', + 'nav_add_games': 'ADD GAMES', + 'nav_downloads': 'DOWNLOADS', + 'nav_backups': 'BACKUP MANAGER', + 'nav_profiles': 'PROFILES & SAVES', + 'nav_tools': 'EXTERNAL TOOLS', + 'nav_collections': 'COLLECTIONS', + 'nav_knowledge': 'KNOWLEDGE', + 'nav_community': 'COMMUNITY HUB', + 'nav_health': 'ARCHIVE HEALTH', + 'nav_settings': 'SETTINGS', + 'nav_about': 'HELP & ABOUT', }, 'es': { 'title': 'UnityScraper - Edición Mejorada', @@ -242,7 +256,7 @@ def _repair_legacy_text(value: str) -> str: """Repair translations that were historically saved with the wrong encoding.""" - if not any(marker in value for marker in ("Ã", "Â", "â", "ã", "ç", "é")): + if not any(marker in value for marker in ("Ã", "Â", "â", "ã", "æ", "ç", "é")): return value for encoding in ("cp1252", "latin-1"): try: @@ -268,7 +282,7 @@ def __init__(self, language: str = 'en'): logger.warning(f"Language '{language}' not supported, using English") language = 'en' self.language = language - self.strings = TRANSLATIONS.get(language, TRANSLATIONS['en']) + self.strings = {**TRANSLATIONS['en'], **TRANSLATIONS.get(language, {})} def get(self, key: str, *args) -> str: """Get translated string with optional formatting""" @@ -283,7 +297,7 @@ def set_language(self, language: str): logger.warning(f"Language '{language}' not supported") return False self.language = language - self.strings = TRANSLATIONS.get(language, TRANSLATIONS['en']) + self.strings = {**TRANSLATIONS['en'], **TRANSLATIONS.get(language, {})} return True @staticmethod @@ -296,9 +310,41 @@ def get_supported_languages() -> list: _translator: Optional[Translator] = None -def init_translator(language: str = 'en') -> Translator: +def load_language_packs(directory: str | Path) -> list[str]: + """Load bounded user language packs without executing code.""" + root = Path(directory) + loaded: list[str] = [] + if not root.is_dir(): + return loaded + for path in sorted(root.glob("*.json")): + if path.stat().st_size > 1024 * 1024: + logger.warning("Ignoring oversized language pack: %s", path) + continue + try: + payload = json.loads(path.read_text(encoding="utf-8")) + code = str(payload["language"]) + strings = payload["strings"] + if not LANGUAGE_CODE_RE.fullmatch(code) or not isinstance(strings, dict): + raise ValueError("invalid language-pack schema") + validated = { + str(key): value for key, value in strings.items() + if isinstance(key, str) and isinstance(value, str) and len(value) <= 2000 + } + except (OSError, json.JSONDecodeError, KeyError, ValueError) as exc: + logger.warning("Ignoring invalid language pack %s: %s", path, exc) + continue + TRANSLATIONS[code] = {**TRANSLATIONS["en"], **validated} + if code not in SUPPORTED_LANGUAGES: + SUPPORTED_LANGUAGES.append(code) + loaded.append(code) + return loaded + + +def init_translator(language: str = 'en', language_pack_dir: str | Path | None = None) -> Translator: """Initialize global translator""" global _translator + if language_pack_dir is not None: + load_language_packs(language_pack_dir) _translator = Translator(language) return _translator diff --git a/knowledge_gui.py b/knowledge_gui.py index 6004d9d..7fcf060 100644 --- a/knowledge_gui.py +++ b/knowledge_gui.py @@ -10,10 +10,11 @@ from knowledge_service import KnowledgeService from knowledge_scheduler import KnowledgeScheduler +from ui_theme import PALETTE -TEXT = "#f2f5f2" -ACCENT = "#72e000" -BORDER = "#26352a" +TEXT = PALETTE.text +ACCENT = PALETTE.accent +BORDER = PALETTE.border class KnowledgePage: @@ -129,10 +130,10 @@ def _build_browse(self, parent: ttk.Frame) -> None: self.detail_text = tk.Text( details, wrap=tk.WORD, - background="#070b08", + background=PALETTE.field, foreground=TEXT, insertbackground=ACCENT, - selectbackground="#315f12", + selectbackground=PALETTE.selection, selectforeground=TEXT, relief=tk.FLAT, highlightthickness=1, diff --git a/main.py b/main.py index 66acf79..4a0a407 100644 --- a/main.py +++ b/main.py @@ -961,6 +961,20 @@ def main(): help='Restore one quarantined duplicate action by ID') parser.add_argument('--dedup-mode', choices=['quarantine', 'hardlink'], default='quarantine', help='Action used with --dedup-apply') + parser.add_argument('--metadata-snapshot-export', type=str, + help='Export portable source-attributed metadata to a .usmeta file') + parser.add_argument('--metadata-snapshot-import', type=str, + help='Merge a portable .usmeta snapshot without personal data') + parser.add_argument('--library-audit', action='store_true', + help='Report missing names, publishers, covers, updates, and MediaIDs') + parser.add_argument('--preservation-report', type=str, + help='Export a privacy-conscious HTML preservation report') + parser.add_argument('--corrections-export', type=str, + help='Export reviewed local metadata corrections as JSON') + parser.add_argument('--extract-stfs', type=str, + help='Extract supported files read-only from an STFS package') + parser.add_argument('--extract-destination', type=str, + help='Destination folder required by --extract-stfs') parser.add_argument( '--scan-backups', type=str, @@ -1159,9 +1173,25 @@ def main(): or args.dedup_preview or args.dedup_apply is not None or args.dedup_restore is not None + or args.metadata_snapshot_export + or args.metadata_snapshot_import + or args.library_audit + or args.preservation_report + or args.corrections_export + or args.extract_stfs ): try: - from community_services import PreservationPlanningService, StorageAndXboxService + from community_services import ( + PackageWorkspaceService, + PreservationPlanningService, + StorageAndXboxService, + ) + from roadmap_services import ( + CorrectionPackageService, + LibraryIntelligenceService, + MetadataSnapshotService, + PreservationReportService, + ) from structured_knowledge import StructuredKnowledgeService from unified_search import UnifiedSearchService @@ -1190,6 +1220,30 @@ def main(): results["dedup_restore"] = preservation.restore_dedup_action( args.dedup_restore ) + if args.metadata_snapshot_export: + results["metadata_snapshot_export"] = MetadataSnapshotService().export( + args.metadata_snapshot_export + ) + if args.metadata_snapshot_import: + results["metadata_snapshot_import"] = MetadataSnapshotService().import_snapshot( + args.metadata_snapshot_import + ) + if args.library_audit: + results["library_audit"] = LibraryIntelligenceService().audit() + if args.preservation_report: + results["preservation_report"] = PreservationReportService().export_html( + args.preservation_report + ) + if args.corrections_export: + results["corrections_export"] = CorrectionPackageService().export( + args.corrections_export + ) + if args.extract_stfs: + if not args.extract_destination: + parser.error("--extract-destination is required with --extract-stfs") + results["stfs_extraction"] = PackageWorkspaceService().extract_read_only( + args.extract_stfs, args.extract_destination + ) print(json.dumps(results, indent=2, default=str)) sys.exit(0) except Exception as e: diff --git a/modern_gui.py b/modern_gui.py index d8b9734..323fec7 100644 --- a/modern_gui.py +++ b/modern_gui.py @@ -26,6 +26,7 @@ DOWNLOADS_DIR, EXPORTS_DIR, GUI_LOG_PATH, + LANGUAGE_PACKS_DIR, TITLEIDS_PATH, describe_storage, ensure_app_dirs, @@ -46,12 +47,14 @@ from knowledge_service import KnowledgeService from knowledge_scheduler import KnowledgeScheduler from knowledge_gui import KnowledgePage +from i18n import SUPPORTED_LANGUAGES, init_translator, t from library_service import GameSummary, LibraryService -from platform_support import desktop_font_family, open_path +from platform_support import open_path from profile_gui import ProfileSavePage from profile_manager import ProfileSaveManager from setup_wizard import run_first_run_wizard from title_catalog import TitleSuggestion, XboxUnityTitleCatalog +from ui_theme import PALETTE, UI_FONT, apply_vs2010_theme from updater import VersionChecker @@ -68,17 +71,16 @@ "for years to come." ) -BG = "#050806" -PANEL = "#0a0f0c" -PANEL_ALT = "#0d1510" -BORDER = "#26352a" -ACCENT = "#72e000" -ACCENT_HOVER = "#8cff18" -TEXT = "#f2f5f2" -MUTED = "#a5b2a8" -DANGER = "#ff5d68" -WARNING = "#ffc857" -UI_FONT = desktop_font_family() +BG = PALETTE.window +PANEL = PALETTE.panel +PANEL_ALT = PALETTE.panel_alt +BORDER = PALETTE.border +ACCENT = PALETTE.accent +ACCENT_HOVER = PALETTE.accent_hot +TEXT = PALETTE.text +MUTED = PALETTE.muted +DANGER = PALETTE.danger +WARNING = PALETTE.warning def _open_path(path: Path) -> None: @@ -124,7 +126,7 @@ def __init__( height=self.banner_height, highlightthickness=0, borderwidth=0, - background="#0b0f0d", + background=PALETTE.window, ) self.canvas.pack(fill=tk.X, expand=True) @@ -151,32 +153,32 @@ def _redraw(self) -> None: if self._source_image is not None: image = self._cover_resize(self._source_image, width, height) - overlay = Image.new("RGB", image.size, "#050807") - image = Image.blend(image, overlay, 0.05) + overlay = Image.new("RGB", image.size, PALETTE.window) + image = Image.blend(image, overlay, 0.45) self._photo = ImageTk.PhotoImage(image) self.canvas.create_image(0, 0, image=self._photo, anchor=tk.NW) else: self.canvas.create_rectangle( - 0, 0, width, height, fill="#0b0f0d", outline="" + 0, 0, width, height, fill=PALETTE.window, outline="" ) self.canvas.create_text( width // 2, height // 2, text=f"Background not loaded: {self.image_path}", anchor=tk.CENTER, - fill="#d6e6d3", + fill=PALETTE.muted, font=(UI_FONT, 10), ) self.canvas.create_rectangle( - 0, height - 5, width, height, fill="#5bd600", outline="" + 0, height - 4, width, height, fill=PALETTE.accent, outline="" ) self.canvas.create_text( 28, 54, text=self.title, anchor=tk.W, - fill="#ffffff", + fill=PALETTE.text, font=(UI_FONT, 23, "bold"), ) self.canvas.create_text( @@ -184,7 +186,7 @@ def _redraw(self) -> None: 94, text=self.subtitle, anchor=tk.W, - fill="#d6e6d3", + fill=PALETTE.muted, width=max(width - 60, 300), font=(UI_FONT, 11), ) @@ -197,7 +199,7 @@ def _cover_resize(image: Image.Image, width: int, height: int) -> Image.Image: (width, height), method=Image.Resampling.LANCZOS, ) - background = Image.new("RGB", (width, height), "#080c0a") + background = Image.new("RGB", (width, height), PALETTE.window) left = (width - fitted.width) // 2 top = (height - fitted.height) // 2 background.paste(fitted, (left, top)) @@ -236,9 +238,9 @@ def __init__( self.listbox = tk.Listbox( self.popup, height=8, - background="#070b08", + background=PALETTE.field, foreground=TEXT, - selectbackground="#315f12", + selectbackground=PALETTE.selection, selectforeground=TEXT, highlightthickness=1, highlightbackground=BORDER, @@ -346,6 +348,8 @@ def __init__(self, root: tk.Tk) -> None: self.root.title(f"UnityScraper {APP_VERSION}") config = self._read_config() + self.language = str(config.get("language", "en")) + init_translator(self.language, LANGUAGE_PACKS_DIR) scale = max(0.8, min(2.0, float(config.get("ui_scale", 1.0)))) if self.accessibility_preferences["large_text"]: scale = min(2.0, scale + 0.25) @@ -354,11 +358,13 @@ def __init__(self, root: tk.Tk) -> None: self.root.minsize(980, 640) self._set_icon() self._configure_style() + self._build_menubar() self._build_shell() if run_first_run_wizard(self.root): self.refresh_library() - self.root.after(750, self._start_catalog_sync_if_stale) + if bool(config.get("sync_title_catalog_on_start", True)): + self.root.after(750, self._start_catalog_sync_if_stale) self.root.after(1500, self._start_scheduled_knowledge_refresh) else: self.root.after(0, self.root.destroy) @@ -374,98 +380,66 @@ def _set_icon(self) -> None: pass def _configure_style(self) -> None: - self.root.configure(background=BG) - style = ttk.Style(self.root) - try: - style.theme_use("clam") - except tk.TclError: - pass + apply_vs2010_theme( + self.root, + high_contrast=bool(self.accessibility_preferences["high_contrast"]), + large_text=bool(self.accessibility_preferences["large_text"]), + ) - style.configure(".", background=PANEL, foreground=TEXT, fieldbackground=PANEL_ALT, - bordercolor=BORDER, darkcolor=PANEL, lightcolor=PANEL, - troughcolor=BG, selectbackground="#315f12", selectforeground=TEXT, - font=(UI_FONT, 10)) - style.configure("TFrame", background=PANEL) - style.configure("Sidebar.TFrame", background="#060a07") - style.configure("Content.TFrame", background=PANEL) - style.configure("TLabel", background=PANEL, foreground=TEXT) - style.configure("Brand.TLabel", background="#060a07", foreground=TEXT, - font=(UI_FONT, 17, "bold")) - style.configure("AccentBrand.TLabel", background="#060a07", foreground=ACCENT, - font=(UI_FONT, 10)) - style.configure("Header.TLabel", background=PANEL, foreground=TEXT, - font=(UI_FONT, 24, "bold")) - style.configure("Subheader.TLabel", background=PANEL, foreground=MUTED, - font=(UI_FONT, 11)) - style.configure("Metric.TLabel", background=PANEL_ALT, foreground=ACCENT, - font=(UI_FONT, 20, "bold")) - style.configure("CardTitle.TLabel", background=PANEL, foreground=ACCENT, - font=(UI_FONT, 11, "bold")) - style.configure("StatusDownloaded.TLabel", background=PANEL, foreground=ACCENT) - style.configure("StatusFailed.TLabel", background=PANEL, foreground=DANGER) - style.configure("StatusPending.TLabel", background=PANEL, foreground=WARNING) - - style.configure("TButton", background=PANEL_ALT, foreground=TEXT, padding=(12, 8), - borderwidth=1, relief="flat") - style.map("TButton", background=[("active", "#152319"), ("pressed", "#1b321e")], - foreground=[("active", ACCENT_HOVER)]) - style.configure("Nav.TButton", anchor=tk.W, padding=(16, 13), background="#080d09", - foreground=TEXT, bordercolor="#1f3321") - style.map("Nav.TButton", background=[("active", "#142017"), ("pressed", "#1c351c")], - foreground=[("active", ACCENT_HOVER)]) - style.configure("Accent.TButton", background="#183a0b", foreground=TEXT, - bordercolor=ACCENT) - style.map("Accent.TButton", background=[("active", "#24580d")], - foreground=[("active", TEXT)]) - - style.configure("TLabelframe", background=PANEL, foreground=ACCENT, - bordercolor=BORDER, relief="solid", borderwidth=1) - style.configure("TLabelframe.Label", background=PANEL, foreground=ACCENT, - font=(UI_FONT, 10, "bold")) - style.configure("TEntry", fieldbackground="#070b08", foreground=TEXT, - insertcolor=ACCENT, bordercolor=BORDER, padding=7) - style.configure("TSpinbox", fieldbackground="#070b08", foreground=TEXT, - insertcolor=ACCENT, bordercolor=BORDER, arrowcolor=ACCENT) - style.configure("Treeview", background="#070b08", fieldbackground="#070b08", - foreground=TEXT, rowheight=27, bordercolor=BORDER) - style.map("Treeview", background=[("selected", "#23480f")], - foreground=[("selected", TEXT)]) - style.configure("Treeview.Heading", background="#101912", foreground=TEXT, - bordercolor=BORDER, relief="flat", font=(UI_FONT, 9, "bold")) - style.map("Treeview.Heading", background=[("active", "#18271b")], - foreground=[("active", ACCENT)]) - style.configure("TNotebook", background=PANEL, bordercolor=BORDER) - style.configure("TNotebook.Tab", background="#0b120d", foreground=MUTED, padding=(12, 7)) - style.map("TNotebook.Tab", background=[("selected", "#173114"), ("active", "#132219")], - foreground=[("selected", ACCENT), ("active", TEXT)]) - style.configure("Vertical.TScrollbar", background="#111a13", troughcolor="#050806", - arrowcolor=ACCENT, bordercolor=BORDER) - - if self.accessibility_preferences["high_contrast"]: - contrast_bg = "#000000" - contrast_text = "#ffffff" - contrast_accent = "#b6ff00" - self.root.configure(background=contrast_bg) - style.configure(".", background=contrast_bg, foreground=contrast_text, - fieldbackground=contrast_bg, bordercolor=contrast_text, - troughcolor=contrast_bg, selectbackground=contrast_accent, - selectforeground=contrast_bg) - for name in ("TFrame", "Content.TFrame", "TLabel", "TLabelframe", - "TLabelframe.Label", "TNotebook"): - style.configure(name, background=contrast_bg, foreground=contrast_text) - style.configure("Sidebar.TFrame", background=contrast_bg) - style.configure("Brand.TLabel", background=contrast_bg, foreground=contrast_text) - style.configure("AccentBrand.TLabel", background=contrast_bg, - foreground=contrast_accent) - style.configure("TButton", background=contrast_bg, foreground=contrast_text, - bordercolor=contrast_text) - style.configure("Nav.TButton", background=contrast_bg, foreground=contrast_text, - bordercolor=contrast_text) - style.configure("Treeview", background=contrast_bg, fieldbackground=contrast_bg, - foreground=contrast_text, bordercolor=contrast_text) - style.configure("Treeview.Heading", background=contrast_bg, - foreground=contrast_accent, bordercolor=contrast_text) - style.configure("TNotebook.Tab", background=contrast_bg, foreground=contrast_text) + def _build_menubar(self) -> None: + menu_options = { + "background": PALETTE.chrome, + "foreground": PALETTE.text, + "activebackground": PALETTE.accent, + "activeforeground": "#FFFFFF", + "borderwidth": 0, + "font": (UI_FONT, 9), + } + bar = ttk.Frame(self.root, style="Toolbar.TFrame", padding=(5, 1)) + bar.pack(side=tk.TOP, fill=tk.X) + + file_menu = tk.Menu(self.root, tearoff=False, **menu_options) + file_menu.add_command(label="Add Games...", command=self.show_add_games) + file_menu.add_command(label="Open Application Data", command=lambda: _open_path(BASE_DIR)) + file_menu.add_separator() + file_menu.add_command(label="Exit", command=self.root.destroy) + view_menu = tk.Menu(self.root, tearoff=False, **menu_options) + view_menu.add_command(label="Library", command=self.show_library) + view_menu.add_command(label="Downloads", command=self.show_downloads) + view_menu.add_command(label="Profiles & Saves", command=self.show_profiles) + view_menu.add_command(label="Knowledge", command=self.show_knowledge) + view_menu.add_command(label="Community Hub", command=self.show_community_hub) + tools_menu = tk.Menu(self.root, tearoff=False, **menu_options) + tools_menu.add_command(label="Backup Manager", command=self.show_backups) + tools_menu.add_command(label="External Tools", command=self.show_external_tools) + tools_menu.add_command(label="Archive Health", command=self.show_health) + tools_menu.add_command(label="Settings", command=self.show_settings) + help_menu = tk.Menu(self.root, tearoff=False, **menu_options) + help_menu.add_command(label="Help & About", command=self.show_about) + help_menu.add_command(label="Check for Updates", command=self._check_updates) + + for label, submenu in ( + ("File", file_menu), + ("View", view_menu), + ("Tools", tools_menu), + ("Help", help_menu), + ): + button = tk.Menubutton( + bar, + text=label, + menu=submenu, + background=PALETTE.chrome, + foreground=PALETTE.text, + activebackground=PALETTE.accent, + activeforeground="#FFFFFF", + borderwidth=0, + relief=tk.FLAT, + font=(UI_FONT, 9), + padx=7, + pady=3, + ) + button.pack(side=tk.LEFT) + self._menu = bar def _build_shell(self) -> None: self._wallpaper_source: Image.Image | None = None @@ -481,8 +455,8 @@ def _build_shell(self) -> None: self.shell.pack(fill=tk.BOTH, expand=True) self._wallpaper_item = self.shell.create_image(0, 0, anchor=tk.NW) - nav = ttk.Frame(self.shell, padding=(14, 18), style="Sidebar.TFrame") - self.content = ttk.Frame(self.shell, padding=18, style="Content.TFrame") + nav = ttk.Frame(self.shell, padding=(8, 10), style="Sidebar.TFrame") + self.content = ttk.Frame(self.shell, padding=12, style="Content.TFrame") self._nav_window = self.shell.create_window(0, 0, anchor=tk.NW, window=nav) self._content_window = self.shell.create_window(0, 0, anchor=tk.NW, window=self.content) @@ -492,18 +466,18 @@ def _build_shell(self) -> None: ) pages = ( - ("LIBRARY", self.show_library), - ("ADD GAMES", self.show_add_games), - ("DOWNLOADS", self.show_downloads), - ("BACKUP MANAGER", self.show_backups), - ("PROFILES & SAVES", self.show_profiles), - ("EXTERNAL TOOLS", self.show_external_tools), - ("COLLECTIONS", self.show_collections), - ("KNOWLEDGE", self.show_knowledge), - ("COMMUNITY HUB", self.show_community_hub), - ("ARCHIVE HEALTH", self.show_health), - ("SETTINGS", self.show_settings), - ("HELP & ABOUT", self.show_about), + (t("nav_library"), self.show_library), + (t("nav_add_games"), self.show_add_games), + (t("nav_downloads"), self.show_downloads), + (t("nav_backups"), self.show_backups), + (t("nav_profiles"), self.show_profiles), + (t("nav_tools"), self.show_external_tools), + (t("nav_collections"), self.show_collections), + (t("nav_knowledge"), self.show_knowledge), + (t("nav_community"), self.show_community_hub), + (t("nav_health"), self.show_health), + (t("nav_settings"), self.show_settings), + (t("nav_about"), self.show_about), ) for index, (label, callback) in enumerate(pages, start=1): shortcut = navigation_shortcut(index) @@ -511,15 +485,12 @@ def _build_shell(self) -> None: if shortcut and self.accessibility_preferences["keyboard_hints"]: display_label = f"{label} Alt+{shortcut}" ttk.Button(nav, text=display_label, command=callback, style="Nav.TButton", width=22).pack( - fill=tk.X, pady=3 + fill=tk.X, pady=1 ) - ttk.Label(nav, text="CONNECTED", style="AccentBrand.TLabel").pack( - side=tk.BOTTOM, anchor=tk.W, padx=6, pady=(4, 0) - ) - ttk.Label(nav, text="Ready", style="AccentBrand.TLabel").pack( - side=tk.BOTTOM, anchor=tk.W, padx=6 - ) + status = ttk.Frame(nav, style="Statusbar.TFrame", padding=(7, 4)) + status.pack(side=tk.BOTTOM, fill=tk.X) + ttk.Label(status, text="Ready", style="Statusbar.TLabel").pack(anchor=tk.W) self.content.columnconfigure(0, weight=1) self.content.rowconfigure(1, weight=1) @@ -539,15 +510,15 @@ def _build_shell(self) -> None: def _resize_shell(self, _event: tk.Event[Any] | None) -> None: width = max(self.shell.winfo_width(), 980) height = max(self.shell.winfo_height(), 640) - nav_width = 230 - gap = 12 + nav_width = 218 + gap = 1 if self._wallpaper_source is not None: image = ImageOps.fit( self._wallpaper_source, (width, height), method=Image.Resampling.LANCZOS, centering=(0.5, 0.5) ) - image = Image.blend(image, Image.new("RGB", image.size, BG), 0.30) + image = Image.blend(image, Image.new("RGB", image.size, BG), 0.72) self._wallpaper_photo = ImageTk.PhotoImage(image) self.shell.itemconfigure(self._wallpaper_item, image=self._wallpaper_photo) @@ -606,7 +577,7 @@ def _page_header(self, title: str, subtitle: str) -> None: ttk.Label(header, text=subtitle, style="Subheader.TLabel").pack( anchor=tk.W, pady=(5, 0) ) - tk.Frame(header, height=2, background=ACCENT).pack(fill=tk.X, pady=(13, 0)) + tk.Frame(header, height=1, background=BORDER).pack(fill=tk.X, pady=(9, 0)) def show_library(self) -> None: self._clear_content() @@ -890,7 +861,7 @@ def show_add_games(self) -> None: text="Selected TitleIDs:", ).grid(row=3, column=0, sticky=tk.W) - self.add_titleids_text = tk.Text(panel, height=8, wrap=tk.WORD, background="#070b08", foreground=TEXT, insertbackground=ACCENT, selectbackground="#315f12", selectforeground=TEXT, relief=tk.FLAT, highlightthickness=1, highlightbackground=BORDER, highlightcolor=ACCENT) + self.add_titleids_text = tk.Text(panel, height=8, wrap=tk.WORD, background=PALETTE.field, foreground=TEXT, insertbackground=ACCENT, selectbackground=PALETTE.selection, selectforeground=TEXT, relief=tk.FLAT, highlightthickness=1, highlightbackground=BORDER, highlightcolor=ACCENT) self.add_titleids_text.grid(row=4, column=0, sticky="ew", pady=8) actions = ttk.Frame(panel) @@ -1152,7 +1123,7 @@ def show_health(self) -> None: command=self._run_health_scan, ).grid(row=0, column=0, sticky=tk.W, pady=(0, 10)) - self.health_text = tk.Text(panel, wrap=tk.WORD, background="#070b08", foreground=TEXT, insertbackground=ACCENT, selectbackground="#315f12", selectforeground=TEXT, relief=tk.FLAT, highlightthickness=1, highlightbackground=BORDER, highlightcolor=ACCENT) + self.health_text = tk.Text(panel, wrap=tk.WORD, background=PALETTE.field, foreground=TEXT, insertbackground=ACCENT, selectbackground=PALETTE.selection, selectforeground=TEXT, relief=tk.FLAT, highlightthickness=1, highlightbackground=BORDER, highlightcolor=ACCENT) self.health_text.grid(row=1, column=0, sticky="nsew") self.health_text.insert( tk.END, @@ -1215,6 +1186,7 @@ def show_settings(self) -> None: self.timeout_var = tk.IntVar(value=int(config.get("timeout", 30))) self.retries_var = tk.IntVar(value=int(config.get("max_retries", 3))) self.scale_var = tk.DoubleVar(value=float(config.get("ui_scale", 1.0))) + self.language_var = tk.StringVar(value=str(config.get("language", "en"))) ttk.Label(panel, text="Archive folder").grid(row=0, column=0, sticky=tk.W) ttk.Entry(panel, textvariable=self.output_var).grid( @@ -1250,11 +1222,17 @@ def show_settings(self) -> None: width=12, ).grid(row=offset, column=1, sticky=tk.W, padx=8) + ttk.Label(panel, text="Language").grid(row=8, column=0, sticky=tk.W, pady=5) + ttk.Combobox( + panel, textvariable=self.language_var, state="readonly", + values=tuple(SUPPORTED_LANGUAGES), width=12, + ).grid(row=8, column=1, sticky=tk.W, padx=8) + ttk.Button( panel, text="Save Settings", command=self._save_settings, - ).grid(row=8, column=2, sticky=tk.E, pady=(16, 0)) + ).grid(row=9, column=2, sticky=tk.E, pady=(16, 0)) def _browse_output(self) -> None: selected = filedialog.askdirectory( @@ -1281,12 +1259,15 @@ def _save_settings(self) -> None: "timeout": self.timeout_var.get(), "max_retries": self.retries_var.get(), "ui_scale": self.scale_var.get(), + "language": self.language_var.get(), } ) CONFIG_PATH.write_text(json.dumps(config, indent=2), encoding="utf-8") self.root.tk.call("tk", "scaling", max(0.8, min(2.0, self.scale_var.get()))) messagebox.showinfo( - "Settings", "Settings saved. Interface scaling applies immediately.", parent=self.root + "Settings", + "Settings saved. Interface scaling applies immediately; language changes apply after restart.", + parent=self.root, ) @staticmethod diff --git a/packaging/linux/io.github.trapemall.UnityScraper.metainfo.xml b/packaging/linux/io.github.trapemall.UnityScraper.metainfo.xml index 5ef4994..298d447 100644 --- a/packaging/linux/io.github.trapemall.UnityScraper.metainfo.xml +++ b/packaging/linux/io.github.trapemall.UnityScraper.metainfo.xml @@ -22,6 +22,6 @@ https://github.com/TrapEmAll/UnityScraper/issues - + diff --git a/packaging/macos/Info.plist b/packaging/macos/Info.plist index e08c366..e9ec825 100644 --- a/packaging/macos/Info.plist +++ b/packaging/macos/Info.plist @@ -8,8 +8,8 @@ CFBundleInfoDictionaryVersion6.0 CFBundleNameUnityScraper CFBundlePackageTypeAPPL - CFBundleShortVersionString1.1.0 - CFBundleVersion2 + CFBundleShortVersionString1.2.0 + CFBundleVersion3 LSMinimumSystemVersion11.0 NSHighResolutionCapable diff --git a/plugin_worker.py b/plugin_worker.py new file mode 100644 index 0000000..f7b4b7d --- /dev/null +++ b/plugin_worker.py @@ -0,0 +1,80 @@ +"""Out-of-process metadata plugin worker used by PluginManager.""" + +from __future__ import annotations + +import importlib.util +import json +import os +import sys +from pathlib import Path + +from plugins import MetadataCollectorPlugin + + +MAX_RESULT_BYTES = 2 * 1024 * 1024 + + +def _limits() -> None: + if os.name == "nt": + return + try: + import resource + + resource.setrlimit(resource.RLIMIT_CPU, (20, 20)) + resource.setrlimit(resource.RLIMIT_AS, (256 * 1024 * 1024, 256 * 1024 * 1024)) + resource.setrlimit(resource.RLIMIT_FSIZE, (4 * 1024 * 1024, 4 * 1024 * 1024)) + except (ImportError, OSError, ValueError): + pass + + +def run(entrypoint: Path, titleid: str, output_path: Path) -> None: + _limits() + spec = importlib.util.spec_from_file_location("unityscraper_isolated_plugin", entrypoint) + if spec is None or spec.loader is None: + raise RuntimeError("Plugin entrypoint could not be loaded") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + plugin_type = next( + ( + value for value in vars(module).values() + if isinstance(value, type) + and issubclass(value, MetadataCollectorPlugin) + and value is not MetadataCollectorPlugin + ), + None, + ) + if plugin_type is None: + raise RuntimeError("Plugin exports no MetadataCollectorPlugin subclass") + plugin = plugin_type() + if not plugin.validate_titleid(titleid): + payload = {"status": "skipped"} + else: + result = plugin.collect(titleid) + if not isinstance(result, dict): + raise TypeError("Plugin collect() must return a dictionary") + payload = {"status": "completed", "data": result} + encoded = json.dumps(payload, default=str).encode("utf-8") + if len(encoded) > MAX_RESULT_BYTES: + raise ValueError("Plugin result exceeds the 2 MiB safety limit") + temporary = output_path.with_suffix(".partial") + temporary.write_bytes(encoded) + temporary.replace(output_path) + + +def main(argv: list[str] | None = None) -> int: + arguments = list(sys.argv[1:] if argv is None else argv) + if len(arguments) != 3: + return 2 + entrypoint, titleid, output = arguments + try: + run(Path(entrypoint).resolve(), titleid, Path(output).resolve()) + except Exception as exc: + encoded = json.dumps({"status": "failed", "error": str(exc)}).encode("utf-8") + if len(encoded) <= MAX_RESULT_BYTES: + Path(output).write_bytes(encoded) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/plugins.py b/plugins.py index 68734f0..1f9f1f3 100644 --- a/plugins.py +++ b/plugins.py @@ -15,9 +15,13 @@ import importlib.util import sys import threading +import subprocess +import tempfile logger = logging.getLogger(__name__) PLUGIN_API_VERSION = 1 +PLUGIN_RESULT_LIMIT = 2 * 1024 * 1024 +PLUGIN_TIMEOUT_SECONDS = 30 @dataclass(frozen=True) @@ -126,15 +130,18 @@ def __init__( enabled_plugins: Optional[List[str]] = None, trusted_hashes: Optional[Mapping[str, str]] = None, allow_legacy: bool = False, + isolated: bool = True, ): self.plugin_dir = Path(plugin_dir) self.plugins: Dict[str, MetadataCollectorPlugin] = {} self.manifests: Dict[str, PluginManifest] = {} self.plugin_ids_by_name: Dict[str, str] = {} self.plugin_locks: Dict[str, threading.Lock] = {} + self.isolated_entries: Dict[str, tuple[PluginManifest, Path]] = {} self.enabled_plugins = set(enabled_plugins or []) self.trusted_hashes = dict(trusted_hashes or {}) self.allow_legacy = allow_legacy + self.isolated = isolated self._load_plugins() def _load_plugins(self): @@ -152,7 +159,12 @@ def _load_plugins(self): expected = self.trusted_hashes.get(manifest.plugin_id) if not expected or file_sha256(entry).casefold() != expected.casefold(): raise ValueError("Plugin entrypoint does not match its approved checksum") - self._load_plugin_file(entry, manifest) + if self.isolated: + self.isolated_entries[manifest.name] = (manifest, entry.resolve()) + self.plugin_ids_by_name[manifest.name] = manifest.plugin_id + self.plugin_locks[manifest.name] = threading.Lock() + else: + self._load_plugin_file(entry, manifest) except Exception as e: logger.warning(f"Failed to load plugin {manifest_path}: {e}") if self.allow_legacy: @@ -209,6 +221,7 @@ def list_available_plugins(self) -> List[Dict[str, Any]]: "permissions": list(manifest.permissions), "enabled": manifest.plugin_id in self.enabled_plugins, "loaded": manifest.name in self.plugins, + "isolated": manifest.name in self.isolated_entries, } for manifest in self.manifests.values() ] @@ -233,19 +246,28 @@ def collect_from_plugin(self, plugin_name: str, titleid: str) -> Optional[Dict[s def collect_enabled(self, titleid: str) -> List[Dict[str, Any]]: """Run enabled collectors independently and retain success/failure details.""" results: List[Dict[str, Any]] = [] - for name, plugin in self.plugins.items(): + names = list(dict.fromkeys([*self.plugins, *self.isolated_entries])) + for name in names: plugin_id = self.plugin_ids_by_name.get(name, name) try: with self.plugin_locks[name]: - if not plugin.validate_titleid(titleid): - results.append({"plugin_id": plugin_id, "name": name, - "status": "skipped"}) - continue - data = plugin.collect(titleid) + if name in self.isolated_entries: + isolated = self._collect_isolated(name, titleid) + if isolated["status"] != "completed": + results.append({"plugin_id": plugin_id, "name": name, **isolated}) + continue + data = isolated["data"] + else: + plugin = self.plugins[name] + if not plugin.validate_titleid(titleid): + results.append({"plugin_id": plugin_id, "name": name, + "status": "skipped"}) + continue + data = plugin.collect(titleid) if not isinstance(data, dict): raise TypeError("Plugin collect() must return a dictionary") encoded = json.dumps(data, default=str) - if len(encoded.encode("utf-8")) > 2 * 1024 * 1024: + if len(encoded.encode("utf-8")) > PLUGIN_RESULT_LIMIT: raise ValueError("Plugin result exceeds the 2 MiB safety limit") except Exception as exc: logger.exception("Plugin %s failed for %s", plugin_id, titleid) @@ -256,6 +278,50 @@ def collect_enabled(self, titleid: str) -> List[Dict[str, Any]]: "status": "completed", "data": data}) return results + def _collect_isolated(self, name: str, titleid: str) -> Dict[str, Any]: + manifest, entry = self.isolated_entries[name] + worker = Path(__file__).resolve().with_name("plugin_worker.py") + if getattr(sys, "frozen", False): + command = [ + sys.executable, "--plugin-worker", str(entry), titleid, + ] + else: + command = [ + sys.executable, str(worker), str(entry), titleid, + ] + with tempfile.TemporaryDirectory(prefix="unityscraper-plugin-") as temp: + output = Path(temp) / "result.json" + command.append(str(output)) + try: + subprocess.run( + command, + cwd=str(entry.parent), + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=PLUGIN_TIMEOUT_SECONDS, + check=False, + ) + except subprocess.TimeoutExpired: + return {"status": "failed", "error": "Plugin execution timed out"} + if not output.is_file(): + return {"status": "failed", "error": "Plugin worker returned no result"} + if output.stat().st_size > PLUGIN_RESULT_LIMIT: + return {"status": "failed", "error": "Plugin result exceeds safety limit"} + try: + payload = json.loads(output.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + return {"status": "failed", "error": f"Invalid plugin result: {exc}"} + if not isinstance(payload, dict) or payload.get("status") not in { + "completed", "skipped", "failed" + }: + return {"status": "failed", "error": "Plugin returned an invalid status"} + if payload.get("status") == "failed": + payload["error"] = str(payload.get("error", "Plugin failed"))[:1000] + payload["worker"] = "isolated-process" + payload["plugin_id"] = manifest.plugin_id + return payload + # Example plugin template (to be saved in plugins/example.py) EXAMPLE_PLUGIN_TEMPLATE = ''' diff --git a/profile_gui.py b/profile_gui.py index 5233f19..35c722e 100644 --- a/profile_gui.py +++ b/profile_gui.py @@ -14,7 +14,13 @@ from platform_support import open_path from profile_intelligence import ProfileIntelligenceService from profile_manager import ProfileSaveManager, mask_identifier -from xenia_bridge import MigrationPlan, candidate_xenia_content_roots +from xenia_bridge import ( + MigrationPlan, + candidate_xenia_content_roots, + find_xenia_installation, + launch_xenia, +) +from ui_theme import PALETTE def _size(value: int) -> str: @@ -345,7 +351,7 @@ def _build_achievements(self, parent: ttk.Frame) -> None: def _build_compare(self, parent: ttk.Frame) -> None: parent.columnconfigure(1, weight=1) - parent.rowconfigure(3, weight=1) + parent.rowconfigure(5, weight=1) self.compare_left_var = tk.StringVar() self.compare_right_var = tk.StringVar() ttk.Label(parent, text="First profile").grid(row=0, column=0, sticky=tk.W) @@ -371,9 +377,9 @@ def _build_compare(self, parent: ttk.Frame) -> None: self.compare_text = tk.Text( parent, wrap=tk.WORD, - background="#070b08", - foreground="#f2f5f2", - insertbackground="#72e000", + background=PALETTE.field, + foreground=PALETTE.text, + insertbackground=PALETTE.accent_hot, relief=tk.FLAT, padx=12, pady=10, @@ -458,6 +464,8 @@ def _build_xenia(self, parent: ttk.Frame) -> None: candidates = [str(path) for path in candidate_xenia_content_roots()] self.xenia_root_var = tk.StringVar(value=candidates[0] if candidates else "") self.xenia_target_var = tk.StringVar() + self.xenia_game_var = tk.StringVar() + self.xenia_fullscreen_var = tk.BooleanVar(value=False) ttk.Label(parent, text="Xenia folder or content root").grid( row=0, column=0, sticky=tk.W ) @@ -473,8 +481,17 @@ def _build_xenia(self, parent: ttk.Frame) -> None: ttk.Entry(parent, textvariable=self.xenia_target_var).grid( row=1, column=1, sticky="ew", padx=8, pady=(8, 0) ) + ttk.Label(parent, text="Game image, folder, or default.xex").grid( + row=2, column=0, sticky=tk.W, pady=(8, 0) + ) + ttk.Entry(parent, textvariable=self.xenia_game_var).grid( + row=2, column=1, sticky="ew", padx=8, pady=(8, 0) + ) + ttk.Button(parent, text="Browse", command=self.choose_xenia_game).grid( + row=2, column=2, pady=(8, 0) + ) controls = ttk.Frame(parent) - controls.grid(row=2, column=0, columnspan=3, sticky="ew", pady=10) + controls.grid(row=3, column=0, columnspan=3, sticky="ew", pady=10) ttk.Button( controls, text="Preview Migration", @@ -488,6 +505,12 @@ def _build_xenia(self, parent: ttk.Frame) -> None: state=tk.DISABLED, ) self.xenia_execute_button.pack(side=tk.LEFT, padx=(8, 0)) + ttk.Checkbutton( + controls, text="Fullscreen", variable=self.xenia_fullscreen_var + ).pack(side=tk.LEFT, padx=(18, 4)) + ttk.Button(controls, text="Launch Game", command=self.launch_xenia_game).pack( + side=tk.LEFT + ) self.migration_tree = ttk.Treeview( parent, columns=("titleid", "file", "action", "reason"), @@ -501,7 +524,7 @@ def _build_xenia(self, parent: ttk.Frame) -> None: ): self.migration_tree.heading(column, text=label) self.migration_tree.column(column, width=width, anchor=tk.W) - self.migration_tree.grid(row=3, column=0, columnspan=3, sticky="nsew") + self.migration_tree.grid(row=5, column=0, columnspan=3, sticky="nsew") ttk.Label( parent, text=( @@ -510,7 +533,7 @@ def _build_xenia(self, parent: ttk.Frame) -> None: ), style="Subheader.TLabel", wraplength=900, - ).grid(row=4, column=0, columnspan=3, sticky="ew", pady=(8, 0)) + ).grid(row=6, column=0, columnspan=3, sticky="ew", pady=(8, 0)) def choose_source(self) -> None: selected = filedialog.askdirectory( @@ -610,6 +633,37 @@ def choose_xenia_root(self) -> None: if path: self.xenia_root_var.set(path) + def choose_xenia_game(self) -> None: + path = filedialog.askopenfilename( + parent=self.root, + title="Choose a game image or executable", + filetypes=(("Xbox games", "*.xex *.iso"), ("All files", "*.*")), + ) + if not path: + path = filedialog.askdirectory(parent=self.root, title="Choose an extracted game") + if path: + self.xenia_game_var.set(path) + + def launch_xenia_game(self) -> None: + installation = find_xenia_installation(self.xenia_root_var.get()) + if installation is None: + messagebox.showerror( + "Xenia", "No Xenia or Xenia Canary executable was found in that folder.", + parent=self.root, + ) + return + try: + result = launch_xenia( + installation, self.xenia_game_var.get(), + fullscreen=self.xenia_fullscreen_var.get(), + ) + except Exception as exc: + messagebox.showerror("Xenia launch failed", str(exc), parent=self.root) + return + self.status_var.set( + f"Launched {result['variant']} for {Path(str(result['game'])).name}." + ) + def preview_xenia_migration(self) -> None: profile_id = self._selected_profile_id() if not profile_id: diff --git a/pyproject.toml b/pyproject.toml index d83f111..2fb8fe2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "unityscraper" -version = "1.1.0b1" +version = "1.2.0b1" description = "Xbox 360 library, knowledge, preservation, and backup manager" readme = "README.md" requires-python = ">=3.10" diff --git a/roadmap_services.py b/roadmap_services.py new file mode 100644 index 0000000..fd2f904 --- /dev/null +++ b/roadmap_services.py @@ -0,0 +1,379 @@ +"""Release-readiness services for metadata, audits, reports, and corrections.""" + +from __future__ import annotations + +import hashlib +import html +import json +import sqlite3 +import zipfile +from collections import defaultdict +from contextlib import contextmanager +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Iterable + +from app_paths import DATABASE_PATH +from database_migrations import ensure_application_schema +from knowledge_base import EntityRecord, Fact, Identifier, KnowledgeRepository, is_unknown + + +SNAPSHOT_SCHEMA = 1 +MAX_SNAPSHOT_EXPANDED = 512 * 1024 * 1024 + + +def utc_now() -> str: + return datetime.now(timezone.utc).replace(microsecond=0).isoformat() + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +class RoadmapRepository: + def __init__(self, db_path: str | Path = DATABASE_PATH) -> None: + self.db_path = Path(db_path) + self.db_path.parent.mkdir(parents=True, exist_ok=True) + with self.connect() as connection: + KnowledgeRepository(connection).ensure_schema() + ensure_application_schema(connection) + + @contextmanager + def connect(self): + connection = sqlite3.connect(self.db_path) + connection.row_factory = sqlite3.Row + try: + yield connection + connection.commit() + except Exception: + connection.rollback() + raise + finally: + connection.close() + + +class MetadataSnapshotService(RoadmapRepository): + """Export and merge portable metadata without profile or filesystem data.""" + + def export(self, destination: str | Path) -> dict[str, Any]: + target = Path(destination).expanduser().resolve() + target.parent.mkdir(parents=True, exist_ok=True) + temporary = target.with_suffix(target.suffix + ".partial") + with self.connect() as connection: + catalog = [dict(row) for row in connection.execute( + "SELECT * FROM xboxunity_title_catalog ORDER BY titleid" + ).fetchall()] + records = self._export_records(connection) + sources = [dict(row) for row in connection.execute( + "SELECT slug, name, homepage_url, license_name, license_url, notes " + "FROM knowledge_sources ORDER BY slug" + ).fetchall()] + payload = { + "schema": SNAPSHOT_SCHEMA, + "application": "UnityScraper", + "created_at": utc_now(), + "contains_personal_data": False, + "sources": sources, + "catalog": catalog, + "records": records, + } + encoded = json.dumps(payload, ensure_ascii=True, separators=(",", ":")).encode("utf-8") + try: + with zipfile.ZipFile(temporary, "w", zipfile.ZIP_DEFLATED) as archive: + archive.writestr("metadata.json", encoded) + temporary.replace(target) + finally: + temporary.unlink(missing_ok=True) + digest = sha256_file(target) + with self.connect() as connection: + cursor = connection.execute( + """INSERT INTO metadata_snapshot_runs( + operation, snapshot_path, created_at, completed_at, catalog_count, + fact_count, status, sha256) VALUES ('export', ?, ?, ?, ?, ?, 'completed', ?)""", + (str(target), utc_now(), utc_now(), len(catalog), + sum(len(item["facts"]) for item in records), digest), + ) + return {"run_id": int(cursor.lastrowid or 0), "path": str(target), + "sha256": digest, "catalog": len(catalog), "records": len(records)} + + def import_snapshot(self, source: str | Path) -> dict[str, Any]: + path = Path(source).expanduser().resolve() + payload = self._read_snapshot(path) + sources = {item["slug"]: item for item in payload.get("sources", [])} + catalog_count = 0 + fact_count = 0 + with self.connect() as connection: + repository = KnowledgeRepository(connection) + source_ids: dict[str, int] = {} + for slug, item in sources.items(): + source_ids[slug] = repository.upsert_source( + slug, item.get("name", slug), item.get("homepage_url", ""), + item.get("license_name", ""), item.get("license_url", ""), + item.get("notes", ""), + ) + for item in payload.get("catalog", []): + if not _valid_titleid(str(item.get("titleid", ""))): + continue + columns = ( + "titleid", "name", "hb_titleid", "title_type", "link_enabled", + "covers_count", "updates_count", "media_id_count", "user_count", + "newest_content", "source_url", "raw_json", "fetched_at", + ) + connection.execute( + f"""INSERT INTO xboxunity_title_catalog({','.join(columns)}) + VALUES ({','.join('?' for _ in columns)}) + ON CONFLICT(titleid) DO UPDATE SET + name=excluded.name, title_type=excluded.title_type, + covers_count=excluded.covers_count, updates_count=excluded.updates_count, + media_id_count=excluded.media_id_count, newest_content=excluded.newest_content, + raw_json=excluded.raw_json, fetched_at=excluded.fetched_at""", + tuple(item.get(column) for column in columns), + ) + catalog_count += 1 + for item in payload.get("records", []): + source_slug = str(item.get("source", "snapshot")) + source_id = source_ids.get(source_slug) + if source_id is None: + source_id = repository.upsert_source( + source_slug, source_slug, notes="Imported metadata snapshot" + ) + source_ids[source_slug] = source_id + record = EntityRecord( + entity_type=str(item.get("entity_type", "reference")), + canonical_name=str(item.get("canonical_name", "")).strip(), + identifiers=tuple( + Identifier(str(value["kind"]), str(value["value"]), + float(value.get("confidence", 1.0))) + for value in item.get("identifiers", []) + if value.get("kind") and value.get("value") + ), + names=tuple(str(value) for value in item.get("names", []) if value), + facts=tuple( + Fact(str(value["property"]), str(value["value"]), + str(value.get("normalized_value", "")), + float(value.get("confidence", 1.0)), + str(value.get("source_url", "")), + str(value.get("source_title", ""))) + for value in item.get("facts", []) + if value.get("property") and value.get("value") + ), + ) + if record.canonical_name: + repository.upsert_entity_record(record, source_id) + fact_count += len(record.facts) + digest = sha256_file(path) + with self.connect() as connection: + cursor = connection.execute( + """INSERT INTO metadata_snapshot_runs( + operation, snapshot_path, created_at, completed_at, catalog_count, + fact_count, status, sha256) VALUES ('import', ?, ?, ?, ?, ?, 'completed', ?)""", + (str(path), utc_now(), utc_now(), catalog_count, fact_count, digest), + ) + return {"run_id": int(cursor.lastrowid or 0), "catalog": catalog_count, + "facts": fact_count, "sha256": digest} + + def _read_snapshot(self, path: Path) -> dict[str, Any]: + if not path.is_file(): + raise FileNotFoundError(path) + with zipfile.ZipFile(path) as archive: + infos = archive.infolist() + if len(infos) != 1 or infos[0].filename != "metadata.json": + raise ValueError("Metadata snapshot must contain only metadata.json") + if infos[0].file_size > MAX_SNAPSHOT_EXPANDED: + raise ValueError("Metadata snapshot exceeds the expanded-size limit") + payload = json.loads(archive.read(infos[0]).decode("utf-8")) + if not isinstance(payload, dict) or payload.get("schema") != SNAPSHOT_SCHEMA: + raise ValueError("Unsupported metadata snapshot schema") + if payload.get("contains_personal_data") is not False: + raise ValueError("Metadata snapshot does not declare a safe data boundary") + return payload + + @staticmethod + def _export_records(connection: sqlite3.Connection) -> list[dict[str, Any]]: + entities = connection.execute( + "SELECT id, entity_type, canonical_name FROM knowledge_entities ORDER BY id" + ).fetchall() + results: list[dict[str, Any]] = [] + for entity in entities: + facts = connection.execute( + """SELECT f.*, s.slug, c.source_url, c.source_title + FROM knowledge_facts f JOIN knowledge_sources s ON s.id=f.source_id + LEFT JOIN fact_citations c ON c.fact_id=f.id WHERE f.entity_id=? + ORDER BY s.slug, f.property""", (entity["id"],) + ).fetchall() + by_source: dict[str, list[sqlite3.Row]] = defaultdict(list) + for fact in facts: + by_source[fact["slug"]].append(fact) + names = [row[0] for row in connection.execute( + "SELECT name FROM entity_names WHERE entity_id=?", (entity["id"],) + ).fetchall()] + for slug, source_facts in by_source.items(): + identifiers = [dict(row) for row in connection.execute( + """SELECT identifier_type kind, identifier_value value, confidence + FROM entity_identifiers i JOIN knowledge_sources s ON s.id=i.source_id + WHERE entity_id=? AND s.slug=?""", (entity["id"], slug) + ).fetchall()] + results.append({ + "entity_type": entity["entity_type"], + "canonical_name": entity["canonical_name"], + "source": slug, + "names": names, + "identifiers": identifiers, + "facts": [{ + "property": row["property"], "value": row["value"], + "normalized_value": row["normalized_value"], + "confidence": row["confidence"], "source_url": row["source_url"] or "", + "source_title": row["source_title"] or "", + } for row in source_facts], + }) + return results + + +class LibraryIntelligenceService(RoadmapRepository): + def audit(self) -> dict[str, Any]: + issues: list[dict[str, str]] = [] + with self.connect() as connection: + rows = connection.execute( + """SELECT t.titleid, t.name, t.publisher, + COUNT(DISTINCT c.id) covers, COUNT(DISTINCT u.id) updates, + COALESCE(MAX(x.covers_count), 0) available_covers, + COALESCE(MAX(x.updates_count), 0) available_updates, + COALESCE(MAX(x.media_id_count), 0) media_ids + FROM titleids t + LEFT JOIN covers c ON c.titleid=t.titleid AND c.status='downloaded' + LEFT JOIN title_updates u ON u.titleid=t.titleid AND u.status='downloaded' + LEFT JOIN xboxunity_title_catalog x ON x.titleid=t.titleid + GROUP BY t.titleid ORDER BY t.name COLLATE NOCASE""" + ).fetchall() + for row in rows: + title = row["name"] or row["titleid"] + if is_unknown(row["name"]) or str(row["name"] or "").upper() == row["titleid"]: + issues.append(_issue(row["titleid"], title, "unknown-name", "Game name is unknown")) + if is_unknown(row["publisher"]): + issues.append(_issue(row["titleid"], title, "unknown-publisher", "Publisher is unknown")) + if row["available_covers"] and not row["covers"]: + issues.append(_issue(row["titleid"], title, "missing-cover", "Cover is available but not archived")) + if row["available_updates"] and not row["updates"]: + issues.append(_issue(row["titleid"], title, "missing-update", "Title updates are available but not archived")) + if row["available_updates"] and not row["media_ids"]: + issues.append(_issue(row["titleid"], title, "unknown-mediaid", "Update compatibility needs a MediaID")) + summary = { + "titles": len(rows), "issues": len(issues), + "unknown_names": sum(item["kind"] == "unknown-name" for item in issues), + "unknown_publishers": sum(item["kind"] == "unknown-publisher" for item in issues), + "missing_covers": sum(item["kind"] == "missing-cover" for item in issues), + "missing_updates": sum(item["kind"] == "missing-update" for item in issues), + } + cursor = connection.execute( + "INSERT INTO library_intelligence_runs(created_at,title_count,issue_count,summary_json) " + "VALUES (?,?,?,?)", (utc_now(), len(rows), len(issues), json.dumps(summary)), + ) + return {"run_id": int(cursor.lastrowid or 0), "summary": summary, "issues": issues} + + +class PreservationReportService(RoadmapRepository): + def export_html(self, destination: str | Path) -> dict[str, Any]: + target = Path(destination).expanduser().resolve() + target.parent.mkdir(parents=True, exist_ok=True) + audit = LibraryIntelligenceService(self.db_path).audit() + with self.connect() as connection: + sources = [dict(row) for row in connection.execute( + """SELECT s.name, s.license_name, COUNT(DISTINCT d.id) documents, + COUNT(DISTINCT f.id) facts + FROM knowledge_sources s LEFT JOIN source_documents d ON d.source_id=s.id + LEFT JOIN knowledge_facts f ON f.source_id=s.id GROUP BY s.id ORDER BY s.name""" + ).fetchall()] + issue_rows = "".join( + f"{html.escape(item['titleid'])}{html.escape(item['title'])}" + f"{html.escape(item['kind'])}{html.escape(item['message'])}" + for item in audit["issues"] + ) + source_rows = "".join( + f"{html.escape(row['name'])}{html.escape(row['license_name'] or 'Unspecified')}" + f"{row['documents']}{row['facts']}" for row in sources + ) + summary = audit["summary"] + target.write_text(f""" +UnityScraper Preservation Report

Xbox 360 Preservation Report

+

Generated {html.escape(utc_now())}. Personal profile identifiers and filesystem paths are excluded.

+
{summary['titles']}Titles +{summary['issues']}Items needing attention +{len(sources)}Knowledge sources
+

Library attention

+{issue_rows}
TitleIDGameTypeDetails

Source provenance

+{source_rows}
SourceLicenseDocumentsFacts
+""", encoding="utf-8") + digest = sha256_file(target) + with self.connect() as connection: + cursor = connection.execute( + "INSERT INTO preservation_report_runs(destination,created_at,report_format,status,sha256) " + "VALUES (?,?,'html','completed',?)", (str(target), utc_now(), digest), + ) + return {"run_id": int(cursor.lastrowid or 0), "path": str(target), "sha256": digest} + + +class CorrectionPackageService(RoadmapRepository): + def export(self, destination: str | Path) -> dict[str, Any]: + target = Path(destination).expanduser().resolve() + target.parent.mkdir(parents=True, exist_ok=True) + with self.connect() as connection: + rows = [dict(row) for row in connection.execute( + """SELECT entity_type, identifier_type, identifier_value, property, value, notes, + updated_at FROM metadata_overrides ORDER BY updated_at""" + ).fetchall()] + payload = {"schema": 1, "created_at": utc_now(), "contains_personal_data": False, + "corrections": rows} + target.write_text(json.dumps(payload, indent=2, ensure_ascii=True), encoding="utf-8") + digest = sha256_file(target) + with self.connect() as connection: + cursor = connection.execute( + "INSERT INTO correction_packages(operation,package_path,created_at,correction_count,status,sha256) " + "VALUES ('export',?,?,?,'completed',?)", (str(target), utc_now(), len(rows), digest), + ) + return {"run_id": int(cursor.lastrowid or 0), "path": str(target), + "corrections": len(rows), "sha256": digest} + + +class HardwareInventoryService(RoadmapRepository): + FIELDS = ("motherboard", "dvd_drive", "nand_type", "dashboard_version", "console_type") + + def save(self, label: str, **values: str) -> dict[str, Any]: + clean_label = label.strip() + if not clean_label: + raise ValueError("A hardware record label is required") + unknown = set(values) - set(self.FIELDS) - {"notes"} + if unknown: + raise ValueError(f"Unsupported hardware fields: {', '.join(sorted(unknown))}") + now = utc_now() + with self.connect() as connection: + cursor = connection.execute( + """INSERT INTO hardware_inventory_records( + label,motherboard,dvd_drive,nand_type,dashboard_version,console_type, + notes,created_at,updated_at) VALUES (?,?,?,?,?,?,?,?,?)""", + (clean_label, *(str(values.get(name, "")).strip() for name in self.FIELDS), + str(values.get("notes", "")).strip(), now, now), + ) + return {"id": int(cursor.lastrowid or 0), "label": clean_label} + + def list(self) -> list[dict[str, Any]]: + with self.connect() as connection: + return [dict(row) for row in connection.execute( + "SELECT * FROM hardware_inventory_records ORDER BY updated_at DESC" + ).fetchall()] + + +def _valid_titleid(value: str) -> bool: + return len(value) == 8 and all(character in "0123456789ABCDEFabcdef" for character in value) + + +def _issue(titleid: str, title: str, kind: str, message: str) -> dict[str, str]: + return {"titleid": titleid, "title": title, "kind": kind, "message": message} diff --git a/setup_wizard.py b/setup_wizard.py index f694679..b79526c 100644 --- a/setup_wizard.py +++ b/setup_wizard.py @@ -16,6 +16,7 @@ ensure_user_titleids_file, ) from platform_support import desktop_font_family +from knowledge_scheduler import KnowledgeScheduler UI_FONT = desktop_font_family() @@ -29,7 +30,7 @@ def __init__(self, parent: tk.Misc) -> None: self.parent = parent self.completed = False self.title("Welcome to UnityScraper") - self.geometry("620x500") + self.geometry("650x610") self.resizable(False, False) self.transient(parent) self.grab_set() @@ -40,6 +41,9 @@ def __init__(self, parent: tk.Misc) -> None: self.output_var = tk.StringVar(value=str(DOWNLOADS_DIR)) self.titleids_var = tk.StringVar() self.collection_var = tk.StringVar() + self.catalog_var = tk.BooleanVar(value=True) + self.knowledge_var = tk.BooleanVar(value=False) + self.refresh_days_var = tk.IntVar(value=7) self._build() @@ -69,7 +73,7 @@ def _build(self) -> None: ttk.Entry(folder_row, textvariable=self.output_var).pack( side=tk.LEFT, fill=tk.X, expand=True ) - ttk.Button(folder_row, text="Browse…", command=self._browse).pack( + ttk.Button(folder_row, text="Browse...", command=self._browse).pack( side=tk.LEFT, padx=(8, 0) ) @@ -92,6 +96,24 @@ def _build(self) -> None: side=tk.LEFT, padx=(8, 0) ) + sources = ttk.LabelFrame(container, text="Local metadata", padding=10) + sources.pack(fill=tk.X, pady=(12, 0)) + ttk.Checkbutton( + sources, + text="Pre-cache the XboxUnity title catalog for offline autocomplete", + variable=self.catalog_var, + ).grid(row=0, column=0, columnspan=3, sticky="w") + ttk.Checkbutton( + sources, + text="Refresh ConsoleMods, XenonLibrary, and Free60 knowledge automatically", + variable=self.knowledge_var, + ).grid(row=1, column=0, columnspan=3, sticky="w", pady=(6, 0)) + ttk.Label(sources, text="Every").grid(row=2, column=0, sticky="w", pady=(6, 0)) + ttk.Spinbox( + sources, from_=1, to=365, width=6, textvariable=self.refresh_days_var + ).grid(row=2, column=1, sticky="w", padx=5, pady=(6, 0)) + ttk.Label(sources, text="days").grid(row=2, column=2, sticky="w", pady=(6, 0)) + ttk.Separator(container).pack(fill=tk.X, pady=22) ttk.Label( @@ -159,6 +181,8 @@ def _finish(self) -> None: else config.get("collection_roots", []) ), "ui_scale": float(config.get("ui_scale", 1.0)), + "sync_title_catalog_on_start": self.catalog_var.get(), + "language": str(config.get("language", "en")), } ) CONFIG_PATH.write_text(json.dumps(config, indent=2), encoding="utf-8") @@ -171,6 +195,10 @@ def _finish(self) -> None: if titleids: TITLEIDS_PATH.write_text(",".join(dict.fromkeys(titleids)), encoding="utf-8") + KnowledgeScheduler().configure( + self.knowledge_var.get(), max(1, self.refresh_days_var.get()) * 24 + ) + FIRST_RUN_PATH.write_text("complete\n", encoding="utf-8") self.completed = True self.destroy() diff --git a/tests.py b/tests.py index 2eafcdf..305a9e5 100644 --- a/tests.py +++ b/tests.py @@ -48,6 +48,7 @@ UnsafeArchiveError, atomic_copy, import_stfs_zip, + extract_stfs_files, inspect_stfs, inspect_xbe, list_stfs_entries, @@ -71,6 +72,20 @@ def test_legacy_translations_are_repaired_at_load_time(self): self.assertEqual(TRANSLATIONS["es"]["settings"], "Configuraci\u00f3n") self.assertEqual(TRANSLATIONS["ja"]["browse"], "\u53c2\u7167") + def test_bounded_language_pack_extends_navigation_with_english_fallback(self): + from i18n import init_translator + + root = Path(tempfile.mkdtemp()) + try: + (root / "nl.json").write_text(json.dumps({ + "language": "nl", "strings": {"nav_library": "BIBLIOTHEEK"} + }), encoding="utf-8") + translator = init_translator("nl", root) + self.assertEqual(translator.get("nav_library"), "BIBLIOTHEEK") + self.assertEqual(translator.get("nav_settings"), "SETTINGS") + finally: + shutil.rmtree(root) + def test_linux_uses_xdg_directories(self): home = Path("/home/tester") paths = resolve_storage_paths( @@ -1125,7 +1140,7 @@ def test_inspect_stfs_reads_profile_ownership_fields(self): self.assertEqual(package.save_game_id, "12345678") def test_stfs_file_table_is_inventoried_read_only(self): - payload = bytearray(0xC000) + payload = bytearray(0xD000) payload[:4] = b"LIVE" payload[0x340:0x344] = (0xA000).to_bytes(4, "big") payload[0x344:0x348] = (1).to_bytes(4, "big") @@ -1149,6 +1164,15 @@ def test_stfs_file_table_is_inventoried_read_only(self): self.assertEqual(entries[0].size, 123) self.assertTrue(entries[0].consecutive) + payload[0x379 + 0x1C:0x379 + 0x20] = (2).to_bytes(4, "big") + payload[0xC000:0xC004] = b"data" + package_path.write_bytes(payload) + destination = self.temp_dir / "extracted" + result = extract_stfs_files(package_path, destination) + self.assertEqual((destination / "savegame.dat").read_bytes()[:4], b"data") + self.assertEqual(result["extracted"][0]["size"], 123) + self.assertTrue(Path(result["manifest"]).is_file()) + def test_rejects_unknown_stfs_content_type(self): with self.assertRaises(InvalidPackageError): inspect_stfs(self._stfs(content_type=0xDEADBEEF)) @@ -1357,6 +1381,27 @@ def test_token_protects_non_health_routes(self): ) self.assertEqual(response.status_code, 200) + def test_scoped_tokens_and_request_limits_are_enforced(self): + api = UnityScraperAPI( + self.scraper, + token_scopes={"reader": ["read"], "writer": ["read", "write"]}, + requests_per_minute=10, + ) + client = api.app.test_client() + self.assertEqual(client.get( + "/api/titleids", headers={"X-API-Key": "reader"} + ).status_code, 200) + self.assertEqual(client.post( + "/api/config", json={"workers": 5}, headers={"X-API-Key": "reader"} + ).status_code, 403) + self.assertEqual(client.post( + "/api/config", json={"workers": 5}, headers={"X-API-Key": "writer"} + ).status_code, 200) + limited = UnityScraperAPI(self.scraper, requests_per_minute=10).app.test_client() + for _ in range(10): + self.assertEqual(limited.get("/api/health").status_code, 200) + self.assertEqual(limited.get("/api/health").status_code, 429) + def test_config_rejects_unknown_and_https_keys(self): client = UnityScraperAPI(self.scraper).app.test_client() response = client.post("/api/config", json={"use_https": True}) @@ -1578,6 +1623,25 @@ def test_xenia_plan_copies_then_skips_identical_save(self): ) self.assertEqual(second.items[0].action, "skip") + @patch("xenia_bridge.subprocess.Popen") + def test_xenia_installation_is_discovered_and_launched_without_a_shell(self, popen): + from xenia_bridge import find_xenia_installation, launch_xenia + + root = self.temp_dir / "xenia" + root.mkdir() + executable = root / ("xenia.exe" if os.name == "nt" else "xenia") + executable.write_bytes(b"binary") + game = root / "game.iso" + game.write_bytes(b"image") + installation = find_xenia_installation(root) + self.assertIsNotNone(installation) + popen.return_value.pid = 42 + result = launch_xenia(installation, game, fullscreen=True) + self.assertEqual(result["pid"], 42) + popen.assert_called_once_with( + [str(executable), str(game), "--fullscreen=true"], cwd=root + ) + def test_knowledge_priority_and_conflict_resolution_are_persistent(self): import sqlite3 from contextlib import closing @@ -1686,7 +1750,7 @@ def test_versioned_migrations_create_all_foundation_tables(self): versions = connection.execute( "SELECT version FROM app_schema_migrations ORDER BY version" ).fetchall() - self.assertEqual([row[0] for row in versions], [1, 2, 3, 4, 5, 6, 7, 8, 9]) + self.assertEqual([row[0] for row in versions], [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) self.assertIn("collection_snapshots", tables) self.assertIn("preservation_matches", tables) self.assertIn("console_transfer_jobs", tables) @@ -1703,6 +1767,84 @@ def test_versioned_migrations_create_all_foundation_tables(self): self.assertIn("scheduled_sync_state", tables) self.assertIn("plugin_collection_runs", tables) self.assertIn("dedup_recovery_records", tables) + self.assertIn("metadata_snapshot_runs", tables) + self.assertIn("library_intelligence_runs", tables) + self.assertIn("preservation_report_runs", tables) + self.assertIn("hardware_inventory_records", tables) + + def test_release_readiness_toolkit_exports_portable_nonpersonal_metadata(self): + import sqlite3 + from contextlib import closing + + from knowledge_base import EntityRecord, Fact, Identifier, KnowledgeRepository + from roadmap_services import ( + CorrectionPackageService, + HardwareInventoryService, + LibraryIntelligenceService, + MetadataSnapshotService, + PreservationReportService, + ) + + self.database.add_titleid("53510804", "Hitman: Absolution", "Unknown") + with closing(sqlite3.connect(self.db_path)) as connection: + repository = KnowledgeRepository(connection) + source_id = repository.upsert_source( + "test-source", "Test Source", license_name="CC0" + ) + repository.upsert_entity_record(EntityRecord( + "game", "Hitman: Absolution", + identifiers=(Identifier("titleid", "53510804"),), + facts=(Fact("publisher", "Square Enix"),), + ), source_id) + connection.execute( + """INSERT INTO xboxunity_title_catalog( + titleid,name,link_enabled,covers_count,updates_count,media_id_count, + user_count,source_url,raw_json,fetched_at) + VALUES ('53510804','Hitman: Absolution',1,2,3,1,1, + 'http://xboxunity.net','{}','now')""" + ) + connection.execute( + """INSERT INTO metadata_overrides( + entity_type,identifier_type,identifier_value,property,value,notes,updated_at) + VALUES ('game','titleid','53510804','publisher','Square Enix','reviewed','now')""" + ) + connection.commit() + + audit = LibraryIntelligenceService(self.db_path).audit() + self.assertEqual(audit["summary"]["titles"], 1) + self.assertTrue(any(row["kind"] == "missing-cover" for row in audit["issues"])) + report = PreservationReportService(self.db_path).export_html( + Path(self.temp_dir) / "report.html" + ) + self.assertTrue(Path(report["path"]).is_file()) + corrections = CorrectionPackageService(self.db_path).export( + Path(self.temp_dir) / "corrections.json" + ) + self.assertEqual(corrections["corrections"], 1) + hardware = HardwareInventoryService(self.db_path) + hardware.save("Living room", motherboard="Trinity", dvd_drive="DG-16D4S") + self.assertEqual(hardware.list()[0]["motherboard"], "Trinity") + + snapshot_path = Path(self.temp_dir) / "metadata.usmeta" + exported = MetadataSnapshotService(self.db_path).export(snapshot_path) + self.assertEqual(exported["catalog"], 1) + imported_db = Path(self.temp_dir) / "imported.db" + DatabaseManager(str(imported_db)) + imported = MetadataSnapshotService(imported_db).import_snapshot(snapshot_path) + self.assertEqual(imported["catalog"], 1) + self.assertEqual(imported["facts"], 1) + + api_scraper = Mock(db=Mock(db_path=self.db_path)) + api_client = UnityScraperAPI(api_scraper).app.test_client() + self.assertEqual(api_client.get("/api/library/audit").status_code, 200) + self.assertEqual(api_client.post("/api/hardware", json={ + "label": "Bench console", "motherboard": "Jasper" + }).status_code, 200) + api_report = Path(self.temp_dir) / "api-report.html" + self.assertEqual(api_client.post("/api/reports/preservation", json={ + "destination": str(api_report) + }).status_code, 200) + self.assertTrue(api_report.is_file()) def test_xex_execution_info_is_parsed(self): from backup_manager import inspect_xex diff --git a/ui_theme.py b/ui_theme.py new file mode 100644 index 0000000..8c4b3f6 --- /dev/null +++ b/ui_theme.py @@ -0,0 +1,207 @@ +"""Shared Visual Studio 2010-inspired desktop theme.""" + +from __future__ import annotations + +import tkinter as tk +from dataclasses import dataclass +from tkinter import ttk + +from platform_support import desktop_font_family + + +@dataclass(frozen=True) +class Palette: + window: str = "#1E1E1E" + chrome: str = "#2D2D30" + panel: str = "#252526" + panel_alt: str = "#333337" + field: str = "#1B1B1C" + border: str = "#3F3F46" + border_hot: str = "#007ACC" + accent: str = "#007ACC" + accent_hot: str = "#1C97EA" + selection: str = "#094771" + text: str = "#F1F1F1" + muted: str = "#B8B8B8" + disabled: str = "#777777" + success: str = "#6A9955" + warning: str = "#DCDCAA" + danger: str = "#F14C4C" + + +PALETTE = Palette() +UI_FONT = desktop_font_family() +MONO_FONT = "Consolas" + + +def apply_vs2010_theme( + root: tk.Misc, + *, + high_contrast: bool = False, + large_text: bool = False, +) -> ttk.Style: + """Apply one compact, square-edged theme to every ttk workspace.""" + palette = PALETTE + style = ttk.Style(root) + try: + style.theme_use("clam") + except tk.TclError: + pass + + font_size = 11 if large_text else 9 + row_height = 30 if large_text else 23 + root.configure(background=palette.window) + style.configure( + ".", + background=palette.panel, + foreground=palette.text, + fieldbackground=palette.field, + bordercolor=palette.border, + darkcolor=palette.border, + lightcolor=palette.border, + troughcolor=palette.window, + selectbackground=palette.selection, + selectforeground=palette.text, + insertcolor=palette.text, + font=(UI_FONT, font_size), + ) + style.configure("TFrame", background=palette.panel) + style.configure("Content.TFrame", background=palette.panel) + style.configure("Sidebar.TFrame", background=palette.chrome) + style.configure("Toolbar.TFrame", background=palette.chrome, relief="flat") + style.configure("Statusbar.TFrame", background=palette.accent) + style.configure("TLabel", background=palette.panel, foreground=palette.text) + style.configure("Toolbar.TLabel", background=palette.chrome, foreground=palette.muted) + style.configure("Statusbar.TLabel", background=palette.accent, foreground="#FFFFFF") + style.configure( + "Brand.TLabel", background=palette.chrome, foreground=palette.text, + font=(UI_FONT, 14, "bold"), + ) + style.configure( + "AccentBrand.TLabel", background=palette.chrome, foreground=palette.accent_hot, + font=(UI_FONT, 9), + ) + style.configure( + "Header.TLabel", background=palette.panel, foreground=palette.text, + font=(UI_FONT, 18, "bold"), + ) + style.configure( + "Subheader.TLabel", background=palette.panel, foreground=palette.muted, + font=(UI_FONT, font_size), + ) + style.configure( + "Metric.TLabel", background=palette.panel_alt, foreground=palette.accent_hot, + font=(UI_FONT, 17, "bold"), + ) + style.configure( + "CardTitle.TLabel", background=palette.panel, foreground=palette.text, + font=(UI_FONT, font_size, "bold"), + ) + style.configure("StatusDownloaded.TLabel", foreground=palette.success) + style.configure("StatusFailed.TLabel", foreground=palette.danger) + style.configure("StatusPending.TLabel", foreground=palette.warning) + style.configure("Title.TLabel", font=(UI_FONT, 16, "bold")) + style.configure("Subtitle.TLabel", foreground=palette.muted) + style.configure("Success.TLabel", foreground=palette.success) + style.configure("Error.TLabel", foreground=palette.danger) + + style.configure( + "TButton", background=palette.panel_alt, foreground=palette.text, + bordercolor=palette.border, padding=(9, 5), borderwidth=1, relief="flat", + ) + style.map( + "TButton", + background=[("pressed", palette.selection), ("active", "#3E3E42")], + bordercolor=[("focus", palette.border_hot), ("active", "#5A5A60")], + foreground=[("disabled", palette.disabled)], + ) + style.configure( + "Nav.TButton", anchor=tk.W, background=palette.chrome, foreground=palette.muted, + bordercolor=palette.chrome, padding=(12, 7), relief="flat", + ) + style.map( + "Nav.TButton", + background=[("pressed", palette.selection), ("active", "#3E3E42")], + foreground=[("pressed", "#FFFFFF"), ("active", "#FFFFFF")], + bordercolor=[("focus", palette.border_hot)], + ) + style.configure( + "Accent.TButton", background=palette.accent, foreground="#FFFFFF", + bordercolor=palette.accent_hot, + ) + style.map("Accent.TButton", background=[("active", palette.accent_hot)]) + style.configure( + "Tool.TButton", background=palette.chrome, foreground=palette.text, + bordercolor=palette.border, padding=(7, 4), + ) + + style.configure( + "TLabelframe", background=palette.panel, foreground=palette.text, + bordercolor=palette.border, relief="solid", borderwidth=1, + ) + style.configure( + "TLabelframe.Label", background=palette.panel, foreground=palette.muted, + font=(UI_FONT, font_size, "bold"), + ) + for widget in ("TEntry", "TSpinbox", "TCombobox"): + style.configure( + widget, fieldbackground=palette.field, foreground=palette.text, + bordercolor=palette.border, arrowcolor=palette.muted, padding=5, + ) + style.map(widget, bordercolor=[("focus", palette.border_hot)]) + style.map( + "TCombobox", + fieldbackground=[("readonly", palette.field)], + foreground=[("readonly", palette.text)], + selectbackground=[("readonly", palette.field)], + selectforeground=[("readonly", palette.text)], + ) + style.configure( + "Treeview", background=palette.field, fieldbackground=palette.field, + foreground=palette.text, rowheight=row_height, bordercolor=palette.border, + relief="flat", + ) + style.map( + "Treeview", background=[("selected", palette.selection)], + foreground=[("selected", "#FFFFFF")], + ) + style.configure( + "Treeview.Heading", background=palette.chrome, foreground=palette.text, + bordercolor=palette.border, relief="raised", padding=(6, 4), + font=(UI_FONT, font_size, "bold"), + ) + style.map("Treeview.Heading", background=[("active", "#3E3E42")]) + style.configure("TNotebook", background=palette.chrome, bordercolor=palette.border) + style.configure( + "TNotebook.Tab", background=palette.chrome, foreground=palette.muted, + bordercolor=palette.border, padding=(11, 6), + ) + style.map( + "TNotebook.Tab", + background=[("selected", palette.panel), ("active", "#3E3E42")], + foreground=[("selected", "#FFFFFF"), ("active", "#FFFFFF")], + bordercolor=[("selected", palette.accent)], + ) + style.configure( + "Vertical.TScrollbar", background=palette.chrome, troughcolor=palette.window, + arrowcolor=palette.muted, bordercolor=palette.border, + ) + style.configure("TCheckbutton", background=palette.panel, foreground=palette.text) + style.configure("TRadiobutton", background=palette.panel, foreground=palette.text) + style.configure( + "Horizontal.TProgressbar", background=palette.accent, + troughcolor=palette.field, bordercolor=palette.border, + ) + + if high_contrast: + style.configure( + ".", background="#000000", foreground="#FFFFFF", fieldbackground="#000000", + bordercolor="#FFFFFF", selectbackground="#FFFFFF", selectforeground="#000000", + ) + for name in ("TFrame", "Content.TFrame", "Sidebar.TFrame", "Toolbar.TFrame", + "TLabel", "Toolbar.TLabel", "TLabelframe", "TLabelframe.Label"): + style.configure(name, background="#000000", foreground="#FFFFFF") + style.configure("Treeview", background="#000000", fieldbackground="#000000") + style.configure("Treeview.Heading", background="#000000", foreground="#FFFFFF") + style.configure("TNotebook.Tab", background="#000000", foreground="#FFFFFF") + return style diff --git a/xenia_bridge.py b/xenia_bridge.py index af6457c..800ad30 100644 --- a/xenia_bridge.py +++ b/xenia_bridge.py @@ -6,6 +6,7 @@ import os import re import shutil +import subprocess from dataclasses import dataclass from pathlib import Path from typing import Iterable @@ -21,6 +22,14 @@ class XeniaBridgeError(RuntimeError): """Raised when a Xenia root or migration plan is unsafe.""" +@dataclass(frozen=True) +class XeniaInstallation: + executable: Path + root: Path + content_root: Path | None + variant: str + + @dataclass(frozen=True) class XeniaSave: profile_id: str @@ -93,6 +102,46 @@ def find_xenia_content_root(path: str | Path | None = None) -> Path | None: return None +def find_xenia_installation(path: str | Path) -> XeniaInstallation | None: + """Find a conventional Xenia executable without searching unrelated folders.""" + selected = Path(path).expanduser().resolve() + candidates: list[Path] = [] + if selected.is_file(): + candidates.append(selected) + root = selected.parent + else: + root = selected.parent if selected.name.casefold() == "content" else selected + names = ("xenia_canary.exe", "xenia.exe", "xenia-canary", "xenia") + candidates.extend(root / name for name in names) + candidates.extend(root.parent / name for name in names if selected.name.casefold() == "content") + executable = next((item for item in candidates if item.is_file()), None) + if executable is None: + return None + content = find_xenia_content_root(root) + variant = "Canary" if "canary" in executable.name.casefold() else "Master" + return XeniaInstallation(executable, executable.parent, content, variant) + + +def launch_xenia( + installation: XeniaInstallation, + game_path: str | Path, + *, + fullscreen: bool = False, +) -> dict[str, object]: + """Launch a user-selected title through an argument list, never a shell.""" + game = Path(game_path).expanduser().resolve() + if not installation.executable.is_file(): + raise FileNotFoundError(installation.executable) + if not game.is_file() and not game.is_dir(): + raise FileNotFoundError(game) + command = [str(installation.executable), str(game)] + if fullscreen: + command.append("--fullscreen=true") + process = subprocess.Popen(command, cwd=installation.root) + return {"pid": process.pid, "variant": installation.variant, + "executable": str(installation.executable), "game": str(game)} + + def scan_xenia_saves(content_root: str | Path) -> tuple[XeniaSave, ...]: """Index visible save packages under a Xenia content tree.""" root = Path(content_root).expanduser().resolve() From a1c87ffe33e569fb1e42886e243fe68110f832f7 Mon Sep 17 00:00:00 2001 From: Sthornberry9 <46094434+Sthornberry9@users.noreply.github.com> Date: Sat, 1 Aug 2026 04:42:11 -0400 Subject: [PATCH 2/2] test: normalize Xenia launch paths on macOS --- tests.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests.py b/tests.py index 305a9e5..c2c290e 100644 --- a/tests.py +++ b/tests.py @@ -1639,7 +1639,12 @@ def test_xenia_installation_is_discovered_and_launched_without_a_shell(self, pop result = launch_xenia(installation, game, fullscreen=True) self.assertEqual(result["pid"], 42) popen.assert_called_once_with( - [str(executable), str(game), "--fullscreen=true"], cwd=root + [ + str(installation.executable), + str(game.resolve()), + "--fullscreen=true", + ], + cwd=installation.root, ) def test_knowledge_priority_and_conflict_resolution_are_persistent(self):