diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index db95f10..6ddea01 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -62,7 +62,7 @@ jobs: shell: pwsh run: | New-Item -ItemType Directory -Path package | Out-Null - Copy-Item dist\UnityScraper.exe, README.md, CHANGELOG.md, LICENSE, DOCS_INDEX.md, BACKUP_MANAGER.md, COLLECTION_INTELLIGENCE.md, CONSOLE_SYNC.md, KNOWLEDGE_SOURCES.md, LINUX.md, PLUGIN_API.md package\ + Copy-Item dist\UnityScraper.exe, README.md, CHANGELOG.md, LICENSE, DOCS_INDEX.md, BACKUP_MANAGER.md, COLLECTION_INTELLIGENCE.md, COMMUNITY_HUB.md, CONSOLE_SYNC.md, KNOWLEDGE_SOURCES.md, LINUX.md, PLUGIN_API.md, PROFILE_INTELLIGENCE.md, PROFILES_AND_SAVES.md, SECURITY.md, THIRD_PARTY_NOTICES.md package\ Compress-Archive -Path package\* -DestinationPath UnityScraper-Windows-x64.zip $hash = (Get-FileHash UnityScraper-Windows-x64.zip -Algorithm SHA256).Hash.ToLower() "$hash *UnityScraper-Windows-x64.zip" | Set-Content UnityScraper-Windows-x64.zip.sha256 diff --git a/API.md b/API.md index 55e210b..06a7164 100644 --- a/API.md +++ b/API.md @@ -45,6 +45,12 @@ when a token is configured. | `GET` | `/api/titleids` | List library TitleIDs | | `GET` | `/api/titleid/` | Get one library record | | `GET` | `/api/search?q=` | Search the library | +| `GET` | `/api/community/search?q=` | Search all local domains; repeat `category` to filter | +| `GET` | `/api/preservation/dedup/actions` | List duplicate actions from the latest or selected plan | +| `POST` | `/api/preservation/dedup/preview` | Create a read-only duplicate preview for JSON `root` | +| `POST` | `/api/preservation/dedup//apply` | Apply `quarantine` or `hardlink` mode | +| `POST` | `/api/preservation/dedup//restore` | Revalidate and restore a quarantined original | +| `GET` | `/api/plugins` | List managed plugins and checksum trust state | | `POST` | `/api/metadata/` | Collect metadata | | `POST` | `/api/download/` | Process downloads | | `GET` | `/api/statistics` | Library statistics | @@ -56,6 +62,9 @@ when a token is configured. | `POST` | `/api/config` | Update allowlisted runtime settings | TitleID routes require exactly eight hexadecimal characters. +Duplicate apply and restore endpoints change local files and therefore require +an explicit action ID created by a prior preview. They retain a recovery copy +until restoration and use the same path and checksum validation as the desktop. ## Configuration diff --git a/CHANGELOG.md b/CHANGELOG.md index 6949a02..8651f81 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,16 @@ Notable changes to UnityScraper are documented here. The project follows ### Added +- Additive schema migration 9 for plugin collection audits and reversible + duplicate recovery records. +- Runtime plugin loading with approved-checksum enforcement, bounded results, + per-plugin failure isolation, provenance, and unknown-only enrichment. +- Background Community Hub operations, actionable unified-search results, and + selectable duplicate apply/restore controls. +- Unified-search and preservation API/CLI operations. +- Read-only FATX partition geometry and Xbox 360 USB-container reporting. +- Bounded read-only STFS file-table inventory in package workspaces. + - Community Hub with unified local search across games, source knowledge, profiles, saves, achievements, files, and structured records. - Additive schema migration 8 for structured knowledge, guided console plans, diff --git a/COMMUNITY_HUB.md b/COMMUNITY_HUB.md index 4dac454..c9abda0 100644 --- a/COMMUNITY_HUB.md +++ b/COMMUNITY_HUB.md @@ -27,9 +27,9 @@ or press `Ctrl+K` to focus unified search. archive export layouts plus checksum manifests. 12. Multi-disc completeness audits based on scanned disc number and count. 13. Duplicate previews using size grouping and SHA-256 verification. -14. Recoverable duplicate actions that quarantine the original and can create - a verified hardlink only after a second hash check. -15. Read-only FATX signature and mounted-storage audits. +14. Selectable, recoverable duplicate actions that quarantine the original, + optionally create a verified hardlink, and restore from the interface. +15. Read-only FATX partition geometry, Xbox 360 USB container, and mounted-storage audits. 16. Original Xbox `default.xbe` discovery alongside Xbox 360 collections. 17. Plugin discovery, checksums, permission display, enable/disable state, and bounded ZIP installation or update with rollback. @@ -46,7 +46,8 @@ or press `Ctrl+K` to focus unified search. - Sync plans are previews until the user confirms queueing. Queued jobs still run through the normal transfer controls. - Duplicate cleanup never deletes the only retained copy. Quarantined files - remain under `.unityscraper-dedup-quarantine` for manual recovery. + remain under `.unityscraper-dedup-quarantine` and can be restored from the + Preservation tab after checksum and path validation. - FATX images are detected read-only. Raw-device and raw-image writes are not implemented. - Package workspaces and ownership changes are previews. CON/LIVE/PIRS rebuild, @@ -65,6 +66,11 @@ disc audits, dedup plans, storage audits, original Xbox records, plugin state, recovery events, dashboard tests, and accessibility preferences. It is additive and preserves existing databases. +Migration 9 adds plugin collection audits and duplicate-recovery records. Enabled +plugins run during normal metadata collection only while their approved entrypoint +checksum still matches. Long Community Hub operations run in a background worker; +search results can be opened with Enter or a double-click. + Imported ConsoleMods, XenonLibrary, Free60, Redump, and No-Intro information continues to retain source, revision, citation, licensing, and conflict data. Redump and No-Intro DAT files remain user-supplied; copyrighted game content is diff --git a/PLUGIN_API.md b/PLUGIN_API.md index 5b9e5f6..43234ba 100644 --- a/PLUGIN_API.md +++ b/PLUGIN_API.md @@ -21,9 +21,17 @@ plugins/ } ``` -The entrypoint exports a `MetadataCollectorPlugin` subclass. The caller must -pass the plugin ID in `enabled_plugins` before code is loaded. Permissions -are disclosure metadata, not an operating-system sandbox, so only enable -plugins whose source and publisher you trust. +The entrypoint exports a `MetadataCollectorPlugin` subclass. Desktop installs +live in the managed application plugin directory and begin disabled. Enabling a +plugin records its SHA-256 checksum; normal metadata collection loads it only +while the manifest ID and approved checksum still match. Each result is limited +to 2 MiB, cover/update counts are bounded, failures are isolated, and every run +is audited in SQLite. Known title and publisher values are never replaced by a +plugin fallback. + +Requested access is disclosure metadata, not an operating-system sandbox. +Plugin code executes with the user's account permissions, so only enable source +and publishers you trust. Editing an enabled entrypoint automatically prevents +it from loading until it is reviewed and enabled again. Root-level legacy Python plugins load only with `allow_legacy=True`. diff --git a/PROFILE_INTELLIGENCE.md b/PROFILE_INTELLIGENCE.md index e90d7fa..9cdfe0f 100644 --- a/PROFILE_INTELLIGENCE.md +++ b/PROFILE_INTELLIGENCE.md @@ -21,9 +21,9 @@ For game GPDs, the application displays achievement ID, title, gamerscore, locked/unlocked state, and a valid online unlock timestamp when present. It also records totals for unlocked achievements and earned/possible gamerscore. -The parser never writes to the source file. It does not unlock achievements, -alter sync records, extract images, edit account settings, or repair malformed -databases. +The parser never writes to the source file. It can validate and export a selected +embedded PNG/JPEG image to a new file, but it does not unlock achievements, +alter sync records, edit account settings, or repair malformed databases. UnityScraper currently reads standalone or extracted GPD files. It does not silently unpack or rewrite the profile's STFS container. diff --git a/PROJECT_STATUS.md b/PROJECT_STATUS.md index 1613b67..a1a93e4 100644 --- a/PROJECT_STATUS.md +++ b/PROJECT_STATUS.md @@ -61,6 +61,11 @@ backup-management, and source-attributed knowledge application. recoverable deduplication, storage audits, original Xbox discovery, plugin controls, recovery actions, dashboard probes, and accessibility preferences. - Additive schema migration 8 and local audit history for every new workspace. +- Additive schema migration 9 for audited plugin collection and reversible + duplicate actions, plus selectable restore controls. +- Background Community Hub jobs, actionable unified-search navigation, CLI/API + parity for search and preservation, FATX geometry reports, and bounded STFS + file-table inventory. - Windows, Linux, and unsigned Apple Silicon macOS CI packaging. ## Validation @@ -93,7 +98,8 @@ backup-management, and source-attributed knowledge application. re-signing remain disabled until complete package verification and reliable cross-platform signing support are available. - FATX and raw-device access remains read-only. Duplicate actions retain a - quarantine copy, and console plans require explicit queue confirmation. + tracked quarantine copy with validated restoration, and console plans require + explicit queue confirmation. ## Future Work diff --git a/README.md b/README.md index 55d7870..85c8696 100644 --- a/README.md +++ b/README.md @@ -120,7 +120,11 @@ checksums, platform notes, and safety guidance. - Adds profile dashboards, read-only package workspaces, ownership previews, save comparison, played-title history, and validated GPD image export. - Manages preferred artwork, multi-disc audits, recoverable duplicate cleanup, - read-only FATX inspection, original Xbox discovery, plugins, and recovery. + read-only FATX geometry inspection, original Xbox discovery, plugins, and recovery. +- Runs long Community Hub operations outside the interface thread, opens unified + search results in their native workspace, and restores quarantined duplicates. +- Inventories STFS file tables in read-only package workspaces without claiming + unsupported package rebuilding or signing. - Stores high-contrast, large-text, reduced-motion, and keyboard-hint settings. See [COMMUNITY_HUB.md](COMMUNITY_HUB.md) for all twenty capabilities and their @@ -323,6 +327,17 @@ python main.py --match-file game.iso # Capture a read-only console inventory python main.py --ftp-host 192.168.1.50 --ftp-snapshot /Hdd1 + +# Search every local knowledge domain +python main.py --search-all "Hitman" + +# Inspect FATX geometry or an Xbox 360 USB container without writing it +python main.py --audit-storage E:\drive.img + +# Preview duplicate recovery actions, then apply or restore one explicitly +python main.py --dedup-preview D:\XboxArchive +python main.py --dedup-apply 42 --dedup-mode quarantine +python main.py --dedup-restore 42 ``` ## Optional REST API diff --git a/api.py b/api.py index aecbb48..de72db2 100644 --- a/api.py +++ b/api.py @@ -13,7 +13,7 @@ from flask import Flask, jsonify, request from flask_cors import CORS -from app_paths import EXPORTS_DIR +from app_paths import DATABASE_PATH, EXPORTS_DIR, PLUGINS_DIR from app_version import DISPLAY_VERSION if TYPE_CHECKING: @@ -134,6 +134,81 @@ def search(): ) ) + @self.app.get("/api/community/search") + def community_search(): + from unified_search import UnifiedSearchService + + query = request.args.get("q", "")[:200] + categories = tuple( + value.strip() for value in request.args.getlist("category") if value.strip() + ) + limit = request.args.get("limit", default=100, type=int) + if len(query.strip()) < 2: + return jsonify({"error": "q must contain at least two characters"}), 400 + if limit is None or limit < 1 or limit > 500: + return jsonify({"error": "limit must be between 1 and 500"}), 400 + return self._execute(lambda: self._search_response( + UnifiedSearchService(self._database_path()).search( + query, categories=categories, limit=limit + ) + )) + + @self.app.get("/api/preservation/dedup/actions") + def dedup_actions(): + from community_services import PreservationPlanningService + + plan_id = request.args.get("plan_id", type=int) + return self._execute(lambda: { + "actions": PreservationPlanningService(self._database_path()).list_dedup_actions( + plan_id + ) + }) + + @self.app.post("/api/preservation/dedup/preview") + def dedup_preview(): + from community_services import PreservationPlanningService + + payload = request.get_json(silent=True) + if not isinstance(payload, dict) or not isinstance(payload.get("root"), str): + return jsonify({"error": "A root directory is required"}), 400 + return self._execute( + lambda: PreservationPlanningService(self._database_path()).create_dedup_plan( + payload["root"] + ) + ) + + @self.app.post("/api/preservation/dedup//apply") + def dedup_apply(action_id: int): + from community_services import PreservationPlanningService + + payload = request.get_json(silent=True) or {} + mode = payload.get("mode", "quarantine") if isinstance(payload, dict) else "" + if mode not in {"quarantine", "hardlink"}: + return jsonify({"error": "mode must be quarantine or hardlink"}), 400 + return self._execute( + lambda: PreservationPlanningService(self._database_path()).apply_dedup_action( + action_id, mode + ) + ) + + @self.app.post("/api/preservation/dedup//restore") + def dedup_restore(action_id: int): + from community_services import PreservationPlanningService + + return self._execute( + lambda: PreservationPlanningService(self._database_path()).restore_dedup_action( + action_id + ) + ) + + @self.app.get("/api/plugins") + def plugins(): + from community_services import PluginControlService + + return self._execute(lambda: { + "plugins": PluginControlService(self._database_path()).discover(PLUGINS_DIR) + }) + @self.app.post("/api/metadata/") def collect_metadata(titleid: str): normalized = self._titleid_or_error(titleid) @@ -277,6 +352,11 @@ def _require_scraper(self) -> "UnityScraper": raise RuntimeError("Scraper not initialized") return self.scraper + def _database_path(self) -> Path: + if self.scraper is None: + return DATABASE_PATH + return Path(getattr(self.scraper.db, "db_path", DATABASE_PATH)) + def _titleid_or_error(self, titleid: str): if self.scraper is None: return jsonify({"error": "Scraper not initialized"}), 400 diff --git a/app_paths.py b/app_paths.py index 8375536..d1c646d 100644 --- a/app_paths.py +++ b/app_paths.py @@ -146,6 +146,7 @@ def xdg_path(variable: str, fallback: Path) -> Path: EXPORTS_DIR = _PATHS.exports DIAGNOSTICS_DIR = _PATHS.diagnostics PROFILE_BACKUPS_DIR = DATA_DIR / "profile_backups" +PLUGINS_DIR = DATA_DIR / "plugins" DATABASE_PATH = DATA_DIR / "unityscraper.db" CONFIG_PATH = CONFIG_DIR / "config.json" @@ -172,6 +173,7 @@ def ensure_app_dirs() -> None: EXPORTS_DIR, DIAGNOSTICS_DIR, PROFILE_BACKUPS_DIR, + PLUGINS_DIR, ): path.mkdir(parents=True, exist_ok=True) diff --git a/backup_manager.py b/backup_manager.py index c8a1c2b..8f1af2c 100644 --- a/backup_manager.py +++ b/backup_manager.py @@ -15,7 +15,7 @@ from dataclasses import asdict, dataclass, field from datetime import datetime, timezone from pathlib import Path, PurePosixPath -from typing import Callable, Iterable, Iterator, Optional +from typing import Any, Callable, Iterable, Iterator, Optional STFS_MAGICS = {b"CON ", b"LIVE", b"PIRS"} @@ -74,12 +74,29 @@ class StfsPackage: device_id: str +@dataclass(frozen=True) +class StfsEntry: + index: int + path: str + name: str + is_directory: bool + consecutive: bool + allocated_blocks: int + starting_block: int + parent_index: int + size: int + + @dataclass(frozen=True) class XbePackage: path: Path title_id: str title_name: str size: int + allowed_media: int + region_flags: int + disc_number: int + version: int @dataclass(frozen=True) @@ -198,6 +215,103 @@ def inspect_stfs(path: str | Path) -> StfsPackage: ) +def _stfs_data_block_number(block: int, magic: bytes, header_size: int, + block_separation: int) -> int: + if block < 0 or block > 0xFFFFFF: + raise InvalidPackageError("STFS block number is outside the supported range") + aligned_header = (header_size + 0xFFF) & 0xFFFFF000 + shift = 1 if aligned_header == 0xB000 else (0 if block_separation & 1 else 1) + base = (block + 0xAA) // 0xAA + if magic == b"CON ": + base <<= shift + result = base + block + if block > 0xAA: + base = (block + 0x70E4) // 0x70E4 + if magic == b"CON ": + base <<= shift + result += base + if block > 0x70E4: + base = (block + 0x4AF768) // 0x4AF768 + if magic == b"CON ": + base <<= shift + result += base + return result + + +def list_stfs_entries(path: str | Path, max_entries: int = 100_000) -> list[StfsEntry]: + """Read the bounded STFS file table without extracting or mutating content.""" + package_path = Path(path) + package_size = package_path.stat().st_size + with package_path.open("rb") as handle: + header = handle.read(0x3AD) + if len(header) < 0x3AD or header[:4] not in STFS_MAGICS: + raise InvalidPackageError("Not a supported STFS package") + if int.from_bytes(header[0x3A9:0x3AD], "big") != 0: + raise InvalidPackageError("SVOD packages do not contain an STFS file table") + header_size = int.from_bytes(header[0x340:0x344], "big") + descriptor = header[0x379:0x39D] + if descriptor[0] != 0x24: + raise InvalidPackageError("STFS volume descriptor is invalid") + block_separation = descriptor[2] + table_blocks = int.from_bytes(descriptor[3:5], "big") + table_start = int.from_bytes(descriptor[5:8], "big") + if table_blocks <= 0 or table_blocks > 0x1000: + raise InvalidPackageError("STFS file-table size is invalid") + aligned_header = (header_size + 0xFFF) & 0xFFFFF000 + raw_entries: list[dict[str, Any]] = [] + for table_index in range(table_blocks): + logical_block = table_start + table_index + physical_block = _stfs_data_block_number( + logical_block, header[:4], header_size, block_separation + ) + offset = aligned_header + physical_block * 0x1000 + if offset + 0x1000 > package_size: + raise InvalidPackageError("STFS file table points outside the package") + handle.seek(offset) + table = handle.read(0x1000) + for entry_offset in range(0, 0x1000, 0x40): + data = table[entry_offset:entry_offset + 0x40] + if not any(data): + continue + flags = data[0x28] + name_length = flags & 0x3F + if name_length == 0 or name_length > 0x28: + continue + name = data[:name_length].decode("utf-8", errors="replace") + raw_entries.append({ + "name": name, + "directory": bool(flags & 0x80), + "consecutive": bool(flags & 0x40), + "blocks": int.from_bytes(data[0x29:0x2C], "little"), + "start": int.from_bytes(data[0x2F:0x32], "little"), + "parent": int.from_bytes(data[0x32:0x34], "big"), + "size": int.from_bytes(data[0x34:0x38], "big"), + }) + if len(raw_entries) > max_entries: + raise InvalidPackageError("STFS file table exceeds the safety limit") + entries: list[StfsEntry] = [] + for index, row in enumerate(raw_entries): + ancestors: list[str] = [] + parent = row["parent"] + visited: set[int] = set() + while parent != 0xFFFF: + if parent >= len(raw_entries) or parent in visited: + ancestors = ["[invalid-parent]"] + break + visited.add(parent) + ancestors.append(raw_entries[parent]["name"]) + parent = raw_entries[parent]["parent"] + full_path = "/".join(reversed(ancestors)) + full_path = f"{full_path}/{row['name']}" if full_path else row["name"] + entries.append(StfsEntry( + index=index, path=full_path, name=row["name"], + is_directory=row["directory"], consecutive=row["consecutive"], + allocated_blocks=row["blocks"], starting_block=row["start"], + parent_index=row["parent"], size=row["size"], + )) + return entries + + def inspect_xbe(path: str | Path) -> XbePackage: """Read TitleID and title from an original Xbox executable certificate.""" package_path = Path(path) @@ -217,7 +331,13 @@ def inspect_xbe(path: str | Path) -> XbePackage: title_id = f"{int.from_bytes(certificate[0x8:0xC], 'little'):08X}" title_name = certificate[0xC:0x5C].decode("utf-16-le", errors="ignore") title_name = title_name.split("\x00", 1)[0].strip() - return XbePackage(package_path, title_id, title_name, package_path.stat().st_size) + return XbePackage( + package_path, title_id, title_name, package_path.stat().st_size, + int.from_bytes(certificate[0x9C:0xA0], "little"), + int.from_bytes(certificate[0xA0:0xA4], "little"), + int.from_bytes(certificate[0xA8:0xAC], "little"), + int.from_bytes(certificate[0xAC:0xB0], "little"), + ) def inspect_xex(path: str | Path) -> XexPackage: diff --git a/build_linux.sh b/build_linux.sh index c2f2709..fb6f905 100755 --- a/build_linux.sh +++ b/build_linux.sh @@ -27,7 +27,9 @@ install -m 0644 packaging/linux/io.github.trapemall.UnityScraper.metainfo.xml \ install -m 0644 assets/UnityScraper.png "$STAGE/unityscraper.png" install -m 0644 README.md CHANGELOG.md LICENSE "$STAGE/" install -m 0644 DOCS_INDEX.md BACKUP_MANAGER.md COLLECTION_INTELLIGENCE.md \ - CONSOLE_SYNC.md KNOWLEDGE_SOURCES.md LINUX.md PLUGIN_API.md "$STAGE/" + COMMUNITY_HUB.md CONSOLE_SYNC.md KNOWLEDGE_SOURCES.md LINUX.md \ + PLUGIN_API.md PROFILE_INTELLIGENCE.md PROFILES_AND_SAVES.md \ + SECURITY.md THIRD_PARTY_NOTICES.md "$STAGE/" tar -C dist -czf "$ARCHIVE" "UnityScraper-Linux-${ARCH}" sha256sum "$ARCHIVE" > "$ARCHIVE.sha256" diff --git a/community_gui.py b/community_gui.py index 4412d73..f901ae0 100644 --- a/community_gui.py +++ b/community_gui.py @@ -3,12 +3,14 @@ from __future__ import annotations import json +import queue import tkinter as tk +from concurrent.futures import Future, ThreadPoolExecutor from pathlib import Path from tkinter import filedialog, messagebox, ttk from typing import Any, Callable -from app_paths import DATABASE_PATH, DOWNLOADS_DIR, PROFILE_BACKUPS_DIR, executable_root +from app_paths import DATABASE_PATH, DOWNLOADS_DIR, PLUGINS_DIR, PROFILE_BACKUPS_DIR from backup_manager import FtpTarget from community_services import ( DASHBOARD_PRESETS, @@ -23,6 +25,7 @@ StorageAndXboxService, ) from profile_intelligence import ProfileIntelligenceService +from platform_support import open_path from structured_knowledge import StructuredKnowledgeService from unified_search import UnifiedSearchService @@ -30,7 +33,13 @@ class CommunityHubPage: """One operational surface for cross-domain community workflows.""" - def __init__(self, root: tk.Tk, parent: ttk.Frame, page_header: Callable) -> None: + def __init__( + self, + root: tk.Tk, + parent: ttk.Frame, + page_header: Callable, + navigate: Callable[[dict[str, Any]], None] | None = None, + ) -> None: self.root = root self.parent = parent self.search_service = UnifiedSearchService() @@ -45,12 +54,18 @@ def __init__(self, root: tk.Tk, parent: ttk.Frame, page_header: Callable) -> Non self.recovery = RecoveryService() self.compatibility = DashboardCompatibilityService() self.accessibility = AccessibilityService() + self.navigate = navigate + self.search_rows: dict[str, dict[str, Any]] = {} + self.task_events: queue.Queue[tuple[str, Future, Callable | None]] = queue.Queue() + self.task_executor = ThreadPoolExecutor(max_workers=2, thread_name_prefix="community") + self.active_task: Future | None = None page_header("Community Hub", "Search, organize, preserve, and safely plan console changes.") self._build() + self.root.after(100, self._poll_tasks) def _build(self) -> None: - notebook = ttk.Notebook(self.parent) - notebook.grid(row=1, column=0, sticky="nsew") + self.notebook = ttk.Notebook(self.parent) + self.notebook.grid(row=1, column=0, sticky="nsew") self.parent.rowconfigure(1, weight=1) for label, builder in ( ("Search", self._build_search), @@ -64,9 +79,19 @@ def _build(self) -> None: ("Compatibility", self._build_compatibility), ("Accessibility", self._build_accessibility), ): - frame = ttk.Frame(notebook, padding=12) - notebook.add(frame, text=label) + frame = ttk.Frame(self.notebook, padding=12) + self.notebook.add(frame, text=label) builder(frame) + task_bar = ttk.Frame(self.parent) + task_bar.grid(row=2, column=0, sticky="ew", pady=(8, 0)) + self.task_status = tk.StringVar(value="Ready") + ttk.Label(task_bar, textvariable=self.task_status, style="Subheader.TLabel").pack( + side=tk.LEFT + ) + self.cancel_task_button = ttk.Button( + task_bar, text="Cancel Pending Task", command=self._cancel_task, state=tk.DISABLED + ) + self.cancel_task_button.pack(side=tk.RIGHT) def _build_search(self, frame: ttk.Frame) -> None: frame.columnconfigure(0, weight=1) @@ -89,16 +114,25 @@ def _build_search(self, frame: ttk.Frame) -> None: self.search_tree.heading(column, text=heading) self.search_tree.column(column, width=width, stretch=column != "category") self.search_tree.grid(row=2, column=0, columnspan=2, sticky="nsew") + self.search_tree.bind("", self._open_search_result) + self.search_tree.bind("", self._open_search_result) def focus_search(self) -> None: self.search_entry.focus_set() def _run_search(self) -> None: - self._fill_tree( - self.search_tree, - self.search_service.search(self.search_var.get()), - lambda row: (row["category"], row["title"], row["subtitle"]), - ) + rows = self.search_service.search(self.search_var.get()) + self.search_rows = {str(index): row for index, row in enumerate(rows)} + self._fill_tree(self.search_tree, rows, + lambda row: (row["category"], row["title"], row["subtitle"])) + + def _open_search_result(self, _event: tk.Event | None = None) -> None: + selection = self.search_tree.selection() + if not selection: + return + row = self.search_rows.get(selection[0]) + if row is not None and self.navigate is not None: + self.navigate(row) def _build_knowledge(self, frame: ttk.Frame) -> None: frame.columnconfigure(0, weight=1) @@ -123,8 +157,10 @@ def _build_knowledge(self, frame: ttk.Frame) -> None: self.knowledge_tree.grid(row=1, column=0, sticky="nsew") def _extract_knowledge(self) -> None: - self._run("Knowledge extraction", lambda: self.structured.extract_cached_documents()) - self._refresh_knowledge() + self._submit( + "Knowledge extraction", self.structured.extract_cached_documents, + lambda _result: self._refresh_knowledge(), + ) def _refresh_knowledge(self) -> None: rows = self.structured.list_records(self.knowledge_type.get()) @@ -137,16 +173,24 @@ def _build_sync(self, frame: ttk.Frame) -> None: self.sync_snapshot = tk.StringVar() self.sync_target = tk.StringVar() self.sync_dashboard = tk.StringVar(value="aurora") + self.sync_snapshot_ids: dict[str, int] = {} + self.sync_target_ids: dict[str, int] = {} self._path_row(frame, 0, "Local content root", self.sync_root, directory=True) - ttk.Label(frame, text="Console snapshot ID").grid(row=1, column=0, sticky="w", pady=5) - ttk.Entry(frame, textvariable=self.sync_snapshot).grid(row=1, column=1, sticky="ew", pady=5) + ttk.Label(frame, text="Console snapshot").grid(row=1, column=0, sticky="w", pady=5) + self.sync_snapshot_combo = ttk.Combobox( + frame, textvariable=self.sync_snapshot, state="readonly" + ) + self.sync_snapshot_combo.grid(row=1, column=1, sticky="ew", pady=5) ttk.Label(frame, text="Dashboard").grid(row=2, column=0, sticky="w", pady=5) ttk.Combobox(frame, textvariable=self.sync_dashboard, state="readonly", values=tuple(DASHBOARD_PRESETS)).grid(row=2, column=1, sticky="w", pady=5) - ttk.Label(frame, text="Saved console target ID (optional)").grid( + ttk.Label(frame, text="Saved console target (optional)").grid( row=3, column=0, sticky="w", pady=5 ) - ttk.Entry(frame, textvariable=self.sync_target).grid( + self.sync_target_combo = ttk.Combobox( + frame, textvariable=self.sync_target, state="readonly" + ) + self.sync_target_combo.grid( row=3, column=1, sticky="ew", pady=5 ) controls = ttk.Frame(frame) @@ -159,19 +203,52 @@ def _build_sync(self, frame: ttk.Frame) -> None: state=tk.DISABLED, ) self.queue_sync_button.pack(side=tk.LEFT, padx=8) + ttk.Button(controls, text="Refresh Sources", command=self._refresh_sync_sources).pack( + side=tk.LEFT + ) self.sync_output = self._output(frame, 5, 2) self.current_sync_plan_id: int | None = None + self._refresh_sync_sources() + + def _refresh_sync_sources(self) -> None: + snapshots = self.console_plans.list_snapshots() + self.sync_snapshot_ids = { + f"#{row['id']} | {row['captured_at']} | {row['root']} | {row['item_count']} items": + int(row["id"]) + for row in snapshots + } + self.sync_snapshot_combo.configure(values=tuple(self.sync_snapshot_ids)) + if self.sync_snapshot.get() not in self.sync_snapshot_ids: + self.sync_snapshot.set(next(iter(self.sync_snapshot_ids), "")) + targets = self.console_plans.list_ftp_targets() + self.sync_target_ids = { + f"{row['name']} | {row['location']}": int(row["id"]) for row in targets + } + target_values = ("", *self.sync_target_ids) + self.sync_target_combo.configure(values=target_values) + if self.sync_target.get() not in target_values: + self.sync_target.set("") def _create_sync_plan(self) -> None: - result = self._run("Sync preview", lambda: self.console_plans.create_plan( - self.sync_root.get(), int(self.sync_snapshot.get()), self.sync_dashboard.get() - )) - self._show_output(self.sync_output, result) - if result is not None: + root = self.sync_root.get() + if self.sync_snapshot.get() not in self.sync_snapshot_ids: + messagebox.showinfo( + "Console sync", "Capture a console inventory snapshot first.", parent=self.root + ) + return + snapshot_id = self.sync_snapshot_ids[self.sync_snapshot.get()] + dashboard = self.sync_dashboard.get() + def completed(result: Any) -> None: + self._show_output(self.sync_output, result) self.current_sync_plan_id = int(result["plan_id"]) self.queue_sync_button.configure( state=tk.NORMAL if result["summary"]["uploads"] else tk.DISABLED ) + self._submit( + "Sync preview", + lambda: self.console_plans.create_plan(root, snapshot_id, dashboard), + completed, + ) def _queue_sync_plan(self) -> None: if self.current_sync_plan_id is None: @@ -183,16 +260,15 @@ def _queue_sync_plan(self) -> None: parent=self.root, ): return - target = self.sync_target.get().strip() - result = self._run( - "Queue sync plan", + target = self.sync_target_ids.get(self.sync_target.get()) + plan_id = self.current_sync_plan_id + self._submit_output( + "Queue sync plan", self.sync_output, lambda: self.console_plans.queue_uploads( - self.current_sync_plan_id, int(target) if target else None + plan_id, target ), + lambda _result: self.queue_sync_button.configure(state=tk.DISABLED), ) - self._show_output(self.sync_output, result) - if result is not None: - self.queue_sync_button.configure(state=tk.DISABLED) def _build_profiles(self, frame: ttk.Frame) -> None: frame.columnconfigure(1, weight=1) @@ -223,52 +299,68 @@ def _build_profiles(self, frame: ttk.Frame) -> None: self.profile_output = self._output(frame, 6, 3) def _profile_dashboard(self) -> None: - self._show_output(self.profile_output, self._run( - "Profile dashboard", lambda: self.profiles.profile_dashboard(self.profile_id.get()))) + profile_id = self.profile_id.get() + self._submit_output("Profile dashboard", self.profile_output, + lambda: self.profiles.profile_dashboard(profile_id)) def _inspect_package(self) -> None: - self._show_output(self.profile_output, self._run( - "Package inspection", lambda: self.packages.inspect(self.package_path.get()))) + package_path = self.package_path.get() + self._submit_output("Package inspection", self.profile_output, + lambda: self.packages.inspect(package_path)) def _package_workspace(self) -> None: destination = filedialog.askdirectory(parent=self.root, title="Choose package workspace") if destination: - self._show_output(self.profile_output, self._run( - "Package workspace", lambda: {"manifest": str( - self.packages.create_workspace(self.package_path.get(), destination))})) + package_path = self.package_path.get() + self._submit_output("Package workspace", self.profile_output, lambda: { + "manifest": str(self.packages.create_workspace( + package_path, destination))}) def _ownership_preview(self) -> None: - self._show_output(self.profile_output, self._run( - "Ownership preview", lambda: self.profiles.preview_ownership_migration( - self.profile_id.get(), self.package_path.get()))) + profile_id = self.profile_id.get() + package_path = self.package_path.get() + self._submit_output("Ownership preview", self.profile_output, + lambda: self.profiles.preview_ownership_migration( + profile_id, package_path)) def _compare_saves(self) -> None: - self._show_output(self.profile_output, self._run( - "Save comparison", lambda: self.profiles.compare_save_files( - self.compare_left.get(), self.compare_right.get()))) + left = self.compare_left.get() + right = self.compare_right.get() + self._submit_output("Save comparison", self.profile_output, + lambda: self.profiles.compare_save_files(left, right)) def _build_preservation(self, frame: ttk.Frame) -> None: frame.columnconfigure(1, weight=1) self.art_titleid = tk.StringVar() self.art_path = tk.StringVar() + self.art_region = tk.StringVar() + self.art_language = tk.StringVar() + self.art_preset = tk.StringVar(value="aurora") ttk.Label(frame, text="Artwork TitleID").grid(row=0, column=0, sticky="w", pady=4) ttk.Entry(frame, textvariable=self.art_titleid).grid(row=0, column=1, sticky="ew", pady=4) self._path_row(frame, 1, "Preferred artwork", self.art_path, directory=False) + metadata = ttk.Frame(frame) + metadata.grid(row=2, column=1, sticky="w", pady=4) + ttk.Label(metadata, text="Region").pack(side=tk.LEFT) + ttk.Entry(metadata, textvariable=self.art_region, width=9).pack(side=tk.LEFT, padx=(4, 10)) + ttk.Label(metadata, text="Language").pack(side=tk.LEFT) + ttk.Entry(metadata, textvariable=self.art_language, width=9).pack(side=tk.LEFT, padx=(4, 0)) controls = ttk.Frame(frame) - controls.grid(row=2, column=1, sticky="w", pady=8) + controls.grid(row=3, column=1, sticky="w", pady=8) ttk.Button(controls, text="Set Artwork", command=self._set_artwork).pack(side=tk.LEFT) - ttk.Button(controls, text="Export Artwork", command=self._export_artwork).pack(side=tk.LEFT, padx=8) - ttk.Button(controls, text="Audit Disc Sets", command=self._audit_discs).pack(side=tk.LEFT) + ttk.Combobox(controls, textvariable=self.art_preset, state="readonly", width=10, + values=tuple(self.artwork.PRESETS)).pack(side=tk.LEFT, padx=8) + ttk.Button(controls, text="Export Artwork", command=self._export_artwork).pack(side=tk.LEFT) + ttk.Button(controls, text="Audit Disc Sets", command=self._audit_discs).pack( + side=tk.LEFT, padx=8 + ) self.dedup_root = tk.StringVar() - self._path_row(frame, 3, "Duplicate scan root", self.dedup_root, directory=True) + self._path_row(frame, 4, "Duplicate scan root", self.dedup_root, directory=True) dedup_controls = ttk.Frame(frame) - dedup_controls.grid(row=4, column=1, sticky="w", pady=8) + dedup_controls.grid(row=5, column=1, sticky="w", pady=8) ttk.Button(dedup_controls, text="Create Dedup Preview", command=self._dedup).pack( side=tk.LEFT ) - self.dedup_action_id = tk.StringVar() - ttk.Label(dedup_controls, text="Action ID").pack(side=tk.LEFT, padx=(14, 4)) - ttk.Entry(dedup_controls, textvariable=self.dedup_action_id, width=8).pack(side=tk.LEFT) self.dedup_mode = tk.StringVar(value="quarantine") ttk.Combobox( dedup_controls, textvariable=self.dedup_mode, state="readonly", width=11, @@ -277,37 +369,88 @@ def _build_preservation(self, frame: ttk.Frame) -> None: ttk.Button(dedup_controls, text="Apply Safely", command=self._apply_dedup).pack( side=tk.LEFT ) - self.preservation_output = self._output(frame, 5, 3) + ttk.Button(dedup_controls, text="Restore Selected", command=self._restore_dedup).pack( + side=tk.LEFT, padx=4 + ) + self.dedup_tree = ttk.Treeview( + frame, columns=("duplicate", "size", "status"), show="headings", height=7 + ) + for column, label, width in ( + ("duplicate", "Duplicate file", 560), ("size", "Bytes", 100), + ("status", "Status", 100), + ): + self.dedup_tree.heading(column, text=label) + self.dedup_tree.column(column, width=width, stretch=column == "duplicate") + self.dedup_tree.grid(row=6, column=0, columnspan=3, sticky="nsew") + self.current_dedup_plan_id: int | None = None + self.preservation_output = self._output(frame, 7, 3) def _set_artwork(self) -> None: - self._show_output(self.preservation_output, self._run( - "Artwork preference", lambda: self.artwork.set_preference( - self.art_titleid.get(), self.art_path.get()))) + titleid = self.art_titleid.get() + path = self.art_path.get() + region = self.art_region.get() + language = self.art_language.get() + self._submit_output("Artwork preference", self.preservation_output, + lambda: self.artwork.set_preference( + titleid, path, region=region, language=language)) def _export_artwork(self) -> None: destination = filedialog.askdirectory(parent=self.root, title="Choose artwork export folder") if destination: - self._show_output(self.preservation_output, self._run( - "Artwork export", lambda: self.artwork.export(destination, "aurora"))) + preset = self.art_preset.get() + self._submit_output("Artwork export", self.preservation_output, + lambda: self.artwork.export(destination, preset)) def _audit_discs(self) -> None: - self._show_output(self.preservation_output, self._run( - "Disc set audit", self.preservation.audit_disc_sets)) + self._submit_output("Disc set audit", self.preservation_output, + self.preservation.audit_disc_sets) def _dedup(self) -> None: - self._show_output(self.preservation_output, self._run( - "Dedup preview", lambda: self.preservation.create_dedup_plan(self.dedup_root.get()))) + root = self.dedup_root.get() + def completed(result: Any) -> None: + self.current_dedup_plan_id = int(result["plan_id"]) + self._refresh_dedup_actions() + self._submit_output("Dedup preview", self.preservation_output, + lambda: self.preservation.create_dedup_plan(root), completed) + + def _refresh_dedup_actions(self) -> None: + self.dedup_tree.delete(*self.dedup_tree.get_children()) + for row in self.preservation.list_dedup_actions(self.current_dedup_plan_id): + status = row.get("recovery_status") or row["status"] + self.dedup_tree.insert("", tk.END, iid=str(row["id"]), values=( + row["duplicate_path"], row["size"], status, + )) def _apply_dedup(self) -> None: + selection = self.dedup_tree.selection() + if not selection: + messagebox.showinfo("Duplicate files", "Select a duplicate first.", parent=self.root) + return if not messagebox.askyesno( "Apply duplicate action", "Revalidate this duplicate and move its original into the recovery quarantine?", parent=self.root, ): return - self._show_output(self.preservation_output, self._run( - "Duplicate action", lambda: self.preservation.apply_dedup_action( - int(self.dedup_action_id.get()), self.dedup_mode.get()))) + action_id = int(selection[0]) + mode = self.dedup_mode.get() + self._submit_output("Duplicate action", self.preservation_output, + lambda: self.preservation.apply_dedup_action(action_id, mode), + lambda _result: self._refresh_dedup_actions()) + + def _restore_dedup(self) -> None: + selection = self.dedup_tree.selection() + if not selection: + messagebox.showinfo("Duplicate files", "Select a duplicate first.", parent=self.root) + return + action_id = int(selection[0]) + if not messagebox.askyesno( + "Restore duplicate", "Restore the quarantined original file?", parent=self.root + ): + return + self._submit_output("Duplicate restore", self.preservation_output, + lambda: self.preservation.restore_dedup_action(action_id), + lambda _result: self._refresh_dedup_actions()) def _build_storage(self, frame: ttk.Frame) -> None: frame.columnconfigure(1, weight=1) @@ -325,18 +468,24 @@ def _build_storage(self, frame: ttk.Frame) -> None: self.storage_output = self._output(frame, 4, 3) def _storage_audit(self) -> None: - self._show_output(self.storage_output, self._run( - "Storage audit", lambda: self.storage.audit_storage(self.storage_path.get()))) + path = self.storage_path.get() + self._submit_output("Storage audit", self.storage_output, + lambda: self.storage.audit_storage(path)) def _scan_xbox(self) -> None: - self._show_output(self.storage_output, self._run( - "Original Xbox scan", lambda: self.storage.scan_original_xbox(self.xbox_root.get()))) + root = self.xbox_root.get() + self._submit_output("Original Xbox scan", self.storage_output, + lambda: self.storage.scan_original_xbox(root)) def _build_plugins(self, frame: ttk.Frame) -> None: frame.columnconfigure(1, weight=1) frame.rowconfigure(2, weight=1) - self.plugin_root = tk.StringVar(value=str(executable_root() / "plugins")) - self._path_row(frame, 0, "Plugin folder", self.plugin_root, directory=True) + self.plugin_root = tk.StringVar(value=str(PLUGINS_DIR)) + ttk.Label(frame, text="Managed plugin folder").grid(row=0, column=0, sticky="w", pady=4) + ttk.Label(frame, textvariable=self.plugin_root).grid(row=0, column=1, sticky="w", pady=4) + ttk.Button(frame, text="Open Folder", command=lambda: open_path(PLUGINS_DIR)).grid( + row=0, column=2, padx=(8, 0), pady=4 + ) controls = ttk.Frame(frame) controls.grid(row=1, column=1, sticky="w", pady=8) ttk.Button(controls, text="Discover", command=self._discover_plugins).pack(side=tk.LEFT) @@ -348,36 +497,39 @@ def _build_plugins(self, frame: ttk.Frame) -> None: ) for column, label, width in (("id", "Plugin", 180), ("version", "Version", 90), ("enabled", "Enabled", 75), ("trusted", "Checksum", 90), - ("permissions", "Permissions", 280)): + ("permissions", "Requested access", 280)): self.plugin_tree.heading(column, text=label) self.plugin_tree.column(column, width=width) self.plugin_tree.grid(row=2, column=0, columnspan=3, sticky="nsew") self.plugin_rows: dict[str, dict[str, Any]] = {} def _discover_plugins(self) -> None: - rows = self._run("Plugin discovery", lambda: self.plugins.discover(self.plugin_root.get())) - if rows is None: - return - self.plugin_rows.clear() - self.plugin_tree.delete(*self.plugin_tree.get_children()) - for row in rows: - plugin_id = row["id"] - self.plugin_rows[plugin_id] = row - self.plugin_tree.insert("", tk.END, iid=plugin_id, values=( - row.get("name", plugin_id), row.get("version", ""), - "Yes" if row.get("enabled") else "No", - "Trusted" if row.get("trusted") else "Review", - ", ".join(row.get("permissions", [])), - )) + plugin_root = self.plugin_root.get() + def completed(rows: Any) -> None: + self.plugin_rows.clear() + self.plugin_tree.delete(*self.plugin_tree.get_children()) + for row in rows: + plugin_id = row["id"] + self.plugin_rows[plugin_id] = row + self.plugin_tree.insert("", tk.END, iid=plugin_id, values=( + row.get("name", plugin_id), row.get("version", ""), + "Yes" if row.get("enabled") else "No", + "Trusted" if row.get("trusted") else "Review", + ", ".join(row.get("permissions", [])), + )) + self._submit("Plugin discovery", lambda: self.plugins.discover(plugin_root), completed) def _install_plugin(self) -> None: archive = filedialog.askopenfilename( parent=self.root, title="Choose a plugin ZIP", filetypes=(("ZIP archives", "*.zip"),) ) if archive: - self._run("Plugin installation", lambda: self.plugins.install_package( - archive, self.plugin_root.get())) - self._discover_plugins() + plugin_root = self.plugin_root.get() + self._submit( + "Plugin installation", + lambda: self.plugins.install_package(archive, plugin_root), + lambda _result: self._discover_plugins(), + ) def _set_plugin(self, enabled: bool) -> None: selection = self.plugin_tree.selection() @@ -414,9 +566,8 @@ def _build_recovery(self, frame: ttk.Frame) -> None: def _scan_recovery(self) -> None: roots = (DOWNLOADS_DIR, PROFILE_BACKUPS_DIR, DATABASE_PATH.parent) - result = self._run("Recovery scan", lambda: self.recovery.scan(roots)) - if result is not None: - self._refresh_recovery() + self._submit("Recovery scan", lambda: self.recovery.scan(roots), + lambda _result: self._refresh_recovery()) def _refresh_recovery(self) -> None: self.recovery_tree.delete(*self.recovery_tree.get_children()) @@ -430,10 +581,10 @@ def _recover_selected(self) -> None: if not selection: messagebox.showinfo("Recovery", "Select a recovery item first.", parent=self.root) return - result = self._run("Recovery", lambda: self.recovery.recover(int(selection[0]))) - if result: + def completed(result: Any) -> None: messagebox.showinfo("Recovery", result["action"], parent=self.root) self._refresh_recovery() + self._submit("Recovery", lambda: self.recovery.recover(int(selection[0])), completed) def _build_compatibility(self, frame: ttk.Frame) -> None: frame.columnconfigure(1, weight=1) @@ -457,11 +608,11 @@ def _build_compatibility(self, frame: ttk.Frame) -> None: self.compat_output = self._output(frame, 5, 2) def _probe(self) -> None: + dashboard = self.compat_dashboard.get() target = FtpTarget(self.compat_host.get(), username=self.compat_user.get(), password=self.compat_password.get()) - self._show_output(self.compat_output, self._run( - "Dashboard probe", lambda: self.compatibility.probe( - self.compat_dashboard.get(), target))) + self._submit_output("Dashboard probe", self.compat_output, + lambda: self.compatibility.probe(dashboard, target)) def _build_accessibility(self, frame: ttk.Frame) -> None: values = self.accessibility.get() @@ -513,12 +664,72 @@ def _show_output(widget: tk.Text, value: Any) -> None: widget.insert(tk.END, json.dumps(value, indent=2, default=str)) widget.configure(state=tk.DISABLED) - def _run(self, title: str, callback: Callable[[], Any]) -> Any: - try: - return callback() - except Exception as exc: - messagebox.showerror(title, str(exc), parent=self.root) - return None + def _submit_output( + self, + title: str, + output: tk.Text, + callback: Callable[[], Any], + after: Callable[[Any], None] | None = None, + ) -> None: + def completed(value: Any) -> None: + self._show_output(output, value) + if after is not None: + after(value) + self._submit(title, callback, completed) + + def _submit( + self, + title: str, + callback: Callable[[], Any], + completed: Callable[[Any], None] | None = None, + ) -> None: + if self.active_task is not None and not self.active_task.done(): + messagebox.showinfo( + "Task in progress", "Wait for the current Community Hub task to finish.", + parent=self.root, + ) + return + future = self.task_executor.submit(callback) + self.active_task = future + self.task_status.set(f"Running: {title}") + self.cancel_task_button.configure(state=tk.NORMAL) + future.add_done_callback( + lambda item: self.task_events.put((title, item, completed)) + ) + + def _cancel_task(self) -> None: + if self.active_task is None or self.active_task.done(): + return + if self.active_task.cancel(): + self.task_status.set("Pending task cancelled") + else: + self.task_status.set("The running operation will finish safely") + self.cancel_task_button.configure(state=tk.DISABLED) + + def _poll_tasks(self) -> None: + if not self.notebook.winfo_exists(): + self.task_executor.shutdown(wait=False, cancel_futures=True) + return + while True: + try: + title, future, completed = self.task_events.get_nowait() + except queue.Empty: + break + self.active_task = None + self.cancel_task_button.configure(state=tk.DISABLED) + if future.cancelled(): + self.task_status.set(f"Cancelled: {title}") + continue + try: + result = future.result() + except Exception as exc: + self.task_status.set(f"Failed: {title}") + messagebox.showerror(title, str(exc), parent=self.root) + else: + self.task_status.set(f"Completed: {title}") + if completed is not None: + completed(result) + self.root.after(100, self._poll_tasks) @staticmethod def _fill_tree(tree: ttk.Treeview, rows: list[dict], values: Callable) -> None: diff --git a/community_services.py b/community_services.py index d3414aa..aa92215 100644 --- a/community_services.py +++ b/community_services.py @@ -18,7 +18,7 @@ from PIL import Image from app_paths import DATABASE_PATH -from backup_manager import FtpTarget, inspect_stfs, inspect_xbe +from backup_manager import FtpTarget, inspect_stfs, inspect_xbe, list_stfs_entries from console_sync import ConsoleSyncService from database_migrations import ensure_application_schema from plugins import PluginManifest @@ -173,6 +173,31 @@ def list_plans(self) -> list[dict[str, Any]]: ).fetchall() return [dict(row) for row in rows] + def list_snapshots(self) -> list[dict[str, Any]]: + with self.connect() as connection: + rows = connection.execute( + """ + SELECT s.*, COUNT(i.id) item_count + FROM console_inventory_snapshots s + LEFT JOIN console_inventory_items i ON i.snapshot_id=s.id + WHERE s.status='completed' + GROUP BY s.id ORDER BY s.captured_at DESC + """ + ).fetchall() + return [dict(row) for row in rows] + + def list_ftp_targets(self) -> list[dict[str, Any]]: + with self.connect() as connection: + exists = connection.execute( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name='backup_targets'" + ).fetchone() + if exists is None: + return [] + rows = connection.execute( + "SELECT id, name, location FROM backup_targets WHERE kind='ftp' ORDER BY name" + ).fetchall() + return [dict(row) for row in rows] + def queue_uploads(self, plan_id: int, target_id: int | None = None) -> dict[str, Any]: """Queue selected upload actions after revalidating their local files.""" with self.connect() as connection: @@ -233,10 +258,18 @@ def inspect(self, package_path: str | Path) -> dict[str, Any]: result = asdict(package) result["path"] = str(package.path) result["sha256"] = _sha256(package.path) + try: + entries = list_stfs_entries(package.path) + except Exception as exc: + result["file_table"] = [] + result["file_table_status"] = f"unavailable: {exc}" + else: + result["file_table"] = [asdict(entry) for entry in entries] + result["file_table_status"] = "read-only bounded inventory" result["mutation_ready"] = False result["required_before_rebuild"] = [ - "complete file-table extraction", "block/hash tree verification", - "rehash", "signature", "post-build verification", + "complete extraction and block/hash tree verification", "rehash", "signature", + "post-build verification", ] return result @@ -453,17 +486,113 @@ def apply_dedup_action(self, action_id: int, mode: str = "quarantine") -> dict[s except Exception: quarantined.replace(duplicate) raise - with self.connect() as connection: - connection.execute( - "UPDATE dedup_actions SET action=?, status='completed' WHERE id=?", - (mode, action_id), - ) + try: + with self.connect() as connection: + connection.execute( + "UPDATE dedup_actions SET action=?, status='completed' WHERE id=?", + (mode, action_id), + ) + connection.execute( + """ + INSERT INTO dedup_recovery_records( + action_id, original_path, quarantine_path, keeper_path, + mode, sha256, created_at, status + ) VALUES (?, ?, ?, ?, ?, ?, ?, 'quarantined') + """, + (action_id, str(duplicate), str(quarantined), str(keeper), + mode, expected, utc_now()), + ) + except Exception: + if mode == "hardlink" and duplicate.exists(): + duplicate.unlink() + if quarantined.exists() and not duplicate.exists(): + quarantined.replace(duplicate) + raise return {"action_id": action_id, "mode": mode, "keeper": str(keeper), "duplicate": str(duplicate), "quarantine": str(quarantined)} + def list_dedup_actions(self, plan_id: int | None = None) -> list[dict[str, Any]]: + with self.connect() as connection: + selected = plan_id + if selected is None: + row = connection.execute( + "SELECT id FROM dedup_plans ORDER BY id DESC LIMIT 1" + ).fetchone() + if row is None: + return [] + selected = int(row["id"]) + rows = connection.execute( + """ + SELECT a.*, r.status recovery_status + FROM dedup_actions a + LEFT JOIN dedup_recovery_records r ON r.action_id=a.id + WHERE a.plan_id=? ORDER BY a.size DESC, a.id + """, + (selected,), + ).fetchall() + return [dict(row) for row in rows] + + def restore_dedup_action(self, action_id: int) -> dict[str, Any]: + """Restore a quarantined duplicate after validating every involved path.""" + with self.connect() as connection: + row = connection.execute( + """ + SELECT r.*, p.root FROM dedup_recovery_records r + JOIN dedup_actions a ON a.id=r.action_id + JOIN dedup_plans p ON p.id=a.plan_id + WHERE r.action_id=? + """, + (action_id,), + ).fetchone() + if row is None: + raise KeyError(action_id) + if row["status"] != "quarantined": + raise ValueError("This duplicate is not waiting in recovery quarantine") + root = Path(row["root"]).resolve() + original = Path(row["original_path"]).resolve() + quarantine = Path(row["quarantine_path"]).resolve() + keeper = Path(row["keeper_path"]).resolve() + original.relative_to(root) + quarantine.relative_to(root) + keeper.relative_to(root) + expected = row["sha256"] + if not quarantine.is_file() or _sha256(quarantine) != expected: + raise ValueError("The quarantined file is missing or changed") + if original.exists(): + if row["mode"] != "hardlink": + raise FileExistsError(original) + if not original.is_file() or _sha256(original) != expected: + raise ValueError("The replacement file changed; restore was refused") + try: + if not original.samefile(keeper): + raise ValueError("The replacement is not the expected hardlink") + except OSError as exc: + raise ValueError("The replacement hardlink could not be verified") from exc + original.unlink() + original.parent.mkdir(parents=True, exist_ok=True) + quarantine.replace(original) + try: + with self.connect() as connection: + connection.execute( + """UPDATE dedup_recovery_records + SET status='restored', restored_at=? WHERE action_id=?""", + (utc_now(), action_id), + ) + connection.execute( + "UPDATE dedup_actions SET status='restored' WHERE id=?", + (action_id,), + ) + except Exception: + original.replace(quarantine) + if row["mode"] == "hardlink": + original.hardlink_to(keeper) + raise + return {"action_id": action_id, "restored": str(original)} + class StorageAndXboxService(CommunityRepository): - FATX_OFFSETS = (0, 0x80000, 0x130EB0000, 0x20000000) + FATX_OFFSETS = (0, 0x7FF000, 0x10C080000, 0x118EB0000, 0x120EB0000, + 0x130EB0000, 0x8000400, 0x8115200, 0x12000400, 0x20000000) def audit_storage(self, source_path: str | Path) -> dict[str, Any]: source = Path(source_path).expanduser().resolve() @@ -471,16 +600,46 @@ def audit_storage(self, source_path: str | Path) -> dict[str, Any]: raise FileNotFoundError(source) filesystem = "mounted-filesystem" if source.is_dir() else "unknown-image" details: dict[str, Any] = {"size": source.stat().st_size} + if source.is_dir(): + usb_parts = sorted( + path for path in (source / "Xbox360").glob("Data[0-9][0-9][0-9][0-9]") + if path.is_file() + ) + if usb_parts: + filesystem = "Xbox 360 USB container" + details["container_parts"] = len(usb_parts) + details["container_size"] = sum(path.stat().st_size for path in usb_parts) + details["container_files"] = [path.name for path in usb_parts] if source.is_file(): + partitions = [] + source_size = source.stat().st_size with source.open("rb") as handle: for offset in self.FATX_OFFSETS: - if offset + 4 > source.stat().st_size: + if offset + 16 > source_size: continue handle.seek(offset) - if handle.read(4) == b"XTAF": - filesystem = "FATX" - details["signature_offset"] = offset - break + header = handle.read(16) + if header[:4] != b"XTAF": + continue + sectors_per_cluster = int.from_bytes(header[8:12], "big") + root_cluster = int.from_bytes(header[12:16], "big") + valid_cluster = ( + sectors_per_cluster > 0 + and sectors_per_cluster <= 0x10000 + and sectors_per_cluster & (sectors_per_cluster - 1) == 0 + ) + partitions.append({ + "offset": offset, + "partition_id": f"{int.from_bytes(header[4:8], 'big'):08X}", + "sectors_per_cluster": sectors_per_cluster, + "cluster_size": sectors_per_cluster * 512, + "root_directory_cluster": root_cluster, + "header_valid": valid_cluster and root_cluster > 0, + }) + if partitions: + filesystem = "FATX" + details["partitions"] = partitions + details["signature_offset"] = partitions[0]["offset"] status = "recognized" if filesystem != "unknown-image" else "unrecognized" with self.connect() as connection: cursor = connection.execute( @@ -505,19 +664,34 @@ def scan_original_xbox(self, root: str | Path) -> list[dict[str, Any]]: package = inspect_xbe(path) except (OSError, ValueError): continue - item = {"titleid": package.title_id, "title_name": package.title_name, - "xbe_path": str(package.path), "size": package.size} + region_names = [ + name for flag, name in ( + (0x1, "North America"), (0x2, "Japan"), + (0x4, "Rest of World"), (0x80000000, "Manufacturing"), + ) if package.region_flags & flag + ] + regions = ", ".join(region_names) or "Unknown" + item = { + "titleid": package.title_id, "title_name": package.title_name, + "xbe_path": str(package.path), "size": package.size, + "region_flags": f"0x{package.region_flags:08X}", + "regions": region_names, "version": package.version, + "disc_number": package.disc_number, + "allowed_media": f"0x{package.allowed_media:08X}", + } connection.execute( """ INSERT INTO original_xbox_records( - titleid, title_name, xbe_path, metadata_json, scanned_at - ) VALUES (?, ?, ?, ?, ?) + titleid, title_name, xbe_path, region_flags, version, + metadata_json, scanned_at + ) VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT(xbe_path) DO UPDATE SET titleid=excluded.titleid, - title_name=excluded.title_name, metadata_json=excluded.metadata_json, + title_name=excluded.title_name, region_flags=excluded.region_flags, + version=excluded.version, metadata_json=excluded.metadata_json, scanned_at=excluded.scanned_at """, - (package.title_id, package.title_name, str(package.path), - json.dumps(item, sort_keys=True), utc_now()), + (package.title_id, package.title_name, str(package.path), regions, + str(package.version), json.dumps(item, sort_keys=True), utc_now()), ) records.append(item) return records diff --git a/database_migrations.py b/database_migrations.py index 667d129..7abd6f6 100644 --- a/database_migrations.py +++ b/database_migrations.py @@ -8,7 +8,7 @@ from pathlib import Path -SCHEMA_VERSION = 8 +SCHEMA_VERSION = 9 def _now() -> str: @@ -69,6 +69,7 @@ def ensure_application_schema(connection: sqlite3.Connection) -> int: (6, "profile and save management", _migration_profiles_and_saves), (7, "profile intelligence and knowledge controls", _migration_roadmap), (8, "community roadmap workspaces", _migration_community_roadmap), + (9, "hardening and plugin runtime", _migration_hardening), ) for version, name, migration in migrations: if version in applied: @@ -702,3 +703,38 @@ def _migration_community_roadmap(connection: sqlite3.Connection) -> None: ); """ ) + + +def _migration_hardening(connection: sqlite3.Connection) -> None: + """Add runtime audits and reversible-action metadata.""" + connection.executescript( + """ + CREATE TABLE IF NOT EXISTS plugin_collection_runs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + plugin_id TEXT NOT NULL, + titleid TEXT NOT NULL, + status TEXT NOT NULL, + started_at TEXT NOT NULL, + completed_at TEXT, + result_json TEXT, + error_message TEXT, + FOREIGN KEY(plugin_id) REFERENCES plugin_states(plugin_id) + ); + CREATE INDEX IF NOT EXISTS idx_plugin_collection_runs_lookup + ON plugin_collection_runs(plugin_id, titleid, started_at); + + CREATE TABLE IF NOT EXISTS dedup_recovery_records ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + action_id INTEGER NOT NULL UNIQUE, + original_path TEXT NOT NULL, + quarantine_path TEXT NOT NULL, + keeper_path TEXT NOT NULL, + mode TEXT NOT NULL, + sha256 TEXT NOT NULL, + created_at TEXT NOT NULL, + restored_at TEXT, + status TEXT NOT NULL DEFAULT 'quarantined', + FOREIGN KEY(action_id) REFERENCES dedup_actions(id) + ); + """ + ) diff --git a/main.py b/main.py index ea17c61..66acf79 100644 --- a/main.py +++ b/main.py @@ -24,11 +24,13 @@ CLI_LOG_PATH, CONFIG_PATH, DOWNLOADS_DIR, + PLUGINS_DIR, ensure_app_dirs, ensure_user_titleids_file, ) from database import DatabaseManager -from plugins import PluginManager +from knowledge_base import is_unknown +from plugins import PluginManager, load_enabled_plugin_configuration from resume import ResumableDownloader logger = logging.getLogger(__name__) @@ -155,12 +157,20 @@ def __init__( self, config: Config, database: Optional[DatabaseManager] = None, + plugin_manager: Optional[PluginManager] = None, ): self.config = config self.rate_limiter = RateLimiter(config.rate_limit) self.session = self._create_session() self.db = database or DatabaseManager() - self.plugin_manager = PluginManager() # Initialize plugin system + if plugin_manager is None: + enabled, trusted = load_enabled_plugin_configuration( + self.db.db_path, PLUGINS_DIR + ) + plugin_manager = PluginManager( + str(PLUGINS_DIR), enabled_plugins=enabled, trusted_hashes=trusted + ) + self.plugin_manager = plugin_manager self.downloader = ResumableDownloader(self.session, config.timeout, config.bandwidth_limit) self._test_connection() @@ -416,12 +426,121 @@ def collect_metadata(self, titleid: str) -> bool: # Batch insert updates if updates_batch: self.db.batch_insert_updates(updates_batch) + + self._collect_plugin_metadata(validated_titleid) # Update database with scrape info self.db.update_scrape_info(validated_titleid) logger.info(f"[OK] Collected metadata for TitleID: {titleid}") return True + + def _collect_plugin_metadata(self, titleid: str) -> None: + """Run explicitly enabled plugins and store bounded, source-labelled results.""" + for result in self.plugin_manager.collect_enabled(titleid): + plugin_id = str(result["plugin_id"]) + now = datetime.now().isoformat() + status = str(result["status"]) + data = result.get("data") if status == "completed" else None + error = str(result.get("error") or "") + try: + if isinstance(data, dict): + self._store_plugin_metadata(titleid, plugin_id, data) + except Exception as exc: + status = "failed" + error = str(exc) + logger.exception("Could not store plugin result from %s", plugin_id) + with self.db.get_connection() as connection: + connection.execute( + """ + INSERT INTO plugin_collection_runs( + plugin_id, titleid, status, started_at, completed_at, + result_json, error_message + ) VALUES (?, ?, ?, ?, ?, ?, ?) + """, + ( + plugin_id, titleid, status, now, datetime.now().isoformat(), + json.dumps(data, sort_keys=True, default=str) if data is not None else None, + error or None, + ), + ) + + def _store_plugin_metadata( + self, titleid: str, plugin_id: str, data: Dict[str, Any] + ) -> None: + title = str(data.get("title") or data.get("name") or "").strip()[:500] + publisher = str(data.get("publisher") or "").strip()[:500] + with self.db.get_connection() as connection: + row = connection.execute( + "SELECT name, publisher, metadata FROM titleids WHERE titleid=?", (titleid,) + ).fetchone() + metadata: Dict[str, Any] = {} + if row and row["metadata"]: + try: + metadata = json.loads(row["metadata"]) + except json.JSONDecodeError: + metadata = {} + current_name = row["name"] if row else None + current_publisher = row["publisher"] if row else None + if title and is_unknown(current_name): + current_name = title + metadata["title_source"] = f"Plugin: {plugin_id}" + if publisher and is_unknown(current_publisher): + current_publisher = publisher + metadata["publisher_source"] = f"Plugin: {plugin_id}" + plugin_sources = metadata.get("plugin_sources") + if not isinstance(plugin_sources, dict): + plugin_sources = {} + metadata["plugin_sources"] = plugin_sources + plugin_sources[plugin_id] = datetime.now().isoformat() + connection.execute( + "UPDATE titleids SET name=?, publisher=?, metadata=? WHERE titleid=?", + (current_name, current_publisher, json.dumps(metadata, sort_keys=True), titleid), + ) + self.db._update_search_index( + connection, titleid, current_name, current_publisher, metadata + ) + + cover_items = data.get("covers", []) + if not isinstance(cover_items, list): + raise TypeError("Plugin covers must be a list") + covers = [] + for item in cover_items[:200]: + if not isinstance(item, dict): + continue + url = str(item.get("cover_url") or item.get("url") or "").strip() + if urlparse(url).scheme not in {"http", "https"}: + continue + covers.append({ + "titleid": titleid, + "cover_url": url, + "cover_type": str(item.get("cover_type") or item.get("type") or "plugin")[:100], + "status": "pending", + "metadata": {**item, "plugin_id": plugin_id}, + }) + if covers: + self.db.batch_insert_covers(covers) + + update_items = data.get("updates", []) + if not isinstance(update_items, list): + raise TypeError("Plugin updates must be a list") + updates = [] + for item in update_items[:500]: + if not isinstance(item, dict): + continue + url = str(item.get("download_url") or item.get("url") or "").strip() + if urlparse(url).scheme not in {"http", "https"}: + continue + updates.append({ + "titleid": titleid, + "media_id": str(item.get("media_id") or item.get("MediaID") or "")[:100], + "version": str(item.get("version") or item.get("Version") or "unknown")[:100], + "download_url": url, + "status": "pending", + "metadata": {**item, "plugin_id": plugin_id}, + }) + if updates: + self.db.batch_insert_updates(updates) def process_titleid(self, titleid: str) -> bool: """Process a single TitleID - download covers and updates""" @@ -449,6 +568,10 @@ def process_titleid(self, titleid: str) -> bool: with open(output_dir / 'updates_data.json', 'w') as f: json.dump(updates_data, f, indent=2) self._download_updates(validated_titleid, updates_data, output_dir) + + self.db.add_titleid(validated_titleid) + self._collect_plugin_metadata(validated_titleid) + self.db.update_scrape_info(validated_titleid) logger.info(f"[OK] Completed downloads for TitleID: {titleid}") return True @@ -822,6 +945,22 @@ def main(): choices=['redump', 'no-intro'], help='Source type for --import-dat' ) + parser.add_argument('--search-all', type=str, + help='Search games, knowledge, profiles, saves, files, and tools') + parser.add_argument('--extract-knowledge', action='store_true', + help='Extract structured records from locally cached wiki documents') + parser.add_argument('--audit-storage', type=str, + help='Inspect a mounted storage path or image read-only') + parser.add_argument('--scan-original-xbox', type=str, + help='Index original Xbox default.xbe files below a folder') + parser.add_argument('--dedup-preview', type=str, + help='Create a checksum-based duplicate preview for a folder') + parser.add_argument('--dedup-apply', type=int, + help='Apply one previewed duplicate action by ID') + parser.add_argument('--dedup-restore', type=int, + help='Restore one quarantined duplicate action by ID') + parser.add_argument('--dedup-mode', choices=['quarantine', 'hardlink'], + default='quarantine', help='Action used with --dedup-apply') parser.add_argument( '--scan-backups', type=str, @@ -1012,6 +1151,51 @@ def main(): logger.error(f"DAT import failed: {e}") sys.exit(1) + if ( + args.search_all + or args.extract_knowledge + or args.audit_storage + or args.scan_original_xbox + or args.dedup_preview + or args.dedup_apply is not None + or args.dedup_restore is not None + ): + try: + from community_services import PreservationPlanningService, StorageAndXboxService + from structured_knowledge import StructuredKnowledgeService + from unified_search import UnifiedSearchService + + results: Dict[str, Any] = {} + if args.search_all: + results["search"] = UnifiedSearchService().search(args.search_all) + if args.extract_knowledge: + results["knowledge"] = StructuredKnowledgeService().extract_cached_documents() + if args.audit_storage: + results["storage"] = StorageAndXboxService().audit_storage(args.audit_storage) + if args.scan_original_xbox: + results["original_xbox"] = StorageAndXboxService().scan_original_xbox( + args.scan_original_xbox + ) + preservation = PreservationPlanningService() + if args.dedup_preview: + results["dedup_preview"] = preservation.create_dedup_plan(args.dedup_preview) + results["dedup_actions"] = preservation.list_dedup_actions( + results["dedup_preview"]["plan_id"] + ) + if args.dedup_apply is not None: + results["dedup_apply"] = preservation.apply_dedup_action( + args.dedup_apply, args.dedup_mode + ) + if args.dedup_restore is not None: + results["dedup_restore"] = preservation.restore_dedup_action( + args.dedup_restore + ) + print(json.dumps(results, indent=2, default=str)) + sys.exit(0) + except Exception as e: + logger.error("Community operation failed: %s", e) + sys.exit(1) + if ( args.analyze_collection or args.aurora_db diff --git a/modern_gui.py b/modern_gui.py index ab89eaa..d8b9734 100644 --- a/modern_gui.py +++ b/modern_gui.py @@ -1100,11 +1100,40 @@ def show_knowledge(self) -> None: def show_community_hub(self, focus_search: bool = False) -> None: self._clear_content() self.community_hub_page = CommunityHubPage( - self.root, self.content, self._page_header + self.root, self.content, self._page_header, self._navigate_search_result ) if focus_search: self.root.after_idle(self.community_hub_page.focus_search) + def _navigate_search_result(self, result: dict[str, Any]) -> None: + """Open a unified-search result in its closest native workspace.""" + category = str(result.get("category", "")).casefold() + target = str(result.get("target", "")) + identifier = str(result.get("identifier", "")) + if category == "game": + self.current_game = identifier + self.show_library() + return + if target.startswith("file:"): + path = Path(target.removeprefix("file:")) + _open_path(path.parent if path.is_file() else path) + return + if category in {"profile", "save", "achievement"}: + self.show_profiles() + self.profile_save_page.search_var.set(result.get("title", identifier)) + self.profile_save_page.refresh() + return + if category == "knowledge": + self.show_knowledge() + self.knowledge_page.search_var.set(result.get("title", "")) + self.knowledge_page.refresh_results() + return + if target.startswith("structured:"): + hub = self.community_hub_page + hub.notebook.select(1) + hub.knowledge_type.set(category) + hub._refresh_knowledge() + def show_health(self) -> None: self._clear_content() self._page_header( diff --git a/packaging/linux/install.sh b/packaging/linux/install.sh index 78acc47..d77f616 100755 --- a/packaging/linux/install.sh +++ b/packaging/linux/install.sh @@ -13,8 +13,7 @@ mkdir -p "$APP_DIR" "$BIN_DIR" "$APPLICATIONS_DIR" "$ICON_DIR" "$METAINFO_DIR" install -m 0755 "$PACKAGE_DIR/unityscraper" "$APP_DIR/unityscraper" install -m 0755 "$PACKAGE_DIR/uninstall.sh" "$APP_DIR/uninstall.sh" -install -m 0644 "$PACKAGE_DIR/README.md" "$PACKAGE_DIR/CHANGELOG.md" \ - "$PACKAGE_DIR/LICENSE" "$APP_DIR/" +install -m 0644 "$PACKAGE_DIR/"*.md "$PACKAGE_DIR/LICENSE" "$APP_DIR/" install -m 0644 "$PACKAGE_DIR/unityscraper.png" \ "$ICON_DIR/io.github.trapemall.UnityScraper.png" install -m 0644 "$PACKAGE_DIR/io.github.trapemall.UnityScraper.metainfo.xml" \ diff --git a/plugins.py b/plugins.py index ed44c14..68734f0 100644 --- a/plugins.py +++ b/plugins.py @@ -5,12 +5,16 @@ import json import logging +import hashlib +import sqlite3 +from contextlib import closing from dataclasses import dataclass from abc import ABC, abstractmethod from pathlib import Path -from typing import List, Dict, Any, Optional +from typing import List, Dict, Any, Optional, Mapping import importlib.util import sys +import threading logger = logging.getLogger(__name__) PLUGIN_API_VERSION = 1 @@ -47,6 +51,48 @@ def load(cls, path: Path) -> "PluginManifest": return manifest +def file_sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def load_enabled_plugin_configuration( + db_path: str | Path, plugin_dir: str | Path +) -> tuple[list[str], dict[str, str]]: + """Return enabled plugin IDs whose entrypoint still matches its trusted hash.""" + database = Path(db_path) + root = Path(plugin_dir) + if not database.is_file() or not root.is_dir(): + return [], {} + try: + with closing(sqlite3.connect(database)) as connection: + rows = connection.execute( + "SELECT plugin_id, trusted_sha256 FROM plugin_states WHERE enabled=1" + ).fetchall() + except sqlite3.Error: + return [], {} + enabled: list[str] = [] + trusted: dict[str, str] = {} + for plugin_id, expected_hash in rows: + try: + manifest = PluginManifest.load(root / plugin_id / "plugin.json") + entry = root / plugin_id / manifest.entrypoint + actual_hash = file_sha256(entry) + except (OSError, ValueError, KeyError, json.JSONDecodeError) as exc: + logger.warning("Enabled plugin %s could not be validated: %s", plugin_id, exc) + continue + expected = str(expected_hash or "").casefold() + if manifest.plugin_id != plugin_id or not expected or actual_hash.casefold() != expected: + logger.warning("Enabled plugin %s changed after approval and was not loaded", plugin_id) + continue + enabled.append(plugin_id) + trusted[plugin_id] = actual_hash + return enabled, trusted + + class MetadataCollectorPlugin(ABC): """Base class for custom metadata collector plugins""" @@ -78,12 +124,16 @@ def __init__( self, plugin_dir: str = "plugins", enabled_plugins: Optional[List[str]] = None, + trusted_hashes: Optional[Mapping[str, str]] = None, allow_legacy: bool = False, ): self.plugin_dir = Path(plugin_dir) self.plugins: Dict[str, MetadataCollectorPlugin] = {} self.manifests: Dict[str, PluginManifest] = {} + self.plugin_ids_by_name: Dict[str, str] = {} + self.plugin_locks: Dict[str, threading.Lock] = {} self.enabled_plugins = set(enabled_plugins or []) + self.trusted_hashes = dict(trusted_hashes or {}) self.allow_legacy = allow_legacy self._load_plugins() @@ -98,7 +148,11 @@ def _load_plugins(self): manifest = PluginManifest.load(manifest_path) self.manifests[manifest.plugin_id] = manifest if manifest.plugin_id in self.enabled_plugins: - self._load_plugin_file(manifest_path.parent / manifest.entrypoint, manifest) + entry = manifest_path.parent / manifest.entrypoint + expected = self.trusted_hashes.get(manifest.plugin_id) + if not expected or file_sha256(entry).casefold() != expected.casefold(): + raise ValueError("Plugin entrypoint does not match its approved checksum") + self._load_plugin_file(entry, manifest) except Exception as e: logger.warning(f"Failed to load plugin {manifest_path}: {e}") if self.allow_legacy: @@ -110,10 +164,14 @@ def _load_plugin_file( self, file_path: Path, manifest: Optional[PluginManifest] = None ): """Load a single plugin file""" - spec = importlib.util.spec_from_file_location(file_path.stem, file_path) + module_name = ( + "unityscraper_plugin_" + manifest.plugin_id.replace(".", "_").replace("-", "_") + if manifest else file_path.stem + ) + spec = importlib.util.spec_from_file_location(module_name, file_path) if spec and spec.loader: module = importlib.util.module_from_spec(spec) - sys.modules[file_path.stem] = module + sys.modules[module_name] = module spec.loader.exec_module(module) # Find and register MetadataCollectorPlugin subclasses @@ -127,7 +185,9 @@ def _load_plugin_file( if manifest: instance.name = manifest.name instance.version = manifest.version + self.plugin_ids_by_name[instance.name] = manifest.plugin_id self.plugins[instance.name] = instance + self.plugin_locks[instance.name] = threading.Lock() logger.info(f"Loaded plugin: {instance.name} v{instance.version}") def get_plugin(self, name: str) -> Optional[MetadataCollectorPlugin]: @@ -170,6 +230,32 @@ def collect_from_plugin(self, plugin_name: str, titleid: str) -> Optional[Dict[s logger.error(f"Plugin {plugin_name} failed: {e}") return None + def collect_enabled(self, titleid: str) -> List[Dict[str, Any]]: + """Run enabled collectors independently and retain success/failure details.""" + results: List[Dict[str, Any]] = [] + for name, plugin in self.plugins.items(): + plugin_id = self.plugin_ids_by_name.get(name, name) + try: + with self.plugin_locks[name]: + if not plugin.validate_titleid(titleid): + results.append({"plugin_id": plugin_id, "name": name, + "status": "skipped"}) + continue + data = plugin.collect(titleid) + if not isinstance(data, dict): + raise TypeError("Plugin collect() must return a dictionary") + encoded = json.dumps(data, default=str) + if len(encoded.encode("utf-8")) > 2 * 1024 * 1024: + raise ValueError("Plugin result exceeds the 2 MiB safety limit") + except Exception as exc: + logger.exception("Plugin %s failed for %s", plugin_id, titleid) + results.append({"plugin_id": plugin_id, "name": name, + "status": "failed", "error": str(exc)}) + else: + results.append({"plugin_id": plugin_id, "name": name, + "status": "completed", "data": data}) + return results + # Example plugin template (to be saved in plugins/example.py) EXAMPLE_PLUGIN_TEMPLATE = ''' diff --git a/tests.py b/tests.py index 64883d3..2eafcdf 100644 --- a/tests.py +++ b/tests.py @@ -50,6 +50,7 @@ import_stfs_zip, inspect_stfs, inspect_xbe, + list_stfs_entries, package_destination, scan_local_target, ) @@ -1123,6 +1124,31 @@ def test_inspect_stfs_reads_profile_ownership_fields(self): self.assertEqual(package.device_id, "11" * 20) self.assertEqual(package.save_game_id, "12345678") + def test_stfs_file_table_is_inventoried_read_only(self): + payload = bytearray(0xC000) + payload[:4] = b"LIVE" + payload[0x340:0x344] = (0xA000).to_bytes(4, "big") + payload[0x344:0x348] = (1).to_bytes(4, "big") + payload[0x360:0x364] = bytes.fromhex("53510804") + payload[0x379] = 0x24 + payload[0x37B] = 1 + payload[0x37C:0x37E] = (1).to_bytes(2, "big") + name = b"savegame.dat" + entry = 0xB000 + payload[entry:entry + len(name)] = name + payload[entry + 0x28] = len(name) | 0x40 + payload[entry + 0x29:entry + 0x2C] = (1).to_bytes(3, "little") + payload[entry + 0x2F:entry + 0x32] = (1).to_bytes(3, "little") + payload[entry + 0x32:entry + 0x34] = (0xFFFF).to_bytes(2, "big") + payload[entry + 0x34:entry + 0x38] = (123).to_bytes(4, "big") + package_path = self.temp_dir / "listing.stfs" + package_path.write_bytes(payload) + + entries = list_stfs_entries(package_path) + self.assertEqual(entries[0].path, "savegame.dat") + self.assertEqual(entries[0].size, 123) + self.assertTrue(entries[0].consecutive) + def test_rejects_unknown_stfs_content_type(self): with self.assertRaises(InvalidPackageError): inspect_stfs(self._stfs(content_type=0xDEADBEEF)) @@ -1215,10 +1241,22 @@ def test_inspect_xbe_certificate(self): title = "Original Game".encode("utf-16-le") start = certificate_offset + 0xC data[start:start + len(title)] = title + data[certificate_offset + 0x9C:certificate_offset + 0xA0] = (0x21).to_bytes( + 4, "little" + ) + data[certificate_offset + 0xA0:certificate_offset + 0xA4] = (0x5).to_bytes( + 4, "little" + ) + data[certificate_offset + 0xA8:certificate_offset + 0xAC] = (2).to_bytes(4, "little") + data[certificate_offset + 0xAC:certificate_offset + 0xB0] = (7).to_bytes(4, "little") path.write_bytes(data) package = inspect_xbe(path) self.assertEqual(package.title_id, "4D530064") self.assertEqual(package.title_name, "Original Game") + self.assertEqual(package.allowed_media, 0x21) + self.assertEqual(package.region_flags, 0x5) + self.assertEqual(package.disc_number, 2) + self.assertEqual(package.version, 7) def test_backup_schema_is_additive_and_omits_passwords(self): repository = BackupRepository(self.temp_dir / "backup.db") @@ -1648,7 +1686,7 @@ def test_versioned_migrations_create_all_foundation_tables(self): versions = connection.execute( "SELECT version FROM app_schema_migrations ORDER BY version" ).fetchall() - self.assertEqual([row[0] for row in versions], [1, 2, 3, 4, 5, 6, 7, 8]) + self.assertEqual([row[0] for row in versions], [1, 2, 3, 4, 5, 6, 7, 8, 9]) self.assertIn("collection_snapshots", tables) self.assertIn("preservation_matches", tables) self.assertIn("console_transfer_jobs", tables) @@ -1663,6 +1701,8 @@ def test_versioned_migrations_create_all_foundation_tables(self): self.assertIn("xenia_migration_runs", tables) self.assertIn("knowledge_source_priorities", tables) self.assertIn("scheduled_sync_state", tables) + self.assertIn("plugin_collection_runs", tables) + self.assertIn("dedup_recovery_records", tables) def test_xex_execution_info_is_parsed(self): from backup_manager import inspect_xex @@ -1781,6 +1821,13 @@ def test_unified_search_spans_games_profiles_and_achievements(self): self.assertEqual(service.search("Hitman")[0]["identifier"], "53510804") self.assertEqual(service.search("Agent47")[0]["category"], "profile") + api_scraper = Mock(db=self.database) + response = UnityScraperAPI(api_scraper).app.test_client().get( + "/api/community/search?q=Hitman" + ) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.get_json()["results"][0]["identifier"], "53510804") + def test_structured_knowledge_extracts_cached_hardware_article(self): import sqlite3 from contextlib import closing @@ -1851,6 +1898,7 @@ def test_console_sync_plan_can_queue_revalidated_uploads(self): connection.commit() snapshot_id = snapshot.lastrowid service = ConsolePlanService(self.db_path) + self.assertEqual(service.list_snapshots()[0]["id"], snapshot_id) plan = service.create_plan(local, snapshot_id) self.assertEqual(plan["summary"]["uploads"], 1) self.assertTrue(any(item["action"] == "review_remote" for item in plan["actions"])) @@ -1890,12 +1938,20 @@ def test_artwork_disc_dedup_and_storage_plans(self): applied = PreservationPlanningService(self.db_path).apply_dedup_action(action_id) self.assertTrue(Path(applied["quarantine"]).is_file()) self.assertFalse(Path(applied["duplicate"]).exists()) + restored = PreservationPlanningService(self.db_path).restore_dedup_action(action_id) + self.assertEqual(Path(restored["restored"]).read_bytes(), b"same") + self.assertFalse(Path(applied["quarantine"]).exists()) fatx = self.temp_dir / "drive.img" - fatx.write_bytes(b"XTAF" + bytes(64)) + fatx.write_bytes( + b"XTAF" + bytes.fromhex("12345678") + + (8).to_bytes(4, "big") + (1).to_bytes(4, "big") + bytes(52) + ) audit = StorageAndXboxService(self.db_path).audit_storage(fatx) self.assertEqual(audit["filesystem"], "FATX") self.assertEqual(audit["access_mode"], "read-only") + self.assertEqual(audit["details"]["partitions"][0]["cluster_size"], 4096) + self.assertTrue(audit["details"]["partitions"][0]["header_valid"]) def test_plugin_recovery_and_accessibility_controls(self): from community_services import AccessibilityService, PluginControlService, RecoveryService @@ -1929,6 +1985,44 @@ def test_plugin_recovery_and_accessibility_controls(self): self.assertTrue(access.get()["high_contrast"]) self.assertTrue(access.get()["reduced_motion"]) + def test_enabled_plugin_requires_approved_checksum_and_enriches_unknowns(self): + from community_services import PluginControlService + from plugins import PluginManager, load_enabled_plugin_configuration + + plugin = self.temp_dir / "plugins" / "catalog" + plugin.mkdir(parents=True) + entry = plugin / "collector.py" + entry.write_text( + "from plugins import MetadataCollectorPlugin\n" + "class Catalog(MetadataCollectorPlugin):\n" + " def validate_titleid(self, titleid): return True\n" + " def collect(self, titleid):\n" + " return {'title': 'Fallback Name', 'publisher': 'Fallback Publisher'}\n", + encoding="utf-8", + ) + (plugin / "plugin.json").write_text(json.dumps({ + "id": "catalog", "name": "Catalog", "version": "1.0", + "api_version": 1, "entrypoint": "collector.py", "permissions": ["metadata"], + }), encoding="utf-8") + PluginControlService(self.db_path).set_state("catalog", True, entry, ["metadata"]) + enabled, trusted = load_enabled_plugin_configuration(self.db_path, plugin.parent) + manager = PluginManager(str(plugin.parent), enabled_plugins=enabled, + trusted_hashes=trusted) + self.assertEqual(manager.collect_enabled("53510804")[0]["status"], "completed") + + self.database.add_titleid("53510804", "Hitman: Absolution", "Unknown") + scraper = UnityScraper.__new__(UnityScraper) + scraper.db = self.database + scraper.plugin_manager = manager + scraper._collect_plugin_metadata("53510804") + title = self.database.get_titleid_info("53510804") + self.assertEqual(title["name"], "Hitman: Absolution") + self.assertEqual(title["publisher"], "Fallback Publisher") + + entry.write_text(entry.read_text(encoding="utf-8") + "# changed\n", encoding="utf-8") + enabled, trusted = load_enabled_plugin_configuration(self.db_path, plugin.parent) + self.assertEqual((enabled, trusted), ([], {})) + def test_package_workspace_is_read_only_and_profile_tools_are_audited(self): from community_services import PackageWorkspaceService from profile_intelligence import ProfileIntelligenceService