diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 20cb9d5..6b8e69a 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -7,6 +7,8 @@ on:
permissions:
contents: write
+ id-token: write
+ attestations: write
jobs:
validate:
@@ -30,6 +32,14 @@ jobs:
python tests.py
python integration_tests.py
+ - name: Generate software bill of materials
+ run: python scripts/generate_sbom.py
+
+ - uses: actions/upload-artifact@v4
+ with:
+ name: release-sbom
+ path: UnityScraper-SBOM.cdx.json
+
windows:
runs-on: windows-latest
needs: validate
@@ -52,7 +62,7 @@ jobs:
shell: pwsh
run: |
New-Item -ItemType Directory -Path package | Out-Null
- Copy-Item dist\UnityScraper.exe, README.md, CHANGELOG.md, LICENSE package\
+ Copy-Item dist\UnityScraper.exe, README.md, CHANGELOG.md, LICENSE, DOCS_INDEX.md, BACKUP_MANAGER.md, COLLECTION_INTELLIGENCE.md, CONSOLE_SYNC.md, KNOWLEDGE_SOURCES.md, LINUX.md, PLUGIN_API.md package\
Compress-Archive -Path package\* -DestinationPath UnityScraper-Windows-x64.zip
$hash = (Get-FileHash UnityScraper-Windows-x64.zip -Algorithm SHA256).Hash.ToLower()
"$hash *UnityScraper-Windows-x64.zip" | Set-Content UnityScraper-Windows-x64.zip.sha256
@@ -103,7 +113,7 @@ jobs:
publish:
runs-on: ubuntu-latest
- needs: [windows, linux]
+ needs: [validate, windows, linux]
steps:
- uses: actions/download-artifact@v4
@@ -112,6 +122,11 @@ jobs:
path: release
merge-multiple: true
+ - name: Attest release artifacts
+ uses: actions/attest-build-provenance@v2
+ with:
+ subject-path: "release/*"
+
- name: Publish GitHub release
env:
GH_TOKEN: ${{ github.token }}
diff --git a/.gitignore b/.gitignore
index 72f91d8..16fc817 100644
--- a/.gitignore
+++ b/.gitignore
@@ -44,6 +44,7 @@ build/
dist/
*.spec.bak
*.zip
+UnityScraper-SBOM.cdx.json
# IDE and operating system
.vs/
diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md
index b54d111..f35caad 100644
--- a/ARCHITECTURE.md
+++ b/ARCHITECTURE.md
@@ -19,6 +19,8 @@ for CLI and optional REST automation. SQLite is the durable local store.
- `modern_gui.py` builds the dark navigation shell and core pages.
- `knowledge_gui.py` renders knowledge search, imports, sources, and conflicts.
- `backup_gui.py` renders inventory, package, FTP, and converter workflows.
+- `collection_gui.py` renders collection analysis, matching, reports, and
+ repair previews.
- `setup_wizard.py` handles first-run storage setup.
GUI operations that can block are dispatched to background threads and return
@@ -31,12 +33,18 @@ results to Tk's main loop.
- `backup_service.py` coordinates scans, installs, exports, verification, FTP,
and audit records.
- `knowledge_sync.py` exposes complete source-import workflows to CLI and GUI.
+- `collection_intelligence.py` coordinates snapshots, exact MediaID matching,
+ health, preservation matching, repair previews, and offline exports.
+- `console_sync.py` owns durable transfer jobs, resumable FTP, remote
+ snapshots, and PC/console comparisons.
+- `database_migrations.py` applies additive schema versions and provides
+ consistent SQLite backup/restore helpers.
### Domain and Adapters
- `main.py` contains the XboxUnity collector and shared configuration.
- `resume.py` handles partial download state and verification.
-- `backup_manager.py` parses public STFS/XBE fields and performs safe
+- `backup_manager.py` parses public STFS/XBE/XEX fields and performs safe
filesystem or FTP operations.
- `knowledge_base.py` defines normalized knowledge records and resolution.
- `consolemods_adapters.py`, `wiki_adapters.py`, and `dat_adapters.py` parse
diff --git a/BACKUP_MANAGER.md b/BACKUP_MANAGER.md
index 4a42e5d..68040ed 100644
--- a/BACKUP_MANAGER.md
+++ b/BACKUP_MANAGER.md
@@ -57,13 +57,15 @@ structural health check, not a substitute for Redump or No-Intro verification.
## FTP console transfer
The Console Transfer tab supports a user-configured FTP server such as Aurora.
-It uses one connection per operation and uploads to a temporary remote name
-before renaming the completed file. The default content root is:
+It persists resumable upload and download jobs, retains partial data, recovers
+interrupted jobs, limits bandwidth, verifies final sizes, and can capture a
+read-only console inventory. The default content root is:
`/Hdd1/Content/0000000000000000`
FTP passwords remain in memory and are deliberately omitted from SQLite.
Traditional FTP is unencrypted, so use it only on a trusted local network.
+See [CONSOLE_SYNC.md](CONSOLE_SYNC.md) for queue and comparison behavior.
## ISO conversion
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 1aa8ffb..8f004e7 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,20 @@ Notable changes to UnityScraper are documented here. The project follows
### Added
+- Versioned additive migrations for collection snapshots, preservation
+ matches, repair plans, console inventories, resumable jobs, overrides, and
+ recovery state.
+- XEX2 identity parsing, mounted-storage discovery, and immutable read-only
+ Aurora database import.
+- Exact MediaID Title Update comparison, collection health scoring, and
+ non-destructive repair-plan previews.
+- Redump/No-Intro file matching, offline HTML reports, manifests, and
+ provenance exports.
+- Persistent resumable FTP upload/download jobs, bandwidth limits, transfer
+ verification, and read-only console snapshots.
+- Database backups, plugin API v1, Italian and Portuguese translation
+ foundations, UI scaling, and keyboard navigation.
+- CycloneDX SBOM generation and GitHub build-provenance attestations.
- Native Linux desktop support with XDG data, configuration, cache, and state
directories.
- Linux x86_64 release bundle with user-level installation, application-menu
@@ -21,6 +35,10 @@ Notable changes to UnityScraper are documented here. The project follows
### Changed
+- Version advanced to `1.0.0-beta.1`.
+- Download queues now use atomic writes and recover interrupted items.
+- Update checks select a platform artifact and require its SHA-256 sidecar
+ before staging it.
- PyInstaller configuration and desktop entry point now support Windows and
Linux from the same source tree.
- Knowledge source snapshots use the platform cache directory.
diff --git a/COLLECTION_INTELLIGENCE.md b/COLLECTION_INTELLIGENCE.md
new file mode 100644
index 0000000..d40eb22
--- /dev/null
+++ b/COLLECTION_INTELLIGENCE.md
@@ -0,0 +1,51 @@
+# Collection Intelligence
+
+UnityScraper 1.0 beta inventories Xbox 360 collections, retains snapshots in
+SQLite, compares TitleIDs and MediaIDs with catalogued Title Updates, and
+produces preservation-oriented reports.
+
+## Sources and Identification
+
+The **Collections** workspace accepts content trees, extracted `Games`
+directories, mounted USB/archive folders, and user-selected Aurora SQLite
+databases. Aurora databases are opened in immutable read-only mode.
+
+Mounted-storage discovery checks Windows drive roots, Linux `/media`,
+`/run/media`, and `/mnt`, and macOS `/Volumes`. XEX2 parsing reads the public
+header fields for TitleID, MediaID, versions, module flags, and disc position.
+It does not decrypt or extract executable content.
+
+Title Update status is conservative:
+
+- `compatible`: TitleID and MediaID both match
+- `media-id-required`: updates exist but the collection MediaID is unknown
+- `incompatible`: the TitleID exists but no MediaID matches
+- `none`: no update is catalogued
+- `unknown`: the TitleID could not be identified
+
+## Preservation and Repair
+
+Files can be hashed with CRC32, MD5, SHA-1, and SHA-256 and matched against
+user-imported Redump or No-Intro DAT metadata. UnityScraper stores metadata,
+hashes, and matches, never game content.
+
+Exports include JSON manifests, offline HTML collection reports, and fact
+provenance with sources and citations. Metadata overrides are stored
+separately from imported facts.
+
+A repair plan is a preview in `repair_plans` and `repair_actions`. Creating
+one does not delete, replace, download, or transfer anything.
+
+## Command Line
+
+```powershell
+python main.py --analyze-collection D:\Xbox360 `
+ --collection-manifest collection.json `
+ --collection-html collection.html `
+ --create-repair-plan
+
+python main.py --aurora-db content.db --collection-manifest aurora.json
+python main.py --match-file game.iso
+python main.py --export-provenance provenance.json
+python main.py --backup-database unityscraper-backup.db
+```
diff --git a/CONSOLE_SYNC.md b/CONSOLE_SYNC.md
new file mode 100644
index 0000000..788098e
--- /dev/null
+++ b/CONSOLE_SYNC.md
@@ -0,0 +1,34 @@
+# Console Sync
+
+Console Sync provides a persistent FTP queue for consoles and dashboards
+whose FTP server the user explicitly configures.
+
+## Behavior
+
+- Upload and download jobs survive restarts in SQLite.
+- Interrupted transfers return as paused.
+- `.partial` files are retained for resume.
+- FTP `REST` is used when the server supports ranged transfer.
+- Uploads are published by renaming the completed partial file.
+- Final sizes are verified; downloads can also require a SHA-256.
+- Each job can have a bytes-per-second bandwidth limit.
+- Passwords remain in memory and are never stored.
+
+Some console FTP servers do not implement ranged uploads correctly. Those
+servers may reject resume; the job retains its state and reports the error.
+
+**Snapshot Console** recursively reads remote metadata without changing
+files. A snapshot can be compared with a local directory to find files only
+on the PC, only on the console, different-sized files, and matching files.
+Discovery has a default 100,000-entry safety limit.
+
+```powershell
+python main.py --ftp-host 192.168.1.50 --ftp-user xbox --ftp-snapshot /Hdd1
+
+python main.py --ftp-host 192.168.1.50 --ftp-user xbox `
+ --ftp-download /Hdd1/Content/file `
+ --ftp-local-path D:\Xbox360\file `
+ --ftp-bandwidth-limit 1048576
+```
+
+Standard FTP is unencrypted. Use it only on a trusted local network.
diff --git a/DOCS_INDEX.md b/DOCS_INDEX.md
index 789a2f8..8b47006 100644
--- a/DOCS_INDEX.md
+++ b/DOCS_INDEX.md
@@ -5,6 +5,10 @@
- [README](README.md) - product overview, installation, core workflows, and CLI
- [Backup Manager](BACKUP_MANAGER.md) - layouts, installation, exports, FTP,
verification, and external conversion
+- [Collection Intelligence](COLLECTION_INTELLIGENCE.md) - storage discovery,
+ XEX identity, Title Update compatibility, preservation, and repair previews
+- [Console Sync](CONSOLE_SYNC.md) - persistent transfers, resume, snapshots,
+ comparison, and verification
- [Knowledge Sources](KNOWLEDGE_SOURCES.md) - imports, provenance, caching, and
source licensing
- [Advanced Features](ADVANCED_FEATURES.md) - rate limits, resume, diagnostics,
@@ -19,6 +23,7 @@
- [Architecture](ARCHITECTURE.md) - modules, layers, schemas, data flows, and
packaging
+- [Plugin API v1](PLUGIN_API.md) - manifests, opt-in loading, and compatibility
- [Contributing](CONTRIBUTING.md) - environment, tests, PR expectations, and
adapter rules
- [Security](SECURITY.md) - private reporting and operational boundaries
diff --git a/PLUGIN_API.md b/PLUGIN_API.md
new file mode 100644
index 0000000..5b9e5f6
--- /dev/null
+++ b/PLUGIN_API.md
@@ -0,0 +1,29 @@
+# Plugin API v1
+
+Metadata collectors use a manifest-based, opt-in API. Disabled plugin code is
+discovered but never imported.
+
+```text
+plugins/
+ example/
+ plugin.json
+ collector.py
+```
+
+```json
+{
+ "id": "org.example.collector",
+ "name": "Example Collector",
+ "version": "1.0.0",
+ "api_version": 1,
+ "entrypoint": "collector.py",
+ "permissions": ["network"]
+}
+```
+
+The entrypoint exports a `MetadataCollectorPlugin` subclass. The caller must
+pass the plugin ID in `enabled_plugins` before code is loaded. Permissions
+are disclosure metadata, not an operating-system sandbox, so only enable
+plugins whose source and publisher you trust.
+
+Root-level legacy Python plugins load only with `allow_legacy=True`.
diff --git a/PROJECT_STATUS.md b/PROJECT_STATUS.md
index beb0bb8..c8415b6 100644
--- a/PROJECT_STATUS.md
+++ b/PROJECT_STATUS.md
@@ -24,18 +24,23 @@ backup-management, and source-attributed knowledge application.
dynamically loaded knowledge adapters.
- Local inventory for Xbox content roots, USB drives, archive folders, Games on
Demand, XBLA, DLC, title updates, and extracted Xbox/Xbox 360 games.
-- Public STFS and XBE header inspection for TitleID, MediaID, content type, disc,
- and display metadata.
+- Public STFS, XBE, and XEX2 header inspection for TitleID, MediaID, versions,
+ content type, disc, and display metadata.
+- Mounted-storage discovery, read-only Aurora database import, exact MediaID
+ Title Update comparison, collection health scoring, and repair previews.
+- Preservation hash matching against imported DAT metadata, offline HTML
+ reports, manifests, provenance exports, and separate local overrides.
- Safe bare-package and ZIP installation with path validation, `.partial`
staging, SHA-256 verification, and atomic final placement.
- Verified archive export with portable JSON manifests and per-file checksums.
-- Aurora-oriented FTP package upload with one connection per operation,
- temporary remote names, and no stored passwords.
+- Persistent resumable FTP upload/download jobs, partial-file recovery,
+ bandwidth limits, verified final sizes, read-only console snapshots, and no
+ stored passwords.
- Explicit external converter integration for user-owned ISO images.
- Local-by-default REST API with token-required remote binding, restricted
browser origins, validated settings, and current version reporting.
-- Cross-platform CI, Windows packaging checks, tagged release archives, and
- SHA-256 release checksums.
+- Cross-platform CI, Windows packaging checks, tagged release archives,
+ SHA-256 checksums, CycloneDX SBOMs, and build-provenance attestations.
- Linux x86_64 packaging, XDG storage, application-menu integration, source
launch scripts, and release artifacts.
- Repository contribution, security, architecture, API, and release
@@ -70,8 +75,7 @@ backup-management, and source-attributed knowledge application.
## Future Work
-- Parse additional XEX fields and link scanned file identifiers directly to
- normalized knowledge entities.
- Add field-specific source-priority controls and conflict resolution actions.
-- Add optional scheduled knowledge refreshes and offline HTML reports.
-- Add resumable FTP queues and optional Aurora database inventory.
+- Add optional scheduled knowledge refreshes.
+- Validate console resume behavior against a broader matrix of dashboard FTP
+ servers and add opt-in remote hash verification where servers expose it.
diff --git a/README.md b/README.md
index 743948c..1965e6c 100644
--- a/README.md
+++ b/README.md
@@ -50,9 +50,20 @@ service exposes. UnityScraper does not silently substitute HTTPS URLs.
`Content/0000000000000000` trees.
- Exports selected backups with per-file SHA-256 values and a preservation
manifest.
-- Uploads packages to a configured Aurora-style FTP server.
+- Queues resumable uploads and downloads to a configured Aurora-style FTP server.
+- Captures read-only console inventories and compares PC and console content.
- Runs a user-selected external ISO converter without bundling converter code.
+### Collection Intelligence and Preservation
+
+- Discovers mounted console, USB, and archive storage.
+- Parses XEX2 identity fields and imports Aurora databases read-only.
+- Compares installed content with catalogued updates using exact MediaIDs.
+- Scores collection health and creates non-destructive repair-plan previews.
+- Matches local hashes against user-imported Redump and No-Intro DAT metadata.
+- Exports preservation manifests, offline HTML reports, and fact provenance.
+- Keeps local metadata overrides separate from source-attributed knowledge.
+
## Install
### Windows Release
@@ -120,6 +131,7 @@ Linux source setup:
| Add Games | Import or enter TitleIDs |
| Downloads | Review and manage download activity |
| Backup Manager | Scan, install, verify, export, convert, and transfer owned content |
+| Collections | Identify storage, compare Title Updates, verify preservation data, and preview repairs |
| Knowledge | Search sources, facts, citations, imports, and conflicts |
| Archive Health | Find missing or inconsistent downloaded files |
| Settings | Configure storage and scraper behavior |
@@ -212,6 +224,24 @@ python main.py --ftp-upload game.live --ftp-host 192.168.1.50
See [BACKUP_MANAGER.md](BACKUP_MANAGER.md) for layouts, conflict behavior,
manifests, FTP considerations, and external converter arguments.
+### Collections and Console Sync
+
+```powershell
+# Analyze a collection and create offline reports
+python main.py --analyze-collection D:\Xbox360 `
+ --collection-manifest collection.json `
+ --collection-html collection.html
+
+# Read an Aurora database without modifying it
+python main.py --aurora-db content.db --collection-manifest aurora.json
+
+# Match a local file against imported preservation DAT hashes
+python main.py --match-file game.iso
+
+# Capture a read-only console inventory
+python main.py --ftp-host 192.168.1.50 --ftp-snapshot /Hdd1
+```
+
## Optional REST API
Start the localhost-only API:
@@ -271,6 +301,9 @@ SHA-256 file.
- [Linux support](LINUX.md)
- [Knowledge sources and licensing](KNOWLEDGE_SOURCES.md)
- [Backup manager](BACKUP_MANAGER.md)
+- [Collection intelligence](COLLECTION_INTELLIGENCE.md)
+- [Console sync](CONSOLE_SYNC.md)
+- [Plugin API v1](PLUGIN_API.md)
- [REST API](API.md)
- [Project status](PROJECT_STATUS.md)
- [Changelog](CHANGELOG.md)
diff --git a/UnityScraper.spec b/UnityScraper.spec
index 2863a23..b4ce347 100644
--- a/UnityScraper.spec
+++ b/UnityScraper.spec
@@ -14,11 +14,17 @@ a = Analysis(
'backup_gui',
'backup_manager',
'backup_service',
+ 'collection_gui',
+ 'collection_intelligence',
+ 'console_sync',
'consolemods_adapters',
+ 'database_migrations',
'dat_adapters',
'knowledge_gui',
'knowledge_service',
'knowledge_sync',
+ 'plugins',
+ 'updater',
'wiki_adapters',
],
hookspath=[],
diff --git a/VERSION b/VERSION
index d21ee86..9a7c866 100644
--- a/VERSION
+++ b/VERSION
@@ -1,6 +1,6 @@
{
- "version": "0.10.0b1",
- "name": "Xbox 360 Knowledge and Backup Manager",
+ "version": "1.0.0b1",
+ "name": "Unified Xbox 360 Collection and Preservation Manager",
"changes": [
"Unified Xbox 360 knowledge browser",
"ConsoleMods, XenonLibrary, and Free60 wiki ingestion",
@@ -9,7 +9,14 @@
"Local Xbox content and Games folder inventory",
"Safe STFS package and ZIP installation",
"Verified exports with preservation manifests",
- "FTP console transfer and external ISO converter support"
+ "Persistent resumable FTP console synchronization",
+ "XEX identity parsing and exact MediaID title-update matching",
+ "Mounted-storage discovery and read-only Aurora database import",
+ "Collection health scoring and repair-plan previews",
+ "Redump and No-Intro local-file hash matching",
+ "Offline HTML reports, manifests, provenance, and metadata overrides",
+ "Versioned migrations, database backups, queue recovery, and plugin API v1",
+ "Platform-aware verified updates, SBOMs, and build attestations"
],
"download_url": "https://github.com/TrapEmAll/UnityScraper/releases",
"release_date": "2026-07-23"
diff --git a/app_version.py b/app_version.py
index 2bc5d2e..35497f9 100644
--- a/app_version.py
+++ b/app_version.py
@@ -1,4 +1,4 @@
"""Single source of truth for UnityScraper version information."""
-APP_VERSION = "0.10.0b1"
-DISPLAY_VERSION = "0.10.0-beta.1"
+APP_VERSION = "1.0.0b1"
+DISPLAY_VERSION = "1.0.0-beta.1"
diff --git a/backup_gui.py b/backup_gui.py
index 03cac48..1222b4b 100644
--- a/backup_gui.py
+++ b/backup_gui.py
@@ -5,6 +5,7 @@
import threading
import tkinter as tk
from pathlib import Path
+from pathlib import PurePosixPath
from tkinter import filedialog, messagebox, ttk
from typing import Callable, Optional
@@ -13,8 +14,10 @@
ExternalConverter,
FtpBackupClient,
FtpTarget,
+ inspect_stfs,
)
from backup_service import BackupService
+from console_sync import ConsoleSyncService
class BackupPage:
@@ -30,6 +33,7 @@ def __init__(
self.root = root
self.parent = parent
self.service = service
+ self.console_sync = ConsoleSyncService(service.repository.db_path)
self.items: dict[str, BackupItem] = {}
self.busy = False
@@ -117,6 +121,12 @@ def _build_inventory(self) -> None:
ttk.Button(
controls, text="Verify Selected", command=self.verify_selected
).pack(side=tk.LEFT, padx=(8, 0))
+ ttk.Button(controls, text="Verify All", command=self.verify_all).pack(
+ side=tk.LEFT, padx=(8, 0)
+ )
+ ttk.Button(controls, text="Export All", command=self.export_all).pack(
+ side=tk.LEFT, padx=(8, 0)
+ )
def _build_transfer(self) -> None:
tab = self.transfer_tab
@@ -151,10 +161,34 @@ def _build_transfer(self) -> None:
)
ttk.Button(
controls,
- text="Upload Package",
- command=self.upload_ftp,
+ text="Queue & Upload",
+ command=self.queue_upload,
style="Accent.TButton",
).pack(side=tk.LEFT, padx=(8, 0))
+ ttk.Button(controls, text="Pause", command=self.pause_transfer).pack(
+ side=tk.LEFT, padx=(8, 0)
+ )
+ ttk.Button(controls, text="Resume", command=self.resume_transfer).pack(
+ side=tk.LEFT, padx=(8, 0)
+ )
+ ttk.Button(controls, text="Snapshot Console", command=self.snapshot_console).pack(
+ side=tk.LEFT, padx=(8, 0)
+ )
+ ttk.Button(controls, text="Compare Latest", command=self.compare_console).pack(
+ side=tk.LEFT, padx=(8, 0)
+ )
+ limit_row = ttk.Frame(tab)
+ limit_row.grid(row=8, column=0, columnspan=2, sticky="ew", pady=(12, 0))
+ ttk.Label(limit_row, text="Bandwidth limit (KiB/s)").pack(side=tk.LEFT)
+ self.ftp_limit_var = tk.StringVar(value="0")
+ ttk.Spinbox(
+ limit_row, from_=0, to=102400, textvariable=self.ftp_limit_var, width=10
+ ).pack(side=tk.LEFT, padx=(8, 0))
+ self.queue_var = tk.StringVar(value="Persistent queue: empty")
+ ttk.Label(tab, textvariable=self.queue_var, style="Subheader.TLabel").grid(
+ row=9, column=0, columnspan=2, sticky=tk.W, pady=(8, 0)
+ )
+ self._refresh_queue_status()
def _build_converter(self) -> None:
tab = self.converter_tab
@@ -341,6 +375,41 @@ def _verify_done(self, issues) -> None:
self.status_var.set("Verification completed with no structural issues.")
messagebox.showinfo("Verification", "No structural issues found.", parent=self.root)
+ def verify_all(self) -> None:
+ if not self.items:
+ messagebox.showinfo("Verification", "Scan a target first.", parent=self.root)
+ return
+ self._run(
+ "Checking all inventoried backups...",
+ lambda: self.service.verify_many(list(self.items.values())),
+ self._verify_batch_done,
+ )
+
+ def _verify_batch_done(self, findings: list[dict]) -> None:
+ self.status_var.set(
+ f"Batch verification completed: {len(findings)} item(s) need attention."
+ )
+ if findings:
+ messagebox.showwarning(
+ "Batch verification",
+ "\n".join(
+ f"{item['path']}: {', '.join(item['issues'])}" for item in findings[:15]
+ ),
+ parent=self.root,
+ )
+
+ def export_all(self) -> None:
+ if not self.items:
+ messagebox.showinfo("Export", "Scan a target first.", parent=self.root)
+ return
+ destination = filedialog.askdirectory(parent=self.root, title="Choose batch export folder")
+ if destination:
+ self._run(
+ "Exporting all inventoried backups...",
+ lambda: self.service.export_many(list(self.items.values()), destination),
+ lambda paths: self._operation_done(f"Exported {len(paths)} item(s)."),
+ )
+
def _ftp_target(self) -> FtpTarget:
host = self.ftp_host_var.get().strip()
if not host:
@@ -372,6 +441,99 @@ def upload_ftp(self) -> None:
),
)
+ def queue_upload(self) -> None:
+ source = filedialog.askopenfilename(parent=self.root, title="Choose STFS package")
+ if not source:
+ return
+ try:
+ package = inspect_stfs(source)
+ remote = str(
+ PurePosixPath(self.ftp_content_var.get().strip())
+ / package.title_id
+ / package.content_directory
+ / Path(source).name
+ )
+ job_id = self.console_sync.enqueue(
+ "upload",
+ source,
+ remote,
+ bandwidth_limit=max(0, int(self.ftp_limit_var.get() or "0")) * 1024,
+ )
+ except Exception as exc:
+ self._failed(exc)
+ return
+ self._refresh_queue_status()
+ self._run(
+ f"Transferring queued job {job_id}...",
+ lambda: self.console_sync.run_job(job_id, self._ftp_target()),
+ lambda result: self._sync_done(result),
+ )
+
+ def pause_transfer(self) -> None:
+ self.console_sync.pause()
+ self.status_var.set("Pause requested. Partial data will be kept for resume.")
+
+ def resume_transfer(self) -> None:
+ paused = [job for job in self.console_sync.list_jobs() if job["status"] == "paused"]
+ if not paused:
+ messagebox.showinfo("Console transfer", "No paused job is available.", parent=self.root)
+ return
+ job_id = int(paused[0]["id"])
+ self.console_sync.resume(job_id)
+ self._run(
+ f"Resuming job {job_id}...",
+ lambda: self.console_sync.run_job(job_id, self._ftp_target()),
+ lambda result: self._sync_done(result),
+ )
+
+ def snapshot_console(self) -> None:
+ self._run(
+ "Reading console inventory...",
+ lambda: self.console_sync.capture_inventory(self._ftp_target(), "/Hdd1"),
+ lambda snapshot_id: self._operation_done(
+ f"Read-only console snapshot {snapshot_id} saved."
+ ),
+ )
+
+ def compare_console(self) -> None:
+ snapshots = [
+ item for item in self.console_sync.list_snapshots()
+ if item["status"] == "completed"
+ ]
+ if not snapshots:
+ messagebox.showinfo(
+ "Console comparison", "Capture a console snapshot first.", parent=self.root
+ )
+ return
+ local = filedialog.askdirectory(parent=self.root, title="Choose matching PC folder")
+ if not local:
+ return
+ snapshot_id = int(snapshots[0]["id"])
+ self._run(
+ f"Comparing with snapshot {snapshot_id}...",
+ lambda: self.console_sync.compare(local, snapshot_id),
+ lambda result: messagebox.showinfo(
+ "PC and console comparison",
+ f"Only on PC: {len(result.only_on_pc)}\n"
+ f"Only on console: {len(result.only_on_console)}\n"
+ f"Different size: {len(result.size_mismatches)}\n"
+ f"Matching: {len(result.matching)}",
+ parent=self.root,
+ ),
+ )
+
+ def _sync_done(self, result: dict) -> None:
+ self._refresh_queue_status()
+ self._operation_done(
+ f"Transfer {result['status']}: {result['transferred_bytes']} / "
+ f"{result['total_bytes']} bytes."
+ )
+
+ def _refresh_queue_status(self) -> None:
+ jobs = self.console_sync.list_jobs()
+ active = sum(job["status"] in {"queued", "transferring", "paused"} for job in jobs)
+ self.queue_var.set(f"Persistent queue: {active} active, {len(jobs)} total")
+
def run_converter(self) -> None:
import shlex
diff --git a/backup_manager.py b/backup_manager.py
index 89ff7cd..b16df5f 100644
--- a/backup_manager.py
+++ b/backup_manager.py
@@ -76,6 +76,19 @@ class XbePackage:
size: int
+@dataclass(frozen=True)
+class XexPackage:
+ path: Path
+ title_id: str
+ media_id: str
+ version: str
+ base_version: str
+ disc_number: int
+ disc_count: int
+ module_flags: int
+ size: int
+
+
@dataclass
class BackupItem:
path: Path
@@ -87,6 +100,8 @@ class BackupItem:
content_type: str = ""
status: str = "ready"
notes: list[str] = field(default_factory=list)
+ disc_number: int = 0
+ disc_count: int = 0
def to_dict(self) -> dict:
result = asdict(self)
@@ -195,6 +210,53 @@ def inspect_xbe(path: str | Path) -> XbePackage:
return XbePackage(package_path, title_id, title_name, package_path.stat().st_size)
+def inspect_xex(path: str | Path) -> XexPackage:
+ """Read the public XEX2 execution-info header without decrypting content."""
+ package_path = Path(path)
+ with package_path.open("rb") as handle:
+ header = handle.read(0x4000)
+ if len(header) < 0x18 or header[:4] != b"XEX2":
+ raise InvalidPackageError(f"{package_path.name} is not an XEX2 executable")
+
+ module_flags = int.from_bytes(header[4:8], "big")
+ optional_count = int.from_bytes(header[0x14:0x18], "big")
+ if optional_count > 4096 or 0x18 + optional_count * 8 > len(header):
+ raise InvalidPackageError("XEX optional-header table is invalid")
+
+ execution_offset = 0
+ for index in range(optional_count):
+ entry = 0x18 + index * 8
+ key = int.from_bytes(header[entry : entry + 4], "big")
+ value = int.from_bytes(header[entry + 4 : entry + 8], "big")
+ if key == 0x00040006:
+ execution_offset = value
+ break
+ if not execution_offset or execution_offset + 24 > len(header):
+ raise InvalidPackageError("XEX execution metadata is unavailable")
+
+ info = header[execution_offset : execution_offset + 24]
+
+ def format_version(value: int) -> str:
+ return (
+ f"{(value >> 28) & 0xF}."
+ f"{(value >> 24) & 0xF}."
+ f"{(value >> 8) & 0xFFFF}."
+ f"{value & 0xFF}"
+ )
+
+ return XexPackage(
+ path=package_path,
+ title_id=info[12:16].hex().upper(),
+ media_id=info[0:4].hex().upper(),
+ version=format_version(int.from_bytes(info[4:8], "big")),
+ base_version=format_version(int.from_bytes(info[8:12], "big")),
+ disc_number=info[18],
+ disc_count=info[19],
+ module_flags=module_flags,
+ size=package_path.stat().st_size,
+ )
+
+
def sha256_file(path: str | Path) -> str:
digest = hashlib.sha256()
with Path(path).open("rb") as handle:
@@ -255,6 +317,8 @@ def scan_local_target(
notes = []
package_names: list[str] = []
media_ids: set[str] = set()
+ disc_numbers: set[int] = set()
+ disc_count = 0
malformed = 0
for type_name in sorted(types & set(CONTENT_TYPES[value][1] for value in CONTENT_TYPES)):
for package_path in (title_dir / type_name).iterdir():
@@ -267,6 +331,9 @@ def scan_local_target(
continue
if package.media_id and package.media_id != "00000000":
media_ids.add(package.media_id)
+ if package.disc_number:
+ disc_numbers.add(package.disc_number)
+ disc_count = max(disc_count, package.disc_count)
candidate_name = package.title_name or package.display_name
if candidate_name:
package_names.append(candidate_name)
@@ -275,6 +342,11 @@ def scan_local_target(
warnings.append(f"{title_id} has support content but no base game")
if malformed:
notes.append(f"{malformed} package header(s) could not be identified")
+ if disc_count:
+ notes.append(
+ f"Discs found: {', '.join(str(value) for value in sorted(disc_numbers))} "
+ f"of {disc_count}"
+ )
name = title_lookup(title_id) if title_lookup else None
items.append(
BackupItem(
@@ -291,6 +363,8 @@ def scan_local_target(
size=directory_size(title_dir),
status=status,
notes=notes,
+ disc_number=min(disc_numbers) if len(disc_numbers) == 1 else 0,
+ disc_count=disc_count,
)
)
@@ -301,22 +375,39 @@ def scan_local_target(
title_name = game_dir.name
media_id = ""
format_name = "Extracted Xbox 360"
- notes: list[str] = []
+ extracted_notes: list[str] = []
+ disc_number = 0
+ disc_count = 0
xbe = game_dir / "default.xbe"
xex = game_dir / "default.xex"
if xbe.is_file():
try:
- info = inspect_xbe(xbe)
- title_id, title_name = info.title_id, info.title_name or title_name
+ xbe_info = inspect_xbe(xbe)
+ title_id = xbe_info.title_id
+ title_name = xbe_info.title_name or title_name
format_name = "Extracted Original Xbox"
except BackupError as exc:
- notes.append(str(exc))
+ extracted_notes.append(str(exc))
else:
+ if xex.is_file():
+ try:
+ xex_info = inspect_xex(xex)
+ title_id = xex_info.title_id
+ media_id = xex_info.media_id if xex_info.media_id != "00000000" else ""
+ disc_number = xex_info.disc_number
+ disc_count = xex_info.disc_count
+ extracted_notes.append(f"XEX version {xex_info.version}")
+ if xex_info.disc_count:
+ extracted_notes.append(
+ f"Disc {xex_info.disc_number} of {xex_info.disc_count}"
+ )
+ except BackupError as exc:
+ extracted_notes.append(str(exc))
folder_match = FOLDER_TITLE_ID_RE.search(game_dir.name)
- if folder_match:
+ if not title_id and folder_match:
title_id = folder_match.group(1).upper()
if not xex.is_file():
- notes.append("default.xex or default.xbe is missing")
+ extracted_notes.append("default.xex or default.xbe is missing")
if title_id and title_lookup:
title_name = title_lookup(title_id) or title_name
items.append(
@@ -327,8 +418,10 @@ def scan_local_target(
format=format_name,
media_id=media_id,
size=directory_size(game_dir),
- status="ready" if not notes else "incomplete",
- notes=notes,
+ status="ready" if not extracted_notes else "incomplete",
+ notes=extracted_notes,
+ disc_number=disc_number,
+ disc_count=disc_count,
)
)
diff --git a/backup_service.py b/backup_service.py
index 2578e1d..067ede4 100644
--- a/backup_service.py
+++ b/backup_service.py
@@ -11,6 +11,7 @@
from typing import Optional
from app_paths import DATABASE_PATH, ensure_app_dirs
+from database_migrations import ensure_application_schema
from backup_manager import (
BackupItem,
FtpBackupClient,
@@ -120,6 +121,7 @@ def connect(self):
def ensure_schema(self) -> None:
with self.connect() as connection:
ensure_backup_schema(connection)
+ ensure_application_schema(connection)
def save_local_target(self, name: str, location: str | Path) -> int:
now = datetime.now(timezone.utc).isoformat()
@@ -360,6 +362,24 @@ def verify(self, item: BackupItem) -> list[str]:
)
return issues
+ def verify_many(self, items: list[BackupItem]) -> list[dict]:
+ """Structurally verify an inventory batch."""
+ findings = []
+ for item in items:
+ issues = self.verify(item)
+ if issues:
+ findings.append({"path": str(item.path), "issues": issues})
+ return findings
+
+ def export_many(
+ self,
+ items: list[BackupItem],
+ destination: str | Path,
+ conflict: str = "skip",
+ ) -> list[Path]:
+ """Export an inventory batch using verified per-item exports."""
+ return [self.export(item, destination, conflict) for item in items]
+
def upload_ftp(
self, source: str | Path, target: FtpTarget
) -> TransferResult:
diff --git a/build_linux.sh b/build_linux.sh
index 02c0e28..c2f2709 100755
--- a/build_linux.sh
+++ b/build_linux.sh
@@ -26,6 +26,8 @@ install -m 0644 packaging/linux/io.github.trapemall.UnityScraper.metainfo.xml \
"$STAGE/io.github.trapemall.UnityScraper.metainfo.xml"
install -m 0644 assets/UnityScraper.png "$STAGE/unityscraper.png"
install -m 0644 README.md CHANGELOG.md LICENSE "$STAGE/"
+install -m 0644 DOCS_INDEX.md BACKUP_MANAGER.md COLLECTION_INTELLIGENCE.md \
+ CONSOLE_SYNC.md KNOWLEDGE_SOURCES.md LINUX.md PLUGIN_API.md "$STAGE/"
tar -C dist -czf "$ARCHIVE" "UnityScraper-Linux-${ARCH}"
sha256sum "$ARCHIVE" > "$ARCHIVE.sha256"
diff --git a/collection_gui.py b/collection_gui.py
new file mode 100644
index 0000000..53f5031
--- /dev/null
+++ b/collection_gui.py
@@ -0,0 +1,311 @@
+"""Desktop collection-intelligence workspace."""
+
+from __future__ import annotations
+
+import threading
+import tkinter as tk
+from pathlib import Path
+from tkinter import filedialog, messagebox, simpledialog, ttk
+from typing import Callable
+
+from collection_intelligence import (
+ CollectionAnalysis,
+ CollectionIntelligenceService,
+ discover_storage_roots,
+)
+
+
+class CollectionPage:
+ def __init__(
+ self,
+ root: tk.Tk,
+ parent: ttk.Frame,
+ service: CollectionIntelligenceService,
+ page_header: Callable[[str, str], None],
+ ) -> None:
+ self.root = root
+ self.parent = parent
+ self.service = service
+ self.analysis: CollectionAnalysis | None = None
+ self.busy = False
+ page_header(
+ "Collection Intelligence",
+ "Identify games, exact title-update compatibility, preservation matches, and repairs.",
+ )
+ self._build()
+
+ def _build(self) -> None:
+ body = ttk.Frame(self.parent)
+ body.grid(row=1, column=0, sticky="nsew")
+ body.columnconfigure(0, weight=1)
+ body.rowconfigure(2, weight=1)
+
+ controls = ttk.Frame(body)
+ controls.grid(row=0, column=0, sticky="ew", pady=(0, 10))
+ controls.columnconfigure(1, weight=1)
+ ttk.Label(controls, text="Collection source").grid(row=0, column=0, padx=(0, 8))
+ self.source_var = tk.StringVar()
+ ttk.Entry(controls, textvariable=self.source_var).grid(
+ row=0, column=1, sticky="ew"
+ )
+ ttk.Button(controls, text="Discover", command=self.discover).grid(
+ row=0, column=2, padx=(8, 0)
+ )
+ ttk.Button(controls, text="Browse", command=self.browse).grid(
+ row=0, column=3, padx=(8, 0)
+ )
+ ttk.Button(
+ controls, text="Analyze", command=self.analyze, style="Accent.TButton"
+ ).grid(row=0, column=4, padx=(8, 0))
+ ttk.Button(controls, text="Import Aurora DB", command=self.import_aurora).grid(
+ row=0, column=5, padx=(8, 0)
+ )
+
+ summary = ttk.Frame(body)
+ summary.grid(row=1, column=0, sticky="ew", pady=(0, 10))
+ self.score_var = tk.StringVar(value="Health: --")
+ self.count_var = tk.StringVar(value="Games: --")
+ self.issue_var = tk.StringVar(value="Issues: --")
+ for variable in (self.score_var, self.count_var, self.issue_var):
+ ttk.Label(summary, textvariable=variable, style="Metric.TLabel", padding=10).pack(
+ side=tk.LEFT, padx=(0, 8)
+ )
+
+ columns = ("titleid", "mediaid", "format", "tu", "status")
+ self.tree = ttk.Treeview(body, columns=columns, show="tree headings")
+ self.tree.heading("#0", text="Game")
+ for column, label in zip(
+ columns, ("TitleID", "MediaID", "Format", "Title update", "Status")
+ ):
+ self.tree.heading(column, text=label)
+ self.tree.column("#0", width=270)
+ self.tree.column("titleid", width=90, anchor=tk.CENTER)
+ self.tree.column("mediaid", width=90, anchor=tk.CENTER)
+ self.tree.column("format", width=180)
+ self.tree.column("tu", width=130)
+ self.tree.column("status", width=100, anchor=tk.CENTER)
+ self.tree.grid(row=2, column=0, sticky="nsew")
+
+ actions = ttk.Frame(body)
+ actions.grid(row=3, column=0, sticky="ew", pady=(10, 0))
+ action_specs = (
+ ("Verify Selected", self.verify_selected),
+ ("Repair Plan", self.repair_plan),
+ ("Edit Metadata", self.edit_metadata),
+ ("Export Manifest", self.export_manifest),
+ ("Export HTML", self.export_html),
+ ("Export Aurora", self.export_aurora),
+ ("Export Provenance", self.export_provenance),
+ )
+ for column_index in range(4):
+ actions.columnconfigure(column_index, weight=1)
+ for index, (label, command) in enumerate(action_specs):
+ ttk.Button(actions, text=label, command=command).grid(
+ row=index // 4,
+ column=index % 4,
+ sticky="ew",
+ padx=(0 if index % 4 == 0 else 8, 0),
+ pady=(0 if index < 4 else 8, 0),
+ )
+ self.status_var = tk.StringVar(value="Choose a folder or discover mounted storage.")
+ ttk.Label(body, textvariable=self.status_var, style="Subheader.TLabel").grid(
+ row=4, column=0, sticky="ew", pady=(8, 0)
+ )
+
+ def discover(self) -> None:
+ roots = discover_storage_roots()
+ if roots:
+ self.source_var.set(str(roots[0]))
+ self.status_var.set(
+ f"Found {len(roots)} mounted location(s); showing the strongest match."
+ )
+ else:
+ self.status_var.set("No mounted collection root was detected.")
+
+ def browse(self) -> None:
+ selected = filedialog.askdirectory(parent=self.root, title="Choose collection root")
+ if selected:
+ self.source_var.set(selected)
+
+ def analyze(self) -> None:
+ source = self.source_var.get().strip()
+ if not source:
+ messagebox.showwarning("Collection source", "Choose a folder first.", parent=self.root)
+ return
+ self._run("Analyzing collection...", lambda: self.service.analyze(source))
+
+ def import_aurora(self) -> None:
+ selected = filedialog.askopenfilename(
+ parent=self.root,
+ title="Choose an Aurora database",
+ filetypes=(("SQLite databases", "*.db *.sqlite *.sqlite3"), ("All files", "*.*")),
+ )
+ if selected:
+ self.source_var.set(selected)
+ self._run("Reading Aurora database...", lambda: self.service.analyze_aurora(selected))
+
+ def _run(self, message: str, operation) -> None:
+ if self.busy:
+ return
+ self.busy = True
+ self.status_var.set(message)
+
+ def worker() -> None:
+ try:
+ result = operation()
+ except Exception as exc:
+ error = exc
+
+ def report_error() -> None:
+ self._failed(error)
+
+ self.root.after(0, report_error)
+ else:
+ self.root.after(0, lambda: self._finished(result))
+
+ threading.Thread(target=worker, daemon=True).start()
+
+ def _failed(self, error: Exception) -> None:
+ self.busy = False
+ self.status_var.set("Analysis failed.")
+ messagebox.showerror("Collection analysis failed", str(error), parent=self.root)
+
+ def _finished(self, analysis: CollectionAnalysis) -> None:
+ self.busy = False
+ self.analysis = analysis
+ self.tree.delete(*self.tree.get_children())
+ for index, item in enumerate(analysis.result.items):
+ match = analysis.compatibility[str(item.path)]
+ self.tree.insert(
+ "",
+ tk.END,
+ iid=str(index),
+ text=item.name,
+ values=(
+ item.title_id,
+ item.media_id,
+ item.format,
+ match.status,
+ item.status,
+ ),
+ )
+ self.score_var.set(f"Health: {analysis.health_score}")
+ self.count_var.set(f"Games: {len(analysis.result.items)}")
+ self.issue_var.set(f"Issues: {len(analysis.issues)}")
+ self.status_var.set(
+ f"Snapshot {analysis.snapshot_id} saved. No repair action has been executed."
+ )
+
+ def _require_analysis(self) -> CollectionAnalysis | None:
+ if self.analysis is None:
+ messagebox.showinfo("Collection", "Analyze a collection first.", parent=self.root)
+ return self.analysis
+
+ def verify_selected(self) -> None:
+ analysis = self._require_analysis()
+ selected = self.tree.selection()
+ if not analysis or not selected:
+ return
+ item = analysis.result.items[int(selected[0])]
+ if not item.path.is_file():
+ messagebox.showinfo(
+ "Verification",
+ "Select a package file to hash-match. Folder verification is represented in the scan.",
+ parent=self.root,
+ )
+ return
+ self._run_hash(item.path)
+
+ def _run_hash(self, path: Path) -> None:
+ self.status_var.set(f"Hashing {path.name}...")
+
+ def worker() -> None:
+ try:
+ matches = self.service.hash_and_match(path)
+ except Exception as exc:
+ error = exc
+
+ def report_error() -> None:
+ self._failed(error)
+
+ self.root.after(0, report_error)
+ else:
+ self.root.after(
+ 0,
+ lambda: self.status_var.set(
+ f"Verification complete: {len(matches)} Redump/No-Intro match(es)."
+ ),
+ )
+
+ threading.Thread(target=worker, daemon=True).start()
+
+ def repair_plan(self) -> None:
+ analysis = self._require_analysis()
+ if analysis:
+ plan_id = self.service.create_repair_plan(analysis)
+ messagebox.showinfo(
+ "Repair plan",
+ f"Preview plan {plan_id} contains {len(analysis.issues)} proposed action(s).\n\n"
+ "Nothing was changed.",
+ parent=self.root,
+ )
+
+ def edit_metadata(self) -> None:
+ analysis = self._require_analysis()
+ selected = self.tree.selection()
+ if not analysis or not selected:
+ return
+ item = analysis.result.items[int(selected[0])]
+ if not item.title_id:
+ messagebox.showinfo(
+ "Metadata override", "This item needs a TitleID first.", parent=self.root
+ )
+ return
+ value = simpledialog.askstring(
+ "Local game name",
+ f"Preferred local name for {item.title_id}:",
+ initialvalue=item.name,
+ parent=self.root,
+ )
+ if value and value.strip():
+ self.service.set_override(item.title_id, "name", value.strip())
+ item.name = value.strip()
+ self.tree.item(selected[0], text=item.name)
+ self.status_var.set(
+ "Local override saved separately; imported source facts were not changed."
+ )
+
+ def export_manifest(self) -> None:
+ analysis = self._require_analysis()
+ if analysis:
+ self.status_var.set(f"Manifest written to {self.service.export_manifest(analysis)}")
+
+ def export_html(self) -> None:
+ analysis = self._require_analysis()
+ if analysis:
+ self.status_var.set(f"HTML report written to {self.service.export_html(analysis)}")
+
+ def export_provenance(self) -> None:
+ selected = filedialog.asksaveasfilename(
+ parent=self.root,
+ title="Export provenance",
+ defaultextension=".json",
+ filetypes=(("JSON", "*.json"),),
+ )
+ if selected:
+ self.status_var.set(
+ f"Provenance written to {self.service.export_provenance(selected)}"
+ )
+
+ def export_aurora(self) -> None:
+ analysis = self._require_analysis()
+ if not analysis:
+ return
+ selected = filedialog.askdirectory(parent=self.root, title="Choose Aurora export root")
+ if selected:
+ try:
+ output = self.service.export_aurora_layout(analysis, selected)
+ except Exception as exc:
+ self._failed(exc)
+ else:
+ self.status_var.set(f"Aurora layout exported to {output}")
diff --git a/collection_intelligence.py b/collection_intelligence.py
new file mode 100644
index 0000000..897e4d7
--- /dev/null
+++ b/collection_intelligence.py
@@ -0,0 +1,651 @@
+"""Collection discovery, compatibility, preservation, and reporting services."""
+
+from __future__ import annotations
+
+import hashlib
+import html
+import json
+import os
+import re
+import sqlite3
+import shutil
+import sys
+import zlib
+from contextlib import closing, contextmanager
+from dataclasses import asdict, dataclass, field
+from datetime import datetime, timezone
+from pathlib import Path
+
+from app_paths import DATABASE_PATH, EXPORTS_DIR, ensure_app_dirs
+from backup_manager import BackupItem, ScanResult, scan_local_target
+from knowledge_base import KnowledgeRepository
+
+
+HEX8_RE = re.compile(r"^[0-9A-Fa-f]{8}$")
+
+
+def utc_now() -> str:
+ return datetime.now(timezone.utc).isoformat()
+
+
+@dataclass(frozen=True)
+class UpdateCompatibility:
+ title_id: str
+ media_id: str
+ status: str
+ newest_version: str = ""
+ update_count: int = 0
+ reason: str = ""
+
+
+@dataclass(frozen=True)
+class CollectionIssue:
+ severity: str
+ code: str
+ target: str
+ message: str
+ suggested_action: str
+ destructive: bool = False
+
+
+@dataclass
+class CollectionAnalysis:
+ result: ScanResult
+ compatibility: dict[str, UpdateCompatibility] = field(default_factory=dict)
+ issues: list[CollectionIssue] = field(default_factory=list)
+ health_score: int = 100
+ snapshot_id: int | None = None
+
+ def to_dict(self) -> dict:
+ return {
+ "schema": 1,
+ "snapshot_id": self.snapshot_id,
+ "health_score": self.health_score,
+ "scan": self.result.to_dict(),
+ "compatibility": {key: asdict(value) for key, value in self.compatibility.items()},
+ "issues": [asdict(issue) for issue in self.issues],
+ }
+
+
+def discover_storage_roots() -> list[Path]:
+ """Return mounted roots that are plausible console, USB, or archive targets."""
+ candidates: list[Path] = []
+ if os.name == "nt":
+ import ctypes
+
+ mask = ctypes.windll.kernel32.GetLogicalDrives()
+ for index in range(26):
+ if mask & (1 << index):
+ candidates.append(Path(f"{chr(65 + index)}:/"))
+ elif sys.platform == "darwin":
+ candidates.extend(_children(Path("/Volumes")))
+ else:
+ user = os.environ.get("USER", "")
+ candidates.extend(_children(Path("/media") / user))
+ candidates.extend(_children(Path("/run/media") / user))
+ candidates.extend(_children(Path("/mnt")))
+
+ scored: list[tuple[int, Path]] = []
+ for path in dict.fromkeys(candidates):
+ if not path.is_dir():
+ continue
+ score = sum(
+ int((path / relative).exists())
+ for relative in ("Content", "Games", "Xbox360", "Aurora", "Data")
+ )
+ scored.append((score, path))
+ return [path for _, path in sorted(scored, key=lambda value: (-value[0], str(value[1])))]
+
+
+def _children(path: Path) -> list[Path]:
+ try:
+ return [child for child in path.iterdir() if child.is_dir()]
+ except OSError:
+ return []
+
+
+def import_aurora_database(path: str | Path) -> ScanResult:
+ """Read a user-selected Aurora SQLite database in immutable, read-only mode."""
+ source = Path(path).expanduser().resolve()
+ if not source.is_file():
+ raise FileNotFoundError(source)
+ uri = f"{source.as_uri()}?mode=ro&immutable=1"
+ items: list[BackupItem] = []
+ with closing(sqlite3.connect(uri, uri=True)) as connection:
+ connection.row_factory = sqlite3.Row
+ tables = [
+ row[0]
+ for row in connection.execute(
+ "SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'"
+ )
+ ]
+ for table in tables:
+ quoted = '"' + table.replace('"', '""') + '"'
+ columns = {
+ row[1].casefold(): row[1]
+ for row in connection.execute(f"PRAGMA table_info({quoted})")
+ }
+ title_col = _column(columns, "titleid", "title_id", "title id")
+ name_col = _column(columns, "name", "title", "displayname", "display_name")
+ if not title_col or not name_col:
+ continue
+ media_col = _column(columns, "mediaid", "media_id")
+ path_col = _column(columns, "path", "file_path", "contentpath", "content_path")
+ selected = [title_col, name_col]
+ selected.extend(value for value in (media_col, path_col) if value)
+ column_sql = ", ".join('"' + value.replace('"', '""') + '"' for value in selected)
+ for row in connection.execute(f"SELECT {column_sql} FROM {quoted}"):
+ title_id = str(row[title_col] or "").strip().upper().replace("0X", "")
+ if title_id and not HEX8_RE.fullmatch(title_id):
+ continue
+ items.append(
+ BackupItem(
+ path=Path(str(row[path_col] or "")) if path_col else source,
+ title_id=title_id,
+ name=str(row[name_col] or title_id or "Unknown"),
+ format="Aurora database",
+ media_id=str(row[media_col] or "").strip().upper() if media_col else "",
+ size=0,
+ )
+ )
+ if items:
+ break
+ warnings = [] if items else ["No compatible Aurora title table was found"]
+ return ScanResult(source, items, warnings, utc_now())
+
+
+def _column(columns: dict[str, str], *aliases: str) -> str | None:
+ return next((columns[alias] for alias in aliases if alias in columns), None)
+
+
+class CollectionIntelligenceService:
+ def __init__(self, db_path: str | Path = DATABASE_PATH) -> None:
+ self.db_path = Path(db_path)
+ if self.db_path == DATABASE_PATH:
+ ensure_app_dirs()
+ else:
+ self.db_path.parent.mkdir(parents=True, exist_ok=True)
+ # Use the application's complete initializer so CLI-only collection
+ # workflows also have the legacy library and normalized knowledge tables.
+ from database import DatabaseManager
+
+ DatabaseManager(str(self.db_path))
+
+ @contextmanager
+ def _connect(self):
+ connection = sqlite3.connect(self.db_path)
+ connection.row_factory = sqlite3.Row
+ try:
+ yield connection
+ connection.commit()
+ except Exception:
+ connection.rollback()
+ raise
+ finally:
+ connection.close()
+
+ def analyze(self, root: str | Path, title_lookup=None) -> CollectionAnalysis:
+ return self.analyze_result(scan_local_target(root, title_lookup), "local")
+
+ def analyze_aurora(self, path: str | Path) -> CollectionAnalysis:
+ return self.analyze_result(import_aurora_database(path), "aurora")
+
+ def analyze_result(self, result: ScanResult, source_kind: str) -> CollectionAnalysis:
+ compatibility: dict[str, UpdateCompatibility] = {}
+ issues: list[CollectionIssue] = []
+ with self._connect() as connection:
+ for item in result.items:
+ compatibility[str(item.path)] = self._compatibility(connection, item)
+ if item.status != "ready":
+ issues.append(
+ CollectionIssue(
+ "warning",
+ "incomplete",
+ str(item.path),
+ f"{item.name} is incomplete",
+ "Review the item and restore its missing base content.",
+ )
+ )
+ if not item.title_id:
+ issues.append(
+ CollectionIssue(
+ "warning",
+ "unknown-titleid",
+ str(item.path),
+ f"{item.name} has no readable TitleID",
+ "Inspect its executable or add a metadata override.",
+ )
+ )
+ if item.title_id and not item.media_id:
+ issues.append(
+ CollectionIssue(
+ "info",
+ "unknown-mediaid",
+ str(item.path),
+ f"{item.name} has no MediaID",
+ "Scan an executable or package before choosing a title update.",
+ )
+ )
+ for note in item.notes:
+ if "missing" in note.casefold() or "could not" in note.casefold():
+ issues.append(
+ CollectionIssue(
+ "warning", "scan-note", str(item.path), note, "Verify this item."
+ )
+ )
+
+ groups: dict[tuple[str, str], list[BackupItem]] = {}
+ for item in result.items:
+ if item.title_id:
+ groups.setdefault((item.title_id, item.media_id), []).append(item)
+ for key, matches in groups.items():
+ if len(matches) > 1:
+ issues.append(
+ CollectionIssue(
+ "info",
+ "duplicate-release",
+ key[0],
+ f"{len(matches)} copies share TitleID {key[0]} and "
+ f"MediaID {key[1] or 'unknown'}",
+ "Hash the copies and keep intentional regional or revision variants.",
+ )
+ )
+ expected = max((item.disc_count for item in matches), default=0)
+ present = {item.disc_number for item in matches if item.disc_number}
+ for item in matches:
+ for note in item.notes:
+ disc_match = re.search(r"Discs found: ([0-9, ]+) of ([0-9]+)", note)
+ if disc_match:
+ present.update(
+ int(value.strip())
+ for value in disc_match.group(1).split(",")
+ if value.strip()
+ )
+ expected = max(expected, int(disc_match.group(2)))
+ missing = sorted(set(range(1, expected + 1)) - present) if expected else []
+ if missing:
+ issues.append(
+ CollectionIssue(
+ "warning",
+ "missing-disc",
+ key[0],
+ f"Missing disc(s) {', '.join(map(str, missing))} of {expected}",
+ "Locate or restore the missing disc backup.",
+ )
+ )
+ score = max(
+ 0,
+ 100
+ - sum(
+ {"error": 15, "warning": 7, "info": 2}.get(issue.severity, 1)
+ for issue in issues
+ ),
+ )
+ analysis = CollectionAnalysis(result, compatibility, issues, score)
+ analysis.snapshot_id = self._save_snapshot(connection, analysis, source_kind)
+ connection.commit()
+ return analysis
+
+ @staticmethod
+ def _compatibility(
+ connection: sqlite3.Connection, item: BackupItem
+ ) -> UpdateCompatibility:
+ if not item.title_id:
+ return UpdateCompatibility("", item.media_id, "unknown", reason="TitleID is unknown")
+ rows = connection.execute(
+ "SELECT media_id, version FROM title_updates WHERE titleid=? ORDER BY version DESC",
+ (item.title_id,),
+ ).fetchall()
+ if not rows:
+ return UpdateCompatibility(
+ item.title_id, item.media_id, "none", reason="No title updates are catalogued"
+ )
+ exact = [
+ row
+ for row in rows
+ if item.media_id and (row["media_id"] or "").upper() == item.media_id
+ ]
+ if exact:
+ return UpdateCompatibility(
+ item.title_id, item.media_id, "compatible", exact[0]["version"] or "", len(exact)
+ )
+ if not item.media_id:
+ return UpdateCompatibility(
+ item.title_id,
+ "",
+ "media-id-required",
+ rows[0]["version"] or "",
+ len(rows),
+ "Updates exist, but exact compatibility requires a MediaID",
+ )
+ return UpdateCompatibility(
+ item.title_id,
+ item.media_id,
+ "incompatible",
+ rows[0]["version"] or "",
+ len(rows),
+ "No catalogued update matches this MediaID",
+ )
+
+ def _save_snapshot(
+ self, connection: sqlite3.Connection, analysis: CollectionAnalysis, source_kind: str
+ ) -> int:
+ result = analysis.result
+ cursor = connection.execute(
+ """
+ INSERT INTO collection_snapshots
+ (source_kind, source_location, label, started_at, completed_at,
+ item_count, total_size, health_score, status, warnings_json)
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'completed', ?)
+ """,
+ (
+ source_kind,
+ str(result.root),
+ result.root.name,
+ result.scanned_at,
+ utc_now(),
+ len(result.items),
+ result.total_size,
+ analysis.health_score,
+ json.dumps(result.warnings),
+ ),
+ )
+ if cursor.lastrowid is None:
+ raise sqlite3.DatabaseError("Collection snapshot did not return an ID")
+ snapshot_id = int(cursor.lastrowid)
+ for item in result.items:
+ match = analysis.compatibility[str(item.path)]
+ connection.execute(
+ """
+ INSERT INTO collection_items
+ (snapshot_id, titleid, media_id, name, format, content_type,
+ path, size, disc_number, disc_count, status, compatibility, notes_json)
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+ """,
+ (
+ snapshot_id,
+ item.title_id,
+ item.media_id,
+ item.name,
+ item.format,
+ item.content_type,
+ str(item.path),
+ item.size,
+ item.disc_number,
+ item.disc_count,
+ item.status,
+ match.status,
+ json.dumps(item.notes),
+ ),
+ )
+ return snapshot_id
+
+ def create_repair_plan(self, analysis: CollectionAnalysis) -> int:
+ """Persist a preview only; no filesystem operation is executed."""
+ with self._connect() as connection:
+ cursor = connection.execute(
+ """
+ INSERT INTO repair_plans(snapshot_id, created_at, status, summary_json)
+ VALUES (?, ?, 'preview', ?)
+ """,
+ (
+ analysis.snapshot_id,
+ utc_now(),
+ json.dumps({"health_score": analysis.health_score, "issues": len(analysis.issues)}),
+ ),
+ )
+ plan_id = int(cursor.lastrowid)
+ for issue in analysis.issues:
+ connection.execute(
+ """
+ INSERT INTO repair_actions
+ (plan_id, action_type, target, reason, destructive, details_json)
+ VALUES (?, ?, ?, ?, ?, ?)
+ """,
+ (
+ plan_id,
+ issue.code,
+ issue.target,
+ issue.message,
+ int(issue.destructive),
+ json.dumps(asdict(issue)),
+ ),
+ )
+ connection.commit()
+ return plan_id
+
+ def set_override(
+ self,
+ identifier_value: str,
+ property_name: str,
+ value: str,
+ *,
+ identifier_type: str = "titleid",
+ entity_type: str = "game",
+ notes: str = "",
+ ) -> None:
+ with self._connect() as connection:
+ connection.execute(
+ """
+ INSERT INTO metadata_overrides
+ (entity_type, identifier_type, identifier_value, property,
+ value, notes, updated_at)
+ VALUES (?, ?, ?, ?, ?, ?, ?)
+ ON CONFLICT(entity_type, identifier_type, identifier_value, property)
+ DO UPDATE SET value=excluded.value, notes=excluded.notes,
+ updated_at=excluded.updated_at
+ """,
+ (
+ entity_type,
+ identifier_type,
+ identifier_value.upper(),
+ property_name,
+ value,
+ notes,
+ utc_now(),
+ ),
+ )
+ connection.commit()
+
+ def list_overrides(self, identifier_value: str | None = None) -> list[dict]:
+ with self._connect() as connection:
+ if identifier_value:
+ rows = connection.execute(
+ """
+ SELECT * FROM metadata_overrides
+ WHERE identifier_value=? ORDER BY property
+ """,
+ (identifier_value.upper(),),
+ ).fetchall()
+ else:
+ rows = connection.execute(
+ "SELECT * FROM metadata_overrides ORDER BY updated_at DESC"
+ ).fetchall()
+ return [dict(row) for row in rows]
+
+ def hash_and_match(self, path: str | Path) -> list[dict]:
+ source = Path(path).resolve()
+ stat = source.stat()
+ crc = 0
+ hashes = (hashlib.md5(), hashlib.sha1(), hashlib.sha256())
+ with source.open("rb") as handle:
+ for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+ crc = zlib.crc32(chunk, crc)
+ for digest in hashes:
+ digest.update(chunk)
+ values = {
+ "crc32": f"{crc & 0xFFFFFFFF:08X}",
+ "md5": hashes[0].hexdigest().upper(),
+ "sha1": hashes[1].hexdigest().upper(),
+ "sha256": hashes[2].hexdigest().upper(),
+ }
+ with self._connect() as connection:
+ connection.execute(
+ """
+ INSERT INTO local_file_hashes
+ (path, size, modified_ns, crc32, md5, sha1, sha256, calculated_at)
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+ ON CONFLICT(path, size, modified_ns) DO UPDATE SET
+ crc32=excluded.crc32, md5=excluded.md5, sha1=excluded.sha1,
+ sha256=excluded.sha256, calculated_at=excluded.calculated_at
+ """,
+ (
+ str(source),
+ stat.st_size,
+ stat.st_mtime_ns,
+ values["crc32"],
+ values["md5"],
+ values["sha1"],
+ values["sha256"],
+ utc_now(),
+ ),
+ )
+ file_hash_id = int(
+ connection.execute(
+ "SELECT id FROM local_file_hashes WHERE path=? AND size=? AND modified_ns=?",
+ (str(source), stat.st_size, stat.st_mtime_ns),
+ ).fetchone()[0]
+ )
+ matches: list[dict] = []
+ for kind, value in values.items():
+ rows = connection.execute(
+ """
+ SELECT e.id, e.entity_type, e.canonical_name, i.identifier_type
+ FROM entity_identifiers i JOIN knowledge_entities e ON e.id=i.entity_id
+ WHERE i.identifier_type=? AND UPPER(i.normalized_value)=?
+ """,
+ (kind, value),
+ ).fetchall()
+ for row in rows:
+ match = dict(row)
+ match["matched_by"] = kind
+ matches.append(match)
+ connection.execute(
+ """
+ INSERT OR IGNORE INTO preservation_matches
+ (file_hash_id, entity_id, identifier_type,
+ identifier_value, matched_at)
+ VALUES (?, ?, ?, ?, ?)
+ """,
+ (file_hash_id, row["id"], kind, value, utc_now()),
+ )
+ connection.commit()
+ return matches
+
+ def export_manifest(
+ self, analysis: CollectionAnalysis, destination: str | Path | None = None
+ ) -> Path:
+ ensure_app_dirs()
+ target = Path(destination) if destination else EXPORTS_DIR / "collection-manifest.json"
+ target.parent.mkdir(parents=True, exist_ok=True)
+ payload = analysis.to_dict()
+ payload.update({"generated_at": utc_now(), "application": "UnityScraper"})
+ payload["metadata_overrides"] = self.list_overrides()
+ target.write_text(json.dumps(payload, indent=2), encoding="utf-8")
+ return target
+
+ def export_aurora_layout(
+ self,
+ analysis: CollectionAnalysis,
+ destination: str | Path,
+ artwork_directory: str | Path | None = None,
+ ) -> Path:
+ """Export an explicit Aurora-friendly Games and Assets layout."""
+ target = Path(destination).expanduser().resolve()
+ games = target / "Games"
+ assets = target / "Assets"
+ games.mkdir(parents=True, exist_ok=True)
+ if artwork_directory:
+ assets.mkdir(parents=True, exist_ok=True)
+ exported: list[dict] = []
+ for item in analysis.result.items:
+ safe_name = re.sub(r'[<>:"/\\|?*]+', "_", item.name).strip(" .") or "Unknown"
+ folder_name = f"{safe_name} [{item.title_id}]" if item.title_id else safe_name
+ output = games / folder_name
+ if output.exists():
+ raise FileExistsError(f"Aurora export destination exists: {output}")
+ if item.path.is_dir() and target.is_relative_to(item.path.resolve()):
+ raise ValueError("Aurora export destination cannot be inside its source")
+ if item.path.is_dir():
+ shutil.copytree(item.path, output)
+ elif item.path.is_file():
+ output.mkdir(parents=True)
+ shutil.copy2(item.path, output / item.path.name)
+ else:
+ continue
+ artwork_files = []
+ if artwork_directory and item.title_id:
+ source_art = Path(artwork_directory)
+ for extension in (".png", ".jpg", ".jpeg"):
+ candidate = source_art / f"{item.title_id}{extension}"
+ if candidate.is_file():
+ artwork_target = assets / item.title_id
+ artwork_target.mkdir(parents=True, exist_ok=True)
+ copied = artwork_target / candidate.name
+ shutil.copy2(candidate, copied)
+ artwork_files.append(str(copied.relative_to(target)))
+ exported.append(
+ {
+ "titleid": item.title_id,
+ "media_id": item.media_id,
+ "source": str(item.path),
+ "destination": str(output.relative_to(target)),
+ "artwork": artwork_files,
+ }
+ )
+ (target / "unityscraper-aurora-manifest.json").write_text(
+ json.dumps({"schema": 1, "generated_at": utc_now(), "items": exported}, indent=2),
+ encoding="utf-8",
+ )
+ return target
+
+ def export_html(
+ self, analysis: CollectionAnalysis, destination: str | Path | None = None
+ ) -> Path:
+ ensure_app_dirs()
+ target = Path(destination) if destination else EXPORTS_DIR / "collection-report.html"
+ rows = []
+ for item in analysis.result.items:
+ compatibility = analysis.compatibility[str(item.path)]
+ rows.append(
+ ""
+ f"| {html.escape(item.name)} | {html.escape(item.title_id)} | "
+ f"{html.escape(item.media_id)} | {html.escape(item.format)} | "
+ f"{html.escape(compatibility.status)} | {item.size} |
"
+ )
+ document = f"""
+UnityScraper Collection
+
+Xbox 360 Collection Report
+Health score: {analysis.health_score}/100
+Generated {html.escape(utc_now())}; source {html.escape(str(analysis.result.root))}
+| Game | TitleID | MediaID | Format |
+Title update | Bytes |
|---|
{''.join(rows)}
+"""
+ target.parent.mkdir(parents=True, exist_ok=True)
+ target.write_text(document, encoding="utf-8")
+ return target
+
+ def export_provenance(self, destination: str | Path) -> Path:
+ target = Path(destination)
+ with self._connect() as connection:
+ KnowledgeRepository(connection).ensure_schema()
+ rows = connection.execute(
+ """
+ SELECT e.entity_type, e.canonical_name, f.property, f.value,
+ f.confidence, s.name source, c.source_url, c.source_title
+ FROM knowledge_facts f
+ JOIN knowledge_entities e ON e.id=f.entity_id
+ JOIN knowledge_sources s ON s.id=f.source_id
+ LEFT JOIN fact_citations c ON c.fact_id=f.id
+ ORDER BY e.entity_type, e.canonical_name, f.property
+ """
+ ).fetchall()
+ target.parent.mkdir(parents=True, exist_ok=True)
+ target.write_text(
+ json.dumps({"generated_at": utc_now(), "facts": [dict(row) for row in rows]}, indent=2),
+ encoding="utf-8",
+ )
+ return target
diff --git a/console_sync.py b/console_sync.py
new file mode 100644
index 0000000..133561f
--- /dev/null
+++ b/console_sync.py
@@ -0,0 +1,501 @@
+"""Persistent, resumable console inventory and FTP synchronization."""
+
+from __future__ import annotations
+
+import ftplib
+import hashlib
+import json
+import posixpath
+import sqlite3
+import threading
+import time
+from contextlib import contextmanager
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path, PurePosixPath
+from typing import Callable, Iterable
+
+from app_paths import DATABASE_PATH
+from backup_manager import FtpBackupClient, FtpTarget
+from backup_service import ensure_backup_schema
+from database_migrations import ensure_application_schema
+
+
+def utc_now() -> str:
+ return datetime.now(timezone.utc).isoformat()
+
+
+class TransferPaused(RuntimeError):
+ """Raised internally after a job reaches a resumable pause point."""
+
+
+@dataclass(frozen=True)
+class ConsoleFile:
+ path: str
+ size: int | None
+ modified_at: str = ""
+ is_directory: bool = False
+
+
+@dataclass(frozen=True)
+class SyncComparison:
+ only_on_pc: tuple[str, ...]
+ only_on_console: tuple[str, ...]
+ size_mismatches: tuple[str, ...]
+ matching: tuple[str, ...]
+
+
+class ConsoleSyncService:
+ """Store sync state in SQLite and execute one explicit job at a time."""
+
+ def __init__(self, db_path: str | Path = DATABASE_PATH) -> None:
+ self.db_path = Path(db_path)
+ self.db_path.parent.mkdir(parents=True, exist_ok=True)
+ self._pause = threading.Event()
+ with self._connect() as connection:
+ ensure_backup_schema(connection)
+ ensure_application_schema(connection)
+ connection.execute(
+ """
+ UPDATE console_transfer_jobs
+ SET status='paused', error_message='Application exited during transfer',
+ updated_at=?
+ WHERE status='transferring'
+ """,
+ (utc_now(),),
+ )
+
+ @contextmanager
+ def _connect(self):
+ connection = sqlite3.connect(self.db_path)
+ connection.row_factory = sqlite3.Row
+ try:
+ yield connection
+ connection.commit()
+ except Exception:
+ connection.rollback()
+ raise
+ finally:
+ connection.close()
+
+ def enqueue(
+ self,
+ direction: str,
+ local_path: str | Path,
+ remote_path: str,
+ *,
+ target_id: int | None = None,
+ priority: int = 100,
+ bandwidth_limit: int = 0,
+ expected_sha256: str = "",
+ ) -> int:
+ if direction not in {"upload", "download"}:
+ raise ValueError("direction must be upload or download")
+ local = Path(local_path).expanduser().resolve()
+ total = local.stat().st_size if direction == "upload" and local.is_file() else 0
+ now = utc_now()
+ with self._connect() as connection:
+ cursor = connection.execute(
+ """
+ INSERT INTO console_transfer_jobs
+ (target_id, direction, local_path, remote_path, total_bytes,
+ status, priority, bandwidth_limit, expected_sha256,
+ created_at, updated_at)
+ VALUES (?, ?, ?, ?, ?, 'queued', ?, ?, ?, ?, ?)
+ """,
+ (
+ target_id,
+ direction,
+ str(local),
+ _remote_path(remote_path),
+ total,
+ priority,
+ max(0, bandwidth_limit),
+ expected_sha256.lower(),
+ now,
+ now,
+ ),
+ )
+ return int(cursor.lastrowid)
+
+ def list_jobs(self, status: str | None = None) -> list[dict]:
+ with self._connect() as connection:
+ if status:
+ rows = connection.execute(
+ """
+ SELECT * FROM console_transfer_jobs
+ WHERE status=? ORDER BY priority, created_at
+ """,
+ (status,),
+ ).fetchall()
+ else:
+ rows = connection.execute(
+ "SELECT * FROM console_transfer_jobs ORDER BY created_at DESC"
+ ).fetchall()
+ return [dict(row) for row in rows]
+
+ def pause(self, job_id: int | None = None) -> None:
+ self._pause.set()
+ if job_id is not None:
+ with self._connect() as connection:
+ connection.execute(
+ """
+ UPDATE console_transfer_jobs SET status='paused', updated_at=?
+ WHERE id=? AND status IN ('queued', 'transferring')
+ """,
+ (utc_now(), job_id),
+ )
+
+ def resume(self, job_id: int) -> None:
+ self._pause.clear()
+ with self._connect() as connection:
+ connection.execute(
+ """
+ UPDATE console_transfer_jobs
+ SET status='queued', error_message=NULL, updated_at=?
+ WHERE id=? AND status IN ('paused', 'failed')
+ """,
+ (utc_now(), job_id),
+ )
+
+ def run_next(
+ self,
+ target: FtpTarget,
+ progress: Callable[[int, int], None] | None = None,
+ ) -> dict | None:
+ with self._connect() as connection:
+ row = connection.execute(
+ """
+ SELECT * FROM console_transfer_jobs
+ WHERE status='queued' ORDER BY priority, created_at LIMIT 1
+ """
+ ).fetchone()
+ return self.run_job(int(row["id"]), target, progress) if row else None
+
+ def run_job(
+ self,
+ job_id: int,
+ target: FtpTarget,
+ progress: Callable[[int, int], None] | None = None,
+ ) -> dict:
+ self._pause.clear()
+ with self._connect() as connection:
+ row = connection.execute(
+ "SELECT * FROM console_transfer_jobs WHERE id=?", (job_id,)
+ ).fetchone()
+ if not row:
+ raise KeyError(f"Unknown transfer job: {job_id}")
+ if row["status"] not in {"queued", "paused", "failed"}:
+ raise ValueError(f"Job {job_id} cannot run from status {row['status']}")
+ connection.execute(
+ """
+ UPDATE console_transfer_jobs
+ SET status='transferring', error_message=NULL, updated_at=? WHERE id=?
+ """,
+ (utc_now(), job_id),
+ )
+ job = dict(row)
+ try:
+ if job["direction"] == "upload":
+ transferred, total = self._upload(job, target, progress)
+ else:
+ transferred, total = self._download(job, target, progress)
+ self._verify(job, target, total)
+ status, error = "completed", None
+ except TransferPaused as exc:
+ transferred = self._progress(job_id)
+ total = int(job["total_bytes"])
+ status, error = "paused", str(exc)
+ except Exception as exc:
+ transferred = self._progress(job_id)
+ total = int(job["total_bytes"])
+ status, error = "failed", str(exc)
+ with self._connect() as connection:
+ connection.execute(
+ """
+ UPDATE console_transfer_jobs SET transferred_bytes=?, total_bytes=?,
+ status=?, error_message=?, updated_at=? WHERE id=?
+ """,
+ (transferred, total, status, error, utc_now(), job_id),
+ )
+ result = connection.execute(
+ "SELECT * FROM console_transfer_jobs WHERE id=?", (job_id,)
+ ).fetchone()
+ return dict(result)
+
+ def _upload(self, job: dict, target: FtpTarget, progress) -> tuple[int, int]:
+ local = Path(job["local_path"])
+ if not local.is_file():
+ raise FileNotFoundError(local)
+ total = local.stat().st_size
+ remote = _remote_path(job["remote_path"])
+ partial = remote + ".partial"
+ started = time.monotonic()
+ with _ftp(target) as ftp:
+ FtpBackupClient._mkdirs(ftp, str(PurePosixPath(remote).parent))
+ offset = _remote_size(ftp, partial) or 0
+ if offset > total:
+ ftp.delete(partial)
+ offset = 0
+ job["_session_offset"] = offset
+ with local.open("rb") as handle:
+ handle.seek(offset)
+
+ def callback(chunk: bytes) -> None:
+ current = handle.tell()
+ self._checkpoint(int(job["id"]), current, total, started, job, progress)
+
+ ftp.storbinary(
+ f"STOR {partial}",
+ handle,
+ blocksize=64 * 1024,
+ callback=callback,
+ rest=offset or None,
+ )
+ if _remote_size(ftp, remote) is not None:
+ ftp.delete(remote)
+ ftp.rename(partial, remote)
+ return total, total
+
+ def _download(self, job: dict, target: FtpTarget, progress) -> tuple[int, int]:
+ local = Path(job["local_path"])
+ local.parent.mkdir(parents=True, exist_ok=True)
+ partial = local.with_suffix(local.suffix + ".partial")
+ remote = _remote_path(job["remote_path"])
+ started = time.monotonic()
+ with _ftp(target) as ftp:
+ total = _remote_size(ftp, remote)
+ if total is None:
+ raise FileNotFoundError(remote)
+ offset = partial.stat().st_size if partial.exists() else 0
+ if offset > total:
+ partial.unlink()
+ offset = 0
+ job["_session_offset"] = offset
+ with partial.open("ab" if offset else "wb") as handle:
+
+ def callback(chunk: bytes) -> None:
+ handle.write(chunk)
+ self._checkpoint(
+ int(job["id"]), handle.tell(), total, started, job, progress
+ )
+
+ ftp.retrbinary(
+ f"RETR {remote}",
+ callback,
+ blocksize=64 * 1024,
+ rest=offset or None,
+ )
+ if partial.stat().st_size != total:
+ raise IOError(f"Transfer size mismatch: {partial.stat().st_size} != {total}")
+ partial.replace(local)
+ return total, total
+
+ def _checkpoint(self, job_id, current, total, started, job, progress) -> None:
+ if self._pause.is_set():
+ raise TransferPaused("Transfer paused; partial data was kept for resume")
+ limit = int(job["bandwidth_limit"] or 0)
+ if limit:
+ expected = (current - int(job.get("_session_offset", 0))) / limit
+ delay = expected - (time.monotonic() - started)
+ if delay > 0:
+ time.sleep(min(delay, 0.25))
+ with self._connect() as connection:
+ connection.execute(
+ """
+ UPDATE console_transfer_jobs SET transferred_bytes=?, total_bytes=?,
+ updated_at=? WHERE id=?
+ """,
+ (current, total, utc_now(), job_id),
+ )
+ if progress:
+ progress(current, total)
+
+ def _verify(self, job: dict, target: FtpTarget, total: int) -> None:
+ if job["direction"] == "upload":
+ with _ftp(target) as ftp:
+ actual = _remote_size(ftp, _remote_path(job["remote_path"]))
+ if actual != total:
+ raise IOError(f"Remote verification failed: {actual} != {total}")
+ else:
+ local = Path(job["local_path"])
+ if local.stat().st_size != total:
+ raise IOError("Local size verification failed")
+ expected = (job["expected_sha256"] or "").lower()
+ if expected and _sha256(local) != expected:
+ raise IOError("Downloaded file failed SHA-256 verification")
+
+ def _progress(self, job_id: int) -> int:
+ with self._connect() as connection:
+ row = connection.execute(
+ "SELECT transferred_bytes FROM console_transfer_jobs WHERE id=?", (job_id,)
+ ).fetchone()
+ return int(row[0]) if row else 0
+
+ def capture_inventory(
+ self,
+ target: FtpTarget,
+ root: str = "/Hdd1",
+ *,
+ target_id: int | None = None,
+ max_entries: int = 100_000,
+ ) -> int:
+ """Capture a read-only remote inventory; no console file is changed."""
+ root = _remote_path(root)
+ with self._connect() as connection:
+ cursor = connection.execute(
+ """
+ INSERT INTO console_inventory_snapshots
+ (target_id, label, root, captured_at, status)
+ VALUES (?, ?, ?, ?, 'running')
+ """,
+ (target_id, target.host, root, utc_now()),
+ )
+ snapshot_id = int(cursor.lastrowid)
+ try:
+ with _ftp(target) as ftp:
+ entries = list(_walk_ftp(ftp, root, max_entries))
+ with self._connect() as connection:
+ for entry in entries:
+ connection.execute(
+ """
+ INSERT INTO console_inventory_items
+ (snapshot_id, remote_path, size, modified_at, is_directory)
+ VALUES (?, ?, ?, ?, ?)
+ """,
+ (
+ snapshot_id,
+ entry.path,
+ entry.size,
+ entry.modified_at,
+ int(entry.is_directory),
+ ),
+ )
+ connection.execute(
+ """
+ UPDATE console_inventory_snapshots
+ SET item_count=?, status='completed' WHERE id=?
+ """,
+ (len(entries), snapshot_id),
+ )
+ return snapshot_id
+ except Exception as exc:
+ with self._connect() as connection:
+ connection.execute(
+ """
+ UPDATE console_inventory_snapshots
+ SET status='failed', error_message=? WHERE id=?
+ """,
+ (str(exc), snapshot_id),
+ )
+ raise
+
+ def list_snapshots(self) -> list[dict]:
+ with self._connect() as connection:
+ rows = connection.execute(
+ """
+ SELECT * FROM console_inventory_snapshots
+ ORDER BY captured_at DESC
+ """
+ ).fetchall()
+ return [dict(row) for row in rows]
+
+ def compare(self, local_root: str | Path, snapshot_id: int) -> SyncComparison:
+ root = Path(local_root).resolve()
+ local = {
+ child.relative_to(root).as_posix(): child.stat().st_size
+ for child in root.rglob("*")
+ if child.is_file()
+ }
+ with self._connect() as connection:
+ snapshot = connection.execute(
+ "SELECT root FROM console_inventory_snapshots WHERE id=?", (snapshot_id,)
+ ).fetchone()
+ if not snapshot:
+ raise KeyError(snapshot_id)
+ remote_root = snapshot["root"].rstrip("/") + "/"
+ rows = connection.execute(
+ """
+ SELECT remote_path, size FROM console_inventory_items
+ WHERE snapshot_id=? AND is_directory=0
+ """,
+ (snapshot_id,),
+ ).fetchall()
+ remote = {
+ row["remote_path"].removeprefix(remote_root): row["size"]
+ for row in rows
+ }
+ local_names, remote_names = set(local), set(remote)
+ common = local_names & remote_names
+ mismatches = tuple(sorted(name for name in common if local[name] != remote[name]))
+ return SyncComparison(
+ tuple(sorted(local_names - remote_names)),
+ tuple(sorted(remote_names - local_names)),
+ mismatches,
+ tuple(sorted(common - set(mismatches))),
+ )
+
+
+def _remote_path(path: str) -> str:
+ normalized = posixpath.normpath("/" + path.replace("\\", "/").lstrip("/"))
+ if normalized == "/.." or normalized.startswith("/../"):
+ raise ValueError("Remote path escapes the configured root")
+ return normalized
+
+
+def _ftp(target: FtpTarget) -> ftplib.FTP:
+ ftp = ftplib.FTP()
+ ftp.connect(target.host, target.port, timeout=target.timeout)
+ ftp.login(target.username, target.password)
+ ftp.voidcmd("TYPE I")
+ return ftp
+
+
+def _remote_size(ftp: ftplib.FTP, path: str) -> int | None:
+ try:
+ return ftp.size(path)
+ except ftplib.error_perm:
+ return None
+
+
+def _walk_ftp(ftp: ftplib.FTP, root: str, limit: int) -> Iterable[ConsoleFile]:
+ pending = [root]
+ seen = 0
+ while pending:
+ directory = pending.pop()
+ try:
+ entries = list(ftp.mlsd(directory))
+ except (ftplib.error_perm, AttributeError):
+ entries = []
+ for name in ftp.nlst(directory):
+ entries.append((PurePosixPath(name).name, {}))
+ for name, facts in entries:
+ if name in {".", ".."}:
+ continue
+ path = _remote_path(posixpath.join(directory, name))
+ entry_type = facts.get("type", "")
+ is_directory = entry_type == "dir"
+ if not entry_type:
+ current = ftp.pwd()
+ try:
+ ftp.cwd(path)
+ is_directory = True
+ except ftplib.error_perm:
+ is_directory = False
+ finally:
+ ftp.cwd(current)
+ size = None if is_directory else _remote_size(ftp, path)
+ yield ConsoleFile(path, size, facts.get("modify", ""), is_directory)
+ seen += 1
+ if seen >= limit:
+ raise RuntimeError(f"Remote inventory exceeded safety limit of {limit} entries")
+ if is_directory:
+ pending.append(path)
+
+
+def _sha256(path: Path) -> str:
+ digest = hashlib.sha256()
+ with path.open("rb") as handle:
+ for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+ digest.update(chunk)
+ return digest.hexdigest()
diff --git a/dat_adapters.py b/dat_adapters.py
index 41c8c35..fd3f6e9 100644
--- a/dat_adapters.py
+++ b/dat_adapters.py
@@ -3,6 +3,7 @@
from __future__ import annotations
import hashlib
+import re
import xml.etree.ElementTree as ET
from pathlib import Path
from typing import Iterable
@@ -83,6 +84,13 @@ def parse_dat(text: str, entity_type: str) -> list[EntityRecord]:
continue
identifiers: list[Identifier] = []
facts: list[Fact] = []
+ alternate_names: tuple[str, ...] = ()
+
+ base_name, inferred = _infer_release_fields(game_name)
+ if base_name and base_name != game_name:
+ alternate_names = (base_name,)
+ facts.append(Fact("release_group", base_name))
+ facts.extend(Fact(key, value) for key, value in inferred.items())
for property_name, xml_name in (
("description", "description"),
@@ -134,6 +142,7 @@ def parse_dat(text: str, entity_type: str) -> list[EntityRecord]:
entity_type=entity_type,
canonical_name=game_name,
identifiers=tuple(identifiers),
+ names=alternate_names,
facts=tuple(unique_facts),
)
)
@@ -145,3 +154,36 @@ def _child_text(node: ET.Element | None, name: str) -> str:
return ""
child = node.find(name)
return (child.text or "").strip() if child is not None else ""
+
+
+def _infer_release_fields(name: str) -> tuple[str, dict[str, str]]:
+ """Extract common No-Intro/Redump naming tags without replacing DAT facts."""
+ tags = re.findall(r"\(([^()]*)\)", name)
+ base = re.sub(r"\s+\([^()]*\)", "", name).strip()
+ inferred: dict[str, str] = {}
+ regions = {
+ "USA",
+ "Europe",
+ "Japan",
+ "World",
+ "Australia",
+ "Asia",
+ "Korea",
+ "China",
+ "Canada",
+ }
+ language_codes = {"En", "Fr", "De", "Es", "It", "Pt", "Ja", "Ko", "Zh", "Ru", "Nl"}
+ for tag in tags:
+ values = [value.strip() for value in tag.split(",")]
+ if values and all(value in language_codes for value in values):
+ inferred.setdefault("languages", ", ".join(values))
+ if tag in regions:
+ inferred.setdefault("region", tag)
+ if re.match(r"^(Rev|Revision|Version|v)\s*", tag, re.IGNORECASE):
+ inferred.setdefault("revision", tag)
+ disc = re.match(r"^Disc\s+(\d+)(?:\s+of\s+(\d+))?$", tag, re.IGNORECASE)
+ if disc:
+ inferred.setdefault("disc_number", disc.group(1))
+ if disc.group(2):
+ inferred.setdefault("disc_count", disc.group(2))
+ return base, inferred
diff --git a/database.py b/database.py
index dec1155..482cc7b 100644
--- a/database.py
+++ b/database.py
@@ -13,6 +13,7 @@
from contextlib import contextmanager
from app_paths import DATABASE_PATH, ensure_app_dirs
from backup_service import ensure_backup_schema
+from database_migrations import ensure_application_schema
from knowledge_base import KnowledgeRepository, is_unknown
logger = logging.getLogger(__name__)
@@ -144,6 +145,7 @@ def init_database(self):
KnowledgeRepository(conn).ensure_schema()
ensure_backup_schema(conn)
+ ensure_application_schema(conn)
logger.info(f"Database initialized at {self.db_path}")
diff --git a/database_migrations.py b/database_migrations.py
new file mode 100644
index 0000000..cbe6b62
--- /dev/null
+++ b/database_migrations.py
@@ -0,0 +1,242 @@
+"""Versioned, additive database migrations and backup helpers."""
+
+from __future__ import annotations
+
+import shutil
+import sqlite3
+from datetime import datetime, timezone
+from pathlib import Path
+
+
+SCHEMA_VERSION = 4
+
+
+def _now() -> str:
+ return datetime.now(timezone.utc).isoformat()
+
+
+def create_database_backup(db_path: str | Path, destination: str | Path | None = None) -> Path:
+ """Create a consistent SQLite backup without modifying the source database."""
+ source_path = Path(db_path)
+ if not source_path.exists():
+ raise FileNotFoundError(source_path)
+ stamp = datetime.now().strftime("%Y%m%d-%H%M%S")
+ target = Path(destination) if destination else source_path.with_suffix(f".{stamp}.bak")
+ if source_path.resolve() == target.resolve():
+ raise ValueError("Backup destination must differ from the active database")
+ target.parent.mkdir(parents=True, exist_ok=True)
+ with sqlite3.connect(source_path) as source, sqlite3.connect(target) as output:
+ source.backup(output)
+ return target
+
+
+def restore_database_backup(backup_path: str | Path, db_path: str | Path) -> Path:
+ """Restore a user-selected backup after validating that it is SQLite."""
+ source = Path(backup_path)
+ target = Path(db_path)
+ if source.resolve() == target.resolve():
+ raise ValueError("Restore source must differ from the active database")
+ with sqlite3.connect(source) as connection:
+ connection.execute("PRAGMA schema_version").fetchone()
+ target.parent.mkdir(parents=True, exist_ok=True)
+ temporary = target.with_suffix(target.suffix + ".restore")
+ shutil.copy2(source, temporary)
+ temporary.replace(target)
+ return target
+
+
+def ensure_application_schema(connection: sqlite3.Connection) -> int:
+ """Apply every additive UnityScraper schema migration."""
+ connection.execute(
+ """
+ CREATE TABLE IF NOT EXISTS app_schema_migrations (
+ version INTEGER PRIMARY KEY,
+ name TEXT NOT NULL,
+ applied_at TEXT NOT NULL
+ )
+ """
+ )
+ applied = {
+ int(row[0])
+ for row in connection.execute("SELECT version FROM app_schema_migrations").fetchall()
+ }
+ migrations = (
+ (1, "collection intelligence", _migration_collection),
+ (2, "preservation records", _migration_preservation),
+ (3, "console synchronization", _migration_console_sync),
+ (4, "user overrides and recovery", _migration_reliability),
+ )
+ for version, name, migration in migrations:
+ if version in applied:
+ continue
+ migration(connection)
+ connection.execute(
+ "INSERT INTO app_schema_migrations(version, name, applied_at) VALUES (?, ?, ?)",
+ (version, name, _now()),
+ )
+ connection.execute(f"PRAGMA user_version = {SCHEMA_VERSION}")
+ return SCHEMA_VERSION
+
+
+def _migration_collection(connection: sqlite3.Connection) -> None:
+ connection.executescript(
+ """
+ CREATE TABLE IF NOT EXISTS collection_snapshots (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ source_kind TEXT NOT NULL,
+ source_location TEXT NOT NULL,
+ label TEXT,
+ started_at TEXT NOT NULL,
+ completed_at TEXT,
+ item_count INTEGER NOT NULL DEFAULT 0,
+ total_size INTEGER NOT NULL DEFAULT 0,
+ health_score INTEGER,
+ status TEXT NOT NULL DEFAULT 'running',
+ warnings_json TEXT
+ );
+ CREATE TABLE IF NOT EXISTS collection_items (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ snapshot_id INTEGER NOT NULL,
+ titleid TEXT,
+ media_id TEXT,
+ name TEXT NOT NULL,
+ format TEXT NOT NULL,
+ content_type TEXT,
+ path TEXT NOT NULL,
+ size INTEGER NOT NULL DEFAULT 0,
+ disc_number INTEGER,
+ disc_count INTEGER,
+ status TEXT NOT NULL,
+ compatibility TEXT,
+ notes_json TEXT,
+ FOREIGN KEY(snapshot_id) REFERENCES collection_snapshots(id)
+ );
+ CREATE INDEX IF NOT EXISTS idx_collection_items_titleid
+ ON collection_items(titleid, media_id);
+ CREATE INDEX IF NOT EXISTS idx_collection_items_snapshot
+ ON collection_items(snapshot_id);
+ """
+ )
+
+
+def _migration_preservation(connection: sqlite3.Connection) -> None:
+ connection.executescript(
+ """
+ CREATE TABLE IF NOT EXISTS local_file_hashes (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ path TEXT NOT NULL,
+ size INTEGER NOT NULL,
+ modified_ns INTEGER NOT NULL,
+ crc32 TEXT,
+ md5 TEXT,
+ sha1 TEXT,
+ sha256 TEXT,
+ calculated_at TEXT NOT NULL,
+ UNIQUE(path, size, modified_ns)
+ );
+ CREATE INDEX IF NOT EXISTS idx_local_hash_sha256 ON local_file_hashes(sha256);
+ CREATE INDEX IF NOT EXISTS idx_local_hash_sha1 ON local_file_hashes(sha1);
+ CREATE TABLE IF NOT EXISTS preservation_matches (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ file_hash_id INTEGER NOT NULL,
+ entity_id INTEGER NOT NULL,
+ identifier_type TEXT NOT NULL,
+ identifier_value TEXT NOT NULL,
+ matched_at TEXT NOT NULL,
+ UNIQUE(file_hash_id, entity_id, identifier_type),
+ FOREIGN KEY(file_hash_id) REFERENCES local_file_hashes(id),
+ FOREIGN KEY(entity_id) REFERENCES knowledge_entities(id)
+ );
+ CREATE TABLE IF NOT EXISTS repair_plans (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ snapshot_id INTEGER,
+ created_at TEXT NOT NULL,
+ status TEXT NOT NULL DEFAULT 'preview',
+ summary_json TEXT,
+ FOREIGN KEY(snapshot_id) REFERENCES collection_snapshots(id)
+ );
+ CREATE TABLE IF NOT EXISTS repair_actions (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ plan_id INTEGER NOT NULL,
+ action_type TEXT NOT NULL,
+ target TEXT NOT NULL,
+ reason TEXT NOT NULL,
+ destructive INTEGER NOT NULL DEFAULT 0,
+ status TEXT NOT NULL DEFAULT 'proposed',
+ details_json TEXT,
+ FOREIGN KEY(plan_id) REFERENCES repair_plans(id)
+ );
+ """
+ )
+
+
+def _migration_console_sync(connection: sqlite3.Connection) -> None:
+ connection.executescript(
+ """
+ CREATE TABLE IF NOT EXISTS console_inventory_snapshots (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ target_id INTEGER,
+ label TEXT,
+ root TEXT NOT NULL,
+ captured_at TEXT NOT NULL,
+ item_count INTEGER NOT NULL DEFAULT 0,
+ status TEXT NOT NULL DEFAULT 'running',
+ error_message TEXT,
+ FOREIGN KEY(target_id) REFERENCES backup_targets(id)
+ );
+ CREATE TABLE IF NOT EXISTS console_inventory_items (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ snapshot_id INTEGER NOT NULL,
+ remote_path TEXT NOT NULL,
+ size INTEGER,
+ modified_at TEXT,
+ is_directory INTEGER NOT NULL DEFAULT 0,
+ titleid TEXT,
+ media_id TEXT,
+ UNIQUE(snapshot_id, remote_path),
+ FOREIGN KEY(snapshot_id) REFERENCES console_inventory_snapshots(id)
+ );
+ CREATE TABLE IF NOT EXISTS console_transfer_jobs (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ target_id INTEGER,
+ direction TEXT NOT NULL CHECK(direction IN ('upload', 'download')),
+ local_path TEXT NOT NULL,
+ remote_path TEXT NOT NULL,
+ total_bytes INTEGER NOT NULL DEFAULT 0,
+ transferred_bytes INTEGER NOT NULL DEFAULT 0,
+ status TEXT NOT NULL DEFAULT 'queued',
+ priority INTEGER NOT NULL DEFAULT 100,
+ bandwidth_limit INTEGER NOT NULL DEFAULT 0,
+ expected_sha256 TEXT,
+ error_message TEXT,
+ created_at TEXT NOT NULL,
+ updated_at TEXT NOT NULL,
+ FOREIGN KEY(target_id) REFERENCES backup_targets(id)
+ );
+ CREATE INDEX IF NOT EXISTS idx_console_jobs_status
+ ON console_transfer_jobs(status, priority, created_at);
+ """
+ )
+
+
+def _migration_reliability(connection: sqlite3.Connection) -> None:
+ connection.executescript(
+ """
+ CREATE TABLE IF NOT EXISTS metadata_overrides (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ entity_type TEXT NOT NULL,
+ identifier_type TEXT NOT NULL,
+ identifier_value TEXT NOT NULL,
+ property TEXT NOT NULL,
+ value TEXT NOT NULL,
+ notes TEXT,
+ updated_at TEXT NOT NULL,
+ UNIQUE(entity_type, identifier_type, identifier_value, property)
+ );
+ CREATE TABLE IF NOT EXISTS recovery_state (
+ key TEXT PRIMARY KEY,
+ value_json TEXT NOT NULL,
+ updated_at TEXT NOT NULL
+ );
+ """
+ )
diff --git a/i18n.py b/i18n.py
index 5f32ae0..b257bd6 100644
--- a/i18n.py
+++ b/i18n.py
@@ -11,7 +11,7 @@
logger = logging.getLogger(__name__)
# Supported languages
-SUPPORTED_LANGUAGES = ['en', 'es', 'fr', 'de', 'ja']
+SUPPORTED_LANGUAGES = ['en', 'es', 'fr', 'de', 'it', 'pt', 'ja']
# Translation strings
TRANSLATIONS = {
@@ -214,6 +214,31 @@
}
}
+TRANSLATIONS['it'] = {
+ **TRANSLATIONS['en'],
+ 'title': 'UnityScraper',
+ 'titleids': 'TitleID:',
+ 'output_dir': 'Cartella di output:',
+ 'browse': 'Sfoglia',
+ 'settings': 'Impostazioni',
+ 'start_download': 'Avvia download',
+ 'stop': 'Ferma',
+ 'test_connection': 'Verifica connessione',
+ 'verify_integrity': 'Verifica integrita file',
+}
+TRANSLATIONS['pt'] = {
+ **TRANSLATIONS['en'],
+ 'title': 'UnityScraper',
+ 'titleids': 'TitleIDs:',
+ 'output_dir': 'Pasta de saida:',
+ 'browse': 'Procurar',
+ 'settings': 'Configuracoes',
+ 'start_download': 'Iniciar download',
+ 'stop': 'Parar',
+ 'test_connection': 'Testar conexao',
+ 'verify_integrity': 'Verificar integridade dos arquivos',
+}
+
class Translator:
"""Language translator for GUI"""
diff --git a/integration_tests.py b/integration_tests.py
index 191e0b7..c526c0e 100644
--- a/integration_tests.py
+++ b/integration_tests.py
@@ -472,19 +472,23 @@ def test_queue_with_speed_monitoring(self):
from queue_manager import DownloadQueue
from resume import DownloadProgress
- queue = DownloadQueue()
-
- # Add item to queue with correct signature
- result = queue.add_item('TESTID00', 'cover', 'http://example.com/file.bin', '/tmp/file.bin', priority=1)
- self.assertTrue(result)
-
- # Create progress tracker (would be used during download)
- progress = DownloadProgress(1000, Path('file.bin'))
- progress.update(500)
-
- # Both should work independently
- self.assertIsNotNone(queue.get_next_item())
- self.assertGreater(progress.percentage, 0)
+ with tempfile.TemporaryDirectory() as temp_dir:
+ queue = DownloadQueue(Path(temp_dir) / "queue.json")
+
+ # Add item to queue with correct signature
+ result = queue.add_item(
+ 'TESTID00', 'cover', 'http://example.com/file.bin',
+ '/tmp/file.bin', priority=1
+ )
+ self.assertTrue(result)
+
+ # Create progress tracker (would be used during download)
+ progress = DownloadProgress(1000, Path('file.bin'))
+ progress.update(500)
+
+ # Both should work independently
+ self.assertIsNotNone(queue.get_next_item())
+ self.assertGreater(progress.percentage, 0)
def test_integrity_checker_with_database(self):
"""Test integrity checker works with database"""
diff --git a/main.py b/main.py
index 6d31b90..9994f1d 100644
--- a/main.py
+++ b/main.py
@@ -879,6 +879,24 @@ def main():
type=str,
help='Output directory for --convert-iso'
)
+ parser.add_argument('--analyze-collection', type=str, help='Analyze a local collection root')
+ parser.add_argument('--aurora-db', type=str, help='Analyze an Aurora database read-only')
+ parser.add_argument('--collection-manifest', type=str, help='Write a preservation manifest')
+ parser.add_argument('--collection-html', type=str, help='Write an offline HTML report')
+ parser.add_argument(
+ '--create-repair-plan', action='store_true',
+ help='Save a non-destructive repair-plan preview for the collection'
+ )
+ parser.add_argument('--match-file', type=str, help='Match a file against imported DAT hashes')
+ parser.add_argument('--export-provenance', type=str, help='Export knowledge provenance as JSON')
+ parser.add_argument('--backup-database', type=str, help='Create a consistent SQLite backup')
+ parser.add_argument('--ftp-snapshot', type=str, help='Capture a read-only remote inventory root')
+ parser.add_argument('--ftp-download', type=str, help='Queue and run a resumable FTP download')
+ parser.add_argument('--ftp-local-path', type=str, help='Local path for an FTP sync operation')
+ parser.add_argument(
+ '--ftp-bandwidth-limit', type=int, default=0,
+ help='FTP sync limit in bytes per second (0 = unlimited)'
+ )
args = parser.parse_args()
@@ -967,6 +985,96 @@ def main():
logger.error(f"DAT import failed: {e}")
sys.exit(1)
+ if (
+ args.analyze_collection
+ or args.aurora_db
+ or args.match_file
+ or args.export_provenance
+ or args.backup_database
+ ):
+ try:
+ from app_paths import DATABASE_PATH
+ from collection_intelligence import CollectionIntelligenceService
+ from database_migrations import create_database_backup
+
+ service = CollectionIntelligenceService()
+ if args.backup_database:
+ backup = create_database_backup(DATABASE_PATH, args.backup_database)
+ logger.info("Database backup written to %s", backup)
+ if args.match_file:
+ matches = service.hash_and_match(args.match_file)
+ logger.info("Preservation matches: %s", json.dumps(matches, indent=2))
+ if args.export_provenance:
+ output = service.export_provenance(args.export_provenance)
+ logger.info("Provenance written to %s", output)
+ if args.analyze_collection or args.aurora_db:
+ analysis = (
+ service.analyze(args.analyze_collection)
+ if args.analyze_collection
+ else service.analyze_aurora(args.aurora_db)
+ )
+ logger.info(
+ "Collection health %s/100: %s item(s), %s issue(s)",
+ analysis.health_score,
+ len(analysis.result.items),
+ len(analysis.issues),
+ )
+ if args.collection_manifest:
+ logger.info(
+ "Manifest written to %s",
+ service.export_manifest(analysis, args.collection_manifest),
+ )
+ if args.collection_html:
+ logger.info(
+ "HTML report written to %s",
+ service.export_html(analysis, args.collection_html),
+ )
+ if args.create_repair_plan:
+ logger.info(
+ "Repair-plan preview saved as %s",
+ service.create_repair_plan(analysis),
+ )
+ sys.exit(0)
+ except Exception as e:
+ logger.error("Collection operation failed: %s", e)
+ sys.exit(1)
+
+ if args.ftp_snapshot or args.ftp_download:
+ if not args.ftp_host:
+ parser.error("--ftp-host is required for console sync")
+ try:
+ from backup_manager import FtpTarget
+ from console_sync import ConsoleSyncService
+
+ target = FtpTarget(
+ host=args.ftp_host,
+ port=args.ftp_port,
+ username=args.ftp_user,
+ password=args.ftp_password,
+ content_root=args.ftp_content_root,
+ )
+ sync = ConsoleSyncService()
+ if args.ftp_snapshot:
+ snapshot = sync.capture_inventory(target, args.ftp_snapshot)
+ logger.info("Console inventory snapshot %s completed", snapshot)
+ if args.ftp_download:
+ if not args.ftp_local_path:
+ parser.error("--ftp-local-path is required with --ftp-download")
+ job = sync.enqueue(
+ "download",
+ args.ftp_local_path,
+ args.ftp_download,
+ bandwidth_limit=args.ftp_bandwidth_limit,
+ )
+ result = sync.run_job(job, target)
+ logger.info("Console transfer %s: %s", job, result["status"])
+ if result["status"] != "completed":
+ sys.exit(1)
+ sys.exit(0)
+ except Exception as e:
+ logger.error("Console sync failed: %s", e)
+ sys.exit(1)
+
if (
args.scan_backups
or args.install_package
diff --git a/modern_gui.py b/modern_gui.py
index 1252b70..772185c 100644
--- a/modern_gui.py
+++ b/modern_gui.py
@@ -8,6 +8,7 @@
from __future__ import annotations
import json
+import sqlite3
import tkinter as tk
import webbrowser
from pathlib import Path
@@ -31,14 +32,18 @@
)
from app_version import DISPLAY_VERSION
from backup_gui import BackupPage
+from collection_gui import CollectionPage
+from collection_intelligence import CollectionIntelligenceService
from backup_service import BackupService
from database import DatabaseManager
+from database_migrations import create_database_backup, restore_database_backup
from diagnostics import create_diagnostics_bundle
from knowledge_service import KnowledgeService
from knowledge_gui import KnowledgePage
from library_service import GameSummary, LibraryService
from platform_support import desktop_font_family, open_path
from setup_wizard import run_first_run_wizard
+from updater import VersionChecker
APP_VERSION = DISPLAY_VERSION
@@ -178,6 +183,7 @@ def __init__(self, root: tk.Tk) -> None:
self.library = LibraryService()
self.knowledge = KnowledgeService()
self.backups = BackupService()
+ self.collections = CollectionIntelligenceService()
self.database = DatabaseManager()
self.current_game: str | None = None
@@ -185,6 +191,9 @@ def __init__(self, root: tk.Tk) -> None:
ensure_user_titleids_file()
self.root.title(f"UnityScraper {APP_VERSION}")
+ config = self._read_config()
+ scale = max(0.8, min(2.0, float(config.get("ui_scale", 1.0))))
+ self.root.tk.call("tk", "scaling", scale)
self.root.geometry("1220x780")
self.root.minsize(980, 640)
self._set_icon()
@@ -302,6 +311,7 @@ def _build_shell(self) -> None:
("ADD GAMES", self.show_add_games),
("DOWNLOADS", self.show_downloads),
("BACKUP MANAGER", self.show_backups),
+ ("COLLECTIONS", self.show_collections),
("KNOWLEDGE", self.show_knowledge),
("ARCHIVE HEALTH", self.show_health),
("SETTINGS", self.show_settings),
@@ -323,6 +333,9 @@ def _build_shell(self) -> None:
self.content.rowconfigure(1, weight=1)
self.shell.bind("", self._resize_shell)
self.root.after_idle(lambda: self._resize_shell(None))
+ for index, (_, callback) in enumerate(pages, start=1):
+ self.root.bind(f"", lambda _event, action=callback: action())
+ self.root.bind("", lambda _event: self.show_library())
self.show_library()
def _resize_shell(self, _event: tk.Event[Any] | None) -> None:
@@ -360,6 +373,15 @@ def show_backups(self) -> None:
self._page_header,
)
+ def show_collections(self) -> None:
+ self._clear_content()
+ self.collection_page = CollectionPage(
+ self.root,
+ self.content,
+ self.collections,
+ self._page_header,
+ )
+
def _page_header(self, title: str, subtitle: str) -> None:
header = ttk.Frame(self.content, style="Content.TFrame")
header.grid(row=0, column=0, sticky="ew", pady=(0, 14))
@@ -825,6 +847,7 @@ def show_settings(self) -> None:
self.rate_var = tk.DoubleVar(value=float(config.get("rate_limit", 0.35)))
self.timeout_var = tk.IntVar(value=int(config.get("timeout", 30)))
self.retries_var = tk.IntVar(value=int(config.get("max_retries", 3)))
+ self.scale_var = tk.DoubleVar(value=float(config.get("ui_scale", 1.0)))
ttk.Label(panel, text="Archive folder").grid(row=0, column=0, sticky=tk.W)
ttk.Entry(panel, textvariable=self.output_var).grid(
@@ -842,6 +865,7 @@ def show_settings(self) -> None:
).grid(row=2, column=0, columnspan=3, sticky=tk.W)
fields = (
+ ("Interface scale", self.scale_var, 0.8, 2.0),
("Parallel workers", self.workers_var, 1, 16),
("Minimum request delay", self.rate_var, 0.1, 5.0),
("Timeout seconds", self.timeout_var, 5, 120),
@@ -889,10 +913,14 @@ def _save_settings(self) -> None:
"rate_limit": self.rate_var.get(),
"timeout": self.timeout_var.get(),
"max_retries": self.retries_var.get(),
+ "ui_scale": self.scale_var.get(),
}
)
CONFIG_PATH.write_text(json.dumps(config, indent=2), encoding="utf-8")
- messagebox.showinfo("Settings", "Settings saved.", parent=self.root)
+ self.root.tk.call("tk", "scaling", max(0.8, min(2.0, self.scale_var.get())))
+ messagebox.showinfo(
+ "Settings", "Settings saved. Interface scaling applies immediately.", parent=self.root
+ )
@staticmethod
def _read_config() -> dict[str, Any]:
@@ -934,6 +962,15 @@ def show_about(self) -> None:
text="Export Diagnostics ZIP",
command=self._export_diagnostics,
).pack(anchor=tk.W, pady=4)
+ ttk.Button(panel, text="Back Up Database", command=self._backup_database).pack(
+ anchor=tk.W, pady=4
+ )
+ ttk.Button(panel, text="Restore Database", command=self._restore_database).pack(
+ anchor=tk.W, pady=4
+ )
+ ttk.Button(panel, text="Check for Updates", command=self._check_updates).pack(
+ anchor=tk.W, pady=4
+ )
ttk.Button(
panel,
text="Open Application Data",
@@ -968,6 +1005,108 @@ def _export_diagnostics(self) -> None:
)
_open_path(bundle.parent)
+ def _backup_database(self) -> None:
+ selected = filedialog.asksaveasfilename(
+ parent=self.root,
+ title="Back up UnityScraper database",
+ defaultextension=".db",
+ filetypes=(("SQLite database", "*.db"), ("All files", "*.*")),
+ )
+ if selected:
+ try:
+ output = create_database_backup(DATABASE_PATH, selected)
+ except (OSError, sqlite3.Error) as exc:
+ messagebox.showerror("Database backup failed", str(exc), parent=self.root)
+ else:
+ messagebox.showinfo("Database backup", f"Created:\n{output}", parent=self.root)
+
+ def _restore_database(self) -> None:
+ selected = filedialog.askopenfilename(
+ parent=self.root,
+ title="Choose UnityScraper database backup",
+ filetypes=(("SQLite database", "*.db *.bak"), ("All files", "*.*")),
+ )
+ if not selected:
+ return
+ if not messagebox.askyesno(
+ "Restore database",
+ "UnityScraper will back up the current database, restore the selected "
+ "file, and then close. Continue?",
+ parent=self.root,
+ ):
+ return
+ try:
+ fallback = create_database_backup(DATABASE_PATH)
+ restore_database_backup(selected, DATABASE_PATH)
+ except (OSError, sqlite3.Error) as exc:
+ messagebox.showerror("Database restore failed", str(exc), parent=self.root)
+ return
+ messagebox.showinfo(
+ "Database restored",
+ f"Restore completed. Previous database backup:\n{fallback}\n\n"
+ "UnityScraper will now close.",
+ parent=self.root,
+ )
+ self.root.destroy()
+
+ def _check_updates(self) -> None:
+ checker = VersionChecker(timeout=10)
+
+ def worker() -> None:
+ has_update, info = checker.check_for_updates()
+ self.root.after(0, lambda: self._show_update(checker, has_update, info))
+
+ import threading
+
+ threading.Thread(target=worker, daemon=True).start()
+
+ def _show_update(self, checker: VersionChecker, has_update: bool, info: dict | None) -> None:
+ if not has_update or not info:
+ messagebox.showinfo("Updates", "You are running the latest release.", parent=self.root)
+ return
+ if not info.get("asset"):
+ messagebox.showinfo(
+ "Update available",
+ VersionChecker.format_update_message(info),
+ parent=self.root,
+ )
+ return
+ if not messagebox.askyesno(
+ "Verified update available",
+ f"UnityScraper {info['version']} is available for this platform.\n\n"
+ "Download and verify the release package now?",
+ parent=self.root,
+ ):
+ return
+ destination = filedialog.askdirectory(parent=self.root, title="Choose download folder")
+ if not destination:
+ return
+
+ def worker() -> None:
+ try:
+ package = checker.download_verified_update(info, destination)
+ except Exception as exc:
+ self.root.after(
+ 0,
+ lambda error=exc: messagebox.showerror(
+ "Update failed", str(error), parent=self.root
+ ),
+ )
+ else:
+ self.root.after(
+ 0,
+ lambda: messagebox.showinfo(
+ "Update verified",
+ f"Verified package downloaded to:\n{package}\n\n"
+ "Close UnityScraper before installing it.",
+ parent=self.root,
+ ),
+ )
+
+ import threading
+
+ threading.Thread(target=worker, daemon=True).start()
+
def _open_log(self) -> None:
if GUI_LOG_PATH.exists():
_open_path(GUI_LOG_PATH)
diff --git a/packaging/linux/io.github.trapemall.UnityScraper.metainfo.xml b/packaging/linux/io.github.trapemall.UnityScraper.metainfo.xml
index 85da907..a3e8484 100644
--- a/packaging/linux/io.github.trapemall.UnityScraper.metainfo.xml
+++ b/packaging/linux/io.github.trapemall.UnityScraper.metainfo.xml
@@ -22,6 +22,6 @@
https://github.com/TrapEmAll/UnityScraper/issues
-
+
diff --git a/plugins.py b/plugins.py
index 48a15b1..ed44c14 100644
--- a/plugins.py
+++ b/plugins.py
@@ -3,7 +3,9 @@
Allows custom metadata collectors and extensions
"""
+import json
import logging
+from dataclasses import dataclass
from abc import ABC, abstractmethod
from pathlib import Path
from typing import List, Dict, Any, Optional
@@ -11,6 +13,38 @@
import sys
logger = logging.getLogger(__name__)
+PLUGIN_API_VERSION = 1
+
+
+@dataclass(frozen=True)
+class PluginManifest:
+ """Stable v1 plugin contract. Plugin code is only loaded when enabled."""
+
+ plugin_id: str
+ name: str
+ version: str
+ api_version: int
+ entrypoint: str
+ permissions: tuple[str, ...] = ()
+
+ @classmethod
+ def load(cls, path: Path) -> "PluginManifest":
+ data = json.loads(path.read_text(encoding="utf-8"))
+ manifest = cls(
+ plugin_id=str(data["id"]),
+ name=str(data["name"]),
+ version=str(data["version"]),
+ api_version=int(data["api_version"]),
+ entrypoint=str(data["entrypoint"]),
+ permissions=tuple(str(value) for value in data.get("permissions", [])),
+ )
+ if manifest.api_version != PLUGIN_API_VERSION:
+ raise ValueError(
+ f"Plugin API {manifest.api_version} is unsupported; expected {PLUGIN_API_VERSION}"
+ )
+ if "/" in manifest.entrypoint or "\\" in manifest.entrypoint:
+ raise ValueError("Plugin entrypoint must be a file in its plugin directory")
+ return manifest
class MetadataCollectorPlugin(ABC):
@@ -40,9 +74,17 @@ def validate_titleid(self, titleid: str) -> bool:
class PluginManager:
"""Manages loading and executing plugins"""
- def __init__(self, plugin_dir: str = "plugins"):
+ def __init__(
+ self,
+ plugin_dir: str = "plugins",
+ enabled_plugins: Optional[List[str]] = None,
+ allow_legacy: bool = False,
+ ):
self.plugin_dir = Path(plugin_dir)
self.plugins: Dict[str, MetadataCollectorPlugin] = {}
+ self.manifests: Dict[str, PluginManifest] = {}
+ self.enabled_plugins = set(enabled_plugins or [])
+ self.allow_legacy = allow_legacy
self._load_plugins()
def _load_plugins(self):
@@ -51,16 +93,22 @@ def _load_plugins(self):
logger.debug(f"Plugin directory not found: {self.plugin_dir}")
return
- for plugin_file in self.plugin_dir.glob("*.py"):
- if plugin_file.name.startswith("_"):
- continue
-
+ for manifest_path in self.plugin_dir.glob("*/plugin.json"):
try:
- self._load_plugin_file(plugin_file)
+ manifest = PluginManifest.load(manifest_path)
+ self.manifests[manifest.plugin_id] = manifest
+ if manifest.plugin_id in self.enabled_plugins:
+ self._load_plugin_file(manifest_path.parent / manifest.entrypoint, manifest)
except Exception as e:
- logger.warning(f"Failed to load plugin {plugin_file.name}: {e}")
+ logger.warning(f"Failed to load plugin {manifest_path}: {e}")
+ if self.allow_legacy:
+ for plugin_file in self.plugin_dir.glob("*.py"):
+ if not plugin_file.name.startswith("_"):
+ self._load_plugin_file(plugin_file)
- def _load_plugin_file(self, file_path: Path):
+ def _load_plugin_file(
+ self, file_path: Path, manifest: Optional[PluginManifest] = None
+ ):
"""Load a single plugin file"""
spec = importlib.util.spec_from_file_location(file_path.stem, file_path)
if spec and spec.loader:
@@ -76,6 +124,9 @@ def _load_plugin_file(self, file_path: Path):
attr is not MetadataCollectorPlugin):
instance = attr()
+ if manifest:
+ instance.name = manifest.name
+ instance.version = manifest.version
self.plugins[instance.name] = instance
logger.info(f"Loaded plugin: {instance.name} v{instance.version}")
@@ -86,6 +137,21 @@ def get_plugin(self, name: str) -> Optional[MetadataCollectorPlugin]:
def list_plugins(self) -> List[str]:
"""List all loaded plugins"""
return list(self.plugins.keys())
+
+ def list_available_plugins(self) -> List[Dict[str, Any]]:
+ """List discovered plugins without importing disabled code."""
+ return [
+ {
+ "id": manifest.plugin_id,
+ "name": manifest.name,
+ "version": manifest.version,
+ "api_version": manifest.api_version,
+ "permissions": list(manifest.permissions),
+ "enabled": manifest.plugin_id in self.enabled_plugins,
+ "loaded": manifest.name in self.plugins,
+ }
+ for manifest in self.manifests.values()
+ ]
def collect_from_plugin(self, plugin_name: str, titleid: str) -> Optional[Dict[str, Any]]:
"""Collect metadata using specific plugin"""
diff --git a/pyproject.toml b/pyproject.toml
index e355604..3292f74 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
[project]
name = "unityscraper"
-version = "0.10.0b1"
+version = "1.0.0b1"
description = "Xbox 360 library, knowledge, preservation, and backup manager"
readme = "README.md"
requires-python = ">=3.10"
diff --git a/queue_manager.py b/queue_manager.py
index ed5ba51..04b98f9 100644
--- a/queue_manager.py
+++ b/queue_manager.py
@@ -5,9 +5,12 @@
import json
import logging
+import os
+import uuid
from pathlib import Path
from typing import List, Dict, Optional
from datetime import datetime
+from app_paths import DATA_DIR, ensure_app_dirs
logger = logging.getLogger(__name__)
@@ -15,8 +18,12 @@
class DownloadQueue:
"""Manage persistent download queue"""
- def __init__(self, queue_file: str = "download_queue.json"):
- self.queue_file = Path(queue_file)
+ def __init__(self, queue_file: str | Path | None = None):
+ if queue_file is None:
+ ensure_app_dirs()
+ self.queue_file = DATA_DIR / "download_queue.json"
+ else:
+ self.queue_file = Path(queue_file)
self.queue: List[Dict] = []
self.load_queue()
@@ -25,7 +32,7 @@ def add_item(self, titleid: str, item_type: str, url: str,
"""Add item to queue"""
try:
item = {
- 'id': f"{titleid}_{item_type}_{len(self.queue)}",
+ 'id': uuid.uuid4().hex,
'titleid': titleid,
'type': item_type, # 'cover' or 'update'
'url': url,
@@ -95,7 +102,6 @@ def retry_failed(self, max_retries: int = 3) -> int:
for item in self.queue:
if item['status'] == 'failed' and item['retry_count'] < max_retries:
item['status'] = 'queued'
- item['retry_count'] += 1
retried += 1
if retried > 0:
self.save_queue()
@@ -143,10 +149,15 @@ def _find_item(self, item_id: str) -> Optional[Dict]:
return None
def save_queue(self):
- """Save queue to JSON file"""
+ """Save the queue atomically so a crash cannot truncate it."""
try:
- with open(self.queue_file, 'w') as f:
- json.dump(self.queue, f, indent=2)
+ self.queue_file.parent.mkdir(parents=True, exist_ok=True)
+ temporary = self.queue_file.with_suffix(self.queue_file.suffix + ".tmp")
+ with temporary.open("w", encoding="utf-8") as handle:
+ json.dump(self.queue, handle, indent=2)
+ handle.flush()
+ os.fsync(handle.fileno())
+ temporary.replace(self.queue_file)
except Exception as e:
logger.error(f"Failed to save queue: {e}")
@@ -154,8 +165,17 @@ def load_queue(self):
"""Load queue from JSON file"""
try:
if self.queue_file.exists():
- with open(self.queue_file, 'r') as f:
+ with self.queue_file.open("r", encoding="utf-8") as f:
self.queue = json.load(f)
+ recovered = 0
+ for item in self.queue:
+ if item.get("status") == "downloading":
+ item["status"] = "queued"
+ item["error"] = "Recovered after an interrupted session"
+ recovered += 1
+ if recovered:
+ self.save_queue()
+ logger.info(f"Recovered {recovered} interrupted queue items")
logger.info(f"Loaded {len(self.queue)} items from queue file")
else:
self.queue = []
diff --git a/scripts/generate_sbom.py b/scripts/generate_sbom.py
new file mode 100644
index 0000000..fe43e6e
--- /dev/null
+++ b/scripts/generate_sbom.py
@@ -0,0 +1,59 @@
+"""Generate a minimal CycloneDX SBOM from the active Python environment."""
+
+from __future__ import annotations
+
+import argparse
+import importlib.metadata
+import json
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT))
+
+from app_version import APP_VERSION # noqa: E402
+
+
+def build_sbom() -> dict:
+ components = []
+ for distribution in sorted(
+ importlib.metadata.distributions(),
+ key=lambda item: (item.metadata.get("Name") or "").casefold(),
+ ):
+ name = distribution.metadata.get("Name")
+ if not name:
+ continue
+ components.append(
+ {
+ "type": "library",
+ "name": name,
+ "version": distribution.version,
+ "purl": f"pkg:pypi/{name.casefold().replace('_', '-')}@{distribution.version}",
+ }
+ )
+ return {
+ "bomFormat": "CycloneDX",
+ "specVersion": "1.5",
+ "version": 1,
+ "metadata": {
+ "timestamp": datetime.now(timezone.utc).isoformat(),
+ "component": {
+ "type": "application",
+ "name": "UnityScraper",
+ "version": APP_VERSION,
+ },
+ },
+ "components": components,
+ }
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--output", type=Path, default=Path("UnityScraper-SBOM.cdx.json"))
+ args = parser.parse_args()
+ args.output.write_text(json.dumps(build_sbom(), indent=2), encoding="utf-8")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/setup_wizard.py b/setup_wizard.py
index 2d5f690..f694679 100644
--- a/setup_wizard.py
+++ b/setup_wizard.py
@@ -29,7 +29,7 @@ def __init__(self, parent: tk.Misc) -> None:
self.parent = parent
self.completed = False
self.title("Welcome to UnityScraper")
- self.geometry("620x420")
+ self.geometry("620x500")
self.resizable(False, False)
self.transient(parent)
self.grab_set()
@@ -39,6 +39,7 @@ def __init__(self, parent: tk.Misc) -> None:
self.output_var = tk.StringVar(value=str(DOWNLOADS_DIR))
self.titleids_var = tk.StringVar()
+ self.collection_var = tk.StringVar()
self._build()
@@ -81,6 +82,16 @@ def _build(self) -> None:
text="Comma-separated, for example: 4D53082D, 584109A8",
).pack(anchor=tk.W)
+ ttk.Label(container, text="Optional collection folder").pack(anchor=tk.W, pady=(16, 0))
+ collection_row = ttk.Frame(container)
+ collection_row.pack(fill=tk.X, pady=(5, 4))
+ ttk.Entry(collection_row, textvariable=self.collection_var).pack(
+ side=tk.LEFT, fill=tk.X, expand=True
+ )
+ ttk.Button(collection_row, text="Browse", command=self._browse_collection).pack(
+ side=tk.LEFT, padx=(8, 0)
+ )
+
ttk.Separator(container).pack(fill=tk.X, pady=22)
ttk.Label(
@@ -111,6 +122,11 @@ def _browse(self) -> None:
if selected:
self.output_var.set(selected)
+ def _browse_collection(self) -> None:
+ selected = filedialog.askdirectory(parent=self, title="Choose Xbox 360 collection")
+ if selected:
+ self.collection_var.set(selected)
+
def _finish(self) -> None:
output = Path(self.output_var.get()).expanduser()
try:
@@ -137,6 +153,12 @@ def _finish(self) -> None:
"rate_limit": float(config.get("rate_limit", 0.35)),
"timeout": int(config.get("timeout", 30)),
"max_retries": int(config.get("max_retries", 3)),
+ "collection_roots": (
+ [self.collection_var.get().strip()]
+ if self.collection_var.get().strip()
+ else config.get("collection_roots", [])
+ ),
+ "ui_scale": float(config.get("ui_scale", 1.0)),
}
)
CONFIG_PATH.write_text(json.dumps(config, indent=2), encoding="utf-8")
diff --git a/tests.py b/tests.py
index bec300a..35bb62f 100644
--- a/tests.py
+++ b/tests.py
@@ -516,6 +516,21 @@ def test_parse_redump_style_dat(self):
self.assertIn(("serial", "AB-1234"), identifiers)
self.assertIn(("crc32", "1234ABCD"), identifiers)
self.assertIn(("sha1", "0123456789ABCDEF0123456789ABCDEF01234567"), identifiers)
+ facts = {(item.property, item.value) for item in records[0].facts}
+ self.assertIn(("release_group", "Example Game"), facts)
+
+ def test_dat_name_tags_add_release_relationship_facts(self):
+ sample = """
+
+
+
+ """
+ record = parse_dat(sample, "disc_release")[0]
+ facts = {(item.property, item.value) for item in record.facts}
+ self.assertIn(("region", "Europe"), facts)
+ self.assertIn(("languages", "En, Fr"), facts)
+ self.assertIn(("revision", "Rev 2"), facts)
+ self.assertIn(("disc_count", "2"), facts)
def test_parse_sitemap_and_article(self):
sitemap = """
@@ -1007,9 +1022,128 @@ def test_config_validates_and_applies_allowlisted_values(self):
self.assertEqual(self.scraper.config.workers, 8)
self.assertEqual(self.scraper.config.rate_limit, 0.5)
self.assertFalse(self.scraper.config.use_https)
-
-
-def run_tests():
+
+
+class TestUnifiedV1Foundation(unittest.TestCase):
+ def setUp(self):
+ self.temp_dir = tempfile.mkdtemp()
+ self.db_path = Path(self.temp_dir) / "library.db"
+ self.database = DatabaseManager(str(self.db_path))
+
+ def tearDown(self):
+ shutil.rmtree(self.temp_dir)
+
+ def test_versioned_migrations_create_all_foundation_tables(self):
+ import sqlite3
+ from contextlib import closing
+
+ with closing(sqlite3.connect(self.db_path)) as connection:
+ tables = {
+ row[0]
+ for row in connection.execute(
+ "SELECT name FROM sqlite_master WHERE type='table'"
+ )
+ }
+ versions = connection.execute(
+ "SELECT version FROM app_schema_migrations ORDER BY version"
+ ).fetchall()
+ self.assertEqual([row[0] for row in versions], [1, 2, 3, 4])
+ self.assertIn("collection_snapshots", tables)
+ self.assertIn("preservation_matches", tables)
+ self.assertIn("console_transfer_jobs", tables)
+ self.assertIn("metadata_overrides", tables)
+
+ def test_xex_execution_info_is_parsed(self):
+ from backup_manager import inspect_xex
+
+ payload = bytearray(0x200)
+ payload[:4] = b"XEX2"
+ payload[4:8] = (8).to_bytes(4, "big")
+ payload[0x14:0x18] = (1).to_bytes(4, "big")
+ payload[0x18:0x1C] = (0x00040006).to_bytes(4, "big")
+ payload[0x1C:0x20] = (0x80).to_bytes(4, "big")
+ payload[0x80:0x84] = bytes.fromhex("11223344")
+ payload[0x84:0x88] = (0x12345678).to_bytes(4, "big")
+ payload[0x88:0x8C] = (0x10000001).to_bytes(4, "big")
+ payload[0x8C:0x90] = bytes.fromhex("4D5307E6")
+ payload[0x92] = 1
+ payload[0x93] = 2
+ xex = Path(self.temp_dir) / "default.xex"
+ xex.write_bytes(payload)
+ result = inspect_xex(xex)
+ self.assertEqual(result.title_id, "4D5307E6")
+ self.assertEqual(result.media_id, "11223344")
+ self.assertEqual((result.disc_number, result.disc_count), (1, 2))
+
+ def test_aurora_database_is_imported_read_only(self):
+ import sqlite3
+ from contextlib import closing
+ from collection_intelligence import import_aurora_database
+
+ aurora = Path(self.temp_dir) / "content.db"
+ with closing(sqlite3.connect(aurora)) as connection:
+ connection.execute(
+ "CREATE TABLE ContentItems(TitleId TEXT, Name TEXT, MediaId TEXT, Path TEXT)"
+ )
+ connection.execute(
+ "INSERT INTO ContentItems VALUES(?, ?, ?, ?)",
+ ("4D5307E6", "Halo 3", "11223344", "/Hdd1/Games/Halo 3"),
+ )
+ connection.commit()
+ result = import_aurora_database(aurora)
+ self.assertEqual(len(result.items), 1)
+ self.assertEqual(result.items[0].title_id, "4D5307E6")
+ self.assertEqual(result.items[0].media_id, "11223344")
+
+ def test_collection_analysis_uses_exact_media_id(self):
+ from backup_manager import BackupItem, ScanResult
+ from collection_intelligence import CollectionIntelligenceService
+
+ self.database.add_titleid("4D5307E6", "Halo 3")
+ self.database.add_title_update(
+ "4D5307E6", "11223344", "6.0.1", "http://xboxunity.net/example"
+ )
+ item = BackupItem(
+ Path(self.temp_dir) / "Halo 3",
+ "4D5307E6",
+ "Halo 3",
+ "Extracted Xbox 360",
+ 100,
+ media_id="11223344",
+ )
+ result = ScanResult(Path(self.temp_dir), [item], [], "2026-07-23T00:00:00Z")
+ analysis = CollectionIntelligenceService(self.db_path).analyze_result(result, "test")
+ self.assertEqual(analysis.compatibility[str(item.path)].status, "compatible")
+ self.assertEqual(analysis.health_score, 100)
+
+ def test_console_queue_recovers_interrupted_job(self):
+ import sqlite3
+ from contextlib import closing
+ from console_sync import ConsoleSyncService
+
+ service = ConsoleSyncService(self.db_path)
+ job_id = service.enqueue("download", Path(self.temp_dir) / "file.bin", "/Hdd1/file.bin")
+ with closing(sqlite3.connect(self.db_path)) as connection:
+ connection.execute(
+ "UPDATE console_transfer_jobs SET status='transferring' WHERE id=?", (job_id,)
+ )
+ connection.commit()
+ recovered = ConsoleSyncService(self.db_path).list_jobs()[0]
+ self.assertEqual(recovered["status"], "paused")
+
+ def test_updater_selects_platform_asset_and_ignores_checksum(self):
+ from updater import VersionChecker
+
+ assets = [
+ {"name": "UnityScraper-Windows-x64.zip.sha256"},
+ {"name": "UnityScraper-Windows-x64.zip"},
+ {"name": "UnityScraper-Linux-x86_64.tar.gz"},
+ ]
+ selected = VersionChecker.select_asset(assets, "Windows")
+ self.assertEqual(selected["name"], "UnityScraper-Windows-x64.zip")
+
+
+def run_tests():
"""Run all tests"""
# Create test suite
loader = unittest.TestLoader()
@@ -1029,6 +1163,7 @@ def run_tests():
suite.addTests(loader.loadTestsFromTestCase(TestIntegration))
suite.addTests(loader.loadTestsFromTestCase(TestBackupManager))
suite.addTests(loader.loadTestsFromTestCase(TestRestAPI))
+ suite.addTests(loader.loadTestsFromTestCase(TestUnifiedV1Foundation))
# Run tests
runner = unittest.TextTestRunner(verbosity=2)
diff --git a/updater.py b/updater.py
index 23bc857..d1dc0f7 100644
--- a/updater.py
+++ b/updater.py
@@ -6,6 +6,8 @@
import logging
import requests
import json
+import hashlib
+import platform
from pathlib import Path
from typing import Optional, Dict, Tuple
from packaging import version
@@ -65,11 +67,13 @@ def _check_github_api(self) -> Optional[Dict]:
return None
if version.parse(latest_version) > version.parse(self.current_version):
+ asset = self.select_asset(data.get("assets", []))
return {
'version': latest_version,
'name': data.get('name'),
'body': data.get('body'),
- 'download_url': data.get('html_url'),
+ 'download_url': asset.get("browser_download_url") if asset else data.get('html_url'),
+ 'asset': asset,
'published_at': data.get('published_at'),
'source': 'github'
}
@@ -77,6 +81,65 @@ def _check_github_api(self) -> Optional[Dict]:
logger.debug(f"GitHub API check failed: {e}")
return None
+
+ @staticmethod
+ def select_asset(assets: list[Dict], system: str | None = None) -> Optional[Dict]:
+ """Choose the packaged release matching the current desktop platform."""
+ current = (system or platform.system()).casefold()
+ machine = platform.machine().casefold()
+ preferred = []
+ if current == "windows":
+ preferred = ["windows-x64.zip", "windows"]
+ elif current == "linux":
+ preferred = ["linux-x86_64.tar.gz", "linux"]
+ elif current == "darwin":
+ preferred = ["macos", "darwin"]
+ for suffix in preferred:
+ for asset in assets:
+ name = str(asset.get("name", "")).casefold()
+ if suffix in name and not name.endswith(".sha256"):
+ if "arm" not in name or "arm" in machine or "aarch64" in machine:
+ return asset
+ return None
+
+ def download_verified_update(
+ self, update_info: Dict, destination: str | Path
+ ) -> Path:
+ """Download a selected package and require its published SHA-256 sidecar."""
+ asset = update_info.get("asset")
+ if not asset or not asset.get("browser_download_url"):
+ raise ValueError("No packaged update is available for this platform")
+ target_dir = Path(destination)
+ target_dir.mkdir(parents=True, exist_ok=True)
+ target = target_dir / asset["name"]
+ self._download(asset["browser_download_url"], target)
+
+ checksum_url = asset["browser_download_url"] + ".sha256"
+ checksum_response = requests.get(checksum_url, timeout=self.timeout)
+ checksum_response.raise_for_status()
+ expected = checksum_response.text.strip().split()[0].lower()
+ if len(expected) != 64:
+ target.unlink(missing_ok=True)
+ raise ValueError("Release checksum is malformed")
+ hasher = hashlib.sha256()
+ with target.open("rb") as handle:
+ for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+ hasher.update(chunk)
+ digest = hasher.hexdigest()
+ if digest != expected:
+ target.unlink(missing_ok=True)
+ raise ValueError("Downloaded update failed SHA-256 verification")
+ return target
+
+ def _download(self, url: str, target: Path) -> None:
+ temporary = target.with_suffix(target.suffix + ".partial")
+ with requests.get(url, timeout=self.timeout, stream=True) as response:
+ response.raise_for_status()
+ with temporary.open("wb") as handle:
+ for chunk in response.iter_content(1024 * 1024):
+ if chunk:
+ handle.write(chunk)
+ temporary.replace(target)
def _check_version_file(self) -> Optional[Dict]:
"""Check version file from repository"""