diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml new file mode 100644 index 0000000..ce8b335 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug.yml @@ -0,0 +1,55 @@ +name: Bug report +description: Report reproducible UnityScraper behavior +title: "[Bug]: " +labels: ["bug"] +body: + - type: input + id: version + attributes: + label: UnityScraper version + placeholder: 0.10.0-beta.1 or commit SHA + validations: + required: true + - type: input + id: environment + attributes: + label: Environment + placeholder: Windows 11, packaged executable or Python 3.12 + validations: + required: true + - type: dropdown + id: area + attributes: + label: Area + options: + - Desktop interface + - XboxUnity metadata or downloads + - Knowledge import + - Backup scan or verification + - Package or ZIP import + - FTP transfer + - REST API + - Packaging or setup + - Other + validations: + required: true + - type: textarea + id: description + attributes: + label: What happened? + description: Include expected behavior and a minimal reproduction. + validations: + required: true + - type: textarea + id: logs + attributes: + label: Sanitized diagnostics + description: Remove credentials, private paths, keys, and copyrighted data. + render: text + - type: checkboxes + id: safety + attributes: + label: Data safety + options: + - label: I removed credentials, keys, copyrighted payloads, and private information. + required: true diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..4094120 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,5 @@ +blank_issues_enabled: false +contact_links: + - name: Private security report + url: https://github.com/TrapEmAll/UnityScraper/security/advisories/new + about: Report archive, filesystem, credential, or remote-access vulnerabilities privately. diff --git a/.github/ISSUE_TEMPLATE/feature.yml b/.github/ISSUE_TEMPLATE/feature.yml new file mode 100644 index 0000000..1ee3898 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature.yml @@ -0,0 +1,31 @@ +name: Feature request +description: Suggest a focused UnityScraper improvement +title: "[Feature]: " +labels: ["enhancement"] +body: + - type: textarea + id: problem + attributes: + label: Problem + description: What workflow or community need is not handled well today? + validations: + required: true + - type: textarea + id: proposal + attributes: + label: Proposed experience + description: Describe the user-facing behavior, not only an implementation. + validations: + required: true + - type: textarea + id: sources + attributes: + label: Sources or formats + description: Link public specifications and state relevant licenses. + - type: checkboxes + id: boundary + attributes: + label: Project boundary + options: + - label: This request does not require distributing game images, firmware, keys, leaked SDK files, or circumvention tooling. + required: true diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..1c2e488 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,22 @@ +## Summary + +Describe the user-visible change and why it is needed. + +## Validation + +- [ ] `python -m ruff check .` +- [ ] `python -m compileall -q .` +- [ ] `python tests.py` +- [ ] Windows build checked when packaging or GUI imports changed + +## Compatibility + +- [ ] Existing SQLite data remains compatible +- [ ] XboxUnity remains HTTP-only +- [ ] New source content retains provenance and licensing information +- [ ] No copyrighted game payloads, firmware, keys, or SDK files are included + +## Documentation + +- [ ] README or focused documentation updated +- [ ] CHANGELOG updated for user-visible changes diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..e5e3023 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,12 @@ +version: 2 +updates: + - package-ecosystem: pip + directory: / + schedule: + interval: monthly + open-pull-requests-limit: 5 + - package-ecosystem: github-actions + directory: / + schedule: + interval: monthly + open-pull-requests-limit: 5 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..d6b6402 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,82 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + +permissions: + contents: read + +concurrency: + group: ci-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + test: + name: Python ${{ matrix.python }} on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] + python: ["3.10", "3.12"] + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + cache: pip + + - name: Install dependencies + run: python -m pip install -r requirements.txt -r requirements-dev.txt + + - name: Check high-confidence Python errors + run: python -m ruff check . + + - name: Compile Python modules + run: python -m compileall -q . + + - name: Check version consistency + run: python scripts/check_version.py + + - name: Run tests + run: python tests.py + + - name: Run legacy integration tests + run: python integration_tests.py + + windows-build: + name: Windows executable + runs-on: windows-latest + needs: test + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + + - name: Install build dependencies + run: python -m pip install -r requirements.txt pyinstaller + + - name: Build + run: python -m PyInstaller --clean --noconfirm UnityScraper.spec + + - name: Generate checksum + shell: pwsh + run: | + $hash = (Get-FileHash dist\UnityScraper.exe -Algorithm SHA256).Hash.ToLower() + "$hash *UnityScraper.exe" | Set-Content dist\UnityScraper.exe.sha256 + + - uses: actions/upload-artifact@v4 + with: + name: UnityScraper-Windows-x64 + path: | + dist/UnityScraper.exe + dist/UnityScraper.exe.sha256 + if-no-files-found: error diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..99df1cb --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,62 @@ +name: Release + +on: + push: + tags: + - "v*" + +permissions: + contents: write + +jobs: + release: + runs-on: windows-latest + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + + - name: Install build dependencies + run: python -m pip install -r requirements.txt pyinstaller + + - name: Test + shell: pwsh + run: | + python scripts/check_version.py --tag "${{ github.ref_name }}" + python tests.py + python integration_tests.py + + - name: Build executable + run: python -m PyInstaller --clean --noconfirm UnityScraper.spec + + - name: Package release + shell: pwsh + run: | + Copy-Item README.md, CHANGELOG.md, LICENSE dist\ + Compress-Archive ` + -Path dist\UnityScraper.exe, dist\README.md, dist\CHANGELOG.md, dist\LICENSE ` + -DestinationPath UnityScraper-Windows-x64.zip + $hash = (Get-FileHash UnityScraper-Windows-x64.zip -Algorithm SHA256).Hash.ToLower() + "$hash *UnityScraper-Windows-x64.zip" | Set-Content UnityScraper-Windows-x64.zip.sha256 + + - name: Publish GitHub release + shell: pwsh + env: + GH_TOKEN: ${{ github.token }} + run: | + $arguments = @( + "release", "create", "${{ github.ref_name }}", + "UnityScraper-Windows-x64.zip", + "UnityScraper-Windows-x64.zip.sha256", + "--title", "UnityScraper ${{ github.ref_name }}", + "--generate-notes", + "--verify-tag" + ) + if ("${{ github.ref_name }}" -match "-") { + $arguments += "--prerelease" + } + & gh @arguments diff --git a/.gitignore b/.gitignore index 587c23a..72f91d8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,60 +1,55 @@ -ο»Ώ################################################################################ -# This .gitignore file was automatically created by Microsoft(R) Visual Studio. -################################################################################ - -/GUI.zip -/.vs -/__pycache__ - -# Documentation and Reports (not part of core codebase) -COMPLETION_REPORT.md -FEATURES_IMPLEMENTED.md -IMPLEMENTATION_DETAILS.md -IMPLEMENTATION_SUMMARY.md -QUICKREF.md -QUICKSTART_V2.md -INDEX.md -.github/copilot-instructions.md - # Python __pycache__/ *.py[cod] *$py.class *.so .Python -env/ -venv/ +.pytest_cache/ +.mypy_cache/ +.ruff_cache/ +.coverage +coverage.xml +htmlcov/ + +# Environments .venv/ -*.log +venv/ +env/ -# Database and local files +# Runtime data +UnityScraperData/ +unityscrape/ +downloads/ +exports/ +diagnostics/ +logs/ *.db +*.db-shm +*.db-wal *.sqlite -unityscraper.log -unityscraper_gui.log +*.sqlite3 +*.log config.json -unityscrape/ download_queue.json +portable.mode -# Build intermediates -build/ - -# Temporary and partial files -*.tmp +# Partial and temporary files *.partial *.meta +*.tmp +*.bak -# OS files -.DS_Store -Thumbs.db +# Build output +build/ +dist/ +*.spec.bak +*.zip -# IDE +# IDE and operating system +.vs/ .vscode/ .idea/ *.swp *.swo - -# IDE Extensions -/.vs/ -.vscode/settings.json -.idea/ +.DS_Store +Thumbs.db diff --git a/ADVANCED_FEATURES.md b/ADVANCED_FEATURES.md index f1b2d63..c7bb862 100644 --- a/ADVANCED_FEATURES.md +++ b/ADVANCED_FEATURES.md @@ -1,258 +1,72 @@ -# UnityScraper Advanced Features - Quick Reference +# Advanced Features -## πŸš€ Quick Start +This guide covers optional operational features. Start with the +[README](README.md) for installation and normal desktop use. -### GUI Mode -```bash -python GUI.py -``` -- Full visual interface with all new features -- Language selector for 5 languages -- Filters, queue viewer, integrity checker - -### API Mode -```bash -python main.py --api-mode --api-port 8000 -``` -- REST API available at `http://127.0.0.1:8000/api/` -- 12 endpoints for full control - -### CLI Mode -```bash -python main.py [--options] -``` - ---- - -## 🌍 Feature Quick Reference - -| Feature | How to Use | Where | -|---------|-----------|-------| -| **Multi-Language** | Click language dropdown | GUI dropdown | -| **File Verification** | Click "Verify Files" button | GUI button | -| **Check Updates** | Click "Check Updates" button | GUI button | -| **Download Queue** | Click "View Queue" button | GUI button | -| **Filters** | Use status/date dropdowns | GUI filter section | -| **API Server** | `python main.py --api-mode` | Terminal | - ---- - -## πŸ”§ Configuration - -### Via GUI: -- Workers: Spinbox (1-16) -- Rate Limit: Spinbox (0.1-5.0 seconds) -- Timeout: Spinbox (5-120 seconds) -- Bandwidth: Spinbox (0-10000 KB/s) -- Language: Dropdown (en/es/fr/de/ja) - -### Via CLI: -```bash -python main.py --workers 8 --rate 0.5 --bandwidth-limit 1000 -``` - -### Via API: -```bash -curl -X POST http://127.0.0.1:8000/api/config \ - -H "Content-Type: application/json" \ - -d '{"workers": 8, "rate_limit": 0.5}' -``` - ---- - -## πŸ“Š API Endpoints - -| Endpoint | Method | Description | -|----------|--------|-------------| -| `/api/health` | GET | Server health check | -| `/api/titleids` | GET | List all TitleIDs | -| `/api/titleid/{id}` | GET | Get TitleID details | -| `/api/search?q=query` | GET | Search TitleIDs | -| `/api/metadata/{id}` | POST | Collect metadata | -| `/api/download/{id}` | POST | Download content | -| `/api/statistics` | GET | Database statistics | -| `/api/failed-items` | GET | List failed downloads | -| `/api/retry-failed` | POST | Retry failed items | -| `/api/verify-integrity` | GET | Check file integrity | -| `/api/export?format=json\|csv` | GET | Export database | -| `/api/config` | GET/POST | Get/update configuration | - ---- - -## πŸ§ͺ Testing - -### Run All Tests: -```bash -python integration_tests.py -``` - -### Run Specific Test Class: -```bash -python -m unittest integration_tests.TestI18nModule -v -``` - -### Test Classes Available: -- `TestI18nModule` - Language support (5 tests) -- `TestUpdaterModule` - Version checking (3 tests) -- `TestQueueManager` - Download queue (8 tests) -- `TestSpeedMonitoring` - Speed tracking (5 tests) -- `TestDatabaseIntegrity` - File verification (6 tests) -- `TestAPIIntegration` - REST API (2 tests) -- `TestFeatureIntegration` - Feature integration (3 tests) - -**Total**: 32 comprehensive integration tests +## Rate and Worker Controls ---- - -## πŸ“¦ Dependencies - -### Required: -- `requests>=2.31.0` - HTTP client -- `urllib3>=2.0.0` - URL library - -### Optional (for new features): -- `flask>=2.3.0` - REST API server -- `flask-cors>=4.0.0` - CORS support -- `packaging>=23.0` - Version parsing - -### Install All: -```bash -pip install -r requirements.txt +```powershell +python main.py 4D5307E6 --metadata-only --workers 4 --rate 0.5 ``` ---- - -## πŸ” Security Notes - -1. **API**: Default localhost only (`127.0.0.1`) - - Change with `--api-host 0.0.0.0` for network access - - Consider adding authentication for production - -2. **Queue**: Stored in JSON file - - Location: `download_queue.json` in current directory - - Contains URLs and download status - -3. **Database**: SQLite with indexes - - Location: `unityscraper.db` - - Contains metadata and history - -4. **Checksums**: SHA256 by default - - Automatic verification if enabled - - CLI: `python main.py --verify-checksums` - ---- - -## 🌍 Language Support - -### Supported Languages: -- πŸ‡ΊπŸ‡Έ **English** (en) -- πŸ‡ͺπŸ‡Έ **Spanish** (es) -- πŸ‡«πŸ‡· **French** (fr) -- πŸ‡©πŸ‡ͺ **German** (de) -- πŸ‡―πŸ‡΅ **Japanese** (ja) - -### Switching Languages: -1. **GUI**: Click language dropdown -2. **Code**: `translator.set_language('es')` +- `--workers` controls concurrent tasks. +- `--rate` sets the minimum interval between service requests. +- `--bandwidth-limit` limits downloads in KB/s; `0` means unlimited. +- `--timeout`, retries, and refresh behavior can also be stored in config. ---- +Be conservative with community services. More workers do not always improve +throughput and can make rate limiting worse. -## πŸ› Troubleshooting +## Resumable Downloads -| Issue | Solution | -|-------|----------| -| Flask not found | `pip install flask flask-cors` | -| Import error | Check all modules in same directory | -| API won't start | Check port (default 8000) not in use | -| Queue not persisting | Check write permissions in directory | -| Integrity check fails | Verify file checksums with `--verify-checksums` | -| Language not changing | Restart GUI and reselect language | - ---- - -## πŸ“š File Structure +UnityScraper writes partial downloads separately and keeps resume metadata. +Completed files are published only after the transfer and configured +verification finish. Failed records remain available for: +```powershell +python main.py --retry-failed ``` -UnityScraper/ -β”œβ”€β”€ main.py # Core scraper -β”œβ”€β”€ GUI.py # Tkinter GUI with all features -β”œβ”€β”€ database.py # SQLite + integrity checking -β”œβ”€β”€ resume.py # Downloads + speed monitor -β”œβ”€β”€ api.py # REST API server -β”œβ”€β”€ i18n.py # Multi-language support -β”œβ”€β”€ updater.py # Version checking -β”œβ”€β”€ queue_manager.py # Persistent queue -β”œβ”€β”€ integration_tests.py # 32 comprehensive tests -β”œβ”€β”€ requirements.txt # Dependencies -└── INTEGRATION_SUMMARY.md # Full documentation -``` - ---- -## 🎯 Common Tasks +## Integrity and Exports -### Check file integrity -```bash -# CLI +```powershell python main.py --verify-integrity - -# GUI -Click "Verify Files" button - -# API -curl http://127.0.0.1:8000/api/verify-integrity +python main.py --export json --export-file library.json +python main.py --export csv --export-file library.csv ``` -### Check for updates -```bash -# GUI -Click "Check Updates" button +Archive Health in the desktop app adds missing-file and database consistency +checks. Backup Manager verification covers Xbox content layouts and abandoned +partial files. -# API -GET http://127.0.0.1:8000/api/health -``` +## Diagnostics -### View download queue -```bash -# GUI -Click "View Queue" button +Use **Help & About > Export Diagnostics** to create a sanitized support bundle. +Review it before sharing. Downloaded content and secrets are not intentionally +included, but filesystem paths can still be sensitive. -# API -curl http://127.0.0.1:8000/api/statistics -``` - -### Filter items by status -```bash -# GUI -Select status from dropdown (all/pending/downloaded/failed) -Select date from dropdown (any/7d/30d/custom) -``` +## Portable Mode -### Export database -```bash -# CLI -python main.py --export json --export-file backup.json +Create `portable.mode` beside the application before launch. Runtime data then +lives under `UnityScraperData` beside the application. Remove the marker to +return to normal per-user storage; existing data is not moved automatically. -# GUI -Click "Export DB" button +## REST API -# API -GET http://127.0.0.1:8000/api/export?format=json -``` - ---- +The API is intended for local automation. It binds to `127.0.0.1` by default, +uses restricted browser origins, validates mutable settings, and requires a +token for non-loopback binds. See [API.md](API.md). -## πŸ’‘ Pro Tips +## External Conversion -1. **Speed**: Increase workers but respect rate limiting -2. **Reliability**: Use `--verify-checksums` for critical downloads -3. **Persistence**: Queue survives app crashes -4. **Languages**: Switch without restarting GUI -5. **API**: Run in background while using GUI -6. **Testing**: Run integration tests after code changes +Backup Manager can invoke a converter selected by the user: ---- +```powershell +python main.py --convert-iso game.iso ` + --converter C:\Tools\converter.exe ` + --converter-arg "{input}" ` + --converter-arg "{output}" ` + --converter-output D:\Converted +``` -**Version**: 1.1.0 with Advanced Features -**Date**: January 24, 2026 -**Status**: βœ… Production Ready +UnityScraper does not bundle a converter or implement copy-protection bypass. diff --git a/API.md b/API.md new file mode 100644 index 0000000..55e210b --- /dev/null +++ b/API.md @@ -0,0 +1,99 @@ +# REST API + +The optional REST API supports local automation. It is not required for the +desktop application. + +## Start + +Localhost mode: + +```powershell +python main.py --api-mode +``` + +The default base URL is: + +```text +http://127.0.0.1:8000/api +``` + +Remote binds require a token: + +```powershell +$env:UNITYSCRAPER_API_TOKEN = "replace-with-a-long-random-token" +python main.py --api-mode --api-host 0.0.0.0 --api-port 8000 +``` + +`--api-token` is also supported, but environment variables avoid exposing a +token in command history and process listings. + +Send credentials using either header: + +```text +Authorization: Bearer +X-API-Key: +``` + +The health endpoint does not require authentication. All other endpoints do +when a token is configured. + +## Endpoints + +| Method | Path | Purpose | +| --- | --- | --- | +| `GET` | `/api/health` | Version, readiness, and authentication status | +| `GET` | `/api/titleids` | List library TitleIDs | +| `GET` | `/api/titleid/` | Get one library record | +| `GET` | `/api/search?q=` | Search the library | +| `POST` | `/api/metadata/` | Collect metadata | +| `POST` | `/api/download/` | Process downloads | +| `GET` | `/api/statistics` | Library statistics | +| `GET` | `/api/failed-items` | Failed downloads | +| `POST` | `/api/retry-failed` | Retry failed downloads | +| `GET` | `/api/verify-integrity` | Verify recorded files | +| `GET` | `/api/export?format=json` | Export JSON or CSV under the exports directory | +| `GET` | `/api/config` | Read safe runtime settings | +| `POST` | `/api/config` | Update allowlisted runtime settings | + +TitleID routes require exactly eight hexadecimal characters. + +## Configuration + +Mutable keys: + +- `workers` +- `rate_limit` +- `timeout` +- `max_retries` +- `retry_backoff` +- `bandwidth_limit` +- `verify_checksums` +- `dry_run` +- `refresh_interval_days` + +Types and ranges are validated. `base_url`, `use_https`, filesystem paths, and +arbitrary object attributes cannot be changed through the API. XboxUnity +remains fixed to `http://xboxunity.net`. + +Example: + +```powershell +$headers = @{ Authorization = "Bearer $env:UNITYSCRAPER_API_TOKEN" } +$body = @{ workers = 4; rate_limit = 0.5 } | ConvertTo-Json +Invoke-RestMethod ` + -Method Post ` + -Uri http://127.0.0.1:8000/api/config ` + -Headers $headers ` + -ContentType application/json ` + -Body $body +``` + +## Network Safety + +- The built-in server uses HTTP, not TLS. +- Use localhost whenever possible. +- For remote use, keep the service on a trusted private network or place it + behind an authenticated TLS reverse proxy. +- Browser CORS is restricted to localhost origins unless the API is embedded + programmatically with an explicit origin list. +- Responses disable caching and include basic content and frame protections. diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md new file mode 100644 index 0000000..3b33c0e --- /dev/null +++ b/ARCHITECTURE.md @@ -0,0 +1,133 @@ +# Architecture + +UnityScraper is a desktop-first Python application with shared service modules +for CLI and optional REST automation. SQLite is the durable local store. + +## Entry Points + +| Entry point | Purpose | +| --- | --- | +| `desktop_app.py` | Primary library-first desktop application | +| `main.py` | XboxUnity, knowledge, backup, export, and API CLI | +| `GUI.py` | Legacy advanced scraper controls | +| `api.py` | Optional local automation API | + +## Layers + +### Presentation + +- `modern_gui.py` builds the dark navigation shell and core pages. +- `knowledge_gui.py` renders knowledge search, imports, sources, and conflicts. +- `backup_gui.py` renders inventory, package, FTP, and converter workflows. +- `setup_wizard.py` handles first-run storage setup. + +GUI operations that can block are dispatched to background threads and return +results to Tk's main loop. + +### Application Services + +- `library_service.py` provides library summaries and archive-health data. +- `knowledge_service.py` provides search, source status, imports, and details. +- `backup_service.py` coordinates scans, installs, exports, verification, FTP, + and audit records. +- `knowledge_sync.py` exposes complete source-import workflows to CLI and GUI. + +### Domain and Adapters + +- `main.py` contains the XboxUnity collector and shared configuration. +- `resume.py` handles partial download state and verification. +- `backup_manager.py` parses public STFS/XBE fields and performs safe + filesystem or FTP operations. +- `knowledge_base.py` defines normalized knowledge records and resolution. +- `consolemods_adapters.py`, `wiki_adapters.py`, and `dat_adapters.py` parse + source-specific data into shared records. +- `knowledge_sources.py` handles cache-aware, rate-limited source retrieval. + +### Persistence + +`database.py` creates the legacy library tables and invokes additive knowledge +and backup schema creation. + +Main schema groups: + +- Library: `titleids`, `title_updates`, `covers`, `download_history` +- Knowledge: sources, documents, revisions, entities, names, identifiers, + facts, citations, relationships, import runs, and conflicts +- Backups: targets, scans, inventory, and operations + +Schema initialization is idempotent. New migrations should preserve existing +data and be covered by tests. + +## Data Flow + +### Knowledge Import + +```text +source discovery + -> rate-limited fetch + -> raw cache + -> source adapter + -> normalized records + -> citations and conflicts + -> preferred fact selection + -> fill unknown library metadata +``` + +Raw documents and retrieval metadata remain attached to their source. A parser +failure is recorded against its import run and should not erase prior data. + +### Backup Import + +```text +user-selected file or ZIP + -> archive/path validation + -> STFS or content-tree identification + -> content-aware destination + -> .partial copy + -> SHA-256 verification + -> atomic publication + -> operation audit record +``` + +Game payloads are not stored in SQLite. Inventory records contain paths, +identifiers, sizes, statuses, and notes. + +## Storage + +Normal Windows data: + +```text +%LOCALAPPDATA%\UnityScraper\ + config\ + data\ + diagnostics\ + downloads\ + exports\ + logs\ +``` + +Portable mode uses `UnityScraperData` beside the application when a +`portable.mode` marker is present. + +Bundled read-only assets are resolved through `app_paths.resource_path`, which +works in source and PyInstaller one-file builds. + +## Security Model + +- XboxUnity is fixed to its HTTP endpoints. +- The REST API is localhost-only by default; remote binds require a token. +- API configuration mutation uses an explicit validated allowlist. +- FTP credentials stay in memory and are omitted from database settings. +- ZIP imports reject traversal, symlinks, excessive entries, and excessive + expanded size. +- Package and export copies use temporary files and verification. +- External converters run only through explicit user configuration. + +See [SECURITY.md](SECURITY.md) for reporting and operational guidance. + +## Packaging + +`UnityScraper.spec` is the canonical PyInstaller definition. Assets and modules +loaded indirectly by the GUI are listed explicitly. GitHub Actions validates a +Windows one-file build on pull requests and publishes ZIP/checksum artifacts for +version tags. diff --git a/BACKGROUND_INSTALL.md b/BACKGROUND_INSTALL.md deleted file mode 100644 index b7cabb8..0000000 --- a/BACKGROUND_INSTALL.md +++ /dev/null @@ -1,35 +0,0 @@ -# Xbox-Inspired UnityScraper Background - -This drop-in adds a custom dark-green, Xbox-inspired UnityScraper -background and displays it as a responsive banner across every page. - -## Install - -Copy the included files into the repository root and allow -`modern_gui.py` to be replaced. - -```powershell -python -m pip install -r requirements-background.txt -python desktop_app.py -``` - -## PyInstaller - -Include the assets folder in the build: - -```powershell -python -m PyInstaller ` - --clean ` - --noconfirm ` - --name UnityScraper ` - --noconsole ` - --onefile ` - --add-data "JSON.txt;." ` - --add-data "assets;assets" ` - --hidden-import PIL.Image ` - --hidden-import PIL.ImageTk ` - desktop_app.py -``` - -The image is loaded through `resource_path(...)`, so it works from the -source checkout and from the packaged executable. diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..88d7b5e --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,46 @@ +# Changelog + +Notable changes to UnityScraper are documented here. The project follows +[Semantic Versioning](https://semver.org/) for published releases. + +## [Unreleased] + +### Added + +- Authenticated remote REST API mode and validated configuration updates. +- Cross-platform CI with Windows executable smoke builds. +- Tagged-release packaging with SHA-256 checksums. +- Contributor, security, community, and architecture documentation. + +### Changed + +- Repository-generated executables are published through releases and CI + artifacts instead of being committed to source control. +- Source checkouts use normal application storage unless the user creates a + `portable.mode` marker. +- Windows setup now creates and uses a project virtual environment. +- Documentation now reflects the library-first desktop application. + +### Removed + +- Stale drop-in background instructions and historical integration reports. +- Duplicate background-only requirement file. + +## [0.10.0-beta.1] - 2026-07-23 + +### Added + +- Source-attributed Xbox 360 knowledge browser. +- ConsoleMods, XenonLibrary, and Free60 wiki ingestion. +- User-supplied Redump and No-Intro DAT imports. +- Normalized entities, identifiers, facts, citations, import runs, and + conflicts. +- Local backup inventory, STFS/XBE inspection, safe package and ZIP imports, + verified exports, FTP transfer, and external ISO converter integration. +- Additive backup target, scan, inventory, and operation tables. + +### Security + +- ZIP traversal and archive symlink protection. +- SHA-256 verified temporary-file publication. +- FTP passwords omitted from persisted target settings. diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..35830a7 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,20 @@ +# Community Standards + +UnityScraper welcomes preservationists, developers, researchers, archivists, +and Xbox community members. + +Participants are expected to: + +- Be respectful, patient, and constructive. +- Critique ideas and code without attacking people. +- Respect privacy, intellectual property, and source licenses. +- Avoid sharing copyrighted payloads, credentials, keys, or private data. +- Give credit for research, testing, documentation, and code. +- Help keep discussions useful to people with different experience levels. + +Harassment, discriminatory conduct, threats, deliberate disruption, and +publication of another person's private information are not acceptable. + +Maintainers may edit or remove contributions and restrict participation when +needed to protect the project and its community. Serious conduct concerns +should be reported privately through the repository owner's GitHub profile. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..ec20fc5 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,64 @@ +# Contributing to UnityScraper + +Thank you for helping improve Xbox 360 preservation tooling. + +## Before You Start + +- Search existing issues and pull requests. +- Use an issue for substantial behavior or schema changes. +- Keep commercial game content, firmware, encryption keys, leaked SDK + material, and circumvention tooling outside the project. +- Confirm that any imported text, metadata, images, or code can legally be + redistributed and preserve its attribution. + +## Development Setup + +UnityScraper requires Python 3.10 or newer. + +```powershell +python -m venv .venv +.\.venv\Scripts\python.exe -m pip install -r requirements.txt -r requirements-dev.txt +``` + +Run the desktop application: + +```powershell +.\.venv\Scripts\python.exe desktop_app.py +``` + +Run validation: + +```powershell +.\.venv\Scripts\python.exe -m ruff check . +.\.venv\Scripts\python.exe -m compileall -q . +.\.venv\Scripts\python.exe tests.py +``` + +## Pull Requests + +- Branch from the current `main`. +- Keep changes focused and retain existing user data compatibility. +- Add tests for parsing, migrations, filesystem operations, and failure paths. +- Do not make live XboxUnity or wiki requests in unit tests. +- Treat database migrations as additive unless a documented migration safely + preserves existing data. +- Keep XboxUnity URLs HTTP-only; this reflects the service endpoints. +- Update user documentation when commands, storage, schemas, or UI workflows + change. + +## Source Adapters + +Every knowledge adapter should: + +1. Identify its source and stated license. +2. Rate-limit requests and cache raw responses. +3. Preserve source URL, revision, retrieval time, and citations. +4. Isolate partial failures in import-run records. +5. Retain conflicting claims instead of silently overwriting them. +6. Avoid downloading or redistributing copyrighted payloads. + +## Reporting Security Problems + +Do not open public issues for vulnerabilities involving arbitrary file writes, +archive traversal, credential exposure, or remote API access. Follow +[SECURITY.md](SECURITY.md). diff --git a/DOCS_INDEX.md b/DOCS_INDEX.md index d5d6cf5..d032aed 100644 --- a/DOCS_INDEX.md +++ b/DOCS_INDEX.md @@ -1,40 +1,37 @@ -# UnityScraper Documentation +# Documentation -## Start Here +## Users -- [README.md](README.md) - installation, launch, and core workflows -- [PROJECT_STATUS.md](PROJECT_STATUS.md) - current implementation and boundaries -- [KNOWLEDGE_SOURCES.md](KNOWLEDGE_SOURCES.md) - knowledge adapters, provenance, - caching, source licenses, and DAT imports -- [BACKUP_MANAGER.md](BACKUP_MANAGER.md) - local inventory, STFS installation, - exports, verification, FTP, and external conversion +- [README](README.md) - product overview, installation, core workflows, and CLI +- [Backup Manager](BACKUP_MANAGER.md) - layouts, installation, exports, FTP, + verification, and external conversion +- [Knowledge Sources](KNOWLEDGE_SOURCES.md) - imports, provenance, caching, and + source licensing +- [Advanced Features](ADVANCED_FEATURES.md) - rate limits, resume, diagnostics, + portable mode, API, and conversion +- [REST API](API.md) - authentication, endpoints, configuration, and safety +- [Project Status](PROJECT_STATUS.md) - completed work, boundaries, and roadmap +- [Changelog](CHANGELOG.md) - release history -## Additional Reference +## Contributors -- [ADVANCED_FEATURES.md](ADVANCED_FEATURES.md) - legacy scraper and download tools -- [INTEGRATION_SUMMARY.md](INTEGRATION_SUMMARY.md) - original feature integration -- [BACKGROUND_INSTALL.md](BACKGROUND_INSTALL.md) - dark-theme background packaging +- [Architecture](ARCHITECTURE.md) - modules, layers, schemas, data flows, and + packaging +- [Contributing](CONTRIBUTING.md) - environment, tests, PR expectations, and + adapter rules +- [Security](SECURITY.md) - private reporting and operational boundaries +- [Community Standards](CODE_OF_CONDUCT.md) - participation expectations -## Developer Map - -| Area | Primary modules | -| --- | --- | -| XboxUnity collection | `main.py`, `database.py`, `resume.py` | -| Desktop shell | `desktop_app.py`, `modern_gui.py` | -| Knowledge model | `knowledge_base.py`, `knowledge_service.py` | -| Source adapters | `consolemods_adapters.py`, `wiki_adapters.py`, `dat_adapters.py` | -| Source synchronization | `knowledge_sources.py`, `knowledge_sync.py` | -| Backup engine | `backup_manager.py`, `backup_service.py` | -| Backup desktop page | `backup_gui.py` | -| Tests | `tests.py` | - -Run the complete offline test suite with: +## Validation ```powershell +python -m pip install -r requirements.txt -r requirements-dev.txt +python -m ruff check . +python -m compileall -q . python tests.py ``` -Build the Windows executable with: +Windows packaging: ```powershell powershell -ExecutionPolicy Bypass -File .\build_windows.ps1 diff --git a/GUI.py b/GUI.py index f1bd07b..b01ff8f 100644 --- a/GUI.py +++ b/GUI.py @@ -571,9 +571,9 @@ def retry(): )) except Exception as e: logging.error(f"Retry error: {e}") - self.root.after(0, lambda: messagebox.showerror( + self.root.after(0, lambda error=str(e): messagebox.showerror( "Error", - f"Retry failed: {str(e)}" + f"Retry failed: {error}" )) thread = threading.Thread(target=retry, daemon=True) @@ -620,9 +620,9 @@ def export(): )) except Exception as e: logging.error(f"Export error: {e}") - self.root.after(0, lambda: messagebox.showerror( + self.root.after(0, lambda error=str(e): messagebox.showerror( "Error", - f"Export failed: {str(e)}" + f"Export failed: {error}" )) thread = threading.Thread(target=export, daemon=True) @@ -658,9 +658,9 @@ def get_stats(): )) except Exception as e: logging.error(f"Stats error: {e}") - self.root.after(0, lambda: messagebox.showerror( + self.root.after(0, lambda error=str(e): messagebox.showerror( "Error", - f"Failed to get statistics: {str(e)}" + f"Failed to get statistics: {error}" )) thread = threading.Thread(target=get_stats, daemon=True) @@ -704,9 +704,9 @@ def verify(): )) except Exception as e: logging.error(f"Integrity check error: {e}") - self.root.after(0, lambda: messagebox.showerror( + self.root.after(0, lambda error=str(e): messagebox.showerror( "Error", - f"Integrity check failed: {str(e)}" + f"Integrity check failed: {error}" )) thread = threading.Thread(target=verify, daemon=True) @@ -717,20 +717,10 @@ def check_for_updates(self): def check(): try: checker = VersionChecker() - update_info = checker.check_for_updates() + has_update, update_info = checker.check_for_updates() - if update_info and update_info.get('new_version'): - message = f""" -New Version Available! - -Current: Unknown -Latest: {update_info.get('version', 'Unknown')} - -Changes: -{update_info.get('changes', 'No changelog available')} - -Download: {update_info.get('download_url', 'GitHub')} -""" + if has_update and update_info: + message = checker.format_update_message(update_info) self.root.after(0, lambda: messagebox.showinfo( "Update Available", message @@ -742,9 +732,9 @@ def check(): )) except Exception as e: logging.error(f"Update check error: {e}") - self.root.after(0, lambda: messagebox.showerror( + self.root.after(0, lambda error=str(e): messagebox.showerror( "Error", - f"Failed to check for updates: {str(e)}" + f"Failed to check for updates: {error}" )) thread = threading.Thread(target=check, daemon=True) @@ -771,9 +761,9 @@ def show_queue(): )) except Exception as e: logging.error(f"Queue error: {e}") - self.root.after(0, lambda: messagebox.showerror( + self.root.after(0, lambda error=str(e): messagebox.showerror( "Error", - f"Failed to display queue: {str(e)}" + f"Failed to display queue: {error}" )) thread = threading.Thread(target=show_queue, daemon=True) @@ -910,9 +900,9 @@ def metadata_thread(self, titleids): )) except Exception as e: logging.error(f"Metadata collection error: {e}") - self.root.after(0, lambda: messagebox.showerror( + self.root.after(0, lambda error=str(e): messagebox.showerror( "Error", - f"Metadata collection failed: {str(e)}" + f"Metadata collection failed: {error}" )) finally: self.root.after(0, self.download_finished) @@ -938,9 +928,9 @@ def download_thread(self, titleids): except Exception as e: logging.error(f"Download error: {e}") - self.root.after(0, lambda: messagebox.showerror( + self.root.after(0, lambda error=str(e): messagebox.showerror( "Error", - f"Download failed: {str(e)}" + f"Download failed: {error}" )) finally: diff --git a/INTEGRATION_SUMMARY.md b/INTEGRATION_SUMMARY.md deleted file mode 100644 index 74c6eeb..0000000 --- a/INTEGRATION_SUMMARY.md +++ /dev/null @@ -1,403 +0,0 @@ -# UnityScraper - Integration Complete - -**Date**: January 24, 2026 -**Status**: βœ… ALL FEATURES INTEGRATED AND TESTED - -## Summary - -All 7 advanced features have been successfully implemented, integrated, and validated. This document provides an overview of the completed work. - ---- - -## 1. GUI Integration (Feature #1-8) - -### Files Modified: `GUI.py` -- **Enhanced window size**: 1100x800 (increased from 900x700) -- **i18n support added**: Full multi-language support with 5 languages - -### New GUI Components Added: - -#### A. Language Selector -- Dropdown menu with 5 language options: English, Spanish, French, German, Japanese -- Real-time language switching with `on_language_change()` handler -- Automatically translates all UI strings - -#### B. Advanced Filters Frame -``` -Filter by Status: [All β–Ό] [Pending β–Ό] [Downloaded β–Ό] [Failed β–Ό] -Filter by Date: [Any β–Ό] [Last 7 days β–Ό] [Last 30 days β–Ό] [Custom β–Ό] -Results: X items (Status: Y, Date: Z) -``` -- `apply_filters()` method filters database items by status and date -- Dynamic result count display -- Integration with database queries - -#### C. New Control Buttons -1. **Verify Files** - Checks file integrity with checksums - - Displays: Total files, verified count, corrupted files, missing files - - Shows detailed list of corrupted/missing files - - `verify_integrity()` handler - -2. **Check Updates** - Checks for application updates - - GitHub API + version file fallback - - Shows version info, changelog, download link - - `check_for_updates()` handler - -3. **View Queue** - Shows download queue status - - Total items, queued, downloading, completed, failed counts - - Persistent across sessions - - `show_download_queue()` handler - -### New Handler Methods: -- `verify_integrity()` - File integrity verification with detailed reporting -- `check_for_updates()` - Version checking with update information -- `show_download_queue()` - Queue status display -- `apply_filters()` - Dynamic filtering by status and date -- `on_language_change()` - Language switching handler -- All handlers run in background threads to prevent GUI freezing - ---- - -## 2. Integration Tests (Feature #2) - -### File Created: `integration_tests.py` (500+ lines) - -#### Test Classes: - -**A. TestI18nModule** (5 tests) -- βœ… Translator initialization -- βœ… Language switching (en, es, fr, de, ja) -- βœ… String translation retrieval -- βœ… All languages loaded correctly -- βœ… Fallback to key when translation missing - -**B. TestUpdaterModule** (3 tests) -- βœ… VersionChecker initialization -- βœ… Update message formatting -- βœ… Version comparison logic - -**C. TestQueueManager** (8 tests) -- βœ… Queue initialization -- βœ… Add and retrieve items -- βœ… Priority-based ordering (high priority items first) -- βœ… Queue persistence across instances -- βœ… Status transitions (queued β†’ downloading β†’ completed/failed) -- βœ… Retry failed items with max retry limit -- βœ… Queue statistics (total, queued, downloading, completed, failed) - -**D. TestSpeedMonitoring** (5 tests) -- βœ… Progress tracker initialization -- βœ… Progress update tracking (percentage, downloaded bytes) -- βœ… Speed calculation (MB/s) -- βœ… Statistics tracking (current, peak, average speeds) -- βœ… ETA calculation - -**E. TestDatabaseIntegrity** (6 tests) -- βœ… Database initialization -- βœ… Add titleid entries -- βœ… Add and verify cover metadata -- βœ… Add and verify update metadata -- βœ… File integrity verification -- βœ… Checksum calculation (SHA256) - -**F. TestAPIIntegration** (2 tests) -- βœ… API initialization (Flask available) -- βœ… All API routes registered - -**G. TestFeatureIntegration** (3 tests) -- βœ… i18n provides all GUI strings -- βœ… Queue items work with speed monitoring -- βœ… Integrity checker works with database - -#### Test Coverage: -- **Total Tests**: 32 comprehensive integration tests -- **Languages Tested**: English, Spanish, French, German, Japanese -- **Validation**: All new features validated independently and in integration -- **Isolation**: Tests use temporary files/databases, no side effects - ---- - -## 3. Feature Summary - -### βœ… Feature #1: Database Integrity Checker -**Files**: `database.py`, `main.py` -- Method: `verify_file_integrity(titleid=None)` -- Returns: verified/corrupted/missing file counts with details -- CLI: `python main.py --verify-integrity` -- GUI: "Verify Files" button shows results in messagebox - -### βœ… Feature #2: Multi-language UI Support (i18n) -**File**: `i18n.py` -- **Languages**: English, Spanish, French, German, Japanese -- **Strings**: 40+ UI strings per language -- **Classes**: `Translator` class with get(), set_language(), format_update_message() -- **Usage**: `t('key')` for translations in GUI -- **Integration**: Language selector dropdown in GUI - -### βœ… Feature #3: Auto-update Checker -**File**: `updater.py` -- **Class**: `VersionChecker` -- **Sources**: GitHub API (primary) + version file URL (fallback) -- **Returns**: version, name, changelog, download URL, published date -- **GUI**: "Check Updates" button shows update notifications -- **Integration**: Graceful failure, no dependencies required - -### βœ… Feature #4: Download Speed Monitor -**File**: `resume.py` (enhanced) -- **Class**: `DownloadProgress` -- **Tracking**: current speed, peak speed, average speed, ETA -- **New Method**: `get_stats()` returns comprehensive statistics dict -- **Display**: `__str__()` shows peak speed and current speed -- **Integration**: Used during resumable downloads - -### βœ… Feature #5: Quick Filters in GUI -**File**: `GUI.py` (new section) -- **Filters**: Status (all/pending/downloaded/failed) + Date (any/7d/30d/custom) -- **Method**: `apply_filters()` queries database with filters -- **Display**: Real-time result count showing filtered items -- **Integration**: Integrated with database queries - -### βœ… Feature #8: Download Queue Persistence -**File**: `queue_manager.py` -- **Class**: `DownloadQueue` -- **Persistence**: JSON-backed, survives app restart -- **Features**: Priority ordering, status tracking, retry logic -- **Methods**: add_item(), get_next_item(), mark_downloading/completed/failed(), retry_failed() -- **Stats**: get_queue_stats() returns queue status -- **GUI**: "View Queue" button shows queue statistics - -### βœ… Feature #10: REST API Mode -**Files**: `api.py`, `main.py`, `requirements.txt` -- **Framework**: Flask with CORS support -- **Endpoints**: 12 HTTP endpoints for full scraper control -- **CLI**: `python main.py --api-mode --api-port 8000 --api-host 127.0.0.1` -- **Routes**: - - GET /api/health, /api/titleids, /api/titleid/{id}, /api/search - - POST /api/metadata/{id}, /api/download/{id} - - GET /api/statistics, /api/failed-items, /api/verify-integrity, /api/export, /api/config - - POST /api/retry-failed, /api/config -- **Integration**: Fully integrated with UnityScraper, DatabaseManager, ResumableDownloader - ---- - -## 4. Code Quality - -### Validation Results: -- βœ… **GUI.py**: No syntax errors -- βœ… **api.py**: No syntax errors -- βœ… **integration_tests.py**: No syntax errors -- βœ… **All dependencies**: Listed in requirements.txt -- βœ… **Type hints**: Consistent with Optional[] annotations -- βœ… **Error handling**: All features have try/except blocks with user feedback -- βœ… **Threading**: All blocking operations run in background threads -- βœ… **Logging**: Comprehensive logging for debugging - -### Dependencies Added: -``` -flask>=2.3.0 # REST API server -flask-cors>=4.0.0 # Cross-origin support -packaging>=23.0 # Version comparison -``` - ---- - -## 5. Usage Examples - -### GUI Usage: -```python -python GUI.py -``` -- Click "Check Updates" to check for new versions -- Click "Verify Files" to check file integrity -- Click "View Queue" to see download queue status -- Select language from dropdown to change UI language -- Use Status/Date filters to filter database items - -### REST API Usage: -```bash -# Start API server -python main.py --api-mode --api-port 8000 - -# Query endpoints -curl http://127.0.0.1:8000/api/health -curl http://127.0.0.1:8000/api/statistics -curl -X POST http://127.0.0.1:8000/api/metadata/555308C5 - -# Update configuration -curl -X POST http://127.0.0.1:8000/api/config \ - -H "Content-Type: application/json" \ - -d '{"workers": 8, "rate_limit": 0.5}' -``` - -### CLI Usage: -```bash -# Verify file integrity -python main.py --verify-integrity - -# Retry failed downloads -python main.py --retry-failed - -# Export database -python main.py --export json --export-file backup.json - -# Run with specific configuration -python main.py --api-mode --workers 8 --rate 0.5 -``` - -### Testing: -```bash -# Run all integration tests -python integration_tests.py - -# Specific test class -python -m unittest integration_tests.TestI18nModule -v -``` - ---- - -## 6. Architecture - -### Component Diagram: -``` -β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” -β”‚ UnityScraper β”‚ -β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ -β”‚ β”‚ -β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ -β”‚ β”‚ GUI.py β”‚ β”‚ api.py β”‚ β”‚ main.py β”‚ β”‚ -β”‚ β”‚(Tkinter)β”‚ β”‚(Flask) β”‚ β”‚ (CLI) β”‚ β”‚ -β”‚ β””β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ -β”‚ β”‚ β”‚ β”‚ β”‚ -β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ -β”‚ β”‚ β”‚ -β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β–Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ -β”‚ β”‚ UnityScraper (Core) β”‚ β”‚ -β”‚ β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ β”‚ -β”‚ β”‚ β€’ collect_metadata() β”‚ β”‚ -β”‚ β”‚ β€’ process_titleid() β”‚ β”‚ -β”‚ β”‚ β€’ download_file() β”‚ β”‚ -β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”˜ β”‚ -β”‚ β”‚ β”‚ β”‚ β”‚ -β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β–Όβ”€β”€β” β”Œβ”€β”€β”€β”€β–Όβ”€β”€β”€β”€β” β”‚ β”‚ -β”‚ β”‚ DatabaseMgr β”‚ β”‚Downloaderβ”‚ β”‚ β”‚ -β”‚ β”‚ β€’ verify_ β”‚ β”‚ β€’ resume β”‚ β”‚ β”‚ -β”‚ β”‚ integrity() β”‚ β”‚ β€’ progress β”‚ β”‚ -β”‚ β”‚ β€’ get_stats() β”‚ β”‚ β€’ speed β”‚ β”‚ β”‚ -β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ β”‚ -β”‚ β”‚ β”‚ -β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”β–Ό β”‚ -β”‚ β”‚ i18n.py β”‚ β”‚ queue_mgr.py β”‚ updater.py β”‚ -β”‚ β”‚ (Languages) β”‚ β”‚ (Persistent) β”‚ (Versions) β”‚ -β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ ─────────── β”‚ -β”‚ β”‚ -β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ -``` - -### Data Flow: -``` -User Input (GUI/CLI/API) - ↓ - UnityScraper (orchestration) - ↓ - β”Œβ”€β”€β”€β”€β”΄β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” - ↓ ↓ ↓ ↓ -Database Download i18n Queue -Manager Manager (UI) Manager - ↓ ↓ ↓ ↓ - SQLite HTTP JSON JSON -``` - ---- - -## 7. Testing Strategy - -### Test Levels: -1. **Syntax Validation**: All files checked with Pylance -2. **Unit Tests**: Each feature tested independently (32 tests) -3. **Integration Tests**: Features tested together -4. **Manual Testing**: GUI, API, CLI verified -5. **Edge Cases**: Tested queue persistence, language switching, file verification - -### Test Results: -- βœ… All syntax validations passed -- βœ… All integration tests prepared and ready -- βœ… No import errors or circular dependencies -- βœ… Type hints consistent throughout -- βœ… Backward compatible (no breaking changes) - ---- - -## 8. Deployment Checklist - -- βœ… All source files created/updated -- βœ… Syntax validation passed -- βœ… Integration tests created (32 tests) -- βœ… Dependencies updated in requirements.txt -- βœ… Documentation created (this file) -- βœ… Type hints validated -- βœ… Error handling implemented -- βœ… Thread safety ensured -- βœ… Backward compatibility maintained -- βœ… GUI integration complete -- βœ… API endpoints available -- βœ… CLI support added - ---- - -## 9. Next Steps (Optional Enhancements) - -1. **Web Dashboard**: Create web UI using Vue.js to interact with REST API -2. **Database Migrations**: Add version tracking for schema changes -3. **Plugin System**: Extend downloaders/processors via plugin architecture -4. **Performance Tuning**: Add caching, connection pooling for API -5. **Authentication**: Add API key/token authentication for security -6. **Monitoring**: Add Prometheus metrics export -7. **Docker**: Create Dockerfile for containerized deployment -8. **CI/CD**: Add GitHub Actions for automated testing - ---- - -## 10. Files Modified/Created - -### New Files: -- βœ… `i18n.py` (300+ lines) - Multi-language support -- βœ… `updater.py` (120+ lines) - Version checking -- βœ… `queue_manager.py` (250+ lines) - Persistent queue -- βœ… `api.py` (250+ lines) - REST API server -- βœ… `integration_tests.py` (500+ lines) - Comprehensive tests - -### Modified Files: -- βœ… `GUI.py` - Added 200+ lines of new features -- βœ… `main.py` - Added API mode support and new CLI flags -- βœ… `database.py` - Added integrity checking methods -- βœ… `resume.py` - Enhanced speed monitoring -- βœ… `requirements.txt` - Added Flask, Flask-CORS, packaging - -### Total Code Added: -- **New Code**: ~1500 lines -- **Enhanced Code**: ~300 lines -- **Tests**: 500+ lines -- **Type Safe**: All code follows type hint conventions -- **Documented**: Comprehensive docstrings throughout - ---- - -## 11. Conclusion - -All 7 advanced features have been successfully implemented, fully integrated into the GUI, and validated with comprehensive integration tests. The UnityScraper now provides: - -- 🌍 Multi-language support in 5 languages -- πŸ” Database integrity verification with checksums -- πŸš€ REST API for external tool integration -- πŸ“Š Advanced filtering and queue management -- πŸ“ˆ Detailed speed monitoring and statistics -- πŸ”„ Persistent download queue across sessions -- ⚑ Version checking and auto-update notifications - -The system is production-ready with robust error handling, comprehensive logging, and thread-safe operations. - ---- - -**Implementation Date**: January 24, 2026 -**Status**: βœ… COMPLETE AND VALIDATED -**Quality**: 100% type-safe, 0 syntax errors, comprehensive tests diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..95be08e --- /dev/null +++ b/LICENSE @@ -0,0 +1,17 @@ +UnityScraper +Copyright (C) 2026 UnityScraper contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, version 3 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +The complete GNU General Public License version 3 is available from: + +https://www.gnu.org/licenses/gpl-3.0.txt + +SPDX-License-Identifier: GPL-3.0-only diff --git a/PROJECT_STATUS.md b/PROJECT_STATUS.md index 90a5a0b..efe49d2 100644 --- a/PROJECT_STATUS.md +++ b/PROJECT_STATUS.md @@ -32,6 +32,12 @@ backup-management, and source-attributed knowledge application. - Aurora-oriented FTP package upload with one connection per operation, temporary remote names, and no stored passwords. - Explicit external converter integration for user-owned ISO images. +- Local-by-default REST API with token-required remote binding, restricted + browser origins, validated settings, and current version reporting. +- Cross-platform CI, Windows packaging checks, tagged release archives, and + SHA-256 release checksums. +- Repository contribution, security, architecture, API, and release + documentation. ## Validation @@ -41,6 +47,8 @@ backup-management, and source-attributed knowledge application. end-to-end local workflow. - Network-backed source syncs remain dependent on each source's availability and access policy. Cached copies are used when available. +- Windows executable artifacts are generated by CI and releases rather than + committed to the source tree. ## Deliberate Boundaries @@ -55,6 +63,8 @@ backup-management, and source-attributed knowledge application. implement copy-protection bypass or bundle third-party converter binaries. - Traditional FTP is intended for trusted local networks. Passwords are kept in memory and omitted from database records. +- Remote API access requires a token but the built-in server does not provide + TLS. Remote deployments need a trusted network or TLS reverse proxy. ## Future Work diff --git a/README.md b/README.md index c650f6c..8a1cc90 100644 --- a/README.md +++ b/README.md @@ -1,432 +1,266 @@ -Screenshot 2026-05-13 210717 -# UnityScraper - -**UnityScraper** is an offline-capable Xbox 360 library, knowledge, update, -artwork, verification, and backup manager. It combines source-attributed -reference metadata with tools for content the user already owns. XboxUnity -Title Updates and cover art use the service's required HTTP endpoints. - -## Features - -* βœ… **Auto-load TitleIDs** from `JSON.txt` file on startup -* βœ… **Metadata-only collection** - fetch and index covers/updates without downloading -* βœ… **SQLite database** for persistent storage and tracking -* βœ… **Download status tracking** (pending, downloaded, failed) -* βœ… **GUI integration** - view available items and download status before downloading -* βœ… **Parallel downloads** with configurable workers and rate limiting -* βœ… **Automatic retries** with exponential backoff -* βœ… **Raw JSON metadata** saved alongside downloaded files -* βœ… **CLI and GUI** - both use the same backend engine -* βœ… **HTTP-only XboxUnity compatibility** matching the service endpoints -* βœ… **ConsoleMods knowledge import** for TitleID, publisher, region, and Multi-ID enrichment -* **Knowledge browser** for ConsoleMods, XenonLibrary, Free60, Redump, and No-Intro metadata -* **Backup inventory** for Xbox content roots, USB drives, archives, and extracted games -* **Safe STFS installs** with content-aware paths, partial files, and SHA-256 verification -* **Preservation exports** with per-file hashes and JSON manifests -* **Console FTP transfer** with in-memory credentials and atomic remote naming -* **External ISO converter integration** without bundling converter code or game content - ---- - -## Requirements - -* Python **3.9+** recommended -* Python packages: - -```bash -pip install -r requirements.txt -``` - -Tkinter is included with standard Python installs. - ---- - -## Quick Start +

+ UnityScraper +

-### Windows Desktop App +# UnityScraper -Run the desktop app directly: +[![CI](https://github.com/TrapEmAll/UnityScraper/actions/workflows/ci.yml/badge.svg)](https://github.com/TrapEmAll/UnityScraper/actions/workflows/ci.yml) +[![Latest release](https://img.shields.io/github/v/release/TrapEmAll/UnityScraper?include_prereleases)](https://github.com/TrapEmAll/UnityScraper/releases) +[![License: GPL-3.0](https://img.shields.io/badge/license-GPL--3.0--only-blue.svg)](LICENSE) -```bat -Run-UnityScraper.bat -``` +UnityScraper is a Windows-focused Xbox 360 library, knowledge, title-update, +artwork, verification, and backup manager. It combines a local SQLite library +with source-attributed community knowledge and tools for content you already +own. -The primary Python entry point is: +The project is currently in beta. Preserve a separate copy of important +archives before running large imports or transfers. -```powershell -python desktop_app.py -``` +## What It Does -The **Knowledge** workspace manages source imports and provenance. The -**Backup Manager** inventories local targets, installs user-supplied packages, -exports verified archives, and transfers packages to a configured console. -See [KNOWLEDGE_SOURCES.md](KNOWLEDGE_SOURCES.md) and -[BACKUP_MANAGER.md](BACKUP_MANAGER.md). +### Library and Downloads -Or build a standalone Windows executable: +- Collects XboxUnity cover and Title Update metadata. +- Reviews results before selectively downloading files. +- Tracks pending, downloaded, failed, and verified content. +- Supports retries, rate limiting, bandwidth limits, and resumable downloads. +- Verifies local archive records and exports JSON or CSV reports. -```powershell -powershell -ExecutionPolicy Bypass -File .\build_windows.ps1 -``` +XboxUnity endpoints are intentionally **HTTP-only** because that is what the +service exposes. UnityScraper does not silently substitute HTTPS URLs. -The built app is written to: +### Xbox 360 Knowledge -```text -dist\UnityScraper.exe -``` +- Imports TitleID, publisher, region, and Multi-ID references from ConsoleMods. +- Caches and indexes Xbox 360 articles from ConsoleMods, XenonLibrary, and + Free60. +- Imports user-supplied Redump and No-Intro XML DAT files. +- Stores entities, identifiers, facts, citations, revisions, import runs, and + conflicts with provenance. +- Fills blank or unknown local metadata without replacing better known values. -All user data is stored locally under: +### Backup Management -```text -%LOCALAPPDATA%\UnityScraper -``` +- Inventories Xbox content roots, USB drives, archive folders, and extracted + `Games` directories. +- Recognizes Games on Demand, Xbox Live Arcade, DLC, Title Updates, extracted + Xbox 360 games, and original Xbox XBE metadata. +- Installs user-supplied STFS packages into content-aware paths. +- Safely imports ZIP archives and complete + `Content/0000000000000000` trees. +- Exports selected backups with per-file SHA-256 values and a preservation + manifest. +- Uploads packages to a configured Aurora-style FTP server. +- Runs a user-selected external ISO converter without bundling converter code. -That folder contains the SQLite database, editable `JSON.txt` TitleID list, -saved config, logs, and downloaded archive files. +## Install -### Option 1: Auto-Load from JSON.txt +### Windows Release -Place a `JSON.txt` file in the same directory with comma-separated TitleIDs: +Download the latest ZIP and checksum from +[GitHub Releases](https://github.com/TrapEmAll/UnityScraper/releases). Verify +the SHA-256 file, extract the ZIP, and run `UnityScraper.exe`. -``` -TESTID00,TESTID01,TESTID02 -``` +Release executables are generated by GitHub Actions. Built binaries are not +stored in the source tree. -Then run: +### Run From Source -```bash -python main.py -``` +Requirements: -This will: -1. Load all TitleIDs from `JSON.txt` -2. Collect metadata (covers, updates) without downloading -3. Store everything in `unityscraper.db` -4. Print: "Metadata collection completed! Check GUI to view and download items." +- Python 3.10 or newer +- Tkinter, normally included with the Windows Python installer -### Option 2: Manual TitleID Entry +Clone the repository and run: -```bash -python main.py -# or -python main.py TESTID00,TESTID01 +```powershell +.\setup.bat +.\Run-UnityScraper.bat ``` -### Option 3: Launch GUI +Manual setup: -```bash -python GUI.py +```powershell +python -m venv .venv +.\.venv\Scripts\python.exe -m pip install -r requirements.txt +.\.venv\Scripts\python.exe desktop_app.py ``` ---- - -## Usage (CLI) - -### Basic Commands +The primary interface is `desktop_app.py`. `GUI.py` remains available from +**Advanced Tools** for older scraper controls. -```bash -# Auto-load from JSON.txt (metadata-only) -python main.py - -# Manual TitleIDs (metadata-only by default) -python main.py TESTID00,TESTID01 +## Application Workspaces -# Download content for specific TitleIDs -python main.py TESTID00 --metadata-only=false +| Workspace | Purpose | +| --- | --- | +| Library | Browse games, covers, MediaIDs, and available updates | +| Add Games | Import or enter TitleIDs | +| Downloads | Review and manage download activity | +| Backup Manager | Scan, install, verify, export, convert, and transfer owned content | +| Knowledge | Search sources, facts, citations, imports, and conflicts | +| Archive Health | Find missing or inconsistent downloaded files | +| Settings | Configure storage and scraper behavior | +| Help & About | Version, diagnostics, storage, and advanced tools | -# Metadata-only explicitly -python main.py TESTID00 --metadata-only -``` +## Storage and Portable Mode -### CLI Options +Normal Windows installations store writable data under: -```bash -python main.py [TITLEIDS] [options] +```text +%LOCALAPPDATA%\UnityScraper ``` -| Option | Description | -| ------------------- | ----------------------------------------------------- | -| `TITLEIDS` | Comma-separated TitleIDs (e.g. `TESTID00,TESTID01`) | -| `--out PATH` | Output directory (default: `unityscrape`) | -| `--workers N` | Parallel workers (default: 4) | -| `--rate SECONDS` | Min seconds between requests (default: 0.35) | -| `--config PATH` | Load config from JSON file | -| `--save-config` | Save current settings to `config.json` | -| `--metadata-only` | Only collect metadata, don't download files | -| `--log-level LEVEL` | DEBUG, INFO, WARNING, ERROR | -| `--force-http` | Use XboxUnity HTTP endpoints (always enabled) | - -### Examples - -```bash -# Auto-load JSON.txt and collect metadata -python main.py - -# Manual entry with custom output directory -python main.py TESTID00 --out D:\Archive --workers 8 - -# Metadata-only mode -python main.py TESTID00 --metadata-only - -# Download with saved config -python main.py --config config.json -``` +This includes the database, configuration, logs, downloads, exports, source +cache, and diagnostics. ---- +To enable portable mode, create an empty file named `portable.mode` beside the +source entry point or packaged executable before launch. Portable data is +written to `UnityScraperData` beside the application. The marker and runtime +data are ignored by Git. -## Usage (GUI) +## Command Line -Launch the GUI: +Display every option: -```bash -python GUI.py +```powershell +python main.py --help ``` -### GUI Features - -* View all TitleIDs in database with metadata -* See download status for each cover and update: - * 🟑 **Pending** - metadata available, not yet downloaded - * 🟒 **Downloaded** - file successfully saved - * πŸ”΄ **Failed** - download failed (can retry) -* Adjust worker count and rate limiting -* Progress tracking with live log output -* Selective download of items -* Search and filter available content +### XboxUnity Metadata ---- - -## Workflow: Metadata Collection β†’ Selective Download - -### Phase 1: Collect Metadata (Fast) +```powershell +# Collect metadata for one or more TitleIDs +python main.py 4D5307E6 --metadata-only -```bash +# Load the bundled/user TitleID list in metadata-only mode python main.py -# Reads JSON.txt β†’ fetches metadata β†’ stores in database -# Takes seconds, no large files downloaded -``` - -Database now contains: -- All available covers with URLs -- All available updates with versions -- Download status for each item - -### Phase 2: Selective Download (Via GUI) - -```bash -python GUI.py -# View all metadata -# Select items to download -# Download marked items -``` - ---- -## Database Schema - -**UnityScraper** uses SQLite (`unityscraper.db`) to store: - -### Tables - -| Table | Purpose | -|-------|---------| -| `titleids` | Tracked TitleIDs with metadata | -| `covers` | Cover art info with status | -| `title_updates` | Update versions with status | -| `download_history` | Download attempts and results | - -### Status Values - -| Status | Meaning | -|--------|---------| -| `pending` | Metadata found, not yet downloaded | -| `downloaded` | File successfully saved | -| `failed` | Download attempt failed | - ---- - -## Output Structure - -``` -unityscrape/ -└── TITLEID/ - β”œβ”€β”€ covers_data.json - β”œβ”€β”€ updates_data.json - β”œβ”€β”€ covers/ - β”‚ β”œβ”€β”€ cover1.jpg - β”‚ └── cover2.png - └── MEDIAID1/ - └── version_3/ - └── update_FILE.bin +# Verify downloaded files recorded in SQLite +python main.py --verify-integrity ``` -Raw JSON responses are **always saved** for reference: -* `covers_data.json` - API response with cover metadata -* `updates_data.json` - API response with update metadata +Providing TitleIDs without `--metadata-only` starts the download workflow. +Review the destination and settings before doing this. ---- +### Knowledge Sources -## Configuration - -### Config File - -Save your settings to `config.json`: +```powershell +# TitleID and Multi-ID enrichment +python main.py --sync-knowledge -```bash -python main.py TESTID00 --workers 8 --rate 0.5 --save-config -``` +# Cache and index reference wikis +python main.py --sync-wikis -This creates `config.json` for future runs: +# Limit a first test sync per source +python main.py --sync-wikis --wiki-limit 25 -```bash -python main.py --config config.json +# Import a user-downloaded preservation DAT +python main.py --import-dat "D:\DATs\xbox360.dat" --dat-source redump ``` ---- - -## TitleID Format - -* **Must be 8 hexadecimal characters** (0-9, A-F) -* Automatically normalized to **uppercase** -* Invalid TitleIDs are skipped with warnings -* Test placeholders: `TESTID00`, `TESTID01`, etc. +### Backup Manager ---- - -## Networking - -* **XboxUnity uses HTTP-only endpoints** -* **Global rate limiting** across all parallel downloads -* **Automatic retries** with exponential backoff -* **429 handling** for rate-limited requests -* Configurable request timeout (default: 30s) +```powershell +# Inventory a target and write a report +python main.py --scan-backups E:\ --backup-report inventory.json ---- +# Install a bare STFS package +python main.py --install-package game.live --backup-target E:\ -## Knowledge Import +# Import supported packages or a validated content tree from ZIP +python main.py --import-package-zip archive.zip --backup-target E:\ -Import ConsoleMods TitleID and Multi-ID reference data, then enrich only unknown -library title/publisher fields: +# Structurally verify a target +python main.py --verify-backups E:\ --backup-report health.json -```bash -python main.py --sync-knowledge +# Upload a package to a console on a trusted local network +python main.py --ftp-upload game.live --ftp-host 192.168.1.50 ``` -Cache and index Xbox 360 articles from ConsoleMods, XenonLibrary, and Free60: +See [BACKUP_MANAGER.md](BACKUP_MANAGER.md) for layouts, conflict behavior, +manifests, FTP considerations, and external converter arguments. -```bash -python main.py --sync-wikis -``` +## Optional REST API -For a smaller first sync, limit each source: +Start the localhost-only API: -```bash -python main.py --sync-wikis --wiki-limit 25 +```powershell +python main.py --api-mode ``` -Import user-downloaded preservation DAT files: +Binding beyond localhost requires an API token: -```bash -python main.py --import-dat "path/to/xbox360.dat" --dat-source redump -python main.py --import-dat "path/to/xbox360-digital.dat" --dat-source no-intro +```powershell +$env:UNITYSCRAPER_API_TOKEN = "replace-with-a-long-random-token" +python main.py --api-mode --api-host 0.0.0.0 ``` -The desktop application's **Knowledge** page provides the same imports through -file pickers, plus global search, source licenses and status, citations, and -conflict review. - -The normalized knowledge schema stores sources, documents, revisions, entities, -identifiers, facts, citations, import runs, and conflicts. See -[`KNOWLEDGE_SOURCES.md`](KNOWLEDGE_SOURCES.md) for migration notes, source and -licensing considerations. - ---- - -## Advanced Usage +Clients send the token as `Authorization: Bearer ` or `X-API-Key`. +Remote HTTP is not encrypted; place it behind a trusted local reverse proxy or +use it only on an isolated network. See [API.md](API.md). -### JSON.txt Format +## Build and Test -Create a file named `JSON.txt` with comma-separated TitleIDs: +Install development dependencies: +```powershell +.\.venv\Scripts\python.exe -m pip install -r requirements-dev.txt ``` -TESTID00,TESTID01,TESTID02,TESTID03 -``` - -On next run, these will automatically load in metadata-only mode. -### Batch Processing +Run the same high-signal checks used in CI: -```bash -# Metadata collection with custom settings -python main.py --rate 0.5 --workers 4 --log-level INFO - -# Then download via GUI at your own pace -python GUI.py +```powershell +.\.venv\Scripts\python.exe -m ruff check . +.\.venv\Scripts\python.exe -m compileall -q . +.\.venv\Scripts\python.exe tests.py ``` -### Resume Failed Downloads - -The database tracks which items failed. Use the GUI to retry without re-scanning metadata. +Build the Windows executable: ---- - -## Testing - -Run the test suite: - -```bash -python -m pytest tests.py -v +```powershell +powershell -ExecutionPolicy Bypass -File .\build_windows.ps1 ``` -Tests use **test-only TitleIDs** (`TESTID00`, `TESTID01`) for safety. +The output appears under `dist\` and is ignored by Git. ---- +## Documentation -## Limitations +- [Documentation index](DOCS_INDEX.md) +- [Architecture](ARCHITECTURE.md) +- [Knowledge sources and licensing](KNOWLEDGE_SOURCES.md) +- [Backup manager](BACKUP_MANAGER.md) +- [REST API](API.md) +- [Project status](PROJECT_STATUS.md) +- [Changelog](CHANGELOG.md) +- [Contributing](CONTRIBUTING.md) +- [Security policy](SECURITY.md) -* No authentication (public endpoints only) -* GUI stop button is **best-effort** -* Resume support for partial files (planned) -* No duplicate detection yet +## Project Boundary ---- +UnityScraper catalogs public knowledge and operates on files supplied by the +user. It does not bundle or download: -## Intended Use +- Commercial game images or copyrighted game payloads +- Xbox firmware or dashboard files +- Encryption keys +- Leaked SDK material +- Copy-protection bypass tools -βœ… **Recommended for:** -* Offline archiving and preservation -* Research and analysis -* Metadata collection -* Personal use +Respect source licenses, service capacity, local law, and the rights of content +owners. Do not attach private paths, credentials, keys, or copyrighted data to +bug reports. -❌ **Not recommended for:** -* High-frequency automated scraping -* Commercial redistribution -* Bypassing site restrictions -* Concurrent instance scraping +## Sources and Attribution -Be respectful of XboxUnity's infrastructure. +Imported knowledge remains linked to its source and stated license. ConsoleMods, +XenonLibrary, Free60, Redump, No-Intro, and XboxUnity are independent projects +and are not affiliated with UnityScraper. ---- +The backup workflow was informed by +[TinyXbox360BackupManager](https://github.com/jeanmatthieud/TinyXbox360BackupManager). +UnityScraper uses an independent Python implementation and does not copy or +bundle that GPL-3.0-only project's Rust source. ## License -No explicit license is currently defined. -If you plan to redistribute or contribute, clarify licensing first. - ---- - -## Author - -Created and maintained by **Sthornberry9** - ---- - -## Next Enhancements - -1. Parse local XEX/STFS headers and match owned files to knowledge entities. -2. Add field-specific source-priority controls for conflict resolution. -3. Add optional scheduled knowledge refreshes. -4. Add exportable offline HTML knowledge reports. - ---- +UnityScraper is licensed under GPL-3.0-only. See [LICENSE](LICENSE). diff --git a/Run-UnityScraper.bat b/Run-UnityScraper.bat index 37da54e..0190e72 100644 --- a/Run-UnityScraper.bat +++ b/Run-UnityScraper.bat @@ -1,5 +1,9 @@ @echo off setlocal cd /d "%~dp0" -python desktop_app.py +if exist ".venv\Scripts\python.exe" ( + ".venv\Scripts\python.exe" desktop_app.py +) else ( + python desktop_app.py +) endlocal diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..0b28e6d --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,39 @@ +# Security Policy + +## Supported Versions + +Security fixes are developed for the current `main` branch and the latest +published beta or stable release. + +## Reporting a Vulnerability + +Use GitHub's private vulnerability reporting feature for this repository when +available. If it is unavailable, contact the maintainer privately through the +contact method shown on the repository owner's GitHub profile. + +Please include: + +- UnityScraper version or commit +- Operating system and Python version +- Affected workflow +- Minimal reproduction steps +- Expected and observed behavior +- Whether untrusted files, ZIP archives, FTP servers, or API requests are + involved + +Do not include copyrighted game data, credentials, encryption keys, or private +filesystem contents in a report. + +## Security Boundaries + +- The REST API binds to localhost by default. Remote binds require an explicit + API token. +- API tokens and FTP passwords are not stored in the UnityScraper database. +- Traditional FTP is unencrypted and should only be used on a trusted local + network. +- ZIP imports reject traversal paths, symlinks, excessive entries, and + unreasonable expanded sizes. +- Package copies use temporary files, verification, and atomic publication. +- External converters run only when explicitly configured by the user. +- UnityScraper does not provide game images, firmware, keys, SDK files, or + copy-protection bypass tools. diff --git a/api.py b/api.py index 70c7c35..aecbb48 100644 --- a/api.py +++ b/api.py @@ -1,274 +1,351 @@ -""" -REST API Server for UnityScraper -Expose scraper functionality via HTTP endpoints -""" +"""Optional REST API for local UnityScraper automation.""" + +from __future__ import annotations import logging -import json +import os +import secrets import threading -from typing import Optional, Dict, Any, TYPE_CHECKING -from flask import Flask, request, jsonify -from flask_cors import CORS +from datetime import datetime, timezone from pathlib import Path +from typing import TYPE_CHECKING, Any, Callable, Optional + +from flask import Flask, jsonify, request +from flask_cors import CORS + +from app_paths import EXPORTS_DIR +from app_version import DISPLAY_VERSION if TYPE_CHECKING: from main import UnityScraper logger = logging.getLogger(__name__) +LOOPBACK_HOSTS = {"127.0.0.1", "localhost", "::1"} +DEFAULT_CORS_ORIGINS = ( + "http://127.0.0.1:*", + "http://localhost:*", +) + class UnityScraperAPI: - """REST API wrapper for UnityScraper""" - - def __init__(self, scraper: Optional['UnityScraper'] = None, port: int = 8000, host: str = "127.0.0.1"): - self.app = Flask(__name__) - CORS(self.app) # Enable CORS - - self.scraper: Optional['UnityScraper'] = scraper + """REST wrapper with local defaults and opt-in remote authentication.""" + + CONFIG_RULES: dict[str, tuple[type, float | None, float | None]] = { + "workers": (int, 1, 32), + "rate_limit": (float, 0.05, 60), + "timeout": (int, 1, 600), + "max_retries": (int, 0, 20), + "retry_backoff": (float, 0, 120), + "bandwidth_limit": (int, 0, None), + "verify_checksums": (bool, None, None), + "dry_run": (bool, None, None), + "refresh_interval_days": (int, 0, 3650), + } + + def __init__( + self, + scraper: Optional["UnityScraper"] = None, + port: int = 8000, + host: str = "127.0.0.1", + token: Optional[str] = None, + cors_origins: Optional[list[str]] = None, + ): + self.scraper = scraper self.port = port self.host = host + self.token = token or os.environ.get("UNITYSCRAPER_API_TOKEN", "").strip() + if host not in LOOPBACK_HOSTS and not self.token: + raise ValueError( + "A token is required when the API is bound beyond localhost" + ) + + self.app = Flask(__name__) + self.app.config["MAX_CONTENT_LENGTH"] = 64 * 1024 + CORS( + self.app, + resources={ + r"/api/*": { + "origins": cors_origins or list(DEFAULT_CORS_ORIGINS) + } + }, + ) self.running = False - + self._register_security() self._register_routes() - - def _register_routes(self): - """Register all API routes""" - - @self.app.route('/api/health', methods=['GET']) + + def _register_security(self) -> None: + @self.app.before_request + def require_token(): + if not self.token or request.path == "/api/health": + return None + supplied = request.headers.get("X-API-Key", "") + authorization = request.headers.get("Authorization", "") + if authorization.startswith("Bearer "): + supplied = authorization[7:] + if not secrets.compare_digest(supplied, self.token): + return jsonify({"error": "Authentication required"}), 401 + return None + + @self.app.after_request + def security_headers(response): + response.headers["Cache-Control"] = "no-store" + response.headers["X-Content-Type-Options"] = "nosniff" + response.headers["X-Frame-Options"] = "DENY" + return response + + def _register_routes(self) -> None: + @self.app.get("/api/health") def health(): - """Health check endpoint""" - return jsonify({ - 'status': 'healthy', - 'version': '1.1.0', - 'scraper_loaded': self.scraper is not None - }) - - @self.app.route('/api/titleids', methods=['GET']) + return jsonify( + { + "status": "healthy", + "version": DISPLAY_VERSION, + "scraper_loaded": self.scraper is not None, + "authentication_required": bool(self.token), + } + ) + + @self.app.get("/api/titleids") def get_titleids(): - """Get all TitleIDs in database""" - try: - if not self.scraper: - return jsonify({'error': 'Scraper not initialized'}), 400 - titleids = self.scraper.db.search_titleids('') - return jsonify({'titleids': titleids}) - except Exception as e: - return jsonify({'error': str(e)}), 500 - - @self.app.route('/api/titleid/', methods=['GET']) - def get_titleid_info(titleid): - """Get info for specific TitleID""" + return self._execute( + lambda: {"titleids": self._require_scraper().db.search_titleids("")} + ) + + @self.app.get("/api/titleid/") + def get_titleid_info(titleid: str): + normalized = self._titleid_or_error(titleid) + if not isinstance(normalized, str): + return normalized try: - if not self.scraper: - return jsonify({'error': 'Scraper not initialized'}), 400 - info = self.scraper.db.get_titleid_info(titleid) - if info: - return jsonify(info) - return jsonify({'error': 'TitleID not found'}), 404 - except Exception as e: - return jsonify({'error': str(e)}), 500 - - @self.app.route('/api/search', methods=['GET']) + info = self._require_scraper().db.get_titleid_info(normalized) + if info is None: + return jsonify({"error": "TitleID not found"}), 404 + return jsonify(info) + except Exception as exc: + return self._server_error(exc) + + @self.app.get("/api/search") def search(): - """Search TitleIDs""" - query = request.args.get('q', '') - try: - if not self.scraper: - return jsonify({'error': 'Scraper not initialized'}), 400 - results = self.scraper.db.search_titleids(query) - return jsonify({'results': results, 'count': len(results)}) - except Exception as e: - return jsonify({'error': str(e)}), 500 - - @self.app.route('/api/metadata/', methods=['POST']) - def collect_metadata(titleid): - """Collect metadata for TitleID""" - try: - if not self.scraper: - return jsonify({'error': 'Scraper not initialized'}), 400 - success = self.scraper.collect_metadata(titleid) - return jsonify({ - 'success': success, - 'titleid': titleid, - 'message': 'Metadata collected' if success else 'Failed to collect metadata' - }) - except Exception as e: - return jsonify({'error': str(e)}), 500 - - @self.app.route('/api/download/', methods=['POST']) - def download_titleid(titleid): - """Download content for TitleID""" - try: - if not self.scraper: - return jsonify({'error': 'Scraper not initialized'}), 400 - success = self.scraper.process_titleid(titleid) - return jsonify({ - 'success': success, - 'titleid': titleid, - 'message': 'Download completed' if success else 'Download failed' - }) - except Exception as e: - return jsonify({'error': str(e)}), 500 - - @self.app.route('/api/statistics', methods=['GET']) - def get_stats(): - """Get database statistics""" - try: - if not self.scraper: - return jsonify({'error': 'Scraper not initialized'}), 400 - stats = self.scraper.db.get_statistics() - return jsonify(stats) - except Exception as e: - return jsonify({'error': str(e)}), 500 - - @self.app.route('/api/failed-items', methods=['GET']) - def get_failed_items(): - """Get all failed downloads""" - titleid = request.args.get('titleid') - try: - if not self.scraper: - return jsonify({'error': 'Scraper not initialized'}), 400 - items = self.scraper.db.get_failed_items(titleid) - return jsonify({'failed_items': items, 'count': len(items)}) - except Exception as e: - return jsonify({'error': str(e)}), 500 - - @self.app.route('/api/retry-failed', methods=['POST']) + query = request.args.get("q", "")[:200] + return self._execute( + lambda: self._search_response( + self._require_scraper().db.search_titleids(query) + ) + ) + + @self.app.post("/api/metadata/") + def collect_metadata(titleid: str): + normalized = self._titleid_or_error(titleid) + if not isinstance(normalized, str): + return normalized + return self._execute( + lambda: self._operation_response( + self._require_scraper().collect_metadata(normalized), + normalized, + "Metadata collected", + "Metadata collection failed", + ) + ) + + @self.app.post("/api/download/") + def download_titleid(titleid: str): + normalized = self._titleid_or_error(titleid) + if not isinstance(normalized, str): + return normalized + return self._execute( + lambda: self._operation_response( + self._require_scraper().process_titleid(normalized), + normalized, + "Download completed", + "Download failed", + ) + ) + + @self.app.get("/api/statistics") + def statistics(): + return self._execute( + lambda: self._require_scraper().db.get_statistics() + ) + + @self.app.get("/api/failed-items") + def failed_items(): + titleid = request.args.get("titleid") + if titleid: + normalized = self._titleid_or_error(titleid) + if not isinstance(normalized, str): + return normalized + titleid = normalized + return self._execute( + lambda: self._failed_response( + self._require_scraper().db.get_failed_items(titleid) + ) + ) + + @self.app.post("/api/retry-failed") def retry_failed(): - """Retry failed downloads""" - titleid = request.args.get('titleid') - try: - if not self.scraper: - return jsonify({'error': 'Scraper not initialized'}), 400 - self.scraper.retry_failed_downloads(titleid) - return jsonify({ - 'success': True, - 'message': 'Retry process started' - }) - except Exception as e: - return jsonify({'error': str(e)}), 500 - - @self.app.route('/api/verify-integrity', methods=['GET']) + titleid = request.args.get("titleid") + if titleid: + normalized = self._titleid_or_error(titleid) + if not isinstance(normalized, str): + return normalized + titleid = normalized + + def retry() -> dict: + self._require_scraper().retry_failed_downloads(titleid) + return {"success": True, "message": "Retry process completed"} + + return self._execute(retry) + + @self.app.get("/api/verify-integrity") def verify_integrity(): - """Verify file integrity""" - titleid = request.args.get('titleid') - try: - if not self.scraper: - return jsonify({'error': 'Scraper not initialized'}), 400 - results = self.scraper.db.verify_file_integrity(titleid) - return jsonify(results) - except Exception as e: - return jsonify({'error': str(e)}), 500 - return jsonify({'error': str(e)}), 500 - - @self.app.route('/api/export', methods=['GET']) + titleid = request.args.get("titleid") + if titleid: + normalized = self._titleid_or_error(titleid) + if not isinstance(normalized, str): + return normalized + titleid = normalized + return self._execute( + lambda: self._require_scraper().db.verify_file_integrity(titleid) + ) + + @self.app.get("/api/export") def export(): - """Export database""" - format = request.args.get('format', 'json') - try: - if not self.scraper: - return jsonify({'error': 'Scraper not initialized'}), 400 - filename = f"export_{format}.{format}" - self.scraper.export_database(format, filename) - return jsonify({ - 'success': True, - 'filename': filename, - 'message': f'Database exported as {format.upper()}' - }) - except Exception as e: - return jsonify({'error': str(e)}), 500 - - @self.app.route('/api/config', methods=['GET']) + export_format = request.args.get("format", "json").lower() + if export_format not in {"json", "csv"}: + return jsonify({"error": "format must be json or csv"}), 400 + timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + output = EXPORTS_DIR / f"unityscraper-{timestamp}.{export_format}" + + def perform_export() -> dict: + self._require_scraper().export_database( + export_format, str(output) + ) + return { + "success": True, + "filename": output.name, + "path": str(output), + } + + return self._execute(perform_export) + + @self.app.get("/api/config") def get_config(): - """Get current configuration""" - try: - if not self.scraper: - return jsonify({'error': 'Scraper not initialized'}), 400 - config = { - 'workers': self.scraper.config.workers, - 'rate_limit': self.scraper.config.rate_limit, - 'timeout': self.scraper.config.timeout, - 'bandwidth_limit': self.scraper.config.bandwidth_limit, - 'use_https': self.scraper.config.use_https, - 'verify_checksums': self.scraper.config.verify_checksums, + def configuration() -> dict: + config = self._require_scraper().config + return { + key: getattr(config, key) + for key in self.CONFIG_RULES + } | { + "base_url": "http://xboxunity.net", + "use_https": False, } - return jsonify(config) - except Exception as e: - return jsonify({'error': str(e)}), 500 - - @self.app.route('/api/config', methods=['POST']) + + return self._execute(configuration) + + @self.app.post("/api/config") def update_config(): - """Update configuration""" + payload = request.get_json(silent=True) + if not isinstance(payload, dict): + return jsonify({"error": "A JSON object is required"}), 400 + unknown = sorted(set(payload) - set(self.CONFIG_RULES)) + if unknown: + return jsonify( + {"error": f"Unsupported configuration keys: {', '.join(unknown)}"} + ), 400 try: - if not self.scraper: - return jsonify({'error': 'Scraper not initialized'}), 400 - data = request.get_json() - for key, value in data.items(): - if hasattr(self.scraper.config, key): - setattr(self.scraper.config, key, value) - return jsonify({ - 'success': True, - 'message': 'Configuration updated' - }) - except Exception as e: - return jsonify({'error': str(e)}), 500 - - def run(self, debug: bool = False): - """Start API server""" + values = { + key: self._validate_config_value(key, value) + for key, value in payload.items() + } + except (TypeError, ValueError) as exc: + return jsonify({"error": str(exc)}), 400 + + def apply_configuration() -> dict: + config = self._require_scraper().config + for key, value in values.items(): + setattr(config, key, value) + config.base_url = "http://xboxunity.net" + config.http_fallback_url = config.base_url + config.use_https = False + return {"success": True, "updated": sorted(values)} + + return self._execute(apply_configuration) + + def _require_scraper(self) -> "UnityScraper": + if self.scraper is None: + raise RuntimeError("Scraper not initialized") + return self.scraper + + def _titleid_or_error(self, titleid: str): + if self.scraper is None: + return jsonify({"error": "Scraper not initialized"}), 400 + normalized = self.scraper.validate_titleid(titleid) + if normalized is None: + return jsonify({"error": "TitleID must be 8 hexadecimal characters"}), 400 + return normalized + + def _execute(self, operation: Callable[[], Any]): + try: + return jsonify(operation()) + except RuntimeError as exc: + return jsonify({"error": str(exc)}), 400 + except Exception as exc: + return self._server_error(exc) + + @staticmethod + def _search_response(results: list) -> dict: + return {"results": results, "count": len(results)} + + @staticmethod + def _failed_response(items: list) -> dict: + return {"failed_items": items, "count": len(items)} + + @staticmethod + def _operation_response( + success: bool, + titleid: str, + success_message: str, + failure_message: str, + ) -> dict: + return { + "success": bool(success), + "titleid": titleid, + "message": success_message if success else failure_message, + } + + @classmethod + def _validate_config_value(cls, key: str, value: Any) -> Any: + expected, minimum, maximum = cls.CONFIG_RULES[key] + if expected is bool: + if not isinstance(value, bool): + raise TypeError(f"{key} must be true or false") + return value + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise TypeError(f"{key} must be numeric") + converted = expected(value) + if minimum is not None and converted < minimum: + raise ValueError(f"{key} must be at least {minimum}") + if maximum is not None and converted > maximum: + raise ValueError(f"{key} must be no greater than {maximum}") + return converted + + @staticmethod + def _server_error(error: Exception): + logger.exception("API request failed") + return jsonify({"error": "The request could not be completed"}), 500 + + def run(self, debug: bool = False) -> None: self.running = True - logger.info(f"Starting API server on {self.host}:{self.port}") - self.app.run(host=self.host, port=self.port, debug=debug, use_reloader=False) - - def run_in_thread(self, debug: bool = False): - """Run API server in background thread""" + logger.info("Starting API server on %s:%s", self.host, self.port) + self.app.run( + host=self.host, + port=self.port, + debug=debug, + use_reloader=False, + ) + + def run_in_thread(self, debug: bool = False) -> threading.Thread: thread = threading.Thread(target=self.run, args=(debug,), daemon=True) thread.start() - logger.info(f"API server started in background thread") return thread - - @staticmethod - def example_client(): - """Example client code""" - import requests - - BASE_URL = "http://127.0.0.1:8000/api" - - # Health check - response = requests.get(f"{BASE_URL}/health") - print("Health:", response.json()) - - # Get statistics - response = requests.get(f"{BASE_URL}/statistics") - print("Statistics:", response.json()) - - # Collect metadata - response = requests.post(f"{BASE_URL}/metadata/555308C5") - print("Metadata collection:", response.json()) - - # Get TitleID info - response = requests.get(f"{BASE_URL}/titleid/555308C5") - print("TitleID info:", response.json()) - - # Search - response = requests.get(f"{BASE_URL}/search?q=test") - print("Search results:", response.json()) - - -# Example usage -if __name__ == '__main__': - logging.basicConfig(level=logging.INFO) - - # Start API server (requires scraper instance) - # from main import UnityScraper, Config - # - # config = Config() - # scraper = UnityScraper(config) - # api = UnityScraperAPI(scraper, port=8000) - # api.run(debug=True) - - print("API server module loaded. Import and use UnityScraperAPI class.") - print("\nExample:") - print(" from api import UnityScraperAPI") - print(" from main import UnityScraper, Config") - print(" scraper = UnityScraper(Config())") - print(" api = UnityScraperAPI(scraper)") - print(" api.run_in_thread()") diff --git a/backup_gui.py b/backup_gui.py index 8f07311..03cac48 100644 --- a/backup_gui.py +++ b/backup_gui.py @@ -228,7 +228,7 @@ def worker() -> None: try: result = operation() except Exception as exc: - self.root.after(0, lambda: self._failed(exc)) + self.root.after(0, lambda error=exc: self._failed(error)) else: self.root.after(0, lambda: self._finished(result, done)) diff --git a/backup_manager.py b/backup_manager.py index f357477..89ff7cd 100644 --- a/backup_manager.py +++ b/backup_manager.py @@ -390,8 +390,14 @@ def atomic_copy( raise ValueError("conflict must be skip, replace, or error") if destination_path.exists(): if conflict == "skip": + source_hash = sha256_file(source_path) + destination_hash = sha256_file(destination_path) return TransferResult( - str(source_path), str(destination_path), 0, sha256_file(destination_path), "skipped" + str(source_path), + str(destination_path), + 0, + destination_hash, + "skipped" if source_hash == destination_hash else "conflict", ) if conflict == "error": raise ConflictError(f"Destination already exists: {destination_path}") @@ -542,12 +548,21 @@ def export_backup_item( output_root = Path(destination_root).expanduser().resolve() safe_name = re.sub(r'[<>:"/\\|?*]', "_", item.name).strip(" .") or item.title_id or "XboxGame" destination = output_root / safe_name + try: + destination.resolve().relative_to(output_root) + except ValueError as exc: + raise BackupError("Export destination escapes the selected root") from exc if destination.exists(): if conflict == "skip": return destination if conflict == "error": raise ConflictError(f"Export destination already exists: {destination}") - shutil.rmtree(destination) + if destination.is_symlink(): + destination.unlink() + elif destination.is_dir(): + shutil.rmtree(destination) + else: + raise ConflictError(f"Export destination is not a directory: {destination}") destination.mkdir(parents=True, exist_ok=True) files = [] source_root = item.path @@ -661,11 +676,23 @@ def callback(chunk: bytes) -> None: ) if remote_exists and conflict == "error": raise ConflictError(f"Remote destination already exists: {remote}") - with source_path.open("rb") as handle: - ftp.storbinary(f"STOR {partial}", handle, blocksize=64 * 1024, callback=callback) - if remote_exists: - ftp.delete(str(remote)) - ftp.rename(str(partial), str(remote)) + try: + with source_path.open("rb") as handle: + ftp.storbinary( + f"STOR {partial}", + handle, + blocksize=64 * 1024, + callback=callback, + ) + if remote_exists: + ftp.delete(str(remote)) + ftp.rename(str(partial), str(remote)) + except Exception: + try: + ftp.delete(str(partial)) + except ftplib.all_errors: + pass + raise return TransferResult( str(source_path), str(remote), diff --git a/build_windows.ps1 b/build_windows.ps1 index 5c47194..9a40a0a 100644 --- a/build_windows.ps1 +++ b/build_windows.ps1 @@ -6,34 +6,28 @@ $ErrorActionPreference = "Stop" $ProjectRoot = Split-Path -Parent $MyInvocation.MyCommand.Path Set-Location $ProjectRoot +$Python = "python" +if (Test-Path ".venv\Scripts\python.exe") { + $Python = ".venv\Scripts\python.exe" +} + if ($Clean) { Remove-Item -Recurse -Force -ErrorAction SilentlyContinue build, dist - Remove-Item -Force -ErrorAction SilentlyContinue UnityScraper.spec } -python -m pip install --upgrade pip -python -m pip install -r requirements.txt -python -m pip install pyinstaller +& $Python -m pip install -r requirements.txt +& $Python -m pip install pyinstaller +& $Python -m PyInstaller --clean --noconfirm UnityScraper.spec + +$Executable = Join-Path $ProjectRoot "dist\UnityScraper.exe" +if (-not (Test-Path $Executable)) { + throw "PyInstaller completed without creating $Executable" +} -python -m PyInstaller ` - --name UnityScraper ` - --noconsole ` - --onefile ` - --icon "assets\UnityScraper.ico" ` - --add-data "JSON.txt;." ` - --add-data "VERSION;." ` - --add-data "assets;assets" ` - --hidden-import backup_gui ` - --hidden-import backup_manager ` - --hidden-import backup_service ` - --hidden-import consolemods_adapters ` - --hidden-import dat_adapters ` - --hidden-import knowledge_gui ` - --hidden-import knowledge_service ` - --hidden-import knowledge_sync ` - --hidden-import wiki_adapters ` - desktop_app.py +$Hash = (Get-FileHash $Executable -Algorithm SHA256).Hash.ToLower() +"$Hash *UnityScraper.exe" | Set-Content "$Executable.sha256" Write-Host "" -Write-Host "Build complete: $ProjectRoot\dist\UnityScraper.exe" -Write-Host "User data is stored under %LOCALAPPDATA%\UnityScraper" +Write-Host "Build complete: $Executable" +Write-Host "Checksum: $Executable.sha256" +Write-Host "User data: %LOCALAPPDATA%\UnityScraper" diff --git a/database.py b/database.py index 7cfee43..dec1155 100644 --- a/database.py +++ b/database.py @@ -3,9 +3,10 @@ SQLite-based indexing and metadata storage for TitleIDs """ -import sqlite3 -import json -import logging +import sqlite3 +import json +import logging +import hashlib from datetime import datetime from pathlib import Path from typing import List, Dict, Optional, Any @@ -17,14 +18,15 @@ logger = logging.getLogger(__name__) -class DatabaseManager: - """Manages SQLite database for TitleID indexing and metadata""" - +class DatabaseManager: + """Manages SQLite database for TitleID indexing and metadata""" + def __init__(self, db_path: str = None): - ensure_app_dirs() if db_path is None: + ensure_app_dirs() db_path = str(DATABASE_PATH) self.db_path = Path(db_path) + self.db_path.parent.mkdir(parents=True, exist_ok=True) self.init_database() @contextmanager diff --git a/dist/UnityScraper.exe b/dist/UnityScraper.exe deleted file mode 100644 index 21353b0..0000000 Binary files a/dist/UnityScraper.exe and /dev/null differ diff --git a/knowledge_gui.py b/knowledge_gui.py index 33696b9..2d23ce4 100644 --- a/knowledge_gui.py +++ b/knowledge_gui.py @@ -338,7 +338,10 @@ def worker() -> None: try: result = operation() except Exception as exc: - self.root.after(0, lambda: self._job_failed(str(exc))) + self.root.after( + 0, + lambda error=str(exc): self._job_failed(error), + ) return self.root.after(0, lambda: self._job_finished(result)) diff --git a/main.py b/main.py index 13c6497..6d31b90 100644 --- a/main.py +++ b/main.py @@ -31,20 +31,22 @@ from plugins import PluginManager from resume import ResumableDownloader -ensure_app_dirs() - -# Configure logging -logging.basicConfig( - level=logging.INFO, - format='%(asctime)s - %(levelname)s - %(message)s', - handlers=[ - logging.StreamHandler(sys.stdout), - logging.FileHandler(CLI_LOG_PATH) - ] -) logger = logging.getLogger(__name__) +def configure_logging() -> None: + """Configure console and file logging when the CLI is actually launched.""" + ensure_app_dirs() + logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(levelname)s - %(message)s', + handlers=[ + logging.StreamHandler(sys.stdout), + logging.FileHandler(CLI_LOG_PATH), + ], + ) + + class Config: """Configuration management with defaults""" def __init__(self, config_file: Optional[str] = None): @@ -149,11 +151,15 @@ def load_titleids_from_json(json_file: str = None) -> List[str]: class UnityScraper: """Main scraper class for XboxUnity's HTTP endpoints.""" - def __init__(self, config: Config): + def __init__( + self, + config: Config, + database: Optional[DatabaseManager] = None, + ): self.config = config self.rate_limiter = RateLimiter(config.rate_limit) self.session = self._create_session() - self.db = DatabaseManager() # Initialize database + self.db = database or DatabaseManager() self.plugin_manager = PluginManager() # Initialize plugin system self.downloader = ResumableDownloader(self.session, config.timeout, config.bandwidth_limit) self._test_connection() @@ -643,6 +649,7 @@ def export_database(self, format: str = 'json', output_file: Optional[str] = Non def main(): + configure_logging() parser = argparse.ArgumentParser( description='UnityScraper - Download Xbox 360 content from XboxUnity', formatter_class=argparse.RawDescriptionHelpFormatter @@ -775,6 +782,15 @@ def main(): default='127.0.0.1', help='API server host (default: 127.0.0.1)' ) + parser.add_argument( + '--api-token', + type=str, + default=None, + help=( + 'API authentication token; prefer the UNITYSCRAPER_API_TOKEN ' + 'environment variable' + ) + ) parser.add_argument( '--sync-knowledge', action='store_true', @@ -877,7 +893,12 @@ def main(): try: from api import UnityScraperAPI scraper = UnityScraper(config) - api = UnityScraperAPI(scraper, port=args.api_port, host=args.api_host) + api = UnityScraperAPI( + scraper, + port=args.api_port, + host=args.api_host, + token=args.api_token, + ) logger.info(f"Starting API server on {args.api_host}:{args.api_port}") logger.info("API documentation available at http://:/api/") api.run(debug=args.log_level == 'DEBUG') diff --git a/modern_gui.py b/modern_gui.py index 4d7b167..a256b98 100644 --- a/modern_gui.py +++ b/modern_gui.py @@ -926,7 +926,9 @@ def show_about(self) -> None: ttk.Label( panel, text=( - "Xbox 360 title-update, cover-art, and preservation manager.\n\n" + "Xbox 360 library, knowledge, update, artwork, and " + "backup manager.\n" + "Licensed under GPL-3.0-only.\n\n" + describe_storage() ), justify=tk.LEFT, @@ -942,6 +944,13 @@ def show_about(self) -> None: text="Open Application Data", command=lambda: _open_path(BASE_DIR), ).pack(anchor=tk.W, pady=4) + ttk.Button( + panel, + text="Open Documentation", + command=lambda: webbrowser.open( + "https://github.com/TrapEmAll/UnityScraper#readme" + ), + ).pack(anchor=tk.W, pady=4) ttk.Button( panel, text="Open GitHub Project", diff --git a/portable.mode b/portable.mode deleted file mode 100644 index 4df10b4..0000000 --- a/portable.mode +++ /dev/null @@ -1,2 +0,0 @@ -Rename this file to portable.mode to store the database, configuration, logs, -diagnostics, and downloads in a UnityScraperData folder beside the executable. diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..e355604 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,39 @@ +[project] +name = "unityscraper" +version = "0.10.0b1" +description = "Xbox 360 library, knowledge, preservation, and backup manager" +readme = "README.md" +requires-python = ">=3.10" +license = { text = "GPL-3.0-only" } +authors = [{ name = "UnityScraper contributors" }] +dependencies = [ + "requests>=2.31.0", + "urllib3>=2.0.0", + "Pillow>=10.0.0", + "Flask>=2.3.0", + "flask-cors>=4.0.0", + "packaging>=23.0", +] + +[project.urls] +Homepage = "https://github.com/TrapEmAll/UnityScraper" +Issues = "https://github.com/TrapEmAll/UnityScraper/issues" +Releases = "https://github.com/TrapEmAll/UnityScraper/releases" + +[tool.pytest.ini_options] +testpaths = ["tests.py"] +addopts = "-ra" + +[tool.ruff] +target-version = "py310" +line-length = 100 +extend-exclude = ["build", "dist", "UnityScraperData"] + +[tool.ruff.lint] +select = ["E9", "F63", "F7", "F82"] + +[tool.mypy] +python_version = "3.10" +ignore_missing_imports = true +warn_unused_configs = true +exclude = "^(build|dist|UnityScraperData)/" diff --git a/requirements-background.txt b/requirements-background.txt deleted file mode 100644 index 750ca02..0000000 --- a/requirements-background.txt +++ /dev/null @@ -1 +0,0 @@ -Pillow>=10.0 diff --git a/resume.py b/resume.py index 105ec4c..c6f4070 100644 --- a/resume.py +++ b/resume.py @@ -61,12 +61,14 @@ def speed_mbps(self) -> float: @property def eta_seconds(self) -> Optional[float]: """Estimate time remaining in seconds""" - if self.downloaded == 0: - return None - elapsed = time.time() - self.start_time - rate = self.downloaded / elapsed - remaining = self.total_size - self.downloaded - return remaining / rate if rate > 0 else None + if self.downloaded == 0: + return None + elapsed = time.time() - self.start_time + if elapsed <= 0: + return 0.0 if self.downloaded >= self.total_size else None + rate = self.downloaded / elapsed + remaining = max(self.total_size - self.downloaded, 0) + return remaining / rate if rate > 0 else None def get_stats(self) -> Dict[str, float]: """Get comprehensive speed statistics""" @@ -425,4 +427,4 @@ def on_progress(progress: DownloadProgress): resume=True ) - print(f"\nDownload {'successful' if success else 'failed'}") \ No newline at end of file + print(f"\nDownload {'successful' if success else 'failed'}") diff --git a/scripts/check_version.py b/scripts/check_version.py new file mode 100644 index 0000000..5c9dacc --- /dev/null +++ b/scripts/check_version.py @@ -0,0 +1,68 @@ +"""Verify that every release-version source agrees.""" + +from __future__ import annotations + +import argparse +import json +import re +from pathlib import Path + +from packaging.version import Version + +ROOT = Path(__file__).resolve().parents[1] + + +def read_versions() -> dict[str, str]: + app_text = (ROOT / "app_version.py").read_text(encoding="utf-8") + app_match = re.search(r'^APP_VERSION\s*=\s*"([^"]+)"', app_text, re.MULTILINE) + if not app_match: + raise RuntimeError("APP_VERSION was not found in app_version.py") + + version_data = json.loads((ROOT / "VERSION").read_text(encoding="utf-8")) + pyproject_text = (ROOT / "pyproject.toml").read_text(encoding="utf-8") + project_match = re.search( + r'^\s*version\s*=\s*"([^"]+)"', + pyproject_text, + re.MULTILINE, + ) + if not project_match: + raise RuntimeError("project.version was not found in pyproject.toml") + + return { + "app_version.py": app_match.group(1), + "VERSION": str(version_data["version"]), + "pyproject.toml": project_match.group(1), + } + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument( + "--tag", + help="Optional release tag to compare, such as v0.10.0-beta.1", + ) + args = parser.parse_args() + + versions = read_versions() + normalized = {name: Version(value) for name, value in versions.items()} + expected = next(iter(normalized.values())) + mismatches = { + name: versions[name] + for name, value in normalized.items() + if value != expected + } + if mismatches: + details = ", ".join(f"{name}={value}" for name, value in versions.items()) + raise SystemExit(f"Version sources disagree: {details}") + + if args.tag and Version(args.tag.removeprefix("v")) != expected: + raise SystemExit( + f"Release tag {args.tag} does not match application version {expected}" + ) + + print(f"Version sources agree: {expected}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/setup.bat b/setup.bat index 2297258..9cf9aaa 100644 --- a/setup.bat +++ b/setup.bat @@ -1,38 +1,42 @@ -#!/bin/bash -# Setup script for UnityScraper Enhanced -# For Windows, save as setup.bat and replace first line with: @echo off - -echo "================================" -echo "UnityScraper Enhanced - Setup" -echo "================================" -echo "" - -# Check Python installation -echo "Checking Python installation..." -python --version -if [ $? -ne 0 ]; then - echo "ERROR: Python not found. Please install Python 3.9 or higher." - exit 1 -fi - -# Install requirements -echo "" -echo "Installing requirements..." -pip install -r requirements.txt - -if [ $? -ne 0 ]; then - echo "ERROR: Failed to install requirements." - exit 1 -fi - -echo "" -echo "================================" -echo "Setup Complete!" -echo "================================" -echo "" -echo "Quick Start:" -echo " CLI: python main.py 555308C5" -echo " GUI: python GUI.py" -echo " Test: python tests.py" -echo "" -echo "See README.md for full documentation." \ No newline at end of file +@echo off +setlocal +cd /d "%~dp0" + +echo ======================================== +echo UnityScraper Setup +echo ======================================== +echo. + +where python >nul 2>nul +if errorlevel 1 ( + echo Python was not found on PATH. + echo Install Python 3.10 or newer, then run this file again. + exit /b 1 +) + +python -c "import sys; raise SystemExit(0 if sys.version_info >= (3, 10) else 1)" +if errorlevel 1 ( + echo UnityScraper requires Python 3.10 or newer. + python --version + exit /b 1 +) + +if not exist ".venv\Scripts\python.exe" ( + echo Creating virtual environment... + python -m venv .venv + if errorlevel 1 exit /b 1 +) + +echo Installing runtime dependencies... +".venv\Scripts\python.exe" -m pip install --upgrade pip +if errorlevel 1 exit /b 1 + +".venv\Scripts\python.exe" -m pip install -r requirements.txt +if errorlevel 1 exit /b 1 + +echo. +echo Setup complete. +echo Run UnityScraper with: +echo .venv\Scripts\python.exe desktop_app.py +echo. +endlocal diff --git a/tests.py b/tests.py index 039c458..869b16b 100644 --- a/tests.py +++ b/tests.py @@ -41,6 +41,8 @@ scan_local_target, ) from backup_service import BackupRepository +from api import UnityScraperAPI +from app_version import DISPLAY_VERSION class TestConfig(unittest.TestCase): @@ -124,8 +126,15 @@ def test_thread(): self.assertGreaterEqual(total_time, 0.2) -class TestUnityScraper(unittest.TestCase): - """Test main scraper functionality""" +class TestUnityScraper(unittest.TestCase): + """Test main scraper functionality""" + + def setUp(self): + self.temp_dir = tempfile.mkdtemp() + self.database = DatabaseManager(Path(self.temp_dir) / "scraper.db") + + def tearDown(self): + shutil.rmtree(self.temp_dir) def test_validate_titleid(self): """Test TitleID validation""" @@ -145,7 +154,7 @@ def test_scraper_initialization(self, mock_test): """Test scraper initialization""" mock_test.return_value = None config = Config() - scraper = UnityScraper(config) + scraper = UnityScraper(config, database=self.database) self.assertIsNotNone(scraper.session) self.assertIsNotNone(scraper.rate_limiter) @@ -161,7 +170,7 @@ def test_make_request_success(self, mock_get): config = Config() with patch.object(UnityScraper, '_test_connection'): - scraper = UnityScraper(config) + scraper = UnityScraper(config, database=self.database) response = scraper._make_request('http://test.com') self.assertIsNotNone(response) @@ -178,7 +187,7 @@ def test_make_request_rate_limit(self, mock_get): config.max_retries = 1 with patch.object(UnityScraper, '_test_connection'): - scraper = UnityScraper(config) + scraper = UnityScraper(config, database=self.database) start = time.time() response = scraper._make_request('http://test.com') @@ -711,6 +720,15 @@ def test_atomic_copy_verifies_and_publishes(self): self.assertEqual(source.read_bytes(), destination.read_bytes()) self.assertFalse(destination.with_name("destination.bin.partial").exists()) + def test_atomic_copy_reports_mismatched_existing_file_as_conflict(self): + source = self.temp_dir / "source.bin" + destination = self.temp_dir / "destination.bin" + source.write_bytes(b"new") + destination.write_bytes(b"existing") + result = atomic_copy(source, destination) + self.assertEqual(result.status, "conflict") + self.assertEqual(destination.read_bytes(), b"existing") + def test_zip_import_rejects_traversal(self): archive = self.temp_dir / "unsafe.zip" with zipfile.ZipFile(archive, "w") as handle: @@ -826,6 +844,81 @@ def storbinary(self, *_args, **_kwargs): result = client.upload_stfs(package) self.assertEqual(result.status, "skipped") self.assertFalse(fake.stored) + + +class TestRestAPI(unittest.TestCase): + """Test API authentication and configuration safety boundaries.""" + + def setUp(self): + class FakeConfig: + workers = 4 + rate_limit = 0.35 + timeout = 30 + max_retries = 3 + retry_backoff = 2.0 + bandwidth_limit = 0 + verify_checksums = False + dry_run = False + refresh_interval_days = 0 + base_url = "http://xboxunity.net" + http_fallback_url = base_url + use_https = False + + class FakeDatabase: + @staticmethod + def search_titleids(_query): + return [] + + class FakeScraper: + config = FakeConfig() + db = FakeDatabase() + + @staticmethod + def validate_titleid(value): + value = value.upper() + if len(value) == 8 and all( + character in "0123456789ABCDEF" for character in value + ): + return value + return None + + self.scraper = FakeScraper() + + def test_remote_bind_requires_token(self): + with self.assertRaises(ValueError): + UnityScraperAPI(self.scraper, host="0.0.0.0") + + def test_health_reports_current_version_without_token(self): + client = UnityScraperAPI(self.scraper, token="secret").app.test_client() + response = client.get("/api/health") + self.assertEqual(response.status_code, 200) + self.assertEqual(response.get_json()["version"], DISPLAY_VERSION) + + def test_token_protects_non_health_routes(self): + client = UnityScraperAPI(self.scraper, token="secret").app.test_client() + self.assertEqual(client.get("/api/titleids").status_code, 401) + response = client.get( + "/api/titleids", + headers={"Authorization": "Bearer secret"}, + ) + self.assertEqual(response.status_code, 200) + + def test_config_rejects_unknown_and_https_keys(self): + client = UnityScraperAPI(self.scraper).app.test_client() + response = client.post("/api/config", json={"use_https": True}) + self.assertEqual(response.status_code, 400) + self.assertIn("Unsupported", response.get_json()["error"]) + + def test_config_validates_and_applies_allowlisted_values(self): + client = UnityScraperAPI(self.scraper).app.test_client() + response = client.post( + "/api/config", + json={"workers": 8, "rate_limit": 0.5}, + ) + self.assertEqual(response.status_code, 200) + self.assertEqual(self.scraper.config.workers, 8) + self.assertEqual(self.scraper.config.rate_limit, 0.5) + self.assertFalse(self.scraper.config.use_https) def run_tests(): @@ -846,6 +939,7 @@ def run_tests(): suite.addTests(loader.loadTestsFromTestCase(TestBatchDownloadManager)) suite.addTests(loader.loadTestsFromTestCase(TestIntegration)) suite.addTests(loader.loadTestsFromTestCase(TestBackupManager)) + suite.addTests(loader.loadTestsFromTestCase(TestRestAPI)) # Run tests runner = unittest.TextTestRunner(verbosity=2)