diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..ada6302 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,58 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: '' +assignees: '' + +--- + +--- +name: Bug report +about: Report a problem with Remove Samples for NZBGet +title: "[Bug]: " +labels: ["bug"] +assignees: [] +--- + +### Pre-flight +- [ ] I updated Remove Samples via NZBGet **Extension Manager** and retested. +- [ ] I read the README and Troubleshooting notes. + +### Environment +- **Platform:** Docker (Linux) | Unraid | Linux | Windows | macOS +- **NZBGet version:** v23.x +- **Remove Samples version:** 1.0.x +- **Python version:** (if known) + +### Extensions order (from _Settings → Categories_) +1) Completion +2) PasswordDetector +3) FakeDetector +4) ExtendedUnpacker +5) Remove Samples +6) Clean + +### Remove Samples settings +- Remove Directories: Yes/No +- Remove Files: Yes/No +- Video threshold (MB): 150 (default) +- Audio threshold (MB): 2 (default) + +### Steps to reproduce +1. +2. +3. + +### Example filenames/sizes +- `movie.sample.mkv` (90 MB) +- `movie.mkv` (1.4 GB) + +### Expected + + +### Actual + + +### Debug log (short snippet) +> Turn **Debug = Yes** temporarily, reproduce once, paste key lines. Redact personal paths. diff --git a/.github/workflows/manifest.yml b/.github/workflows/manifest.yml index f526463..60b45ea 100644 --- a/.github/workflows/manifest.yml +++ b/.github/workflows/manifest.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Check manifest.json exists run: | diff --git a/.github/workflows/prospector.yml b/.github/workflows/prospector.yml index 61ef6e3..f5555e4 100644 --- a/.github/workflows/prospector.yml +++ b/.github/workflows/prospector.yml @@ -14,10 +14,10 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: '3.8' @@ -30,7 +30,7 @@ jobs: run: prospector --zero-exit main.py - name: Upload Prospector report - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v5 if: always() with: name: prospector-report diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index bd59f94..32e8868 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -17,10 +17,10 @@ jobs: python-version: ['3.8', '3.9', '3.10', '3.11'] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} diff --git a/.gitignore b/.gitignore index b2841dd..1735ff8 100644 --- a/.gitignore +++ b/.gitignore @@ -206,4 +206,7 @@ marimo/_static/ marimo/_lsp/ __marimo__/ -.vscode/ \ No newline at end of file +.vscode/ + +# Kilo Code (keep everything private) +.kilocode/ \ No newline at end of file diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..a5ecbc1 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,147 @@ +# AGENTS.md — RemoveSamples-NZBGet + +Briefing for coding agents and new maintainers working on this repo. Keep changes safe, small, and well tested. + +## 1) What this project does +RemoveSamples-NZBGet is a Post-Processing extension for NZBGet v23+. It finds and removes “sample” clips and junk assets from completed downloads using name patterns and size thresholds. It only performs destructive actions when the overall download status indicates success. + +Goals: +- Idempotent and predictable runs +- Contract correctness +- Clear, minimal logs with a one-line summary +- Cross-platform behavior on Windows and Linux + +Execution order: after ExtendedUnpacker, before Clean and before media managers scan the payload. + +## 2) Contract snapshot + +Exit codes: +- `POSTPROCESS_SUCCESS = 93` +- `POSTPROCESS_ERROR = 94` +- `POSTPROCESS_NONE = 95` + +Env vars read at runtime: + +**Required** +- `NZBPP_DIRECTORY` +- `NZBPP_STATUS` with fallback `NZBPP_TOTALSTATUS` + +**Optional** +- `NZBPP_NZBNAME` +- `NZBPP_CATEGORY` + +Required options provided as env vars: +- `NZBPO_REMOVEDIRECTORIES` +- `NZBPO_REMOVEFILES` +- `NZBPO_DEBUG` +- `NZBPO_VIDEOSIZETHRESHOLDMB` +- `NZBPO_VIDEOEXTS` +- `NZBPO_AUDIOSIZETHRESHOLDMB` +- `NZBPO_AUDIOEXTS` + +Optional toggles (read only if present, some land in v1.1.0): +- `NZBPO_TESTMODE` +- `NZBPO_BLOCKIMPORTDURINGTEST` +- `NZBPO_RELATIVEPERCENT` +- `NZBPO_PROTECTEDPATHS` +- `NZBPO_DENYPATTERNS` +- `NZBPO_IMAGESAMPLES` +- `NZBPO_JUNKEXTRAS` +- `NZBPO_CATEGORYTHRESHOLDS` +- `NZBPO_QUARANTINEMODE` +- `NZBPO_QUARANTINEMAXAGEDAYS` + +If you need a new `NZBPO_*` key, open an issue first. + +## 3) Detection model +- Name patterns: case-insensitive matches for words like `sample`, common dot variants, and typical scene junk. Never hard-code titles. Prefer pattern lists that are user configurable. +- Size thresholds: treat very small video or audio as samples. Defaults come from the options above. Relative-percent can be used when enabled. +- Category overrides: when enabled, thresholds may vary by `NZBPP_CATEGORY`. + +## 4) Safety rules +- If `NZBPP_STATUS` or `NZBPP_TOTALSTATUS` is not success, exit 95 without changing files. +- **Perform deletions or moves only when `NZBPO_REMOVEFILES=yes` or `NZBPO_REMOVEDIRECTORIES=yes`. When `NZBPO_TESTMODE=yes`, never modify files (simulate only).** +- Never traverse above `NZBPP_DIRECTORY`. Reject symlinks. +- Respect `PROTECTEDPATHS` and deny lists. When quarantine is enabled, move to `_samples_quarantine` and optionally purge by age. +- Keep the Windows UTF-8 console safeguard near the top of `main.py` so debug logs do not throw `UnicodeEncodeError`. +- Never traverse above `NZBPP_DIRECTORY`. Reject symlinks and Windows junctions. + +## 5) Logs +- `[INFO]` actions, `[ERROR]` failures, `[DEBUG]` only when `NZBPO_DEBUG` is true. +- Always print one final summary line, for example: + `Summary: removed 2 files, 1 dir; quarantined 0; errors 0; exit=93` +- Return the correct exit code. + +## 6) Local test recipes + +Linux or macOS (bash): +```bash +export NZBPP_DIRECTORY="/abs/path/to/testdir" +export NZBPP_STATUS=SUCCESS +export NZBPO_REMOVEDIRECTORIES=yes +export NZBPO_REMOVEFILES=yes +export NZBPO_DEBUG=no +export NZBPO_VIDEOSIZETHRESHOLDMB=150 +export NZBPO_VIDEOEXTS=".mkv,.mp4,.avi,.mov,.ts,.m4v" +export NZBPO_AUDIOSIZETHRESHOLDMB=2 +export NZBPO_AUDIOEXTS=".mp3,.flac,.m4a,.ogg,.wav" +export NZBPO_TESTMODE=yes +# Optional toggles: +# export NZBPO_BLOCKIMPORTDURINGTEST=yes +# export NZBPO_QUARANTINEMODE=yes +# export NZBPO_QUARANTINEMAXAGEDAYS=30 + +python3 main.py; echo "exit=$?" +``` + +Windows PowerShell: +```powershell +$env:NZBPP_DIRECTORY = "C:\Path\To\testdir" +$env:NZBPP_STATUS = "SUCCESS" +$env:NZBPO_REMOVEDIRECTORIES = "yes" +$env:NZBPO_REMOVEFILES = "yes" +$env:NZBPO_DEBUG = "no" +$env:NZBPO_VIDEOSIZETHRESHOLDMB = "150" +$env:NZBPO_VIDEOEXTS = ".mkv,.mp4,.avi,.mov,.ts,.m4v" +$env:NZBPO_AUDIOSIZETHRESHOLDMB = "2" +$env:NZBPO_AUDIOEXTS = ".mp3,.flac,.m4a,.ogg,.wav" +$env:NZBPO_TESTMODE = "yes" +# Optional toggles (uncomment as needed) +# $env:NZBPO_BLOCKIMPORTDURINGTEST = "yes" +# $env:NZBPO_QUARANTINEMODE = "yes" +# $env:NZBPO_QUARANTINEMAXAGEDAYS = "30" + +python .\main.py; echo "exit=$LASTEXITCODE" +``` + +Test corpus guidelines: +- Include nested `sample` folders, tiny audio, a few image assets, long and Unicode paths. Verify summary text and exit code. + +## 7) Coding guidelines +- Keep env parsing helpers tight and explicit. Do not assume casing or presence. +- Avoid OS-specific path assumptions. Use `pathlib` where possible. +- Do not hard-code user policy. Use options and flags. + +## 8) Versioning and PR checklist +- Increment version when behavior changes and update the changelog. +- PR title in imperative mood and scoped, for example: `fix: honor ProtectedPaths under dry run`. +- Include what you tested, on which OS, and the observed exit code. +- Acceptance criteria: + - Contract header values unchanged + - Required env keys and option names intact + - One summary line in logs + - Windows UTF-8 safeguard present + - Local smoke test completed + +## 9) Common tasks +- Adjust thresholds: change option defaults and descriptions, not the detection core. +- Add deny pattern: extend `NZBPO_DENYPATTERNS` list, not hard-coded checks. +- Protect a file or dir: add to `NZBPO_PROTECTEDPATHS`. +- Enable quarantine: set `QuarantineMode=Yes`, optionally `QuarantineMaxAgeDays>0`. + +## 10) Ownership +If uncertain, run a one-minute local simulation and include 2–3 bullets of findings in the PR. + +Maintainer: Anunnaki-Astronaut + +URL: https://github.com/Anunnaki-Astronaut/RemoveSamples-NZBGet \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..8ebd9b4 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,19 @@ +# Changelog + +## v1.1.0 + +This release introduces several powerful new features for more flexible and safer sample detection, along with a new Test Mode for previewing changes. + +### New Features +* **Test Mode**: A new safe-run option that logs what the script *would* remove without changing any files. Perfect for tuning settings. +* **Block Import During Test**: An optional companion to Test Mode that tells NZBGet to report a failure, preventing media managers from importing a download during a test run. +* **Relative Size % Detection**: A new detection method that identifies a video file as a sample if its size is below a certain percentage of the largest video in the same download. This allows for dynamic thresholds based on the content (e.g., 8% of a 10 GB movie vs. 8% of a 2 GB TV episode). +* **Category Thresholds**: Users can now override the global Relative Size % for specific NZBGet categories. +* **Protected Names/Paths**: A new option to specify file and directory names/patterns that should *never* be removed, giving users full control to protect important assets like subtitles or posters. +* **Deny Patterns**: An extra list of user-defined patterns to flag additional files for removal. +* **Image Samples & Junk Extras**: Optional toggles to remove common image-based samples and other release clutter like `.url` files. +* **Quarantine Mode**: Instead of permanently deleting files, this mode moves them to a `_samples_quarantine` subfolder for review. +* **Quarantine Max Age**: An automatic purge option to delete files from the quarantine folder after a specified number of days. + +### Notes +* Change history for versions prior to v1.1.0 can be found in the release tags on GitHub. \ No newline at end of file diff --git a/README.md b/README.md index e0e5838..f210fa0 100644 --- a/README.md +++ b/README.md @@ -1,256 +1,165 @@ -# RemoveSamples - NZBGet Extension +# Remove Samples • NZBGet Extension [![Tests](https://github.com/Anunnaki-Astronaut/RemoveSamples-NZBGet/actions/workflows/tests.yml/badge.svg)](https://github.com/Anunnaki-Astronaut/RemoveSamples-NZBGet/actions/workflows/tests.yml) [![Prospector](https://github.com/Anunnaki-Astronaut/RemoveSamples-NZBGet/actions/workflows/prospector.yml/badge.svg)](https://github.com/Anunnaki-Astronaut/RemoveSamples-NZBGet/actions/workflows/prospector.yml) [![Manifest Check](https://github.com/Anunnaki-Astronaut/RemoveSamples-NZBGet/actions/workflows/manifest.yml/badge.svg)](https://github.com/Anunnaki-Astronaut/RemoveSamples-NZBGet/actions/workflows/manifest.yml) -**Modern NZBGet extension** for intelligent sample file detection and removal. Automatically cleans sample files and directories before Sonarr/Radarr/Lidarr/Prowlarr processing. +NZBGet extension that removes "sample" files and folders before Sonarr/Radarr/Lidarr/Prowlarr see the download. Keeps your library clean while protecting real content with conservative defaults. -> 🔄 **Replaces the legacy DeleteSamples.py script** with modern extension format and advanced detection algorithms. - -## 🚀 Quick Start +--- + +## Overview + +Scene releases often include short sample clips, promo images, and other junk alongside the real media. If those make it into your library you get: + +* noisy episode/movie folders +* extra files in Plex/Jellyfin/Kodi +* higher chance of grabbing the wrong file in manual imports + +**Remove Samples** runs after unpacking and before your media managers. It identifies sample-like content using filename patterns, size thresholds, and optional relative-size rules, then removes (or quarantines) the junk so only real media moves downstream. + +--- + +## Key features + +* **Sample-aware filename matching** + Detects common patterns like `sample`, `SAMPLE`, `.sample.`, `_sample_`, etc., using word- and separator-aware matching to avoid false positives. + +* **Size-based detection for video & audio** + Treats very small video and audio files under your thresholds as samples (e.g. tiny preview clips). + +* **Relative Size % detection (optional)** + Flags a video as a sample when its size is below a certain percentage of the largest video in the same download. This gives you dynamic thresholds that scale with the release. + +* **Per-category overrides** + Category-specific thresholds let you tune behavior differently for TV, movies, music, etc., while keeping safe global defaults. + +* **Safety tools for testing and recovery** + + * **Test Mode** – dry-run logging that shows what *would* be removed without touching the files. + * **Block Import During Test** – optional companion to Test Mode that tells NZBGet to report a failure so media managers don’t import during a test run. + * **Quarantine Mode** – instead of deleting, moves samples to a `_samples_quarantine` subfolder for manual review. + * **Quarantine Max Age** – optional automatic cleanup of old quarantine files after a configurable number of days. + +* **Protected Paths & Deny Patterns** + + * **Protected paths/names** let you explicitly shield things like subtitles, artwork, or NFOs so they are never removed, even if they look like samples. + * **Deny patterns** are a configurable list of extra patterns you always want treated as junk. -**📖 [Complete Documentation](../../wiki/Home)** | **🚀 [Installation Guide](../../wiki/02_Installation_Guide)** | **⚙️ [Configuration Reference](../../wiki/03_Configuration_Reference)** - -## ✨ Key Features - -- 🎯 **Smart Detection**: Advanced pattern matching with word boundary detection -- 📁 **Directory Cleanup**: Removes entire sample directories (`samples/`, `SAMPLE/`) -- 🎬 **Video Support**: Configurable size thresholds for different quality levels -- 🎵 **Audio Support**: Separate detection logic for audio samples -- ⚙️ **Modern Interface**: GUI dropdown configuration (no file editing!) -- 🐳 **Docker Ready**: Works with all popular NZBGet Docker containers -- 🔧 **Flexible**: Independent control over file and directory removal -- 🛡️ **Enterprise-Grade**: Automated security scanning and dependency monitoring - -## 🆚 Why Choose RemoveSamples? - -**vs DeleteSamples.py (Legacy Script)** - -| Feature | DeleteSamples.py | RemoveSamples | -|---------|-----------------|---------------| -| **Configuration** | ❌ Manual file editing | ✅ Modern dropdown interface | -| **Directory Removal** | ❌ Files only | ✅ Files AND directories | -| **Extension Format** | ❌ Legacy script | ✅ Modern NZBGet extension | -| **Pattern Detection** | ❌ Basic substring | ✅ Advanced pattern matching | -| **Audio Support** | ❌ Limited | ✅ Full configurable support | -| **Maintenance** | ❌ Abandoned (6+ years) | ✅ Active development | - -**[See detailed comparison →](../../wiki/09_Comparison_DeleteSamples)** - -## 📦 Installation - -### Method 1: Extension Manager (Recommended) -1. Open NZBGet web interface -2. Go to **Settings** → **Extension Manager** -3. Find "RemoveSamples" in the list -4. Click **Install** - -### Method 2: Manual Installation -```bash -# Download and extract to NZBGet scripts directory -mkdir -p /path/to/nzbget/scripts/RemoveSamples/ -# Copy main.py and manifest.json -chmod 755 main.py && chmod 644 manifest.json -``` - -### Method 3: Docker/Unraid -```bash -# For Unraid NZBGet containers -cd /mnt/user/appdata/nzbget/scripts/ -mkdir -p RemoveSamples -# Download files and set permissions for nobody:users -``` +* **Image & extras cleanup (optional)** + Optional toggles to remove common screenshot/image samples and other minor extras left behind by some releases. -**📖 [Detailed installation instructions for all platforms →](../../wiki/02_Installation_Guide)** - -## ⚙️ Configuration - -### Basic Settings (Dropdown Interface) -``` -Remove Directories: Yes # Delete sample directories -Remove Files: Yes # Delete sample files -Debug: No # Enable for troubleshooting -``` - -### Advanced Thresholds -``` -Video Size Threshold: 150 MB # 720p: 50MB, 1080p: 100MB, 4K: 300MB+ -Audio Size Threshold: 2 MB # ~30 seconds of 320kbps MP3 -``` - -### Recommended Settings by Use Case - -**Conservative (New Users)** -``` -Video: 300 MB | Audio: 5 MB | Debug: Yes -``` +--- + +## Install + +**NZBGet → Settings → Extension Manager** + +1. Find **Remove Samples** in the list. +2. Click the download/install icon. +3. That’s it. + +--- + +## Basic configuration + +**NZBGet → Settings → Extension Manager → Remove Samples** -**Balanced (Most Users)** -``` -Video: 150 MB | Audio: 2 MB | Debug: No -``` - -**Aggressive (High Volume)** -``` -Video: 50 MB | Audio: 1 MB | Debug: No -``` - -**📖 [Complete configuration guide →](../../wiki/03_Configuration_Reference)** +For most users, the defaults are a safe starting point: -## 🔄 Workflow Integration +* **Video Size Threshold (MB):** `150` + Small video files under this size are considered candidates for sample detection. +* **Audio Size Threshold (MB):** `2` + Small audio files (e.g., preview tracks) are treated as samples. +* **Remove Directories:** `Yes` + Removes entire folders that look like sample directories. +* **Remove Files:** `Yes` + Removes individual files that match sample patterns. +* **Debug:** `No` + Leave off for daily use. Turn on temporarily when tuning settings or diagnosing behavior. -### Recommended Script Order -``` -1. PasswordDetector (if used) -2. FakeDetector (if used) -3. RemoveSamples ← Place here -4. Clean (if used) -5. Other scripts -``` - -### Media Manager Integration -- **Sonarr**: Cleaner TV episode imports, no sample episodes -- **Radarr**: No trailer/sample files in movie folders -- **Lidarr**: No 30-second preview tracks in albums -- **Prowlarr**: Consistent cleanup across all content types - -**📖 [Complete workflow integration guide →](../../wiki/05_Workflow_Integration)** - -## 📊 Sample Detection Examples - -### ✅ Will Be Removed -``` -Movie.Name.2023.sample.mkv # Pattern + size match -sample.mp4 # Clear sample file -preview_sample.avi # Sample pattern -samples/ # Sample directory -Small.video.under.150MB.mkv # Size-based detection -``` - -### ❌ Will Be Preserved -``` -Movie.Name.2023.1080p.mkv # Normal size, no pattern -soundtrack.mp3 # No sample pattern -behind-the-scenes.mp4 # Above size threshold -Movie.Title.SAMPLE.2023.mkv # If "SAMPLE" in original title -``` +### Recommended defaults & safety notes -## 🔍 Detection Logic +* Start with the bundled defaults; they are intentionally conservative. +* **Relative Size %** defaults to **8%**, which provides a good balance for most content. Most users can leave this and **Category Thresholds** at their defaults. +* **Protected Paths** always win: if a file matches a protected pattern (for example `*.srt` for subtitles), it will **never** be removed, even if it also looks like a sample. +* When experimenting with new thresholds or patterns, enable **Test Mode** first so you can review log output before allowing deletions or quarantine moves. -### Pattern Matching -- **Word boundary detection**: `\bsample\b` prevents false positives -- **Multiple separators**: `.sample.`, `_sample.`, `-sample.` -- **Directory patterns**: Comprehensive sample directory detection - -### Size-Based Detection -- **Separate thresholds** for video/audio files -- **Configurable extensions** for each media type -- **Smart combination** of pattern and size detection - -**📖 [Complete detection logic documentation →](../../wiki/06_Detection_Logic)** - -## 🐳 Docker & Container Support +--- + +## Extension order in NZBGet -**Fully compatible with popular Docker containers:** -- ✅ `linuxserver/nzbget` (Recommended) -- ✅ `nzbget/nzbget` (Official) -- ✅ Unraid Community Applications NZBGet -- ✅ Custom Docker Compose setups +**NZBGet → Settings → Categories → ``.Extensions** -**Container-specific installation guides available in documentation.** +Place **RemoveSamples** **after** unpacking and **before** any final cleanup or media managers. -## 🚨 Troubleshooting +**Example (working setup):** -### Quick Diagnostics -```bash -# Enable debug mode -Settings → Extension Manager → RemoveSamples → Debug: Yes - -# Check logs -Settings → Logging → Messages +1. **Completion** – Verifies download completeness before processing +2. **PasswordDetector** – Detects password-protected archives early +3. **FakeDetector** – Flags fake/corrupted releases +4. **ExtendedUnpacker** – Extracts nested zip/rar archives +5. **RemoveSamples** – Removes sample files/folders **after unpack** +6. **Clean** – Final tidy-up -# Verify installation -ls -la /path/to/scripts/RemoveSamples/ -# Should show: main.py (executable) and manifest.json -``` +**Why order matters** -### Common Issues -- **Extension not appearing**: Check file permissions and restart NZBGet -- **Files not removed**: Verify thresholds and enable debug mode -- **Docker permissions**: Use container-appropriate user/group +* Remove Samples runs **after unpack**, so it can see real files. +* It runs **before Clean**, so samples are removed before final cleanup. +* Upstream detection scripts run first to catch bad releases early. -**📖 [Complete troubleshooting guide →](../../wiki/07_Troubleshooting_Guide)** +--- -## 📞 Support & Documentation +## Quick test / first-run checklist -- **📖 Complete Wiki**: [Comprehensive Documentation](../../wiki/01_Home) -- **🐛 Bug Reports**: [GitHub Issues](https://github.com/Anunnaki-Astronaut/RemoveSamples-NZBGet/issues) -- **💬 Discussions**: [GitHub Discussions](https://github.com/Anunnaki-Astronaut/RemoveSamples-NZBGet/discussions) -- **🔒 Security Issues**: anunnaki.astronaut@machinamindmeld.com -- **❓ FAQ**: [Frequently Asked Questions](../../wiki/08_FAQ) +**Recommended first step – Test Mode only** -## 🛡️ Security & Quality +1. In Extension Manager, set **Test Mode = Yes**. +2. (Optional) Enable **Block Import (Test Mode) = Yes** if you want to prevent Sonarr/Radarr/Lidarr/Prowlarr from seeing the test download. +3. Process a known-good test download. +4. Open **NZBGet → Messages** and review the log lines: -RemoveSamples is built with enterprise-grade practices: -- **Automated security scanning** with CodeQL -- **Dependency vulnerability monitoring** with Dependabot -- **Comprehensive test coverage** with automated CI/CD -- **Professional code review** workflow + * size checks for video/audio + * matches on filename patterns + * summary line showing how many files/dirs would be removed or quarantined +5. Once you’re satisfied, set **Test Mode = No** (and **Block Import (Test Mode) = No** if you enabled it) to allow real removals or quarantine moves. -## 🏆 Official Recognition +**When to use Debug** -**🎉 RemoveSamples is now officially available in the NZBGet Extension Manager!** +* Turn **Debug = Yes** **by itself** (with Test Mode left at `No`) when you need deeper, per-item decision details to understand *why* something was or wasn’t treated as a sample. +* After troubleshooting, set **Debug = No** again for normal operation. -*RemoveSamples has been accepted by the NZBGet team and is available for one-click installation through the official Extension Manager.* +--- -## 📋 Requirements +## Detection logic (short) -- **NZBGet**: Version 14.0 or later (21.0+ recommended) -- **Python**: 3.8+ installed on your system -- **Permissions**: Execute permission on main.py +* **Word-boundary matching:** uses patterns like `\bsample\b` to avoid false positives inside longer words. +* **Separator-aware:** catches `.sample.`, `_sample_`, `-sample-`, and similar separators in filenames. +* **Size checks:** very small video/audio files under your thresholds are considered sample candidates. +* **Relative-size checks (optional):** flags videos that are much smaller than the main video in the same download when Relative Size % is enabled. -## 🔧 Development +--- -### Running Tests -```bash -python -m unittest tests.py -v -``` +## Windows debug console note -### Code Quality Checks -```bash -pip install prospector -prospector main.py -``` +If you previously saw a Unicode/console encoding error with **Debug** enabled on Windows, update to the latest version via Extension Manager. The script now uses UTF-8 console output on Windows so Debug works normally. -### Contributing -1. Fork the repository -2. Create a feature branch: `git checkout -b feature-name` -3. Make your changes with tests -4. Ensure all tests pass: `python -m unittest tests.py -v` -5. Submit a pull request +--- -**📖 [Development documentation →](../../wiki/10_Contributing)** +## NZBGet versions / requirements -## 📄 License +* **NZBGet:** v23+ recommended +* **Python:** 3.8+ (required) -GNU General Public License v2.0 - see [LICENSE](LICENSE) file for details. +--- -## 📈 Changelog +## Support -### v1.0.1 - Official Release -- ✅ **Official NZBGet adoption** - Available in Extension Manager -- ✅ Modern NZBGet extension format with manifest.json -- ✅ GUI dropdown configuration interface -- ✅ Advanced pattern matching with word boundaries -- ✅ Comprehensive sample detection (files + directories) -- ✅ Configurable video/audio size thresholds -- ✅ Full test coverage with automated CI/CD -- ✅ Docker and Unraid compatibility -- ✅ Enterprise-grade security practices -- ✅ Complete documentation wiki +* **Bug Reports**: [https://github.com/Anunnaki-Astronaut/RemoveSamples-NZBGet/issues](https://github.com/Anunnaki-Astronaut/RemoveSamples-NZBGet/issues) +* **Discussions**: [https://github.com/Anunnaki-Astronaut/RemoveSamples-NZBGet/discussions](https://github.com/Anunnaki-Astronaut/RemoveSamples-NZBGet/discussions) --- -**Ready to get started?** → **[Installation Guide](../../wiki/Installation-Guide)** -**Need help?** → **[FAQ](../../wiki/FAQ)** | **[Troubleshooting](../../wiki/Troubleshooting-Guide)** \ No newline at end of file +## License + +**GNU General Public License v2.0** – see the LICENSE file for details. \ No newline at end of file diff --git a/main.py b/main.py index 8657ba4..10697b8 100644 --- a/main.py +++ b/main.py @@ -27,17 +27,141 @@ import re import shutil import sys +import fnmatch +import time from pathlib import Path -# --- NZBGet exit codes ----------------------------------------------------- +# === BEGIN_NZBGET_V23_CONTRACT === +# Exit constants (DO NOT RENAME OR CHANGE VALUES) POSTPROCESS_SUCCESS = 93 POSTPROCESS_ERROR = 94 POSTPROCESS_NONE = 95 +# Required NZBGet envs +# - NZBPP_DIRECTORY +# - NZBPP_STATUS (fallback to NZBPP_TOTALSTATUS) +# - NZBPP_NZBNAME (optional) +# +# Required options for v1.0.3+ +# - NZBPO_REMOVEDIRECTORIES +# - NZBPO_REMOVEFILES +# - NZBPO_DEBUG +# - NZBPO_VIDEOSIZETHRESHOLDMB +# - NZBPO_VIDEOEXTS +# - NZBPO_AUDIOSIZETHRESHOLDMB +# - NZBPO_AUDIOEXTS +# +# Allowed optional options (v1.1.0+) +# - NZBPO_TESTMODE +# - NZBPO_BLOCKIMPORTDURINGTEST +# - NZBPO_RELATIVEPERCENT +# - NZBPO_PROTECTEDPATHS +# - NZBPO_DENYPATTERNS +# - NZBPO_IMAGESAMPLES +# - NZBPO_JUNKEXTRAS +# - NZBPO_CATEGORYTHRESHOLDS +# - NZBPO_QUARANTINEMODE +# - NZBPO_QUARANTINEMAXAGEDAYS +# === END_NZBGET_V23_CONTRACT === + + +# Force UTF-8 console on Windows to avoid UnicodeEncodeError in debug logs +def _enable_utf8_windows(): + if os.name == "nt": + try: + sys.stdout.reconfigure(encoding="utf-8", errors="replace") + sys.stderr.reconfigure(encoding="utf-8", errors="replace") + except Exception: + pass # Continue silently if UTF-8 can't be forced + + +_enable_utf8_windows() + + +# ---------- Helpers ---------- +def _env_bool(key, default=False): + val = os.environ.get(key) + if val is None: + return default + return str(val).strip().lower() in ("1", "true", "yes", "on", "y", "t", "enabled", "enable") + + +def _env_int(key, default=0): + try: + return int(str(os.environ.get(key, str(default))).strip()) + except (ValueError, TypeError): + return default + + +def _env_str(key, default=""): + return str(os.environ.get(key, default)).strip() + + +def _parse_exts(s): + if not s: + return set() + out = set() + for t in re.split(r"[,\s;]+", s): + t = t.strip().lower() + if not t: + continue + if not t.startswith("."): + t = "." + t + out.add(t) + return out + + +def _csv_list(s): + if not s: + return [] + return [x.strip() for x in re.split(r"[,;\n]+", s) if x.strip()] + + +def info(msg): print(f"[INFO] {msg}", flush=True) +def error(msg): print(f"[ERROR] {msg}", flush=True) +def debug_enabled(): return _env_bool("NZBPO_DEBUG", False) +def debug(msg): + if debug_enabled(): + print(f"[DEBUG] {msg}", flush=True) + + +def _size_mb(p: Path) -> float: + try: + return p.stat().st_size / (1024 * 1024) + except OSError: + return 0.0 + + +def _matches_any(base: Path, p: Path, patterns): + if not patterns: + return False + # Use resolved paths for accurate relative_to checks + try: + rel = str(p.resolve().relative_to(base.resolve())).replace("\\", "/") + except ValueError: + rel = str(p.resolve()).replace("\\", "/") # Fallback if not relative + name = p.name + for pat in patterns: + if fnmatch.fnmatch(name, pat) or fnmatch.fnmatch(rel, pat): + return True + return False + + +def _parse_cat_overrides(s: str): + out = {} + for part in [t.strip() for t in s.split(",") if t.strip()]: + if "=" in part: + k, v = part.split("=", 1) + try: + out[k.strip().lower()] = int(v.strip()) + except ValueError: + pass + return out + print("[DETAIL] RemoveSamples extension started") sys.stdout.flush() -# Check required options +# Check required options from v1.0.3 REQUIRED_OPTIONS = ( "NZBPO_REMOVEDIRECTORIES", "NZBPO_REMOVEFILES", @@ -51,139 +175,262 @@ for optname in REQUIRED_OPTIONS: if optname not in os.environ: error_msg = ( - f"[ERROR] Option {optname[6:]} is missing in configuration " + f"Option {optname[6:]} is missing in configuration " f"file. Please check script settings" ) - print(error_msg) + error(error_msg) sys.exit(POSTPROCESS_ERROR) -# ── Read NZBGet variables & user options ─────────────────────────────────── -DEBUG = os.environ.get('NZBPO_DEBUG', 'No').lower() in ( - 'yes', 'y', 'true', '1' -) -REM_DIRS = os.environ.get('NZBPO_REMOVEDIRECTORIES', 'Yes').lower() in ( - 'yes', 'y', 'true', '1' -) -REM_FILES = os.environ.get('NZBPO_REMOVEFILES', 'Yes').lower() in ( - 'yes', 'y', 'true', '1' -) -VID_LIMIT = int(os.environ.get('NZBPO_VIDEOSIZETHRESHOLDMB', '150') or '0') -AUD_LIMIT = int(os.environ.get('NZBPO_AUDIOSIZETHRESHOLDMB', '2') or '0') - -VIDEO_EXTS = { - e if e.startswith('.') else f'.{e}' - for e in os.environ.get( - 'NZBPO_VIDEOEXTS', - '.mkv,.mp4,.avi,.mov,.wmv,.flv,.webm,.ts,.m4v,.vob' - ).split(',') if e.strip() -} - -AUDIO_EXTS = { - e if e.startswith('.') else f'.{e}' - for e in os.environ.get( - 'NZBPO_AUDIOEXTS', - '.wav,.aiff,.mp3,.flac,.m4a,.ogg,.aac,.alac,.ape,.opus,.wma' - ).split(',') if e.strip() -} -DL_DIR = os.environ.get('NZBPP_DIRECTORY') -DL_STATUS = os.environ.get('NZBPP_STATUS', '') -DL_NAME = os.environ.get('NZBPP_NZBNAME', '') - - -def log(level, message): - """Log a message with the specified level.""" - print(f"[{level}] {message}") - - -def debug_log(message): - """Log a debug message if debug mode is enabled.""" - if DEBUG: - log("DEBUG", message) - - -# ── Regex patterns ───────────────────────────────────────────────────────── -FILE_PATTERNS = [ - r'\bsample\b', r'\.sample\.', r'^sample\.', r'_sample\.', r'-sample\.', - r'sample[_-]' -] -DIR_PATTERNS = [r'\bsamples?\b', r'^samples?$'] - -FILE_RE = [re.compile(pattern, re.I) for pattern in FILE_PATTERNS] -DIR_RE = [re.compile(pattern, re.I) for pattern in DIR_PATTERNS] - - -def is_sample_file(path): - """Check if a file is considered a sample based on name and size.""" - name = path.name - if any(regex.search(name) for regex in FILE_RE): - return True - - ext = path.suffix.lower() - try: - size_mb = path.stat().st_size / (1 << 20) - except OSError: - return False - - if ext in VIDEO_EXTS and VID_LIMIT and size_mb <= VID_LIMIT: - return True - if ext in AUDIO_EXTS and AUD_LIMIT and size_mb <= AUD_LIMIT: - return True - return False - - -def is_sample_dir(dir_name): - """Check if a directory name matches sample patterns.""" - return any(regex.search(dir_name) for regex in DIR_RE) - - -def remove_samples(root): - """Remove sample files and directories from the specified root path.""" - removed = [] - for path in sorted(root.rglob('*'), reverse=True): - if path.is_file() and REM_FILES and is_sample_file(path): - path.unlink(missing_ok=True) - removed.append(str(path)) - log("INFO", f"Removed file {path.relative_to(root)}") - elif path.is_dir() and REM_DIRS and is_sample_dir(path.name): - shutil.rmtree(path, ignore_errors=True) - removed.append(str(path)) - log("INFO", f"Removed dir {path.relative_to(root)}") - return removed +# ---------- Inputs ---------- +DL_STATUS = os.environ.get("NZBPP_STATUS", "") or os.environ.get("NZBPP_TOTALSTATUS", "") +DEST_DIR = Path(_env_str("NZBPP_DIRECTORY", ".")).resolve() +NZB_NAME = _env_str("NZBPP_NZBNAME", "") +CATEGORY = _env_str("NZBPP_CATEGORY", "") + +# v1.0.3 options +REMOVE_DIRS = _env_bool("NZBPO_REMOVEDIRECTORIES", False) +REMOVE_FILES = _env_bool("NZBPO_REMOVEFILES", False) +VID_LIMIT = _env_int("NZBPO_VIDEOSIZETHRESHOLDMB", 150) +AUD_LIMIT = _env_int("NZBPO_AUDIOSIZETHRESHOLDMB", 2) +VIDEO_EXTS = _parse_exts(_env_str("NZBPO_VIDEOEXTS", ".mkv,.mp4,.avi,.mov,.ts,.m4v,.wmv")) +AUDIO_EXTS = _parse_exts(_env_str("NZBPO_AUDIOEXTS", ".mp3,.flac,.aac,.m4a,.ogg,.opus,.wav")) + +# v1.1.0 optional features +TEST_MODE = _env_bool("NZBPO_TESTMODE", False) +BLOCK_IMPORT_DURING_TEST = _env_bool("NZBPO_BLOCKIMPORTDURINGTEST", False) +RELATIVE_PERCENT = _env_int("NZBPO_RELATIVEPERCENT", -1) +PROTECTED_PATHS = _csv_list(_env_str("NZBPO_PROTECTEDPATHS", "")) +DENY_PATTERNS = _csv_list(_env_str("NZBPO_DENYPATTERNS", "")) +IMAGE_SAMPLES = _env_bool("NZBPO_IMAGESAMPLES", False) +JUNK_EXTRAS = _env_bool("NZBPO_JUNKEXTRAS", False) +CATEGORY_THRESH = _env_str("NZBPO_CATEGORYTHRESHOLDS", "") +QUARANTINE_MODE = _env_bool("NZBPO_QUARANTINEMODE", False) +QUARANTINE_MAX_AGE_DAYS = _env_int("NZBPO_QUARANTINEMAXAGEDAYS", 0) + +# Name patterns for "sample" +SAMPLE_NAME_RE_FILE = re.compile(r"(?i)(?:^|[\\/_\.\-\s])sample(?:s)?(?:$|[\\/_\.\-\s])") +SAMPLE_NAME_RE_DIR = re.compile(r"(?i)(?:^|[\\/_\.\-\s])samples?(?:$|[\\/_\.\-\s])") def main(): """Main entry point for the RemoveSamples extension.""" - # Check if running in post-processing mode - if not DL_DIR: - error_msg = ( - "NZBPP_DIRECTORY missing - script must run in " - "post-processing mode" - ) - log("ERROR", error_msg) + if not _env_str("NZBPP_DIRECTORY"): + error("NZBPP_DIRECTORY missing - script must run in post-processing mode") sys.exit(POSTPROCESS_ERROR) if not DL_STATUS.upper().startswith('SUCCESS'): - log("INFO", f"Status {DL_STATUS}; skipping.") + info(f"Status {DL_STATUS}; skipping.") + info("Summary: 0 removed (status not SUCCESS). Mode: " + ("TEST" if TEST_MODE else "LIVE")) sys.exit(POSTPROCESS_NONE) - if not os.path.exists(DL_DIR): - log("ERROR", f"Destination directory doesn't exist: {DL_DIR}") + if not DEST_DIR.exists(): + error(f"Destination directory not found: {DEST_DIR}") sys.exit(POSTPROCESS_NONE) - log("INFO", f'Processing "{DL_NAME}" in {DL_DIR}') - - debug_info = ( - f"dirs={REM_DIRS}, files={REM_FILES}, " - f"vid≤{VID_LIMIT} MB, aud≤{AUD_LIMIT} MB, " - f"vExt={sorted(VIDEO_EXTS)}, aExt={sorted(AUDIO_EXTS)}" + info(f'Processing "{NZB_NAME}" in {DEST_DIR}') + + # Category override for relative percent + eff_relative_percent = RELATIVE_PERCENT + cat_map = _parse_cat_overrides(CATEGORY_THRESH) + if CATEGORY and cat_map and CATEGORY.lower() in cat_map: + eff_relative_percent = cat_map[CATEGORY.lower()] + info(f"Category override: {CATEGORY} → Relative Size % = {eff_relative_percent}") + + # Heads-up notes + if TEST_MODE and QUARANTINE_MODE: + info("Heads-up: Quarantine is ignored while Test Mode is ON.") + if eff_relative_percent >= 0 and VID_LIMIT >= 400 and eff_relative_percent <= 5: + info("Heads-up: High video threshold with low relative percent may skip everything.") + if BLOCK_IMPORT_DURING_TEST and not TEST_MODE: + info("Reminder: BlockImportDuringTest is designed for Test Mode.") + + # ---------- Scan ---------- + files_considered = 0 + dirs_considered = 0 + file_candidates = [] + dir_candidates = [] + errors = 0 + + def is_video(p: Path) -> bool: return p.suffix.lower() in VIDEO_EXTS + def is_audio(p: Path) -> bool: return p.suffix.lower() in AUDIO_EXTS + def is_image(p: Path) -> bool: return p.suffix.lower() in {".jpg", ".jpeg", ".png", ".bmp", ".gif", ".webp"} + + all_files = [] + all_dirs = [] + for p in DEST_DIR.rglob("*"): + try: + if p.is_symlink(): + debug(f"Skip symlink: {p}") + continue + if p.is_file(): + all_files.append(p) + elif p.is_dir(): + all_dirs.append(p) + except Exception as ex: + errors += 1 + error(f"Scan error at {p}: {ex}") + + files_considered = len(all_files) + dirs_considered = len(all_dirs) + + largest_video_bytes = 0 + if eff_relative_percent >= 0: + for f in all_files: + if is_video(f): + try: + s = f.stat().st_size + if s > largest_video_bytes: + largest_video_bytes = s + except OSError: + pass + + # Directories by name pattern + for d in all_dirs: + try: + if SAMPLE_NAME_RE_DIR.search(d.name): + if _matches_any(DEST_DIR, d, PROTECTED_PATHS): + debug(f"Protected directory: {d}") + else: + dir_candidates.append(d) + debug(f"Candidate dir: {d}") + except Exception as ex: + errors += 1 + error(f"Dir scan error at {d}: {ex}") + + # Files by name, size, optional extras + for f in all_files: + try: + if _matches_any(DEST_DIR, f, PROTECTED_PATHS): + debug(f"Protected file: {f}") + continue + + mb = _size_mb(f) + name_hit = SAMPLE_NAME_RE_FILE.search(f.name) is not None + small_video = is_video(f) and (VID_LIMIT > 0 and mb < float(VID_LIMIT)) + small_audio = is_audio(f) and (AUD_LIMIT > 0 and mb < float(AUD_LIMIT)) + + rel_hit = False + if is_video(f) and eff_relative_percent >= 0 and largest_video_bytes > 0: + try: + pct = int(round((f.stat().st_size / largest_video_bytes) * 100)) + rel_hit = (pct <= eff_relative_percent) + except OSError: + rel_hit = False + + deny_hit = _matches_any(DEST_DIR, f, DENY_PATTERNS) + image_hit = IMAGE_SAMPLES and is_image(f) and (("sample" in f.name.lower()) or deny_hit) + junk_hit = JUNK_EXTRAS and (deny_hit or f.suffix.lower() in {".url", ".webloc"} or f.name.lower().endswith("readme.txt")) + + if name_hit or small_video or small_audio or rel_hit or image_hit or junk_hit or deny_hit: + file_candidates.append((f, mb, name_hit, small_video, small_audio, rel_hit, image_hit, junk_hit, deny_hit)) + debug(f"Candidate file: {f} | {mb:.1f} MB | name={name_hit} smallV={small_video} smallA={small_audio} rel%={rel_hit} img={image_hit} junk={junk_hit} deny={deny_hit}") + + except Exception as ex: + errors += 1 + error(f"File scan error at {f}: {ex}") + + if TEST_MODE and BLOCK_IMPORT_DURING_TEST and (file_candidates or dir_candidates): + info("BlockImportDuringTest=ON with candidates → reporting 94 to prevent import (no deletions performed).") + info("Summary: 0 removed (blocked during Test). Mode: TEST") + sys.exit(POSTPROCESS_ERROR) + + # ---------- Act ---------- + removed_files = 0 + removed_dirs = 0 + removed_mb_total = 0.0 + quar_dir = DEST_DIR / "_samples_quarantine" + + def _safe_move(src: Path): + dst = quar_dir / src.resolve().relative_to(DEST_DIR) + dst.parent.mkdir(parents=True, exist_ok=True) + shutil.move(str(src), str(dst)) + + # Directories + if REMOVE_DIRS: + for d in sorted(dir_candidates, key=lambda p: len(str(p)), reverse=True): + try: + rel = d.resolve().relative_to(DEST_DIR) + if TEST_MODE: + info(f"[TEST] Would remove directory: {rel}") + elif QUARANTINE_MODE: + moved_any = False + for p in d.rglob("*"): + if p.is_file(): + try: + _safe_move(p) + moved_any = True + except Exception as ex: + errors += 1; error(f"Quarantine move failed {p}: {ex}") + shutil.rmtree(d, ignore_errors=True) + if moved_any: removed_dirs += 1; info(f"[QUARANTINE] Directory contents moved: {rel}") + else: + shutil.rmtree(d) + removed_dirs += 1 + info(f"Removed directory: {rel}") + except Exception as ex: + errors += 1; error(f"Failed to process directory: {d} - {ex}") + else: + debug("Configured to keep directories (NZBPO_REMOVEDIRECTORIES=No)") + + # Files + if REMOVE_FILES: + for f_tuple in file_candidates: + f, mb, *_ = f_tuple + try: + # Re-check existence as parent dir might be gone + if not f.exists(): + continue + rel = f.resolve().relative_to(DEST_DIR) + if TEST_MODE: + info(f"[TEST] Would remove file: {rel} ({mb:.1f} MB)") + elif QUARANTINE_MODE: + _safe_move(f) + removed_files += 1; removed_mb_total += mb + info(f"[QUARANTINE] {rel} ({mb:.1f} MB)") + else: + f.unlink() + removed_files += 1; removed_mb_total += mb + info(f"Removed file: {rel} ({mb:.1f} MB)") + except Exception as ex: + errors += 1; error(f"Failed to process file: {f} - {ex}") + else: + debug("Configured to keep files (NZBPO_REMOVEFILES=No)") + + mode = "TEST" if TEST_MODE else ("LIVE+QUARANTINE" if QUARANTINE_MODE else "LIVE") + info( + f"Summary: removed {removed_files} files / {removed_dirs} dirs " + f"({removed_mb_total:.1f} MB). Mode: {mode}. " + f"FilesChecked={files_considered} DirsChecked={dirs_considered} " + f"Candidates={len(file_candidates)+len(dir_candidates)} " + f"Rel%={'disabled' if eff_relative_percent < 0 else eff_relative_percent} " + f"VideoMB>={VID_LIMIT}" ) - debug_log(debug_info) - removed_count = len(remove_samples(Path(DL_DIR))) - log("INFO", f"Removed {removed_count} sample item(s)") + if QUARANTINE_MODE and QUARANTINE_MAX_AGE_DAYS > 0 and quar_dir.exists(): + cutoff = time.time() - (QUARANTINE_MAX_AGE_DAYS * 86400) + try: + for p in quar_dir.rglob("*"): + try: + if p.is_file() and p.stat().st_mtime < cutoff: + p.unlink(); debug(f"Purged old quarantine file: {p.name}") + except Exception as ex: + error(f"Quarantine purge failed at {p}: {ex}") + # Clean up empty subdirs + for sub in sorted(quar_dir.rglob("*"), key=lambda p: len(str(p)), reverse=True): + if sub.is_dir() and not any(sub.iterdir()): + sub.rmdir() + except Exception: + pass + + if errors > 0: + sys.exit(POSTPROCESS_ERROR) + + if removed_files == 0 and removed_dirs == 0: + # Exit 95 if no action was taken. In Test Mode, this is the expected outcome + # if candidates were found but no removal occurred. + sys.exit(POSTPROCESS_NONE) - print("[DETAIL] RemoveSamples extension completed successfully") sys.exit(POSTPROCESS_SUCCESS) diff --git a/manifest.json b/manifest.json index 045cb4e..7ef242a 100644 --- a/manifest.json +++ b/manifest.json @@ -1,100 +1,208 @@ { - "main": "main.py", - "name": "RemoveSamples", - "homepage": "https://github.com/Anunnaki-Astronaut/RemoveSamples-NZBGet", - "kind": "POST-PROCESSING", - "displayName": "Remove Samples", - "version": "1.0.1", - "author": "Anunnaki-Astronaut", - "license": "GNU", - "about": "Modern NZBGet extension for intelligent sample file detection and removal. Automatically cleans sample files/directories before Sonarr/Radarr/Lidarr/Prowlarr processing.", - "queueEvents": "", - "requirements": [ - "This script requires Python 3.8+ to be installed on your system." - ], -"description": [ - "RemoveSamples intelligently detects and removes sample files and directories", - "using advanced pattern matching and configurable size thresholds.", - "", - "🎯 SETUP:", - "1. Settings → Categories → [Your Category] → ExtensionScripts → Add 'RemoveSamples'", - "2. Place RemoveSamples AFTER unpack but BEFORE media manager processing" - ], - "options": [ + "main": "main.py", + "name": "RemoveSamples", + "homepage": "https://github.com/Anunnaki-Astronaut/RemoveSamples-NZBGet", + "kind": "POST-PROCESSING", + "displayName": "Remove Samples", + "version": "1.1.0", + "nzbgetMinVersion": "23", + "author": "Anunnaki-Astronaut", + "license": "GPL-2.0-only", + "about": "Detects and removes sample files/folders; includes an optional Test Mode and Quarantine.", + "queueEvents": "", + "requirements": [ + "This script requires Python 3.8+ to be installed on your system." + ], + "description": [ + "Remove Samples identifies and removes sample files and directories using name/size heuristics and configurable thresholds.", + "SETUP: Settings → Categories → [Your Category] → ExtensionScripts → Add 'RemoveSamples'. Place it AFTER unpack and BEFORE your media manager." + ], + "options": [ + { + "name": "RemoveDirectories", + "displayName": "Remove Directories", + "value": "Yes", + "description": [ + "Delete entire directories with sample patterns (samples/, SAMPLE/, etc.).", + "Recommended: Yes — removes complete sample folder structures." + ], + "select": ["Yes", "No"] + }, + { + "name": "RemoveFiles", + "displayName": "Remove Files", + "value": "Yes", + "description": [ + "Delete individual files containing sample patterns in filename or by size threshold.", + "Recommended: Yes — removes files like movie.sample.mkv." + ], + "select": ["Yes", "No"] + }, + { + "name": "TestMode", + "displayName": "Test Mode", + "value": "No", + "description": [ + "Safe preview. Shows what would be removed; makes no changes. Use while tuning, then turn off.", + "Temporarily set thresholds (e.g., Video Size Threshold) so tests catch likely samples.", + "See results in NZBGet → Messages and in the NZB’s History log." + ], + "select": ["Yes", "No"] + }, + { + "name": "BlockImportDuringTest", + "displayName": "Block Import (Test Mode)", + "value": "No", + "description": [ + "Use with Test Mode to prevent your media manager (e.g., Sonarr/Radarr) from importing this download.", + "No files are deleted; this helps keep test runs out of your library.", + "Depending on manager settings, the release may be marked failed/blacklisted.", + "Recommended: Enable only for specific test cases, then turn off." + ], + "select": ["Yes", "No"] + }, + { + "name": "Debug", + "displayName": "Debug", + "value": "No", + "description": [ + "Advanced troubleshooting: prints per-item decision lines and configuration details.", + "Not required for Test Mode (Test Mode already prints a summary to Messages/History).", + "Enable only during setup or when investigating issues; leave Off for normal use." + ], + "select": ["Yes", "No"] + }, + { + "name": "VideoSizeThresholdMB", + "displayName": "Video Size Threshold (MB)", + "value": 150, + "description": [ + "Maximum size (MB) for video files to be considered samples (only if no explicit name match).", + "Common presets: 50 (480p), 100 (720p), 150 (1080p), 300 (4K).", + "Higher values = more aggressive detection (range: 1–1000)." + ], + "select": [1, 1000] + }, + { + "name": "RelativePercent", + "displayName": "Relative Size %", + "value": 8, + "description": [ + "Treats a video as a sample if it is ≤ this percent of the largest video in the download.", + "Example: 8% of a 10 GB movie is 800 MB; 8% of a 2 GB episode is 160 MB.", + "Default is 8%. Set to 0 to disable relative size checks (absolute MB thresholds still apply)." + ], + "select": [0, 100] + }, + { + "name": "VideoExts", + "displayName": "Video Extensions", + "value": ".mkv,.mp4,.avi,.mov,.wmv,.flv,.webm,.ts,.m4v,.vob,.mpg,.mpeg,.iso", + "description": [ + "Comma-separated video extensions for size-based detection.", + "Default covers most common formats. Add rare formats if needed." + ], + "select": [] + }, + { + "name": "AudioSizeThresholdMB", + "displayName": "Audio Size Threshold (MB)", + "value": 2, + "description": [ + "Maximum size (MB) for audio files to be considered samples (only if no explicit name match).", + "Common presets: 1 (short clips), 2 (≈30s @ 320kbps), 4 (≈1min), 8 (≈2min).", + "Set to 0 to disable audio size detection (range: 0–100)." + ], + "select": [0, 100] + }, + { + "name": "AudioExts", + "displayName": "Audio Extensions", + "value": ".mp3,.flac,.aac,.ogg,.wma,.m4a,.opus,.wav,.alac,.ape", + "description": [ + "Comma-separated audio extensions for size-based detection.", + "Covers lossless (FLAC, WAV) and compressed (MP3, AAC) formats." + ], + "select": [] + }, { - "name": "RemoveDirectories", - "displayName": "Remove Directories", - "value": "Yes", - "description": [ - "Delete entire directories with sample patterns (samples/, SAMPLE/, etc.)", - "Recommended: Yes - removes complete sample folder structures" - ], - "select": ["Yes", "No"] - }, - { - "name": "RemoveFiles", - "displayName": "Remove Files", - "value": "Yes", - "description": [ - "Delete individual files containing sample patterns in filename", - "Recommended: Yes - removes files like movie.sample.mkv" - ], - "select": ["Yes", "No"] - }, -{ - "name": "Debug", - "displayName": "Debug", - "value": "No", - "description": [ - "Enable detailed logging for troubleshooting and configuration testing", - "Set to Yes only during setup or when investigating issues", - "Testing Guide: https://github.com/Anunnaki-Astronaut/RemoveSamples-NZBGet/wiki/04_Testing_Guide" - ], - "select": ["Yes", "No"] - }, -{ - "name": "VideoSizeThresholdMB", - "displayName": "Video Size Threshold (MB)", - "value": 150, -"description": [ - "Maximum size (MB) for video files to be considered samples", - "Common presets: 50 (480p samples), 100 (720p samples), 150 (1080p samples), 300 (4K samples)", - "Higher values = more aggressive detection (range: 1-1000)" - ], -"select": [1, 1000] - }, - { - "name": "VideoExts", - "displayName": "Video Extensions", - "value": ".mkv,.mp4,.avi,.mov,.wmv,.flv,.webm,.ts,.m4v,.vob,.mpg,.mpeg,.iso", - "description": [ - "Comma-separated video file extensions for size-based detection", - "Default covers most common formats. Add rare formats if needed." - ], - "select": [] - }, -{ - "name": "AudioSizeThresholdMB", - "displayName": "Audio Size Threshold (MB)", - "value": 2, -"description": [ - "Maximum size (MB) for audio files to be considered samples", - "Common presets: 1 (short clips), 2 (30s @ 320kbps), 4 (1min @ 320kbps), 8 (2min samples)", - "Set to 0 to disable audio size detection (range: 0-100)" - ], -"select": [0, 100] - }, - { - "name": "AudioExts", - "displayName": "Audio Extensions", - "value": ".mp3,.flac,.aac,.ogg,.wma,.m4a,.opus,.wav,.alac,.ape", - "description": [ - "Comma-separated audio file extensions for size-based detection", - "Covers lossless (FLAC, WAV) and compressed (MP3, AAC) formats" - ], - "select": [] - } - ], - "commands": [], - "taskTime": "" -} + "name": "ProtectedPaths", + "displayName": "Protected Names/Paths", + "value": "poster.jpg,*/subs/*,*.srt", + "description": [ + "Never remove items matching these names or glob patterns (comma-separated).", + "Example default: poster.jpg, */subs/*, *.srt", + "Heads-up: Protected paths always win over Deny Patterns and other removal logic." + ], + "select": [] + }, + { + "name": "DenyPatterns", + "displayName": "Deny Patterns", + "value": "*sample*.jpg,proof*.txt", + "description": [ + "Extra glob patterns to mark files for removal (comma-separated).", + "Example default: *sample*.jpg, proof*.txt", + "Heads-up: Protected Paths will still prevent removal if an item matches both." + ], + "select": [] + }, + { + "name": "ImageSamples", + "displayName": "Image Samples", + "value": "No", + "description": [ + "Also remove image files that look like samples (e.g., sample.jpg).", + "Default: No — enable after thresholds are tuned.", + "Heads-up: Avoid broad Protected rules like *.jpg; Protected always wins." + ], + "select": ["Yes", "No"] + }, + { + "name": "JunkExtras", + "displayName": "Junk Extras", + "value": "No", + "description": [ + "Remove common release clutter using a built-in, conservative list.", + "Currently targets: *.url, *.webloc, and readme.txt.", + "Enable to clean up these common extra files." + ], + "select": ["Yes", "No"] + }, + { + "name": "CategoryThresholds", + "displayName": "Category Thresholds", + "value": "", + "description": [ + "Per-category override for Relative Size %, in `category=percent` pairs (e.g., `movies=8,tv=12,anime=15`).", + "If blank, all categories use the global Relative Size % (default 8%).", + "Unknown categories also fall back to the global default.", + "Most users should leave this empty." + ], + "select": [] + }, + { + "name": "QuarantineMode", + "displayName": "Quarantine Mode", + "value": "No", + "description": [ + "Move candidates into '_samples_quarantine' inside the NZB folder instead of deleting.", + "Good training wheels for live runs; review then delete manually or via Max Age.", + "Heads-up: Ignored while Test Mode is ON." + ], + "select": ["Yes", "No"] + }, + { + "name": "QuarantineMaxAgeDays", + "displayName": "Quarantine Max Age (days)", + "value": 7, + "description": [ + "If > 0, purge files in '_samples_quarantine' older than N days on each run.", + "Set to 0 to disable auto-purge." + ], + "select": [0, 365] + } + ], + "commands": [], + "taskTime": "" +} \ No newline at end of file diff --git a/prototypes/main_v110_prototype.py b/prototypes/main_v110_prototype.py new file mode 100644 index 0000000..a448349 --- /dev/null +++ b/prototypes/main_v110_prototype.py @@ -0,0 +1,396 @@ +#!/usr/bin/env python3 +""" +RemoveSamples NZBGet extension (v23+) +- Contract constants preserved +- Implements new optional features that match your manifest.json option names +""" + +import os, re, sys, shutil, fnmatch, time +from pathlib import Path + +# === BEGIN_NZBGET_V23_CONTRACT === +# Exit constants (DO NOT RENAME OR CHANGE VALUES) +POSTPROCESS_SUCCESS = 93 +POSTPROCESS_ERROR = 94 +POSTPROCESS_NONE = 95 + +# Required NZBGet envs +# - NZBPP_DIRECTORY +# - NZBPP_STATUS (fallback to NZBPP_TOTALSTATUS) +# - NZBPP_NZBNAME (optional) +# +# Required options +# - NZBPO_REMOVEDIRECTORIES +# - NZBPO_REMOVEFILES +# - NZBPO_DEBUG +# - NZBPO_VIDEOSIZETHRESHOLDMB +# - NZBPO_VIDEOEXTS +# - NZBPO_AUDIOSIZETHRESHOLDMB +# - NZBPO_AUDIOEXTS +# +# Allowed optional options +# - NZBPO_TESTMODE +# - NZBPO_BLOCKIMPORTDURINGTEST +# - NZBPO_RELATIVEPERCENT +# - NZBPO_PROTECTEDPATHS +# - NZBPO_DENYPATTERNS +# - NZBPO_IMAGESAMPLES +# - NZBPO_JUNKEXTRAS +# - NZBPO_CATEGORYTHRESHOLDS +# - NZBPO_QUARANTINEMODE +# - NZBPO_QUARANTINEMAXAGEDAYS +# === END_NZBGET_V23_CONTRACT === + +# Force UTF-8 console on Windows to avoid UnicodeEncodeError in debug logs +def _enable_utf8_windows(): + if os.name == "nt": + try: + sys.stdout.reconfigure(encoding="utf-8", errors="replace") + sys.stderr.reconfigure(encoding="utf-8", errors="replace") + except Exception: + pass +_enable_utf8_windows() + +# ---------- Helpers ---------- +def _env_bool(key, default=False): + val = os.environ.get(key) + if val is None: + return default + return str(val).strip().lower() in ("1","true","yes","on","y","t","enabled","enable") + +def _env_int(key, default=0): + try: + return int(str(os.environ.get(key, str(default))).strip()) + except Exception: + return default + +def _env_str(key, default=""): + return str(os.environ.get(key, default)).strip() + +def _parse_exts(s): + if not s: + return set() + out = set() + for t in re.split(r"[,\s;]+", s): + if not t: + continue + t = t.lower() + if not t.startswith("."): + t = "." + t + out.add(t) + return out + +def _csv_list(s): + if not s: + return [] + return [x.strip() for x in re.split(r"[,\n;]+", s) if x.strip()] + +def info(msg): print(f"[INFO] {msg}", flush=True) +def error(msg): print(f"[ERROR] {msg}", flush=True) +def debug_enabled(): return _env_bool("NZBPO_DEBUG", False) +def debug(msg): + if debug_enabled(): + print(f"[DEBUG] {msg}", flush=True) + +def _size_mb(p: Path) -> float: + try: + return p.stat().st_size / (1024 * 1024) + except Exception: + return 0.0 + +def _matches_any(base: Path, p: Path, patterns): + if not patterns: + return False + rel = str(p.resolve().relative_to(base)).replace("\\", "/") + name = p.name + for pat in patterns: + if fnmatch.fnmatch(name, pat) or fnmatch.fnmatch(rel, pat): + return True + return False + +def _parse_cat_overrides(s: str): + out = {} + for part in [t.strip() for t in s.split(",") if t.strip()]: + if "=" in part: + k, v = part.split("=", 1) + try: + out[k.strip().lower()] = int(v.strip()) + except: + pass + return out + +# ---------- Inputs ---------- +DL_STATUS = os.environ.get("NZBPP_STATUS", "") or os.environ.get("NZBPP_TOTALSTATUS", "") +DEST_DIR = Path(_env_str("NZBPP_DIRECTORY", ".")).resolve() +NZB_NAME = _env_str("NZBPP_NZBNAME", "") +CATEGORY = _env_str("NZBPP_CATEGORY", "") + +REMOVE_DIRS = _env_bool("NZBPO_REMOVEDIRECTORIES", False) +REMOVE_FILES = _env_bool("NZBPO_REMOVEFILES", False) +VIDEO_MIN_MB = _env_int("NZBPO_VIDEOSIZETHRESHOLDMB", 200) +AUDIO_MIN_MB = _env_int("NZBPO_AUDIOSIZETHRESHOLDMB", 10) +VIDEO_EXTS = _parse_exts(_env_str("NZBPO_VIDEOEXTS", ".mkv,.mp4,.avi,.mov,.ts,.m4v,.wmv")) +AUDIO_EXTS = _parse_exts(_env_str("NZBPO_AUDIOEXTS", ".mp3,.flac,.aac,.m4a,.ogg,.opus,.wav")) + +TEST_MODE = _env_bool("NZBPO_TESTMODE", False) +BLOCK_IMPORT_DURING_TEST = _env_bool("NZBPO_BLOCKIMPORTDURINGTEST", False) + +# New optional features, mapped to your manifest option names +RELATIVE_PERCENT = _env_int("NZBPO_RELATIVEPERCENT", -1) # -1 disables relative check +PROTECTED_PATHS = _csv_list(_env_str("NZBPO_PROTECTEDPATHS", "")) +DENY_PATTERNS = _csv_list(_env_str("NZBPO_DENYPATTERNS", "")) +IMAGE_SAMPLES = _env_bool("NZBPO_IMAGESAMPLES", False) +JUNK_EXTRAS = _env_bool("NZBPO_JUNKEXTRAS", False) +CATEGORY_THRESH = _env_str("NZBPO_CATEGORYTHRESHOLDS", "") +QUARANTINE_MODE = _env_bool("NZBPO_QUARANTINEMODE", False) +QUARANTINE_MAX_AGE_DAYS = _env_int("NZBPO_QUARANTINEMAXAGEDAYS", 0) + +# Name patterns for "sample" +SAMPLE_NAME_RE_FILE = re.compile(r"(?i)(?:^|[\\/_\.\-\s])sample(?:s)?(?:$|[\\/_\.\-\s])") +SAMPLE_NAME_RE_DIR = re.compile(r"(?i)(?:^|[\\/_\.\-\s])samples?(?:$|[\\/_\.\-\s])") + +# ---------- Early sanity ---------- +if not DEST_DIR.exists(): + error(f"Destination directory not found: {DEST_DIR}") + sys.exit(POSTPROCESS_ERROR) + +if DL_STATUS and DL_STATUS.upper() != "SUCCESS": + info(f"Status {DL_STATUS}; skipping.") + info("Summary: 0 removed (status not SUCCESS). Mode: " + ("TEST" if TEST_MODE else "LIVE")) + sys.exit(POSTPROCESS_NONE) + +info(f'Processing "{NZB_NAME}" in {DEST_DIR}') + +# Category override for relative percent +EFF_RELATIVE_PERCENT = RELATIVE_PERCENT +cat_map = _parse_cat_overrides(CATEGORY_THRESH) +if CATEGORY and cat_map and CATEGORY.lower() in cat_map: + EFF_RELATIVE_PERCENT = cat_map[CATEGORY.lower()] + info(f"Category override: {CATEGORY} → Relative Size % = {EFF_RELATIVE_PERCENT}") + +# Heads-up notes +if TEST_MODE and QUARANTINE_MODE: + info("Heads-up: Quarantine is ignored while Test Mode is ON.") +if EFF_RELATIVE_PERCENT >= 0 and VIDEO_MIN_MB >= 400 and EFF_RELATIVE_PERCENT <= 5: + info("Heads-up: Very high video threshold with very low relative percent may skip everything.") +if BLOCK_IMPORT_DURING_TEST and not TEST_MODE: + info("Reminder: BlockImportDuringTest is designed for Test Mode.") + +# ---------- Scan ---------- +files_considered = 0 +dirs_considered = 0 +file_candidates = [] +dir_candidates = [] +errors = 0 + +def is_video(p: Path) -> bool: + return p.suffix.lower() in VIDEO_EXTS + +def is_audio(p: Path) -> bool: + return p.suffix.lower() in AUDIO_EXTS + +def is_image(p: Path) -> bool: + return p.suffix.lower() in {".jpg",".jpeg",".png",".bmp",".gif",".webp"} + +# Gather lists +all_files = [] +all_dirs = [] +for p in DEST_DIR.rglob("*"): + try: + if p.is_symlink(): + debug(f"Skip symlink: {p}") + continue + if p.is_file(): + all_files.append(p) + elif p.is_dir(): + all_dirs.append(p) + except Exception as ex: + errors += 1 + error(f"Scan error at {p}: {ex}") + +files_considered = len(all_files) +dirs_considered = len(all_dirs) + +# Largest video size for relative percent logic +largest_video_bytes = 0 +for f in all_files: + if is_video(f): + try: + s = f.stat().st_size + if s > largest_video_bytes: + largest_video_bytes = s + except Exception: + pass + +# Directories by name pattern +for d in all_dirs: + try: + if SAMPLE_NAME_RE_DIR.search(d.name): + if _matches_any(DEST_DIR, d, PROTECTED_PATHS): + debug(f"Protected directory: {d}") + else: + dir_candidates.append(d) + debug(f"Candidate dir: {d}") + except Exception as ex: + errors += 1 + error(f"Dir scan error at {d}: {ex}") + +# Files by name, size, optional extras +for f in all_files: + try: + if _matches_any(DEST_DIR, f, PROTECTED_PATHS): + debug(f"Protected file: {f}") + continue + + mb = _size_mb(f) + name_hit = SAMPLE_NAME_RE_FILE.search(f.name) is not None + small_video = is_video(f) and (mb < float(VIDEO_MIN_MB)) + small_audio = is_audio(f) and (mb < float(AUDIO_MIN_MB)) + + rel_hit = False + if is_video(f) and EFF_RELATIVE_PERCENT >= 0 and largest_video_bytes > 0: + try: + pct = int(round((f.stat().st_size / largest_video_bytes) * 100)) + rel_hit = (pct <= EFF_RELATIVE_PERCENT) + except Exception: + rel_hit = False + + deny_hit = _matches_any(DEST_DIR, f, DENY_PATTERNS) + + image_hit = False + if IMAGE_SAMPLES and is_image(f): + image_hit = ("sample" in f.name.lower()) or deny_hit + + junk_hit = False + if JUNK_EXTRAS: + junk_hit = deny_hit or f.suffix.lower() in {".url",".webloc"} or f.name.lower().endswith("readme.txt") + + if name_hit or small_video or small_audio or rel_hit or image_hit or junk_hit or deny_hit: + file_candidates.append((f, mb, name_hit, small_video, small_audio, rel_hit, image_hit, junk_hit, deny_hit)) + debug(f"Candidate file: {f} | {mb:.1f} MB | name={name_hit} smallV={small_video} smallA={small_audio} rel%={rel_hit} img={image_hit} junk={junk_hit} deny={deny_hit}") + + except Exception as ex: + errors += 1 + error(f"File scan error at {f}: {ex}") + +# Optional block during test: return 94 early if candidates found +if TEST_MODE and BLOCK_IMPORT_DURING_TEST and (file_candidates or dir_candidates): + info("BlockImportDuringTest=ON with candidates → reporting 94 to prevent import (no deletions performed).") + info("Summary: 0 removed (blocked during Test). Mode: TEST") + sys.exit(POSTPROCESS_ERROR) + +# ---------- Act ---------- +removed_files = 0 +removed_dirs = 0 +removed_mb_total = 0.0 + +QUAR_DIR = DEST_DIR / "_samples_quarantine" + +def _safe_move(src: Path): + dst = QUAR_DIR / src.resolve().relative_to(DEST_DIR) + dst.parent.mkdir(parents=True, exist_ok=True) + shutil.move(str(src), str(dst)) + +# Directories +if REMOVE_DIRS: + for d in dir_candidates: + rel = d.resolve().relative_to(DEST_DIR) + if TEST_MODE: + info(f"[TEST] Would remove directory: {rel}") + else: + if QUARANTINE_MODE: + # Move contents into quarantine, then remove empty dirs + moved_any = False + for p in d.rglob("*"): + if p.is_file(): + try: + _safe_move(p) + moved_any = True + except Exception as ex: + errors += 1 + error(f"Quarantine move failed {p}: {ex}") + # attempt to remove the now-empty tree + try: + for sub in sorted(d.rglob("*"), reverse=True): + if sub.is_dir(): + try: + sub.rmdir() + except: + pass + d.rmdir() + except: + pass + if moved_any: + removed_dirs += 1 + info(f"[QUARANTINE] Directory contents moved: {rel}") + else: + try: + shutil.rmtree(d) + removed_dirs += 1 + info(f"Removed directory: {rel}") + except Exception as ex: + errors += 1 + error(f"Failed to remove directory: {rel} - {ex}") +else: + debug("Configured to keep directories (NZBPO_REMOVEDIRECTORIES=No)") + +# Files +if REMOVE_FILES: + for f_tuple in file_candidates: + f, mb, *_ = f_tuple + rel = f.resolve().relative_to(DEST_DIR) + if TEST_MODE: + info(f"[TEST] Would remove file: {rel} ({mb:.1f} MB)") + else: + if QUARANTINE_MODE: + try: + QUAR_DIR.mkdir(parents=True, exist_ok=True) + _safe_move(f) + removed_files += 1 + removed_mb_total += mb + info(f"[QUARANTINE] {rel} ({mb:.1f} MB)") + except Exception as ex: + errors += 1 + error(f"Quarantine move failed: {rel} - {ex}") + else: + try: + f.unlink() + removed_files += 1 + removed_mb_total += mb + info(f"Removed file: {rel} ({mb:.1f} MB)") + except Exception as ex: + errors += 1 + error(f"Failed to remove file: {rel} - {ex}") +else: + debug("Configured to keep files (NZBPO_REMOVEFILES=No)") + +mode = "TEST" if TEST_MODE else ("LIVE+QUARANTINE" if QUARANTINE_MODE else "LIVE") +info( + f"Summary: removed {removed_files} files / {removed_dirs} dirs " + f"({removed_mb_total:.1f} MB). Mode: {mode}. " + f"FilesChecked={files_considered} DirsChecked={dirs_considered} " + f"Candidates={len(file_candidates)+len(dir_candidates)} " + f"Rel%={'disabled' if EFF_RELATIVE_PERCENT < 0 else EFF_RELATIVE_PERCENT} " + f"VideoMB>={VIDEO_MIN_MB}" +) + +if errors > 0: + sys.exit(POSTPROCESS_ERROR) + +if removed_files == 0 and removed_dirs == 0 and len(file_candidates) == 0 and len(dir_candidates) == 0: + sys.exit(POSTPROCESS_NONE) + +# Optional quarantine purge after action +if QUARANTINE_MODE and QUARANTINE_MAX_AGE_DAYS > 0 and QUAR_DIR.exists(): + cutoff = time.time() - (QUARANTINE_MAX_AGE_DAYS * 86400) + try: + for p in QUAR_DIR.rglob("*"): + try: + if p.is_file() and p.stat().st_mtime < cutoff: + p.unlink() + except Exception as ex: + error(f"Quarantine purge failed at {p}: {ex}") + except Exception: + pass + +sys.exit(POSTPROCESS_SUCCESS) \ No newline at end of file diff --git a/prototypes/manifest_v110_prototype.json b/prototypes/manifest_v110_prototype.json new file mode 100644 index 0000000..789dcec --- /dev/null +++ b/prototypes/manifest_v110_prototype.json @@ -0,0 +1,209 @@ +{ + "main": "main.py", + "name": "RemoveSamples", + "homepage": "https://github.com/Anunnaki-Astronaut/RemoveSamples-NZBGet", + "kind": "POST-PROCESSING", + "displayName": "Remove Samples", + "version": "1.1.0", + "nzbgetMinVersion": "23", + "author": "Anunnaki-Astronaut", + "license": "GPL-2.0-only", + "about": "Detects and removes sample files/folders; includes an optional Test Mode.", + "queueEvents": "", + "requirements": [ + "This script requires Python 3.8+ to be installed on your system." + ], + "description": [ + "Remove Samples identifies and removes sample files and directories using name/size heuristics and configurable thresholds.", + "SETUP: Settings → Categories → [Your Category] → ExtensionScripts → Add 'RemoveSamples'. Place it AFTER unpack and BEFORE your media manager." + ], + "options": [ + { + "name": "RemoveDirectories", + "displayName": "Remove Directories", + "value": "Yes", + "description": [ + "Delete entire directories with sample patterns (samples/, SAMPLE/, etc.).", + "Recommended: Yes — removes complete sample folder structures." + ], + "select": ["Yes", "No"] + }, + { + "name": "RemoveFiles", + "displayName": "Remove Files", + "value": "Yes", + "description": [ + "Delete individual files containing sample patterns in filename or by size threshold.", + "Recommended: Yes — removes files like movie.sample.mkv." + ], + "select": ["Yes", "No"] + }, + { + "name": "TestMode", + "displayName": "Test Mode", + "value": "No", + "description": [ + "Safe preview. Shows what would be removed; makes no changes. Use while tuning, then turn off.", + "Temporarily set thresholds (e.g., Video Size Threshold) so tests catch likely samples.", + "See results in NZBGet → Messages and in the NZB’s History log.", + "Heads-up: Avoid combining with… Debug (On) — can produce very verbose logs." + ], + "select": ["Yes", "No"] + }, + { + "name": "BlockImportDuringTest", + "displayName": "Block Import", + "value": "No", + "description": [ + "Use with Test Mode to prevent your media manager (e.g., Sonarr/Radarr) from importing this download.", + "No files are deleted; this helps keep test runs out of your library.", + "Depending on manager settings, the release may be marked failed/blacklisted.", + "Recommended: Enable only for specific test cases, then turn off." + ], + "select": ["Yes", "No"] + }, + { + "name": "Debug", + "displayName": "Debug", + "value": "No", + "description": [ + "Advanced troubleshooting: prints per-item decision lines and configuration details.", + "Not required for Test Mode (Test Mode already prints a summary to Messages/History).", + "Enable only during setup or when investigating issues; leave Off for normal use.", + "Heads-up: Avoid combining with… Test Mode (On) for routine use — can produce very verbose logs." + ], + "select": ["Yes", "No"] + }, + { + "name": "VideoSizeThresholdMB", + "displayName": "Video Size Threshold (MB)", + "value": 150, + "description": [ + "Maximum size (MB) for video files to be considered samples (only if no explicit name match).", + "Common presets: 50 (480p), 100 (720p), 150 (1080p), 300 (4K).", + "Higher values = more aggressive detection (range: 1–1000)." + ], + "select": [1, 1000] + }, + { + "name": "VideoExts", + "displayName": "Video Extensions", + "value": ".mkv,.mp4,.avi,.mov,.wmv,.flv,.webm,.ts,.m4v,.vob,.mpg,.mpeg,.iso", + "description": [ + "Comma-separated video extensions for size-based detection.", + "Default covers most common formats. Add rare formats if needed." + ], + "select": [] + }, + { + "name": "AudioSizeThresholdMB", + "displayName": "Audio Size Threshold (MB)", + "value": 2, + "description": [ + "Maximum size (MB) for audio files to be considered samples (only if no explicit name match).", + "Common presets: 1 (short clips), 2 (≈30s @ 320kbps), 4 (≈1min), 8 (≈2min).", + "Set to 0 to disable audio size detection (range: 0–100)." + ], + "select": [0, 100] + }, + { + "name": "AudioExts", + "displayName": "Audio Extensions", + "value": ".mp3,.flac,.aac,.ogg,.wma,.m4a,.opus,.wav,.alac,.ape", + "description": [ + "Comma-separated audio extensions for size-based detection.", + "Covers lossless (FLAC, WAV) and compressed (MP3, AAC) formats." + ], + "select": [] + }, + { + "name": "RelativePercent", + "displayName": "Relative Size %", + "value": 8, + "description": [ + "Treat a file as a sample if it is ≤ this percent of the largest video in the NZB folder.", + "Lower = stricter (catches more); Higher = keeps more extras. Start around 8%.", + "Heads-up: Avoid over-strict combos with very high 'Video Size Threshold (MB)'." + ], + "select": [0, 100] + }, + { + "name": "ProtectedPaths", + "displayName": "Protected Names/Paths", + "value": "poster.jpg,*/subs/*,*.srt", + "description": [ + "Never remove items matching these names or globs (comma-separated).", + "Examples: poster.jpg, */subs/*, *.srt", + "Heads-up: Protected wins over Image Samples / Junk Extras / Deny Patterns." + ], + "select": [] + }, + { + "name": "DenyPatterns", + "displayName": "Deny Patterns", + "value": "*sample*.jpg,proof*.txt", + "description": [ + "Extra patterns eligible for removal when matched (comma-separated).", + "Examples: *sample*.jpg, proof*.txt", + "Heads-up: Still blocked by Protected if both match." + ], + "select": [] + }, + { + "name": "ImageSamples", + "displayName": "Image Samples", + "value": "No", + "description": [ + "Also remove image files that look like samples (e.g., sample.jpg).", + "Default: No — enable after thresholds are tuned.", + "Heads-up: Avoid broad Protected rules like *.jpg; Protected always wins." + ], + "select": ["Yes", "No"] + }, + { + "name": "JunkExtras", + "displayName": "Junk Extras", + "value": "No", + "description": [ + "Remove common release clutter (e.g., .url, spam .txt). Conservative list.", + "Default: No — enable after thresholds are tuned.", + "Heads-up: Protected rules covering these files will prevent removal." + ], + "select": ["Yes", "No"] + }, + { + "name": "CategoryThresholds", + "displayName": "Category Thresholds", + "value": "", + "description": [ + "Per-category Relative % overrides (key=value, comma-separated), e.g.: movies=8,tv=10,anime=14", + "Unknown categories fall back to the global Relative Size %.", + "Keeps a single install behaving correctly across Movies/TV/Anime." + ], + "select": [] + }, + { + "name": "QuarantineMode", + "displayName": "Quarantine Mode", + "value": "No", + "description": [ + "Move candidates into '_samples_quarantine' inside the NZB folder instead of deleting.", + "Good training wheels for live runs; review then delete manually or via Max Age.", + "Heads-up: Ignored while Test Mode is ON." + ], + "select": ["Yes", "No"] + }, + { + "name": "QuarantineMaxAgeDays", + "displayName": "Quarantine Max Age (days)", + "value": 7, + "description": [ + "If > 0, purge files in '_samples_quarantine' older than N days on each run.", + "Set to 0 to disable auto-purge." + ], + "select": [0, 365] + } + ], + "commands": [], + "taskTime": "" +} \ No newline at end of file diff --git a/tests.py b/tests.py index 403e404..846351a 100644 --- a/tests.py +++ b/tests.py @@ -1,158 +1,171 @@ #!/usr/bin/env python3 # -# Tests for RemoveSamples Extension +# Tests for RemoveSamples Extension (v1.1.0 semantics) # -# Copyright (C) 2025 Anunnaki-Astronaut -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. +# - Uses POSTPROCESS_SUCCESS (93) for runs that actually delete something +# - Uses POSTPROCESS_NONE (95) for runs that do no destructive work # +import os import sys -import unittest import shutil -import os -import subprocess -import pathlib import tempfile -from os.path import dirname - -ROOT_DIR = dirname(__file__) +import subprocess +import unittest +from pathlib import Path POSTPROCESS_SUCCESS = 93 POSTPROCESS_ERROR = 94 POSTPROCESS_NONE = 95 -def get_python(): - if os.name == "nt": - return "python" - return "python3" +ROOT_DIR = Path(__file__).resolve().parent +SCRIPT_PATH = ROOT_DIR / "main.py" -def set_defaults(test_dir): - """Set default environment variables for testing""" + +def set_defaults(test_dir: str) -> None: + """Set default NZBGet environment variables for tests.""" + # Core NZBGet runtime envs + os.environ["NZBPP_DIRECTORY"] = test_dir + os.environ["NZBPP_STATUS"] = "SUCCESS" + os.environ["NZBPP_NZBNAME"] = "Test-NZB" + + # Required options (mirror manifest defaults) os.environ["NZBPO_REMOVEDIRECTORIES"] = "Yes" os.environ["NZBPO_REMOVEFILES"] = "Yes" os.environ["NZBPO_DEBUG"] = "No" os.environ["NZBPO_VIDEOSIZETHRESHOLDMB"] = "150" - os.environ["NZBPO_VIDEOEXTS"] = ".mkv,.mp4,.avi,.mov,.wmv,.flv,.webm,.ts,.m4v,.vob" + os.environ["NZBPO_VIDEOEXTS"] = ( + ".mkv,.mp4,.avi,.mov,.wmv,.flv,.webm,.ts,.m4v,.vob" + ) os.environ["NZBPO_AUDIOSIZETHRESHOLDMB"] = "2" - os.environ["NZBPO_AUDIOEXTS"] = ".wav,.aiff,.mp3,.flac,.m4a,.ogg,.aac,.alac,.ape,.opus,.wma" - os.environ["NZBPP_DIRECTORY"] = test_dir - os.environ["NZBPP_STATUS"] = "SUCCESS" - os.environ["NZBPP_NZBNAME"] = "Test.Download" + os.environ["NZBPO_AUDIOEXTS"] = ( + ".wav,.aiff,.mp3,.flac,.m4a,.ogg,.aac,.alac,.ape,.opus,.wma" + ) + + # Optional toggles default off + os.environ["NZBPO_TESTMODE"] = "No" + os.environ["NZBPO_BLOCKIMPORTDURINGTEST"] = "No" + def run_script(): - """Run the main.py script and return output, return code, and error""" - sys.stdout.flush() + """Run main.py as a subprocess and capture output and exit code.""" proc = subprocess.Popen( - [get_python(), ROOT_DIR + "/main.py"], + [sys.executable, str(SCRIPT_PATH)], stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=os.environ.copy(), ) out, err = proc.communicate() - ret_code = proc.returncode - return (out.decode(), int(ret_code), err.decode()) + return out.decode("utf-8"), int(proc.returncode), err.decode("utf-8") + class TestRemoveSamples(unittest.TestCase): - - def setUp(self): - """Set up test environment""" - self.test_dir = tempfile.mkdtemp() + def setUp(self) -> None: + self.test_dir = tempfile.mkdtemp(prefix="rs_tests_") set_defaults(self.test_dir) - - def tearDown(self): - """Clean up test environment""" + + def tearDown(self) -> None: shutil.rmtree(self.test_dir, ignore_errors=True) - - def test_script_success(self): - """Test that script runs successfully with default settings""" - [output, code, error] = run_script() - self.assertEqual(code, POSTPROCESS_SUCCESS) + + # ---- Basic control-flow -------------------------------------------- + + def test_script_success_no_work_done(self): + """Empty dir with defaults should run and exit with POSTPROCESS_NONE.""" + output, code, error = run_script() + self.assertEqual(code, POSTPROCESS_NONE) self.assertIn("RemoveSamples extension started", output) - self.assertIn("RemoveSamples extension completed successfully", output) - + # v1.1.0 uses a summary line instead of "completed successfully" + self.assertIn("Summary: removed 0 files / 0 dirs", output) + def test_missing_directory(self): - """Test script behavior when download directory doesn't exist""" + """Missing NZBPP_DIRECTORY should be handled gracefully.""" os.environ["NZBPP_DIRECTORY"] = "/nonexistent/directory" - [output, code, error] = run_script() + output, code, error = run_script() + # Current behavior: treat as NONE and log an error line. self.assertEqual(code, POSTPROCESS_NONE) - self.assertIn("doesn't exist", output) - - def test_sample_file_detection(self): - """Test that sample files are properly detected""" - # Create test files - test_file = pathlib.Path(self.test_dir) / "sample.mkv" - test_file.write_text("test content") - - [output, code, error] = run_script() - self.assertEqual(code, POSTPROCESS_SUCCESS) - self.assertFalse(test_file.exists()) # Should be removed - + self.assertIn("Destination directory not found", output) + + def test_failed_status_skip(self): + """If NZBPP_STATUS != SUCCESS, script should skip processing.""" + os.environ["NZBPP_STATUS"] = "FAILURE" + output, code, error = run_script() + self.assertEqual(code, POSTPROCESS_NONE) + self.assertIn("skipping", output.lower()) + + # ---- Sample detection ---------------------------------------------- + def test_sample_directory_detection(self): - """Test that sample directories are properly detected""" - # Create test directory - sample_dir = pathlib.Path(self.test_dir) / "samples" + """Directories with 'sample' in the name should be removed.""" + sample_dir = Path(self.test_dir) / "Sample" sample_dir.mkdir() - (sample_dir / "test.txt").write_text("test content") - - [output, code, error] = run_script() + (sample_dir / "test.txt").write_text("content", encoding="utf-8") + + output, code, error = run_script() + self.assertEqual(code, POSTPROCESS_SUCCESS) + self.assertFalse(sample_dir.exists()) + + def test_sample_file_detection(self): + """Files with sample pattern in filename should be removed.""" + sample_file = Path(self.test_dir) / "movie.sample.mkv" + sample_file.write_text("content", encoding="utf-8") + + output, code, error = run_script() self.assertEqual(code, POSTPROCESS_SUCCESS) - self.assertFalse(sample_dir.exists()) # Should be removed - + self.assertFalse(sample_file.exists()) + + def test_small_audio_file_detection(self): + """Very small audio files under threshold should be treated as samples.""" + small_audio = Path(self.test_dir) / "track01.mp3" + small_audio.write_bytes(b"x" * 1024) # 1KB < 2MB threshold + + output, code, error = run_script() + self.assertEqual(code, POSTPROCESS_SUCCESS) + self.assertFalse(small_audio.exists()) + def test_small_video_file_detection(self): - """Test that small video files are detected as samples""" - # Create small video file (under threshold) - small_video = pathlib.Path(self.test_dir) / "movie.mkv" - small_video.write_bytes(b"x" * 1024) # 1KB file, well under 150MB threshold - - [output, code, error] = run_script() + """Very small video files under threshold should be treated as samples.""" + small_video = Path(self.test_dir) / "movie.mkv" + small_video.write_bytes(b"x" * 1024) # 1KB < 150MB threshold + + output, code, error = run_script() self.assertEqual(code, POSTPROCESS_SUCCESS) - self.assertFalse(small_video.exists()) # Should be removed as sample - + self.assertFalse(small_video.exists()) + + # ---- Non-sample and disabled behavior ------------------------------ + def test_normal_files_preserved(self): - """Test that normal files are not removed""" - # Create normal file - normal_file = pathlib.Path(self.test_dir) / "movie.mkv" - normal_file.write_bytes(b"x" * (200 * 1024 * 1024)) # 200MB file, over threshold - - [output, code, error] = run_script() - self.assertEqual(code, POSTPROCESS_SUCCESS) - self.assertTrue(normal_file.exists()) # Should be preserved - + """Large non-sample videos should not be removed.""" + normal_file = Path(self.test_dir) / "movie.mkv" + normal_file.write_bytes(b"x" * (200 * 1024 * 1024)) # 200MB > 150MB + + output, code, error = run_script() + # Nothing removed -> NONE + self.assertEqual(code, POSTPROCESS_NONE) + self.assertTrue(normal_file.exists()) + def test_disabled_file_removal(self): - """Test that file removal can be disabled""" + """When REMOVEFILES is No, even obvious sample files are preserved.""" os.environ["NZBPO_REMOVEFILES"] = "No" - - # Create sample file - sample_file = pathlib.Path(self.test_dir) / "sample.mkv" - sample_file.write_text("test content") - - [output, code, error] = run_script() - self.assertEqual(code, POSTPROCESS_SUCCESS) - self.assertTrue(sample_file.exists()) # Should be preserved when disabled - + + sample_file = Path(self.test_dir) / "movie.sample.mkv" + sample_file.write_text("content", encoding="utf-8") + + output, code, error = run_script() + self.assertEqual(code, POSTPROCESS_NONE) + self.assertTrue(sample_file.exists()) + def test_disabled_directory_removal(self): - """Test that directory removal can be disabled""" + """When REMOVEDIRECTORIES is No, sample directories are preserved.""" os.environ["NZBPO_REMOVEDIRECTORIES"] = "No" - - # Create sample directory - sample_dir = pathlib.Path(self.test_dir) / "samples" + + sample_dir = Path(self.test_dir) / "Sample" sample_dir.mkdir() - - [output, code, error] = run_script() - self.assertEqual(code, POSTPROCESS_SUCCESS) - self.assertTrue(sample_dir.exists()) # Should be preserved when disabled - - def test_failed_status_skip(self): - """Test that script skips processing when status is not SUCCESS""" - os.environ["NZBPP_STATUS"] = "FAILURE" - - [output, code, error] = run_script() + (sample_dir / "test.txt").write_text("content", encoding="utf-8") + + output, code, error = run_script() self.assertEqual(code, POSTPROCESS_NONE) - self.assertIn("skipping", output) + self.assertTrue(sample_dir.exists()) + if __name__ == "__main__": - unittest.main() \ No newline at end of file + unittest.main()