diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..bbb7ebe --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,94 @@ +name: CI + +on: + push: + branches: ["main", "master"] + pull_request: + branches: ["main", "master"] + +env: + CARGO_TERM_COLOR: always + RUSTFLAGS: "-D warnings" + +permissions: + contents: read + +jobs: + # ───────────────────────────────────────────────────────────────────────────── + # Format check + # ───────────────────────────────────────────────────────────────────────────── + fmt: + name: Rustfmt + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Install Rust (stable + rustfmt) + uses: dtolnay/rust-toolchain@stable + with: + components: rustfmt + + - name: Check formatting + run: cargo fmt --all -- --check + + # ───────────────────────────────────────────────────────────────────────────── + # Lint (clippy) + # ───────────────────────────────────────────────────────────────────────────── + clippy: + name: Clippy + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Install Rust (stable + clippy) + uses: dtolnay/rust-toolchain@stable + with: + components: clippy + + - name: Cache Cargo + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: clippy-${{ hashFiles('**/Cargo.lock') }} + restore-keys: clippy- + + - name: Run Clippy + run: cargo clippy --all-targets --all-features + + # ───────────────────────────────────────────────────────────────────────────── + # Build + test on multiple platforms + # ───────────────────────────────────────────────────────────────────────────── + build: + name: Build · ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + + steps: + - uses: actions/checkout@v4 + + - name: Install Rust (stable) + uses: dtolnay/rust-toolchain@stable + + - name: Cache Cargo + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: build-${{ matrix.os }}-${{ hashFiles('**/Cargo.lock') }} + restore-keys: build-${{ matrix.os }}- + + - name: Build + run: cargo build --locked + + - name: Test + run: cargo test --locked diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index eed6164..13bb7bf 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -95,6 +95,7 @@ jobs: mkdir "$STAGING" cp target/${{ matrix.target }}/release/mineru "$STAGING/" cp README.md "$STAGING/" + cp skill/SKILL.md "$STAGING/" tar czf "${STAGING}.tar.gz" "$STAGING" echo "ASSET=${STAGING}.tar.gz" >> "$GITHUB_ENV" @@ -106,6 +107,7 @@ jobs: New-Item -ItemType Directory $staging Copy-Item "target\${{ matrix.target }}\release\mineru.exe" "$staging\" Copy-Item "README.md" "$staging\" + Copy-Item "skill\SKILL.md" "$staging\" Compress-Archive -Path "$staging\*" -DestinationPath "$staging.zip" "ASSET=$staging.zip" | Out-File -FilePath $env:GITHUB_ENV -Append @@ -152,6 +154,7 @@ jobs: dl/x86_64/mineru lipo -info "$STAGING/mineru" cp README.md "$STAGING/" + cp skill/SKILL.md "$STAGING/" tar czf "${STAGING}.tar.gz" "$STAGING" echo "ASSET=${STAGING}.tar.gz" >> "$GITHUB_ENV" diff --git a/README.md b/README.md index 193e8cd..521eaeb 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,10 @@ # mineru-cli +[![CI](https://github.com/neipor/mineru-cli/actions/workflows/ci.yml/badge.svg)](https://github.com/neipor/mineru-cli/actions/workflows/ci.yml) +[![Release](https://github.com/neipor/mineru-cli/actions/workflows/release.yml/badge.svg)](https://github.com/neipor/mineru-cli/actions/workflows/release.yml) +[![Latest Release](https://img.shields.io/github/v/release/neipor/mineru-cli)](https://github.com/neipor/mineru-cli/releases/latest) +[![License: MIT](https://img.shields.io/badge/license-MIT-blue)](LICENSE) + A Rust CLI tool that extracts text from **PDFs, images, and Office documents** using the [MinerU OCR](https://huggingface.co/spaces/opendatalab/MinerU) HuggingFace Space, and outputs **LLM-friendly Markdown** (or JSON / plain text). ``` @@ -22,23 +27,69 @@ echo "$(mineru scan.pdf)" | llm summarize ## Installation -### Prebuilt binaries (recommended) +### One-liner install (macOS / Linux — recommended) -Download from [GitHub Releases](https://github.com/neipor/mineru-cli/releases/latest): +These commands download the binary and place it in `/usr/local/bin/` so it is +immediately available on your `PATH`. -| Platform | File | -|---|---| -| macOS Apple Silicon | `mineru-aarch64-apple-darwin.tar.gz` | -| macOS Intel | `mineru-x86_64-apple-darwin.tar.gz` | -| macOS Universal | `mineru-universal-apple-darwin.tar.gz` | -| Linux x86_64 | `mineru-x86_64-unknown-linux-musl.tar.gz` | -| Linux ARM64 | `mineru-aarch64-unknown-linux-musl.tar.gz` | -| Windows x86_64 | `mineru-x86_64-pc-windows-msvc.zip` | +```bash +# macOS — Universal binary (Intel + Apple Silicon) +curl -fsSL https://github.com/neipor/mineru-cli/releases/latest/download/mineru-universal-apple-darwin.tar.gz \ + | tar -xz --strip-components=1 -C /usr/local/bin/ mineru-*/mineru +chmod +x /usr/local/bin/mineru && mineru --version +``` + +```bash +# Linux x86_64 (static musl binary, no libc dependency) +curl -fsSL https://github.com/neipor/mineru-cli/releases/latest/download/mineru-x86_64-unknown-linux-musl.tar.gz \ + | tar -xz --strip-components=1 -C /usr/local/bin/ mineru-*/mineru +chmod +x /usr/local/bin/mineru && mineru --version +``` ```bash -# macOS Apple Silicon -curl -L https://github.com/neipor/mineru-cli/releases/latest/download/mineru-aarch64-apple-darwin.tar.gz \ - | tar -xz -C /usr/local/bin/ +# Linux ARM64 (Raspberry Pi 4/5, AWS Graviton, Oracle Ampere, …) +curl -fsSL https://github.com/neipor/mineru-cli/releases/latest/download/mineru-aarch64-unknown-linux-musl.tar.gz \ + | tar -xz --strip-components=1 -C /usr/local/bin/ mineru-*/mineru +chmod +x /usr/local/bin/mineru && mineru --version +``` + +> **No root?** Replace `/usr/local/bin/` with `$HOME/.local/bin/` and ensure +> `export PATH="$HOME/.local/bin:$PATH"` is in your `~/.bashrc` / `~/.zshrc`. + +### Windows (PowerShell) + +```powershell +$ver = (Invoke-RestMethod "https://api.github.com/repos/neipor/mineru-cli/releases/latest").tag_name +$url = "https://github.com/neipor/mineru-cli/releases/download/$ver/mineru-$ver-x86_64-pc-windows-msvc.zip" +Invoke-WebRequest -Uri $url -OutFile "$env:TEMP\mineru.zip" +Expand-Archive "$env:TEMP\mineru.zip" -DestinationPath "$env:TEMP\mineru-pkg" -Force + +# Install to %LOCALAPPDATA%\Programs\mineru and add to PATH +$dest = "$env:LOCALAPPDATA\Programs\mineru" +New-Item -ItemType Directory -Force $dest | Out-Null +Copy-Item "$env:TEMP\mineru-pkg\*\mineru.exe" $dest +$cur = [Environment]::GetEnvironmentVariable("Path","User") +if ($cur -notlike "*$dest*") { + [Environment]::SetEnvironmentVariable("Path","$cur;$dest","User") + Write-Host "Restart your terminal to use mineru." +} +& "$dest\mineru.exe" --version +``` + +### cargo install (requires Rust 1.85+) + +```bash +# Install from crates.io — binary lands in ~/.cargo/bin/ (already on PATH) +cargo install mineru-cli + +# Or build from source +git clone https://github.com/neipor/mineru-cli.git +cd mineru-cli +cargo build --release +# System-wide: +sudo cp target/release/mineru /usr/local/bin/ +# User-level (no root): +cp target/release/mineru ~/.local/bin/ ``` ### OpenClaw / ClawHub @@ -49,13 +100,16 @@ openclaw skills install mineru-ocr-cli clawhub install mineru-ocr-cli ``` -### Build from source (requires Rust 1.85+) +### Available binaries -```bash -git clone https://github.com/neipor/mineru-cli.git -cd mineru-cli && cargo build --release -cp target/release/mineru /usr/local/bin/ -``` +| Platform | Architecture | File | +|---|---|---| +| macOS Universal | Intel + Apple Silicon | `mineru-*-universal-apple-darwin.tar.gz` | +| macOS | Apple Silicon (M1/M2/M3/M4) | `mineru-*-aarch64-apple-darwin.tar.gz` | +| macOS | Intel | `mineru-*-x86_64-apple-darwin.tar.gz` | +| Linux | x86_64 (static musl) | `mineru-*-x86_64-unknown-linux-musl.tar.gz` | +| Linux | ARM64 (static musl) | `mineru-*-aarch64-unknown-linux-musl.tar.gz` | +| Windows | x86_64 | `mineru-*-x86_64-pc-windows-msvc.zip` | ## 🌐 Network (China mainland / restricted regions) diff --git a/skill/SKILL.md b/skill/SKILL.md index b4ec24d..f1b39d5 100644 --- a/skill/SKILL.md +++ b/skill/SKILL.md @@ -1,94 +1,163 @@ --- name: mineru-ocr-cli -description: Convert PDFs, scanned images, and Office documents to LLM-friendly Markdown using the free MinerU OCR API. Extracts equations, tables, and images. No API key required. +description: > + Convert PDFs, scanned images, and Office documents to LLM-friendly Markdown + using the free MinerU OCR API. Extracts equations, tables, and images with + GPU-backed accuracy. No API key required. Supports batch processing, multiple + output formats (Markdown / JSON / plain text), and LLM pipe workflows. version: 1.0.0 metadata: openclaw: emoji: "📄" homepage: https://github.com/neipor/mineru-cli + tags: [pdf, ocr, markdown, llm, document, extract] requires: bins: - mineru install: - - kind: brew - formula: mineru + - kind: release + url: "https://github.com/neipor/mineru-cli/releases/latest" bins: [mineru] config: env: MINERU_SERVER_URL: - description: "Custom Gradio server URL. Override the default HuggingFace Space if it is blocked in your region (see Network section below)." + description: > + Custom Gradio server URL. Overrides the default HuggingFace Space + when it is blocked in your region (e.g. mainland China). Set to the + base URL of any self-hosted or mirrored MinerU Gradio instance. required: false default: "https://opendatalab-mineru.hf.space" HTTPS_PROXY: - description: "Standard HTTPS proxy. Set this to route requests through a proxy server." + description: > + Standard HTTPS proxy URL (e.g. http://127.0.0.1:7890). The tool + honours this automatically — useful for regions where hf.space is + inaccessible. Common ports: Clash=7890, V2Ray=10809, SS=1080. required: false HTTP_PROXY: - description: "Standard HTTP proxy fallback." + description: > + Standard HTTP proxy fallback. Used when HTTPS_PROXY is not set. required: false --- # mineru-ocr-cli -Convert **PDFs, scanned images, and Office documents** to **LLM-friendly Markdown** using the [MinerU OCR](https://huggingface.co/spaces/opendatalab/MinerU) API — free, no API key, GPU-backed (L40S), supports LaTeX formulas and tables. +Convert **PDFs, scanned images, and Office documents** to **LLM-friendly +Markdown** using the [MinerU OCR HuggingFace Space](https://huggingface.co/spaces/opendatalab/MinerU) +— free, no API key, GPU-backed (L40S), with support for LaTeX formulas, tables, +and embedded images. + +``` +mineru paper.pdf # Markdown to stdout +mineru scan.pdf --ocr --lang en # Force OCR +mineru *.pdf -f json -o ./output/ # Batch → JSON files +mineru report.pdf -f plain -q | llm "Summarize:" +``` + +--- ## What this skill does When you run `mineru `, it: -1. Uploads the file to the MinerU HuggingFace Space -2. Waits for GPU-backed OCR processing (formulas, tables, images, text) -3. Downloads the result ZIP and extracts the Markdown + all images -4. Outputs clean Markdown ready for LLM consumption +1. **Validates** the file extension against supported formats +2. **Uploads** the file to the MinerU HuggingFace Space via multipart POST +3. **Queues** a conversion job on the remote GPU (L40S) +4. **Streams** the live status via Gradio 6 SSE queue events +5. **Downloads** the result ZIP and extracts Markdown + all images +6. **Formats** the output (Markdown / JSON / plain text) for LLM consumption -**Supported formats:** PDF, DOCX, DOC, PPT, PPTX, PNG, JPG, JPEG, WebP, BMP, TIFF +**Supported input formats:** PDF, DOCX, DOC, PPT, PPTX, PNG, JPG, JPEG, WebP, BMP, TIFF --- ## Installation -### Option A — Download prebuilt binary (recommended, no Rust needed) +### Option A — One-liner install (macOS / Linux, recommended) -Go to [GitHub Releases](https://github.com/neipor/mineru-cli/releases/latest) and download the binary for your platform: +Pick the command for your platform and paste it into your terminal. +The binary is placed in `/usr/local/bin/` which is already on your `PATH`. -| Platform | File | -|---|---| -| macOS Apple Silicon (M1/M2/M3) | `mineru-aarch64-apple-darwin.tar.gz` | -| macOS Intel | `mineru-x86_64-apple-darwin.tar.gz` | -| macOS Universal (both) | `mineru-universal-apple-darwin.tar.gz` | -| Linux x86_64 | `mineru-x86_64-unknown-linux-musl.tar.gz` | -| Linux ARM64 | `mineru-aarch64-unknown-linux-musl.tar.gz` | -| Windows x86_64 | `mineru-x86_64-pc-windows-msvc.zip` | +```bash +# macOS — Universal binary (Intel + Apple Silicon) +curl -fsSL https://github.com/neipor/mineru-cli/releases/latest/download/mineru-universal-apple-darwin.tar.gz \ + | tar -xz --strip-components=1 -C /usr/local/bin/ mineru-*/mineru +chmod +x /usr/local/bin/mineru && mineru --version +``` ```bash -# Example: macOS Apple Silicon -curl -L https://github.com/neipor/mineru-cli/releases/latest/download/mineru-aarch64-apple-darwin.tar.gz \ - | tar -xz -C /usr/local/bin/ -chmod +x /usr/local/bin/mineru -mineru --version +# Linux x86_64 (static musl — works on Ubuntu, Debian, Alpine, RHEL, …) +curl -fsSL https://github.com/neipor/mineru-cli/releases/latest/download/mineru-x86_64-unknown-linux-musl.tar.gz \ + | tar -xz --strip-components=1 -C /usr/local/bin/ mineru-*/mineru +chmod +x /usr/local/bin/mineru && mineru --version ``` ```bash -# Example: Linux x86_64 -curl -L https://github.com/neipor/mineru-cli/releases/latest/download/mineru-x86_64-unknown-linux-musl.tar.gz \ - | tar -xz -C /usr/local/bin/ -chmod +x /usr/local/bin/mineru -mineru --version +# Linux ARM64 (Raspberry Pi 4/5, AWS Graviton, Oracle Ampere, …) +curl -fsSL https://github.com/neipor/mineru-cli/releases/latest/download/mineru-aarch64-unknown-linux-musl.tar.gz \ + | tar -xz --strip-components=1 -C /usr/local/bin/ mineru-*/mineru +chmod +x /usr/local/bin/mineru && mineru --version ``` +> **Tip — install without root:** Replace `/usr/local/bin/` with `$HOME/.local/bin/` +> and make sure `$HOME/.local/bin` is in your `PATH` (add +> `export PATH="$HOME/.local/bin:$PATH"` to `~/.bashrc` or `~/.zshrc`). + +### Option B — Windows (PowerShell) + ```powershell -# Windows (PowerShell) -Invoke-WebRequest -Uri https://github.com/neipor/mineru-cli/releases/latest/download/mineru-x86_64-pc-windows-msvc.zip -OutFile mineru.zip -Expand-Archive mineru.zip -DestinationPath $env:USERPROFILE\bin\ -# Add $env:USERPROFILE\bin to your PATH +# Download and extract +$ver = (Invoke-RestMethod "https://api.github.com/repos/neipor/mineru-cli/releases/latest").tag_name +$url = "https://github.com/neipor/mineru-cli/releases/download/$ver/mineru-$ver-x86_64-pc-windows-msvc.zip" +Invoke-WebRequest -Uri $url -OutFile "$env:TEMP\mineru.zip" +Expand-Archive "$env:TEMP\mineru.zip" -DestinationPath "$env:TEMP\mineru-pkg" -Force + +# Copy to a directory that is already on PATH (System32 alternative: %LOCALAPPDATA%\Programs\mineru) +$dest = "$env:LOCALAPPDATA\Programs\mineru" +New-Item -ItemType Directory -Force $dest | Out-Null +Copy-Item "$env:TEMP\mineru-pkg\*\mineru.exe" $dest + +# Permanently add to user PATH (takes effect in new shells) +$currentPath = [Environment]::GetEnvironmentVariable("Path", "User") +if ($currentPath -notlike "*$dest*") { + [Environment]::SetEnvironmentVariable("Path", "$currentPath;$dest", "User") + Write-Host "Added $dest to PATH — restart your terminal." +} + +# Verify +& "$dest\mineru.exe" --version ``` -### Option B — Build from source (requires Rust 1.85+) +### Option C — Build from source (requires Rust 1.85+) ```bash git clone https://github.com/neipor/mineru-cli.git cd mineru-cli cargo build --release -cp target/release/mineru /usr/local/bin/ + +# Install to /usr/local/bin/ (system-wide) +sudo cp target/release/mineru /usr/local/bin/ + +# Or install to ~/.cargo/bin/ (user-level, already on PATH after `rustup` setup) +cargo install --path . +``` + +### Option D — cargo install (crates.io) + +```bash +cargo install mineru-cli +# Binary is placed at ~/.cargo/bin/mineru +``` + +> If `~/.cargo/bin` is not on your `PATH`, add `export PATH="$HOME/.cargo/bin:$PATH"` +> to your shell profile. + +--- + +## Verify the installation + +```bash +mineru --version # should print: mineru x.y.z +mineru --help # full option reference ``` --- @@ -97,130 +166,286 @@ cp target/release/mineru /usr/local/bin/ > **⚠️ China mainland and some regions: HuggingFace is blocked.** > -> The default server (`https://opendatalab-mineru.hf.space`) requires access to `hf.space` domains. -> These are **not accessible from mainland China** and some corporate networks without a proxy. +> The default server (`https://opendatalab-mineru.hf.space`) requires access to +> `hf.space` domains, which are **not reachable from mainland China** and some +> corporate networks without a proxy. ### Workaround 1 — System proxy (simplest) -Set a proxy before running the tool. The tool respects standard proxy env vars: +The tool automatically honours standard `HTTPS_PROXY` / `HTTP_PROXY` env vars: ```bash -# macOS / Linux +# macOS / Linux — set once for the session export HTTPS_PROXY=http://127.0.0.1:7890 # replace with your proxy port mineru document.pdf -# Or inline, single command +# Or inline for a single command HTTPS_PROXY=http://127.0.0.1:7890 mineru document.pdf ``` -Common proxy ports: Clash=7890, V2Ray/Xray=10809, Shadowsocks=1080, Trojan=7890. +```powershell +# Windows PowerShell +$env:HTTPS_PROXY = "http://127.0.0.1:7890" +mineru document.pdf +``` + +Common proxy ports: Clash / ClashX = **7890** · V2Ray / Xray = **10809** · +Shadowsocks = **1080** · Trojan = **7890** ### Workaround 2 — `MINERU_SERVER_URL` env var -If you have a self-hosted or mirrored MinerU Gradio instance: +If you have access to a self-hosted or mirrored MinerU Gradio instance: ```bash export MINERU_SERVER_URL=https://your-mirror.example.com mineru document.pdf ``` -Or per-command with `--server-url`: +Or pass it per-command: ```bash mineru document.pdf --server-url https://your-mirror.example.com ``` -### Workaround 3 — Self-host MinerU - -Deploy your own MinerU Gradio Space: +### Workaround 3 — Self-host MinerU (requires a CUDA GPU) ```bash -# Requires a machine with CUDA GPU pip install mineru[full] python -m mineru.cli.gradio_app --server-name 0.0.0.0 --server-port 7860 -# Then use: --server-url http://your-machine:7860 +mineru document.pdf --server-url http://localhost:7860 ``` -See the [MinerU GitHub](https://github.com/opendatalab/MinerU) for Docker deployment. +See the [MinerU GitHub](https://github.com/opendatalab/MinerU) for full Docker +and cloud deployment instructions. --- -## Usage +## Full option reference ``` mineru [OPTIONS] ... +Arguments: + ... One or more input files (globs work in most shells: *.pdf) + Options: - -f, --format Output format: markdown (default), json, plain - -p, --pages Max pages to process [default: 20] - --ocr Force OCR mode (for scanned documents) - --no-formulas Disable LaTeX formula recognition - --no-tables Disable table recognition - -l, --lang OCR language [default: ch] - -b, --backend pipeline | vlm-auto-engine | hybrid-auto-engine - -o, --output-dir Save output folder here (markdown + images/) - --embed-images Inline images as base64 data URIs - -q, --quiet Suppress progress; pipe-friendly - --server-url Custom Gradio server [default: HuggingFace Space] - -h, --help Print help + -f, --format + Output format [default: markdown] + Possible values: markdown, json, plain + + -p, --pages + Maximum pages to process per document [default: 20] + The public MinerU Space enforces a hard 20-page limit. + + --ocr + Force OCR mode. Use when native text extraction fails (scanned PDFs, + image-only documents). Slightly slower but more thorough. + + --no-formulas + Disable LaTeX formula recognition (faster for documents without math). + + --no-tables + Disable table structure recognition. + + -l, --lang + OCR language hint [default: "ch (Chinese, English, Chinese Traditional)"] + Common values: ch, en, fr, de, ja, ko, ar + Affects OCR character set; does not restrict document content. + + -b, --backend + Processing backend [default: hybrid-auto-engine] + Possible values: + pipeline — Traditional multi-model pipeline, hallucination-free + vlm-auto-engine — VLM-based, Chinese/English only, highest accuracy + hybrid-auto-engine — Hybrid, best overall multi-language (default) + + -o, --output-dir + Save output Markdown (or JSON / txt) plus an images/ sub-folder here. + When omitted, content goes to stdout and image refs are shown as tags. + + --embed-images + Inline all images as base64 data URIs in the Markdown output. + Useful for multimodal LLMs that accept self-contained Markdown. + Ignored when --output-dir is set (images are always saved as files). + + --server-url + Custom Gradio server base URL [env: MINERU_SERVER_URL] + [default: https://opendatalab-mineru.hf.space] + + -q, --quiet + Suppress all progress output. Only the document content is written to + stdout — ideal for shell pipelines. + + -h, --help Print help + -V, --version Print version ``` --- -## Examples for LLM workflows +## Usage examples + +### Basic extraction ```bash -# Summarize a PDF with any LLM CLI -mineru paper.pdf -q | llm "Summarize this paper in 3 bullet points" +# Extract a PDF to stdout (Markdown) +mineru paper.pdf + +# Extract only the first 5 pages +mineru report.pdf --pages 5 -# Extract and save with images to a folder +# Force OCR on a scanned document +mineru scan.pdf --ocr + +# English-only document, disable formula/table detection for speed +mineru letter.pdf --lang en --no-formulas --no-tables +``` + +### Save to files + +```bash +# Save Markdown + images to ./output/ mineru report.pdf -o ./output/ -# OCR a scanned document (Chinese) -mineru scan.pdf --ocr --lang ch +# Save as JSON (includes metadata block) +mineru report.pdf -f json -o ./output/ -# Process multiple PDFs to JSON -mineru *.pdf -f json -o ./extracted/ +# Batch: convert all PDFs in current directory +mineru *.pdf -o ./extracted/ -# Pipe to ollama -mineru doc.pdf -f plain -q | ollama run llama3 "What are the key points?" +# Batch with progress suppressed (CI / scripting) +mineru *.pdf -o ./extracted/ -q +``` -# Use with a proxy (China mainland) +### LLM pipe workflows + +```bash +# Summarize with the llm CLI +mineru paper.pdf -q | llm "Summarize in 5 bullet points" + +# Chat with a document via ollama +mineru doc.pdf -f plain -q | ollama run llama3 "What are the key findings?" + +# Feed a PDF to GPT-4 via the openai CLI +mineru paper.pdf -q | openai api chat.completions.create \ + -m gpt-4o --message "$(cat)" + +# Embed images for multimodal LLMs +mineru figure_heavy_paper.pdf --embed-images -q | llm -m gpt-4o "Describe the figures" +``` + +### Advanced + +```bash +# VLM backend for highest Chinese accuracy +mineru chinese_report.pdf -b vlm-auto-engine -l ch + +# Process a PPTX presentation +mineru slides.pptx -o ./slides_output/ + +# Via proxy (China mainland) HTTPS_PROXY=http://127.0.0.1:7890 mineru document.pdf -o ./output/ + +# Self-hosted server +mineru document.pdf --server-url http://192.168.1.100:7860 +``` + +--- + +## Output formats + +### `markdown` (default) — LLM-optimised + +```markdown + + +# Attention Is All You Need + +## Abstract +The dominant sequence transduction models... + +$$\text{Attention}(Q, K, V) = \text{softmax}\!\left(\frac{QK^T}{\sqrt{d_k}}\right)V$$ +``` + +Headings, LaTeX math (`\(...\)` / `\[...\]`), tables, and lists are preserved. + +### `json` — structured with metadata + +```json +{ + "meta": { + "source_file": "paper.pdf", + "processed_at": "2026-04-03T14:26:00Z", + "backend": "hybrid-auto-engine", + "pages": 3, + "ocr": false, + "formula": true, + "table": true, + "language": "ch (Chinese, English, Chinese Traditional)" + }, + "content": "# Attention Is All You Need\n\n...", + "status_log": ["Preparing request...", "Processing on server (0.0s)", "Completed"] +} ``` -## Output structure (with `-o`) +### `plain` — stripped text + +Markdown syntax removed — suitable for pure-text pipelines or word-count tools. + +--- + +## Output directory layout (when `-o` is used) ``` output/ ├── document.md ← Markdown with relative image links └── document/ └── images/ - ├── abc123.jpg ← Extracted images + ├── abc123.jpg ← Extracted images referenced from the .md └── def456.jpg ``` -## Output structure (stdout, default) - -Clean Markdown is printed to stdout. Image references are replaced with text tags: +When writing to stdout (no `-o`), image references appear as compact tags: ``` [🖼 Image 1: figure1.jpg (45 KB)] ``` -This keeps the output compact and LLM-readable. Use `--embed-images` for base64 inline images when you need multimodal LLM input. +Use `--embed-images` to replace these with base64 `data:` URIs for multimodal +LLM input. --- ## Backends -| Backend | Use case | +| Backend | Best for | Notes | +|---|---|---| +| `hybrid-auto-engine` | General use, multi-language | **Default** | +| `pipeline` | Hallucination-sensitive workloads | Traditional multi-model, no generative steps | +| `vlm-auto-engine` | Highest accuracy (Chinese / English) | VLM-based; slower but most precise | + +--- + +## Limits and performance + +| Constraint | Value | |---|---| -| `hybrid-auto-engine` | Best overall, multi-language **(default)** | -| `pipeline` | Traditional pipeline, no hallucinations | -| `vlm-auto-engine` | Highest accuracy (Chinese/English only) | +| Max pages (public Space) | 20 per document | +| Typical processing time | 1–5 seconds / page on L40S GPU | +| API key required | None | +| Internet required | Yes (calls HuggingFace Space) | +| Max file size | ~50 MB (Space limit) | + +--- + +## How it works (internals) + +The tool calls the public [MinerU HuggingFace Space](https://huggingface.co/spaces/opendatalab/MinerU) +via the Gradio 6 queue API: -## Limits +1. **Upload** — `POST /gradio_api/upload` with the file as multipart/form-data +2. **Queue** — `POST /gradio_api/queue/join` (fn_index=8, `convert_to_markdown_stream`) +3. **Stream** — `GET /gradio_api/queue/data` SSE stream, parsing Gradio 6 patch events +4. **Extract** — downloads the result ZIP, extracts the `.md` file and all images +5. **Format** — cleans and re-formats Markdown for LLM consumption -- Max **20 pages** per document (public Space limit) -- Processing: ~1–5 seconds/page on L40S GPU -- No API key needed +The Space runs `opendatalab/PDF-Extract-Kit-1.0` and +`opendatalab/MinerU2.5-2509-1.2B` models on an L40S GPU.