From c49e3fd046e03d9d842eed223df753860fa3907a Mon Sep 17 00:00:00 2001 From: Howard Su Date: Thu, 18 Jun 2026 17:12:02 +0800 Subject: [PATCH] ci(release): add Windows release package workflow - New workflow (.github/workflows/release-windows.yml) builds dflash_server.exe fat binary (sm_75/86/89/120) and packages with run.py + user manual - Add docs/USER_MANUAL.md: friendly getting-started guide for end users - CMakeLists.txt: static CUDA runtime on Windows for portable binary - Triggered on v* tags, push to main, PRs, and workflow_dispatch - Creates GitHub Release with zip artifact on tag pushes --- .github/workflows/release-windows.yml | 103 ++++++++++++++ docs/USER_MANUAL.md | 186 ++++++++++++++++++++++++++ server/CMakeLists.txt | 5 + 3 files changed, 294 insertions(+) create mode 100644 .github/workflows/release-windows.yml create mode 100644 docs/USER_MANUAL.md diff --git a/.github/workflows/release-windows.yml b/.github/workflows/release-windows.yml new file mode 100644 index 000000000..221ad8dd4 --- /dev/null +++ b/.github/workflows/release-windows.yml @@ -0,0 +1,103 @@ +name: Release (Windows) + +on: + push: + tags: ['v*'] + branches: [main] + paths: + - 'server/**' + - '.github/workflows/release-windows.yml' + pull_request: + branches: [main] + paths: + - 'server/**' + - '.github/workflows/release-windows.yml' + workflow_dispatch: + +jobs: + build-windows: + name: Build Windows release package + runs-on: windows-latest + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + submodules: recursive + token: ${{ secrets.SUBMODULE_PAT || secrets.GITHUB_TOKEN }} + + - uses: Jimver/cuda-toolkit@3d45d157f327c09c04b50ee6ccdea2d9d017ec76 # v0.2.35 + with: + cuda: '12.8.0' + method: network + sub-packages: '["nvcc", "cudart", "thrust", "visual_studio_integration"]' + non-cuda-sub-packages: '["libcublas-dev"]' + + - name: Configure CMake + run: | + cmake -B server/build -S server ` + -DCMAKE_CUDA_ARCHITECTURES="75;86;89;120" ` + -DCMAKE_BUILD_TYPE=Release ` + -DDFLASH27B_ENABLE_BSA=OFF ` + -DDFLASH27B_FA_ALL_QUANTS=ON ` + -DDFLASH27B_TESTS=OFF ` + -DDFLASH27B_SERVER=ON + + - name: Build dflash_server + run: cmake --build server/build --config Release --target dflash_server -j $env:NUMBER_OF_PROCESSORS + + - name: Assemble release package + shell: pwsh + run: | + $pkg = "dflash-server-windows-x64" + New-Item -ItemType Directory -Force -Path $pkg/scripts + New-Item -ItemType Directory -Force -Path $pkg/share + New-Item -ItemType Directory -Force -Path $pkg/docs + + # Binary + Copy-Item server/build/Release/dflash_server.exe $pkg/ -ErrorAction SilentlyContinue + if (-not (Test-Path "$pkg/dflash_server.exe")) { + Copy-Item server/build/dflash_server.exe $pkg/ + } + + # Status page + Copy-Item server/share/status.html $pkg/share/ + + # Python launcher + Copy-Item server/scripts/run.py $pkg/scripts/ + + # User manual + Copy-Item docs/USER_MANUAL.md $pkg/docs/ + + # Version info + $tag = "${{ github.ref_name }}" + Set-Content -Path "$pkg/VERSION.txt" -Value "DFlash Server $tag`nBuilt from ${{ github.sha }}" + + # Zip + Compress-Archive -Path $pkg/* -DestinationPath "$pkg.zip" + echo "PACKAGE_NAME=$pkg" >> $env:GITHUB_ENV + + - name: Upload release artifact + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: ${{ env.PACKAGE_NAME }} + path: ${{ env.PACKAGE_NAME }}.zip + + release: + name: Create GitHub Release + needs: build-windows + runs-on: ubuntu-latest + if: startsWith(github.ref, 'refs/tags/v') + permissions: + contents: write + steps: + - name: Download artifact + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: dflash-server-windows-x64 + + - name: Create Release + uses: softprops/action-gh-release@da05d552573ad5aba039eaac05058a918a7bf631 # v2.2.2 + with: + files: dflash-server-windows-x64.zip + generate_release_notes: true + draft: false + prerelease: ${{ contains(github.ref_name, 'rc') || contains(github.ref_name, 'beta') }} diff --git a/docs/USER_MANUAL.md b/docs/USER_MANUAL.md new file mode 100644 index 000000000..a1c2e9815 --- /dev/null +++ b/docs/USER_MANUAL.md @@ -0,0 +1,186 @@ +# DFlash Server — User Manual + +> **Fast AI inference on your Windows PC with an NVIDIA GPU.** + +DFlash is a high-performance local AI server. It runs large language models +(like Qwen 27B) on your GPU using speculative decoding — a technique that +generates text 2–5× faster than standard inference. Once running, it exposes an +OpenAI-compatible API that works with any chat client. + +--- + +## System Requirements + +| Component | Minimum | Recommended | +|-----------|---------|-------------| +| OS | Windows 10 (64-bit) | Windows 11 | +| GPU | NVIDIA GTX 1060 (6 GB) | RTX 3090 / 4090 (24 GB) | +| VRAM | 8 GB (smaller models) | 24 GB (Qwen 27B Q4) | +| RAM | 16 GB | 32 GB | +| Disk | 20 GB free | 40 GB free | +| NVIDIA Driver | 535+ | Latest Game Ready or Studio | +| Python | 3.10+ | 3.12 | + +> **Note:** You do NOT need to install the CUDA Toolkit. The server binary +> includes everything it needs. Just make sure your NVIDIA driver is up to date. + +--- + +## Quick Start + +### Step 1: Download Models + +You need two model files — a **target** (the big model) and a **draft** (a +small helper that speeds things up). + +Install the Hugging Face CLI if you don't have it: + +```powershell +pip install huggingface-hub[cli] +``` + +Download the models (about 18 GB total): + +```powershell +# Create a folder for models +mkdir models\draft + +# Download the main model (~16 GB) +huggingface-cli download unsloth/Qwen3.6-27B-GGUF Qwen3.6-27B-Q4_K_M.gguf --local-dir models + +# Download the draft model (~1.8 GB) +huggingface-cli download Lucebox/Qwen3.6-27B-DFlash-GGUF dflash-draft-3.6-q4_k_m.gguf --local-dir models\draft +``` + +### Step 2: Start the Server + +The simplest way is to run the server directly: + +```powershell +.\dflash_server.exe models\Qwen3.6-27B-Q4_K_M.gguf ^ + --draft models\draft\dflash-draft-3.6-q4_k_m.gguf ^ + --ddtree --ddtree-budget 22 --fa-window 2048 --port 8080 +``` + +You should see output indicating the model is loaded and the server is +listening on `http://localhost:8080`. + +### Step 3: Talk to the Server + +Open another terminal and send a request: + +```powershell +curl http://localhost:8080/v1/chat/completions ^ + -H "Content-Type: application/json" ^ + -d "{\"model\": \"qwen\", \"messages\": [{\"role\": \"user\", \"content\": \"Hello! What can you do?\"}]}" +``` + +Or use any OpenAI-compatible client by pointing it at `http://localhost:8080`. + +--- + +## Using the Python Launcher + +For a more convenient experience, use the included `run.py` script. It handles +tokenization and chat templates automatically. + +First, install the required Python package: + +```powershell +pip install transformers +``` + +Then run: + +```powershell +python scripts\run.py --prompt "Write a Python function to sort a list" +``` + +You can also pipe input: + +```powershell +echo "Explain quantum computing in simple terms" | python scripts\run.py +``` + +### Useful Options + +| Option | Description | +|--------|-------------| +| `--prompt "..."` | The text prompt to send | +| `--n-gen 512` | Maximum tokens to generate (default: 256) | +| `--target path` | Path to target model (default: `models/Qwen3.6-27B-Q4_K_M.gguf`) | +| `--draft path` | Path to draft model (default: `models/draft/`) | +| `--budget 22` | DDTree budget — higher = more speculative (default: 22) | +| `--system "..."` | System prompt | +| `--kv-tq3` | Enable TQ3 KV cache for longer context (up to 256K) | + +--- + +## Server API Endpoints + +Once running, the server provides these endpoints: + +| Endpoint | Description | +|----------|-------------| +| `GET /health` | Health check (returns 200 when ready) | +| `GET /v1/models` | List available models | +| `POST /v1/chat/completions` | OpenAI Chat Completions API | +| `POST /v1/responses` | OpenAI Responses API (for Codex) | +| `POST /v1/messages` | Anthropic Messages API (for Claude Code) | + +--- + +## Connecting Clients + +DFlash works as a drop-in backend for popular AI tools: + +- **Open WebUI**: Set the OpenAI API base URL to `http://localhost:8080/v1` +- **Continue (VS Code)**: Add a custom model with base URL `http://localhost:8080` +- **Codex CLI**: Set `OPENAI_BASE_URL=http://localhost:8080/v1` +- **Any OpenAI SDK**: Point `base_url` to `http://localhost:8080/v1` + +--- + +## Troubleshooting + +### "CUDA error" or "no CUDA device" + +- Make sure your NVIDIA driver is version 535 or newer +- Run `nvidia-smi` in a terminal to verify your GPU is detected +- Restart your PC after a driver update + +### Out of memory (OOM) + +- Close other GPU-heavy applications (games, other AI tools) +- Use a smaller model or quantization +- Add `--kv-tq3` to reduce KV cache memory usage + +### Port already in use + +- Change the port: `--port 8081` +- Or find what's using port 8080: `netstat -ano | findstr 8080` + +### Server starts but generation is slow + +- Make sure you're using both `--ddtree` and `--draft` flags +- Check that your GPU is not thermal throttling (`nvidia-smi` shows temperature) +- Close background GPU workloads + +### Python launcher can't find the binary + +- Run from the folder where `dflash_server.exe` is located +- Or set the path: `python scripts\run.py --bin path\to\dflash_server.exe` + +--- + +## Performance Tips + +- **DDTree budget**: The default of 22 works well for most tasks. Higher values + (e.g., 32) may help for code generation but use more VRAM. +- **FA window**: `--fa-window 2048` is optimal for most use cases. Only increase + if you need the model to attend to very long prior context. +- **TQ3 KV cache**: Use `--kv-tq3` if you need very long context (32K+). It + uses ~3× less memory than the default F16 cache with minimal quality loss. +- **Power limit**: For sustained workloads, setting a power limit + (`nvidia-smi -pl 220` on RTX 3090) can improve efficiency without + significant speed loss. diff --git a/server/CMakeLists.txt b/server/CMakeLists.txt index 5bfcb5ea7..a13d11e0e 100644 --- a/server/CMakeLists.txt +++ b/server/CMakeLists.txt @@ -175,6 +175,11 @@ endif() if(WIN32) set(BUILD_SHARED_LIBS OFF CACHE BOOL "Static ggml on Windows (internal CUDA symbols not dllexported)" FORCE) + # Static CUDA runtime on Windows so the release binary is portable + # (no cudart64_*.dll needed alongside the executable). + if(DFLASH27B_GPU_BACKEND STREQUAL "cuda") + set(CMAKE_CUDA_RUNTIME_LIBRARY Static CACHE STRING "" FORCE) + endif() endif() if(WIN32 AND NOT CMAKE_ASM_COMPILER)