From c258b4877a13ffceacbbca1ed12c199aebba20e8 Mon Sep 17 00:00:00 2001 From: Patrick Walukagga Date: Mon, 15 Jun 2026 13:25:08 +0300 Subject: [PATCH] Oprheus 3B TTS model --- .gitignore | 15 +- modal-deploy/orpheus-3B/MODAL_DEPLOYMENT.md | 393 ++++++++ modal-deploy/orpheus-3B/MODEL_CARD.md | 522 ++++++++++ .../orpheus-3B/MODEL_CARD_MULTILINGUAL.md | 663 +++++++++++++ .../Orpheus_3B_Sunbird_Luganda.ipynb | 862 +++++++++++++++++ ...Orpheus_3B_Sunbird_Luganda_Inference.ipynb | 484 ++++++++++ ...us_3B_Sunbird_Luganda_vLLM_Inference.ipynb | 618 ++++++++++++ .../Orpheus_3B_Sunbird_Multilingual.ipynb | 902 ++++++++++++++++++ modal-deploy/orpheus-3B/api/.env.example | 16 + modal-deploy/orpheus-3B/api/.gitignore | 2 + modal-deploy/orpheus-3B/api/README.md | 227 +++++ modal-deploy/orpheus-3B/api/__init__.py | 0 modal-deploy/orpheus-3B/api/config.py | 72 ++ modal-deploy/orpheus-3B/api/errors.py | 159 +++ modal-deploy/orpheus-3B/api/logging_setup.py | 36 + modal-deploy/orpheus-3B/api/main.py | 106 ++ modal-deploy/orpheus-3B/api/modal_client.py | 172 ++++ modal-deploy/orpheus-3B/api/models.py | 139 +++ modal-deploy/orpheus-3B/api/requirements.txt | 11 + modal-deploy/orpheus-3B/api/routes.py | 302 ++++++ .../orpheus-3B/api/scripts/__init__.py | 0 modal-deploy/orpheus-3B/api/scripts/smoke.py | 43 + .../api/scripts/test_english_tts.py | 151 +++ .../orpheus-3B/api/scripts/test_long_text.py | 244 +++++ modal-deploy/orpheus-3B/api/speakers.py | 115 +++ modal-deploy/orpheus-3B/api/storage.py | 115 +++ modal-deploy/orpheus-3B/api/tests/__init__.py | 0 modal-deploy/orpheus-3B/api/tests/conftest.py | 130 +++ .../orpheus-3B/api/tests/test_config.py | 74 ++ .../orpheus-3B/api/tests/test_errors.py | 97 ++ .../orpheus-3B/api/tests/test_modal_client.py | 376 ++++++++ .../orpheus-3B/api/tests/test_models.py | 101 ++ .../orpheus-3B/api/tests/test_routes_batch.py | 141 +++ .../api/tests/test_routes_health.py | 65 ++ .../api/tests/test_routes_speakers.py | 36 + .../orpheus-3B/api/tests/test_routes_tts.py | 195 ++++ .../orpheus-3B/api/tests/test_speakers.py | 165 ++++ .../orpheus-3B/api/tests/test_storage.py | 138 +++ modal-deploy/orpheus-3B/article.txt | 15 + ...blogpost-orpheus-multilingual-release.docx | Bin 0 -> 20161 bytes modal-deploy/orpheus-3B/conftest.py | 8 + modal-deploy/orpheus-3B/luganda_article.txt | 14 + modal-deploy/orpheus-3B/modal_deploy.py | 668 +++++++++++++ modal-deploy/orpheus-3B/optimizer-review.md | 584 ++++++++++++ modal-deploy/orpheus-3B/tests/__init__.py | 1 + .../orpheus-3B/tests/test_chunking.py | 209 ++++ 46 files changed, 9385 insertions(+), 1 deletion(-) create mode 100644 modal-deploy/orpheus-3B/MODAL_DEPLOYMENT.md create mode 100644 modal-deploy/orpheus-3B/MODEL_CARD.md create mode 100644 modal-deploy/orpheus-3B/MODEL_CARD_MULTILINGUAL.md create mode 100644 modal-deploy/orpheus-3B/Orpheus_3B_Sunbird_Luganda.ipynb create mode 100644 modal-deploy/orpheus-3B/Orpheus_3B_Sunbird_Luganda_Inference.ipynb create mode 100644 modal-deploy/orpheus-3B/Orpheus_3B_Sunbird_Luganda_vLLM_Inference.ipynb create mode 100644 modal-deploy/orpheus-3B/Orpheus_3B_Sunbird_Multilingual.ipynb create mode 100644 modal-deploy/orpheus-3B/api/.env.example create mode 100644 modal-deploy/orpheus-3B/api/.gitignore create mode 100644 modal-deploy/orpheus-3B/api/README.md create mode 100644 modal-deploy/orpheus-3B/api/__init__.py create mode 100644 modal-deploy/orpheus-3B/api/config.py create mode 100644 modal-deploy/orpheus-3B/api/errors.py create mode 100644 modal-deploy/orpheus-3B/api/logging_setup.py create mode 100644 modal-deploy/orpheus-3B/api/main.py create mode 100644 modal-deploy/orpheus-3B/api/modal_client.py create mode 100644 modal-deploy/orpheus-3B/api/models.py create mode 100644 modal-deploy/orpheus-3B/api/requirements.txt create mode 100644 modal-deploy/orpheus-3B/api/routes.py create mode 100644 modal-deploy/orpheus-3B/api/scripts/__init__.py create mode 100644 modal-deploy/orpheus-3B/api/scripts/smoke.py create mode 100644 modal-deploy/orpheus-3B/api/scripts/test_english_tts.py create mode 100644 modal-deploy/orpheus-3B/api/scripts/test_long_text.py create mode 100644 modal-deploy/orpheus-3B/api/speakers.py create mode 100644 modal-deploy/orpheus-3B/api/storage.py create mode 100644 modal-deploy/orpheus-3B/api/tests/__init__.py create mode 100644 modal-deploy/orpheus-3B/api/tests/conftest.py create mode 100644 modal-deploy/orpheus-3B/api/tests/test_config.py create mode 100644 modal-deploy/orpheus-3B/api/tests/test_errors.py create mode 100644 modal-deploy/orpheus-3B/api/tests/test_modal_client.py create mode 100644 modal-deploy/orpheus-3B/api/tests/test_models.py create mode 100644 modal-deploy/orpheus-3B/api/tests/test_routes_batch.py create mode 100644 modal-deploy/orpheus-3B/api/tests/test_routes_health.py create mode 100644 modal-deploy/orpheus-3B/api/tests/test_routes_speakers.py create mode 100644 modal-deploy/orpheus-3B/api/tests/test_routes_tts.py create mode 100644 modal-deploy/orpheus-3B/api/tests/test_speakers.py create mode 100644 modal-deploy/orpheus-3B/api/tests/test_storage.py create mode 100644 modal-deploy/orpheus-3B/article.txt create mode 100644 modal-deploy/orpheus-3B/blogpost-orpheus-multilingual-release.docx create mode 100644 modal-deploy/orpheus-3B/conftest.py create mode 100644 modal-deploy/orpheus-3B/luganda_article.txt create mode 100644 modal-deploy/orpheus-3B/modal_deploy.py create mode 100644 modal-deploy/orpheus-3B/optimizer-review.md create mode 100644 modal-deploy/orpheus-3B/tests/__init__.py create mode 100644 modal-deploy/orpheus-3B/tests/test_chunking.py diff --git a/.gitignore b/.gitignore index 5d50df6a..ba13f5cb 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,17 @@ runpod.toml test/* vllm-base/vllm-* .DS_Store -.ipynb_checkpoints \ No newline at end of file +.ipynb_checkpoints + +# Orpheus-3B local artifacts +modal-deploy/orpheus-3B/_build_notebook.py +modal-deploy/orpheus-3B/_build_inference_notebook.py +modal-deploy/orpheus-3B/_build_multilingual_notebook.py +modal-deploy/orpheus-3B/_build_vllm_inference_notebook.py +modal-deploy/orpheus-3B/outputs/ +modal-deploy/orpheus-3B/inference_outputs/ +modal-deploy/orpheus-3B/Orpheus_(3B)_TTS.ipynb +service-account-key.json + +budget.json +notification.json \ No newline at end of file diff --git a/modal-deploy/orpheus-3B/MODAL_DEPLOYMENT.md b/modal-deploy/orpheus-3B/MODAL_DEPLOYMENT.md new file mode 100644 index 00000000..5ecb16a1 --- /dev/null +++ b/modal-deploy/orpheus-3B/MODAL_DEPLOYMENT.md @@ -0,0 +1,393 @@ +# Deploying Orpheus-3B Multilingual TTS to Modal + +This guide walks through deploying the finetuned multilingual Orpheus-3B TTS +([`sunbird/orpheus-3b-tts-multilingual`](https://huggingface.co/sunbird/orpheus-3b-tts-multilingual)) +to [Modal](https://modal.com) as a vLLM-backed, FastAPI HTTP service. + +The deployment exposes: + +| Endpoint | Method | Purpose | +|---------------------|--------|-------------------------------------------------------------------------| +| `/tts` | POST | Single utterance, returns `audio/wav` | +| `/tts/batch` | POST | Multi-utterance batched synthesis (multi-speaker), returns base64 wavs | +| `/speakers` | GET | Static catalog of speaker IDs grouped by language | +| `/health` | GET | Liveness probe with model + sample-rate info | + +All endpoints accept a `speaker_id` per request, so a single replica serves +any speaker the finetuned checkpoint knows (multi-language, multi-speaker). + +Files used by this guide: +- [orpheus-3B/modal_deploy.py](modal_deploy.py) — the Modal app +- [orpheus-3B/Orpheus_3B_Sunbird_Luganda_vLLM_Inference.ipynb](Orpheus_3B_Sunbird_Luganda_vLLM_Inference.ipynb) — reference notebook the script mirrors + +--- + +## 1. Prerequisites + +### 1.1 Tooling + +```bash +# Modal CLI (also used to manage secrets, volumes, deploys) +pip install -U modal + +# Log in once per workstation (opens a browser tab) +modal token new +``` + +You will also need a HuggingFace account with read access to the Orpheus +checkpoint (the published Sunbird repos are public, so a generic read token +works). + +### 1.2 Modal secret for HuggingFace + +The script expects a Modal secret named `huggingface-secret` containing an +`HF_TOKEN`. Create it once: + +```bash +modal secret create huggingface-secret HF_TOKEN=hf_xxx_your_read_token +``` + +Confirm with: + +```bash +modal secret list | grep huggingface-secret +``` + +### 1.3 (Optional) Override defaults at deploy time + +Defaults baked into the script can be overridden with environment variables +that Modal reads from your shell at deploy time: + +| Env var | Default | Effect | +|-------------------------------|------------------------------------------|---------------------------------------| +| `ORPHEUS_MODEL_ID` | `sunbird/orpheus-3b-tts-multilingual` | HF repo or local path to load | +| `ORPHEUS_DEFAULT_SPEAKER` | `salt_lug_0001` | Used when a request omits `speaker_id`| +| `ORPHEUS_MAX_MODEL_LEN` | `4096` | vLLM KV-cache budget | +| `ORPHEUS_GPU` | `L40S` | Modal GPU type — `A10G`, `L40S`, `H100`| +| `ORPHEUS_MAX_INPUTS` | `16` | Concurrent requests per replica | + +Example: + +```bash +ORPHEUS_GPU=H100 ORPHEUS_MAX_INPUTS=32 modal deploy orpheus-3B/modal_deploy.py +``` + +--- + +## 2. (Recommended) Warm the HuggingFace cache volume + +The first time the container starts it will download ~6 GB of model weights. +A one-shot warmup writes them into a persistent Modal Volume so future cold +starts skip the download: + +```bash +modal run orpheus-3B/modal_deploy.py::download_model +``` + +This creates two volumes if missing: `orpheus-hf-cache` and `orpheus-vllm-cache`. +The first stores HuggingFace snapshots; the second caches vLLM's compiled +CUDA graphs across container restarts. + +To warm a custom checkpoint, pass `--model-id`: + +```bash +modal run orpheus-3B/modal_deploy.py::download_model --model-id patrickcmd/orpheus-3b-tts-multilingual +``` + +--- + +## 3. Deploy + +```bash +modal deploy orpheus-3B/modal_deploy.py +``` + +Modal prints the public URL when the deploy completes. It looks like: + +``` +https://--orpheus-3b-tts-orpheustts-web.modal.run +``` + +This is the **base URL** for the four endpoints below. + +The first request after deploy triggers a cold start: pulling the image and +loading the model (~30–60 s on a warm cache, several minutes on a cold one). +Subsequent requests are near-instant until the scaledown window (15 minutes +of idle) elapses. + +--- + +## 4. Test the deployed endpoints + +Replace `BASE_URL` below with the URL printed by `modal deploy`. + +### 4.1 Health check + +```bash +BASE_URL='https://--orpheus-3b-tts-orpheustts-web.modal.run' + +curl -s "$BASE_URL/health" | jq +``` + +Expected output: + +```json +{ + "status": "ok", + "model": "sunbird/orpheus-3b-tts-multilingual", + "max_model_len": 4096, + "sample_rate": 24000 +} +``` + +### 4.2 Speaker catalog + +```bash +curl -s "$BASE_URL/speakers" | jq +``` + +The catalog is static (edit `SPEAKERS_BY_LANGUAGE` in `modal_deploy.py` to +reflect the speakers you actually finetuned on). Any speaker tag the +checkpoint was trained on will work even if it is not listed here. + +### 4.3 Single utterance (`audio/wav` response) + +```bash +curl -s -X POST "$BASE_URL/tts" \ + -H 'Content-Type: application/json' \ + -d '{ + "text": "Mwattu, oli otya?", + "speaker_id": "salt_lug_0001", + "seed": 42 + }' \ + --output luganda_hello.wav + +afplay luganda_hello.wav # macOS — or `aplay` on Linux +``` + +Response headers carry the duration and speaker: + +```bash +curl -sI -X POST "$BASE_URL/tts" \ + -H 'Content-Type: application/json' \ + -d '{"text":"Mwattu, oli otya?","speaker_id":"salt_lug_0001"}' +``` + +``` +HTTP/2 200 +content-type: audio/wav +x-sample-rate: 24000 +x-duration-seconds: 1.583 +x-speaker-id: salt_lug_0001 +``` + +### 4.4 Batched multi-speaker synthesis + +`POST /tts/batch` accepts an `items` array, each carrying its own +`speaker_id`. vLLM batches them all through one GPU pass — much faster +than calling `/tts` N times. + +```bash +curl -s -X POST "$BASE_URL/tts/batch" \ + -H 'Content-Type: application/json' \ + -d '{ + "items": [ + {"text": "Mwattu, oli otya?", "speaker_id": "salt_lug_0001"}, + {"text": "Habari ya asubuhi.", "speaker_id": "waxal_swa_0006"}, + {"text": "Itye ma ber?", "speaker_id": "salt_ach_0001"}, + {"text": "Abu ngesi ilip itunga.", "speaker_id": "salt_teo_0001"} + ] + }' \ + | jq -r '.results[] | "\(.speaker_id) \(.duration_sec)s -> \(.audio_wav_b64[0:40])..."' +``` + +Response shape: + +```json +{ + "results": [ + { + "text": "Mwattu, oli otya?", + "speaker_id": "salt_lug_0001", + "sample_rate": 24000, + "duration_sec": 1.58, + "audio_wav_b64": "UklGRiQAAABXQVZFZm10IBAAAAAB..." + } + ] +} +``` + +### 4.5 Python client (single + batch) + +```python +"""orpheus_modal_client.py — minimal Python client for the deployed service.""" +import base64 +import requests + +BASE_URL = "https://--orpheus-3b-tts-orpheustts-web.modal.run" + + +def tts(text: str, speaker_id: str, out_path: str, **decode) -> None: + r = requests.post( + f"{BASE_URL}/tts", + json={"text": text, "speaker_id": speaker_id, **decode}, + timeout=120, + ) + r.raise_for_status() + with open(out_path, "wb") as f: + f.write(r.content) + print(f"{speaker_id}: {text!r} -> {out_path}" + f" ({r.headers.get('x-duration-seconds')}s)") + + +def tts_batch(items: list[dict], out_dir: str = ".") -> None: + import pathlib + pathlib.Path(out_dir).mkdir(parents=True, exist_ok=True) + r = requests.post(f"{BASE_URL}/tts/batch", json={"items": items}, timeout=300) + r.raise_for_status() + for i, result in enumerate(r.json()["results"]): + wav_bytes = base64.b64decode(result["audio_wav_b64"]) + path = f"{out_dir}/{i:02d}_{result['speaker_id']}.wav" + with open(path, "wb") as f: + f.write(wav_bytes) + print(f"{path}: {result['duration_sec']:.2f}s") + + +if __name__ == "__main__": + tts("Mwattu, oli otya?", "salt_lug_0001", "hello_lug.wav", seed=42) + tts_batch( + [ + {"text": "Mwattu, oli otya?", "speaker_id": "salt_lug_0001"}, + {"text": "Habari ya asubuhi.", "speaker_id": "waxal_swa_0006"}, + {"text": "Itye ma ber?", "speaker_id": "salt_ach_0001"}, + ], + out_dir="modal_out", + ) +``` + +Run: + +```bash +python orpheus_modal_client.py +``` + +--- + +## 5. Decoding parameters + +Same knobs as the inference notebook — all optional on the request: + +| Field | Default | Notes | +|----------------------|---------|------------------------------------------------------------------------------------| +| `temperature` | `0.6` | Lower = more deterministic prosody; higher = more variation, can be unstable | +| `top_p` | `0.95` | Nucleus sampling cutoff | +| `repetition_penalty` | `1.1` | Discourages duplicate audio tokens (long pauses / chant artifacts) | +| `max_tokens` | `1200` | Hard cap on audio tokens generated (~1200 = ~10 s of audio) | +| `seed` | `null` | Pass an int for reproducible output | + +Run a sweep with the notebook's `#4` cell pattern if you want to tune for a +particular voice/text combo before locking these values into the client. + +--- + +## 6. Scaling, GPU choice, and cost + +| GPU | VRAM | Cost relative | When to use | +|--------|-------|---------------|----------------------------------------------------------| +| `A10G` | 24 GB | low | Light traffic, latency-tolerant | +| `L40S` | 48 GB | medium | **Default.** Best cost/throughput for Orpheus-3B bf16 | +| `H100` | 80 GB | high | Highest throughput, large batches, low p95 latency | + +Other knobs in the script: + +- `@modal.concurrent(max_inputs=16)` — how many in-flight requests a single + replica accepts before Modal starts a second one. Raising this leans on + vLLM's continuous batching for throughput; lower it if you see GPU OOM. +- `scaledown_window=15 * MINUTES` — how long a replica stays warm with no + traffic. Shorten to save cost; lengthen to avoid cold starts during + bursty workloads. +- `min_containers=0` — set to `1` to keep one replica always warm (no cold + starts, but you pay 24/7). +- `gpu_memory_utilization=0.85` in `LLM(...)` — drop to `0.70` if you hit + OOM during model load on smaller GPUs. + +--- + +## 7. Operations + +### 7.1 Tail logs + +```bash +modal app logs orpheus-3b-tts +``` + +### 7.2 Stop the app (no further requests served) + +```bash +modal app stop orpheus-3b-tts +``` + +### 7.3 Inspect / list / delete the cache volumes + +```bash +modal volume list +modal volume ls orpheus-hf-cache +modal volume rm orpheus-hf-cache # wipes the HF cache; next deploy redownloads +``` + +### 7.4 Redeploy after editing `modal_deploy.py` + +```bash +modal deploy orpheus-3B/modal_deploy.py +``` + +Modal does a rolling update — the old replicas keep serving until the new +ones are healthy. + +--- + +## 8. Troubleshooting + +### `AssertionError: ... org_vocab_size` on container start + +Your checkpoint's `config.json` has `vocab_size = 156940` but the actual +embedding weight is `[156939, 3072]`. This is the Unsloth `push_to_hub_merged` +off-by-one. The script already passes `hf_overrides={"vocab_size": 156939}` +to vLLM to fix this. If you're loading a third-party Orpheus checkpoint and +still hit the error, edit `VOCAB_SIZE` at the top of `modal_deploy.py` to +match the actual embedding shape. + +(The training notebooks for both single-speaker and multilingual finetunes +now patch `config.json` to `156939` both locally and on the hub after +`push_to_hub_merged`, so newly trained checkpoints don't need the override.) + +### Long cold starts on first request + +The very first deploy pulls the container image and downloads the model. +Run `modal run orpheus-3B/modal_deploy.py::download_model` once before +deploying to push the weights into the persistent volume. + +### Repeated 502s under load + +Bump `@modal.concurrent(max_inputs=...)` carefully — vLLM's per-request KV +budget is fixed, so very high concurrency on a single GPU eventually +overflows. Either lower `MAX_MODEL_LEN` or `max_tokens`, or let Modal scale +horizontally by leaving `max_inputs` modest (8–16) and serving more replicas. + +### `HF_TOKEN not set` / 401 from HuggingFace + +The `huggingface-secret` Modal secret is missing or the token expired. +Re-run `modal secret create huggingface-secret HF_TOKEN=hf_...`. + +--- + +## 9. What this deployment does *not* do + +- No streaming. The endpoint waits for the full utterance before returning. + If you need token-streaming TTS, return Server-Sent Events from a + generator inside the FastAPI route and decode SNAC in fixed-size windows. +- No request authentication. Add a `@web_app.middleware("http")` check on a + shared API key, or wrap the URL behind your own auth proxy. +- No long-form chunking. Inputs longer than ~30–40 seconds of speech may + hit `max_tokens`. Chunk on sentence boundaries on the client side for + long passages. diff --git a/modal-deploy/orpheus-3B/MODEL_CARD.md b/modal-deploy/orpheus-3B/MODEL_CARD.md new file mode 100644 index 00000000..27839ec3 --- /dev/null +++ b/modal-deploy/orpheus-3B/MODEL_CARD.md @@ -0,0 +1,522 @@ +--- +license: apache-2.0 +language: + - lg +library_name: transformers +pipeline_tag: text-to-speech +tags: + - text-to-speech + - tts + - orpheus + - luganda + - low-resource-language + - sunbird + - snac + - unsloth +datasets: + - Sunbird/tts +base_model: unsloth/orpheus-3b-0.1-pretrained +--- + +# Orpheus-3B Sunbird Luganda TTS — `salt_lug_0001` + +A Luganda text-to-speech model fine-tuned from +[`unsloth/orpheus-3b-0.1-pretrained`](https://huggingface.co/unsloth/orpheus-3b-0.1-pretrained) +on the [`Sunbird/tts`](https://huggingface.co/datasets/Sunbird/tts) corpus +(`lug` config), filtered to a single speaker (`salt_lug_0001`). + +The model speaks **Luganda** in the voice of `salt_lug_0001`. It accepts +arbitrary Luganda text and emits 24 kHz mono speech via the +[SNAC](https://huggingface.co/hubertsiuzdak/snac_24khz) audio codec. + +## Quick links + +- **Base model:** [`unsloth/orpheus-3b-0.1-pretrained`](https://huggingface.co/unsloth/orpheus-3b-0.1-pretrained) (Llama-3 architecture) +- **Audio codec:** [`hubertsiuzdak/snac_24khz`](https://huggingface.co/hubertsiuzdak/snac_24khz) (24 kHz, 7 codes per ~12 ms frame) +- **Training dataset:** [`Sunbird/tts`](https://huggingface.co/datasets/Sunbird/tts) — `lug` (Luganda) config, speaker `salt_lug_0001` +- **Training framework:** [Unsloth](https://github.com/unslothai/unsloth) + HuggingFace Trainer + +## TL;DR + +```python +# After installing the dependencies (see "Inference" below) +wav = synthesize("Mwattu, oli otya?", speaker_id="salt_lug_0001") +# 1-D numpy float32 at 24 kHz mono — write with soundfile +``` + +--- + +## Inference + +The model wraps every prompt in a multi-speaker tagged format: + +``` +[SOH] + tokenize("salt_lug_0001: ") + [EOT, EOH] +``` + +and the model autoregressively emits Llama-3 special tokens followed by +SNAC audio codes that decode to a 24 kHz waveform. Two reference +implementations follow. + +### Option A — `transformers` + `unsloth` (single request) + +Best for development, notebook-driven iteration, and small batch sizes. + +**Install:** + +```bash +pip install unsloth snac soundfile torchcodec "datasets>=3.4.1,<4.0.0" +``` + +**Run:** + +```python +import os +import numpy as np +import torch +import soundfile as sf +from unsloth import FastLanguageModel +from snac import SNAC + +MODEL_ID = "sunbird/orpheus-3b-tts-salt-lug-0001" +SPEAKER_ID = "salt_lug_0001" + +# Special tokens — must match the training format +END_OF_TEXT = 128009 +START_OF_SPEECH = 128257 +END_OF_SPEECH = 128258 +START_OF_HUMAN = 128259 +END_OF_HUMAN = 128260 +PAD_TOKEN = 128263 +AUDIO_TOKEN_LO = 128266 +AUDIO_TOKEN_HI = 128266 + 7 * 4096 # exclusive + +# 1) Load the LM (LoRA already merged into 16-bit weights at training time) +model, tokenizer = FastLanguageModel.from_pretrained( + model_name = MODEL_ID, + max_seq_length = 4096, + dtype = None, # auto bf16 / fp16 + load_in_4bit = False, # set True to halve VRAM at slight quality cost + token = os.environ.get("HF_TOKEN"), +) +FastLanguageModel.for_inference(model) + +# 2) Load SNAC decoder (CPU is fine — frees GPU for the LM) +snac_model = SNAC.from_pretrained("hubertsiuzdak/snac_24khz").to("cpu") + + +def _redistribute_codes(code_list: list[int]) -> torch.Tensor: + layer_1, layer_2, layer_3 = [], [], [] + for i in range(len(code_list) // 7): + layer_1.append(code_list[7*i]) + layer_2.append(code_list[7*i + 1] - 4096) + layer_3.append(code_list[7*i + 2] - 2*4096) + layer_3.append(code_list[7*i + 3] - 3*4096) + layer_2.append(code_list[7*i + 4] - 4*4096) + layer_3.append(code_list[7*i + 5] - 5*4096) + layer_3.append(code_list[7*i + 6] - 6*4096) + if not layer_1: + return torch.zeros(1, 1, 12000) # ~0.5s silence fallback + clamp = lambda vals: [max(0, min(4095, v)) for v in vals] + codes = [torch.tensor(clamp(layer_1)).unsqueeze(0), + torch.tensor(clamp(layer_2)).unsqueeze(0), + torch.tensor(clamp(layer_3)).unsqueeze(0)] + return snac_model.decode(codes) + + +def synthesize(text: str, speaker_id: str = SPEAKER_ID, + *, max_new_tokens: int = 1200, + temperature: float = 0.6, top_p: float = 0.95, + repetition_penalty: float = 1.1, + seed: int | None = None) -> np.ndarray: + if seed is not None: + torch.manual_seed(seed) + + # Build prompt: [SOH] + tokenizer("speaker_id: text") + [EOT, EOH] + tagged = f"{speaker_id}: {text}" + text_ids = tokenizer(tagged, return_tensors="pt").input_ids + soh = torch.tensor([[START_OF_HUMAN]], dtype=torch.int64) + end = torch.tensor([[END_OF_TEXT, END_OF_HUMAN]], dtype=torch.int64) + input_ids = torch.cat([soh, text_ids, end], dim=1).to("cuda") + attention_mask = torch.ones_like(input_ids) + + generated = model.generate( + input_ids = input_ids, attention_mask = attention_mask, + max_new_tokens = max_new_tokens, + do_sample = True, + temperature = temperature, top_p = top_p, + repetition_penalty = repetition_penalty, + eos_token_id = END_OF_SPEECH, use_cache = True, + ) + + # Crop on last SOS, filter to audio token range, redistribute, decode + sos_indices = (generated == START_OF_SPEECH).nonzero(as_tuple=True) + cropped = generated[:, sos_indices[1][-1].item() + 1:] if len(sos_indices[1]) > 0 else generated + row = cropped[0] + audio_only = row[(row >= AUDIO_TOKEN_LO) & (row < AUDIO_TOKEN_HI)] + n = (audio_only.size(0) // 7) * 7 + code_list = [t.item() - AUDIO_TOKEN_LO for t in audio_only[:n]] + waveform = _redistribute_codes(code_list) + return waveform.detach().squeeze().to("cpu").numpy().astype(np.float32) + + +# 3) Use it +wav = synthesize("Mwattu, Mukama yeebazibwe.", seed=42) +sf.write("output.wav", wav, 24000) +print(f"saved {len(wav)/24000:.2f}s of audio at 24 kHz") +``` + +### Option B — `vllm` (high throughput, batched, deployment) + +Best for serving traffic. PagedAttention + continuous batching gives +roughly **5–10× faster** single-request latency and **10–100× higher** +throughput on batched requests vs. the `transformers` path. + +> **Important:** vLLM ships its own torch/transformers and conflicts +> with Unsloth's pinned versions. Use a fresh Python environment for +> vLLM serving — do not install on top of an Unsloth env. + +**Install:** + +```bash +pip install vllm snac soundfile torchcodec "datasets>=3.4.1,<4.0.0" +``` + +**Run:** + +```python +import os +import numpy as np +import torch +import soundfile as sf +from snac import SNAC +from transformers import AutoTokenizer +from vllm import LLM, SamplingParams + +MODEL_ID = "sunbird/orpheus-3b-tts-salt-lug-0001" +SPEAKER_ID = "salt_lug_0001" + +END_OF_TEXT = 128009 +START_OF_SPEECH = 128257 +END_OF_SPEECH = 128258 +START_OF_HUMAN = 128259 +END_OF_HUMAN = 128260 +AUDIO_TOKEN_LO = 128266 +AUDIO_TOKEN_HI = 128266 + 7 * 4096 + +# 1) Load LM into vLLM +llm = LLM( + model = MODEL_ID, + dtype = "bfloat16", + max_model_len = 4096, + gpu_memory_utilization = 0.85, +) +tokenizer = AutoTokenizer.from_pretrained(MODEL_ID, token=os.environ.get("HF_TOKEN")) +snac_model = SNAC.from_pretrained("hubertsiuzdak/snac_24khz").to("cpu") + + +def _build_prompt_token_ids(text: str, speaker_id: str) -> list[int]: + tagged = f"{speaker_id}: {text}" + text_ids = tokenizer.encode(tagged, add_special_tokens=True) + return [START_OF_HUMAN] + text_ids + [END_OF_TEXT, END_OF_HUMAN] + + +def _codes_to_waveform(generated_token_ids: list[int]) -> np.ndarray: + ids = torch.tensor(generated_token_ids, dtype=torch.int64) + sos_pos = (ids == START_OF_SPEECH).nonzero(as_tuple=True)[0] + if len(sos_pos) > 0: + ids = ids[sos_pos[-1].item() + 1:] + audio = ids[(ids >= AUDIO_TOKEN_LO) & (ids < AUDIO_TOKEN_HI)] + n = (audio.size(0) // 7) * 7 + cl = [t.item() - AUDIO_TOKEN_LO for t in audio[:n]] + l1, l2, l3 = [], [], [] + for i in range(len(cl) // 7): + l1.append(cl[7*i]) + l2.append(cl[7*i+1] - 4096); l3.append(cl[7*i+2] - 2*4096) + l3.append(cl[7*i+3] - 3*4096); l2.append(cl[7*i+4] - 4*4096) + l3.append(cl[7*i+5] - 5*4096); l3.append(cl[7*i+6] - 6*4096) + if not l1: + return np.zeros(12000, dtype=np.float32) + cb = lambda v: [max(0, min(4095, x)) for x in v] + codes = [torch.tensor(cb(l1)).unsqueeze(0), + torch.tensor(cb(l2)).unsqueeze(0), + torch.tensor(cb(l3)).unsqueeze(0)] + return snac_model.decode(codes).detach().squeeze().cpu().numpy().astype(np.float32) + + +def synthesize(text: str, speaker_id: str = SPEAKER_ID, + *, max_tokens: int = 1200, + temperature: float = 0.6, top_p: float = 0.95, + repetition_penalty: float = 1.1, + seed: int | None = None) -> np.ndarray: + sp = SamplingParams( + temperature = temperature, top_p = top_p, + repetition_penalty = repetition_penalty, + max_tokens = max_tokens, + stop_token_ids = [END_OF_SPEECH], + skip_special_tokens = False, # we need raw token_ids for SNAC + seed = seed, + ) + pids = _build_prompt_token_ids(text, speaker_id) + out = llm.generate([{"prompt_token_ids": pids}], sp) + return _codes_to_waveform(list(out[0].outputs[0].token_ids)) + + +def synthesize_batch(items: list[dict], **kwargs) -> list[np.ndarray]: + """items: list of {"text": str, "speaker_id": str (optional)}""" + sp = SamplingParams( + temperature = kwargs.get("temperature", 0.6), + top_p = kwargs.get("top_p", 0.95), + repetition_penalty = kwargs.get("repetition_penalty", 1.1), + max_tokens = kwargs.get("max_tokens", 1200), + stop_token_ids = [END_OF_SPEECH], + skip_special_tokens = False, + seed = kwargs.get("seed"), + ) + prompts = [{"prompt_token_ids": _build_prompt_token_ids( + it["text"], it.get("speaker_id", SPEAKER_ID))} + for it in items] + outputs = llm.generate(prompts, sp) + return [_codes_to_waveform(list(o.outputs[0].token_ids)) for o in outputs] + + +# 2) Use it — single +wav = synthesize("Mwattu, Mukama yeebazibwe.", seed=42) +sf.write("output.wav", wav, 24000) + +# 3) Use it — batched (much higher throughput) +items = [ + {"text": "Mwattu, oli otya?"}, + {"text": "Webale nyo okwagala Uganda."}, + {"text": "Tunaagenda mu Kampala olwa leero."}, +] +wavs = synthesize_batch(items, seed=123) +for i, w in enumerate(wavs): + sf.write(f"batch_{i:02d}.wav", w, 24000) +``` + +### Generation parameters + +| Param | Default | What it does | +|---|---|---| +| `temperature` | 0.6 | Lower = more deterministic, slightly flatter prosody. | +| `top_p` | 0.95 | Nucleus sampling. Don't drop below 0.9 — produces robotic audio. | +| `repetition_penalty` | 1.1 | Discourages stuck-on-one-frame artefacts. 1.0 disables it. | +| `max_new_tokens` / `max_tokens` | 1200 | ≈ 9–10 s of audio. Raise for longer utterances. | +| `seed` | `None` | Pass an int for reproducible output across runs. | + +--- + +## Token format + +The tokenizer is Llama-3's, with Orpheus's audio-codebook special tokens +laid out above the standard text vocabulary: + +| Token | ID | Purpose | +|---|---|---| +| `<\|begin_of_text\|>` | 128000 | Llama-3 BOS (auto-prepended by tokenizer) | +| `<\|end_of_text\|>` | 128009 | end of human turn (text portion) | +| `START_OF_SPEECH` | 128257 | model emits this just before audio codes | +| `END_OF_SPEECH` | 128258 | model emits this when it finishes — used as `eos_token_id` / `stop_token_ids` | +| `START_OF_HUMAN` | 128259 | wrap the text prompt | +| `END_OF_HUMAN` | 128260 | wrap the text prompt | +| `START_OF_AI` | 128261 | model emits this to begin its response | +| `END_OF_AI` | 128262 | model emits this when fully done | +| `PAD_TOKEN` | 128263 | left-padding for batched generation | +| audio codebook | 128266 + N·4096 | SNAC codes, N ∈ {0..6} for 7-frame layout | + +**Training prompt structure** (and what the model expects at inference): + +``` +[SOH] + tokenize("salt_lug_0001: ") + [EOT] + [EOH] +↳ model autoregressively emits: +[SOA] + [SOS] + audio_codes... + [EOS] + [EOA] +``` + +To recover audio: find the **last** `START_OF_SPEECH` (128257) in the +output, take everything after it, drop any token outside the audio +codebook range, group into 7-token frames, undo the per-position offsets, +and feed the three layers to `SNAC.decode`. Both inference snippets above +implement this end-to-end. + +--- + +## Training details + +| Setting | Value | +|---|---| +| Base model | `unsloth/orpheus-3b-0.1-pretrained` (raw pretrained, not the `-ft` voice-actor variant) | +| Adapter | LoRA r=64, α=64, dropout=0, bias=none | +| Target modules | `q_proj`, `k_proj`, `v_proj`, `o_proj`, `gate_proj`, `up_proj`, `down_proj` | +| Optimizer | `adamw_8bit`, weight decay 0.001 | +| LR schedule | linear, lr=2e-4, warmup steps=5 | +| Per-device batch size | 1 (with `gradient_accumulation_steps=4`, effective batch = 4) | +| Epochs | 3 | +| `max_seq_length` | 4096 | +| Precision | bfloat16 weights, 16-bit LoRA | +| Seed | 3407 | +| Hardware | single NVIDIA RTX 4090 (24 GB) | +| Gradient checkpointing | Unsloth's optimised variant | +| Final save | LoRA merged into 16-bit weights via `save_pretrained_merged(save_method="merged_16bit")` | + +The pretrained variant of Orpheus was chosen over the `-ft` voice-actor +variant because that variant has a strong English-voice-actor prior that +fights low-resource-language fine-tuning. + +### Data prep summary + +1. Load `Sunbird/tts` config `lug`, splits `train` and `test`. +2. Filter both splits to `speaker_id == "salt_lug_0001"`. +3. Tag each row with `source = "salt_lug_0001"` (multi-speaker prompt + format — every row carries the speaker tag, so adding more speakers + later is continued training, not a re-architecture). +4. Cast `audio` column to 24 kHz via `Audio(sampling_rate=24000)`. +5. Drop rows whose tokenised text alone exceeds `max_seq_length`. +6. Encode each audio clip with `hubertsiuzdak/snac_24khz` → 7 codes per + frame, flattened with per-layer offsets `(+128266, +4096, +2·4096, …)`. +7. Filter out rows with empty/None codes; drop consecutive duplicate + frames. +8. Build `input_ids = [SOH] + text_ids + [EOT] + [EOH] + [SOA] + [SOS] + audio_codes + [EOS] + [EOA]`. +9. Drop rows whose total tokenised length still exceeds `max_seq_length`. + +--- + +## Evaluation + +Quality was evaluated qualitatively on the held-out `test` split for +`salt_lug_0001` (un-seen utterances, same speaker). Generated audio is +compared A/B against the ground-truth recordings; samples are saved to +`outputs/inference_samples/` by both the +[`Orpheus_3B_Sunbird_Luganda.ipynb`](https://github.com/SunbirdAI/Qwen3-TTS/blob/main/orpheus-3B/Orpheus_3B_Sunbird_Luganda.ipynb) +training notebook and the +[`Orpheus_3B_Sunbird_Luganda_Inference.ipynb`](https://github.com/SunbirdAI/Qwen3-TTS/blob/main/orpheus-3B/Orpheus_3B_Sunbird_Luganda_Inference.ipynb) +inference notebook. + +We did **not** run automated metrics (WER on a downstream STT, MOS +prediction, etc.) for this release. Numbers will be added if/when those +become part of the evaluation pipeline. + +--- + +## Intended uses & out-of-scope + +**Intended:** + +- Luganda voice synthesis for accessibility, language learning, + human–computer interaction, audio content creation, and downstream + speech research on low-resource Bantu languages. +- A reference checkpoint for the Sunbird/tts → Orpheus-3B fine-tuning + pipeline; reproducible training recipe in the + [companion notebooks](https://github.com/SunbirdAI/Qwen3-TTS/tree/main/orpheus-3B). + +**Out of scope:** + +- **Voice impersonation / deception.** The model imitates the timbre of a + consenting Sunbird voice donor (`salt_lug_0001`). Do not use the + generated audio to impersonate identifiable real persons or to + produce content that could mislead listeners about who is speaking. +- **High-stakes decisions.** Generated speech may contain pronunciation + errors, prosodic artefacts, or hallucinated phrases — do not deploy + in safety-critical contexts (medical, legal, emergency) without + human review. +- **Languages other than Luganda.** This single-speaker checkpoint was + trained only on `salt_lug_0001` Luganda data. For other Sunbird + languages or speakers, see `sunbird/orpheus-3b-tts-multilingual` + (when available) or fine-tune your own from the same recipe. + +--- + +## Limitations & risks + +- **Single-speaker.** Only `salt_lug_0001` is faithfully reproduced. The + prompt format supports `f"{speaker_id}: {text}"` but the model has not + seen any other speaker_id during training. +- **Vocabulary coverage.** Limited to the lexicon present in the + `Sunbird/tts (lug)` training subset (~hundreds of utterances). + Unfamiliar words, code-switching with English, and out-of-distribution + proper nouns may produce artefacts. +- **Long utterances.** The model was trained on utterances up to ~16 s + of audio (`max_seq_length=4096`). Generation may degrade or truncate + beyond ~10 s of speech. +- **Sampling variance.** With `do_sample=True`, identical prompts can + produce noticeably different deliveries between runs. Pass `seed=` for + reproducibility. +- **No emotion/style control.** Unlike the upstream `orpheus-3b-0.1-ft`, + this fine-tune was not exposed to in-text emotion tags + (``, ``, …). Such tags will be tokenised as ordinary + text and produce no special prosodic effect. +- **Bias.** Inherits any biases present in the Sunbird/tts corpus and in + Llama-3's pretraining; we have not audited these systematically. + +--- + +## Hardware requirements + +| Mode | Min VRAM | Recommended | +|---|---|---| +| `transformers` + Unsloth, fp16 | 8 GB (with `load_in_4bit=True`) | 16 GB | +| `transformers` + Unsloth, bf16 | 14 GB | 24 GB | +| vLLM, bf16, `max_model_len=4096` | 14 GB | 24 GB | + +Audio decoding via SNAC runs on CPU and adds ~50–150 ms per utterance. + +--- + +## License & attribution + +This fine-tune is released under **Apache-2.0**, matching the upstream +[`unsloth/orpheus-3b-0.1-pretrained`](https://huggingface.co/unsloth/orpheus-3b-0.1-pretrained) +license. It transitively inherits obligations from: + +- The [Orpheus-TTS](https://github.com/canopyai/Orpheus-TTS) project (CanopyAI). +- The [Llama-3](https://llama.meta.com/llama3/) base architecture and weights — Meta Llama 3 Community License. +- The [SNAC](https://github.com/hubertsiuzdak/snac) audio codec (Hubert Siuzdak, MIT). +- The [`Sunbird/tts`](https://huggingface.co/datasets/Sunbird/tts) dataset. + +If you redistribute the merged weights, please carry these attributions +forward. + +--- + +## Citation + +If you use this model in your work, please cite both the dataset and the +fine-tuning project: + +```bibtex +@misc{sunbird_orpheus3b_lug_2026, + title = {Orpheus-3B Sunbird Luganda TTS (salt_lug_0001)}, + author = {Sunbird AI}, + year = {2026}, + howpublished = {\url{https://huggingface.co/sunbird/orpheus-3b-tts-salt-lug-0001}}, +} + +@misc{sunbird_tts_dataset, + title = {Sunbird Speech Dataset}, + author = {Sunbird AI}, + howpublished = {\url{https://huggingface.co/datasets/Sunbird/tts}}, +} + +@misc{orpheus_tts_2025, + title = {Orpheus-TTS}, + author = {Canopy Labs}, + year = {2025}, + howpublished = {\url{https://github.com/canopyai/Orpheus-TTS}}, +} +``` + +--- + +## Adapting this card + +If you fine-tune additional speakers / languages from the same recipe, +the card above is mostly drop-in — just swap: + +- `MODEL_ID` and `SPEAKER_ID` constants in both code blocks. +- The `language:` field in the YAML metadata. +- The "Training data summary" section. +- The dataset citation. + +For the multi-speaker variant trained on the full `Sunbird/tts` corpus, +also remove the "Single-speaker" limitation note and add a row to the +prompt examples showing several speaker_ids in one batch. diff --git a/modal-deploy/orpheus-3B/MODEL_CARD_MULTILINGUAL.md b/modal-deploy/orpheus-3B/MODEL_CARD_MULTILINGUAL.md new file mode 100644 index 00000000..2efb14ea --- /dev/null +++ b/modal-deploy/orpheus-3B/MODEL_CARD_MULTILINGUAL.md @@ -0,0 +1,663 @@ +--- +license: apache-2.0 +language: + - ach + - af + - en + - ee + - ff + - ha + - ig + - ki + - rw + - lgg + - ln + - lg + - luo + - nyn + - st + - sw + - teo + - tn + - xh + - yo +library_name: transformers +pipeline_tag: text-to-speech +tags: + - text-to-speech + - tts + - orpheus + - multilingual + - multi-speaker + - african-languages + - low-resource-language + - sunbird + - snac + - unsloth +datasets: + - Sunbird/tts +base_model: unsloth/orpheus-3b-0.1-pretrained +--- + +# Orpheus-3B Sunbird Multilingual TTS + +A multilingual, multi-speaker text-to-speech model fine-tuned from +[`unsloth/orpheus-3b-0.1-pretrained`](https://huggingface.co/unsloth/orpheus-3b-0.1-pretrained) +on the **full** [`Sunbird/tts`](https://huggingface.co/datasets/Sunbird/tts) +corpus — 20 language configurations and every speaker present in the +dataset. + +The model accepts arbitrary text and emits 24 kHz mono speech via the +[SNAC](https://huggingface.co/hubertsiuzdak/snac_24khz) audio codec. +Voice selection happens at the prompt level: prepend the chosen +`speaker_id` followed by `": "` to your text, and the model produces +audio in that speaker's voice. + +## Quick links + +- **Base model:** [`unsloth/orpheus-3b-0.1-pretrained`](https://huggingface.co/unsloth/orpheus-3b-0.1-pretrained) (Llama-3 architecture) +- **Audio codec:** [`hubertsiuzdak/snac_24khz`](https://huggingface.co/hubertsiuzdak/snac_24khz) (24 kHz, 7 codes per ~12 ms frame) +- **Training dataset:** [`Sunbird/tts`](https://huggingface.co/datasets/Sunbird/tts) — all 20 configs, all speakers +- **Training framework:** [Unsloth](https://github.com/unslothai/unsloth) + HuggingFace Trainer + +## Languages covered + +Speaker IDs encode both the source corpus (`salt_*`, `waxal_*`, `slr32_*`, +`slr129_*`, `bateesa_*`) and the language. Languages marked with an em dash +in the Speaker IDs column are present in the model's training mix but do +not currently expose individual voice IDs in this checkpoint. + +| Config | Language | ISO 639-1 | Region | Speaker IDs | +|---|---|---|---|---| +| `ach` | Acholi | — | Uganda, South Sudan | `salt_ach_0001`
`waxal_ach_0001`
`waxal_ach_0005`
`waxal_ach_0006`
`waxal_ach_0008` | +| `afr` | Afrikaans | af | South Africa, Namibia | `slr32_afr_0009` | +| `eng` | English | en | (control language) | `salt_eng_0001`
`salt_eng_0002`
`salt_eng_0003` | +| `ewe` | Ewe | ee | Ghana, Togo | `slr129_ewe_0001` | +| `ful` | Fulah | ff | West Africa (Sahel) | `waxal_ful_0003`
`waxal_ful_0004`
`waxal_ful_0006` | +| `hau` | Hausa | ha | Nigeria, Niger, Chad | `waxal_hau_0004`
`waxal_hau_0006`
`waxal_hau_0007`
`waxal_hau_0008` | +| `ibo` | Igbo | ig | Nigeria | `waxal_ibo_0003`
`waxal_ibo_0005`
`waxal_ibo_0008` | +| `kik` | Kikuyu | ki | Kenya | `waxal_kik_0003`
`waxal_kik_0004` | +| `kin` | Kinyarwanda | rw | Rwanda | `bateesa_kin_0001` | +| `lgg` | Lugbara | — | Uganda, DRC | — | +| `lin` | Lingala | ln | DRC, Republic of Congo | `slr129_lin_0001` | +| `lug` | Luganda | lg | Uganda | `salt_lug_0001`
`waxal_lug_0002`
`waxal_lug_0003`
`waxal_lug_0004`
`waxal_lug_0005`
`waxal_lug_0006`
`waxal_lug_0007`
`waxal_lug_0008` | +| `luo` | Luo (Dholuo) | — | Kenya, Tanzania | `waxal_luo_0001`
`waxal_luo_0002`
`waxal_luo_0003`
`waxal_luo_0004` | +| `nyn` | Runyankole | — | Uganda | `salt_nyn_0001`
`waxal_nyn_0003`
`waxal_nyn_0004`
`waxal_nyn_0007`
`waxal_nyn_0008` | +| `sot` | Sesotho | st | Lesotho, South Africa | — | +| `swa` | Swahili | sw | East Africa | `waxal_swa_0006`
`waxal_swa_0007` | +| `teo` | Ateso | — | Uganda, Kenya | `salt_teo_0001` | +| `tsn` | Setswana | tn | Botswana, South Africa | — | +| `xho` | Xhosa | xh | South Africa | `slr32_xho_0012` | +| `yor` | Yoruba | yo | Nigeria, Benin | `waxal_yor_0002`
`waxal_yor_0006`
`waxal_yor_0008` | + +Per-language quality scales with the amount of training data Sunbird +collected for that language; some configs have many more speaker hours +than others. **Audition the test split** for each language before relying +on a particular speaker — see the discovery snippet below. + +## TL;DR + +```python +# After installing the dependencies (see "Inference" below) +wav = synthesize("Mwattu, oli otya?", speaker_id="salt_lug_0001") # Luganda +wav = synthesize("Habari yako rafiki.", speaker_id="waxal_swa_0006") # Swahili +wav = synthesize("Bawo ni, ọrẹ mi?", speaker_id="waxal_yor_0002") # Yoruba +``` + +The model has no explicit "language" knob — the language identity +travels via the speaker tag, since each `__` voice +(`salt_*`, `waxal_*`, `slr32_*`, `slr129_*`, `bateesa_*`) was recorded in +exactly one language. + +--- + +## Discovering speaker IDs + +The exact speaker_ids in each config can be enumerated from the dataset: + +```python +from collections import defaultdict +from datasets import load_dataset, get_dataset_config_names + +CONFIGS = get_dataset_config_names("Sunbird/tts") # the 20 languages + +speakers_by_lang = defaultdict(set) +for cfg in CONFIGS: + ds = load_dataset("Sunbird/tts", cfg, split="train") + for sid in ds["speaker_id"]: + speakers_by_lang[cfg].add(sid) + +for cfg, sids in sorted(speakers_by_lang.items()): + print(f"{cfg}: {len(sids)} speaker(s) — {sorted(sids)[:3]}{'...' if len(sids) > 3 else ''}") +``` + +Speaker IDs encode both the source corpus and the language — e.g. +`salt_lug_0001` (a Luganda speaker from SALT), `waxal_yor_0002` (a Yoruba +speaker from WaxalNLP), `slr32_xho_0012` (an OpenSLR Xhosa speaker), +`bateesa_kin_0001` (a Kinyarwanda speaker from the Bateesa corpus). See +the **Languages covered** table above for the full list of IDs currently +exposed by this checkpoint. Pass any one of them as `speaker_id` to either +inference function below. + +--- + +## Inference + +The model wraps every prompt in a multi-speaker tagged format: + +``` +[SOH] + tokenize(": ") + [EOT, EOH] +``` + +and the model autoregressively emits Llama-3 special tokens followed by +SNAC audio codes that decode to a 24 kHz waveform. Two reference +implementations follow. + +### Option A — `transformers` + `unsloth` (single request) + +Best for development, notebook-driven iteration, and small batch sizes. + +**Install:** + +```bash +pip install unsloth snac soundfile torchcodec "datasets>=3.4.1,<4.0.0" +``` + +**Run:** + +```python +import os +import numpy as np +import torch +import soundfile as sf +from unsloth import FastLanguageModel +from snac import SNAC + +MODEL_ID = "sunbird/orpheus-3b-tts-multilingual" + +# Special tokens — must match the training format +END_OF_TEXT = 128009 +START_OF_SPEECH = 128257 +END_OF_SPEECH = 128258 +START_OF_HUMAN = 128259 +END_OF_HUMAN = 128260 +PAD_TOKEN = 128263 +AUDIO_TOKEN_LO = 128266 +AUDIO_TOKEN_HI = 128266 + 7 * 4096 # exclusive + +# 1) Load the LM (LoRA already merged into 16-bit weights at training time) +model, tokenizer = FastLanguageModel.from_pretrained( + model_name = MODEL_ID, + max_seq_length = 4096, + dtype = None, # auto bf16 / fp16 + load_in_4bit = False, # set True to halve VRAM at slight quality cost + token = os.environ.get("HF_TOKEN"), +) +FastLanguageModel.for_inference(model) + +# 2) Load SNAC decoder (CPU is fine — frees GPU for the LM) +snac_model = SNAC.from_pretrained("hubertsiuzdak/snac_24khz").to("cpu") + + +def _redistribute_codes(code_list: list[int]) -> torch.Tensor: + layer_1, layer_2, layer_3 = [], [], [] + for i in range(len(code_list) // 7): + layer_1.append(code_list[7*i]) + layer_2.append(code_list[7*i + 1] - 4096) + layer_3.append(code_list[7*i + 2] - 2*4096) + layer_3.append(code_list[7*i + 3] - 3*4096) + layer_2.append(code_list[7*i + 4] - 4*4096) + layer_3.append(code_list[7*i + 5] - 5*4096) + layer_3.append(code_list[7*i + 6] - 6*4096) + if not layer_1: + return torch.zeros(1, 1, 12000) # ~0.5s silence fallback + clamp = lambda vals: [max(0, min(4095, v)) for v in vals] + codes = [torch.tensor(clamp(layer_1)).unsqueeze(0), + torch.tensor(clamp(layer_2)).unsqueeze(0), + torch.tensor(clamp(layer_3)).unsqueeze(0)] + return snac_model.decode(codes) + + +def synthesize(text: str, speaker_id: str, + *, max_new_tokens: int = 1200, + temperature: float = 0.6, top_p: float = 0.95, + repetition_penalty: float = 1.1, + seed: int | None = None) -> np.ndarray: + """Synthesize speech for `text` in the voice of `speaker_id`. + + `speaker_id` must be one of the speakers seen during training, + e.g. "salt_lug_0001" (Luganda) or "waxal_swa_0007" (Swahili). + """ + if seed is not None: + torch.manual_seed(seed) + + tagged = f"{speaker_id}: {text}" + text_ids = tokenizer(tagged, return_tensors="pt").input_ids + soh = torch.tensor([[START_OF_HUMAN]], dtype=torch.int64) + end = torch.tensor([[END_OF_TEXT, END_OF_HUMAN]], dtype=torch.int64) + input_ids = torch.cat([soh, text_ids, end], dim=1).to("cuda") + attention_mask = torch.ones_like(input_ids) + + generated = model.generate( + input_ids = input_ids, attention_mask = attention_mask, + max_new_tokens = max_new_tokens, + do_sample = True, + temperature = temperature, top_p = top_p, + repetition_penalty = repetition_penalty, + eos_token_id = END_OF_SPEECH, use_cache = True, + ) + + # Crop on last SOS, filter to audio token range, redistribute, decode + sos_indices = (generated == START_OF_SPEECH).nonzero(as_tuple=True) + cropped = generated[:, sos_indices[1][-1].item() + 1:] if len(sos_indices[1]) > 0 else generated + row = cropped[0] + audio_only = row[(row >= AUDIO_TOKEN_LO) & (row < AUDIO_TOKEN_HI)] + n = (audio_only.size(0) // 7) * 7 + code_list = [t.item() - AUDIO_TOKEN_LO for t in audio_only[:n]] + waveform = _redistribute_codes(code_list) + return waveform.detach().squeeze().to("cpu").numpy().astype(np.float32) + + +# 3) Use it — pick a speaker per language +wav = synthesize("Mwattu, Mukama yeebazibwe.", speaker_id="salt_lug_0001", seed=42) +sf.write("luganda.wav", wav, 24000) + +wav = synthesize("Habari yako rafiki.", speaker_id="waxal_swa_0006", seed=42) +sf.write("swahili.wav", wav, 24000) +``` + +### Option B — `vllm` (high throughput, batched, deployment) + +Best for serving traffic. PagedAttention + continuous batching gives +roughly **5–10× faster** single-request latency and **10–100× higher** +throughput on batched requests vs. the `transformers` path. Multi-speaker +batching (different `speaker_id`s in one call) gets the full benefit. + +> **Important:** vLLM ships its own torch/transformers and conflicts +> with Unsloth's pinned versions. Use a fresh Python environment for +> vLLM serving — do not install on top of an Unsloth env. + +**Install:** + +```bash +pip install vllm snac soundfile torchcodec "datasets>=3.4.1,<4.0.0" +``` + +**Run:** + +```python +import os +import numpy as np +import torch +import soundfile as sf +from snac import SNAC +from transformers import AutoTokenizer +from vllm import LLM, SamplingParams + +MODEL_ID = "sunbird/orpheus-3b-tts-multilingual" + +END_OF_TEXT = 128009 +START_OF_SPEECH = 128257 +END_OF_SPEECH = 128258 +START_OF_HUMAN = 128259 +END_OF_HUMAN = 128260 +AUDIO_TOKEN_LO = 128266 +AUDIO_TOKEN_HI = 128266 + 7 * 4096 + +# 1) Load +llm = LLM( + model = MODEL_ID, + dtype = "bfloat16", + max_model_len = 4096, + gpu_memory_utilization = 0.85, +) +tokenizer = AutoTokenizer.from_pretrained(MODEL_ID, token=os.environ.get("HF_TOKEN")) +snac_model = SNAC.from_pretrained("hubertsiuzdak/snac_24khz").to("cpu") + + +def _build_prompt_token_ids(text: str, speaker_id: str) -> list[int]: + tagged = f"{speaker_id}: {text}" + text_ids = tokenizer.encode(tagged, add_special_tokens=True) + return [START_OF_HUMAN] + text_ids + [END_OF_TEXT, END_OF_HUMAN] + + +def _codes_to_waveform(generated_token_ids: list[int]) -> np.ndarray: + ids = torch.tensor(generated_token_ids, dtype=torch.int64) + sos_pos = (ids == START_OF_SPEECH).nonzero(as_tuple=True)[0] + if len(sos_pos) > 0: + ids = ids[sos_pos[-1].item() + 1:] + audio = ids[(ids >= AUDIO_TOKEN_LO) & (ids < AUDIO_TOKEN_HI)] + n = (audio.size(0) // 7) * 7 + cl = [t.item() - AUDIO_TOKEN_LO for t in audio[:n]] + l1, l2, l3 = [], [], [] + for i in range(len(cl) // 7): + l1.append(cl[7*i]) + l2.append(cl[7*i+1] - 4096); l3.append(cl[7*i+2] - 2*4096) + l3.append(cl[7*i+3] - 3*4096); l2.append(cl[7*i+4] - 4*4096) + l3.append(cl[7*i+5] - 5*4096); l3.append(cl[7*i+6] - 6*4096) + if not l1: + return np.zeros(12000, dtype=np.float32) + cb = lambda v: [max(0, min(4095, x)) for x in v] + codes = [torch.tensor(cb(l1)).unsqueeze(0), + torch.tensor(cb(l2)).unsqueeze(0), + torch.tensor(cb(l3)).unsqueeze(0)] + return snac_model.decode(codes).detach().squeeze().cpu().numpy().astype(np.float32) + + +def synthesize(text: str, speaker_id: str, + *, max_tokens: int = 1200, + temperature: float = 0.6, top_p: float = 0.95, + repetition_penalty: float = 1.1, + seed: int | None = None) -> np.ndarray: + sp = SamplingParams( + temperature = temperature, top_p = top_p, + repetition_penalty = repetition_penalty, + max_tokens = max_tokens, + stop_token_ids = [END_OF_SPEECH], + skip_special_tokens = False, + seed = seed, + ) + pids = _build_prompt_token_ids(text, speaker_id) + out = llm.generate([{"prompt_token_ids": pids}], sp) + return _codes_to_waveform(list(out[0].outputs[0].token_ids)) + + +def synthesize_batch(items: list[dict], **kwargs) -> list[np.ndarray]: + """items: list of {"text": str, "speaker_id": str} — different speakers + can be mixed in one batch.""" + sp = SamplingParams( + temperature = kwargs.get("temperature", 0.6), + top_p = kwargs.get("top_p", 0.95), + repetition_penalty = kwargs.get("repetition_penalty", 1.1), + max_tokens = kwargs.get("max_tokens", 1200), + stop_token_ids = [END_OF_SPEECH], + skip_special_tokens = False, + seed = kwargs.get("seed"), + ) + prompts = [{"prompt_token_ids": _build_prompt_token_ids(it["text"], it["speaker_id"])} + for it in items] + outputs = llm.generate(prompts, sp) + return [_codes_to_waveform(list(o.outputs[0].token_ids)) for o in outputs] + + +# 2) Single — pick a speaker per language +wav = synthesize("Mwattu, oli otya?", speaker_id="salt_lug_0001", seed=42) +sf.write("luganda.wav", wav, 24000) + +# 3) Batched — different languages and speakers in one GPU pass +items = [ + {"text": "Mwattu, oli otya?", "speaker_id": "salt_lug_0001"}, + {"text": "Habari yako rafiki.", "speaker_id": "waxal_swa_0006"}, + {"text": "Bawo ni, ọrẹ mi?", "speaker_id": "waxal_yor_0002"}, + {"text": "Sannu, ina kwana?", "speaker_id": "waxal_hau_0004"}, + {"text": "Goeie môre, hoe gaan dit?", "speaker_id": "slr32_afr_0009"}, +] +wavs = synthesize_batch(items, seed=123) +for i, (it, w) in enumerate(zip(items, wavs)): + sf.write(f"batch_{i:02d}_{it['speaker_id']}.wav", w, 24000) +``` + +### Generation parameters + +| Param | Default | What it does | +|---|---|---| +| `temperature` | 0.6 | Lower = more deterministic, slightly flatter prosody. | +| `top_p` | 0.95 | Nucleus sampling. Don't drop below 0.9 — produces robotic audio. | +| `repetition_penalty` | 1.1 | Discourages stuck-on-one-frame artefacts. 1.0 disables it. | +| `max_new_tokens` / `max_tokens` | 1200 | ≈ 9–10 s of audio. Raise for longer utterances. | +| `seed` | `None` | Pass an int for reproducible output across runs. | + +--- + +## Token format + +The tokenizer is Llama-3's, with Orpheus's audio-codebook special tokens +laid out above the standard text vocabulary: + +| Token | ID | Purpose | +|---|---|---| +| `<\|begin_of_text\|>` | 128000 | Llama-3 BOS (auto-prepended by tokenizer) | +| `<\|end_of_text\|>` | 128009 | end of human turn (text portion) | +| `START_OF_SPEECH` | 128257 | model emits this just before audio codes | +| `END_OF_SPEECH` | 128258 | model emits this when it finishes — used as `eos_token_id` / `stop_token_ids` | +| `START_OF_HUMAN` | 128259 | wrap the text prompt | +| `END_OF_HUMAN` | 128260 | wrap the text prompt | +| `START_OF_AI` | 128261 | model emits this to begin its response | +| `END_OF_AI` | 128262 | model emits this when fully done | +| `PAD_TOKEN` | 128263 | left-padding for batched generation | +| audio codebook | 128266 + N·4096 | SNAC codes, N ∈ {0..6} for 7-frame layout | + +**Training prompt structure** (and what the model expects at inference): + +``` +[SOH] + tokenize("__: ") + [EOT] + [EOH] +↳ model autoregressively emits: +[SOA] + [SOS] + audio_codes... + [EOS] + [EOA] +``` + +To recover audio: find the **last** `START_OF_SPEECH` (128257) in the +output, take everything after it, drop any token outside the audio +codebook range, group into 7-token frames, undo the per-position offsets, +and feed the three layers to `SNAC.decode`. Both inference snippets above +implement this end-to-end. + +--- + +## Training details + +| Setting | Value | +|---|---| +| Base model | `unsloth/orpheus-3b-0.1-pretrained` (raw pretrained, not the `-ft` voice-actor variant) | +| Adapter | LoRA r=64, α=64, dropout=0, bias=none | +| Target modules | `q_proj`, `k_proj`, `v_proj`, `o_proj`, `gate_proj`, `up_proj`, `down_proj` | +| Optimizer | `adamw_8bit`, weight decay 0.001 | +| LR schedule | linear, lr=2e-4, warmup steps=5 | +| Per-device batch size | 1 (with `gradient_accumulation_steps=4`, effective batch = 4) | +| Epochs | 3 | +| `max_seq_length` | 4096 | +| `save_total_limit` | 2 | +| Precision | bfloat16 weights, 16-bit LoRA | +| Seed | 3407 | +| Hardware | single NVIDIA RTX 4090 (24 GB) | +| Gradient checkpointing | Unsloth's optimised variant | +| Final save | LoRA merged into 16-bit weights via `save_pretrained_merged(save_method="merged_16bit")` | + +The pretrained variant of Orpheus was chosen over the `-ft` voice-actor +variant because that variant has a strong English-voice-actor prior that +fights low-resource-language fine-tuning. + +### Data prep summary + +1. Load all 20 configs of `Sunbird/tts` (`get_dataset_config_names`) + and `concatenate_datasets` their `train` and `test` splits into one + training set and one held-out evaluation set. **No** speaker filter. +2. Tag each row with `source = example["speaker_id"]` (per-row, not + constant) — the model learns the multi-speaker prompt format + `f"{speaker_id}: {text}"` across every speaker it sees. +3. Cast `audio` to 24 kHz via `Audio(sampling_rate=24000)`. +4. Drop rows whose tokenised text alone exceeds `max_seq_length` — + saves expensive SNAC encoding on rows that would be filtered out + downstream. +5. Encode each remaining audio clip with `hubertsiuzdak/snac_24khz` → + 7 codes per frame, flattened with per-layer offsets + `(+128266, +4096, +2·4096, …)`. +6. Filter out rows with empty/None codes; drop consecutive duplicate + frames. +7. Build `input_ids = [SOH] + text_ids + [EOT] + [EOH] + [SOA] + [SOS] + audio_codes + [EOS] + [EOA]`. +8. Drop rows whose total tokenised length exceeds `max_seq_length` + (safety net for rows where text fits but text + audio together + overflow the budget). + +--- + +## Evaluation + +Quality was evaluated qualitatively on a diverse held-out test sample: +during training, up to 10 utterances are pulled from +`ds_test.shuffle(seed=42)` covering as many distinct speaker_ids as +possible. Generated audio is saved next to the ground-truth recording +under `inference_samples/sample__.wav` so each language +/ voice combination can be auditioned individually. + +We did **not** run automated metrics (WER on a downstream STT, MOS +prediction, language-confusion eval, etc.) for this release. Numbers +will be added if/when those become part of the evaluation pipeline. + +**Important caveat — quality varies by language.** The training corpus +is unbalanced across the 20 configs; languages with more speaker hours +in `Sunbird/tts` get more training signal and produce more natural +speech. Audition the per-language samples before relying on a specific +voice for production traffic. + +--- + +## Intended uses & out-of-scope + +**Intended:** + +- Multilingual voice synthesis for accessibility, language learning, + human–computer interaction, audio content creation, and downstream + speech research on the 20 covered languages. +- A reference checkpoint for the Sunbird/tts → Orpheus-3B multilingual + fine-tuning pipeline; reproducible training recipe in + [`Orpheus_3B_Sunbird_Multilingual.ipynb`](https://github.com/SunbirdAI/Qwen3-TTS/blob/main/orpheus-3B/Orpheus_3B_Sunbird_Multilingual.ipynb). + +**Out of scope:** + +- **Voice impersonation / deception.** The model imitates the timbres + of consenting Sunbird voice donors. Do not use the generated audio + to impersonate identifiable real persons or to produce content that + could mislead listeners about who is speaking. +- **High-stakes decisions.** Generated speech may contain pronunciation + errors, prosodic artefacts, or hallucinated phrases — do not deploy + in safety-critical contexts (medical, legal, emergency) without + human review. +- **Languages outside the 20 configs.** The model has no signal for + languages not present in `Sunbird/tts`; sending German text to any + speaker will produce garbled output, not "German with a Luganda + accent". +- **Code-switching.** Each speaker_id was recorded in a single language; + the model has not seen mixed-language utterances and will likely + produce phonetic artefacts at language boundaries within one prompt. +- **Cross-language voice transfer.** Sending Acholi text to + `salt_lug_0001` (a Luganda speaker_id) is undefined behaviour. The + model has no language-conditioning input separate from the speaker + tag, so language identity travels via the speaker_id. Use a speaker + whose `__` middle field matches the language of + your text (see the **Languages covered** table for the full list). + +--- + +## Limitations & risks + +- **Quality varies by language.** Per-language data volume in + `Sunbird/tts` is unbalanced. Languages with fewer hours produce + noticeably less natural speech. Run the per-language test-split + audit (script below) before committing to a particular voice. +- **No language conditioning.** There is no `language` token; the + model relies entirely on the speaker_id to disambiguate. Mismatching + speaker_id and text language is undefined behaviour (see above). +- **Vocabulary coverage.** Limited to the lexicon present in each + config's training subset. Unfamiliar words, code-switching, and + out-of-distribution proper nouns may produce artefacts. +- **Long utterances.** The model was trained on utterances up to ~16 s + of audio (`max_seq_length=4096`). Generation may degrade or + truncate beyond ~10 s of speech. +- **Sampling variance.** With `do_sample=True`, identical prompts can + produce noticeably different deliveries between runs. Pass `seed=` + for reproducibility. +- **No emotion/style control.** Unlike the upstream `orpheus-3b-0.1-ft`, + this fine-tune was not exposed to in-text emotion tags + (``, ``, …). Such tags will be tokenised as ordinary + text and produce no special prosodic effect. +- **Bias.** Inherits any biases present in the Sunbird/tts corpus and + in Llama-3's pretraining; we have not audited these systematically + per language. + +### Quick per-language audit script + +```python +from datasets import load_dataset, Audio, get_dataset_config_names +import soundfile as sf +from pathlib import Path + +CONFIGS = get_dataset_config_names("Sunbird/tts") +out_dir = Path("language_audit"); out_dir.mkdir(exist_ok=True) + +for cfg in CONFIGS: + ds = load_dataset("Sunbird/tts", cfg, split="test") + ds = ds.cast_column("audio", Audio(sampling_rate=24000)) + row = ds[0] + sid, text = row["speaker_id"], row["text"] + print(f"{cfg}: {sid} -> {text[:80]}") + wav = synthesize(text, speaker_id=sid, seed=0) + sf.write(out_dir / f"{cfg}_{sid}.wav", wav, 24000) + sf.write(out_dir / f"{cfg}_{sid}_groundtruth.wav", + row["audio"]["array"], 24000) +``` + +--- + +## Hardware requirements + +| Mode | Min VRAM | Recommended | +|---|---|---| +| `transformers` + Unsloth, fp16 | 8 GB (with `load_in_4bit=True`) | 16 GB | +| `transformers` + Unsloth, bf16 | 14 GB | 24 GB | +| vLLM, bf16, `max_model_len=4096` | 14 GB | 24 GB | + +Audio decoding via SNAC runs on CPU and adds ~50–150 ms per utterance. + +--- + +## License & attribution + +This fine-tune is released under **Apache-2.0**, matching the upstream +[`unsloth/orpheus-3b-0.1-pretrained`](https://huggingface.co/unsloth/orpheus-3b-0.1-pretrained) +license. It transitively inherits obligations from: + +- The [Orpheus-TTS](https://github.com/canopyai/Orpheus-TTS) project (CanopyAI). +- The [Llama-3](https://llama.meta.com/llama3/) base architecture and weights — Meta Llama 3 Community License. +- The [SNAC](https://github.com/hubertsiuzdak/snac) audio codec (Hubert Siuzdak, MIT). +- The [`Sunbird/tts`](https://huggingface.co/datasets/Sunbird/tts) dataset and the SALT voice donors who contributed recordings. + +If you redistribute the merged weights, please carry these attributions +forward. + +--- + +## Citation + +If you use this model in your work, please cite both the dataset and the +fine-tuning project: + +```bibtex +@misc{sunbird_orpheus3b_multilingual_2026, + title = {Orpheus-3B Sunbird Multilingual TTS}, + author = {Sunbird AI}, + year = {2026}, + howpublished = {\url{https://huggingface.co/sunbird/orpheus-3b-tts-multilingual}}, +} + +@misc{sunbird_tts_dataset, + title = {Sunbird Speech Dataset}, + author = {Sunbird AI}, + howpublished = {\url{https://huggingface.co/datasets/Sunbird/tts}}, +} + +@misc{orpheus_tts_2025, + title = {Orpheus-TTS}, + author = {Canopy Labs}, + year = {2025}, + howpublished = {\url{https://github.com/canopyai/Orpheus-TTS}}, +} +``` + +--- + +## Single-speaker variant + +If you only need one specific voice and want a smaller, more focused +checkpoint, see +[`sunbird/orpheus-3b-tts-salt-lug-0001`](https://huggingface.co/sunbird/orpheus-3b-tts-salt-lug-0001) +— same recipe, scoped to a single Luganda speaker. diff --git a/modal-deploy/orpheus-3B/Orpheus_3B_Sunbird_Luganda.ipynb b/modal-deploy/orpheus-3B/Orpheus_3B_Sunbird_Luganda.ipynb new file mode 100644 index 00000000..1974b818 --- /dev/null +++ b/modal-deploy/orpheus-3B/Orpheus_3B_Sunbird_Luganda.ipynb @@ -0,0 +1,862 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "14c3e39a", + "metadata": {}, + "source": [ + "# Orpheus-3B Sunbird Luganda Finetune\n", + "\n", + "Finetune `unsloth/orpheus-3b-0.1-pretrained` on the Sunbird/tts Luganda subset filtered to `speaker_id == \"salt_lug_0001\"`, with MLflow tracking and a final push to `sunbird/orpheus-3b-tts-salt-lug-0001`.\n", + "\n", + "**Reference**: `Orpheus_(3B)_TTS.ipynb` and https://unsloth.ai/docs/basics/text-to-speech-tts-fine-tuning" + ] + }, + { + "cell_type": "markdown", + "id": "e56825bf", + "metadata": {}, + "source": [ + "## #0 Setup & secrets" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e9fff530", + "metadata": {}, + "outputs": [], + "source": [ + "%%capture\n", + "import os, re\n", + "if \"COLAB_\" not in \"\".join(os.environ.keys()):\n", + " !pip install unsloth\n", + "else:\n", + " import torch; v = re.match(r'[\\d]{1,}\\.[\\d]{1,}', str(torch.__version__)).group(0)\n", + " xformers = 'xformers==' + {'2.10':'0.0.34','2.9':'0.0.33.post1','2.8':'0.0.32.post2'}.get(v, '0.0.34')\n", + " !pip install sentencepiece protobuf \"datasets==4.3.0\" \"huggingface_hub>=0.34.0\" hf_transfer\n", + " !pip install --no-deps unsloth_zoo bitsandbytes accelerate {xformers} peft trl triton unsloth\n", + " !pip install --no-deps --upgrade \"torchao>=0.16.0\"\n", + "!pip install transformers==4.56.2\n", + "!pip install --no-deps trl==0.22.2\n", + "!pip install snac torchcodec \"datasets>=3.4.1,<4.0.0\"\n", + "# `--ignore-installed blinker` works around vast.ai/Debian images where\n", + "# blinker is installed via apt (no RECORD file) and pip refuses to upgrade it.\n", + "!pip install --ignore-installed blinker mlflow soundfile" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "25e7f577", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "from getpass import getpass\n", + "\n", + "# HF + MLflow credentials (interactive)\n", + "os.environ[\"HF_TOKEN\"] = getpass(\"HF_TOKEN: \")\n", + "os.environ.setdefault(\"MLFLOW_TRACKING_URI\", \"https://mlflow.sunbird.ai/\")\n", + "os.environ.setdefault(\"MLFLOW_EXPERIMENT_NAME\", \"orpheus-3b-tts-finetuning\")\n", + "os.environ[\"MLFLOW_TRACKING_USERNAME\"] = getpass(\"MLFLOW_TRACKING_USERNAME: \")\n", + "os.environ[\"MLFLOW_TRACKING_PASSWORD\"] = getpass(\"MLFLOW_TRACKING_PASSWORD: \")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b132a14e", + "metadata": {}, + "outputs": [], + "source": [ + "import locale\n", + "from pathlib import Path\n", + "\n", + "import torch\n", + "import soundfile as sf\n", + "from datasets import load_dataset, Audio\n", + "from snac import SNAC\n", + "from IPython.display import display, Audio as AudioDisplay\n", + "\n", + "locale.getpreferredencoding = lambda: \"UTF-8\"\n", + "\n", + "OUTPUT_DIR = Path(\"orpheus-3B/outputs\")\n", + "OUTPUT_DIR.mkdir(parents=True, exist_ok=True)\n", + "(OUTPUT_DIR / \"inference_samples\").mkdir(parents=True, exist_ok=True)" + ] + }, + { + "cell_type": "markdown", + "id": "64be186c", + "metadata": {}, + "source": [ + "## #1 Load model + LoRA" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "913d3c16", + "metadata": {}, + "outputs": [], + "source": [ + "from unsloth import FastLanguageModel\n", + "\n", + "# 4096 tokens ≈ 16 s of audio — covers most Sunbird utterances after\n", + "# SNAC tokenisation. Longer rows are dropped by the length filter in #2.6.\n", + "# Drop to 2048 if you're on <16 GB VRAM; bump to 8192 if you see many\n", + "# rows dropped at the #2.6 filter.\n", + "MAX_SEQ_LEN = 4096\n", + "\n", + "model, tokenizer = FastLanguageModel.from_pretrained(\n", + " model_name = \"unsloth/orpheus-3b-0.1-pretrained\",\n", + " max_seq_length = MAX_SEQ_LEN,\n", + " dtype = None, # auto-detect bf16 on Ampere+, fp16 on T4\n", + " load_in_4bit = False, # LoRA 16-bit for higher accuracy on TTS\n", + " token = os.environ[\"HF_TOKEN\"],\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8d7f3324", + "metadata": {}, + "outputs": [], + "source": [ + "model = FastLanguageModel.get_peft_model(\n", + " model,\n", + " r = 64,\n", + " target_modules = [\"q_proj\", \"k_proj\", \"v_proj\", \"o_proj\",\n", + " \"gate_proj\", \"up_proj\", \"down_proj\"],\n", + " lora_alpha = 64,\n", + " lora_dropout = 0,\n", + " bias = \"none\",\n", + " use_gradient_checkpointing = \"unsloth\",\n", + " random_state = 3407,\n", + " use_rslora = False,\n", + " loftq_config = None,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "506f79be", + "metadata": {}, + "source": [ + "## #2 Data prep" + ] + }, + { + "cell_type": "markdown", + "id": "1e72507f", + "metadata": {}, + "source": [ + "### #2.1 Load Sunbird/tts (lug) train + test, filter to salt_lug_0001" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3c444278", + "metadata": {}, + "outputs": [], + "source": [ + "SPEAKER_ID = \"salt_lug_0001\"\n", + "\n", + "ds_train_full = load_dataset(\"Sunbird/tts\", \"lug\", split=\"train\")\n", + "ds_test_full = load_dataset(\"Sunbird/tts\", \"lug\", split=\"test\")\n", + "\n", + "ds_train = ds_train_full.filter(lambda r: r[\"speaker_id\"] == SPEAKER_ID)\n", + "ds_test = ds_test_full.filter(lambda r: r[\"speaker_id\"] == SPEAKER_ID)\n", + "\n", + "print(f\"train: {len(ds_train_full)} -> {len(ds_train)} after speaker filter\")\n", + "print(f\"test: {len(ds_test_full)} -> {len(ds_test)} after speaker filter\")\n", + "assert len(ds_train) > 0, f\"no train rows for speaker_id={SPEAKER_ID}\"" + ] + }, + { + "cell_type": "markdown", + "id": "24c0fed6", + "metadata": {}, + "source": [ + "### #2.2 Tag every row with `source = SPEAKER_ID` (multi-speaker prompt format)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f2531934", + "metadata": {}, + "outputs": [], + "source": [ + "def add_source(example):\n", + " example[\"source\"] = SPEAKER_ID\n", + " return example\n", + "\n", + "ds_train = ds_train.map(add_source)\n", + "ds_test = ds_test.map(add_source)\n", + "ds_train[0].keys()" + ] + }, + { + "cell_type": "markdown", + "id": "d0348de2", + "metadata": {}, + "source": [ + "### #2.3 Cast `audio` to 24 kHz (Orpheus / SNAC expected sample rate)\n", + "\n", + "Sunbird/tts stores audio as `{\"bytes\": , \"path\": ...}` (decode=False). Casting to `Audio(sampling_rate=24000)` switches the column to lazy-decode mode — on access `row[\"audio\"]` returns `{\"array\": np.ndarray, \"sampling_rate\": 24000, \"path\": str}`, which is what `add_codes` consumes in #2.4. No manual `soundfile`/`librosa` decoding is needed." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1882bd6e", + "metadata": {}, + "outputs": [], + "source": [ + "ds_train = ds_train.cast_column(\"audio\", Audio(sampling_rate=24000))\n", + "ds_test = ds_test.cast_column(\"audio\", Audio(sampling_rate=24000))\n", + "# Sanity check — should print 24000 (and KeyError if the cast didn't take).\n", + "ds_train[0][\"audio\"][\"sampling_rate\"]" + ] + }, + { + "cell_type": "markdown", + "id": "987b8cac", + "metadata": {}, + "source": [ + "### #2.4 SNAC tokenisation (24 kHz, 7 codes per frame, per-layer offsets)\n", + "\n", + "**Pre-filter**: before encoding, drop any row whose tokenised text alone exceeds `MAX_SEQ_LEN`. Such a row could never fit in context (audio codes only push it further over), so SNAC-encoding it would be wasted work." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f5de9e77", + "metadata": {}, + "outputs": [], + "source": [ + "# Pre-filter overlong text rows BEFORE the expensive SNAC pass.\n", + "# A row whose text alone tokenises to > MAX_SEQ_LEN cannot fit even\n", + "# without audio codes; the #2.6 length filter would drop it anyway,\n", + "# but only after we paid for SNAC encoding.\n", + "n_before = len(ds_train)\n", + "ds_train = ds_train.filter(\n", + " lambda r: len(tokenizer.encode(r[\"text\"], add_special_tokens=False)) <= MAX_SEQ_LEN\n", + ")\n", + "n_dropped = n_before - len(ds_train)\n", + "print(f\"text pre-filter (text tokens <= {MAX_SEQ_LEN}): \"\n", + " f\"{n_before} -> {len(ds_train)} ({n_dropped} dropped)\")\n", + "\n", + "snac_model = SNAC.from_pretrained(\"hubertsiuzdak/snac_24khz\").to(\"cuda\")\n", + "\n", + "def tokenise_audio(waveform):\n", + " waveform = torch.from_numpy(waveform).unsqueeze(0).to(dtype=torch.float32)\n", + " waveform = waveform.unsqueeze(0).to(\"cuda\")\n", + " with torch.inference_mode():\n", + " codes = snac_model.encode(waveform)\n", + " all_codes = []\n", + " for i in range(codes[0].shape[1]):\n", + " all_codes.append(codes[0][0][i].item() + 128266)\n", + " all_codes.append(codes[1][0][2*i].item() + 128266 + 4096)\n", + " all_codes.append(codes[2][0][4*i].item() + 128266 + 2*4096)\n", + " all_codes.append(codes[2][0][(4*i)+1].item() + 128266 + 3*4096)\n", + " all_codes.append(codes[1][0][(2*i)+1].item() + 128266 + 4*4096)\n", + " all_codes.append(codes[2][0][(4*i)+2].item() + 128266 + 5*4096)\n", + " all_codes.append(codes[2][0][(4*i)+3].item() + 128266 + 6*4096)\n", + " return all_codes\n", + "\n", + "def add_codes(example):\n", + " codes_list = None\n", + " try:\n", + " answer_audio = example.get(\"audio\")\n", + " if answer_audio and \"array\" in answer_audio:\n", + " codes_list = tokenise_audio(answer_audio[\"array\"])\n", + " except Exception as e:\n", + " print(f\"Skipping row due to error: {e}\")\n", + " example[\"codes_list\"] = codes_list\n", + " return example\n", + "\n", + "ds_train = ds_train.map(add_codes, remove_columns=[\"audio\"])" + ] + }, + { + "cell_type": "markdown", + "id": "18a964ad", + "metadata": {}, + "source": [ + "### #2.5 Drop empty/None code rows and remove duplicate frames" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b2e76e87", + "metadata": {}, + "outputs": [], + "source": [ + "ds_train = ds_train.filter(lambda x: x[\"codes_list\"] is not None)\n", + "ds_train = ds_train.filter(lambda x: len(x[\"codes_list\"]) > 0)\n", + "\n", + "def remove_duplicate_frames(example):\n", + " vals = example[\"codes_list\"]\n", + " if len(vals) % 7 != 0:\n", + " raise ValueError(\"Input list length must be divisible by 7\")\n", + " result = vals[:7]\n", + " for i in range(7, len(vals), 7):\n", + " if vals[i] != result[-7]:\n", + " result.extend(vals[i:i+7])\n", + " example[\"codes_list\"] = result\n", + " return example\n", + "\n", + "ds_train = ds_train.map(remove_duplicate_frames)\n", + "print(f\"after dedupe: {len(ds_train)} rows\")" + ] + }, + { + "cell_type": "markdown", + "id": "41773d13", + "metadata": {}, + "source": [ + "### #2.6 Build training input_ids — multi-speaker tagged prompt format" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cf21ced5", + "metadata": {}, + "outputs": [], + "source": [ + "TOKENISER_LENGTH = 128256\n", + "END_OF_TEXT = 128009\n", + "START_OF_SPEECH = TOKENISER_LENGTH + 1 # 128257\n", + "END_OF_SPEECH = TOKENISER_LENGTH + 2 # 128258\n", + "START_OF_HUMAN = TOKENISER_LENGTH + 3 # 128259\n", + "END_OF_HUMAN = TOKENISER_LENGTH + 4 # 128260\n", + "START_OF_AI = TOKENISER_LENGTH + 5 # 128261\n", + "END_OF_AI = TOKENISER_LENGTH + 6 # 128262\n", + "PAD_TOKEN = TOKENISER_LENGTH + 7 # 128263\n", + "AUDIO_TOKENS_START = TOKENISER_LENGTH + 10 # 128266\n", + "\n", + "def create_input_ids(example):\n", + " text_prompt = f\"{example['source']}: {example['text']}\"\n", + " text_ids = tokenizer.encode(text_prompt, add_special_tokens=True)\n", + " text_ids.append(END_OF_TEXT)\n", + " input_ids = (\n", + " [START_OF_HUMAN]\n", + " + text_ids\n", + " + [END_OF_HUMAN]\n", + " + [START_OF_AI]\n", + " + [START_OF_SPEECH]\n", + " + example[\"codes_list\"]\n", + " + [END_OF_SPEECH]\n", + " + [END_OF_AI]\n", + " )\n", + " return {\n", + " \"input_ids\": input_ids,\n", + " \"labels\": input_ids,\n", + " \"attention_mask\": [1] * len(input_ids),\n", + " }\n", + "\n", + "ds_train = ds_train.map(create_input_ids, remove_columns=[\"text\", \"codes_list\"])\n", + "ds_train = ds_train.remove_columns(\n", + " [c for c in ds_train.column_names if c not in (\"input_ids\", \"labels\", \"attention_mask\")]\n", + ")\n", + "\n", + "# Drop rows whose tokenised length exceeds MAX_SEQ_LEN (the model's max\n", + "# context). Long Sunbird utterances + SNAC's ~80-frame/sec rate can\n", + "# blow past 2048 easily; without this filter trainer.train() crashes\n", + "# inside unsloth_fused_ce_loss when its auto-truncation hits a\n", + "# torch.compile graph break.\n", + "n_before = len(ds_train)\n", + "ds_train = ds_train.filter(lambda r: len(r[\"input_ids\"]) <= MAX_SEQ_LEN)\n", + "n_dropped = n_before - len(ds_train)\n", + "print(f\"length filter (<= {MAX_SEQ_LEN} tokens): {n_before} -> {len(ds_train)} \"\n", + " f\"({n_dropped} dropped for being too long)\")\n", + "\n", + "print(\"train columns:\", ds_train.column_names)\n", + "print(\"first row len:\", len(ds_train[0][\"input_ids\"]))" + ] + }, + { + "cell_type": "markdown", + "id": "618c5f67", + "metadata": {}, + "source": [ + "### #2.7 Hold the test split aside un-tokenised — only `text` is used at inference" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9da29890", + "metadata": {}, + "outputs": [], + "source": [ + "test_prompts = [row[\"text\"] for row in ds_test.select(range(min(5, len(ds_test))))]\n", + "print(f\"held-aside test prompts ({len(test_prompts)}):\")\n", + "for p in test_prompts:\n", + " print(\" -\", p[:100])" + ] + }, + { + "cell_type": "markdown", + "id": "13d23f5d", + "metadata": {}, + "source": [ + "## #3 Train\n", + "\n", + "Default is a smoke run (`max_steps=60`). For a real Luganda voice fine-tune, set `max_steps=-1` (the TrainingArguments default that disables step-capping) and uncomment `num_train_epochs=3`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ba68704d", + "metadata": {}, + "outputs": [], + "source": [ + "from transformers import TrainingArguments, Trainer, DataCollatorForSeq2Seq\n", + "\n", + "# Llama-style tokenizers ship without a pad token. Reuse Orpheus's\n", + "# PAD_TOKEN (128263) so the collator can pad variable-length input_ids /\n", + "# attention_mask cleanly when batch size > 1.\n", + "if tokenizer.pad_token_id is None:\n", + " tokenizer.pad_token_id = PAD_TOKEN\n", + "\n", + "# DataCollatorForSeq2Seq pads input_ids/attention_mask to the longest\n", + "# sequence in the batch and pads labels with -100 (ignored by loss).\n", + "data_collator = DataCollatorForSeq2Seq(tokenizer=tokenizer, padding=True)\n", + "\n", + "trainer = Trainer(\n", + " model = model,\n", + " train_dataset = ds_train,\n", + " data_collator = data_collator,\n", + " args = TrainingArguments(\n", + " per_device_train_batch_size = 1,\n", + " gradient_accumulation_steps = 4,\n", + " warmup_steps = 5,\n", + " max_steps = 60, # SMOKE DEFAULT\n", + " # max_steps = -1, # uncomment for the real run\n", + " # num_train_epochs = 3, # uncomment for the real run\n", + " learning_rate = 2e-4,\n", + " logging_steps = 1,\n", + " optim = \"adamw_8bit\",\n", + " weight_decay = 0.001,\n", + " lr_scheduler_type = \"linear\",\n", + " seed = 3407,\n", + " output_dir = str(OUTPUT_DIR),\n", + " report_to = \"mlflow\",\n", + " ),\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "1c035e09", + "metadata": {}, + "source": [ + "### GPU memory snapshot before training" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f1931212", + "metadata": {}, + "outputs": [], + "source": [ + "gpu_stats = torch.cuda.get_device_properties(0)\n", + "start_gpu_memory = round(torch.cuda.max_memory_reserved() / 1024 / 1024 / 1024, 3)\n", + "max_memory = round(gpu_stats.total_memory / 1024 / 1024 / 1024, 3)\n", + "print(f\"GPU = {gpu_stats.name}. Max memory = {max_memory} GB.\")\n", + "print(f\"{start_gpu_memory} GB of memory reserved.\")" + ] + }, + { + "cell_type": "markdown", + "id": "6930b340", + "metadata": {}, + "source": [ + "### Run training" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fe001e23", + "metadata": {}, + "outputs": [], + "source": [ + "trainer_stats = trainer.train()" + ] + }, + { + "cell_type": "markdown", + "id": "77526595", + "metadata": {}, + "source": [ + "### GPU memory snapshot after training" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9a26b68a", + "metadata": {}, + "outputs": [], + "source": [ + "used_memory = round(torch.cuda.max_memory_reserved() / 1024 / 1024 / 1024, 3)\n", + "used_for_lora = round(used_memory - start_gpu_memory, 3)\n", + "used_pct = round(used_memory / max_memory * 100, 3)\n", + "lora_pct = round(used_for_lora / max_memory * 100, 3)\n", + "print(f\"{trainer_stats.metrics['train_runtime']} seconds used for training.\")\n", + "print(f\"{round(trainer_stats.metrics['train_runtime']/60, 2)} minutes used for training.\")\n", + "print(f\"Peak reserved memory = {used_memory} GB.\")\n", + "print(f\"Peak reserved memory for training = {used_for_lora} GB.\")\n", + "print(f\"Peak reserved memory % of max memory = {used_pct} %.\")\n", + "print(f\"Peak reserved memory for training % of max memory = {lora_pct} %.\")" + ] + }, + { + "cell_type": "markdown", + "id": "71668118", + "metadata": {}, + "source": [ + "## #4 Inference" + ] + }, + { + "cell_type": "markdown", + "id": "e53414e2", + "metadata": {}, + "source": [ + "### #4.1 Build prompt list (smoke from train split + held-aside test)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "001544de", + "metadata": {}, + "outputs": [], + "source": [ + "chosen_voice = SPEAKER_ID # multi-speaker tag we trained with\n", + "\n", + "# Smoke prompts: pull two short transcripts from the train split (still in\n", + "# ds_train_full pre-filter — our transformed `ds_train` no longer has `text`).\n", + "_train_speaker = ds_train_full.filter(lambda r: r[\"speaker_id\"] == SPEAKER_ID)\n", + "smoke_prompts = [_train_speaker[i][\"text\"] for i in range(min(2, len(_train_speaker)))]\n", + "all_prompts = smoke_prompts + test_prompts\n", + "print(f\"running inference on {len(all_prompts)} prompts\")\n", + "for p in all_prompts:\n", + " print(\" -\", p[:100])" + ] + }, + { + "cell_type": "markdown", + "id": "4486b9cd", + "metadata": {}, + "source": [ + "### #4.2 Tokenise + pad + generate" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c6e213c1", + "metadata": {}, + "outputs": [], + "source": [ + "FastLanguageModel.for_inference(model)\n", + "snac_model.to(\"cpu\") # free GPU for the LM\n", + "\n", + "prompts_tagged = [f\"{chosen_voice}: {p}\" for p in all_prompts]\n", + "all_input_ids = [tokenizer(p, return_tensors=\"pt\").input_ids for p in prompts_tagged]\n", + "\n", + "start_token = torch.tensor([[128259]], dtype=torch.int64) # SOH\n", + "end_tokens = torch.tensor([[128009, 128260]], dtype=torch.int64) # EOT, EOH\n", + "\n", + "modified = [torch.cat([start_token, ids, end_tokens], dim=1) for ids in all_input_ids]\n", + "max_len = max(t.shape[1] for t in modified)\n", + "\n", + "padded, masks = [], []\n", + "for t in modified:\n", + " pad = max_len - t.shape[1]\n", + " padded.append(torch.cat([torch.full((1, pad), 128263, dtype=torch.int64), t], dim=1))\n", + " masks.append(torch.cat([torch.zeros((1, pad), dtype=torch.int64),\n", + " torch.ones((1, t.shape[1]), dtype=torch.int64)], dim=1))\n", + "\n", + "input_ids = torch.cat(padded, dim=0).to(\"cuda\")\n", + "attention_mask = torch.cat(masks, dim=0).to(\"cuda\")\n", + "\n", + "generated_ids = model.generate(\n", + " input_ids = input_ids,\n", + " attention_mask = attention_mask,\n", + " max_new_tokens = 1200,\n", + " do_sample = True,\n", + " temperature = 0.6,\n", + " top_p = 0.95,\n", + " repetition_penalty = 1.1,\n", + " num_return_sequences = 1,\n", + " eos_token_id = 128258,\n", + " use_cache = True,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "251df722", + "metadata": {}, + "source": [ + "### #4.3 Crop, strip, redistribute SNAC codes, decode" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "015cc459", + "metadata": {}, + "outputs": [], + "source": [ + "TOKEN_TO_FIND = 128257 # last SOS\n", + "AUDIO_TOKEN_LO = 128266 # first audio-codebook id\n", + "AUDIO_TOKEN_HI = 128266 + 7 * 4096 # exclusive (156938)\n", + "\n", + "indices = (generated_ids == TOKEN_TO_FIND).nonzero(as_tuple=True)\n", + "if len(indices[1]) > 0:\n", + " last = indices[1][-1].item()\n", + " cropped = generated_ids[:, last + 1:]\n", + "else:\n", + " print(\"WARNING: no start-of-speech token (128257) in generated output.\"\n", + " \" Smoke runs (max_steps=60) often produce garbage — switch to the\"\n", + " \" full-run epochs block in #3 for a real fine-tune.\")\n", + " cropped = generated_ids\n", + "\n", + "# Keep only tokens that fall in the audio-codebook range. This strips EOS\n", + "# and any non-audio garbage tokens a poorly-trained model may emit,\n", + "# preventing IndexError in SNAC's codebook lookup downstream.\n", + "rows = [row[(row >= AUDIO_TOKEN_LO) & (row < AUDIO_TOKEN_HI)] for row in cropped]\n", + "\n", + "code_lists = []\n", + "for row in rows:\n", + " n = (row.size(0) // 7) * 7\n", + " if n == 0:\n", + " print(\"WARNING: row has no usable audio tokens after filtering;\"\n", + " \" emitting silence. (Smoke runs commonly hit this.)\")\n", + " trimmed = [t.item() - AUDIO_TOKEN_LO for t in row[:n]]\n", + " code_lists.append(trimmed)\n", + "\n", + "def redistribute_codes(code_list):\n", + " \"\"\"Inverse of the SNAC encoding from #2.4.\n", + "\n", + " Each frame is 7 tokens; positions 0-6 carry layer_1 / layer_2(even) /\n", + " layer_3(0) / layer_3(1) / layer_2(odd) / layer_3(2) / layer_3(3),\n", + " each with its own +N*4096 offset. We subtract the offsets and clamp\n", + " to [0, 4095] as a final safety net for tokens that landed in the\n", + " audio range but at the wrong intra-frame position.\n", + " \"\"\"\n", + " layer_1, layer_2, layer_3 = [], [], []\n", + " for i in range(len(code_list) // 7):\n", + " layer_1.append(code_list[7*i])\n", + " layer_2.append(code_list[7*i + 1] - 4096)\n", + " layer_3.append(code_list[7*i + 2] - 2*4096)\n", + " layer_3.append(code_list[7*i + 3] - 3*4096)\n", + " layer_2.append(code_list[7*i + 4] - 4*4096)\n", + " layer_3.append(code_list[7*i + 5] - 5*4096)\n", + " layer_3.append(code_list[7*i + 6] - 6*4096)\n", + " if not layer_1:\n", + " # No valid frames — return ~0.5s of silence at 24 kHz so the #4.4\n", + " # display loop still runs end-to-end on garbage output.\n", + " return torch.zeros(1, 1, 12000)\n", + " def clamp(vals):\n", + " return [max(0, min(4095, v)) for v in vals]\n", + " codes = [torch.tensor(clamp(layer_1)).unsqueeze(0),\n", + " torch.tensor(clamp(layer_2)).unsqueeze(0),\n", + " torch.tensor(clamp(layer_3)).unsqueeze(0)]\n", + " return snac_model.decode(codes)\n", + "\n", + "samples = [redistribute_codes(cl) for cl in code_lists]" + ] + }, + { + "cell_type": "markdown", + "id": "ff45665e", + "metadata": {}, + "source": [ + "### #4.4 Display + save WAVs to `outputs/inference_samples/`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7f52d052", + "metadata": {}, + "outputs": [], + "source": [ + "samples_dir = OUTPUT_DIR / \"inference_samples\"\n", + "samples_dir.mkdir(parents=True, exist_ok=True)\n", + "\n", + "for idx, (prompt, audio_hat) in enumerate(zip(all_prompts, samples)):\n", + " wav = audio_hat.detach().squeeze().to(\"cpu\").numpy()\n", + " out_path = samples_dir / f\"sample_{idx:02d}.wav\"\n", + " sf.write(str(out_path), wav, 24000)\n", + " print(f\"[{idx}] {prompt[:80]} -> {out_path}\")\n", + " display(AudioDisplay(wav, rate=24000))" + ] + }, + { + "cell_type": "markdown", + "id": "16bc0761", + "metadata": {}, + "source": [ + "## #5 Save & push" + ] + }, + { + "cell_type": "markdown", + "id": "f332b166", + "metadata": {}, + "source": [ + "### #5.1 LoRA adapters (local)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f5eac3bd", + "metadata": {}, + "outputs": [], + "source": [ + "lora_dir = str(OUTPUT_DIR / \"orpheus_lora\")\n", + "model.save_pretrained(lora_dir)\n", + "tokenizer.save_pretrained(lora_dir)\n", + "print(\"LoRA saved to\", lora_dir)" + ] + }, + { + "cell_type": "markdown", + "id": "2c70beb0", + "metadata": {}, + "source": [ + "### #5.2 Merged 16-bit checkpoint (local)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4762c3fa", + "metadata": {}, + "outputs": [], + "source": [ + "merged_dir = str(OUTPUT_DIR / \"orpheus_finetune_16bit\")\n", + "model.save_pretrained_merged(\n", + " merged_dir, tokenizer, save_method=\"merged_16bit\",\n", + ")\n", + "print(\"merged 16-bit saved to\", merged_dir)" + ] + }, + { + "cell_type": "markdown", + "id": "46902cbb", + "metadata": {}, + "source": [ + "### #5.2.1 Patch local `config.json` to match embedding shape\n", + "\n", + "Unsloth's `save_pretrained_merged` runs a pad-token shim that adds `<|finetune_right_pad_id|>` to the tokenizer when `pad_token` is missing or equals `eos_token`. That bumps `len(tokenizer)` by 1 and Unsloth writes `config.vocab_size = len(tokenizer)`, but the merged 16-bit weights are not resized — so `config.vocab_size` ends up off-by-one vs. `embed_tokens.weight.shape[0]`. `transformers` tolerates this, but vLLM asserts equality and refuses to load. We re-derive `vocab_size` from the actual embedding tensor and rewrite `config.json`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "326bb2e0", + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "from pathlib import Path as _Path\n", + "\n", + "_embed_size = model.get_input_embeddings().num_embeddings\n", + "_cfg_path = _Path(merged_dir) / \"config.json\"\n", + "_cfg = json.loads(_cfg_path.read_text())\n", + "if _cfg.get(\"vocab_size\") != _embed_size:\n", + " print(f\"fix config.vocab_size: {_cfg.get('vocab_size')} -> {_embed_size}\")\n", + " _cfg[\"vocab_size\"] = _embed_size\n", + " _cfg_path.write_text(json.dumps(_cfg, indent=2))\n", + "else:\n", + " print(f\"config.vocab_size already matches embedding ({_embed_size})\")" + ] + }, + { + "cell_type": "markdown", + "id": "a6e3cb45", + "metadata": {}, + "source": [ + "### #5.3 Push merged 16-bit to Hugging Face Hub" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "84b866e4", + "metadata": {}, + "outputs": [], + "source": [ + "HF_REPO_ID = \"sunbird/orpheus-3b-tts-salt-lug-0001\"\n", + "model.push_to_hub_merged(\n", + " HF_REPO_ID, tokenizer,\n", + " save_method=\"merged_16bit\",\n", + " token=os.environ[\"HF_TOKEN\"],\n", + ")\n", + "print(\"pushed to https://huggingface.co/\" + HF_REPO_ID)" + ] + }, + { + "cell_type": "markdown", + "id": "099a32b5", + "metadata": {}, + "source": [ + "### #5.3.1 Patch hub `config.json` to match embedding shape\n", + "\n", + "`push_to_hub_merged` re-runs the same pad-token shim while writing to the hub, so the local fix in #5.2.1 doesn't propagate. Upload a corrected `config.json` directly so vLLM can load this repo without `hf_overrides`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "02366ac9", + "metadata": {}, + "outputs": [], + "source": [ + "from huggingface_hub import HfApi\n", + "\n", + "_local_cfg = _Path(merged_dir) / \"config.json\" # already fixed in #5.2.1\n", + "HfApi().upload_file(\n", + " path_or_fileobj=str(_local_cfg),\n", + " path_in_repo=\"config.json\",\n", + " repo_id=HF_REPO_ID,\n", + " commit_message=f\"fix(config): vocab_size -> {_embed_size} to match embed weights\",\n", + " token=os.environ[\"HF_TOKEN\"],\n", + ")\n", + "print(f\"hub config.json patched: vocab_size -> {_embed_size}\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/modal-deploy/orpheus-3B/Orpheus_3B_Sunbird_Luganda_Inference.ipynb b/modal-deploy/orpheus-3B/Orpheus_3B_Sunbird_Luganda_Inference.ipynb new file mode 100644 index 00000000..3fb840bf --- /dev/null +++ b/modal-deploy/orpheus-3B/Orpheus_3B_Sunbird_Luganda_Inference.ipynb @@ -0,0 +1,484 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "56f6aa20", + "metadata": {}, + "source": [ + "# Orpheus-3B Sunbird Luganda — Inference\n", + "\n", + "Standalone inference notebook for the finetuned Luganda voice at [`sunbird/orpheus-3b-tts-salt-lug-0001`](https://huggingface.co/sunbird/orpheus-3b-tts-salt-lug-0001). No training, no data prep — just load the model and synthesize speech from arbitrary Luganda text.\n", + "\n", + "**Use cases:**\n", + "- Audition the model on sentences it has never seen.\n", + "- A/B against the held-out `test` split for quality eyeballing.\n", + "- Lift the `synthesize()` function into a FastAPI / vLLM deployment with no notebook-specific code to remove.\n", + "\n", + "**Companion notebook**: `Orpheus_3B_Sunbird_Luganda.ipynb` trains and pushes the model this notebook consumes." + ] + }, + { + "cell_type": "markdown", + "id": "e56825bf", + "metadata": {}, + "source": [ + "## #0 Setup & secrets" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "436719ba", + "metadata": {}, + "outputs": [], + "source": [ + "%%capture\n", + "import os, re\n", + "if \"COLAB_\" not in \"\".join(os.environ.keys()):\n", + " !pip install unsloth\n", + "else:\n", + " import torch; v = re.match(r'[\\d]{1,}\\.[\\d]{1,}', str(torch.__version__)).group(0)\n", + " xformers = 'xformers==' + {'2.10':'0.0.34','2.9':'0.0.33.post1','2.8':'0.0.32.post2'}.get(v, '0.0.34')\n", + " !pip install sentencepiece protobuf \"datasets==4.3.0\" \"huggingface_hub>=0.34.0\" hf_transfer\n", + " !pip install --no-deps unsloth_zoo bitsandbytes accelerate {xformers} peft trl triton unsloth\n", + " !pip install --no-deps --upgrade \"torchao>=0.16.0\"\n", + "!pip install transformers==4.56.2\n", + "!pip install snac torchcodec \"datasets>=3.4.1,<4.0.0\"\n", + "!pip install soundfile" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a03f1ff6", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "from getpass import getpass\n", + "\n", + "# HF token only — needed to pull the model from sunbird/orpheus-3b-tts-salt-lug-0001.\n", + "# Skip the prompt if the model is already on disk locally and HF_TOKEN is set in env.\n", + "if not os.environ.get(\"HF_TOKEN\"):\n", + " os.environ[\"HF_TOKEN\"] = getpass(\"HF_TOKEN: \")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b6ac2815", + "metadata": {}, + "outputs": [], + "source": [ + "import locale\n", + "from pathlib import Path\n", + "\n", + "import numpy as np\n", + "import torch\n", + "import soundfile as sf\n", + "from datasets import load_dataset, Audio\n", + "from snac import SNAC\n", + "from IPython.display import display, Audio as AudioDisplay\n", + "\n", + "locale.getpreferredencoding = lambda: \"UTF-8\"\n", + "\n", + "INFERENCE_OUTPUT_DIR = Path(\"orpheus-3B/inference_outputs\")\n", + "INFERENCE_OUTPUT_DIR.mkdir(parents=True, exist_ok=True)" + ] + }, + { + "cell_type": "markdown", + "id": "c326e73a", + "metadata": {}, + "source": [ + "## #1 Load the finetuned model + SNAC decoder\n", + "\n", + "Set `MODEL_ID` to either an HF repo or a local checkpoint path (`outputs/orpheus_finetune_16bit/` is what the training notebook produces). `SPEAKER_ID` must match the multi-speaker tag the model was trained with — `salt_lug_0001` for the default Luganda voice. `LOAD_IN_4BIT=True` cuts VRAM in half for deployment on smaller GPUs at a small quality cost." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "80b08cd2", + "metadata": {}, + "outputs": [], + "source": [ + "'''\n", + "from huggingface_hub import HfApi, hf_hub_download\n", + "import json, pathlib\n", + "\n", + "REPO = MODEL_ID # e.g. \"patrickcmd/orpheus-3b-tts-multilingual\"\n", + "\n", + "local_cfg = pathlib.Path(hf_hub_download(REPO, \"config.json\", token=os.environ[\"HF_TOKEN\"]))\n", + "cfg = json.loads(local_cfg.read_text())\n", + "print(\"current config.vocab_size =\", cfg[\"vocab_size\"])\n", + "\n", + "cfg[\"vocab_size\"] = 156939\n", + "out = pathlib.Path(\"config_fixed.json\")\n", + "out.write_text(json.dumps(cfg, indent=2))\n", + "\n", + "HfApi().upload_file(\n", + " path_or_fileobj=str(out),\n", + " path_in_repo=\"config.json\",\n", + " repo_id=REPO,\n", + " commit_message=\"fix(config): vocab_size 156940 -> 156939 to match embed weights\",\n", + " token=os.environ[\"HF_TOKEN\"],\n", + ")\n", + "print(\"hub config.json patched — re-run #1 load cell.\")\n", + "\n", + "'''" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "88544146", + "metadata": {}, + "outputs": [], + "source": [ + "'''\n", + "import os, json\n", + "from pathlib import Path\n", + "from huggingface_hub import snapshot_download\n", + "\n", + "snap = Path(snapshot_download(repo_id=MODEL_ID, token=os.environ[\"HF_TOKEN\"]))\n", + "patched = Path(\"orpheus-3B/outputs/orpheus_patched\")\n", + "patched.mkdir(parents=True, exist_ok=True)\n", + "\n", + "for name in os.listdir(snap):\n", + " src, dst = snap / name, patched / name\n", + " if dst.exists() or dst.is_symlink():\n", + " continue\n", + " if name == \"config.json\":\n", + " cfg = json.loads(src.read_text())\n", + " if cfg.get(\"vocab_size\") != 156939:\n", + " print(f\"patch: vocab_size {cfg.get('vocab_size')} -> 156939\")\n", + " cfg[\"vocab_size\"] = 156939\n", + " dst.write_text(json.dumps(cfg, indent=2))\n", + " else:\n", + " dst.symlink_to(src.resolve())\n", + "\n", + "# Then load from the local patched dir instead of the hub repo:\n", + "model, tokenizer = FastLanguageModel.from_pretrained(\n", + " model_name = str(patched),\n", + " max_seq_length = MAX_SEQ_LEN,\n", + " dtype = None,\n", + " load_in_4bit = LOAD_IN_4BIT,\n", + " token = os.environ.get(\"HF_TOKEN\"),\n", + ")\n", + "FastLanguageModel.for_inference(model)\n", + "\n", + "'''" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8d94055b", + "metadata": {}, + "outputs": [], + "source": [ + "from unsloth import FastLanguageModel\n", + "\n", + "MODEL_ID = \"sunbird/orpheus-3b-tts-salt-lug-0001\"\n", + "# Or load locally:\n", + "# MODEL_ID = \"orpheus-3B/outputs/orpheus_finetune_16bit\"\n", + "\n", + "SPEAKER_ID = \"salt_lug_0001\"\n", + "LOAD_IN_4BIT = False # True → ~6 GB VRAM, slight quality drop\n", + "MAX_SEQ_LEN = 2048\n", + "\n", + "model, tokenizer = FastLanguageModel.from_pretrained(\n", + " model_name = MODEL_ID,\n", + " max_seq_length = MAX_SEQ_LEN,\n", + " dtype = None, # auto bf16 / fp16\n", + " load_in_4bit = LOAD_IN_4BIT,\n", + " token = os.environ.get(\"HF_TOKEN\"),\n", + ")\n", + "FastLanguageModel.for_inference(model) # 2x faster generate\n", + "print(f\"Loaded LM: {MODEL_ID}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7e44b94b", + "metadata": {}, + "outputs": [], + "source": [ + "# SNAC decoder runs on CPU — frees GPU for the LM, decoding is fast enough.\n", + "snac_model = SNAC.from_pretrained(\"hubertsiuzdak/snac_24khz\").to(\"cpu\")\n", + "print(\"Loaded SNAC 24 kHz decoder (on CPU)\")" + ] + }, + { + "cell_type": "markdown", + "id": "31451b0a", + "metadata": {}, + "source": [ + "## #2 Token constants + helpers\n", + "\n", + "Same special-token layout as the training notebook. The audio codebook spans `[128266, 128266 + 7·4096) = [128266, 156938)`; anything outside that range in the model's output is treated as garbage and filtered out before SNAC decoding." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "00e94cf0", + "metadata": {}, + "outputs": [], + "source": [ + "# Special tokens (must match #2.6 of the training notebook)\n", + "START_OF_TEXT = 128000\n", + "END_OF_TEXT = 128009\n", + "START_OF_SPEECH = 128257\n", + "END_OF_SPEECH = 128258\n", + "START_OF_HUMAN = 128259\n", + "END_OF_HUMAN = 128260\n", + "PAD_TOKEN = 128263\n", + "AUDIO_TOKEN_LO = 128266\n", + "AUDIO_TOKEN_HI = 128266 + 7 * 4096 # exclusive (156938)\n", + "\n", + "if tokenizer.pad_token_id is None:\n", + " tokenizer.pad_token_id = PAD_TOKEN\n", + "\n", + "\n", + "def _build_input_ids(prompt: str, speaker_id: str) -> torch.Tensor:\n", + " \"\"\"Wrap `f\"{speaker_id}: {prompt}\"` in [SOH] ... [EOT, EOH].\"\"\"\n", + " tagged = f\"{speaker_id}: {prompt}\"\n", + " text_ids = tokenizer(tagged, return_tensors=\"pt\").input_ids\n", + " soh = torch.tensor([[START_OF_HUMAN]], dtype=torch.int64)\n", + " end = torch.tensor([[END_OF_TEXT, END_OF_HUMAN]], dtype=torch.int64)\n", + " return torch.cat([soh, text_ids, end], dim=1)\n", + "\n", + "\n", + "def _redistribute_codes(code_list: list[int]) -> torch.Tensor:\n", + " \"\"\"Inverse of the SNAC encoding. Returns a (1, 1, n_samples) waveform.\"\"\"\n", + " layer_1, layer_2, layer_3 = [], [], []\n", + " for i in range(len(code_list) // 7):\n", + " layer_1.append(code_list[7*i])\n", + " layer_2.append(code_list[7*i + 1] - 4096)\n", + " layer_3.append(code_list[7*i + 2] - 2*4096)\n", + " layer_3.append(code_list[7*i + 3] - 3*4096)\n", + " layer_2.append(code_list[7*i + 4] - 4*4096)\n", + " layer_3.append(code_list[7*i + 5] - 5*4096)\n", + " layer_3.append(code_list[7*i + 6] - 6*4096)\n", + " if not layer_1:\n", + " return torch.zeros(1, 1, 12000) # ~0.5s silence fallback\n", + " def clamp(vals):\n", + " return [max(0, min(4095, v)) for v in vals]\n", + " codes = [torch.tensor(clamp(layer_1)).unsqueeze(0),\n", + " torch.tensor(clamp(layer_2)).unsqueeze(0),\n", + " torch.tensor(clamp(layer_3)).unsqueeze(0)]\n", + " return snac_model.decode(codes)" + ] + }, + { + "cell_type": "markdown", + "id": "5d45db11", + "metadata": {}, + "source": [ + "## #3 `synthesize()` — the deployable entry point\n", + "\n", + "Single function: `text in → 24 kHz mono float32 numpy array out`. No notebook globals beyond `model`, `tokenizer`, `snac_model` (loaded in #1) and the helpers from #2 — so to deploy you just lift #1 + #2 + #3 into a Python module.\n", + "\n", + "**Sampling defaults** (`temperature=0.6`, `top_p=0.95`, `repetition_penalty=1.1`) come from the upstream Orpheus reference notebook. `max_new_tokens=1200` gives ~9–10 seconds of audio at SNAC's 7-codes/frame rate; raise it if your prompts run longer." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "69bfd85e", + "metadata": {}, + "outputs": [], + "source": [ + "def synthesize(\n", + " text: str,\n", + " speaker_id: str = SPEAKER_ID,\n", + " *,\n", + " max_new_tokens: int = 1200,\n", + " temperature: float = 0.6,\n", + " top_p: float = 0.95,\n", + " repetition_penalty: float = 1.1,\n", + " seed: int | None = None,\n", + ") -> np.ndarray:\n", + " \"\"\"Synthesize speech for `text`.\n", + "\n", + " Returns a 1-D float32 numpy array at 24 kHz mono.\n", + " Pass an int `seed` for reproducible output.\n", + " \"\"\"\n", + " if seed is not None:\n", + " torch.manual_seed(seed)\n", + "\n", + " input_ids = _build_input_ids(text, speaker_id).to(\"cuda\")\n", + " attention_mask = torch.ones_like(input_ids)\n", + "\n", + " generated = model.generate(\n", + " input_ids = input_ids,\n", + " attention_mask = attention_mask,\n", + " max_new_tokens = max_new_tokens,\n", + " do_sample = True,\n", + " temperature = temperature,\n", + " top_p = top_p,\n", + " repetition_penalty = repetition_penalty,\n", + " eos_token_id = END_OF_SPEECH,\n", + " use_cache = True,\n", + " )\n", + "\n", + " # Crop on last SOS — discards the echoed prompt and any pre-speech tokens.\n", + " sos_indices = (generated == START_OF_SPEECH).nonzero(as_tuple=True)\n", + " if len(sos_indices[1]) > 0:\n", + " last = sos_indices[1][-1].item()\n", + " cropped = generated[:, last + 1:]\n", + " else:\n", + " print(\"WARNING: no start-of-speech token in output; output may be silence.\")\n", + " cropped = generated\n", + "\n", + " # Filter to audio-codebook range only (strips EOS + any garbage tokens).\n", + " row = cropped[0]\n", + " audio_only = row[(row >= AUDIO_TOKEN_LO) & (row < AUDIO_TOKEN_HI)]\n", + " n = (audio_only.size(0) // 7) * 7\n", + " code_list = [t.item() - AUDIO_TOKEN_LO for t in audio_only[:n]]\n", + "\n", + " waveform = _redistribute_codes(code_list)\n", + " return waveform.detach().squeeze().to(\"cpu\").numpy().astype(np.float32)" + ] + }, + { + "cell_type": "markdown", + "id": "659be12d", + "metadata": {}, + "source": [ + "## #4 Free-text inference — try your own Luganda sentence\n", + "\n", + "Edit the `prompt` variable below and re-run the cell to hear the model on any new text. Output is saved to `orpheus-3B/inference_outputs/free_text.wav` and displayed inline." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "624a0350", + "metadata": {}, + "outputs": [], + "source": [ + "prompt = \"Mwattu, oli otya?\" # ← put any Luganda sentence here\n", + "\n", + "wav = synthesize(prompt, seed=42)\n", + "out_path = INFERENCE_OUTPUT_DIR / \"free_text.wav\"\n", + "sf.write(str(out_path), wav, 24000)\n", + "print(f\"{prompt}\")\n", + "print(f\" -> {out_path} ({len(wav)/24000:.2f}s)\")\n", + "display(AudioDisplay(wav, rate=24000))" + ] + }, + { + "cell_type": "markdown", + "id": "96f5ab1f", + "metadata": {}, + "source": [ + "## #5 Held-out test split — A/B vs ground truth\n", + "\n", + "The `test` split was never used during training. For each of the first 5 utterances, this cell synthesizes the model's version, saves it next to the ground-truth recording, and displays both inline so you can A/B them." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4042fbc8", + "metadata": {}, + "outputs": [], + "source": [ + "ds_test = load_dataset(\"Sunbird/tts\", \"lug\", split=\"test\")\n", + "ds_test = ds_test.filter(lambda r: r[\"speaker_id\"] == SPEAKER_ID)\n", + "ds_test = ds_test.cast_column(\"audio\", Audio(sampling_rate=24000))\n", + "print(f\"{len(ds_test)} test rows for {SPEAKER_ID}\")\n", + "\n", + "n_compare = min(5, len(ds_test))\n", + "for i in range(n_compare):\n", + " row = ds_test[i]\n", + " text = row[\"text\"]\n", + " print(f\"\\n[{i}] {text[:120]}\")\n", + "\n", + " wav = synthesize(text, seed=42 + i)\n", + " synth_path = INFERENCE_OUTPUT_DIR / f\"test_{i:02d}_synth.wav\"\n", + " truth_path = INFERENCE_OUTPUT_DIR / f\"test_{i:02d}_groundtruth.wav\"\n", + " sf.write(str(synth_path), wav, 24000)\n", + " sf.write(str(truth_path), row[\"audio\"][\"array\"], 24000)\n", + " print(f\" synth -> {synth_path} ({len(wav)/24000:.2f}s)\")\n", + " print(f\" ground truth -> {truth_path}\")\n", + " print(\" [synth]\")\n", + " display(AudioDisplay(wav, rate=24000))\n", + " print(\" [ground truth]\")\n", + " display(AudioDisplay(row[\"audio\"][\"array\"], rate=24000))" + ] + }, + { + "cell_type": "markdown", + "id": "e52edd12", + "metadata": {}, + "source": [ + "## #6 Deployment notes\n", + "\n", + "The `synthesize()` function in #3 has no notebook-specific dependencies — it only uses `model`, `tokenizer`, `snac_model` (loaded in #1) and the constants/helpers in #2. To turn this into a service:\n", + "\n", + "1. Copy #1 + #2 + #3 into a Python module (e.g. `orpheus_inference.py`). The `model = ...`, `tokenizer = ...`, `snac_model = ...` lines become module-level globals loaded at import time.\n", + "2. Wrap `synthesize()` in your transport of choice — see the FastAPI snippet below for a minimal HTTP endpoint.\n", + "3. Deploy on a GPU instance (≥8 GB VRAM with `LOAD_IN_4BIT=True`, ≥16 GB without). One worker per GPU; `synthesize()` is single-stream because `model.generate` holds the GPU.\n", + "\n", + "**Latency rough numbers** (RTX 4090, ~5 word sentence):\n", + "- generate: ~2–4 s wall clock\n", + "- SNAC decode (CPU): ~50–150 ms\n", + "- Total: ~3 s for ~5 s of audio (real-time factor ~0.6)\n", + "\n", + "For higher throughput, batch inference (multiple prompts in one `generate` call with left-padding — see #4.2 of the training notebook for the pattern) or serve with **vLLM** which supports Llama-architecture models out of the box." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "58563c08", + "metadata": {}, + "outputs": [], + "source": [ + "# FastAPI deployment snippet — DO NOT RUN inside this notebook.\n", + "# Save as `serve.py`, then `uvicorn serve:app --host 0.0.0.0 --port 8000`.\n", + "FASTAPI_SNIPPET = '''\n", + "import io\n", + "import soundfile as sf\n", + "from fastapi import FastAPI\n", + "from fastapi.responses import Response\n", + "from pydantic import BaseModel\n", + "\n", + "from orpheus_inference import synthesize # #1 + #2 + #3 in a .py module\n", + "\n", + "app = FastAPI()\n", + "\n", + "class TTSRequest(BaseModel):\n", + " text: str\n", + " speaker_id: str = \"salt_lug_0001\"\n", + " seed: int | None = None\n", + "\n", + "@app.post(\"/tts\", responses={200: {\"content\": {\"audio/wav\": {}}}})\n", + "def tts(req: TTSRequest):\n", + " wav = synthesize(req.text, req.speaker_id, seed=req.seed)\n", + " buf = io.BytesIO()\n", + " sf.write(buf, wav, 24000, format=\"WAV\", subtype=\"PCM_16\")\n", + " return Response(content=buf.getvalue(), media_type=\"audio/wav\")\n", + "'''\n", + "print(FASTAPI_SNIPPET)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/modal-deploy/orpheus-3B/Orpheus_3B_Sunbird_Luganda_vLLM_Inference.ipynb b/modal-deploy/orpheus-3B/Orpheus_3B_Sunbird_Luganda_vLLM_Inference.ipynb new file mode 100644 index 00000000..7a3e331d --- /dev/null +++ b/modal-deploy/orpheus-3B/Orpheus_3B_Sunbird_Luganda_vLLM_Inference.ipynb @@ -0,0 +1,618 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "25d6ad63", + "metadata": {}, + "source": [ + "# Orpheus-3B Sunbird Luganda — vLLM Inference\n", + "\n", + "High-throughput inference notebook for the finetuned Luganda voice [`sunbird/orpheus-3b-tts-salt-lug-0001`](https://huggingface.co/sunbird/orpheus-3b-tts-salt-lug-0001) served by [vLLM](https://docs.vllm.ai). vLLM provides PagedAttention + continuous batching, giving roughly 5–10× faster single-request latency and 10–100× throughput on batched requests vs. plain HuggingFace `model.generate`.\n", + "\n", + "**vs. `Orpheus_3B_Sunbird_Luganda_Inference.ipynb`:** same Orpheus token layout, same SNAC decode path. Only the LM step changes from `transformers` + `unsloth` to `vllm.LLM`. We also showcase batch inference (cell #5) and an OpenAI-compatible deployment server snippet (#7).\n", + "\n", + "**Companion notebook**: `Orpheus_3B_Sunbird_Luganda.ipynb` trains and pushes the merged 16-bit model this notebook serves.\n", + "\n", + "**Why a separate notebook?** vLLM ships its own Torch + transformers and conflicts with unsloth's pinned versions. Use a fresh Python env / GPU instance for vLLM; do not run this notebook in the same environment that ran the training notebook." + ] + }, + { + "cell_type": "markdown", + "id": "9b5f38c6", + "metadata": {}, + "source": [ + "## #0 Setup & secrets\n", + "\n", + "vLLM bundles its own torch / transformers, so the install is much smaller than the training notebook. We only add `snac` + `soundfile` for audio decoding and `datasets` for pulling the test split." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f4a45806", + "metadata": {}, + "outputs": [], + "source": [ + "%%capture\n", + "!pip install vllm\n", + "!pip install snac torchcodec \"datasets>=3.4.1,<4.0.0\"\n", + "# `--ignore-installed blinker` works around vast.ai/Debian images where\n", + "# blinker is installed via apt (no RECORD file) and pip refuses to upgrade it.\n", + "!pip install --ignore-installed blinker soundfile" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b4844e94", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "from getpass import getpass\n", + "\n", + "# HF token to pull the finetuned model. Skip the prompt if HF_TOKEN\n", + "# is already set in the env.\n", + "if not os.environ.get(\"HF_TOKEN\"):\n", + " os.environ[\"HF_TOKEN\"] = getpass(\"HF_TOKEN: \")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e05b7097", + "metadata": {}, + "outputs": [], + "source": [ + "import locale\n", + "from pathlib import Path\n", + "\n", + "import numpy as np\n", + "import torch\n", + "import soundfile as sf\n", + "from datasets import load_dataset, Audio\n", + "from snac import SNAC\n", + "from transformers import AutoTokenizer\n", + "from vllm import LLM, SamplingParams\n", + "from IPython.display import display, Audio as AudioDisplay\n", + "\n", + "locale.getpreferredencoding = lambda: \"UTF-8\"\n", + "\n", + "INFERENCE_OUTPUT_DIR = Path(\"orpheus-3B/inference_outputs/vllm\")\n", + "INFERENCE_OUTPUT_DIR.mkdir(parents=True, exist_ok=True)" + ] + }, + { + "cell_type": "markdown", + "id": "e306c468", + "metadata": {}, + "source": [ + "## #1 Load the finetuned model into vLLM + SNAC decoder\n", + "\n", + "`MODEL_ID` accepts either an HF repo or a local directory (e.g. `orpheus-3B/outputs/orpheus_finetune_16bit`). vLLM wants a **merged** checkpoint — pure LoRA adapters need to be merged first (the training notebook's #5.2 already produces a merged 16-bit copy you can point this at).\n", + "\n", + "**Tuning knobs:**\n", + "- `gpu_memory_utilization=0.85` leaves ~15% headroom for SNAC + activation tensors. Drop to `0.70` if you get OOM during model load on a 16 GB GPU.\n", + "- `max_model_len=4096` matches the training context length. vLLM pre-allocates KV cache for this; larger = more VRAM consumed even when prompts are short.\n", + "- `enforce_eager=False` keeps CUDA graph compilation on (faster decode); set `True` to skip the ~30 s warm-up at startup." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1f10161e", + "metadata": {}, + "outputs": [], + "source": [ + "MODEL_ID = \"sunbird/orpheus-3b-tts-salt-lug-0001\"\n", + "# Or load a local merged checkpoint:\n", + "# MODEL_ID = \"orpheus-3B/outputs/orpheus_finetune_16bit\"\n", + "\n", + "SPEAKER_ID = \"salt_lug_0001\"\n", + "MAX_MODEL_LEN = 4096\n", + "\n", + "# Orpheus-3B uses vocab_size=156939 (Llama-3 base 128256 + 7*4096 audio\n", + "# codebook + 11 special tokens). Unsloth's `push_to_hub_merged` shim can\n", + "# add a pad token to the tokenizer at save time, bumping `len(tokenizer)`\n", + "# to 156940 and writing that into config.json — while leaving the actual\n", + "# embedding weight at [156939, 3072]. transformers tolerates the\n", + "# off-by-one; vLLM asserts equality and crashes. Override here so vLLM\n", + "# aligns config.vocab_size with the real embedding shape.\n", + "llm = LLM(\n", + " model = MODEL_ID,\n", + " dtype = \"bfloat16\", # auto-detected on Ampere+\n", + " max_model_len = MAX_MODEL_LEN,\n", + " gpu_memory_utilization = 0.85,\n", + " enforce_eager = False,\n", + " trust_remote_code = False,\n", + " hf_overrides = {\"vocab_size\": 156939},\n", + ")\n", + "print(f\"vLLM loaded: {MODEL_ID}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "98ba8d08", + "metadata": {}, + "outputs": [], + "source": [ + "tokenizer = AutoTokenizer.from_pretrained(MODEL_ID, token=os.environ.get(\"HF_TOKEN\"))\n", + "print(f\"Tokenizer vocab size: {tokenizer.vocab_size}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "40efe602", + "metadata": {}, + "outputs": [], + "source": [ + "# SNAC decoder runs on CPU — frees GPU for vLLM's KV cache.\n", + "snac_model = SNAC.from_pretrained(\"hubertsiuzdak/snac_24khz\").to(\"cpu\")\n", + "print(\"SNAC 24 kHz decoder loaded (on CPU)\")" + ] + }, + { + "cell_type": "markdown", + "id": "dfbd6df4", + "metadata": {}, + "source": [ + "## #2 Token constants + helpers\n", + "\n", + "Same special-token layout as the training notebook. The audio codebook spans `[128266, 128266 + 7·4096) = [128266, 156938)`; anything outside that range in vLLM's output is treated as garbage and filtered before SNAC decoding." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cdd730ff", + "metadata": {}, + "outputs": [], + "source": [ + "# Special tokens (must match #2.6 of the training notebook)\n", + "END_OF_TEXT = 128009\n", + "START_OF_SPEECH = 128257\n", + "END_OF_SPEECH = 128258\n", + "START_OF_HUMAN = 128259\n", + "END_OF_HUMAN = 128260\n", + "PAD_TOKEN = 128263\n", + "AUDIO_TOKEN_LO = 128266\n", + "AUDIO_TOKEN_HI = 128266 + 7 * 4096 # exclusive (156938)\n", + "\n", + "\n", + "def build_prompt_token_ids(text: str, speaker_id: str) -> list[int]:\n", + " \"\"\"[SOH] + tokenizer(speaker_id: text) + [EOT, EOH].\"\"\"\n", + " tagged = f\"{speaker_id}: {text}\"\n", + " text_ids = tokenizer.encode(tagged, add_special_tokens=True)\n", + " return [START_OF_HUMAN] + text_ids + [END_OF_TEXT, END_OF_HUMAN]\n", + "\n", + "\n", + "def codes_to_waveform(generated_token_ids: list[int]) -> np.ndarray:\n", + " \"\"\"vLLM-output token_ids → 24 kHz mono float32 waveform.\n", + "\n", + " The last SOS marker (128257) appears in the model's output before\n", + " the audio codes start. After cropping past it, we keep only\n", + " audio-range tokens, redistribute through SNAC layers, and decode.\n", + " \"\"\"\n", + " ids = torch.tensor(generated_token_ids, dtype=torch.int64)\n", + "\n", + " # Crop on last SOS — discards [SOA, SOS] preamble and any\n", + " # pre-speech meta tokens vLLM may have emitted.\n", + " sos_positions = (ids == START_OF_SPEECH).nonzero(as_tuple=True)[0]\n", + " if len(sos_positions) > 0:\n", + " ids = ids[sos_positions[-1].item() + 1:]\n", + " else:\n", + " print(\"WARNING: no SOS in vLLM output — will likely produce silence.\")\n", + "\n", + " # Filter to audio-codebook range (strips EOS + any garbage tokens).\n", + " audio_only = ids[(ids >= AUDIO_TOKEN_LO) & (ids < AUDIO_TOKEN_HI)]\n", + " n = (audio_only.size(0) // 7) * 7\n", + " code_list = [t.item() - AUDIO_TOKEN_LO for t in audio_only[:n]]\n", + "\n", + " layer_1, layer_2, layer_3 = [], [], []\n", + " for i in range(len(code_list) // 7):\n", + " layer_1.append(code_list[7*i])\n", + " layer_2.append(code_list[7*i + 1] - 4096)\n", + " layer_3.append(code_list[7*i + 2] - 2*4096)\n", + " layer_3.append(code_list[7*i + 3] - 3*4096)\n", + " layer_2.append(code_list[7*i + 4] - 4*4096)\n", + " layer_3.append(code_list[7*i + 5] - 5*4096)\n", + " layer_3.append(code_list[7*i + 6] - 6*4096)\n", + " if not layer_1:\n", + " return np.zeros(12000, dtype=np.float32) # ~0.5s silence fallback\n", + "\n", + " def clamp(vals):\n", + " return [max(0, min(4095, v)) for v in vals]\n", + " codes = [torch.tensor(clamp(layer_1)).unsqueeze(0),\n", + " torch.tensor(clamp(layer_2)).unsqueeze(0),\n", + " torch.tensor(clamp(layer_3)).unsqueeze(0)]\n", + " waveform = snac_model.decode(codes)\n", + " return waveform.detach().squeeze().to(\"cpu\").numpy().astype(np.float32)" + ] + }, + { + "cell_type": "markdown", + "id": "4d462a99", + "metadata": {}, + "source": [ + "## #3 `synthesize()` — single-prompt entry point\n", + "\n", + "Mirrors the HF-inference notebook's API: `text in → 24 kHz float32 waveform out`. Internally calls `llm.generate(...)` with the speaker-tagged token-IDs prompt and SNAC-decodes the result. For multi-prompt throughput use `synthesize_batch` in #5 instead — calling `synthesize` in a Python loop forfeits vLLM's continuous batching." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1b4cb381", + "metadata": {}, + "outputs": [], + "source": [ + "def synthesize(\n", + " text: str,\n", + " speaker_id: str = SPEAKER_ID,\n", + " *,\n", + " max_tokens: int = 1200,\n", + " temperature: float = 0.6,\n", + " top_p: float = 0.95,\n", + " repetition_penalty: float = 1.1,\n", + " seed: int | None = None,\n", + ") -> np.ndarray:\n", + " \"\"\"Synthesize speech for `text`.\n", + "\n", + " Returns a 1-D float32 numpy array at 24 kHz mono.\n", + " Pass an int `seed` for reproducible output.\n", + " \"\"\"\n", + " sampling_params = SamplingParams(\n", + " temperature = temperature,\n", + " top_p = top_p,\n", + " repetition_penalty = repetition_penalty,\n", + " max_tokens = max_tokens,\n", + " stop_token_ids = [END_OF_SPEECH],\n", + " skip_special_tokens = False, # we need raw token_ids\n", + " seed = seed,\n", + " )\n", + " prompt_ids = build_prompt_token_ids(text, speaker_id)\n", + " outputs = llm.generate(\n", + " [{\"prompt_token_ids\": prompt_ids}],\n", + " sampling_params,\n", + " )\n", + " generated = list(outputs[0].outputs[0].token_ids)\n", + " return codes_to_waveform(generated)" + ] + }, + { + "cell_type": "markdown", + "id": "76114ea2", + "metadata": {}, + "source": [ + "## #4 Free-text inference — try your own Luganda sentence\n", + "\n", + "Edit `prompt` and re-run. The first request pays a one-time CUDA-graph compile cost (~30 s); subsequent calls are near-instant for short utterances." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1a3d96fe", + "metadata": {}, + "outputs": [], + "source": [ + "prompt = \"Mwattu, oli otya?\" # ← put any Luganda sentence here\n", + "\n", + "wav = synthesize(prompt, seed=42)\n", + "out_path = INFERENCE_OUTPUT_DIR / \"free_text.wav\"\n", + "sf.write(str(out_path), wav, 24000)\n", + "print(f\"{prompt}\")\n", + "print(f\" -> {out_path} ({len(wav)/24000:.2f}s)\")\n", + "display(AudioDisplay(wav, rate=24000))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2ce11c68", + "metadata": {}, + "outputs": [], + "source": [ + "import itertools, time\n", + "\n", + "PROMPT = \"Mwattu, oli otya? Webale nnyo okwagala Uganda.\"\n", + "SEED = 42 # fixed seed isolates the effect of (temp, top_p) from RNG noise\n", + "\n", + "GRID = list(itertools.product(\n", + " [0.3, 0.6, 0.8, 1.0], # temperature axis\n", + " [0.80, 0.90, 0.95], # top_p axis\n", + "))\n", + "\n", + "prompt_ids = build_prompt_token_ids(PROMPT, SPEAKER_ID)\n", + "prompts = [{\"prompt_token_ids\": prompt_ids} for _ in GRID]\n", + "sps = [\n", + " SamplingParams(\n", + " temperature=t, top_p=p,\n", + " repetition_penalty=1.1, max_tokens=1200,\n", + " stop_token_ids=[END_OF_SPEECH], skip_special_tokens=False,\n", + " seed=SEED,\n", + " )\n", + " for t, p in GRID\n", + "]\n", + "\n", + "t0 = time.perf_counter()\n", + "outs = llm.generate(prompts, sps)\n", + "print(f\"sweep of {len(GRID)} variants in one batch: {time.perf_counter()-t0:.2f}s\")\n", + "\n", + "for (t, p), out in zip(GRID, outs):\n", + " wav = codes_to_waveform(list(out.outputs[0].token_ids))\n", + " out_path = INFERENCE_OUTPUT_DIR / f\"sweep_t{t:.1f}_p{p:.2f}.wav\"\n", + " sf.write(str(out_path), wav, 24000)\n", + " print(f\"\\nT={t} top_p={p} -> {out_path.name} ({len(wav)/24000:.2f}s)\")\n", + " display(AudioDisplay(wav, rate=24000))\n" + ] + }, + { + "cell_type": "markdown", + "id": "893fa6e8", + "metadata": {}, + "source": [ + "## #5 Batch inference — vLLM's main throughput advantage\n", + "\n", + "vLLM's continuous batching schedules many in-flight requests through one shared GPU pass. Passing `N` prompts to a single `llm.generate(...)` call is dramatically faster than calling `synthesize` `N` times in a loop, because the per-request overhead (kernel launches, host↔device sync) is amortized.\n", + "\n", + "Each prompt can specify its own `speaker_id`, so this also demonstrates multi-speaker batching for the multilingual checkpoint." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cbed20dc", + "metadata": {}, + "outputs": [], + "source": [ + "def synthesize_batch(\n", + " items: list[dict],\n", + " *,\n", + " max_tokens: int = 1200,\n", + " temperature: float = 0.6,\n", + " top_p: float = 0.95,\n", + " repetition_penalty: float = 1.1,\n", + " seed: int | None = None,\n", + ") -> list[np.ndarray]:\n", + " \"\"\"Synthesize many prompts in one vLLM batch.\n", + "\n", + " Each `items[i]` is a dict with keys:\n", + " - `text`: the transcript\n", + " - `speaker_id`: optional, defaults to SPEAKER_ID\n", + " Returns a list of 1-D float32 numpy waveforms, same length\n", + " and order as `items`.\n", + " \"\"\"\n", + " sampling_params = SamplingParams(\n", + " temperature = temperature,\n", + " top_p = top_p,\n", + " repetition_penalty = repetition_penalty,\n", + " max_tokens = max_tokens,\n", + " stop_token_ids = [END_OF_SPEECH],\n", + " skip_special_tokens = False,\n", + " seed = seed,\n", + " )\n", + " prompts = [\n", + " {\"prompt_token_ids\": build_prompt_token_ids(\n", + " it[\"text\"], it.get(\"speaker_id\", SPEAKER_ID))}\n", + " for it in items\n", + " ]\n", + " outputs = llm.generate(prompts, sampling_params)\n", + " return [codes_to_waveform(list(o.outputs[0].token_ids)) for o in outputs]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1490608a", + "metadata": {}, + "outputs": [], + "source": [ + "import time\n", + "\n", + "demo_items = [\n", + " {\"text\": \"Mwattu, oli otya?\"},\n", + " {\"text\": \"Webale nyo okwagala Uganda.\"},\n", + " {\"text\": \"Tunaagenda mu Kampala olwa leero.\"},\n", + " {\"text\": \"Twebale ku kutu.\"},\n", + "]\n", + "\n", + "t0 = time.perf_counter()\n", + "wavs = synthesize_batch(demo_items, seed=123)\n", + "elapsed = time.perf_counter() - t0\n", + "total_audio_sec = sum(len(w) / 24000 for w in wavs)\n", + "rtf = elapsed / total_audio_sec if total_audio_sec > 0 else float('inf')\n", + "print(f\"batch of {len(demo_items)}: {elapsed:.2f}s wall, {total_audio_sec:.2f}s audio,\"\n", + " f\" real-time-factor = {rtf:.3f} (lower is better, < 1 means faster than realtime)\")\n", + "\n", + "for idx, (item, wav) in enumerate(zip(demo_items, wavs)):\n", + " out_path = INFERENCE_OUTPUT_DIR / f\"batch_{idx:02d}.wav\"\n", + " sf.write(str(out_path), wav, 24000)\n", + " print(f\"[{idx}] {item['text']} -> {out_path}\")\n", + " display(AudioDisplay(wav, rate=24000))" + ] + }, + { + "cell_type": "markdown", + "id": "b0903768", + "metadata": {}, + "source": [ + "## #6 Held-out test split — A/B vs ground truth\n", + "\n", + "Pulls the first 5 utterances from `Sunbird/tts (lug)` test split, runs them through `synthesize_batch`, and saves each synthesized version next to its ground-truth recording. Same evaluation pattern as the HF-inference notebook — useful as a head-to-head check that vLLM produces equivalent audio quality to `transformers` generate (it should)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9a8da458", + "metadata": {}, + "outputs": [], + "source": [ + "ds_test = load_dataset(\"Sunbird/tts\", \"lug\", split=\"test\")\n", + "ds_test = ds_test.filter(lambda r: r[\"speaker_id\"] == SPEAKER_ID)\n", + "ds_test = ds_test.cast_column(\"audio\", Audio(sampling_rate=24000))\n", + "print(f\"{len(ds_test)} test rows for {SPEAKER_ID}\")\n", + "\n", + "n_compare = min(5, len(ds_test))\n", + "items = [{\"text\": ds_test[i][\"text\"], \"speaker_id\": SPEAKER_ID}\n", + " for i in range(n_compare)]\n", + "\n", + "t0 = time.perf_counter()\n", + "wavs = synthesize_batch(items, seed=42)\n", + "print(f\"\\nvLLM batch of {n_compare}: {time.perf_counter() - t0:.2f}s wall\\n\")\n", + "\n", + "for i in range(n_compare):\n", + " text = items[i][\"text\"]\n", + " print(f\"[{i}] {text[:120]}\")\n", + " synth_path = INFERENCE_OUTPUT_DIR / f\"test_{i:02d}_synth.wav\"\n", + " truth_path = INFERENCE_OUTPUT_DIR / f\"test_{i:02d}_groundtruth.wav\"\n", + " sf.write(str(synth_path), wavs[i], 24000)\n", + " sf.write(str(truth_path), ds_test[i][\"audio\"][\"array\"], 24000)\n", + " print(f\" synth -> {synth_path}\")\n", + " print(f\" ground truth -> {truth_path}\")\n", + " print(\" [synth]\")\n", + " display(AudioDisplay(wavs[i], rate=24000))\n", + " print(\" [ground truth]\")\n", + " display(AudioDisplay(ds_test[i][\"audio\"][\"array\"], rate=24000))\n", + " print()" + ] + }, + { + "cell_type": "markdown", + "id": "3cce9924", + "metadata": {}, + "source": [ + "## #7 Production deployment with `vllm serve`\n", + "\n", + "vLLM ships with an OpenAI-compatible HTTP server. For TTS we want a custom endpoint that returns `audio/wav` rather than JSON token streams, so the cleanest pattern is to wrap the in-process `LLM` instance behind FastAPI (same as the HF-inference notebook) — but with vLLM's batched generate doing the heavy lifting.\n", + "\n", + "Below is a **runnable** FastAPI server template. Save the snippet to `vllm_tts_server.py`, then `uvicorn vllm_tts_server:app --host 0.0.0.0 --port 8000`. It supports both single-text and batched (multi-text) requests on a single GPU.\n", + "\n", + "**Why not `vllm serve`?** That CLI exposes the OpenAI Chat Completions API and serializes outputs as text/JSON. For TTS you need raw token IDs back so you can run SNAC decoding — the wrapper below is the simplest way." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dd135035", + "metadata": {}, + "outputs": [], + "source": [ + "VLLM_FASTAPI_SNIPPET = '''\n", + "# vllm_tts_server.py — minimal Orpheus-3B TTS server backed by vLLM.\n", + "# Run: uvicorn vllm_tts_server:app --host 0.0.0.0 --port 8000\n", + "import io\n", + "import os\n", + "from typing import Optional\n", + "\n", + "import numpy as np\n", + "import soundfile as sf\n", + "import torch\n", + "from fastapi import FastAPI\n", + "from fastapi.responses import Response\n", + "from pydantic import BaseModel\n", + "from snac import SNAC\n", + "from transformers import AutoTokenizer\n", + "from vllm import LLM, SamplingParams\n", + "\n", + "MODEL_ID = os.environ.get(\"ORPHEUS_MODEL_ID\", \"sunbird/orpheus-3b-tts-salt-lug-0001\")\n", + "DEFAULT_SPEAKER = os.environ.get(\"ORPHEUS_SPEAKER\", \"salt_lug_0001\")\n", + "MAX_MODEL_LEN = int(os.environ.get(\"ORPHEUS_MAX_MODEL_LEN\", \"4096\"))\n", + "\n", + "END_OF_TEXT, START_OF_SPEECH, END_OF_SPEECH = 128009, 128257, 128258\n", + "START_OF_HUMAN, END_OF_HUMAN = 128259, 128260\n", + "AUDIO_TOKEN_LO, AUDIO_TOKEN_HI = 128266, 128266 + 7 * 4096\n", + "\n", + "llm = LLM(model=MODEL_ID, dtype=\"bfloat16\", max_model_len=MAX_MODEL_LEN,\n", + " gpu_memory_utilization=0.85,\n", + " hf_overrides={\"vocab_size\": 156939})\n", + "tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)\n", + "snac_model = SNAC.from_pretrained(\"hubertsiuzdak/snac_24khz\").to(\"cpu\")\n", + "\n", + "def _build_prompt_ids(text, speaker):\n", + " tagged = f\"{speaker}: {text}\"\n", + " text_ids = tokenizer.encode(tagged, add_special_tokens=True)\n", + " return [START_OF_HUMAN] + text_ids + [END_OF_TEXT, END_OF_HUMAN]\n", + "\n", + "def _codes_to_wav(token_ids):\n", + " ids = torch.tensor(token_ids, dtype=torch.int64)\n", + " sos = (ids == START_OF_SPEECH).nonzero(as_tuple=True)[0]\n", + " if len(sos) > 0: ids = ids[sos[-1].item() + 1:]\n", + " audio = ids[(ids >= AUDIO_TOKEN_LO) & (ids < AUDIO_TOKEN_HI)]\n", + " n = (audio.size(0) // 7) * 7\n", + " cl = [t.item() - AUDIO_TOKEN_LO for t in audio[:n]]\n", + " l1, l2, l3 = [], [], []\n", + " for i in range(len(cl) // 7):\n", + " l1.append(cl[7*i])\n", + " l2.append(cl[7*i+1] - 4096); l3.append(cl[7*i+2] - 2*4096)\n", + " l3.append(cl[7*i+3] - 3*4096); l2.append(cl[7*i+4] - 4*4096)\n", + " l3.append(cl[7*i+5] - 5*4096); l3.append(cl[7*i+6] - 6*4096)\n", + " if not l1: return np.zeros(12000, dtype=np.float32)\n", + " cb = lambda v: [max(0, min(4095, x)) for x in v]\n", + " codes = [torch.tensor(cb(l1)).unsqueeze(0), torch.tensor(cb(l2)).unsqueeze(0),\n", + " torch.tensor(cb(l3)).unsqueeze(0)]\n", + " return snac_model.decode(codes).detach().squeeze().cpu().numpy().astype(np.float32)\n", + "\n", + "app = FastAPI()\n", + "\n", + "class TTSReq(BaseModel):\n", + " text: str\n", + " speaker_id: str = DEFAULT_SPEAKER\n", + " seed: Optional[int] = None\n", + " temperature: float = 0.6\n", + " top_p: float = 0.95\n", + " repetition_penalty: float = 1.1\n", + " max_tokens: int = 1200\n", + "\n", + "@app.post(\"/tts\", responses={200: {\"content\": {\"audio/wav\": {}}}})\n", + "def tts(req: TTSReq):\n", + " sp = SamplingParams(\n", + " temperature=req.temperature, top_p=req.top_p,\n", + " repetition_penalty=req.repetition_penalty,\n", + " max_tokens=req.max_tokens, stop_token_ids=[END_OF_SPEECH],\n", + " skip_special_tokens=False, seed=req.seed,\n", + " )\n", + " pids = _build_prompt_ids(req.text, req.speaker_id)\n", + " out = llm.generate([{\"prompt_token_ids\": pids}], sp)\n", + " wav = _codes_to_wav(list(out[0].outputs[0].token_ids))\n", + " buf = io.BytesIO()\n", + " sf.write(buf, wav, 24000, format=\"WAV\", subtype=\"PCM_16\")\n", + " return Response(content=buf.getvalue(), media_type=\"audio/wav\")\n", + "\n", + "class TTSBatchReq(BaseModel):\n", + " items: list[TTSReq]\n", + "\n", + "@app.post(\"/tts/batch\")\n", + "def tts_batch(req: TTSBatchReq):\n", + " sp = SamplingParams(\n", + " temperature=req.items[0].temperature, top_p=req.items[0].top_p,\n", + " repetition_penalty=req.items[0].repetition_penalty,\n", + " max_tokens=req.items[0].max_tokens,\n", + " stop_token_ids=[END_OF_SPEECH], skip_special_tokens=False,\n", + " )\n", + " prompts = [{\"prompt_token_ids\": _build_prompt_ids(it.text, it.speaker_id)} for it in req.items]\n", + " outs = llm.generate(prompts, sp)\n", + " wavs = [_codes_to_wav(list(o.outputs[0].token_ids)) for o in outs]\n", + " # Multipart return omitted for brevity — typically zip the wavs or stream them.\n", + " return {\"sample_rates\": [24000] * len(wavs), \"durations_sec\": [len(w)/24000 for w in wavs]}\n", + "'''\n", + "print(VLLM_FASTAPI_SNIPPET)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/modal-deploy/orpheus-3B/Orpheus_3B_Sunbird_Multilingual.ipynb b/modal-deploy/orpheus-3B/Orpheus_3B_Sunbird_Multilingual.ipynb new file mode 100644 index 00000000..ae7cbdd4 --- /dev/null +++ b/modal-deploy/orpheus-3B/Orpheus_3B_Sunbird_Multilingual.ipynb @@ -0,0 +1,902 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "52dc3b6d", + "metadata": {}, + "source": [ + "# Orpheus-3B Sunbird Multilingual Finetune\n", + "\n", + "Finetune `unsloth/orpheus-3b-0.1-pretrained` on the **full** Sunbird/tts corpus — every available language config and every speaker — with MLflow tracking and a final push to `sunbird/orpheus-3b-tts-multilingual`.\n", + "\n", + "**Differences from `Orpheus_3B_Sunbird_Luganda.ipynb`:**\n", + "- #2.1 discovers all language configs via `get_dataset_config_names(\"Sunbird/tts\")` and concatenates them into one `ds_train` / `ds_test`. No language or speaker filter.\n", + "- #2.2 sets `source = example[\"speaker_id\"]` so every row carries its own speaker tag (per-row, not constant). The model learns the multi-speaker prompt format `f\"{speaker_id}: {text}\"` across all speakers it sees.\n", + "- #2.7 holds aside up to 10 *diverse* test prompts (one per unique speaker) for inference quality eyeballing.\n", + "- #3 writes checkpoints to `orpheus-3B/outputs/multilingual/`.\n", + "- #5 pushes to `sunbird/orpheus-3b-tts-multilingual`.\n", + "\n", + "**Resource note:** The combined dataset is several GB and tens of thousands of utterances. Expect 4–8 hours of training on an RTX 4090 at `num_train_epochs=3`. The smoke default (`max_steps=60`) still completes in a few minutes for pipeline verification.\n", + "\n", + "**References**: `Orpheus_(3B)_TTS.ipynb` and https://unsloth.ai/docs/basics/text-to-speech-tts-fine-tuning" + ] + }, + { + "cell_type": "markdown", + "id": "e56825bf", + "metadata": {}, + "source": [ + "## #0 Setup & secrets" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e9fff530", + "metadata": {}, + "outputs": [], + "source": [ + "%%capture\n", + "import os, re\n", + "if \"COLAB_\" not in \"\".join(os.environ.keys()):\n", + " !pip install unsloth\n", + "else:\n", + " import torch; v = re.match(r'[\\d]{1,}\\.[\\d]{1,}', str(torch.__version__)).group(0)\n", + " xformers = 'xformers==' + {'2.10':'0.0.34','2.9':'0.0.33.post1','2.8':'0.0.32.post2'}.get(v, '0.0.34')\n", + " !pip install sentencepiece protobuf \"datasets==4.3.0\" \"huggingface_hub>=0.34.0\" hf_transfer\n", + " !pip install --no-deps unsloth_zoo bitsandbytes accelerate {xformers} peft trl triton unsloth\n", + " !pip install --no-deps --upgrade \"torchao>=0.16.0\"\n", + "!pip install transformers==4.56.2\n", + "!pip install --no-deps trl==0.22.2\n", + "!pip install snac torchcodec \"datasets>=3.4.1,<4.0.0\"\n", + "# `--ignore-installed blinker` works around vast.ai/Debian images where\n", + "# blinker is installed via apt (no RECORD file) and pip refuses to upgrade it.\n", + "!pip install --ignore-installed blinker mlflow soundfile librosa" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dafb66bd", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "from getpass import getpass\n", + "\n", + "# HF + MLflow credentials (interactive)\n", + "os.environ[\"HF_TOKEN\"] = getpass(\"HF_TOKEN: \")\n", + "os.environ.setdefault(\"MLFLOW_TRACKING_URI\", \"https://mlflow.sunbird.ai/\")\n", + "os.environ.setdefault(\"MLFLOW_EXPERIMENT_NAME\", \"orpheus-3b-tts-multilingual-finetuning\")\n", + "os.environ[\"MLFLOW_TRACKING_USERNAME\"] = getpass(\"MLFLOW_TRACKING_USERNAME: \")\n", + "os.environ[\"MLFLOW_TRACKING_PASSWORD\"] = getpass(\"MLFLOW_TRACKING_PASSWORD: \")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "90af50a4", + "metadata": {}, + "outputs": [], + "source": [ + "import locale\n", + "from collections import Counter\n", + "from pathlib import Path\n", + "\n", + "import torch\n", + "import soundfile as sf\n", + "from datasets import load_dataset, Audio, concatenate_datasets, get_dataset_config_names\n", + "from snac import SNAC\n", + "from IPython.display import display, Audio as AudioDisplay\n", + "\n", + "locale.getpreferredencoding = lambda: \"UTF-8\"\n", + "\n", + "OUTPUT_DIR = Path(\"orpheus-3B/outputs/multilingual\")\n", + "OUTPUT_DIR.mkdir(parents=True, exist_ok=True)\n", + "(OUTPUT_DIR / \"inference_samples\").mkdir(parents=True, exist_ok=True)" + ] + }, + { + "cell_type": "markdown", + "id": "64be186c", + "metadata": {}, + "source": [ + "## #1 Load model + LoRA" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "913d3c16", + "metadata": {}, + "outputs": [], + "source": [ + "from unsloth import FastLanguageModel\n", + "\n", + "# 4096 tokens ≈ 16 s of audio — covers most Sunbird utterances after\n", + "# SNAC tokenisation. Longer rows are dropped by the length filter in #2.6.\n", + "# Drop to 2048 if you're on <16 GB VRAM; bump to 8192 if you see many\n", + "# rows dropped at the #2.6 filter.\n", + "MAX_SEQ_LEN = 4096\n", + "\n", + "model, tokenizer = FastLanguageModel.from_pretrained(\n", + " model_name = \"unsloth/orpheus-3b-0.1-pretrained\",\n", + " max_seq_length = MAX_SEQ_LEN,\n", + " dtype = None, # auto-detect bf16 on Ampere+, fp16 on T4\n", + " load_in_4bit = False, # LoRA 16-bit for higher accuracy on TTS\n", + " token = os.environ[\"HF_TOKEN\"],\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8d7f3324", + "metadata": {}, + "outputs": [], + "source": [ + "model = FastLanguageModel.get_peft_model(\n", + " model,\n", + " r = 64,\n", + " target_modules = [\"q_proj\", \"k_proj\", \"v_proj\", \"o_proj\",\n", + " \"gate_proj\", \"up_proj\", \"down_proj\"],\n", + " lora_alpha = 64,\n", + " lora_dropout = 0,\n", + " bias = \"none\",\n", + " use_gradient_checkpointing = \"unsloth\",\n", + " random_state = 3407,\n", + " use_rslora = False,\n", + " loftq_config = None,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "b4992eab", + "metadata": {}, + "source": [ + "## #2 Data prep — multilingual, multi-speaker" + ] + }, + { + "cell_type": "markdown", + "id": "8517e151", + "metadata": {}, + "source": [ + "### #2.1 Discover and load all language configs\n", + "\n", + "Pulls every config available on `Sunbird/tts` and concatenates the train / test splits into single datasets. To restrict to a subset, override `CONFIGS` with an explicit list (e.g. `CONFIGS = [\"lug\", \"ach\"]`)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ba203adc", + "metadata": {}, + "outputs": [], + "source": [ + "CONFIGS = get_dataset_config_names(\"Sunbird/tts\", token=os.environ.get(\"HF_TOKEN\"))\n", + "# Or override: CONFIGS = [\"lug\", \"ach\", \"nyn\"]\n", + "print(f\"Available language configs: {CONFIGS}\")\n", + "\n", + "train_parts, test_parts = [], []\n", + "for cfg in CONFIGS:\n", + " print(f\"Loading {cfg}...\")\n", + " train_parts.append(load_dataset(\"Sunbird/tts\", cfg, split=\"train\"))\n", + " test_parts.append(load_dataset(\"Sunbird/tts\", cfg, split=\"test\"))\n", + "\n", + "ds_train_full = concatenate_datasets(train_parts)\n", + "ds_test_full = concatenate_datasets(test_parts)\n", + "ds_train = ds_train_full # no speaker filter\n", + "ds_test = ds_test_full\n", + "\n", + "print(f\"\\ntrain rows: {len(ds_train)}\")\n", + "print(f\"test rows: {len(ds_test)}\")\n", + "\n", + "speaker_counts = Counter(ds_train[\"speaker_id\"])\n", + "print(f\"\\n{len(speaker_counts)} unique speakers in train; top 10 by row count:\")\n", + "for sid, n in speaker_counts.most_common(10):\n", + " print(f\" {sid}: {n}\")\n", + "\n", + "lang_counts = Counter(ds_train[\"language\"])\n", + "print(f\"\\n{len(lang_counts)} unique languages in train:\")\n", + "for lang, n in lang_counts.most_common():\n", + " print(f\" {lang}: {n}\")" + ] + }, + { + "cell_type": "markdown", + "id": "e7b76996", + "metadata": {}, + "source": [ + "### #2.2 Tag every row with `source = speaker_id` (per-row, not constant)\n", + "\n", + "The training prompt becomes `f\"{speaker_id}: {text}\"`, so the model learns to condition on the speaker tag and produce that speaker's voice at inference time. Unlike the single-speaker notebook (where every row had the same constant tag), here `source` varies per row across the entire speaker pool." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9c2f6954", + "metadata": {}, + "outputs": [], + "source": [ + "def add_source(example):\n", + " example[\"source\"] = example[\"speaker_id\"]\n", + " return example\n", + "\n", + "ds_train = ds_train.map(add_source)\n", + "ds_test = ds_test.map(add_source)\n", + "ds_train[0].keys()" + ] + }, + { + "cell_type": "markdown", + "id": "a023e31f", + "metadata": {}, + "source": [ + "### #2.3 Cast `audio` to 24 kHz (Orpheus / SNAC expected sample rate)\n", + "\n", + "Sunbird/tts stores audio as `{\"bytes\": , \"path\": ...}` (decode=False). Casting to `Audio(sampling_rate=24000)` switches the column to lazy-decode mode — on access `row[\"audio\"]` returns `{\"array\": np.ndarray, \"sampling_rate\": 24000, \"path\": str}`, which is what `add_codes` consumes in #2.4." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1882bd6e", + "metadata": {}, + "outputs": [], + "source": [ + "ds_train = ds_train.cast_column(\"audio\", Audio(sampling_rate=24000))\n", + "ds_test = ds_test.cast_column(\"audio\", Audio(sampling_rate=24000))\n", + "# Sanity check — should print 24000 (and KeyError if the cast didn't take).\n", + "ds_train[0][\"audio\"][\"sampling_rate\"]" + ] + }, + { + "cell_type": "markdown", + "id": "86ad623a", + "metadata": {}, + "source": [ + "### #2.4 SNAC tokenisation (24 kHz, 7 codes per frame, per-layer offsets)\n", + "\n", + "Same SNAC encoding as the single-speaker notebook. With tens of thousands of utterances this step is the slowest part of data prep — 30–90 minutes on an RTX 4090. HF `datasets.map` caches the result to disk, so re-running the notebook skips re-encoding.\n", + "\n", + "**Pre-filter**: before encoding, drop any row whose tokenised text alone exceeds `MAX_SEQ_LEN`. Such a row could never fit in context (audio codes only push it further over), so SNAC-encoding it would be wasted work." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f5de9e77", + "metadata": {}, + "outputs": [], + "source": [ + "# Pre-filter overlong text rows BEFORE the expensive SNAC pass.\n", + "# A row whose text alone tokenises to > MAX_SEQ_LEN cannot fit even\n", + "# without audio codes; the #2.6 length filter would drop it anyway,\n", + "# but only after we paid for SNAC encoding.\n", + "n_before = len(ds_train)\n", + "ds_train = ds_train.filter(\n", + " lambda r: len(tokenizer.encode(r[\"text\"], add_special_tokens=False)) <= MAX_SEQ_LEN\n", + ")\n", + "n_dropped = n_before - len(ds_train)\n", + "print(f\"text pre-filter (text tokens <= {MAX_SEQ_LEN}): \"\n", + " f\"{n_before} -> {len(ds_train)} ({n_dropped} dropped)\")\n", + "\n", + "snac_model = SNAC.from_pretrained(\"hubertsiuzdak/snac_24khz\").to(\"cuda\")\n", + "\n", + "def tokenise_audio(waveform):\n", + " waveform = torch.from_numpy(waveform).unsqueeze(0).to(dtype=torch.float32)\n", + " waveform = waveform.unsqueeze(0).to(\"cuda\")\n", + " with torch.inference_mode():\n", + " codes = snac_model.encode(waveform)\n", + " all_codes = []\n", + " for i in range(codes[0].shape[1]):\n", + " all_codes.append(codes[0][0][i].item() + 128266)\n", + " all_codes.append(codes[1][0][2*i].item() + 128266 + 4096)\n", + " all_codes.append(codes[2][0][4*i].item() + 128266 + 2*4096)\n", + " all_codes.append(codes[2][0][(4*i)+1].item() + 128266 + 3*4096)\n", + " all_codes.append(codes[1][0][(2*i)+1].item() + 128266 + 4*4096)\n", + " all_codes.append(codes[2][0][(4*i)+2].item() + 128266 + 5*4096)\n", + " all_codes.append(codes[2][0][(4*i)+3].item() + 128266 + 6*4096)\n", + " return all_codes\n", + "\n", + "def add_codes(example):\n", + " codes_list = None\n", + " try:\n", + " answer_audio = example.get(\"audio\")\n", + " if answer_audio and \"array\" in answer_audio:\n", + " codes_list = tokenise_audio(answer_audio[\"array\"])\n", + " except Exception as e:\n", + " print(f\"Skipping row due to error: {e}\")\n", + " example[\"codes_list\"] = codes_list\n", + " return example\n", + "\n", + "ds_train = ds_train.map(add_codes, remove_columns=[\"audio\"])" + ] + }, + { + "cell_type": "markdown", + "id": "18a964ad", + "metadata": {}, + "source": [ + "### #2.5 Drop empty/None code rows and remove duplicate frames" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b2e76e87", + "metadata": {}, + "outputs": [], + "source": [ + "ds_train = ds_train.filter(lambda x: x[\"codes_list\"] is not None)\n", + "ds_train = ds_train.filter(lambda x: len(x[\"codes_list\"]) > 0)\n", + "\n", + "def remove_duplicate_frames(example):\n", + " vals = example[\"codes_list\"]\n", + " if len(vals) % 7 != 0:\n", + " raise ValueError(\"Input list length must be divisible by 7\")\n", + " result = vals[:7]\n", + " for i in range(7, len(vals), 7):\n", + " if vals[i] != result[-7]:\n", + " result.extend(vals[i:i+7])\n", + " example[\"codes_list\"] = result\n", + " return example\n", + "\n", + "ds_train = ds_train.map(remove_duplicate_frames)\n", + "print(f\"after dedupe: {len(ds_train)} rows\")" + ] + }, + { + "cell_type": "markdown", + "id": "9a303c09", + "metadata": {}, + "source": [ + "### #2.6 Build training input_ids — multi-speaker tagged prompt format\n", + "\n", + "The `create_input_ids` body is identical to the single-speaker notebook — `f\"{example['source']}: {example['text']}\"`. The only difference is that `example['source']` now varies per row (it's the row's own `speaker_id`, set in #2.2)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cf21ced5", + "metadata": {}, + "outputs": [], + "source": [ + "TOKENISER_LENGTH = 128256\n", + "END_OF_TEXT = 128009\n", + "START_OF_SPEECH = TOKENISER_LENGTH + 1 # 128257\n", + "END_OF_SPEECH = TOKENISER_LENGTH + 2 # 128258\n", + "START_OF_HUMAN = TOKENISER_LENGTH + 3 # 128259\n", + "END_OF_HUMAN = TOKENISER_LENGTH + 4 # 128260\n", + "START_OF_AI = TOKENISER_LENGTH + 5 # 128261\n", + "END_OF_AI = TOKENISER_LENGTH + 6 # 128262\n", + "PAD_TOKEN = TOKENISER_LENGTH + 7 # 128263\n", + "AUDIO_TOKENS_START = TOKENISER_LENGTH + 10 # 128266\n", + "\n", + "def create_input_ids(example):\n", + " text_prompt = f\"{example['source']}: {example['text']}\"\n", + " text_ids = tokenizer.encode(text_prompt, add_special_tokens=True)\n", + " text_ids.append(END_OF_TEXT)\n", + " input_ids = (\n", + " [START_OF_HUMAN]\n", + " + text_ids\n", + " + [END_OF_HUMAN]\n", + " + [START_OF_AI]\n", + " + [START_OF_SPEECH]\n", + " + example[\"codes_list\"]\n", + " + [END_OF_SPEECH]\n", + " + [END_OF_AI]\n", + " )\n", + " return {\n", + " \"input_ids\": input_ids,\n", + " \"labels\": input_ids,\n", + " \"attention_mask\": [1] * len(input_ids),\n", + " }\n", + "\n", + "ds_train = ds_train.map(create_input_ids, remove_columns=[\"text\", \"codes_list\"])\n", + "ds_train = ds_train.remove_columns(\n", + " [c for c in ds_train.column_names if c not in (\"input_ids\", \"labels\", \"attention_mask\")]\n", + ")\n", + "\n", + "# Drop rows whose tokenised length exceeds MAX_SEQ_LEN (the model's max\n", + "# context). Long Sunbird utterances + SNAC's ~80-frame/sec rate can\n", + "# blow past 2048 easily; without this filter trainer.train() crashes\n", + "# inside unsloth_fused_ce_loss when its auto-truncation hits a\n", + "# torch.compile graph break.\n", + "n_before = len(ds_train)\n", + "ds_train = ds_train.filter(lambda r: len(r[\"input_ids\"]) <= MAX_SEQ_LEN)\n", + "n_dropped = n_before - len(ds_train)\n", + "print(f\"length filter (<= {MAX_SEQ_LEN} tokens): {n_before} -> {len(ds_train)} \"\n", + " f\"({n_dropped} dropped for being too long)\")\n", + "\n", + "print(\"train columns:\", ds_train.column_names)\n", + "print(\"first row len:\", len(ds_train[0][\"input_ids\"]))" + ] + }, + { + "cell_type": "markdown", + "id": "cd1d6fba", + "metadata": {}, + "source": [ + "### #2.7 Hold diverse test prompts aside — 1 per speaker, capped at 10\n", + "\n", + "Pulls a held-out evaluation set spanning as many speakers / languages as possible, so #4 inference can demonstrate the model's per-speaker conditioning. Shuffles the test split first to avoid bias toward whichever language was concatenated first." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "95963066", + "metadata": {}, + "outputs": [], + "source": [ + "MAX_TEST_PROMPTS = 10\n", + "ds_test_shuffled = ds_test.shuffle(seed=42)\n", + "\n", + "test_prompts = []\n", + "_seen_speakers = set()\n", + "for row in ds_test_shuffled:\n", + " sid = row[\"speaker_id\"]\n", + " if sid in _seen_speakers:\n", + " continue\n", + " _seen_speakers.add(sid)\n", + " test_prompts.append({\n", + " \"speaker_id\": sid,\n", + " \"language\": row[\"language\"],\n", + " \"text\": row[\"text\"],\n", + " })\n", + " if len(test_prompts) >= MAX_TEST_PROMPTS:\n", + " break\n", + "\n", + "print(f\"held-aside test prompts ({len(test_prompts)} unique speakers):\")\n", + "for p in test_prompts:\n", + " print(f\" [{p['language']}/{p['speaker_id']}] {p['text'][:80]}\")" + ] + }, + { + "cell_type": "markdown", + "id": "976d9acf", + "metadata": {}, + "source": [ + "## #3 Train\n", + "\n", + "Default is a smoke run (`max_steps=60`). For a real multilingual fine-tune, set `max_steps=-1` and uncomment `num_train_epochs=3`. On RTX 4090 with the full Sunbird corpus, the real run is roughly 4–8 hours depending on row count. Consider raising `per_device_train_batch_size` to 4 (and dropping `grad_accum` to 1) for ~2× wall-clock speedup, since the 4090 has VRAM headroom." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f27259a1", + "metadata": {}, + "outputs": [], + "source": [ + "from transformers import TrainingArguments, Trainer, DataCollatorForSeq2Seq\n", + "\n", + "# Llama-style tokenizers ship without a pad token. Reuse Orpheus's\n", + "# PAD_TOKEN (128263) so the collator can pad variable-length input_ids /\n", + "# attention_mask cleanly when batch size > 1.\n", + "if tokenizer.pad_token_id is None:\n", + " tokenizer.pad_token_id = PAD_TOKEN\n", + "\n", + "# DataCollatorForSeq2Seq pads input_ids/attention_mask to the longest\n", + "# sequence in the batch and pads labels with -100 (ignored by loss).\n", + "data_collator = DataCollatorForSeq2Seq(tokenizer=tokenizer, padding=True)\n", + "\n", + "trainer = Trainer(\n", + " model = model,\n", + " train_dataset = ds_train,\n", + " data_collator = data_collator,\n", + " args = TrainingArguments(\n", + " per_device_train_batch_size = 1,\n", + " gradient_accumulation_steps = 4,\n", + " warmup_steps = 5,\n", + " # max_steps = 60, # SMOKE DEFAULT\n", + " max_steps = -1, # uncomment for the real run\n", + " num_train_epochs = 3, # uncomment for the real run\n", + " learning_rate = 2e-4,\n", + " logging_steps = 1,\n", + " optim = \"adamw_8bit\",\n", + " weight_decay = 0.001,\n", + " lr_scheduler_type = \"linear\",\n", + " seed = 3407,\n", + " output_dir = str(OUTPUT_DIR),\n", + " report_to = \"mlflow\",\n", + " save_total_limit = 2, # cap disk usage\n", + " ),\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "1c035e09", + "metadata": {}, + "source": [ + "### GPU memory snapshot before training" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f1931212", + "metadata": {}, + "outputs": [], + "source": [ + "gpu_stats = torch.cuda.get_device_properties(0)\n", + "start_gpu_memory = round(torch.cuda.max_memory_reserved() / 1024 / 1024 / 1024, 3)\n", + "max_memory = round(gpu_stats.total_memory / 1024 / 1024 / 1024, 3)\n", + "print(f\"GPU = {gpu_stats.name}. Max memory = {max_memory} GB.\")\n", + "print(f\"{start_gpu_memory} GB of memory reserved.\")" + ] + }, + { + "cell_type": "markdown", + "id": "6930b340", + "metadata": {}, + "source": [ + "### Run training" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fe001e23", + "metadata": {}, + "outputs": [], + "source": [ + "trainer_stats = trainer.train()" + ] + }, + { + "cell_type": "markdown", + "id": "77526595", + "metadata": {}, + "source": [ + "### GPU memory snapshot after training" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9a26b68a", + "metadata": {}, + "outputs": [], + "source": [ + "used_memory = round(torch.cuda.max_memory_reserved() / 1024 / 1024 / 1024, 3)\n", + "used_for_lora = round(used_memory - start_gpu_memory, 3)\n", + "used_pct = round(used_memory / max_memory * 100, 3)\n", + "lora_pct = round(used_for_lora / max_memory * 100, 3)\n", + "print(f\"{trainer_stats.metrics['train_runtime']} seconds used for training.\")\n", + "print(f\"{round(trainer_stats.metrics['train_runtime']/60, 2)} minutes used for training.\")\n", + "print(f\"Peak reserved memory = {used_memory} GB.\")\n", + "print(f\"Peak reserved memory for training = {used_for_lora} GB.\")\n", + "print(f\"Peak reserved memory % of max memory = {used_pct} %.\")\n", + "print(f\"Peak reserved memory for training % of max memory = {lora_pct} %.\")" + ] + }, + { + "cell_type": "markdown", + "id": "71668118", + "metadata": {}, + "source": [ + "## #4 Inference" + ] + }, + { + "cell_type": "markdown", + "id": "c51222a5", + "metadata": {}, + "source": [ + "### #4.1 Use the diverse test_prompts from #2.7\n", + "\n", + "Each entry has its own `speaker_id`, so the model gets prompted with `f\"{speaker_id}: {text}\"` per prompt — different voices for different prompts." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "48b10f0a", + "metadata": {}, + "outputs": [], + "source": [ + "all_prompts = test_prompts # list of {speaker_id, language, text}\n", + "print(f\"running inference on {len(all_prompts)} prompts:\")\n", + "for p in all_prompts:\n", + " print(f\" [{p['language']}/{p['speaker_id']}] {p['text'][:100]}\")" + ] + }, + { + "cell_type": "markdown", + "id": "67135948", + "metadata": {}, + "source": [ + "### #4.2 Tokenise + pad + generate (per-prompt speaker)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0649bb6d", + "metadata": {}, + "outputs": [], + "source": [ + "FastLanguageModel.for_inference(model)\n", + "snac_model.to(\"cpu\") # free GPU for the LM\n", + "\n", + "prompts_tagged = [f\"{p['speaker_id']}: {p['text']}\" for p in all_prompts]\n", + "all_input_ids = [tokenizer(p, return_tensors=\"pt\").input_ids for p in prompts_tagged]\n", + "\n", + "start_token = torch.tensor([[128259]], dtype=torch.int64) # SOH\n", + "end_tokens = torch.tensor([[128009, 128260]], dtype=torch.int64) # EOT, EOH\n", + "\n", + "modified = [torch.cat([start_token, ids, end_tokens], dim=1) for ids in all_input_ids]\n", + "max_len = max(t.shape[1] for t in modified)\n", + "\n", + "padded, masks = [], []\n", + "for t in modified:\n", + " pad = max_len - t.shape[1]\n", + " padded.append(torch.cat([torch.full((1, pad), 128263, dtype=torch.int64), t], dim=1))\n", + " masks.append(torch.cat([torch.zeros((1, pad), dtype=torch.int64),\n", + " torch.ones((1, t.shape[1]), dtype=torch.int64)], dim=1))\n", + "\n", + "input_ids = torch.cat(padded, dim=0).to(\"cuda\")\n", + "attention_mask = torch.cat(masks, dim=0).to(\"cuda\")\n", + "\n", + "generated_ids = model.generate(\n", + " input_ids = input_ids,\n", + " attention_mask = attention_mask,\n", + " max_new_tokens = 1200,\n", + " do_sample = True,\n", + " temperature = 0.6,\n", + " top_p = 0.95,\n", + " repetition_penalty = 1.1,\n", + " num_return_sequences = 1,\n", + " eos_token_id = 128258,\n", + " use_cache = True,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "251df722", + "metadata": {}, + "source": [ + "### #4.3 Crop, strip, redistribute SNAC codes, decode" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "638b884b", + "metadata": {}, + "outputs": [], + "source": [ + "TOKEN_TO_FIND = 128257 # last SOS\n", + "AUDIO_TOKEN_LO = 128266 # first audio-codebook id\n", + "AUDIO_TOKEN_HI = 128266 + 7 * 4096 # exclusive (156938)\n", + "\n", + "indices = (generated_ids == TOKEN_TO_FIND).nonzero(as_tuple=True)\n", + "if len(indices[1]) > 0:\n", + " last = indices[1][-1].item()\n", + " cropped = generated_ids[:, last + 1:]\n", + "else:\n", + " print(\"WARNING: no start-of-speech token (128257) in generated output.\"\n", + " \" Smoke runs (max_steps=60) often produce garbage — switch to the\"\n", + " \" full-run epochs block in #3 for a real fine-tune.\")\n", + " cropped = generated_ids\n", + "\n", + "rows = [row[(row >= AUDIO_TOKEN_LO) & (row < AUDIO_TOKEN_HI)] for row in cropped]\n", + "\n", + "code_lists = []\n", + "for row in rows:\n", + " n = (row.size(0) // 7) * 7\n", + " if n == 0:\n", + " print(\"WARNING: row has no usable audio tokens after filtering;\"\n", + " \" emitting silence. (Smoke runs commonly hit this.)\")\n", + " trimmed = [t.item() - AUDIO_TOKEN_LO for t in row[:n]]\n", + " code_lists.append(trimmed)\n", + "\n", + "def redistribute_codes(code_list):\n", + " layer_1, layer_2, layer_3 = [], [], []\n", + " for i in range(len(code_list) // 7):\n", + " layer_1.append(code_list[7*i])\n", + " layer_2.append(code_list[7*i + 1] - 4096)\n", + " layer_3.append(code_list[7*i + 2] - 2*4096)\n", + " layer_3.append(code_list[7*i + 3] - 3*4096)\n", + " layer_2.append(code_list[7*i + 4] - 4*4096)\n", + " layer_3.append(code_list[7*i + 5] - 5*4096)\n", + " layer_3.append(code_list[7*i + 6] - 6*4096)\n", + " if not layer_1:\n", + " return torch.zeros(1, 1, 12000) # ~0.5s silence fallback\n", + " def clamp(vals):\n", + " return [max(0, min(4095, v)) for v in vals]\n", + " codes = [torch.tensor(clamp(layer_1)).unsqueeze(0),\n", + " torch.tensor(clamp(layer_2)).unsqueeze(0),\n", + " torch.tensor(clamp(layer_3)).unsqueeze(0)]\n", + " return snac_model.decode(codes)\n", + "\n", + "samples = [redistribute_codes(cl) for cl in code_lists]" + ] + }, + { + "cell_type": "markdown", + "id": "6d73db63", + "metadata": {}, + "source": [ + "### #4.4 Display + save WAVs to `outputs/multilingual/inference_samples/`\n", + "\n", + "Filename includes the speaker_id so multilingual outputs are easy to identify on disk." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0e864a7a", + "metadata": {}, + "outputs": [], + "source": [ + "samples_dir = OUTPUT_DIR / \"inference_samples\"\n", + "samples_dir.mkdir(parents=True, exist_ok=True)\n", + "\n", + "for idx, (p, audio_hat) in enumerate(zip(all_prompts, samples)):\n", + " wav = audio_hat.detach().squeeze().to(\"cpu\").numpy()\n", + " sid = p[\"speaker_id\"]\n", + " out_path = samples_dir / f\"sample_{idx:02d}_{sid}.wav\"\n", + " sf.write(str(out_path), wav, 24000)\n", + " print(f\"[{idx}] [{p['language']}/{sid}] {p['text'][:80]} -> {out_path}\")\n", + " display(AudioDisplay(wav, rate=24000))" + ] + }, + { + "cell_type": "markdown", + "id": "16bc0761", + "metadata": {}, + "source": [ + "## #5 Save & push" + ] + }, + { + "cell_type": "markdown", + "id": "f332b166", + "metadata": {}, + "source": [ + "### #5.1 LoRA adapters (local)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f5eac3bd", + "metadata": {}, + "outputs": [], + "source": [ + "lora_dir = str(OUTPUT_DIR / \"orpheus_lora\")\n", + "model.save_pretrained(lora_dir)\n", + "tokenizer.save_pretrained(lora_dir)\n", + "print(\"LoRA saved to\", lora_dir)" + ] + }, + { + "cell_type": "markdown", + "id": "2c70beb0", + "metadata": {}, + "source": [ + "### #5.2 Merged 16-bit checkpoint (local)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4762c3fa", + "metadata": {}, + "outputs": [], + "source": [ + "merged_dir = str(OUTPUT_DIR / \"orpheus_finetune_16bit\")\n", + "model.save_pretrained_merged(\n", + " merged_dir, tokenizer, save_method=\"merged_16bit\",\n", + ")\n", + "print(\"merged 16-bit saved to\", merged_dir)" + ] + }, + { + "cell_type": "markdown", + "id": "46902cbb", + "metadata": {}, + "source": [ + "### #5.2.1 Patch local `config.json` to match embedding shape\n", + "\n", + "Unsloth's `save_pretrained_merged` runs a pad-token shim that adds `<|finetune_right_pad_id|>` to the tokenizer when `pad_token` is missing or equals `eos_token`. That bumps `len(tokenizer)` by 1 and Unsloth writes `config.vocab_size = len(tokenizer)`, but the merged 16-bit weights are not resized — so `config.vocab_size` ends up off-by-one vs. `embed_tokens.weight.shape[0]`. `transformers` tolerates this, but vLLM asserts equality and refuses to load. We re-derive `vocab_size` from the actual embedding tensor and rewrite `config.json`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "326bb2e0", + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "from pathlib import Path as _Path\n", + "\n", + "_embed_size = model.get_input_embeddings().num_embeddings\n", + "_cfg_path = _Path(merged_dir) / \"config.json\"\n", + "_cfg = json.loads(_cfg_path.read_text())\n", + "if _cfg.get(\"vocab_size\") != _embed_size:\n", + " print(f\"fix config.vocab_size: {_cfg.get('vocab_size')} -> {_embed_size}\")\n", + " _cfg[\"vocab_size\"] = _embed_size\n", + " _cfg_path.write_text(json.dumps(_cfg, indent=2))\n", + "else:\n", + " print(f\"config.vocab_size already matches embedding ({_embed_size})\")" + ] + }, + { + "cell_type": "markdown", + "id": "a6e3cb45", + "metadata": {}, + "source": [ + "### #5.3 Push merged 16-bit to Hugging Face Hub" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5508ea70", + "metadata": {}, + "outputs": [], + "source": [ + "HF_REPO_ID = \"sunbird/orpheus-3b-tts-multilingual\"\n", + "model.push_to_hub_merged(\n", + " HF_REPO_ID, tokenizer,\n", + " save_method=\"merged_16bit\",\n", + " token=os.environ[\"HF_TOKEN\"],\n", + ")\n", + "print(\"pushed to https://huggingface.co/\" + HF_REPO_ID)" + ] + }, + { + "cell_type": "markdown", + "id": "099a32b5", + "metadata": {}, + "source": [ + "### #5.3.1 Patch hub `config.json` to match embedding shape\n", + "\n", + "`push_to_hub_merged` re-runs the same pad-token shim while writing to the hub, so the local fix in #5.2.1 doesn't propagate. Upload a corrected `config.json` directly so vLLM can load this repo without `hf_overrides`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "02366ac9", + "metadata": {}, + "outputs": [], + "source": [ + "from huggingface_hub import HfApi\n", + "\n", + "_local_cfg = _Path(merged_dir) / \"config.json\" # already fixed in #5.2.1\n", + "HfApi().upload_file(\n", + " path_or_fileobj=str(_local_cfg),\n", + " path_in_repo=\"config.json\",\n", + " repo_id=HF_REPO_ID,\n", + " commit_message=f\"fix(config): vocab_size -> {_embed_size} to match embed weights\",\n", + " token=os.environ[\"HF_TOKEN\"],\n", + ")\n", + "print(f\"hub config.json patched: vocab_size -> {_embed_size}\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/modal-deploy/orpheus-3B/api/.env.example b/modal-deploy/orpheus-3B/api/.env.example new file mode 100644 index 00000000..5bc0e374 --- /dev/null +++ b/modal-deploy/orpheus-3B/api/.env.example @@ -0,0 +1,16 @@ +# Required +ORPHEUS_MODAL_URL=https://your-org--orpheus-3b-tts-orpheustts-web.modal.run +GCS_BUCKET_NAME=your-bucket-name +GOOGLE_APPLICATION_CREDENTIALS=/absolute/path/to/service-account.json + +# Optional (defaults shown) +GCS_OBJECT_PREFIX=tts +GCS_SIGNED_URL_EXPIRY_MINUTES=30 +MODAL_REQUEST_TIMEOUT_SECONDS=180 +MODAL_CONNECT_TIMEOUT_SECONDS=10 +MODAL_RETRY_BACKOFF_SECONDS=0.5 +MAX_BATCH_SIZE=16 +SPEAKERS_CACHE_TTL_SECONDS=60 +LOG_LEVEL=INFO +API_HOST=0.0.0.0 +API_PORT=8000 diff --git a/modal-deploy/orpheus-3B/api/.gitignore b/modal-deploy/orpheus-3B/api/.gitignore new file mode 100644 index 00000000..341009fa --- /dev/null +++ b/modal-deploy/orpheus-3B/api/.gitignore @@ -0,0 +1,2 @@ +.venv/ +.pytest_cache/ diff --git a/modal-deploy/orpheus-3B/api/README.md b/modal-deploy/orpheus-3B/api/README.md new file mode 100644 index 00000000..02e2dc5c --- /dev/null +++ b/modal-deploy/orpheus-3B/api/README.md @@ -0,0 +1,227 @@ +# Orpheus-3B TTS FastAPI Service + +A thin HTTP gateway in front of the Modal-deployed Orpheus-3B inference app. + +- Calls Modal `/tts`, `/tts/batch`, `/speakers`, `/health` via async httpx. +- Uploads each generated WAV to Google Cloud Storage. +- Returns a v4 presigned URL (default 30-minute expiry) plus metadata and latency timings. +- Validates `speaker_id` (and optional `language`) against the live catalog up front, so bad inputs fail at the API tier with a 400 instead of consuming GPU time on Modal. + +## Quick start + +```bash +cd orpheus-3B/api +python -m venv .venv && source .venv/bin/activate +pip install -r requirements.txt +cp .env.example .env +# edit .env with your Modal URL, GCS bucket name, and GCP service account JSON path +cd .. # back to orpheus-3B/ +uvicorn api.main:app --host 0.0.0.0 --port 8000 +``` + +OpenAPI: . + +## Environment variables + +See `.env.example`. Required: + +- `ORPHEUS_MODAL_URL` — base URL of the deployed Modal app. +- `GCS_BUCKET_NAME` — existing GCS bucket; the API writes objects, it does **not** create/delete buckets or set lifecycle rules. +- `GOOGLE_APPLICATION_CREDENTIALS` — absolute path to a service account JSON key file. The SA needs `roles/storage.objectCreator` on the bucket (for uploads) and `roles/storage.objectViewer` (or `storage.buckets.get`) for `/healthz`'s readiness probe. + +Optional vars (with defaults) are documented inline in `.env.example`. + +## Endpoints + +### `POST /tts` — single synthesis + +```bash +curl -s -X POST http://localhost:8000/tts \ + -H 'Content-Type: application/json' \ + -d '{"text": "Mwattu, oli otya?", "speaker_id": "salt_lug_0001"}' | jq +``` + +Response: +```json +{ + "audio_url": "https://storage.googleapis.com/...signed...", + "audio_url_expires_at": "2026-05-12T15:30:00Z", + "speaker_id": "salt_lug_0001", + "language": "lug", + "sample_rate": 24000, + "duration_seconds": 2.45, + "audio_size_bytes": 117648, + "gcs_object": "tts/2026-05-12/.wav", + "request_id": "", + "timings_ms": { + "inference_ms": 1820.5, + "upload_ms": 234.1, + "signed_url_ms": 12.0, + "total_ms": 2095.6 + } +} +``` + +### `POST /tts/batch` — multi-item synthesis + +```bash +curl -s -X POST http://localhost:8000/tts/batch \ + -H 'Content-Type: application/json' \ + -d '{ + "items": [ + {"text": "Hi there", "speaker_id": "salt_eng_0001", "language": "eng"}, + {"text": "Mwattu, oli otya?", "speaker_id": "salt_lug_0001", "language": "lug"}, + {"text": "Habari yako?", "speaker_id": "waxal_swa_0006", "language": "swa"} + ] + }' | jq +``` + +Per-item results carry `status: "ok"` or `status: "error"`. The batch returns 200 if ≥1 item succeeded, 502 if all failed. + +### `GET /speakers` and `GET /speakers/{language}` + +```bash +curl -s http://localhost:8000/speakers | jq +curl -s http://localhost:8000/speakers/lug | jq +``` + +### `GET /health` and `/healthz` + +- `/health` — liveness; cheap, no upstream calls. Always 200 when the process is up. +- `/healthz` — readiness; pings Modal `/health` and probes GCS. 200 when both healthy, 503 with per-upstream status otherwise. + +## Selecting a speaker (recommended client flow) + +1. `GET /speakers` — get the full `by_language` map. +2. Show a language picker; once the user picks (say) `lug`, call `GET /speakers/lug`. +3. Show a speaker picker scoped to that language. +4. Submit `POST /tts` with `language` set, for an extra safety check. + +## Python client example (httpx) + +```python +import httpx + +with httpx.Client(base_url="http://localhost:8000", timeout=httpx.Timeout(180.0, connect=10.0)) as c: + r = c.post("/tts", json={ + "text": "Mwattu, oli otya?", + "speaker_id": "salt_lug_0001", + "language": "lug", + }) + r.raise_for_status() + data = r.json() + print("download:", data["audio_url"]) + print(f"duration: {data['duration_seconds']:.2f}s") + print(f"inference: {data['timings_ms']['inference_ms']:.0f} ms") +``` + +## Running tests + +### Unit / integration suite (hermetic) + +```bash +cd orpheus-3B +pytest api/tests/ -v +``` + +73 tests, all hermetic — no real Modal, no real GCS (httpx `MockTransport` + an in-memory fake storage client). This is what runs in CI. + +### Manual scripts against a live deployment + +Three small clients live in `orpheus-3B/api/scripts/`. They all read `API_BASE` from the environment (default `http://localhost:8000`) and exercise the real running API + real Modal + real GCS. Useful for sanity-checking a fresh deploy or new speaker before relying on it. + +Activate the venv first: + +```bash +cd /path/to/Qwen3-TTS +source orpheus-3B/api/.venv/bin/activate +``` + +#### `smoke.py` — minimal end-to-end check + +Hits `/health`, `/healthz`, `/speakers`, then a single `/tts` (Luganda) so you can confirm in under a minute that all four paths work. + +```bash +python orpheus-3B/api/scripts/smoke.py + +# different host +API_BASE=https://my-api.example.com python orpheus-3B/api/scripts/smoke.py +``` + +#### `test_english_tts.py` — single + batch with an English speaker + +Hits `/tts` with one sentence and `/tts/batch` with five sentences using `salt_eng_0002`. Prints `request_id`, durations, audio sizes, presigned URLs, expiry times, and stage-by-stage timings (`inference_ms` / `upload_ms` / `signed_url_ms` / `total_ms`). + +```bash +python orpheus-3B/api/scripts/test_english_tts.py + +# also stream each WAV to /tmp/orpheus-english-tts/ so you can listen to them +DOWNLOAD=1 python orpheus-3B/api/scripts/test_english_tts.py +``` + +Useful env flags: + +| Var | Default | Purpose | +|---|---|---| +| `API_BASE` | `http://localhost:8000` | Different host | +| `DOWNLOAD` | (unset) | Set to `1` to save each WAV locally | +| `OUT_DIR` | `/tmp/orpheus-english-tts` | Where to save WAVs when `DOWNLOAD=1` | + +#### `test_long_text.py` — long-form text via split + batch + stitch + +Recommended pattern for paragraphs / articles / scripts: each `/tts` call is bounded by `max_tokens` (default 1200, hard cap 4096), so a long single input can produce truncated audio. This script: + +1. Splits the input on sentence boundaries (`. ! ?`); overly long sentences are further split on `,` / `;` / spaces so no chunk exceeds 500 chars. +2. Groups sentences into batches of `BATCH_SIZE` (default 8, must stay ≤ server `MAX_BATCH_SIZE`) and sends each batch to `/tts/batch` — vLLM continuous batching means N sentences in one batch is roughly 1.5× the latency of one sentence, not N×. +3. Downloads each per-item WAV (presigned URL), asserts 24 kHz / mono / 16-bit, and concatenates the PCM frames with a configurable silence between sentences (default 250 ms). +4. Writes one stitched WAV using stdlib `wave` — no extra dependencies. + +```bash +# inline text +python orpheus-3B/api/scripts/test_long_text.py \ + --speaker salt_eng_0002 --language eng \ + --text "Good morning everyone. Today we will cover three things. First, last week's progress. Second, the upcoming launch and timeline. Finally, open questions about marketing." + +# from a file +python orpheus-3B/api/scripts/test_long_text.py \ + --speaker salt_eng_0002 --language eng \ + --file my_article.txt \ + --out /tmp/article.wav +``` + +Useful env flags: + +| Var | Default | Purpose | +|---|---|---| +| `API_BASE` | `http://localhost:8000` | Different host | +| `BATCH_SIZE` | `8` | Items per `/tts/batch` call. Keep ≤ server `MAX_BATCH_SIZE`. | +| `SILENCE_MS` | `250` | Silence inserted between sentences. Bump to 400 for slower pacing. | + +One failed sentence is logged and skipped — the rest still produce audio. Exit code is `1` if any sentence failed. + +## Errors + +All error responses share this shape: + +```json +{"error": "", "detail": "", "request_id": ""} +``` + +| Status | Code | When | +|---|---|---| +| 400 | `invalid_speaker` | `speaker_id` not in catalog | +| 400 | `unknown_language` | `language` not in catalog | +| 400 | `invalid_speaker_for_language` | `speaker_id` exists but is from a different language than `language` | +| 422 | `invalid_request` | Pydantic validation (e.g. text too long, batch empty) | +| 502 | `upstream_unavailable` | Modal failed twice | +| 502 | `storage_unavailable` | GCS upload or signing failed | +| 504 | `upstream_timeout` | Modal `read` timeout twice (usually a long cold-start) | +| 503 | (no code; `/healthz` only) | Readiness probe reporting degraded | + +`X-Request-ID` is set on every response. Pass `X-Request-ID: ` to thread your own ID through logs. + +## Out of scope + +- API-level auth (deploy behind a gateway). +- Streaming response (`/tts/stream` is planned separately). +- Cloud deploy (Dockerfile, Cloud Run/GKE manifests) — separate spec. diff --git a/modal-deploy/orpheus-3B/api/__init__.py b/modal-deploy/orpheus-3B/api/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/modal-deploy/orpheus-3B/api/config.py b/modal-deploy/orpheus-3B/api/config.py new file mode 100644 index 00000000..b5208560 --- /dev/null +++ b/modal-deploy/orpheus-3B/api/config.py @@ -0,0 +1,72 @@ +"""Environment-driven settings for the Orpheus TTS FastAPI service. + +Reads required vars from the environment; missing required → startup fails +with a clear message. Optional vars have sensible defaults documented in +.env.example. +""" + +from functools import lru_cache +from pathlib import Path + +from pydantic import Field, field_validator +from pydantic_settings import BaseSettings, SettingsConfigDict + + +class Settings(BaseSettings): + model_config = SettingsConfigDict( + env_file=".env", + env_file_encoding="utf-8", + case_sensitive=False, + extra="ignore", + ) + + # Required + orpheus_modal_url: str = Field(..., description="Base URL of the Modal app") + gcs_bucket_name: str = Field(..., description="Existing GCS bucket name") + google_application_credentials: str = Field( + ..., description="Path to service account JSON key file" + ) + + # GCS + gcs_object_prefix: str = Field("tts") + gcs_signed_url_expiry_minutes: int = Field(30, ge=1, le=7 * 24 * 60) + + # Modal client + modal_request_timeout_seconds: float = Field(180.0, gt=0) + modal_connect_timeout_seconds: float = Field(10.0, gt=0) + modal_retry_backoff_seconds: float = Field(0.5, ge=0) + + # Batching / caching + max_batch_size: int = Field(16, ge=1, le=128) + speakers_cache_ttl_seconds: int = Field(60, ge=1) + + # Logging / serving + log_level: str = Field("INFO") + api_host: str = Field("0.0.0.0") + api_port: int = Field(8000, ge=1, le=65535) + + @field_validator("google_application_credentials") + @classmethod + def _creds_file_exists(cls, v: str) -> str: + p = Path(v) + if not p.is_file(): + raise ValueError( + f"google_application_credentials file not found: {v}" + ) + return v + + @field_validator("orpheus_modal_url") + @classmethod + def _strip_trailing_slash(cls, v: str) -> str: + return v.rstrip("/") + + +@lru_cache(maxsize=1) +def get_settings() -> Settings: + """Cached accessor — Settings is instantiated on first call and reused. + + Tests should instantiate Settings() directly with monkeypatched env vars + rather than going through this function, so the cache doesn't leak state + across tests. + """ + return Settings() # type: ignore[call-arg] diff --git a/modal-deploy/orpheus-3B/api/errors.py b/modal-deploy/orpheus-3B/api/errors.py new file mode 100644 index 00000000..ca2baa4f --- /dev/null +++ b/modal-deploy/orpheus-3B/api/errors.py @@ -0,0 +1,159 @@ +"""Exception hierarchy + FastAPI exception handlers + X-Request-ID middleware.""" + +import contextvars +import logging +import uuid +from typing import Awaitable, Callable + +from fastapi import FastAPI, HTTPException, Request +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse + +logger = logging.getLogger("orpheus_api") + +# Contextvar so log records and downstream code can read the current id. +_request_id: contextvars.ContextVar[str] = contextvars.ContextVar( + "request_id", default="" +) + + +def get_request_id() -> str: + return _request_id.get() or "no-request-id" + + +# ----- Exceptions ----- + +class APIError(Exception): + status_code: int = 500 + error_code: str = "internal_error" + + def __init__(self, detail: str = "") -> None: + super().__init__(detail) + self.detail = detail or self.__class__.__name__ + + +class InvalidSpeakerError(APIError): + status_code = 400 + error_code = "invalid_speaker" + + +class UnknownLanguageError(APIError): + status_code = 400 + error_code = "unknown_language" + + +class InvalidSpeakerForLanguageError(APIError): + status_code = 400 + error_code = "invalid_speaker_for_language" + + +class UpstreamUnavailableError(APIError): + status_code = 502 + error_code = "upstream_unavailable" + + +class UpstreamTimeoutError(APIError): + status_code = 504 + error_code = "upstream_timeout" + + +class StorageUnavailableError(APIError): + status_code = 502 + error_code = "storage_unavailable" + + +# ----- Middleware ----- + +async def request_id_middleware( + request: Request, call_next: Callable[[Request], Awaitable] +): + incoming = request.headers.get("x-request-id") + rid = incoming or uuid.uuid4().hex + request.state.request_id = rid + token = _request_id.set(rid) + try: + response = await call_next(request) + finally: + _request_id.reset(token) + response.headers["X-Request-ID"] = rid + return response + + +# ----- Handlers ----- + +def register_exception_handlers(app: FastAPI) -> None: + @app.exception_handler(HTTPException) + async def _http_exc_handler(request: Request, exc: HTTPException): + rid = getattr(request.state, "request_id", "no-request-id") + if isinstance(exc.detail, dict) and "error" in exc.detail: + return JSONResponse( + status_code=exc.status_code, + content={ + "error": exc.detail.get("error", "http_error"), + "detail": exc.detail.get("detail", ""), + "request_id": rid, + }, + headers={"X-Request-ID": rid}, + ) + return JSONResponse( + status_code=exc.status_code, + content={ + "error": "http_error", + "detail": str(exc.detail), + "request_id": rid, + }, + headers={"X-Request-ID": rid}, + ) + + @app.exception_handler(APIError) + async def _api_error_handler(request: Request, exc: APIError): + rid = getattr(request.state, "request_id", "no-request-id") + logger.warning( + "api_error", + extra={"request_id": rid, "code": exc.error_code, "detail": exc.detail}, + ) + return JSONResponse( + status_code=exc.status_code, + content={"error": exc.error_code, "detail": exc.detail, "request_id": rid}, + headers={"X-Request-ID": rid}, + ) + + @app.exception_handler(RequestValidationError) + async def _validation_handler(request: Request, exc: RequestValidationError): + rid = getattr(request.state, "request_id", "no-request-id") + detail = _format_validation_errors(exc) + logger.warning( + "validation_error", + extra={"request_id": rid, "detail": detail}, + ) + return JSONResponse( + status_code=422, + content={ + "error": "invalid_request", + "detail": detail, + "request_id": rid, + }, + headers={"X-Request-ID": rid}, + ) + + @app.exception_handler(Exception) + async def _unhandled_handler(request: Request, exc: Exception): + rid = getattr(request.state, "request_id", "no-request-id") + logger.exception("unhandled_exception", extra={"request_id": rid}) + return JSONResponse( + status_code=500, + content={ + "error": "internal_error", + "detail": "An unexpected error occurred.", + "request_id": rid, + }, + headers={"X-Request-ID": rid}, + ) + + +def _format_validation_errors(exc: RequestValidationError) -> str: + parts = [] + for err in exc.errors(): + loc = ".".join(str(p) for p in err.get("loc", [])) + parts.append(f"{loc}: {err.get('msg', 'invalid')}") + return "; ".join(parts) or "validation failed" diff --git a/modal-deploy/orpheus-3B/api/logging_setup.py b/modal-deploy/orpheus-3B/api/logging_setup.py new file mode 100644 index 00000000..588d816d --- /dev/null +++ b/modal-deploy/orpheus-3B/api/logging_setup.py @@ -0,0 +1,36 @@ +"""Configure stdlib logging for the API. + +A single-line, structured-ish format with request_id when present. +Avoids pulling in structlog/loguru — stdlib only. +""" + +import logging +import sys + +from api.errors import get_request_id + + +class _RequestIdFilter(logging.Filter): + def filter(self, record: logging.LogRecord) -> bool: + rid = getattr(record, "request_id", None) or get_request_id() + record.request_id = rid + return True + + +def setup_logging(level: str = "INFO") -> None: + root = logging.getLogger() + root.setLevel(level.upper()) + + # Wipe existing handlers if reloaders attached extras. + for h in list(root.handlers): + root.removeHandler(h) + + handler = logging.StreamHandler(stream=sys.stdout) + handler.setLevel(level.upper()) + handler.addFilter(_RequestIdFilter()) + handler.setFormatter( + logging.Formatter( + "%(asctime)s %(levelname)s %(name)s [%(request_id)s] %(message)s" + ) + ) + root.addHandler(handler) diff --git a/modal-deploy/orpheus-3B/api/main.py b/modal-deploy/orpheus-3B/api/main.py new file mode 100644 index 00000000..ac8d12a7 --- /dev/null +++ b/modal-deploy/orpheus-3B/api/main.py @@ -0,0 +1,106 @@ +"""FastAPI app entry point. + +Owns the lifecycle of the httpx AsyncClient, the GCS storage Client, and +the SpeakersCache. Routes pull these from app.state via dependencies. +""" + +import logging +from contextlib import asynccontextmanager + +import httpx +from dotenv import load_dotenv +from fastapi import FastAPI +from google.cloud import storage as gcs_storage + +# Load .env into os.environ at import time so google.auth.default() can see +# GOOGLE_APPLICATION_CREDENTIALS for local SA-key signing. pydantic-settings +# reads .env into the Settings object only — it does NOT export to os.environ, +# so the GCS auth layer (which calls os.getenv directly) would otherwise fall +# through to gcloud user ADC and break signed URLs. On Cloud Run / GKE the SA +# path env var is unset and Workload Identity takes over via the metadata +# server instead. Missing .env is a no-op. +load_dotenv() + +from api.config import get_settings +from api.errors import register_exception_handlers, request_id_middleware +from api.logging_setup import setup_logging +from api.modal_client import ModalClient +from api.routes import router +from api.speakers import SpeakersCache +from api.storage import StorageBackend + +logger = logging.getLogger("orpheus_api") + + +@asynccontextmanager +async def lifespan(app: FastAPI): + settings = get_settings() + setup_logging(settings.log_level) + + timeout = httpx.Timeout( + connect=settings.modal_connect_timeout_seconds, + read=settings.modal_request_timeout_seconds, + write=10.0, + pool=10.0, + ) + transport = httpx.AsyncHTTPTransport(retries=1) + http = httpx.AsyncClient( + base_url=settings.orpheus_modal_url, + timeout=timeout, + transport=transport, + ) + modal = ModalClient(client=http, retry_backoff_seconds=settings.modal_retry_backoff_seconds) + speakers = SpeakersCache(modal=modal, ttl_seconds=settings.speakers_cache_ttl_seconds) + gcs_client = gcs_storage.Client() + storage = StorageBackend( + client=gcs_client, + bucket_name=settings.gcs_bucket_name, + object_prefix=settings.gcs_object_prefix, + signed_url_expiry_minutes=settings.gcs_signed_url_expiry_minutes, + ) + + app.state.settings = settings + app.state.http = http + app.state.modal = modal + app.state.speakers = speakers + app.state.storage = storage + + await speakers.try_warm() + logger.info( + "startup_complete", + extra={ + "modal_url": settings.orpheus_modal_url, + "bucket": settings.gcs_bucket_name, + "speakers_warm": speakers.is_warm, + }, + ) + try: + yield + finally: + await http.aclose() + # google-cloud-storage holds a requests.Session + connection pool. + # close() is idempotent and safe even if it was never opened. + try: + gcs_client.close() + except Exception as exc: # noqa: BLE001 + logger.warning("gcs_client_close_failed: %s", exc) + + +def create_app() -> FastAPI: + app = FastAPI( + title="Orpheus-3B TTS API", + version="0.1.0", + description=( + "Multilingual, multi-speaker TTS gateway. Calls a Modal-deployed " + "vLLM inference server, uploads audio to GCS, and returns a " + "presigned URL that expires in 30 minutes." + ), + lifespan=lifespan, + ) + app.middleware("http")(request_id_middleware) + register_exception_handlers(app) + app.include_router(router) + return app + + +app = create_app() diff --git a/modal-deploy/orpheus-3B/api/modal_client.py b/modal-deploy/orpheus-3B/api/modal_client.py new file mode 100644 index 00000000..acdd5cb0 --- /dev/null +++ b/modal-deploy/orpheus-3B/api/modal_client.py @@ -0,0 +1,172 @@ +"""Async httpx wrapper for the Modal-deployed Orpheus inference app. + +Owns timeout & retry policy. Exposes: +- health() +- speakers() +- tts(...) -> TTSAudio +- tts_batch([...]) -> list[TTSAudio] + +Retry policy: one app-level retry on (httpx.ReadTimeout, 5xx) with a +backoff of `retry_backoff_seconds + jitter`. After exhaustion, the right +APIError subclass is raised so the route layer doesn't have to know the +upstream details. +""" + +import asyncio +import base64 +import binascii +import logging +import random +from dataclasses import dataclass + +import httpx + +from api.errors import UpstreamTimeoutError, UpstreamUnavailableError + +logger = logging.getLogger("orpheus_api.modal") + +_RETRY_STATUSES = {502, 503, 504} + + +@dataclass +class TTSAudio: + audio_bytes: bytes + sample_rate: int + duration_seconds: float + speaker_id: str + chunks: int | None = None + + +class ModalClient: + def __init__( + self, + client: httpx.AsyncClient, + retry_backoff_seconds: float = 0.5, + ) -> None: + self.client = client + self.backoff = retry_backoff_seconds + + async def health(self) -> bool: + try: + r = await self.client.get("/health", timeout=10.0) + except httpx.HTTPError as exc: + logger.warning("modal_health_unreachable: %s", exc) + return False + return r.is_success + + async def speakers(self) -> dict: + return await self._json_get("/speakers") + + async def tts( + self, + *, + text: str, + speaker_id: str, + seed: int | None, + temperature: float, + top_p: float, + repetition_penalty: float, + max_tokens: int, + ) -> TTSAudio: + body = { + "text": text, + "speaker_id": speaker_id, + "seed": seed, + "temperature": temperature, + "top_p": top_p, + "repetition_penalty": repetition_penalty, + "max_tokens": max_tokens, + } + resp = await self._post_with_retry("/tts", json=body) + sr = int(resp.headers.get("X-Sample-Rate", "24000")) + dur = float(resp.headers.get("X-Duration-Seconds", "0")) + chunks_hdr = resp.headers.get("X-Chunks") + try: + chunks = int(chunks_hdr) if chunks_hdr is not None else None + except ValueError: + chunks = None + return TTSAudio( + audio_bytes=resp.content, + sample_rate=sr, + duration_seconds=dur, + speaker_id=resp.headers.get("X-Speaker-Id", speaker_id), + chunks=chunks, + ) + + async def tts_batch(self, items: list[dict]) -> list[TTSAudio]: + body = {"items": items} + resp = await self._post_with_retry("/tts/batch", json=body) + try: + data = resp.json() + except ValueError as exc: + raise UpstreamUnavailableError( + f"non-JSON response from /tts/batch: {exc}" + ) from exc + results = [] + for idx, r in enumerate(data.get("results", [])): + try: + wav = base64.b64decode(r["audio_wav_b64"], validate=True) + except (KeyError, ValueError, binascii.Error) as exc: + raise UpstreamUnavailableError( + f"malformed result at index {idx}: {exc}" + ) from exc + results.append( + TTSAudio( + audio_bytes=wav, + sample_rate=int(r.get("sample_rate", 24000)), + duration_seconds=float(r.get("duration_sec", 0.0)), + speaker_id=r.get("speaker_id", ""), + ) + ) + return results + + # ----- internals ----- + + async def _json_get(self, path: str) -> dict: + resp = await self._with_retry(lambda: self.client.get(path)) + return resp.json() + + async def _post_with_retry(self, path: str, *, json: dict) -> httpx.Response: + return await self._with_retry(lambda: self.client.post(path, json=json)) + + async def _with_retry(self, op): + for attempt in (1, 2): + try: + resp = await op() + except httpx.ReadTimeout as exc: + if attempt == 1: + await self._sleep_backoff() + continue + raise UpstreamTimeoutError( + f"Modal request timed out after {attempt} attempt(s)" + ) from exc + except httpx.HTTPError as exc: + if attempt == 1: + await self._sleep_backoff() + continue + raise UpstreamUnavailableError( + f"Modal request failed: {exc}" + ) from exc + + if resp.status_code in _RETRY_STATUSES and attempt == 1: + await self._sleep_backoff() + continue + if resp.is_success: + return resp + # 4xx (after our pre-flight checks) → bug; surface as 502. + # Includes Modal's 422 text_too_long from the chunker cap — + # currently unreachable because the gateway's TTSRequest enforces + # max_length=2000 on `text` (≈10 chunks at the 220-char target, + # well under MAX_CHUNKS_PER_REQUEST=48). Decouple cautiously. + raise UpstreamUnavailableError( + f"Modal returned {resp.status_code}: {resp.text[:200]}" + ) + + # Unreachable: the loop above always returns on success or raises on + # the second failed attempt. + raise AssertionError("unreachable") + + async def _sleep_backoff(self) -> None: + if self.backoff <= 0: + return + await asyncio.sleep(self.backoff + random.uniform(0, self.backoff / 2)) diff --git a/modal-deploy/orpheus-3B/api/models.py b/modal-deploy/orpheus-3B/api/models.py new file mode 100644 index 00000000..51aa5378 --- /dev/null +++ b/modal-deploy/orpheus-3B/api/models.py @@ -0,0 +1,139 @@ +"""Pydantic v2 request/response models for the Orpheus TTS API.""" + +from datetime import datetime +from typing import Literal, Optional + +from pydantic import BaseModel, Field, HttpUrl, computed_field + +# ----- TTS request ----- + +class TTSRequest(BaseModel): + text: str = Field(..., min_length=1, max_length=2000, description="Text to synthesize.") + speaker_id: str = Field( + "salt_lug_0001", + description="Speaker tag from the finetune set (see GET /speakers).", + ) + language: Optional[str] = Field( + None, + description=( + "Optional ISO 639-3 language code (e.g. 'lug', 'eng'). " + "If set, speaker_id must belong to it." + ), + ) + seed: Optional[int] = Field(None, description="RNG seed for reproducibility.") + temperature: float = Field(0.6, ge=0.0, le=2.0) + top_p: float = Field(0.95, gt=0.0, le=1.0) + repetition_penalty: float = Field(1.1, ge=1.0, le=2.0) + max_tokens: int = Field(1200, ge=64, le=4096) + + +# ----- Batch (size is also enforced at the route layer against config.max_batch_size) ----- + +class TTSBatchRequest(BaseModel): + items: list[TTSRequest] = Field(..., min_length=1, max_length=128) + + +# ----- Timings ----- + +class Timings(BaseModel): + inference_ms: float + upload_ms: float + signed_url_ms: float + total_ms: float + + +class BatchTimings(BaseModel): + inference_ms: float + upload_ms: float + total_ms: float + + +# ----- TTS response (single) ----- + +class TTSResponse(BaseModel): + audio_url: HttpUrl + audio_url_expires_at: datetime + speaker_id: str + language: Optional[str] = None + sample_rate: int = 24000 + duration_seconds: float + chunks: Optional[int] = None + audio_size_bytes: int + gcs_object: str + request_id: str + timings_ms: Timings + + +# ----- Batch results ----- + +class TTSBatchItemResult(BaseModel): + index: int + status: Literal["ok", "error"] + speaker_id: str + + # success-only + audio_url: Optional[HttpUrl] = None + audio_url_expires_at: Optional[datetime] = None + language: Optional[str] = None + sample_rate: int = 24000 + duration_seconds: Optional[float] = None + audio_size_bytes: Optional[int] = None + gcs_object: Optional[str] = None + request_id: Optional[str] = None + + # error-only + error_code: Optional[str] = None + error_detail: Optional[str] = None + + +class TTSBatchResponse(BaseModel): + results: list[TTSBatchItemResult] + timings_ms: BatchTimings + request_id: str + + +# ----- Catalog ----- + +class SpeakersResponse(BaseModel): + default: str + by_language: dict[str, list[str]] + + @computed_field + @property + def total(self) -> int: + return sum(len(v) for v in self.by_language.values()) + + @computed_field + @property + def languages(self) -> list[str]: + return sorted(self.by_language.keys()) + + +class LanguageSpeakersResponse(BaseModel): + language: str + speakers: list[str] + + @computed_field + @property + def count(self) -> int: + return len(self.speakers) + + +# ----- Health ----- + +class HealthResponse(BaseModel): + status: Literal["ok"] = "ok" + service: Literal["orpheus-tts-api"] = "orpheus-tts-api" + + +class ReadinessResponse(BaseModel): + status: Literal["ok", "degraded"] + upstreams: dict[str, str] # {"modal": "ok"|"", "gcs": "ok"|""} + + +# ----- Errors ----- + +class ErrorResponse(BaseModel): + error: str + detail: str + request_id: str diff --git a/modal-deploy/orpheus-3B/api/requirements.txt b/modal-deploy/orpheus-3B/api/requirements.txt new file mode 100644 index 00000000..81d0bcaa --- /dev/null +++ b/modal-deploy/orpheus-3B/api/requirements.txt @@ -0,0 +1,11 @@ +fastapi==0.136.* +uvicorn[standard]==0.46.* +httpx==0.28.* +google-cloud-storage>=2.18,<4.0 +pydantic>=2.5,<3.0 +pydantic-settings>=2.5,<3.0 +python-dotenv>=1.2.2 + +# dev/test +pytest>=9.0.3 +pytest-asyncio>=1.3.0 diff --git a/modal-deploy/orpheus-3B/api/routes.py b/modal-deploy/orpheus-3B/api/routes.py new file mode 100644 index 00000000..b1733649 --- /dev/null +++ b/modal-deploy/orpheus-3B/api/routes.py @@ -0,0 +1,302 @@ +"""HTTP route handlers.""" + +import asyncio +import time +from typing import Annotated + +from fastapi import APIRouter, Body, HTTPException, Request +from fastapi.responses import JSONResponse + +from api.errors import ( + InvalidSpeakerError, + InvalidSpeakerForLanguageError, + StorageUnavailableError, + UnknownLanguageError, + get_request_id, +) +from api.models import ( + BatchTimings, + HealthResponse, + LanguageSpeakersResponse, + ReadinessResponse, + SpeakersResponse, + Timings, + TTSBatchItemResult, + TTSBatchRequest, + TTSBatchResponse, + TTSRequest, + TTSResponse, +) + +router = APIRouter() + + +# ----- /speakers ----- + +@router.get( + "/speakers", + response_model=SpeakersResponse, + summary="List all speakers grouped by language.", + description=( + "Returns the full speaker catalog. Use this to populate a " + "client-side picker. `total` and `languages` are derived " + "convenience fields." + ), +) +async def get_speakers(request: Request) -> SpeakersResponse: + cat = await request.app.state.speakers.get() + return SpeakersResponse(default=cat.default, by_language=cat.by_language) + + +@router.get( + "/speakers/{language}", + response_model=LanguageSpeakersResponse, + summary="List speakers for one language.", + description=( + "Convenience endpoint for two-step pickers: pick a language, then " + "list its speakers. Returns 404 unknown_language if the code is " + "not in the catalog." + ), + responses={ + 404: {"description": "language code not found in catalog"}, + }, +) +async def get_speakers_for_language( + language: str, request: Request +) -> LanguageSpeakersResponse: + cat = await request.app.state.speakers.get() + if language not in cat.by_language: + raise HTTPException( + status_code=404, + detail={ + "error": "unknown_language", + "detail": ( + f"language '{language}' not supported; " + f"supported: {sorted(cat.by_language)}" + ), + }, + ) + return LanguageSpeakersResponse( + language=language, speakers=cat.by_language[language] + ) + + +# ----- /tts ----- + +@router.post( + "/tts", + response_model=TTSResponse, + summary="Synthesize speech for one input.", + description=( + "Calls the Modal vLLM inference app, uploads the generated WAV to " + "Google Cloud Storage, and returns a v4 presigned download URL " + "valid for the configured expiry window (default 30 minutes), " + "together with metadata and stage-by-stage latency timings." + ), + responses={ + 400: {"description": "invalid_speaker | unknown_language | invalid_speaker_for_language"}, + 422: {"description": "request validation error"}, + 502: {"description": "upstream_unavailable | storage_unavailable"}, + 504: {"description": "upstream_timeout"}, + }, +) +async def tts(req: Annotated[TTSRequest, Body()], request: Request) -> TTSResponse: + speakers = request.app.state.speakers + modal = request.app.state.modal + storage = request.app.state.storage + + await speakers.validate_speaker(req.speaker_id, language=req.language) + + t_total = time.monotonic() + t_inf = time.monotonic() + audio = await modal.tts( + text=req.text, + speaker_id=req.speaker_id, + seed=req.seed, + temperature=req.temperature, + top_p=req.top_p, + repetition_penalty=req.repetition_penalty, + max_tokens=req.max_tokens, + ) + inference_ms = (time.monotonic() - t_inf) * 1000.0 + + upload = await storage.upload_wav( + audio_bytes=audio.audio_bytes, content_type="audio/wav" + ) + + total_ms = (time.monotonic() - t_total) * 1000.0 + resolved_language = await speakers.language_for(req.speaker_id) + + return TTSResponse( + audio_url=upload.audio_url, + audio_url_expires_at=upload.audio_url_expires_at, + speaker_id=req.speaker_id, + language=resolved_language, + sample_rate=audio.sample_rate, + duration_seconds=audio.duration_seconds, + chunks=audio.chunks, + audio_size_bytes=upload.audio_size_bytes, + gcs_object=upload.gcs_object, + request_id=get_request_id(), + timings_ms=Timings( + inference_ms=inference_ms, + upload_ms=upload.upload_ms, + signed_url_ms=upload.signed_url_ms, + total_ms=total_ms, + ), + ) + + +@router.post( + "/tts/batch", + response_model=TTSBatchResponse, + summary="Synthesize speech for a batch of inputs.", + description=( + "Calls Modal's batched inference endpoint (a single vLLM " + "continuous-batched pass) and uploads each generated WAV to GCS " + "in parallel. Per-item failures are reported in the response with " + "`status: \"error\"`; the request as a whole returns 200 if at " + "least one item succeeds, 502 if every item failed." + ), + responses={ + 400: {"description": "invalid_speaker | unknown_language | invalid_speaker_for_language (any item)"}, + 422: {"description": "request validation error"}, + 502: {"description": "all items failed upstream or storage"}, + 504: {"description": "upstream_timeout"}, + }, +) +async def tts_batch( + req: Annotated[TTSBatchRequest, Body()], request: Request +) -> TTSBatchResponse: + settings = request.app.state.settings + speakers = request.app.state.speakers + modal = request.app.state.modal + storage = request.app.state.storage + + if len(req.items) > settings.max_batch_size: + raise HTTPException( + status_code=422, + detail={ + "error": "invalid_request", + "detail": ( + f"batch size {len(req.items)} exceeds MAX_BATCH_SIZE " + f"{settings.max_batch_size}" + ), + }, + ) + + # Pre-flight validation — fail fast for the entire batch on first bad item. + for idx, item in enumerate(req.items): + try: + await speakers.validate_speaker(item.speaker_id, language=item.language) + except (InvalidSpeakerError, UnknownLanguageError, InvalidSpeakerForLanguageError) as exc: + exc.detail = f"item index {idx}: {exc.detail}" + raise + + t_total = time.monotonic() + t_inf = time.monotonic() + audios = await modal.tts_batch( + [ + { + "text": it.text, + "speaker_id": it.speaker_id, + "seed": it.seed, + "temperature": it.temperature, + "top_p": it.top_p, + "repetition_penalty": it.repetition_penalty, + "max_tokens": it.max_tokens, + } + for it in req.items + ] + ) + inference_ms = (time.monotonic() - t_inf) * 1000.0 + + t_up = time.monotonic() + uploads = await asyncio.gather( + *[ + storage.upload_wav(audio_bytes=a.audio_bytes, content_type="audio/wav") + for a in audios + ], + return_exceptions=True, + ) + upload_ms = (time.monotonic() - t_up) * 1000.0 + + results: list[TTSBatchItemResult] = [] + ok_count = 0 + for i, (item, audio, up) in enumerate(zip(req.items, audios, uploads)): + if isinstance(up, Exception): + code = getattr(up, "error_code", "storage_unavailable") + results.append( + TTSBatchItemResult( + index=i, + status="error", + speaker_id=item.speaker_id, + error_code=code, + error_detail=str(up), + ) + ) + continue + ok_count += 1 + results.append( + TTSBatchItemResult( + index=i, + status="ok", + speaker_id=item.speaker_id, + language=await speakers.language_for(item.speaker_id), + audio_url=up.audio_url, + audio_url_expires_at=up.audio_url_expires_at, + sample_rate=audio.sample_rate, + duration_seconds=audio.duration_seconds, + audio_size_bytes=up.audio_size_bytes, + gcs_object=up.gcs_object, + request_id=get_request_id(), + ) + ) + + total_ms = (time.monotonic() - t_total) * 1000.0 + if ok_count == 0: + raise StorageUnavailableError( + f"all {len(req.items)} batch items failed during upload" + ) + + return TTSBatchResponse( + results=results, + timings_ms=BatchTimings( + inference_ms=inference_ms, upload_ms=upload_ms, total_ms=total_ms + ), + request_id=get_request_id(), + ) + + +# ----- /health / /healthz ----- + +@router.get( + "/health", + response_model=HealthResponse, + summary="Liveness probe (no upstream calls).", +) +async def health() -> HealthResponse: + return HealthResponse() + + +@router.get( + "/healthz", + response_model=ReadinessResponse, + summary="Readiness probe — checks Modal and GCS.", + responses={503: {"description": "one or more upstreams unreachable"}}, +) +async def healthz(request: Request): + modal_ok = await request.app.state.modal.health() + gcs_ok = await request.app.state.storage.check_bucket() + upstreams = { + "modal": "ok" if modal_ok else "unreachable", + "gcs": "ok" if gcs_ok else "unreachable", + } + if modal_ok and gcs_ok: + return ReadinessResponse(status="ok", upstreams=upstreams) + return JSONResponse( + status_code=503, + content=ReadinessResponse( + status="degraded", upstreams=upstreams + ).model_dump(), + ) diff --git a/modal-deploy/orpheus-3B/api/scripts/__init__.py b/modal-deploy/orpheus-3B/api/scripts/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/modal-deploy/orpheus-3B/api/scripts/smoke.py b/modal-deploy/orpheus-3B/api/scripts/smoke.py new file mode 100644 index 00000000..cd344fdd --- /dev/null +++ b/modal-deploy/orpheus-3B/api/scripts/smoke.py @@ -0,0 +1,43 @@ +"""Manual smoke test — hits a live API at $API_BASE (default http://localhost:8000). + +Usage: + API_BASE=http://localhost:8000 python orpheus-3B/api/scripts/smoke.py +""" + +import os +import sys + +import httpx + + +BASE = os.environ.get("API_BASE", "http://localhost:8000").rstrip("/") + + +def main() -> int: + with httpx.Client(base_url=BASE, timeout=httpx.Timeout(200.0, connect=10.0)) as c: + print(f"--> GET /health") + print(c.get("/health").json()) + + print(f"--> GET /healthz") + print(c.get("/healthz").json()) + + print(f"--> GET /speakers") + speakers = c.get("/speakers").json() + print(f" total: {speakers['total']}, languages: {speakers['languages']}") + + print(f"--> POST /tts") + r = c.post( + "/tts", + json={"text": "Mwattu, oli otya?", "speaker_id": "salt_lug_0001", "language": "lug"}, + ) + r.raise_for_status() + body = r.json() + print(f" audio_url: {body['audio_url'][:80]}...") + print(f" duration: {body['duration_seconds']:.2f}s") + print(f" timings: {body['timings_ms']}") + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/modal-deploy/orpheus-3B/api/scripts/test_english_tts.py b/modal-deploy/orpheus-3B/api/scripts/test_english_tts.py new file mode 100644 index 00000000..98c4f89f --- /dev/null +++ b/modal-deploy/orpheus-3B/api/scripts/test_english_tts.py @@ -0,0 +1,151 @@ +"""Hit the running FastAPI service against an English speaker. + +Tests: +- POST /tts with one sentence +- POST /tts/batch with five sentences + +Speaker: salt_eng_0002 (language: eng). + +Usage: + # API running locally: + python orpheus-3B/api/scripts/test_english_tts.py + + # Different host: + API_BASE=http://host:8000 python orpheus-3B/api/scripts/test_english_tts.py + + # Also download the WAVs to /tmp: + DOWNLOAD=1 python orpheus-3B/api/scripts/test_english_tts.py +""" + +import os +import sys +from pathlib import Path + +import httpx + +BASE = os.environ.get("API_BASE", "http://localhost:8000").rstrip("/") +DOWNLOAD = os.environ.get("DOWNLOAD") in ("1", "true", "yes") +OUT_DIR = Path(os.environ.get("OUT_DIR", "/tmp/orpheus-english-tts")) + +SPEAKER_ID = "salt_eng_0002" +LANGUAGE = "eng" + +SINGLE_TEXT = "Good morning, my name is Patrick and I work on text to speech systems." + +BATCH_TEXTS = [ + "The sun rose over the misty hills as the village began to wake.", + "Please bring me a glass of water and the morning newspaper.", + "Artificial intelligence has changed how we build software products.", + "She laughed at the joke and then quickly returned to her work.", + "Travel safely and remember to call us when you arrive home.", +] + + +def _short(url: str, n: int = 90) -> str: + return url if len(url) <= n else url[: n - 3] + "..." + + +def _download(client: httpx.Client, url: str, dest: Path) -> None: + dest.parent.mkdir(parents=True, exist_ok=True) + with client.stream("GET", url) as r: + r.raise_for_status() + with dest.open("wb") as f: + for chunk in r.iter_bytes(): + f.write(chunk) + print(f" saved → {dest} ({dest.stat().st_size:,} bytes)") + + +def test_single(client: httpx.Client) -> dict: + print(f"\n--> POST /tts (speaker={SPEAKER_ID}, language={LANGUAGE})") + print(f" text: {SINGLE_TEXT!r}") + r = client.post( + "/tts", + json={ + "text": SINGLE_TEXT, + "speaker_id": SPEAKER_ID, + "language": LANGUAGE, + }, + ) + r.raise_for_status() + body = r.json() + t = body["timings_ms"] + print(f" request_id: {body['request_id']}") + print(f" duration_sec: {body['duration_seconds']:.2f}") + print(f" audio_size: {body['audio_size_bytes']:,} bytes") + print(f" gcs_object: {body['gcs_object']}") + print(f" audio_url: {_short(body['audio_url'])}") + print(f" expires_at: {body['audio_url_expires_at']}") + print( + f" timings_ms: inference={t['inference_ms']:.0f} " + f"upload={t['upload_ms']:.0f} sign={t['signed_url_ms']:.0f} " + f"total={t['total_ms']:.0f}" + ) + if DOWNLOAD: + _download(client, body["audio_url"], OUT_DIR / "single.wav") + return body + + +def test_batch(client: httpx.Client) -> dict: + print(f"\n--> POST /tts/batch ({len(BATCH_TEXTS)} items, speaker={SPEAKER_ID})") + for i, text in enumerate(BATCH_TEXTS): + print(f" [{i}] {text!r}") + r = client.post( + "/tts/batch", + json={ + "items": [ + {"text": text, "speaker_id": SPEAKER_ID, "language": LANGUAGE} + for text in BATCH_TEXTS + ] + }, + ) + r.raise_for_status() + body = r.json() + t = body["timings_ms"] + print(f" request_id: {body['request_id']}") + print( + f" timings_ms: inference={t['inference_ms']:.0f} " + f"upload={t['upload_ms']:.0f} total={t['total_ms']:.0f}" + ) + for item in body["results"]: + idx = item["index"] + if item["status"] != "ok": + print( + f" [{idx}] ERROR code={item.get('error_code')} " + f"detail={item.get('error_detail')}" + ) + continue + print( + f" [{idx}] ok dur={item['duration_seconds']:.2f}s " + f"size={item['audio_size_bytes']:,} " + f"obj={item['gcs_object']}" + ) + print(f" url: {_short(item['audio_url'])}") + if DOWNLOAD: + _download(client, item["audio_url"], OUT_DIR / f"batch_{idx}.wav") + return body + + +def main() -> int: + print(f"API base: {BASE}") + timeout = httpx.Timeout(300.0, connect=10.0) + with httpx.Client(base_url=BASE, timeout=timeout) as client: + # Quick liveness check up front so we fail fast on a misconfigured server. + try: + health = client.get("/health") + health.raise_for_status() + print(f"health: {health.json()}") + except httpx.HTTPError as exc: + print(f"server not reachable at {BASE}: {exc}") + return 1 + + test_single(client) + test_batch(client) + + if DOWNLOAD: + print(f"\nWAVs saved under {OUT_DIR}") + print("\nOK") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/modal-deploy/orpheus-3B/api/scripts/test_long_text.py b/modal-deploy/orpheus-3B/api/scripts/test_long_text.py new file mode 100644 index 00000000..91de2d70 --- /dev/null +++ b/modal-deploy/orpheus-3B/api/scripts/test_long_text.py @@ -0,0 +1,244 @@ +"""Synthesize a large chunk of text by splitting into sentences, +batching the calls, and stitching the WAVs together locally. + +Why this approach: +- Each /tts call has a max_tokens cap; very long inputs risk truncated audio. +- vLLM does continuous batching on the server — N sentences in one + /tts/batch call is much faster than N sequential /tts calls. +- Per-sentence chunks keep each generation safely within token bounds + and give you natural points to insert breath silence. + +Output: a single concatenated 24 kHz mono 16-bit PCM WAV. + +Usage: + # Inline text: + python orpheus-3B/api/scripts/test_long_text.py \\ + --speaker salt_eng_0002 --language eng \\ + --text "Your long paragraph here. It can span many sentences." + + # From a file: + python orpheus-3B/api/scripts/test_long_text.py \\ + --speaker salt_eng_0002 --language eng \\ + --file path/to/article.txt \\ + --out /tmp/long.wav + +Env vars: + API_BASE (default http://localhost:8000) + BATCH_SIZE (default 8, must be <= server MAX_BATCH_SIZE) + SILENCE_MS (default 250) — silence inserted between sentences +""" + +import argparse +import io +import os +import re +import sys +import time +import wave +from pathlib import Path + +import httpx + +BASE = os.environ.get("API_BASE", "http://localhost:8000").rstrip("/") +BATCH_SIZE = int(os.environ.get("BATCH_SIZE", "8")) +SILENCE_MS = int(os.environ.get("SILENCE_MS", "250")) + +# Hard guard: the model itself caps text at 2000 chars per item. +# We further enforce a softer per-sentence limit so generation never +# pushes against max_tokens on a single chunk. +PER_ITEM_CHAR_LIMIT = 500 + + +# ----- text chunking ----- + +_SENTENCE_BOUNDARY = re.compile(r"(?<=[.!?])\s+(?=[A-Z\"'(])") + + +def split_sentences(text: str) -> list[str]: + """Naive sentence splitter — good enough for prose, no NLTK dependency. + + Splits on `.`, `!`, `?` followed by whitespace and a capital letter. + Long sentences are further split on commas/semicolons if they exceed + PER_ITEM_CHAR_LIMIT, so no single chunk is too big for the model. + """ + text = re.sub(r"\s+", " ", text).strip() + if not text: + return [] + raw = _SENTENCE_BOUNDARY.split(text) + out: list[str] = [] + for s in raw: + s = s.strip() + if not s: + continue + while len(s) > PER_ITEM_CHAR_LIMIT: + # Try a clause boundary first. + split_idx = s.rfind(", ", 0, PER_ITEM_CHAR_LIMIT) + if split_idx < PER_ITEM_CHAR_LIMIT // 2: + split_idx = s.rfind("; ", 0, PER_ITEM_CHAR_LIMIT) + if split_idx < PER_ITEM_CHAR_LIMIT // 2: + split_idx = s.rfind(" ", 0, PER_ITEM_CHAR_LIMIT) + if split_idx <= 0: + split_idx = PER_ITEM_CHAR_LIMIT + out.append(s[: split_idx + 1].strip()) + s = s[split_idx + 1 :].strip() + if s: + out.append(s) + return out + + +def chunked(items: list, n: int) -> list[list]: + return [items[i : i + n] for i in range(0, len(items), n)] + + +# ----- WAV helpers (24 kHz mono 16-bit PCM, matches Orpheus output) ----- + +SAMPLE_RATE = 24000 +CHANNELS = 1 +SAMPLE_WIDTH = 2 # bytes + + +def read_wav_pcm(data: bytes) -> bytes: + """Read a WAV and return the raw PCM frames. Assert the expected format.""" + with wave.open(io.BytesIO(data), "rb") as w: + assert w.getnchannels() == CHANNELS, f"channels={w.getnchannels()}" + assert w.getsampwidth() == SAMPLE_WIDTH, f"sampwidth={w.getsampwidth()}" + assert w.getframerate() == SAMPLE_RATE, f"sr={w.getframerate()}" + return w.readframes(w.getnframes()) + + +def silence_pcm(ms: int) -> bytes: + """Return `ms` milliseconds of silence as 16-bit PCM frames.""" + n_frames = SAMPLE_RATE * ms // 1000 + return b"\x00\x00" * n_frames + + +def write_wav(path: Path, pcm: bytes) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with wave.open(str(path), "wb") as w: + w.setnchannels(CHANNELS) + w.setsampwidth(SAMPLE_WIDTH) + w.setframerate(SAMPLE_RATE) + w.writeframes(pcm) + + +# ----- API calls ----- + +def call_batch( + client: httpx.Client, items: list[dict], n_done: int, n_total: int +) -> list[dict]: + print(f" [{n_done + 1}-{n_done + len(items)} / {n_total}] POST /tts/batch...") + t0 = time.monotonic() + r = client.post("/tts/batch", json={"items": items}) + r.raise_for_status() + body = r.json() + elapsed = time.monotonic() - t0 + t = body["timings_ms"] + print( + f" wall {elapsed:.1f}s | server inference={t['inference_ms']:.0f}ms " + f"upload={t['upload_ms']:.0f}ms total={t['total_ms']:.0f}ms" + ) + return body["results"] + + +def download_audio(client: httpx.Client, url: str) -> bytes: + r = client.get(url) + r.raise_for_status() + return r.content + + +# ----- main ----- + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + src = ap.add_mutually_exclusive_group(required=True) + src.add_argument("--text", help="Inline text to synthesize.") + src.add_argument("--file", type=Path, help="Path to a text file.") + ap.add_argument("--speaker", default="salt_eng_0002") + ap.add_argument("--language", default="eng") + ap.add_argument( + "--out", + type=Path, + default=Path("/tmp/orpheus-long.wav"), + help="Output WAV path (default %(default)s).", + ) + args = ap.parse_args() + + text = args.text if args.text else args.file.read_text(encoding="utf-8") + sentences = split_sentences(text) + if not sentences: + print("no sentences found", file=sys.stderr) + return 2 + + print(f"API base: {BASE}") + print(f"Speaker / lang: {args.speaker} / {args.language}") + print(f"Total chars: {len(text):,}") + print(f"Sentences: {len(sentences)}") + print(f"Batches: {-(-len(sentences) // BATCH_SIZE)} of up to {BATCH_SIZE}") + print(f"Silence between: {SILENCE_MS} ms") + print(f"Output WAV: {args.out}") + print() + + base_item = { + "speaker_id": args.speaker, + "language": args.language, + } + items_all = [{**base_item, "text": s} for s in sentences] + + # 300s budget covers Modal cold start. Subsequent batches are fast. + timeout = httpx.Timeout(300.0, connect=10.0) + pcm_parts: list[bytes] = [] + failures = 0 + + with httpx.Client(base_url=BASE, timeout=timeout) as client: + # Liveness check up front. + try: + client.get("/health").raise_for_status() + except httpx.HTTPError as exc: + print(f"server not reachable at {BASE}: {exc}", file=sys.stderr) + return 1 + + t_wall = time.monotonic() + done = 0 + for batch in chunked(items_all, BATCH_SIZE): + results = call_batch(client, batch, done, len(items_all)) + # results come back in original order; rely on that to stitch. + for res in results: + idx = done + res["index"] + if res["status"] != "ok": + failures += 1 + print( + f" [{idx}] FAIL code={res.get('error_code')} " + f"detail={res.get('error_detail')}", + file=sys.stderr, + ) + continue + wav_bytes = download_audio(client, res["audio_url"]) + pcm_parts.append(read_wav_pcm(wav_bytes)) + if SILENCE_MS > 0: + pcm_parts.append(silence_pcm(SILENCE_MS)) + done += len(batch) + + wall = time.monotonic() - t_wall + + # Drop the trailing silence so the file doesn't end on a gap. + if pcm_parts and SILENCE_MS > 0: + pcm_parts.pop() + + if not pcm_parts: + print("no audio produced", file=sys.stderr) + return 1 + + full_pcm = b"".join(pcm_parts) + write_wav(args.out, full_pcm) + duration_sec = len(full_pcm) / (SAMPLE_RATE * SAMPLE_WIDTH) + + print() + print(f"OK: {len(sentences) - failures}/{len(sentences)} sentences synthesized") + print(f" wall time: {wall:.1f}s") + print(f" audio length: {duration_sec:.2f}s") + print(f" output: {args.out} ({args.out.stat().st_size:,} bytes)") + return 0 if failures == 0 else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/modal-deploy/orpheus-3B/api/speakers.py b/modal-deploy/orpheus-3B/api/speakers.py new file mode 100644 index 00000000..522bf380 --- /dev/null +++ b/modal-deploy/orpheus-3B/api/speakers.py @@ -0,0 +1,115 @@ +"""TTL-cached speakers catalog + per-request validation helpers. + +The catalog is fetched lazily on first call and refreshed when older than +ttl_seconds. If the upstream fetch fails, the cache stays unwarmed and +validation falls open (requests proceed; unknown-speaker errors then +surface from Modal as 502). +""" + +import asyncio +import logging +import time +from dataclasses import dataclass, field + +from api.errors import ( + InvalidSpeakerError, + InvalidSpeakerForLanguageError, + UnknownLanguageError, +) + +logger = logging.getLogger("orpheus_api.speakers") + + +@dataclass +class Catalog: + default: str + by_language: dict[str, list[str]] + speaker_to_language: dict[str, str] = field(default_factory=dict) + + @classmethod + def from_payload(cls, payload: dict) -> "Catalog": + by_lang = payload.get("by_language", {}) or {} + s2l: dict[str, str] = {} + for lang, speakers in by_lang.items(): + for sp in speakers: + s2l[sp] = lang + return cls( + default=payload.get("default", ""), + by_language=by_lang, + speaker_to_language=s2l, + ) + + +class SpeakersCache: + def __init__(self, modal, ttl_seconds: int) -> None: + self.modal = modal + self.ttl = ttl_seconds + self._catalog: Catalog | None = None + self._loaded_at: float = 0.0 + self._lock = asyncio.Lock() + + @property + def is_warm(self) -> bool: + return self._catalog is not None + + async def try_warm(self) -> None: + """Attempt initial load. Never raises — failure leaves cache cold.""" + try: + await self._refresh() + except Exception as exc: # noqa: BLE001 + logger.warning("speakers_cache_warm_failed: %s", exc) + + async def get(self) -> Catalog: + if self._catalog is None or (time.monotonic() - self._loaded_at) > self.ttl: + await self._refresh() + assert self._catalog is not None # _refresh raises on failure + return self._catalog + + async def _refresh(self) -> None: + async with self._lock: + # Double-check after acquiring lock. + if ( + self._catalog is not None + and (time.monotonic() - self._loaded_at) <= self.ttl + ): + return + payload = await self.modal.speakers() + self._catalog = Catalog.from_payload(payload) + self._loaded_at = time.monotonic() + + async def language_for(self, speaker_id: str) -> str | None: + """Reverse lookup: speaker_id -> language. + + Reads the cached catalog directly without triggering a refresh. + Returns None if the cache is cold or the speaker is unknown. + Never raises — callers use this in success paths after TTS + generation has already completed, so a transient Modal blip must + not fail the response. + """ + if self._catalog is None: + return None + return self._catalog.speaker_to_language.get(speaker_id) + + async def validate_speaker( + self, speaker_id: str, *, language: str | None + ) -> None: + """Raises 400-class APIError if invalid. Falls open if cache unwarmed.""" + if not self.is_warm: + return # fail-open + cat = await self.get() + if speaker_id not in cat.speaker_to_language: + raise InvalidSpeakerError( + f"speaker_id '{speaker_id}' not found; see /speakers" + ) + if language is not None: + if language not in cat.by_language: + raise UnknownLanguageError( + f"language '{language}' not supported; " + f"supported: {sorted(cat.by_language)}" + ) + actual = cat.speaker_to_language[speaker_id] + if actual != language: + raise InvalidSpeakerForLanguageError( + f"speaker '{speaker_id}' is for language '{actual}', " + f"not '{language}'; see /speakers/{language}" + ) diff --git a/modal-deploy/orpheus-3B/api/storage.py b/modal-deploy/orpheus-3B/api/storage.py new file mode 100644 index 00000000..62e687d8 --- /dev/null +++ b/modal-deploy/orpheus-3B/api/storage.py @@ -0,0 +1,115 @@ +"""GCS upload + v4 signed URL helpers. + +google-cloud-storage is sync. We wrap each blocking call in +asyncio.to_thread so the FastAPI event loop is free during uploads — +this is what lets `/tts/batch` upload in parallel via asyncio.gather. +""" + +import asyncio +import datetime as dt +import logging +import time +import uuid +from dataclasses import dataclass + +from api.errors import StorageUnavailableError + +logger = logging.getLogger("orpheus_api.storage") + + +@dataclass +class UploadResult: + gcs_object: str + audio_url: str + audio_url_expires_at: dt.datetime + audio_size_bytes: int + upload_ms: float + signed_url_ms: float + + +class StorageBackend: + def __init__( + self, + *, + client, + bucket_name: str, + object_prefix: str, + signed_url_expiry_minutes: int, + ) -> None: + self._client = client + self.bucket_name = bucket_name + self.object_prefix = object_prefix.strip("/") + self.signed_expiry = dt.timedelta(minutes=signed_url_expiry_minutes) + + def _object_name(self) -> str: + date = dt.datetime.now(dt.timezone.utc).strftime("%Y-%m-%d") + return f"{self.object_prefix}/{date}/{uuid.uuid4().hex}.wav" + + async def upload_wav( + self, *, audio_bytes: bytes, content_type: str = "audio/wav" + ) -> UploadResult: + name = self._object_name() + try: + return await asyncio.to_thread( + self._upload_and_sign, name, audio_bytes, content_type + ) + except StorageUnavailableError: + raise + except Exception as exc: # noqa: BLE001 + raise StorageUnavailableError(f"GCS failure: {exc}") from exc + + def _upload_and_sign( + self, name: str, audio_bytes: bytes, content_type: str + ) -> UploadResult: + bucket = self._client.bucket(self.bucket_name) + blob = bucket.blob(name) + # Hint to any CDN/proxy not to cache the (short-lived) signed-URL + # response. The metadata setter on real GCS blobs accepts a dict; + # guard narrowly against the rare case where a custom blob impl + # exposes a non-writable property. + try: + blob.metadata = {"Cache-Control": "private, max-age=0"} + except (AttributeError, TypeError) as exc: + logger.debug("blob_metadata_unset: %s", exc) + + t0 = _monotonic_ms() + try: + blob.upload_from_string( + audio_bytes, content_type=content_type, timeout=60 + ) + except Exception as exc: # noqa: BLE001 + raise StorageUnavailableError(f"upload failed: {exc}") from exc + t1 = _monotonic_ms() + + try: + url = blob.generate_signed_url( + version="v4", + expiration=self.signed_expiry, + method="GET", + ) + except Exception as exc: # noqa: BLE001 + raise StorageUnavailableError(f"sign failed: {exc}") from exc + t2 = _monotonic_ms() + + return UploadResult( + gcs_object=name, + audio_url=url, + audio_url_expires_at=dt.datetime.now(dt.timezone.utc) + self.signed_expiry, + audio_size_bytes=len(audio_bytes), + upload_ms=t1 - t0, + signed_url_ms=t2 - t1, + ) + + async def check_bucket(self) -> bool: + try: + return await asyncio.to_thread(self._check_bucket_blocking) + except Exception as exc: # noqa: BLE001 + logger.warning("gcs_check_failed: %s", exc) + return False + + def _check_bucket_blocking(self) -> bool: + return bool(self._client.bucket(self.bucket_name).exists()) + + +def _monotonic_ms() -> float: + return time.monotonic() * 1000.0 diff --git a/modal-deploy/orpheus-3B/api/tests/__init__.py b/modal-deploy/orpheus-3B/api/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/modal-deploy/orpheus-3B/api/tests/conftest.py b/modal-deploy/orpheus-3B/api/tests/conftest.py new file mode 100644 index 00000000..02534325 --- /dev/null +++ b/modal-deploy/orpheus-3B/api/tests/conftest.py @@ -0,0 +1,130 @@ +"""Shared pytest fixtures: TestClient with mocked Modal + fake GCS.""" + +import base64 +import struct +from typing import Callable + +import httpx +import pytest +from fastapi.testclient import TestClient + + +def make_wav(n_samples: int = 2400) -> bytes: + sample_rate = 24000 + n_channels = 1 + bits = 16 + byte_rate = sample_rate * n_channels * bits // 8 + block_align = n_channels * bits // 8 + data = b"\x00\x00" * n_samples + riff = b"RIFF" + struct.pack(" httpx.Response: + if request.url.path == "/speakers": + return httpx.Response(200, json=SPEAKERS_PAYLOAD) + if request.url.path == "/health": + return httpx.Response(200, json={"status": "ok"}) + return httpx.Response(404) + + return handler diff --git a/modal-deploy/orpheus-3B/api/tests/test_config.py b/modal-deploy/orpheus-3B/api/tests/test_config.py new file mode 100644 index 00000000..29e01aad --- /dev/null +++ b/modal-deploy/orpheus-3B/api/tests/test_config.py @@ -0,0 +1,74 @@ +import os +from pathlib import Path + +import pytest +from pydantic import ValidationError + + +def _clear_env(monkeypatch, keep=()): + """Remove every ORPHEUS_/GCS_/GOOGLE_APPLICATION_CREDENTIALS env var, + except those in `keep`, before instantiating Settings.""" + for var in list(os.environ): + if var.startswith(("ORPHEUS_", "GCS_", "MODAL_", "API_", "SPEAKERS_")): + if var not in keep: + monkeypatch.delenv(var, raising=False) + if var == "GOOGLE_APPLICATION_CREDENTIALS" and var not in keep: + monkeypatch.delenv(var, raising=False) + + +def test_settings_load_with_required_vars(monkeypatch, tmp_path): + _clear_env(monkeypatch) + creds = tmp_path / "sa.json" + creds.write_text("{}") + monkeypatch.setenv("ORPHEUS_MODAL_URL", "https://example.modal.run") + monkeypatch.setenv("GCS_BUCKET_NAME", "my-bucket") + monkeypatch.setenv("GOOGLE_APPLICATION_CREDENTIALS", str(creds)) + + from api.config import Settings + s = Settings() + assert s.orpheus_modal_url == "https://example.modal.run" + assert s.gcs_bucket_name == "my-bucket" + assert s.google_application_credentials == str(creds) + assert s.gcs_signed_url_expiry_minutes == 30 + assert s.max_batch_size == 16 + + +def test_settings_missing_required_raises(monkeypatch): + _clear_env(monkeypatch) + from api.config import Settings + # _env_file=None disables .env reading for this test so it stays hermetic + # even when a developer has a real .env in the cwd. + with pytest.raises(ValidationError) as excinfo: + Settings(_env_file=None) + # error should name at least one of the missing required fields + msg = str(excinfo.value).lower() + assert "orpheus_modal_url" in msg or "gcs_bucket_name" in msg + + +def test_settings_credentials_file_must_exist(monkeypatch, tmp_path): + _clear_env(monkeypatch) + monkeypatch.setenv("ORPHEUS_MODAL_URL", "https://example.modal.run") + monkeypatch.setenv("GCS_BUCKET_NAME", "my-bucket") + monkeypatch.setenv( + "GOOGLE_APPLICATION_CREDENTIALS", str(tmp_path / "does-not-exist.json") + ) + from api.config import Settings + with pytest.raises(ValidationError) as excinfo: + Settings() + assert "google_application_credentials" in str(excinfo.value).lower() + + +def test_settings_overrides(monkeypatch, tmp_path): + _clear_env(monkeypatch) + creds = tmp_path / "sa.json" + creds.write_text("{}") + monkeypatch.setenv("ORPHEUS_MODAL_URL", "https://example.modal.run") + monkeypatch.setenv("GCS_BUCKET_NAME", "my-bucket") + monkeypatch.setenv("GOOGLE_APPLICATION_CREDENTIALS", str(creds)) + monkeypatch.setenv("GCS_SIGNED_URL_EXPIRY_MINUTES", "60") + monkeypatch.setenv("MAX_BATCH_SIZE", "32") + + from api.config import Settings + s = Settings() + assert s.gcs_signed_url_expiry_minutes == 60 + assert s.max_batch_size == 32 diff --git a/modal-deploy/orpheus-3B/api/tests/test_errors.py b/modal-deploy/orpheus-3B/api/tests/test_errors.py new file mode 100644 index 00000000..6356ea91 --- /dev/null +++ b/modal-deploy/orpheus-3B/api/tests/test_errors.py @@ -0,0 +1,97 @@ +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient + + +def _build_app(): + """Mini FastAPI app wired with our handlers, used only by tests.""" + from api.errors import ( + APIError, + InvalidSpeakerError, + StorageUnavailableError, + UpstreamTimeoutError, + UpstreamUnavailableError, + register_exception_handlers, + request_id_middleware, + ) + + app = FastAPI() + app.middleware("http")(request_id_middleware) + register_exception_handlers(app) + + @app.get("/raise/{kind}") + def raise_kind(kind: str): + if kind == "invalid_speaker": + raise InvalidSpeakerError("speaker x not found") + if kind == "upstream_unavailable": + raise UpstreamUnavailableError("modal said 502") + if kind == "upstream_timeout": + raise UpstreamTimeoutError("modal slow") + if kind == "storage_unavailable": + raise StorageUnavailableError("gcs down") + if kind == "generic": + raise RuntimeError("boom") + return {"ok": True} + + return app + + +def test_request_id_generated_if_missing(): + client = TestClient(_build_app()) + r = client.get("/raise/ok-path-doesnt-exist") # 404 from FastAPI + assert "x-request-id" in {k.lower() for k in r.headers} + + +def test_request_id_echoed_if_provided(): + client = TestClient(_build_app()) + r = client.get("/raise/ok", headers={"X-Request-ID": "client-123"}) + # 404 because /raise/ok isn't defined either, but middleware still runs + assert r.headers.get("x-request-id") == "client-123" + + +@pytest.mark.parametrize( + "kind,expected_status,expected_code", + [ + ("invalid_speaker", 400, "invalid_speaker"), + ("upstream_unavailable", 502, "upstream_unavailable"), + ("upstream_timeout", 504, "upstream_timeout"), + ("storage_unavailable", 502, "storage_unavailable"), + ("generic", 500, "internal_error"), + ], +) +def test_handlers_map_errors_to_responses(kind, expected_status, expected_code): + # raise_server_exceptions=False is required so TestClient does not re-raise the + # generic Exception case; we want to verify the handler turns it into a 500. + client = TestClient(_build_app(), raise_server_exceptions=False) + r = client.get(f"/raise/{kind}", headers={"X-Request-ID": "abc"}) + assert r.status_code == expected_status + body = r.json() + assert body["error"] == expected_code + assert body["request_id"] == "abc" + assert "detail" in body + # generic error must NOT leak the underlying message + if kind == "generic": + assert "boom" not in body["detail"].lower() + + +def test_validation_error_uses_uniform_shape(): + from api.errors import register_exception_handlers, request_id_middleware + from pydantic import BaseModel + + app = FastAPI() + app.middleware("http")(request_id_middleware) + register_exception_handlers(app) + + class Body(BaseModel): + x: int + + @app.post("/v") + def v(b: Body): + return {"ok": True} + + client = TestClient(app) + r = client.post("/v", json={"x": "not-an-int"}) + assert r.status_code == 422 + body = r.json() + assert body["error"] == "invalid_request" + assert "request_id" in body diff --git a/modal-deploy/orpheus-3B/api/tests/test_modal_client.py b/modal-deploy/orpheus-3B/api/tests/test_modal_client.py new file mode 100644 index 00000000..7ef9672a --- /dev/null +++ b/modal-deploy/orpheus-3B/api/tests/test_modal_client.py @@ -0,0 +1,376 @@ +import asyncio +import base64 +import io +import json +import struct +from typing import Callable + +import httpx +import pytest + +from api.errors import UpstreamTimeoutError, UpstreamUnavailableError +from api.modal_client import ModalClient + + +def _wav_bytes(n_samples: int = 2400) -> bytes: + """Minimal mono 16-bit 24 kHz WAV with `n_samples` of silence.""" + sample_rate = 24000 + n_channels = 1 + bits = 16 + byte_rate = sample_rate * n_channels * bits // 8 + block_align = n_channels * bits // 8 + data = b"\x00\x00" * n_samples + riff = b"RIFF" + struct.pack(" httpx.Response: + assert req.url.path == "/health" + return httpx.Response(200, json={"status": "ok"}) + + async with _make_client(handler) as ac: + mc = ModalClient(client=ac, retry_backoff_seconds=0.0) + ok = await mc.health() + assert ok is True + + +@pytest.mark.asyncio +async def test_speakers_returns_payload(): + payload = { + "default": "salt_lug_0001", + "by_language": {"lug": ["salt_lug_0001"], "eng": ["salt_eng_0001"]}, + } + + def handler(req: httpx.Request) -> httpx.Response: + assert req.url.path == "/speakers" + return httpx.Response(200, json=payload) + + async with _make_client(handler) as ac: + mc = ModalClient(client=ac, retry_backoff_seconds=0.0) + out = await mc.speakers() + assert out == payload + + +@pytest.mark.asyncio +async def test_tts_returns_wav_and_duration(): + wav = _wav_bytes() + + def handler(req: httpx.Request) -> httpx.Response: + assert req.url.path == "/tts" + body = json.loads(req.content) + assert body["text"] == "hi" + assert body["speaker_id"] == "salt_lug_0001" + return httpx.Response( + 200, + content=wav, + headers={ + "Content-Type": "audio/wav", + "X-Sample-Rate": "24000", + "X-Duration-Seconds": "1.234", + "X-Speaker-Id": "salt_lug_0001", + }, + ) + + async with _make_client(handler) as ac: + mc = ModalClient(client=ac, retry_backoff_seconds=0.0) + result = await mc.tts( + text="hi", + speaker_id="salt_lug_0001", + seed=None, + temperature=0.6, + top_p=0.95, + repetition_penalty=1.1, + max_tokens=1200, + ) + assert result.audio_bytes == wav + assert result.duration_seconds == pytest.approx(1.234) + assert result.sample_rate == 24000 + + +@pytest.mark.asyncio +async def test_tts_parses_x_chunks_header(): + wav = _wav_bytes() + + def handler(req: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, + content=wav, + headers={ + "Content-Type": "audio/wav", + "X-Sample-Rate": "24000", + "X-Duration-Seconds": "5.0", + "X-Speaker-Id": "salt_lug_0001", + "X-Chunks": "6", + }, + ) + + async with _make_client(handler) as ac: + mc = ModalClient(client=ac, retry_backoff_seconds=0.0) + result = await mc.tts( + text="long", + speaker_id="salt_lug_0001", + seed=None, + temperature=0.6, + top_p=0.95, + repetition_penalty=1.1, + max_tokens=1200, + ) + assert result.chunks == 6 + + +@pytest.mark.asyncio +async def test_tts_chunks_defaults_to_none_when_header_absent(): + wav = _wav_bytes() + + def handler(req: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, + content=wav, + headers={ + "Content-Type": "audio/wav", + "X-Sample-Rate": "24000", + "X-Duration-Seconds": "1.0", + # no X-Chunks header (simulates older Modal deploy) + }, + ) + + async with _make_client(handler) as ac: + mc = ModalClient(client=ac, retry_backoff_seconds=0.0) + result = await mc.tts( + text="hi", + speaker_id="salt_lug_0001", + seed=None, + temperature=0.6, + top_p=0.95, + repetition_penalty=1.1, + max_tokens=1200, + ) + assert result.chunks is None + + +@pytest.mark.asyncio +async def test_tts_chunks_is_none_when_header_is_not_an_integer(): + wav = _wav_bytes() + + def handler(req: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, + content=wav, + headers={ + "Content-Type": "audio/wav", + "X-Sample-Rate": "24000", + "X-Duration-Seconds": "1.0", + "X-Chunks": "abc", # malformed; simulates upstream bug + }, + ) + + async with _make_client(handler) as ac: + mc = ModalClient(client=ac, retry_backoff_seconds=0.0) + result = await mc.tts( + text="hi", + speaker_id="salt_lug_0001", + seed=None, + temperature=0.6, + top_p=0.95, + repetition_penalty=1.1, + max_tokens=1200, + ) + assert result.chunks is None + + +@pytest.mark.asyncio +async def test_tts_retries_once_on_502_then_succeeds(): + wav = _wav_bytes() + calls = {"n": 0} + + def handler(req: httpx.Request) -> httpx.Response: + calls["n"] += 1 + if calls["n"] == 1: + return httpx.Response(502, text="bad gateway") + return httpx.Response( + 200, content=wav, headers={"X-Duration-Seconds": "0.5"} + ) + + async with _make_client(handler) as ac: + mc = ModalClient(client=ac, retry_backoff_seconds=0.0) + result = await mc.tts( + text="hi", + speaker_id="salt_lug_0001", + seed=None, + temperature=0.6, + top_p=0.95, + repetition_penalty=1.1, + max_tokens=1200, + ) + assert result.audio_bytes == wav + assert calls["n"] == 2 + + +@pytest.mark.asyncio +async def test_tts_persistent_502_raises_upstream_unavailable(): + def handler(req: httpx.Request) -> httpx.Response: + return httpx.Response(502, text="bad gateway") + + async with _make_client(handler) as ac: + mc = ModalClient(client=ac, retry_backoff_seconds=0.0) + with pytest.raises(UpstreamUnavailableError): + await mc.tts( + text="hi", speaker_id="salt_lug_0001", + seed=None, temperature=0.6, top_p=0.95, + repetition_penalty=1.1, max_tokens=1200, + ) + + +@pytest.mark.asyncio +async def test_tts_read_timeout_then_success(): + wav = _wav_bytes() + calls = {"n": 0} + + def handler(req: httpx.Request) -> httpx.Response: + calls["n"] += 1 + if calls["n"] == 1: + raise httpx.ReadTimeout("timeout") + return httpx.Response( + 200, content=wav, headers={"X-Duration-Seconds": "0.5"} + ) + + async with _make_client(handler) as ac: + mc = ModalClient(client=ac, retry_backoff_seconds=0.0) + result = await mc.tts( + text="hi", speaker_id="salt_lug_0001", + seed=None, temperature=0.6, top_p=0.95, + repetition_penalty=1.1, max_tokens=1200, + ) + assert calls["n"] == 2 + + +@pytest.mark.asyncio +async def test_tts_persistent_timeout_raises_upstream_timeout(): + def handler(req: httpx.Request) -> httpx.Response: + raise httpx.ReadTimeout("timeout") + + async with _make_client(handler) as ac: + mc = ModalClient(client=ac, retry_backoff_seconds=0.0) + with pytest.raises(UpstreamTimeoutError): + await mc.tts( + text="hi", speaker_id="salt_lug_0001", + seed=None, temperature=0.6, top_p=0.95, + repetition_penalty=1.1, max_tokens=1200, + ) + + +@pytest.mark.asyncio +async def test_tts_batch_parses_b64_results(): + wav1 = _wav_bytes(1200) + wav2 = _wav_bytes(2400) + payload = { + "results": [ + { + "text": "a", + "speaker_id": "salt_lug_0001", + "sample_rate": 24000, + "duration_sec": 0.05, + "audio_wav_b64": base64.b64encode(wav1).decode("ascii"), + }, + { + "text": "b", + "speaker_id": "salt_eng_0001", + "sample_rate": 24000, + "duration_sec": 0.10, + "audio_wav_b64": base64.b64encode(wav2).decode("ascii"), + }, + ] + } + + def handler(req: httpx.Request) -> httpx.Response: + assert req.url.path == "/tts/batch" + return httpx.Response(200, json=payload) + + async with _make_client(handler) as ac: + mc = ModalClient(client=ac, retry_backoff_seconds=0.0) + items = [ + {"text": "a", "speaker_id": "salt_lug_0001", "seed": None, + "temperature": 0.6, "top_p": 0.95, "repetition_penalty": 1.1, "max_tokens": 1200}, + {"text": "b", "speaker_id": "salt_eng_0001", "seed": None, + "temperature": 0.6, "top_p": 0.95, "repetition_penalty": 1.1, "max_tokens": 1200}, + ] + out = await mc.tts_batch(items) + assert len(out) == 2 + assert out[0].audio_bytes == wav1 + assert out[0].duration_seconds == pytest.approx(0.05) + assert out[1].audio_bytes == wav2 + + +@pytest.mark.asyncio +async def test_tts_batch_malformed_base64_raises_upstream_unavailable(): + def handler(req: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, + json={ + "results": [ + { + "text": "a", + "speaker_id": "salt_lug_0001", + "sample_rate": 24000, + "duration_sec": 0.05, + "audio_wav_b64": "this is not valid base64!@#$", + } + ] + }, + ) + + async with _make_client(handler) as ac: + mc = ModalClient(client=ac, retry_backoff_seconds=0.0) + with pytest.raises(UpstreamUnavailableError): + await mc.tts_batch( + [ + { + "text": "a", "speaker_id": "salt_lug_0001", + "seed": None, "temperature": 0.6, "top_p": 0.95, + "repetition_penalty": 1.1, "max_tokens": 1200, + } + ] + ) + + +@pytest.mark.asyncio +async def test_tts_batch_missing_audio_field_raises_upstream_unavailable(): + def handler(req: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, + json={ + "results": [ + { + "text": "a", + "speaker_id": "salt_lug_0001", + "sample_rate": 24000, + "duration_sec": 0.05, + # audio_wav_b64 missing entirely + } + ] + }, + ) + + async with _make_client(handler) as ac: + mc = ModalClient(client=ac, retry_backoff_seconds=0.0) + with pytest.raises(UpstreamUnavailableError): + await mc.tts_batch( + [ + { + "text": "a", "speaker_id": "salt_lug_0001", + "seed": None, "temperature": 0.6, "top_p": 0.95, + "repetition_penalty": 1.1, "max_tokens": 1200, + } + ] + ) diff --git a/modal-deploy/orpheus-3B/api/tests/test_models.py b/modal-deploy/orpheus-3B/api/tests/test_models.py new file mode 100644 index 00000000..b80f7685 --- /dev/null +++ b/modal-deploy/orpheus-3B/api/tests/test_models.py @@ -0,0 +1,101 @@ +from datetime import datetime, timezone + +import pytest +from pydantic import ValidationError + + +def test_tts_request_defaults(): + from api.models import TTSRequest + r = TTSRequest(text="hello") + assert r.speaker_id == "salt_lug_0001" + assert r.language is None + assert r.temperature == 0.6 + assert r.top_p == 0.95 + assert r.repetition_penalty == 1.1 + assert r.max_tokens == 1200 + assert r.seed is None + + +def test_tts_request_text_required(): + from api.models import TTSRequest + with pytest.raises(ValidationError): + TTSRequest(text="") + + +def test_tts_request_text_too_long(): + from api.models import TTSRequest + with pytest.raises(ValidationError): + TTSRequest(text="x" * 2001) + + +@pytest.mark.parametrize( + "field,value", + [ + ("temperature", -0.1), + ("temperature", 2.1), + ("top_p", 0.0), + ("top_p", 1.1), + ("repetition_penalty", 0.99), + ("repetition_penalty", 2.01), + ("max_tokens", 63), + ("max_tokens", 4097), + ], +) +def test_tts_request_bounds(field, value): + from api.models import TTSRequest + with pytest.raises(ValidationError): + TTSRequest(text="ok", **{field: value}) + + +def test_tts_batch_request_size_bounds(): + from api.models import TTSBatchRequest, TTSRequest + with pytest.raises(ValidationError): + TTSBatchRequest(items=[]) + items = [TTSRequest(text=f"t{i}") for i in range(129)] + with pytest.raises(ValidationError): + TTSBatchRequest(items=items) + + +def test_timings_total(): + from api.models import Timings + t = Timings(inference_ms=100.0, upload_ms=50.0, signed_url_ms=10.0, total_ms=160.0) + assert t.total_ms == 160.0 + + +def test_tts_response_ok(): + from api.models import Timings, TTSResponse + resp = TTSResponse( + audio_url="https://storage.googleapis.com/x?token=y", + audio_url_expires_at=datetime.now(timezone.utc), + speaker_id="salt_lug_0001", + language="lug", + sample_rate=24000, + duration_seconds=2.4, + audio_size_bytes=115_244, + gcs_object="tts/2026-05-12/abc.wav", + request_id="abc-123", + timings_ms=Timings(inference_ms=1000, upload_ms=200, signed_url_ms=10, total_ms=1210), + ) + assert str(resp.audio_url).startswith("https://") + + +def test_speakers_response_total_computed(): + from api.models import SpeakersResponse + r = SpeakersResponse( + default="salt_lug_0001", + by_language={"lug": ["salt_lug_0001", "salt_lug_0002"], "eng": ["salt_eng_0001"]}, + ) + assert r.total == 3 + assert r.languages == ["eng", "lug"] + + +def test_language_speakers_response(): + from api.models import LanguageSpeakersResponse + r = LanguageSpeakersResponse(language="lug", speakers=["salt_lug_0001"]) + assert r.count == 1 + + +def test_error_response_shape(): + from api.models import ErrorResponse + e = ErrorResponse(error="invalid_speaker", detail="not found", request_id="abc") + assert e.error == "invalid_speaker" diff --git a/modal-deploy/orpheus-3B/api/tests/test_routes_batch.py b/modal-deploy/orpheus-3B/api/tests/test_routes_batch.py new file mode 100644 index 00000000..619587d6 --- /dev/null +++ b/modal-deploy/orpheus-3B/api/tests/test_routes_batch.py @@ -0,0 +1,141 @@ +import base64 +import json + +import httpx +import pytest +from fastapi.testclient import TestClient + +from api.tests.conftest import SPEAKERS_PAYLOAD, build_test_app, make_wav + + +def _batch_handler(*, audios: list[bytes], status: int = 200): + """Return /tts/batch JSON payload with per-item audios.""" + + def handler(req: httpx.Request) -> httpx.Response: + if req.url.path == "/speakers": + return httpx.Response(200, json=SPEAKERS_PAYLOAD) + if req.url.path == "/health": + return httpx.Response(200, json={"status": "ok"}) + if req.url.path == "/tts/batch": + if status != 200: + return httpx.Response(status, text="upstream error") + body = json.loads(req.content) + assert len(audios) == len(body["items"]) + return httpx.Response( + 200, + json={ + "results": [ + { + "text": it["text"], + "speaker_id": it["speaker_id"], + "sample_rate": 24000, + "duration_sec": 1.0, + "audio_wav_b64": base64.b64encode(a).decode("ascii"), + } + for it, a in zip(body["items"], audios) + ] + }, + ) + return httpx.Response(404) + + return handler + + +def test_batch_happy_path(): + wavs = [make_wav(1000), make_wav(2000), make_wav(3000)] + app = build_test_app(_batch_handler(audios=wavs)) + client = TestClient(app) + r = client.post( + "/tts/batch", + json={ + "items": [ + {"text": "a", "speaker_id": "salt_lug_0001"}, + {"text": "b", "speaker_id": "salt_eng_0001"}, + {"text": "c", "speaker_id": "waxal_swa_0006"}, + ] + }, + ) + assert r.status_code == 200, r.text + body = r.json() + assert len(body["results"]) == 3 + for i, res in enumerate(body["results"]): + assert res["index"] == i + assert res["status"] == "ok" + assert res["audio_size_bytes"] == len(wavs[i]) + assert res["audio_url"].startswith("https://signed.example/") + assert res["gcs_object"].startswith("tts/") + assert res["sample_rate"] == 24000 + assert body["timings_ms"]["inference_ms"] >= 0 + assert body["timings_ms"]["upload_ms"] >= 0 + + +def test_batch_rejects_empty(): + app = build_test_app(_batch_handler(audios=[])) + client = TestClient(app) + r = client.post("/tts/batch", json={"items": []}) + assert r.status_code == 422 + + +def test_batch_rejects_unknown_speaker_in_any_item(): + wavs = [make_wav()] + app = build_test_app(_batch_handler(audios=wavs)) + client = TestClient(app) + r = client.post( + "/tts/batch", + json={ + "items": [ + {"text": "a", "speaker_id": "salt_lug_0001"}, + {"text": "b", "speaker_id": "salt_zzz_9999"}, + ] + }, + ) + assert r.status_code == 400 + body = r.json() + assert body["error"] == "invalid_speaker" + assert "1" in body["detail"] + + +def test_batch_all_modal_502_returns_502(): + app = build_test_app(_batch_handler(audios=[], status=502)) + client = TestClient(app) + r = client.post( + "/tts/batch", + json={"items": [{"text": "a", "speaker_id": "salt_lug_0001"}]}, + ) + assert r.status_code == 502 + + +def test_batch_per_item_gcs_failure_does_not_sink_others(monkeypatch): + """If GCS upload for one item raises, the other items still return OK.""" + wavs = [make_wav(1000), make_wav(2000)] + app = build_test_app(_batch_handler(audios=wavs)) + + original = app.state.storage.upload_wav + calls = {"n": 0} + + async def flaky_upload(*, audio_bytes, content_type): + calls["n"] += 1 + if calls["n"] == 2: + from api.errors import StorageUnavailableError + raise StorageUnavailableError("fake gcs failure on item 2") + return await original(audio_bytes=audio_bytes, content_type=content_type) + + app.state.storage.upload_wav = flaky_upload # type: ignore[assignment] + + client = TestClient(app) + r = client.post( + "/tts/batch", + json={ + "items": [ + {"text": "a", "speaker_id": "salt_lug_0001"}, + {"text": "b", "speaker_id": "salt_eng_0001"}, + ] + }, + ) + assert r.status_code == 200, r.text + body = r.json() + statuses = [r["status"] for r in body["results"]] + assert statuses.count("ok") == 1 + assert statuses.count("error") == 1 + failed = [r for r in body["results"] if r["status"] == "error"][0] + assert failed["error_code"] == "storage_unavailable" diff --git a/modal-deploy/orpheus-3B/api/tests/test_routes_health.py b/modal-deploy/orpheus-3B/api/tests/test_routes_health.py new file mode 100644 index 00000000..d3365ae5 --- /dev/null +++ b/modal-deploy/orpheus-3B/api/tests/test_routes_health.py @@ -0,0 +1,65 @@ +import httpx +from fastapi.testclient import TestClient + +from api.tests.conftest import SPEAKERS_PAYLOAD, build_test_app + + +def _healthy_handler(): + def handler(req: httpx.Request) -> httpx.Response: + if req.url.path == "/health": + return httpx.Response(200, json={"status": "ok"}) + if req.url.path == "/speakers": + return httpx.Response(200, json=SPEAKERS_PAYLOAD) + return httpx.Response(404) + + return handler + + +def _modal_down_handler(): + def handler(req: httpx.Request) -> httpx.Response: + if req.url.path == "/speakers": + return httpx.Response(200, json=SPEAKERS_PAYLOAD) + return httpx.Response(503, text="modal down") + + return handler + + +def test_health_liveness_no_upstream(): + app = build_test_app(_healthy_handler()) + client = TestClient(app) + r = client.get("/health") + assert r.status_code == 200 + assert r.json() == {"status": "ok", "service": "orpheus-tts-api"} + + +def test_healthz_all_upstreams_ok(): + app = build_test_app(_healthy_handler()) + client = TestClient(app) + r = client.get("/healthz") + assert r.status_code == 200 + body = r.json() + assert body["status"] == "ok" + assert body["upstreams"]["modal"] == "ok" + assert body["upstreams"]["gcs"] == "ok" + + +def test_healthz_modal_down_returns_503(): + app = build_test_app(_modal_down_handler()) + client = TestClient(app) + r = client.get("/healthz") + assert r.status_code == 503 + body = r.json() + assert body["status"] == "degraded" + assert body["upstreams"]["modal"] != "ok" + + +def test_healthz_gcs_down_returns_503(): + app = build_test_app(_healthy_handler()) + # break the fake bucket + app.state.storage._client.bucket_obj._exists = False + client = TestClient(app) + r = client.get("/healthz") + assert r.status_code == 503 + body = r.json() + assert body["status"] == "degraded" + assert body["upstreams"]["gcs"] != "ok" diff --git a/modal-deploy/orpheus-3B/api/tests/test_routes_speakers.py b/modal-deploy/orpheus-3B/api/tests/test_routes_speakers.py new file mode 100644 index 00000000..62eb393c --- /dev/null +++ b/modal-deploy/orpheus-3B/api/tests/test_routes_speakers.py @@ -0,0 +1,36 @@ +from fastapi.testclient import TestClient + +from api.tests.conftest import SPEAKERS_PAYLOAD, build_test_app + + +def test_get_speakers_returns_catalog(speakers_handler): + app = build_test_app(speakers_handler) + client = TestClient(app) + r = client.get("/speakers") + assert r.status_code == 200 + body = r.json() + assert body["default"] == SPEAKERS_PAYLOAD["default"] + assert body["by_language"] == SPEAKERS_PAYLOAD["by_language"] + assert body["total"] == 4 + assert body["languages"] == ["eng", "lug", "swa"] + + +def test_get_speakers_by_language_ok(speakers_handler): + app = build_test_app(speakers_handler) + client = TestClient(app) + r = client.get("/speakers/lug") + assert r.status_code == 200 + body = r.json() + assert body["language"] == "lug" + assert body["speakers"] == ["salt_lug_0001", "salt_lug_0002"] + assert body["count"] == 2 + + +def test_get_speakers_by_language_unknown_returns_404(speakers_handler): + app = build_test_app(speakers_handler) + client = TestClient(app) + r = client.get("/speakers/xyz") + assert r.status_code == 404 + body = r.json() + assert body["error"] == "unknown_language" + assert "xyz" in body["detail"] diff --git a/modal-deploy/orpheus-3B/api/tests/test_routes_tts.py b/modal-deploy/orpheus-3B/api/tests/test_routes_tts.py new file mode 100644 index 00000000..4a156616 --- /dev/null +++ b/modal-deploy/orpheus-3B/api/tests/test_routes_tts.py @@ -0,0 +1,195 @@ +import base64 +import json +from urllib.parse import urlparse + +import httpx +import pytest +from fastapi.testclient import TestClient + +from api.tests.conftest import SPEAKERS_PAYLOAD, build_test_app, make_wav + + +def _modal_handler( + *, audio: bytes, duration: float = 1.5, x_chunks: str | None = None +): + def handler(req: httpx.Request) -> httpx.Response: + if req.url.path == "/speakers": + return httpx.Response(200, json=SPEAKERS_PAYLOAD) + if req.url.path == "/health": + return httpx.Response(200, json={"status": "ok"}) + if req.url.path == "/tts": + headers = { + "Content-Type": "audio/wav", + "X-Sample-Rate": "24000", + "X-Duration-Seconds": str(duration), + "X-Speaker-Id": json.loads(req.content)["speaker_id"], + } + if x_chunks is not None: + headers["X-Chunks"] = x_chunks + return httpx.Response(200, content=audio, headers=headers) + return httpx.Response(404) + + return handler + + +def test_tts_happy_path(): + wav = make_wav() + app = build_test_app(_modal_handler(audio=wav, duration=2.0)) + client = TestClient(app) + r = client.post( + "/tts", + json={"text": "Mwattu", "speaker_id": "salt_lug_0001"}, + ) + assert r.status_code == 200, r.text + body = r.json() + assert body["speaker_id"] == "salt_lug_0001" + assert body["language"] == "lug" + assert body["sample_rate"] == 24000 + assert body["duration_seconds"] == pytest.approx(2.0) + assert body["audio_size_bytes"] == len(wav) + assert body["gcs_object"].startswith("tts/") + assert body["audio_url"].startswith("https://signed.example/") + t = body["timings_ms"] + assert t["inference_ms"] >= 0 + assert t["upload_ms"] >= 0 + assert t["total_ms"] >= 0 + assert "request_id" in body + + +def test_tts_response_includes_chunks_when_upstream_sets_header(): + wav = make_wav() + app = build_test_app(_modal_handler(audio=wav, duration=2.0, x_chunks="4")) + client = TestClient(app) + r = client.post( + "/tts", + json={"text": "long text", "speaker_id": "salt_lug_0001"}, + ) + assert r.status_code == 200, r.text + body = r.json() + assert body["chunks"] == 4 + + +def test_tts_response_chunks_is_null_when_upstream_omits_header(): + wav = make_wav() + app = build_test_app(_modal_handler(audio=wav, duration=1.0)) # no x_chunks + client = TestClient(app) + r = client.post( + "/tts", + json={"text": "short", "speaker_id": "salt_lug_0001"}, + ) + assert r.status_code == 200, r.text + body = r.json() + assert body["chunks"] is None + + +def test_tts_language_field_validated_ok(): + wav = make_wav() + app = build_test_app(_modal_handler(audio=wav)) + client = TestClient(app) + r = client.post( + "/tts", + json={"text": "hi", "speaker_id": "salt_lug_0001", "language": "lug"}, + ) + assert r.status_code == 200, r.text + + +def test_tts_unknown_speaker_400(): + wav = make_wav() + app = build_test_app(_modal_handler(audio=wav)) + client = TestClient(app) + r = client.post( + "/tts", + json={"text": "hi", "speaker_id": "salt_zzz_9999"}, + ) + assert r.status_code == 400 + assert r.json()["error"] == "invalid_speaker" + + +def test_tts_unknown_language_400(): + wav = make_wav() + app = build_test_app(_modal_handler(audio=wav)) + client = TestClient(app) + r = client.post( + "/tts", + json={"text": "hi", "speaker_id": "salt_lug_0001", "language": "xyz"}, + ) + assert r.status_code == 400 + assert r.json()["error"] == "unknown_language" + + +def test_tts_speaker_for_language_mismatch_400(): + wav = make_wav() + app = build_test_app(_modal_handler(audio=wav)) + client = TestClient(app) + r = client.post( + "/tts", + json={"text": "hi", "speaker_id": "salt_lug_0001", "language": "eng"}, + ) + assert r.status_code == 400 + assert r.json()["error"] == "invalid_speaker_for_language" + + +def test_tts_modal_502_returns_502(): + def handler(req): + if req.url.path == "/speakers": + return httpx.Response(200, json=SPEAKERS_PAYLOAD) + if req.url.path == "/health": + return httpx.Response(200, json={"status": "ok"}) + return httpx.Response(502, text="bad gateway") + + app = build_test_app(handler) + client = TestClient(app) + r = client.post("/tts", json={"text": "hi", "speaker_id": "salt_lug_0001"}) + assert r.status_code == 502 + assert r.json()["error"] == "upstream_unavailable" + + +def test_tts_modal_422_text_too_long_surfaces_as_502(): + # Modal raises 422 text_too_long when the chunker exceeds + # MAX_CHUNKS_PER_REQUEST=48. The gateway's TTSRequest.max_length=2000 + # makes this practically unreachable, but the cross-layer mapping + # (Modal 4xx → gateway 502 upstream_unavailable via _with_retry's + # catch-all) is worth pinning down by test in case either limit moves. + def handler(req): + if req.url.path == "/speakers": + return httpx.Response(200, json=SPEAKERS_PAYLOAD) + if req.url.path == "/health": + return httpx.Response(200, json={"status": "ok"}) + return httpx.Response( + 422, + json={ + "detail": { + "error": "text_too_long", + "detail": "text produced 60 chunks (limit 48)", + } + }, + ) + + app = build_test_app(handler) + client = TestClient(app) + r = client.post("/tts", json={"text": "hi", "speaker_id": "salt_lug_0001"}) + assert r.status_code == 502 + assert r.json()["error"] == "upstream_unavailable" + + +def test_tts_request_id_echoed(): + wav = make_wav() + app = build_test_app(_modal_handler(audio=wav)) + client = TestClient(app) + r = client.post( + "/tts", + json={"text": "hi", "speaker_id": "salt_lug_0001"}, + headers={"X-Request-ID": "client-rid"}, + ) + assert r.status_code == 200 + assert r.headers["x-request-id"] == "client-rid" + assert r.json()["request_id"] == "client-rid" + + +def test_tts_validation_error_422_on_empty_text(): + wav = make_wav() + app = build_test_app(_modal_handler(audio=wav)) + client = TestClient(app) + r = client.post("/tts", json={"text": "", "speaker_id": "salt_lug_0001"}) + assert r.status_code == 422 + assert r.json()["error"] == "invalid_request" diff --git a/modal-deploy/orpheus-3B/api/tests/test_speakers.py b/modal-deploy/orpheus-3B/api/tests/test_speakers.py new file mode 100644 index 00000000..d4b0fb65 --- /dev/null +++ b/modal-deploy/orpheus-3B/api/tests/test_speakers.py @@ -0,0 +1,165 @@ +import asyncio + +import pytest + + +class _FakeModal: + def __init__(self, payload, *, fail_first: bool = False): + self.payload = payload + self.calls = 0 + self.fail_first = fail_first + + async def speakers(self) -> dict: + self.calls += 1 + if self.fail_first and self.calls == 1: + raise RuntimeError("modal down") + return self.payload + + +_PAYLOAD = { + "default": "salt_lug_0001", + "by_language": { + "lug": ["salt_lug_0001", "salt_lug_0002"], + "eng": ["salt_eng_0001"], + }, +} + + +@pytest.mark.asyncio +async def test_cache_loads_and_returns_catalog(): + from api.speakers import SpeakersCache + + fake = _FakeModal(_PAYLOAD) + cache = SpeakersCache(modal=fake, ttl_seconds=60) + cat = await cache.get() + assert cat.default == "salt_lug_0001" + assert "lug" in cat.by_language + assert fake.calls == 1 + + +@pytest.mark.asyncio +async def test_cache_reuses_within_ttl(): + from api.speakers import SpeakersCache + + fake = _FakeModal(_PAYLOAD) + cache = SpeakersCache(modal=fake, ttl_seconds=60) + await cache.get() + await cache.get() + assert fake.calls == 1 + + +@pytest.mark.asyncio +async def test_cache_refreshes_after_ttl(): + from api.speakers import SpeakersCache + + fake = _FakeModal(_PAYLOAD) + cache = SpeakersCache(modal=fake, ttl_seconds=0) # always expired + await cache.get() + await cache.get() + assert fake.calls == 2 + + +@pytest.mark.asyncio +async def test_cache_fail_open_when_modal_down_on_startup(): + from api.speakers import SpeakersCache + + fake = _FakeModal(_PAYLOAD, fail_first=True) + cache = SpeakersCache(modal=fake, ttl_seconds=60) + # try_warm() must not raise; should leave cache empty + await cache.try_warm() + assert cache.is_warm is False + + +@pytest.mark.asyncio +async def test_validate_known_speaker_passes(): + from api.speakers import SpeakersCache + + fake = _FakeModal(_PAYLOAD) + cache = SpeakersCache(modal=fake, ttl_seconds=60) + await cache.try_warm() + await cache.validate_speaker("salt_lug_0001", language=None) + await cache.validate_speaker("salt_lug_0001", language="lug") + # also returns the resolved language + lang = await cache.language_for("salt_lug_0001") + assert lang == "lug" + + +@pytest.mark.asyncio +async def test_validate_unknown_speaker_raises(): + from api.errors import InvalidSpeakerError + from api.speakers import SpeakersCache + + fake = _FakeModal(_PAYLOAD) + cache = SpeakersCache(modal=fake, ttl_seconds=60) + await cache.try_warm() + with pytest.raises(InvalidSpeakerError): + await cache.validate_speaker("salt_zzz_9999", language=None) + + +@pytest.mark.asyncio +async def test_validate_unknown_language_raises(): + from api.errors import UnknownLanguageError + from api.speakers import SpeakersCache + + fake = _FakeModal(_PAYLOAD) + cache = SpeakersCache(modal=fake, ttl_seconds=60) + await cache.try_warm() + with pytest.raises(UnknownLanguageError): + await cache.validate_speaker("salt_lug_0001", language="zzz") + + +@pytest.mark.asyncio +async def test_validate_speaker_for_language_mismatch_raises(): + from api.errors import InvalidSpeakerForLanguageError + from api.speakers import SpeakersCache + + fake = _FakeModal(_PAYLOAD) + cache = SpeakersCache(modal=fake, ttl_seconds=60) + await cache.try_warm() + with pytest.raises(InvalidSpeakerForLanguageError): + await cache.validate_speaker("salt_lug_0001", language="eng") + + +@pytest.mark.asyncio +async def test_validate_falls_open_when_not_warm(): + from api.speakers import SpeakersCache + + fake = _FakeModal(_PAYLOAD, fail_first=True) + cache = SpeakersCache(modal=fake, ttl_seconds=60) + await cache.try_warm() # leaves cache unwarmed + assert cache.is_warm is False + # validate_speaker must NOT raise when cache is unwarmed (fail-open) + await cache.validate_speaker("anything", language=None) + await cache.validate_speaker("anything", language="anylang") + + +@pytest.mark.asyncio +async def test_language_for_does_not_refresh_or_raise(): + """language_for must read self._catalog directly without refreshing. + + Even if the TTL has expired and Modal would now fail, language_for + must return the cached value (or None) without raising. + """ + from api.speakers import SpeakersCache + + fake = _FakeModal(_PAYLOAD) + cache = SpeakersCache(modal=fake, ttl_seconds=60) + await cache.try_warm() + assert fake.calls == 1 + + # Make subsequent modal.speakers() raise. If language_for tried to + # refresh, this would surface; we want None or a stale-cache hit + # instead. + async def boom(): + raise RuntimeError("modal exploded") + + fake.speakers = boom # type: ignore[assignment] + + lang = await cache.language_for("salt_lug_0001") + assert lang == "lug" + # No additional modal calls were made. + assert fake.calls == 1 + + # Unknown speaker returns None, not a raise. + lang_unknown = await cache.language_for("salt_zzz_9999") + assert lang_unknown is None diff --git a/modal-deploy/orpheus-3B/api/tests/test_storage.py b/modal-deploy/orpheus-3B/api/tests/test_storage.py new file mode 100644 index 00000000..4f626c4c --- /dev/null +++ b/modal-deploy/orpheus-3B/api/tests/test_storage.py @@ -0,0 +1,138 @@ +import datetime as dt + +import pytest + + +class _FakeBlob: + def __init__(self, name: str, bucket: "_FakeBucket"): + self.name = name + self.bucket = bucket + self.uploaded_bytes: bytes | None = None + self.uploaded_content_type: str | None = None + self.uploaded_metadata: dict | None = None + self.signed_url_calls: list[dict] = [] + self.metadata: dict = {} + + def upload_from_string(self, data, content_type, timeout=None): + self.uploaded_bytes = data + self.uploaded_content_type = content_type + self.bucket.blobs[self.name] = self + + def generate_signed_url(self, **kwargs): + self.signed_url_calls.append(kwargs) + return f"https://signed.example/{self.bucket.name}/{self.name}?exp=test" + + +class _FakeBucket: + def __init__(self, name: str): + self.name = name + self.blobs: dict[str, _FakeBlob] = {} + self._exists = True + + def blob(self, name: str) -> _FakeBlob: + return _FakeBlob(name, self) + + def exists(self) -> bool: + return self._exists + + +class _FakeStorage: + def __init__(self): + self.bucket_obj = _FakeBucket("my-bucket") + + def bucket(self, name: str) -> _FakeBucket: + assert name == "my-bucket" + return self.bucket_obj + + +@pytest.mark.asyncio +async def test_upload_and_sign_returns_url_and_path(): + from api.storage import StorageBackend + + fake = _FakeStorage() + backend = StorageBackend( + client=fake, + bucket_name="my-bucket", + object_prefix="tts", + signed_url_expiry_minutes=30, + ) + audio = b"RIFF....fake-wav" + result = await backend.upload_wav(audio_bytes=audio, content_type="audio/wav") + + # path shape: tts/YYYY-MM-DD/.wav + parts = result.gcs_object.split("/") + assert parts[0] == "tts" + assert len(parts[1]) == 10 and parts[1].count("-") == 2 # YYYY-MM-DD + assert parts[2].endswith(".wav") + + blob = fake.bucket_obj.blobs[result.gcs_object] + assert blob.uploaded_bytes == audio + assert blob.uploaded_content_type == "audio/wav" + assert result.audio_url.startswith("https://signed.example/") + assert result.audio_url_expires_at > dt.datetime.now(dt.timezone.utc) + # 30-minute expiry + delta = result.audio_url_expires_at - dt.datetime.now(dt.timezone.utc) + assert 25 * 60 < delta.total_seconds() < 32 * 60 + + +@pytest.mark.asyncio +async def test_signed_url_uses_v4_get_with_correct_expiry(): + from api.storage import StorageBackend + + fake = _FakeStorage() + backend = StorageBackend( + client=fake, + bucket_name="my-bucket", + object_prefix="tts", + signed_url_expiry_minutes=45, + ) + audio = b"RIFF....fake-wav" + result = await backend.upload_wav(audio_bytes=audio, content_type="audio/wav") + blob = fake.bucket_obj.blobs[result.gcs_object] + assert blob.signed_url_calls + call = blob.signed_url_calls[0] + assert call["version"] == "v4" + assert call["method"] == "GET" + assert int(call["expiration"].total_seconds()) == 45 * 60 + + +@pytest.mark.asyncio +async def test_upload_failure_raises_storage_unavailable(): + from api.errors import StorageUnavailableError + from api.storage import StorageBackend + + class _BoomBucket(_FakeBucket): + def blob(self, name): + blob = super().blob(name) + def boom(*a, **kw): + raise RuntimeError("gcs down") + blob.upload_from_string = boom # type: ignore[assignment] + return blob + + class _BoomStorage(_FakeStorage): + def __init__(self): + self.bucket_obj = _BoomBucket("my-bucket") + + backend = StorageBackend( + client=_BoomStorage(), + bucket_name="my-bucket", + object_prefix="tts", + signed_url_expiry_minutes=30, + ) + with pytest.raises(StorageUnavailableError): + await backend.upload_wav(audio_bytes=b"x", content_type="audio/wav") + + +@pytest.mark.asyncio +async def test_check_bucket_returns_true_when_exists(): + from api.storage import StorageBackend + + fake = _FakeStorage() + backend = StorageBackend( + client=fake, bucket_name="my-bucket", + object_prefix="tts", signed_url_expiry_minutes=30, + ) + assert await backend.check_bucket() is True + + fake.bucket_obj._exists = False + assert await backend.check_bucket() is False diff --git a/modal-deploy/orpheus-3B/article.txt b/modal-deploy/orpheus-3B/article.txt new file mode 100644 index 00000000..71e2dc08 --- /dev/null +++ b/modal-deploy/orpheus-3B/article.txt @@ -0,0 +1,15 @@ +Not since the World-Wide Web emerged 30 years ago has a new technology promised to change our world so fundamentally and so swiftly as AI does. Today’s AI tools such as ChatGPT and AlphaGo are just a hint of what is to come. The future of AI is going to be quite a journey, and the 2023 CHRISTMAS LECTURES, broadcast on BBC Four and iPlayer in late December, were a guided tour. + +AI has increasingly grabbed the headlines in recent years – generating excitement and concern in equal measure – but what should we believe? In these lectures Mike revealed how AI works and how it will affect our lives – and will tackle head-on our hopes and fears for this most fascinating of fields. + +Throughout the three Lectures, Mike explored the big questions facing AI research and unravelled the myths about how this ground-breaking technology really works. How can a machine be taught to play a game or translate from one language to another? He shows how computer programmes inspired by the human brain can be taught and even teach themselves. And he tells the surprising story of how tools like the latest chatbots work and illustrate the mind-boggling scale of modern AI systems – that can be used in everything from gaming to movie-making to designing drugs. + +When Alan Turing suggested the ‘Turing test’, to see if a computer’s responses to questions could be mistaken for a human’s, it seemed like a far-off, perhaps even impossible goal. We played a high stakes ‘Turing test’ in the Ri Theatre, along with other exciting experiments from the AI frontier, to find out just how far we are from Turing’s dream. + +Although we think of AI as something in the future, Mike's lectures show us the role that AI already plays in our lives – without us even being aware of it. He will demonstrate the huge impact AI is having in fields ranging from medicine to football to astrophysics – and even our creative lives - in art, photography, and music. + +Yet while AI will create many exciting opportunities, advances in the field have raised many fears – some justified, others not. There are concerns about identifying what’s fake and what’s real, as AI becomes increasingly clever. Mike will talk us through the dangers that AI raises, from machines adopting human prejudices, through to machines that operate as our boss at work, up to questions about whether AI might be a risk to humanity. + +And finally, Mike addresses the really big question of AI: can it ever truly be like us, or are humans unique? + +Across the series Michael was joined by some major figures from the AI world, including scientists from the world’s leading AI companies. He also introduces a range of robot friends, who will demonstrate what robots today can do – and what they can’t. He may have even surprised the audience with some Deep Fake guests who are not quite what they seem. \ No newline at end of file diff --git a/modal-deploy/orpheus-3B/blogpost-orpheus-multilingual-release.docx b/modal-deploy/orpheus-3B/blogpost-orpheus-multilingual-release.docx new file mode 100644 index 0000000000000000000000000000000000000000..5e1ed50bdd5b2e9375657cf40e0a5a4dd38ebf36 GIT binary patch literal 20161 zcmd43WpG?e5+y8VW@ct)%VK6`W@cuo#mvksSb-GK@)ppS6fp%7X}Yo8-~AzZlJRM^1otsh zM53x%(TZACwKy(HH&J3t5*Q$2-QI6fISN&x4WU@=>s`{l-Mu#sShcmCab-1&oGL}e zw5Eq`MMIG}QlAXJ*1wuoD36ogj*r-CkI(gC+B-~G0nt-BvnR&&#zW&4`>Q%w8FGh);`f{wEPXFSPlsksQR zXg(D)?v>v+cOJcKr#}|s_x(;1$yNKPea04WB)KRgY4cE)p{5>iF@khuAw(db!w%um z#M!cIu!C6YPllX)BEv{($|OW%S1k}wVWU_=EaXChKon=tJAPeQ6Fs>dxM43>%SA6Q zH5zK2I4Wkkrd^WOp~!hn6mpXFN-Dt(T$&dyu+kOQ33x;-;w-ajy-K?Rl^sNCugv!` zbNsE7$#MZ&y$CpYJjZeY-@S??d+dRUDIi)WMc^f!crBNEws0XJ+8Bi(IT0z? z4UV`t4{hxN^hu@eZIi1Ti;+A2jM$YYCnxMso&dexp8j?WO-{5tghj)wsZuXuP5j$VkbsQgQwDRaSb`TYr1N~*rE!Vj@Mj#KmP}%4yon)Gk58)K9 zenCwbr=j+P!Eik&e9^ZYcBA-nVyJ7>1-z0U^~YF~g$VtRLi^%}$%T*`9#+>XmgJU7 z{bgSm!$BFZrFU}HCN=Wa zpLyyZLo9|^R!qRcv|+yuu!#v*pYn|Rr!4EWr>#WD-HWxlCLhSFRn33Nf9}6GbylS#K_1{Oh8nc?I&lL zD^(YjzOh}InVX%PufAcYl&e>u5>gF^=?(HI6a5v+^&$yGFYllGasE6rVA#yIZHceA z+BsA3{U+5Y43Mw5g!1~$PaNU(DikVIV~?B^lQwUs%im-3C}s4aD(*d6Jf$8Nq>MeM*{93ZhDIOA^S+Y4HUuI_5r zrZ0NrgrUz%m2UbC*)N^@2a@JfMg4^0=^^bWRl)w{%A!m!E0v{EuDt$!X;ojGW%KBX zHY4-+AR4$fZ|guGtnA3Od51yLi2R#q#?Dx;?>Q&fdj^JhM7yx;Wi9j<7k0nS#n^*7Jy=h_h-N9WneNQ?&4KJ; zN%yubIZnBXmUEwu;idT~zBB&QOI^Kk_h7&dxVLs%+M1qM#U=y)VhI3e{uTh(sbKi= zE54kurfKKU&AvtH>Q*YwzGeRLFy+y%WuDv#M3CiYpOVkGG26Oas@|?6r!q9^y0T$g zGj3z(x#G>5-dOdf#~=PS1p8;cMV`(F9-l`~yGJHg*z`e)Q>ChVn$?XV>xvR;_S5so zi3rx6Wvg~c2qvKcN~JNd&!fw{C!Ejp?BM_`7C))i)WDoNNH?_XaRJmmd``}eo{xI% zyo*oj?>$QMI6r3x@eUpXk5qP=>oZ>Z%j+RS7czuT1KA#4be{}9wvUJ3V^S1wrg0U4 z5w8o~ugj0+S@o-m;@ql8XQtlj?cRMb43D=Ew@8;KnZ6m}_7F%`sVcRleaIeD)5>_i z#%H6k?C}uW!y5MOB?bqWHt*bT5PHSYNc&N{>lR^4U8Pi1ML-hc&(kBx0-LUbPWQw3BG7UgFdDcC;G4!H$ z1Lf}QyVd%1%{V^L1U+2D6>OZv`qIFym!Ej_DR!T%^s{E=+dBjQ;lM4ZCAkkS;>H7m zT0|l__xWh={1nxoFiUTL27Zf@rGsh8GLefvft-3eyH?|HwDYIxkDI$B# zzJ&XJ>|*g0YL5C!hAEkC>m%MfZM~Kjr-y&Z0ong-sfaINFJ31WFB#PbG<`W>Sf|x7 zEL~Jn$>#X01q1H{o8CuJ^_PK7IaiC9bp=-oLaW|G3u8fJD^O0CSURsNbWS`osY+ww zC-Ss(c)+m~4mlIv5i!TBFHF;F5&6cMFfB}nX$7Ccrb^<+zP&$XvP~-~1DJY4jZRlaKVlKn3wK~xGKgxlQsG!SM``RP6KhzESViiD zS$&WI?E};S|2)=fl5z`V8dRr37!8ey`8V4H=2im~XImRyh)Mu^B%k^k59?|jaaYiIBt3K$nAh)0+&xsh(hg4JQkaz@?WO(QpA@7A0e#&XlAAeS&Daj?w~*Bye;X z?rz6@`57S6!>kg5xm1H%N5|4j@xK2o00i!T*0>-m%1ii5rW6z>TMbY2$N&$SsPyA2 z{(3HwvxBpcYjc#pX%Bz|y~>)<7(-6VmSk#BsOf`F7E!T|by7mc=|pOlr|$y>NR_Mh zFadAYl<+3A(AeM*O=-&r>oWT|#LQ6kW+jh0wOiX-&2a zWvZt2Yg1U!#Q=n{sgXm%i+SW~F=YY)-Kem##-Kg3IvM5=THhV000M=mr-cudF*4NNIsY`KmD8-?%nQ}vP{p!=NxPar=`I-b5+st>sE zHH>>5D$2Xa(2Y3G@-##LfOB}`^r`7(U>_v#Qhd~+4saY|&O6~ozlz;X3$4oc>RqEK zHISTIdrkS(If=&VWp2rHsQ;SUa)-t6vF82L{qZ@9ZNOWs3>5Ox4*Xc4GDZS@1DxH! zf-fH{$0H2n1l5LsfEjJUU^*XKBwD}_UJW)qU6Wu*KO$jyTD{xV6W63Z>@Rm zY|d+9gGV|of?2?wWa%Xq-IVJO+<5cZ@BPO~z6cQEpK322&%9~ZE4g|M9syg2&N$FA zn=bos3AZV0cQCKrcSDk! zy>q6*1SK|ni5V708|&kJ`>hsd5_-?VfVu^?mgt`u7|ZhzlAj&V%aos_@Y zk&wS?OJ`97&u0B27jB(pOU`}fjuF7Opzk&I4VEn|WHg4W3QQZFX_zfrdTr5ii^lyp zO6^Z&e8dc%@zT!#@TpZ47XZd(T21+$!Vm5(zvIPYx`ugmH3yq@C;XYwP?wzlg6ga&c<(1#}EbN|YCe zm&1w@OR%we^bV&Xcx$MvoB{BQnKMkVaV;*X#NLJ#01hoD~e#ch#A+Ig4mn5xx;GlBcMIT>>4bq7Z6e7LNOu`!#N{br1pJ(1ynb zS-qg*s86rink>rTd&QvOulpx}yfu(0kxH;A_gvb$L*PI9=AR%CuxnAzeNbJSwBKzW za(Bw0-W<$MR(trWf4k61&~qr-kd1#Q&n3Nqw&)GIfcA)`M>)fSGIzeq`x%<@e$|OU+3A{6LYTr2Es8?0XXNkX*~saPd*EDp0L>RO?BJ{A!F%ulHV;iJdtQ#Wyq(|*F|a_|Afb2f*BKmT*9u%{sp1h|?S_~$ZTkr#;D0?=f( z0ED#^1S`hHBHL^Z4qO`a9?k}TH@y|%Hv8m0FzyD%6iwZ%XpBr7csiSL0=p6@Z>Vb#CBmSF=x( zV+_$@urjd9M9`TckgBr61%v`S_@Z9j?<*WDd*kJ&2b^^Dr)i$S%p~HHTUGVVN$tepakn;W?vli5L&BANpk9ioZB)jZ2bSO?~_N$EFl2g)gpS} zEVMG2%OmVkVJ7ZdsVa!M>zP1fX-sH=)TMoIH}ZwQ-pxi>tj5?eqiR1@tv z>_@WX$`fSICPyF@w0VO(EI~UURjIcp0jt0YHlqqGqNzmDC z(i?;9sUbREJqPr4X$|^XBpg%g{_6AJzHW0HM>(*nqfG@-Ew{0Ss~iDVkrhOb47kFx zvM>1N*@V#d?rnaL2M@BPhHPI_-vLps&HDhA$QA;X2BK=AWDQqo;Ihg6YfA_g?!^I~ zt+z{g>vKG8hz!hK9obn%E@;m6$>ik6BB{i@feSr&rRdFqfn;?Jc|Agr;bk8GUVhv? z*2o*!{SSga{ME2WOA?Z5MKywjp_M1R5gXb})mS0J7wE~2BSS=mUie@aUBbHT1J^@g z$H2bZ_`Y!n?(PKkAF(H|BikPYXgj`20e$j1qSRh+wc?@*0kq*f3O@9=frp}G9?{;& zz@1@YYrIsd_S+J)K@%Q`>}JS1!{}(8J1jMlHkVrkm-`(Cb~$Wh(?%eQI7i;>((|eL zFreKy!8FAmxWP0{a_kVQRI*=7G;6wqrBX0a&0H?jcb6l4@gw`xHQZ;DH||`t668n? z#kppuT~w~(P0I9yvom=9-d!P?#$!=B@T-dvME`jjWW{q|M;js^qh$z_zrnNEG6XKIM9jFw%T zvpD(o7kFYw7CfK73 zvtmA!%-*=>`5S$+%_QGUp zhSGnr&nNM}Ly@rG`!j%7#{aT|tNaZ+d?Qa*15R{%X9Q)Z-mvGa^Nm_6g^W4|~b zbvY5~knxe-p4x#!WAe~|j-w^-a)Z(rV-8Y9@))oHX}wCRYZoE!bXJ*Pe`TiOszma~ zZqH)$f{gRROz7O~jMni#*E;A{H0~GNr~bw^F;S3WUL-n#k-IubrxgIuW2Jj5y-P+f z?edif$ff4SU@N+~a1(*Q&Z5Gey+fGgYwq>N%NPA(qDG7Hk|vPGfw_(!56Ar)Myu@? zMC!voXN+kEivi`KofLebQSw{(e2`+Y8ah9&<9gn6_}s$a#sZT*I~GYZh<#;QkNoh; z_#l=jQzxJUVRvO>bl{^kES{j$`ewTDS$sep3P{PKtA*tfWLd>_leHk?frG;W?qGN9 zIgGqOCgBFzKU;>!;sceS16%!wlJT0@v-@_Kl_7*k%;=wvI>PDI6Yo=ytl(gqCDfoM z)*rk`(HF{*ROAVQ?m_BUejj4=7kFi$`2oKKjKC90(5!*v$|lDzIV??f?R*|zeb-}j z0onGU?0y-PAnaXg0f6xf7Wdi9RrD?+1Jpr>?ts`~J|2}2z#x4O*(9=tF>-AX+`kX% zXnmtUIH95Wbx-9bAa`Ju9Xd}SQ)oQ9S~dg3wC~& z7)l5YW#nu?Z~}HQ`czPInXQ03906s{<7DW3$&OcOYU-_lNHSc~Gj{-o$^jdS78GQb z7EK3xa82KR?Dqh+_tl|-ALb|#|{UK*Wg;ZP@kKPDe>uZjFh^%6C0C!fahh}44c14|-1 ziKBoO$BQ{-I~42!(I?blSE>cLb@Y? z1JwfNacIixUB-Hx7rY4v8!r(^d1a$p8*5Z@*Z+&N0vp2CUxn#B+pHZZiZ|XDw4uj| zz(40B^fry);BJP=MbkjJzSdOn`~ z7d{_)K9Wp)LVapUc9f!Kf;2l(lq1E^)Q!sypE$ zKf|a#(%9sJ6SjXpu0Js-YIUDqk`;SL9 zR-RNA0!uy5wpB4Qz9&T`luxf$`}0a!2_!p|Gq#acKGr0t2-48Q5F`aZi4UT6Abv0S z9bRly?IxM(&Pse3qDjbQXIQ*ihj(qkoKJxo8-|Rob2KzhP7_FViKU3AGS(En#sw~T z`{!LeQQ25Rri9jnCI*2}9NqN+4MGui;1Bo?T%&gz0ZH(lC|w2yjR6xTEH2c~p#VvU z9lNn8;(XeP)LM6#L6?#6ipn3Pe1fjVlvKuTBEZI6ONVV-KIpyXC<|2wL~?Ve z!y!P94;-q`Y*-4!*blEF5~_ylmD_S$XC37N@Q*eWS8PW@Z9-Elz^X@;Qap4M1OjYc zQuL^r$WhcwUXH(IC?eSPiNG{7nj%#(0ujSnVO$|OsrT_%l~VX`M3d5kX6F#w!J>V&Yn+VCM0;SDq`Xkm;+rSf*wrn8Wm@_q=-czlOeHYV2lD`jX8bf+|xNt5}l{w z>B-Q`L0y%50u*JaTLv^|akVq%QQP1_9dCE-XNOpT!0piN!mnndLoO6#=eGJQsQ9Gn zcwCHYnO|t3;coagx>>3fs)E3`bi{U9X<<6U?o|)xx)N*343EanY8~GWF!ikf=CCZG z_Y{qOiD}NgNugk57Y|vT6UR_0@R&~IRUPkQ#REB60#xAieKQB+#s72RU9xFhFzL!3 z0^kG?EoPd2DB(Si?Wp7hiu<8`uqhHlUKugNZ*3w(=07y&6F+;d?9I5h0(<1uP*;{L zH)>|?>|7;^O_%_x8mY&em7>PfTDhgSopDIx5%?aLRf-4UaH!YKTbcxB}Y(k zIMrl|N1GR9plSP-_Y6|%?$7V#=pw+M{Q&@KwMm5R!oU4Ko+7H0tjKu}Kx6rwbho{^ zkWXg_5w^S#y+`0x`#~5q_OV}nV`Pe~&<0^RgZjb1k3H29V^1P;8Y98}iBy8Yk~FU7 zi+EOqBJbuci9^Vp`6;3Xr44n=>_C`kfSMK?@}*i3c4$kiDjf zymn`|QZ1l0-BS;(U89ymACh9kggm-#nk0Z+VCTY~QI8-Gzl-R#q?p~budXumM9kJ& z)_ciXm&jom#SIvwrl#x=lHjgZ1{7Tvr5iu_oB4ud10u+ZLuwG{X2#&3W?v;oL)zZq zJ~M`wfYa6~%1&So1P*AsQV=4~GBQ{qM0NykiZ!{fxF)WKtbT+5~qzk zA5Xo|yFg|7Ip%d&*+J50YuetH$jr9zjNKG2D?@iPUc3YM*4+1H#;Tr1*_em*KubV@ z%=o~%?1X~LCP2Ed>)A=~IePhZ`kxW$e&*HfKZt037Caz<^aWx0TO{lufw1;|e`m!M z{R2QMcrK6N$sCUY@DV`497`FT0-H_wByN*4ziar?5mdM>TFP=#khH1JxqtxVPoHzV zHWoJ~FKEP!Ds@@zWtp+^WjsV3ZN{Bb3S5XD!IJ2AX>P5*}yx=3y^74R;K4h`Eh5h{WoqV0aNQe$d@)H4#jq^7G zvn!F%6S0$)q=^{Mut_VDV2X?iHm+ZHm!a%d@~XCsHC?n<6%yq&Y*#?tC`Gw2r$AL# z-Xsx?9RpTYt=h$^cS=&f)uLwW$%R!VI{-TYr{lRowLE&#FER0M%n4L9fA5~~2_S{# z_v5;ZC#@Tc!lR{@`KAZHl}`DqQ_UHDX%Sk+NYdYPSX8>Ex^6(yALVIb5XBy)4B#TV ze`2m^%MAr^%RqO&v(GVLo3yXTNxpn2R!d{YbF-Z86#T8%1;DY}(S`s!s~65Z9Ct8C zl{|sgXrbIejiit5YzS87G86JhC4BvcPTvL)3Sg5|J;8pisI-V)#|sV$XJGp*-dVwt zwg@R|8L!E3a{I;zuZ#)SS=l$TxRVkixbp2_PQQ7(cNZkA&8Rj`ZCFNJH!OUu0hxWo za>f4e#YXZO05QQkoHX(<>%OcK&jfLlvd!Di`LjJ?n`PxYyaD!6pPv0|zASx82{P2X z6O?<^T2d9uLc$rCB8G5NSUyuEtTw7jS5_TDT!yxkk^^n|y4g~>$vjQjuP5%Img#ZH zI=(12+D%{uT%w-mUUt8J*M@lJrx$|75$tVjADoJGF+l#rx{2WYCiA>DUV}eO!zJxy zE<6`v!YfkyXV-oeitHv$z*YYfL*yPvZMoyzmMoTBEtN4QHon>dS5;1)s3s6+8@5LeIoc8>riQVs;Z60_t@D6X(G{PaK#-OoxvM(n$AbjorA+Y z3SkYoD<=_CH{Hodoj%qH*Ez^jaSLROOxsQzt1xdX9cvXycK$gh)8pGjpbD=IjNO@3%;lHoABVpuBaL5vF5&TJgp+Yr81CrL15qmn|Y1gJp$UBpA z<1OQ``iLtrA4qJsNVHlb9=q+?iB;SO|0h&N*^slLbkM{jzr?<8^n!6 z(jlxOm+J!S*JdQ?bGwwrdO+E+^}DX3VE-Sx+MMtbmqFD6-UISK7OI(jS5s)9#LqxX zKX_1SC@)us=$oypwx27C%C85DA4#06lYu@B&InMf%yoq5=I^&GiCPyaIO}Ta%($H| zb%(2u>WNX!CK}RLf!hxGwA568DrDCpiWK<(G`qs^?I2HDy9R(;^9-=7OYW1n^LpMC z5F2e1T6=N~G|l?=EqS#?O>FCSUjdFcP;K>umfoPin3YTXOi96}J1nyR!vdBI^P9~5 zawk%D%nsA;lWIDwYj9u`d`&Qa&Ews7Y<*f#0z?e0EaE-Ifgu{3hL%0H8-$+<8(;*d z27(d1GXicnfWYV%BUA@h@SC0j@bF``aokrdP4-wmH2_fGbWATiHh4avz0o+%sAJ`9 zzk{2I#Pt@U)ol!#1q9BTx!Zg!`%{6QIM(qJr8lcer%F?}0qENROKBj`e#C1t4UR*>U7H)I1$(^#)RD zIPLOSrehGRZ&@`(X){gOY5}B1#S9TMKwK5jxrdSaF9Opdk4GIDn6`8PvCl(=ob@pq zvCW+9T#nc^)jpBs?P>>gnrh__B?T8F%B|fYi0h1t%DrSGT^LZs=#@NJR)|`I#nXgn zFIL-8z@Ton!-k?zp%Y1OGLvsU?5GgcAX%}9>DL(E1JwcUhy{Kj!hPASn1gefJ`Os^__dxcFOvaV|)Z${owfT&3KsBwpX z`q!R7nk_1zo2$I=P@17Esc&s@Hth6BQkA-~O6qp6h6#>NwoG@<%SJg{0XP>7H%-^j zlsgIWF6E{}z_v=xmf?6U7Xj6=S=_I`FmBnnO;yr{%)~B@=jdIW@?43hADc#kG07@g z%2TPl-ex|j-eVaxn|}jiC_iQ)O!)c$_r1Y1?_8KNf2BQcr-BFcJaV*xb@`9a@E+^o zGc1e?LJg~x=?tW)g$d8MKJp&6s$xGqMr@T|0n-r3atnr7Iv*npjPtQuhbTpoFC0{e^bC za8~HE`5)k2S97YLcj`@_h7lUUk~T`R<~!x$1c;VKQ%=QKwa{P=qS(*b-Ydcj1pyLqQ4(X(&8Pd24(F}J`?F~0V4z+FHagJD{H5Q zq~Fbu+^1<|+UqWyh@kTiLE*p)PZGPm!7sJZK09~fe%IyEB6SZX9VnU$e-z8y9#zf; zzfJ)y%1LZi_45brsR!)IS80vvTcm!c#f4>ZyyA;H#3m*@ZG0Y+kbk@#OQJf1em$6b z`DPpf+-ivojA(*>L5S!2X*+qq5gG`BSdD|IQ=UO<9bdWDe&}i79A)Z~Wk$!)ZGmh_ z5C<{$Z9mX#X?77{zsxFjJ=cxezP}|>L#$tpELTrTUF5(SEm$~n1Ty!KtiC`jK8)6u z6q3~(5fjtx!!;7wep)k-xl6%KgsQ1(a-X@*DazKktlqAOez)L9Qte&3W-m<{g3}t6 z-CMDwNdVIjFGBRdtSLJ=6817zVd(%Ijz}UkJE%iyr>B=~Zp^TRye9@X;!Z1stsyp_wOW&~ znrkVc(m%qSonC-6w$i`_rwI4VEQ^#d9ngRU)owyd3vo^@8%q>}N_34Pu2D^I&V-f1 zWJ8FC1BO*-4YPoSoX}&>j3y}}L3M_;+NNNjTXR$qC>wLbXuZ!_}y6wjI)7x2WdZo z+Z1c`qLKMP8y2iB-3hKmW;gkoW=lySUMyL1hBe$IO=s2FT!ZVmcK(d3#B^AHoo;Hh z@=m5w62SK?K|iHnpAzh>crl4w~R4FWGY!gra%Z|=47a=0j)7!h`eD4TK-c6lg{-My?^ zZ$P$IUioOX%8^gd@Sa33ij*&Nluc{iWB!=YiK@XJ98aU(ng)mOcX|!_OqfyQ%E#3| zhljhOUPFyJ;jyH+sF1vM9ZrprDdeXj)pJ`m9}Kk=Rh|M#n1*0OsgpQ=usoj2cuR!) z-jj+PC>ptP^!MVzoBuJ8k}m3?ML)j03~8m!QKo@HKOq1QAB(cMvp{qS>8p(IE_O!S z*FJv@KOYVG$rxoBqP}RymtwjTzbk^rf?M(ja`kYtA8>nV#j%K5;*0A>vEb)HfZ|Il z1*Ec#>p1{tLT;)~#qDHW9Ghw7m{<7;L4>YCDc-Zyrp+CkN@_hT{#|wcw22GZxuLTF@wzvD{N<4*#7IU3b6bk`4?6MD}HNe`TBgOU&5b$@DLyOG(tV zUtmHD{fs<({Utd8BT_*Lk;xVTBf=Wk=6tcR)UV$lzx1%ZUo5*6vFUJ{eay%28JD$g zZas=RSlp#OXQieDBIi}I=_g9K{<;TB4h>N5_kg7J3KFsRBo(Ij4Y`_w3@;~6ICrc_ zH55e*FEFDU1Jy_{%W!CKyPJcV9BDHb9Q|&DXrq@ZZ3YZLq0(Ta>o+OJEJ;jaahZby zA;-q5+hMA=Qme;H9T?KIS zRYIY%Pp(!h0{+-x@>(3rrhLH|0PawsyBsZb*2YKIbcbjj8C7O!+Qta4s8e@GT(ef~F(gH) zxl>-TRB)&KIDw(0D-V6w{(S=_-{CLz6DRz1MW@Ay02gh{a_RVdWPE1mnDOG<;hj@A z-MFdZX4BzFN9CCnx3_-x>t@{3_zg*O8u7=vm$NB8xlj7r!^Dw4@ISTvkECH@e51(O zuf+DvFSm*EPtR%RYHMWbWNByqmmU|@r<}5wko?fao{w3Q7J4!|5Q&0q z*ya#Pp~Jn;y27^BtjogqHS&s4IIwtat@cazvqK(_*W(s1VWt_cuv}s2(2@B-wklxp zCXEx2_~UwC$EL=RxTHvPiD$oj7sH=`|{JLB%%$eO+b8?ovX@ZBC%*qjdY0h zh*SsaUoA)mHmJ;nSc<3Ariq1bIcDVGyh7PBP^9kr4LXBy#6*bF_;dub*+QzN+sK-< ztPtC!H%ca&!<|FhYe02bRzrpuz;y_cGOTPeaOq?Of{#j4?my}4grqu=MK5j3Fg*pPFQQ{ z+Z-A#r%%0F_)f2jzYI;bN~i{UWAWHjwF)(QrZXDraVgDQfO*}7BBq)W)!MS=1)=OPw(JvbdRg40Z69 zwv`_2(XIEa12>`U{g!gnOYnPO+w7Hj`rkQkF&*hkJX%<0wxl|da zJlTuAL6_J1Q<$skTE@KNgg+L#54wm>dbj-19`(B~Cbpg+@K>Hp)+3BxpwB zFM7NE$x=}7J6tZlzL#COovt++8~QJmSci%C`yrh#H9PE!z5RAQ{XX^kcK_*}cV=B8 zFC*yUKBYj1o-@eW6t%?2s5?(Ak;v+s@#306@x27{FO(9}8l2Ziz#j$HyLT<85)M-b z^yePEuB(X@TQVx0$QJVpNSL_?PZ3fm|FE~Gic~vW6OPMxJDT7W#;6irsZihB zQ~t$dtEGTu>%{+vliT`x48J-xigFFRB7)^vN5wWmbbu)F3ONd&Dt*3KrhdyMA~?b8 zfS|FSv6M2q`GWpd9wM7%aoBPvCM#&I+q@$rr(;^%OTvwiYCMC|kDO-JLB4R8zFzoA z(~bt?`sdWSQ&tbS zduhTZrsQrOIzFqul*cV|8sUGsjCuFBR@<*-@cwGJU+wR@nt!Z`g{h&5sT1?RN?~N9 z1rb1q;Y4eNI@}`}qL~LK?GOy^f#B7M5C95BeBE@qA3vWye)`?n@o%u9J33_3VPWX! zN(Kdw`10x)=pNx-{R%5!R%Itec17~8RTp_kU_Di#nINj2XL(@O?v?tG4ZJbAz+V4^ z+_?q)zn-O;y}ir-PiFan4wn&k zfG!d$gSUJ+^L}S>x@E7Xi3C`y*9Hg6AhG6T5d((;j-VQ0I*PV{zBL#YaUw&66r&n1 zMwlDC>$*#tPgDS!@Es%>+DYZwDwPxkuoDy?t*of~is1cj6^WtKH5|~>oSf`5^^Sre zUA8L}RUi>-&fg%X0!vXoEa)7h*U2T5O1l(cldG%(c^+@kM=PxocsL%|Prr}RhbBX zW|Xk%vY3&~{CQh44ZQ;!1A2#GhYVVJuZNVH&=s!4xg$q4&S=ix44FIcU_jq$)ZR^- zZ=nd66@H8!grNlV4-!Poqm|25bgq)`rJT;Lc60fEa&0$9T7VRt2H(*~#Ju-M83!4g z_iP`UX}1TDWt0+OuI7oqE}Pu_S6Fl{?GSzQ#nJlz9bNy1{=?LDx&6P;MSko18w@zA zu(VtoVJ%9RSdpNYOU@`9G1C!srA+^LrN2_OKnvQ^-e@HEH@=@6S2^-0Qsl0RBm#u&QRxPiRfCL^ zpfxD+Nsd1^E=kS?zP zKtji?JeO{9etyPH!uC_~()#J(5=V=R%p*TA`)27C8h(@*6*xAP!AEAz_aZ%=zrLKd z)~)w$4_LV{SJT~G;NYXTm$ETy4g1>CL!l|Wz+GuD1PxER4(oqUsFq0sw-$J}d9d=8 ziX(YSt9nSq!LJ^is7c0bU52-3TNf5J#Ac%8xkex@SPX55Y~1fzJJiS4Tm%bSvH z`*ar%Y|%$t`$(Sr1o}@GsjQ%mTJr@Dt}mL8`sIAIh3)NJzA72@RXiO`opt~6;>~J( zcI!-V@8|=|#A+P_3UHEMD@*%p!cV}FBD`yKit#5+@2^U3M!mY(X4bB(FVD}pA~4ry zM`JBR1VU^lONXk>bg&LE&5lzYZx0TgZ4CAtRbl3c^*c(K%ZIU}{^%r?_!`)>_d8e> z%o>t-p<*5tAEOY1$A`C33MWu_^i&U2oK;r}6E!7!>+$kGmP_g@wmM7c{P-Nb>XM8c zQ*YJ>lIJNuGVU-;t=~fu_U@vGf1!#+&am!%*89Lgfl6Tq1Bn|?MuI{V2=@3}U zL7IGJH?RpmQ4HAhimx-%_dZ=hU2V}URB}wE0P7u5I`Dr|vt&dRP#>1vMsRVWL!VCg zjAr@sgRnf?ZM9>IY$FLE_*{&KvNj@^YJs{btzyTbJb0QeB>xbB?zN_iGBUma5(8Xd$!VIKL#jT97^_BVSV>H(?|@~Y*aEK$ zK)b3AGLo&My6d#h8!wPJyk>G@AsX~FJTbUOscduJm?%i*_!2= zrVi96fW$8tbmM96DR_otBUf?UJUhwQe1O2GVmKzKTuw6ve$kual45qs)QHe_O)#*@ zyucz)BKIi|N()Dzw!;S7=E7T<;ca~ma+(VMLX!gLZBoVw35QnfND&u4L{{!TCNAj6 z<yqOTqY@PE_ZT$ z@Qm>hQ=@)tq~c{_(8k*A2Er=$`vn}=W2FLe3te9&<-5IXY+iQp#K2Y|05g>vqZfH2 z(Nqv))xaWbrtvqVJ@yEcMQ0Wl<~jd`)^G7KQQojDDnT0D}R@Yjp_SBY2p`3|Juv>d!D9tCjSEIRPwsr zKGEML?fiYQ9gE|U3sD4WT~(Qzkoj06<-J6`k4G2oK2Xyws>vLjUCd<}_8%F{)%G7& zDNX&J(4|DHEaVYPcv6;b;4gQthBrwYHcPq;GL}S8dHG%c*T~#$oot#+f-c)$=!FRm zL+6?TVc3kdI?}`K6zDJ&?3N%OAXq(p00srO%pRj0B!M~8y;YoL!{giFb`($J-HE}V zfJ3f8g<%i4X>M}H(J$FG^bKe#OSEVvsRv(OH2v1*Ts>e2kxSZxm(zrkl1E{7ij9AY zJhzVr{W-!%H7AqFeVr!dtg>-Fatu_}3e?$I&E12?=2kj3VG(!4-VptaW^HxrX!XIu zyq>A~GcfZwju$kxi%KPgUb^l~@>4sh2$S;kwyxD;2Vfct?f}+mq8TsBv9h{*ckriA zdzcvrr;Y~d=Z2+ywMpLX84W{j7C(tK2vUdU!U?}$=?llXDq)vtCDBt+y162B*i}*b z8-+HqeAxw5RN+1_KH9>`y<#PvOnfOeLHfmd)52BWL)agA6@%Q@sqg=1T>jT-2LG=0 z|JSgT%^M#t_=2VL*Xa%af~B*mi_6!})Zf0dBzf~Mb^_2Jn)M~=)QIXU*LI;@dN5iM zZG|)-U&SC%3?f^fSdWX{Z~HEgz^0xkc*84RJoJ0rf61f^Mv=E#qkdYmRDr9azG7wj z`*$^cO$^k0uzdwpn<|pdu6Ba)a-~9HW>TPGr|Uu^TdD2?zDPCAaO)~jhq*;d`s~A+ znq~zb2f^E@pQG-)gxLzrC?~uIhzS>tkKd*|0ch2_b@3;xY#Tx=E`Fjeh?3u}ixyH> zUKXPr(SL)adJmo%SjbRQ3cs38LyersQ9sGKs2Q$N-{!-4I$&vEq*?oLLOuaQXkqLyKOQEVOdY%#8zm6VB_KPFat-%ePpw9P$3$J1N!=B;4INqr zM708KXD?@m?eO%>-g`EQZ{TU{ck%}|+uu%94c$%hHSe5)Ybo+`oPhA*I*HIvl%K6W zYd1RXN#J({UT`91g<1tVr45h7pP_FRywM-ZgC~}1awLCCk6bh;H0oqsNY-$ZUjWLI z$4HBAfl4jbhfmg*t7QHKK$YT`x)n`gyt_U6FW5zeat{uD>2vri9Ql_%jqPp!Iz8hb z`uw%&v`2&mv7_zCrJksv%(>7+nR9XPu@0Lb(9iY;mp|93uS71HmLnG=enz&Pl(yVE zupr46MarH^s~PN*5(u86fz0s4?&1I1wGjkxl|cDLC28$Yioy4dzX@GUNa6InA9P;r zSq1bcyN0MKT`}-n%E2aCmACcwNvL_TuqckSD>KL3`6&)ToDcJZrZ4rqQ{|F`?uq(A9~=hglw%U<;N>HP3_ref!Z{D&VN zEXi_v`b?OA0)vQPK$6wR1N_InWjWZqxp3~G@OHO?Q}-UGJXz8z$vDm8P@1Oe3(>O8 zCluRsq`OPvR`{Nudvt~J-u=1Cdn8IPG#|^q9M_(_cIs!b8B_mnTlY3M@t?sG&4zyT z@nz&H5Ur>|j`@KuYLef^U6TbSO=sY222wjMuOuWfDF-~AG56x#yh92i>>t7v?`UtG z*_*lMo{o3fep?2ntuB1aA3nIUU4HGh>_@$KMFl?ne_E6CJ-pz}ah6RlwD(1&&5>Lm z5M96K$M@^UlMYL|zT8^!!uM!?+mE{Kd#9NG^3YjoJ9UB9y(2fKIZepo@rX*wuKBgb zu%YLrWFJ$C#oXJiP8q3N*k9-D>9jcw+yHz?o9o)v&}Xt&C42g7<5(U~>X;{W`_h8g zxQlucCDH+si>?^%*v)?F*Tl%ZwvGv*4u`Dtf{pvTsmv2f5ZWM~N1rW(X=GrCL)HeHJ4DxxK1+qr zuBs1pGRmA4LO-Uffo)tU0J&P#2sWON&;xNY`lvrl7XyPQTn{(}qiaMi@<8L`2oPb6 zl0uNic}X8b1dwt>aS5^^Skn`F!yIAKL6}L% z&2)5!ptl1N+9x_g9fI68K-Z4jAFZw^{y;Vat3S|deS|}F+@L0<0f$84Z2){OL5q@_o literal 0 HcmV?d00001 diff --git a/modal-deploy/orpheus-3B/conftest.py b/modal-deploy/orpheus-3B/conftest.py new file mode 100644 index 00000000..4b2a798b --- /dev/null +++ b/modal-deploy/orpheus-3B/conftest.py @@ -0,0 +1,8 @@ +"""Top-level pytest conftest: makes the orpheus-3B/api package importable +when pytest is run from this directory.""" +import sys +from pathlib import Path + +_HERE = Path(__file__).resolve().parent +if str(_HERE) not in sys.path: + sys.path.insert(0, str(_HERE)) diff --git a/modal-deploy/orpheus-3B/luganda_article.txt b/modal-deploy/orpheus-3B/luganda_article.txt new file mode 100644 index 00000000..9758758e --- /dev/null +++ b/modal-deploy/orpheus-3B/luganda_article.txt @@ -0,0 +1,14 @@ +Obulwadde obuluma Sheikh Nuhu Muzaata Batte bwazze nga musujja okukkakkana ng'ali ku kitanda mu IHK Kampala. Assiddwa mu kifo ekyenjawulo yekka era okutuusa eggulo yabadde tekkirizibwa bantu ng'abasawo bakyalinda ebiva mu kumukebera corona n'endwadde endala. "Yaleeteddwa mu ddwaaliro ku Ssande ng'ali bubi. + +Kati ali bulungiko okusinga we yajjidde. Twamukebedde corona n'endwadde endala kye tuvudde tumuteeka yekka nga tulinda ebinaava mu kukebera", omu ku basawo abamujjanjaba bwe yategeezezza. Mutabani we amujjanjaba Sulaiman Sowed era eyamututte mu ddwaaliro yagambye nti kitaawe abadde mugonvugonvu okumala wiiki nga bamujjanjabira waka nga balowooza musujja gwa nsiri. "Ku Ssande obulwadde bwamunyiikiridde kwe kumuleeta mu ddwaaliro. + +Tulindiridde ebinaava mu kumukebera", bwe yagambye. Kyokka omusawo yagambye nti okutuusa nga bafunye ebivudde mu kukebera tebannakakasa bulwadde. Okwogera bino amawulire gaabadde gasaasaanye nga Muzaata bwe yatwaliddwa mu ddwaaliro ng'alumizibwa nnyo mu lubuto nga kyandiba nga yalidde obutwa. + +Ate eky'okumwawula n'assibwa yekka nakyo kyataputiddwa ng'alina corona. Bino abasawo baabigaanyi ne bategeeza nti bya kuteebereza. Muzaata abeera Kawempe-Keti Falaawo. Mu ddwaaliro lya IHK e Namuwongo gy'ali wassiddwaawo abaserikale okuziyiza abaagala okulaba Muzaata. Sheikh Muhamood Kibaate amyuka Supreme Mufti e Kibuli yagambye nti bamanyi ebya Muzaata okulwala. + +Yatwaliddwa mu ddwaaliro ku ku Ssande ekiro. Yasambazze ebigambibwa nti yalidde obutwa n'agamba nti balinze lipoota y'abasawo entongole. Kyokka n'agattako nti yakebeddwa corona kyokka bakyalinze ebinaava mu kumukebera. N'agumya abantu: Nnasembye okwogera naye ku Lwokubiri ku makya ng'embeera egenda etereera. Muzaata ye mwogezi w'e Kibuli. Mukyala we Klutum Nabunya Muzaata yagambye nti ekikulu kusabira Muzaata, Allah amussuuse n'okulinda ekituufu ekiva mu basawo. + +Sheikh Muzaatay'ani? +Azaalibwa omugenzi Sheikh. Adam Muzaata eyali omusomi w'edduwa mu Lufula zooni e Bwaise okumpi n'e Kimombaasa. Yasooka kusomera waka oluvannyuma kitaawe n'amutwala ku Bilal Islamic School e Bwaise eno yasomerayo katono n'amutwala ku Bilal Quran Islamic School e Wandegeya. Y'akulira Dawa (okusaasaanya eddiini) e Kibuli. + +Omu ku bayigiriza Obusiraamu abasinga ettutumu. Okubuulira kwe atera okukugeraageranya n'embeera eriwo mu ggwanga n'ajuliza kulaani. Ate ku mikolo gy'ebyobuwangwa, okwanjula oba embaga akozesa ebigambo ebisanyusa naddala abakyala okugeza "mukazi wo omuwanga ku ssente; ojjukiranga okumutwalako out" n'ebirala. \ No newline at end of file diff --git a/modal-deploy/orpheus-3B/modal_deploy.py b/modal-deploy/orpheus-3B/modal_deploy.py new file mode 100644 index 00000000..f3369be8 --- /dev/null +++ b/modal-deploy/orpheus-3B/modal_deploy.py @@ -0,0 +1,668 @@ +"""Deploy Orpheus-3B multilingual TTS to Modal with vLLM. + +Mirrors the inference pipeline from `Orpheus_3B_Sunbird_Luganda_vLLM_Inference.ipynb`: +same special-token layout, same SNAC decode, same `hf_overrides` fix for the +Unsloth pad-token off-by-one. Wraps vLLM behind a FastAPI ASGI app on Modal so +clients get `audio/wav` back instead of token IDs. + +Multi-language + multi-speaker: every request carries its own `speaker_id`, +so a single deployed replica serves any speaker the finetuned checkpoint +knows (e.g. `salt_lug_0001`, `salt_swa_0001`, ...). vLLM's continuous +batching lets `/tts/batch` mix speakers in a single GPU pass. + +Deploy: + modal deploy orpheus-3B/modal_deploy.py + +Tear down: + modal app stop orpheus-3b-tts + +See `MODAL_DEPLOYMENT.md` for full instructions. +""" + +# Note: we deliberately do NOT use `from __future__ import annotations`. +# It would turn every type hint into a string. FastAPI then calls +# `typing.get_type_hints()` to resolve those strings against the +# function's `__globals__` — and the Pydantic request models (TTSReq / +# TTSBatchReq) are defined inside `OrpheusTTS.web()`, so they are NOT in +# `__globals__`. The resolution silently fails and FastAPI falls back to +# treating `req` as a query parameter, returning 422 +# (`loc: ["query", "req"]`) on every POST. + +import logging +import os +import re + +import modal + +# --------------------------------------------------------------------------- +# Configuration +# --------------------------------------------------------------------------- + +APP_NAME = "orpheus-3b-tts" + +# Default HF repo. Override at deploy time with: +# ORPHEUS_MODEL_ID=patrickcmd/orpheus-3b-tts-multilingual modal deploy ... +MODEL_ID = os.environ.get( + "ORPHEUS_MODEL_ID", "sunbird/orpheus-3b-tts-multilingual" +) +SNAC_MODEL_ID = "hubertsiuzdak/snac_24khz" + +# Orpheus-3B vocab = Llama-3 base (128256) + 11 specials + 7*4096 audio +# codebooks = 156939. Unsloth's `push_to_hub_merged` shim can bump +# `config.vocab_size` to 156940 while leaving the embedding at +# [156939, 3072]. transformers tolerates it; vLLM asserts equality and +# crashes. Override on load to align config with the real weight shape. +VOCAB_SIZE = 156939 +MAX_MODEL_LEN = int(os.environ.get("ORPHEUS_MAX_MODEL_LEN", "4096")) + +# GPU choice. L40S (48 GB) is a good cost/perf default for a 3B bf16 model +# with continuous batching. A10G (24 GB) works but is slower; H100 is fastest. +GPU = os.environ.get("ORPHEUS_GPU", "L40S") + +# Per-replica concurrency. vLLM batches in-flight requests on one GPU, so +# raising this lifts batched throughput. Watch GPU memory if you raise it. +MAX_CONCURRENT_INPUTS = int(os.environ.get("ORPHEUS_MAX_INPUTS", "16")) + +MINUTES = 60 # seconds + +# --------------------------------------------------------------------------- +# Token constants (must match the training notebook's #2.6 / inference §2) +# --------------------------------------------------------------------------- + +END_OF_TEXT = 128009 +START_OF_SPEECH = 128257 +END_OF_SPEECH = 128258 +START_OF_HUMAN = 128259 +END_OF_HUMAN = 128260 +AUDIO_TOKEN_LO = 128266 +AUDIO_TOKEN_HI = 128266 + 7 * 4096 # exclusive, = 156938 + + +# --------------------------------------------------------------------------- +# Long-text chunking constants +# +# `_chunk_text` packs sentences into chunks ≤ MAX_CHARS_PER_CHUNK as a soft +# target. A chunk may exceed it only when (a) the orphan-tail merge keeps the +# join below CHUNK_HARD_CEILING or (b) a single indivisible word exceeds the +# hard ceiling (logged as a warning). +# --------------------------------------------------------------------------- + +MAX_CHARS_PER_CHUNK = 220 +CHUNK_HARD_CEILING = 280 +MAX_CHUNKS_PER_REQUEST = 48 + +_SENTENCE_RE = re.compile(r"(?<=[.!?…።])\s+") +_CLAUSE_RE = re.compile(r"(?<=[,;:])\s+") + +_chunker_log = logging.getLogger("orpheus.chunker") + + +def _chunk_text( + text: str, + max_chars: int = MAX_CHARS_PER_CHUNK, + hard_ceiling: int = CHUNK_HARD_CEILING, +) -> list[str]: + """Sentence-aware chunker for long-text TTS. + + Returns a list of chunks. Each chunk fits within `max_chars` as a soft + target. A chunk may exceed `max_chars` only when an oversized single word + cannot be split further; such chunks are logged as warnings if they also + exceed `hard_ceiling`. Empty input returns `[""]`. + """ + normalized = " ".join(text.split()) + if not normalized: + return [""] + sentences = [s for s in _SENTENCE_RE.split(normalized) if s.strip()] + atoms: list[str] = [] + for s in sentences: + if len(s) <= max_chars: + atoms.append(s) + else: + atoms.extend(_expand(s, max_chars, hard_ceiling)) + chunks = _pack(atoms, max_chars) + # Orphan-tail merge. + if len(chunks) >= 2 and len(chunks[-1]) < 30: + merged = chunks[-2] + " " + chunks[-1] + if len(merged) <= hard_ceiling: + chunks = chunks[:-2] + [merged] + return chunks + + +def _expand(sentence: str, max_chars: int, hard_ceiling: int) -> list[str]: + """Decompose an oversized sentence into clause- or word-level atoms. + + Clause-level split is taken only when *every* clause fits `max_chars`. + If any clause is still oversized, the entire sentence falls through to + word-level — clause structure is discarded rather than mixed. + """ + clauses = [c for c in _CLAUSE_RE.split(sentence) if c.strip()] + if len(clauses) > 1 and all(len(c) <= max_chars for c in clauses): + return clauses + words = sentence.split() + for w in words: + if len(w) > hard_ceiling: + _chunker_log.warning( + "chunker: word length %d exceeds hard_ceiling=%d (%r...)", + len(w), + hard_ceiling, + w[:50], + ) + return words + + +def _pack(atoms: list[str], max_chars: int) -> list[str]: + """Greedy packer. + + An atom that fits joins the current chunk; otherwise the current chunk + is closed and the atom starts a new one. An atom larger than max_chars + occupies its own chunk untouched — never split here. (The caller is + expected to have decomposed sentences into clause/word atoms first.) + """ + chunks: list[str] = [] + current = "" + for atom in atoms: + if not atom: + continue + if not current: + current = atom + elif len(current) + 1 + len(atom) <= max_chars: + current = current + " " + atom + else: + chunks.append(current) + current = atom + if current: + chunks.append(current) + return chunks + + +def _concat_wavs(wavs, pad_ms: int = 120): + """Concatenate per-chunk 24 kHz float32 waveforms with a silence pad. + + `wavs` is a list of `np.ndarray` (float32 mono). Returns one np.ndarray + of the same dtype. A `pad_ms`-millisecond zero-padded gap is inserted + between adjacent chunks (no pad before the first chunk or after the + last). + """ + import numpy as np + + if not wavs: + return np.zeros(0, dtype=np.float32) + if len(wavs) == 1: + return wavs[0] + pad = np.zeros(int(24000 * pad_ms / 1000), dtype=np.float32) + out: list = [] + for i, w in enumerate(wavs): + if i: + out.append(pad) + out.append(w) + return np.concatenate(out) + + +# --------------------------------------------------------------------------- +# Static catalog of speakers exposed via GET /speakers. +# +# These are the speaker tags the multilingual Orpheus-3B checkpoint was +# finetuned on (Sunbird/tts configs: lug, ach, lgg, nyn, teo, swa, ...). +# Add or remove entries to match whatever your own training run produced. +# Clients only need to send a `speaker_id` that exists in the checkpoint; +# this catalog is purely informational. +# --------------------------------------------------------------------------- + +# 40 speakers across 16 languages from the multilingual training run's +# held-aside test split. Source corpora: SALT, WAXAL, SLR32, SLR129, +# BATEESA. Language codes are ISO 639-3. +SPEAKERS_BY_LANGUAGE: dict[str, list[str]] = { + "ach": [ # Acholi + "salt_ach_0001", + "waxal_ach_0001", + "waxal_ach_0005", + "waxal_ach_0006", + "waxal_ach_0008", + ], + "afr": [ # Afrikaans + "slr32_afr_0009", + ], + "eng": [ # English + "salt_eng_0001", + "salt_eng_0002", + "salt_eng_0003", + ], + "ewe": [ # Ewe + "slr129_ewe_0001", + ], + "ful": [ # Fula / Pulaar + "waxal_ful_0003", + "waxal_ful_0004", + "waxal_ful_0006", + ], + "hau": [ # Hausa + "waxal_hau_0004", + "waxal_hau_0006", + "waxal_hau_0007", + "waxal_hau_0008", + ], + "ibo": [ # Igbo + "waxal_ibo_0003", + "waxal_ibo_0005", + "waxal_ibo_0008", + ], + "kik": [ # Kikuyu + "waxal_kik_0003", + "waxal_kik_0004", + ], + "kin": [ # Kinyarwanda + "bateesa_kin_0001", + ], + "lin": [ # Lingala + "slr129_lin_0001", + ], + "lug": [ # Luganda + "salt_lug_0001", + "waxal_lug_0002", + "waxal_lug_0003", + "waxal_lug_0004", + "waxal_lug_0005", + "waxal_lug_0006", + "waxal_lug_0007", + "waxal_lug_0008", + ], + "luo": [ # Dholuo + "waxal_luo_0001", + "waxal_luo_0002", + "waxal_luo_0003", + "waxal_luo_0004", + ], + "nyn": [ # Runyankole + "salt_nyn_0001", + "waxal_nyn_0003", + "waxal_nyn_0004", + "waxal_nyn_0007", + "waxal_nyn_0008", + ], + "swa": [ # Swahili + "waxal_swa_0006", + "waxal_swa_0007", + ], + "teo": [ # Ateso + "salt_teo_0001", + ], + "xho": [ # Xhosa + "slr32_xho_0012", + ], + "yor": [ # Yoruba + "waxal_yor_0002", + "waxal_yor_0006", + "waxal_yor_0008", + ], +} +DEFAULT_SPEAKER_ID = os.environ.get("ORPHEUS_DEFAULT_SPEAKER", "salt_lug_0001") + + +# --------------------------------------------------------------------------- +# Modal image, volumes, secrets +# --------------------------------------------------------------------------- + +vllm_image = ( + modal.Image.from_registry( + "nvidia/cuda:12.8.0-devel-ubuntu22.04", add_python="3.12" + ) + .entrypoint([]) + .apt_install("libsndfile1") + # transformers is capped below 4.56 on purpose: vLLM 0.10.0's tokenizer + # cache layer reads `tokenizer.all_special_tokens_extended`, which was + # refactored out into a `TokenizersBackend` object in transformers 4.56. + # Loading then raises: + # AttributeError: TokenizersBackend has no attribute + # all_special_tokens_extended + # The lower bound matches vLLM 0.10.0's own minimum (>=4.53.2). + .uv_pip_install( + "vllm==0.10.0", + "transformers>=4.53.2,<4.56.0", + "huggingface-hub>=0.27.0", + "snac==1.2.1", + "soundfile==0.12.1", + "numpy<2.0", + "fastapi[standard]==0.115.0", + "pydantic>=2.5,<3.0", + extra_index_url="https://download.pytorch.org/whl/cu128", + extra_options="--index-strategy unsafe-best-match", + ) + .env( + { + "HF_XET_HIGH_PERFORMANCE": "1", # faster HF downloads + "HF_HUB_ENABLE_HF_TRANSFER": "1", + } + ) +) + +hf_cache_vol = modal.Volume.from_name( + "orpheus-hf-cache", create_if_missing=True +) +vllm_cache_vol = modal.Volume.from_name( + "orpheus-vllm-cache", create_if_missing=True +) + +hf_secret = modal.Secret.from_name( + "huggingface-secret", required_keys=["HF_TOKEN"] +) + +app = modal.App(APP_NAME) + + +# --------------------------------------------------------------------------- +# One-shot warmup: prefetch model weights into the HF cache volume. +# +# Run manually before the first deploy to keep cold starts short: +# modal run orpheus-3B/modal_deploy.py::download_model +# --------------------------------------------------------------------------- + +@app.function( + image=vllm_image, + secrets=[hf_secret], + volumes={"/root/.cache/huggingface": hf_cache_vol}, + timeout=30 * MINUTES, +) +def download_model(model_id: str = MODEL_ID) -> None: + from huggingface_hub import snapshot_download + + print(f"downloading {model_id}") + snapshot_download( + model_id, + # keep the cache tidy: skip the pickled .bin if safetensors exist + ignore_patterns=["*.pt", "*.bin"], + ) + print(f"downloading {SNAC_MODEL_ID}") + snapshot_download(SNAC_MODEL_ID) + print("warmup complete") + + +# --------------------------------------------------------------------------- +# Inference class +# --------------------------------------------------------------------------- + +@app.cls( + image=vllm_image, + gpu=GPU, + secrets=[hf_secret], + volumes={ + "/root/.cache/huggingface": hf_cache_vol, + "/root/.cache/vllm": vllm_cache_vol, + }, + scaledown_window=15 * MINUTES, + timeout=20 * MINUTES, # generous cold-start budget for first model load + min_containers=0, + max_containers=3, # hard cap on horizontal scale-out +) +@modal.concurrent(max_inputs=MAX_CONCURRENT_INPUTS) +class OrpheusTTS: + @modal.enter() + def load(self) -> None: + """Load vLLM + SNAC + tokenizer once per container.""" + from snac import SNAC + from transformers import AutoTokenizer + from vllm import LLM + + print(f"loading vLLM model: {MODEL_ID}") + self.llm = LLM( + model=MODEL_ID, + dtype="bfloat16", + max_model_len=MAX_MODEL_LEN, + gpu_memory_utilization=0.85, + enforce_eager=True, + trust_remote_code=False, + hf_overrides={"vocab_size": VOCAB_SIZE}, + ) + self.tokenizer = AutoTokenizer.from_pretrained(MODEL_ID) + # SNAC runs on CPU — frees GPU for vLLM's KV cache. + self.snac = SNAC.from_pretrained(SNAC_MODEL_ID).to("cpu") + print("Orpheus-3B + SNAC ready") + + # ----- helpers ----- + + def _build_prompt_ids(self, text: str, speaker_id: str) -> list[int]: + """[SOH] + tokenizer(speaker_id: text) + [EOT, EOH].""" + tagged = f"{speaker_id}: {text}" + text_ids = self.tokenizer.encode(tagged, add_special_tokens=True) + return [START_OF_HUMAN, *text_ids, END_OF_TEXT, END_OF_HUMAN] + + def _codes_to_wav(self, token_ids: list[int]): + """vLLM-output token_ids -> 24 kHz mono float32 numpy waveform.""" + import numpy as np + import torch + + ids = torch.tensor(token_ids, dtype=torch.int64) + + # Crop on the LAST SOS, discarding any pre-speech preamble. + sos = (ids == START_OF_SPEECH).nonzero(as_tuple=True)[0] + if len(sos) > 0: + ids = ids[sos[-1].item() + 1 :] + + # Keep only tokens in the audio codebook range. + audio = ids[(ids >= AUDIO_TOKEN_LO) & (ids < AUDIO_TOKEN_HI)] + n = (audio.size(0) // 7) * 7 + code_list = [t.item() - AUDIO_TOKEN_LO for t in audio[:n]] + + l1, l2, l3 = [], [], [] + for i in range(len(code_list) // 7): + l1.append(code_list[7 * i]) + l2.append(code_list[7 * i + 1] - 4096) + l3.append(code_list[7 * i + 2] - 2 * 4096) + l3.append(code_list[7 * i + 3] - 3 * 4096) + l2.append(code_list[7 * i + 4] - 4 * 4096) + l3.append(code_list[7 * i + 5] - 5 * 4096) + l3.append(code_list[7 * i + 6] - 6 * 4096) + if not l1: + return np.zeros(12000, dtype=np.float32) + + def clamp(vals): + return [max(0, min(4095, v)) for v in vals] + + codes = [ + torch.tensor(clamp(l1)).unsqueeze(0), + torch.tensor(clamp(l2)).unsqueeze(0), + torch.tensor(clamp(l3)).unsqueeze(0), + ] + wav = self.snac.decode(codes) + return wav.detach().squeeze().to("cpu").numpy().astype(np.float32) + + # ----- HTTP ----- + + @modal.asgi_app() + def web(self): + import base64 + import io + from typing import Annotated, Optional + + import soundfile as sf + from fastapi import Body, FastAPI, HTTPException + from fastapi.responses import JSONResponse, Response + from pydantic import BaseModel, Field + from vllm import SamplingParams + + api = FastAPI( + title="Orpheus-3B Multilingual TTS", + description=( + "Multi-language, multi-speaker TTS backed by vLLM. " + "Each request carries a speaker_id matching one of the " + "speaker tags the finetuned checkpoint was trained on." + ), + ) + + class TTSReq(BaseModel): + text: str = Field(..., description="Text to synthesize.") + speaker_id: str = Field( + DEFAULT_SPEAKER_ID, + description="Speaker tag from the finetune set, e.g. salt_lug_0001.", + ) + seed: Optional[int] = Field( + None, description="RNG seed for reproducibility." + ) + temperature: float = Field(0.6, ge=0.0, le=2.0) + top_p: float = Field(0.95, gt=0.0, le=1.0) + repetition_penalty: float = Field(1.1, ge=1.0, le=2.0) + max_tokens: int = Field(1200, ge=64, le=MAX_MODEL_LEN) + + class TTSBatchReq(BaseModel): + items: list[TTSReq] + + def _sampling(req: TTSReq, seed_offset: int = 0) -> SamplingParams: + seed = None if req.seed is None else req.seed + seed_offset + return SamplingParams( + temperature=req.temperature, + top_p=req.top_p, + repetition_penalty=req.repetition_penalty, + max_tokens=req.max_tokens, + stop_token_ids=[END_OF_SPEECH], + skip_special_tokens=False, + seed=seed, + ) + + def _wav_bytes(wav) -> bytes: + buf = io.BytesIO() + sf.write(buf, wav, 24000, format="WAV", subtype="PCM_16") + return buf.getvalue() + + @api.get("/health") + def health(): + return { + "status": "ok", + "model": MODEL_ID, + "max_model_len": MAX_MODEL_LEN, + "sample_rate": 24000, + } + + @api.get("/speakers") + def speakers(): + return { + "default": DEFAULT_SPEAKER_ID, + "by_language": SPEAKERS_BY_LANGUAGE, + } + + @api.post( + "/tts", + responses={200: {"content": {"audio/wav": {}}}}, + response_class=Response, + ) + def tts(req: Annotated[TTSReq, Body()]): + chunks = _chunk_text(req.text) + if len(chunks) > MAX_CHUNKS_PER_REQUEST: + raise HTTPException( + status_code=422, + detail={ + "error": "text_too_long", + "detail": ( + f"text produced {len(chunks)} chunks " + f"(limit {MAX_CHUNKS_PER_REQUEST}); " + "shorten the input or split it client-side" + ), + }, + ) + + if len(chunks) == 1: + # Fast path — byte-for-byte identical to the prior + # single-shot implementation. Uses req.text (raw) so any + # exotic whitespace/casing in the original input is + # preserved verbatim through the tokenizer. + sp = _sampling(req) + prompt_ids = self._build_prompt_ids(req.text, req.speaker_id) + outs = self.llm.generate( + [{"prompt_token_ids": prompt_ids}], sp + ) + wav = self._codes_to_wav(list(outs[0].outputs[0].token_ids)) + else: + prompts = [ + { + "prompt_token_ids": self._build_prompt_ids( + c, req.speaker_id + ) + } + for c in chunks + ] + sps = [ + _sampling(req, seed_offset=i) for i in range(len(chunks)) + ] + print( + f"tts chunks={len(chunks)} " + f"text_len={len(req.text)} " + f"speaker={req.speaker_id}" + ) + outs = self.llm.generate(prompts, sps) + wavs = [ + self._codes_to_wav(list(o.outputs[0].token_ids)) + for o in outs + ] + wav = _concat_wavs(wavs, pad_ms=120) + + return Response( + content=_wav_bytes(wav), + media_type="audio/wav", + headers={ + "X-Sample-Rate": "24000", + "X-Duration-Seconds": f"{len(wav) / 24000:.3f}", + "X-Speaker-Id": req.speaker_id, + "X-Chunks": str(len(chunks)), + }, + ) + + @api.post("/tts/batch") + def tts_batch(req: Annotated[TTSBatchReq, Body()]): + if not req.items: + raise HTTPException( + status_code=400, detail="items must be non-empty" + ) + # Use the first item's decoding params for the whole batch. + # vLLM allows per-request SamplingParams too — pass a list if you + # need heterogeneous decoding settings across the batch. + sp = _sampling(req.items[0]) + prompts = [ + { + "prompt_token_ids": self._build_prompt_ids( + it.text, it.speaker_id + ) + } + for it in req.items + ] + outs = self.llm.generate(prompts, sp) + results = [] + for it, out in zip(req.items, outs): + wav = self._codes_to_wav(list(out.outputs[0].token_ids)) + results.append( + { + "text": it.text, + "speaker_id": it.speaker_id, + "sample_rate": 24000, + "duration_sec": len(wav) / 24000, + "audio_wav_b64": base64.b64encode( + _wav_bytes(wav) + ).decode("ascii"), + } + ) + return JSONResponse({"results": results}) + + return api + + +# --------------------------------------------------------------------------- +# Local entry point: a quick smoke test you can run with +# modal run orpheus-3B/modal_deploy.py +# --------------------------------------------------------------------------- + +@app.local_entrypoint() +def smoke_test( + text: str = "Mwattu, oli otya?", + speaker_id: str = DEFAULT_SPEAKER_ID, +) -> None: + """Round-trip one synthesis through the deployed class without HTTP. + + Useful for confirming the model loads and the SNAC pipeline works + without going through the FastAPI layer. + """ + chunks = _chunk_text(text) + print(f"text = {text!r}") + print(f"speaker = {speaker_id!r}") + print(f"model = {MODEL_ID}") + print(f"chunks = {len(chunks)} (cap={MAX_CHUNKS_PER_REQUEST})") + print( + "deploy with `modal deploy orpheus-3B/modal_deploy.py` and then" + " curl the /tts endpoint — see MODAL_DEPLOYMENT.md for examples." + ) diff --git a/modal-deploy/orpheus-3B/optimizer-review.md b/modal-deploy/orpheus-3B/optimizer-review.md new file mode 100644 index 00000000..238ca2ee --- /dev/null +++ b/modal-deploy/orpheus-3B/optimizer-review.md @@ -0,0 +1,584 @@ +# Question 1: How many epochs are best for finetuning a tts model? + +There isn’t a single “best” number of epochs for fine-tuning a TTS model—it depends heavily on your **dataset size, model size, and overfitting behavior**. But I can give you practical ranges and how to decide (this is what actually matters in real training). + +--- + +## 🔑 The short answer + +* **Small dataset (≤ 5 hours audio):** 30–100 epochs +* **Medium dataset (5–50 hours):** 10–30 epochs +* **Large dataset (50+ hours):** 3–10 epochs + +But don’t stop here—epochs alone are a weak signal. What matters is **when to stop training**. + +--- + +## 🧠 What really determines the “right” number? + +### 1. Dataset size (biggest factor) + +* Small data → needs more epochs to learn patterns +* Large data → fewer epochs, otherwise overfitting + +--- + +### 2. Overfitting signals (critical for TTS) + +TTS models overfit **fast**, especially on: + +* single-speaker datasets +* clean studio audio + +Watch for: + +* 🔊 Audio becomes **too sharp / robotic** +* 🗣️ Model memorizes phrases instead of generalizing +* 📉 Validation loss starts increasing + +--- + +### 3. Steps vs Epochs (more important for you) + +For TTS (especially with models like **Tacotron, FastSpeech, VITS, Bark-style models**), people usually track: + +* **training steps (iterations)** instead of epochs + +Typical ranges: + +* **20k – 200k steps** depending on dataset + +👉 Example: + +* 10 hours dataset → ~50k–100k steps often enough +* 1–2 hours dataset → may need 100k+ steps (but risky overfit) + +--- + +## 📊 Practical strategy (what you should actually do) + +### ✅ Use early stopping + checkpoints + +Instead of guessing epochs: + +1. Save checkpoints every N steps (e.g. every 1k–5k) +2. Periodically generate sample audio +3. Stop when: + + * quality stops improving + * or starts degrading + +--- + +### ✅ Monitor these metrics + +* **Loss (training vs validation)** +* **Mel spectrogram alignment quality** +* **Generated audio naturalness** + +--- + +### ✅ Rule of thumb (very practical) + +> “Train until it sounds best, not until epochs are exhausted.” + +--- + +## ⚠️ Common mistakes (I see this a lot) + +* ❌ Training for fixed epochs without listening to outputs +* ❌ Using too many epochs on small datasets → voice memorization +* ❌ Ignoring validation set +* ❌ Not shuffling or cleaning data (hurts convergence more than epochs) + +--- + +## 🧪 For your setup (based on your work with TTS + ML pipelines) + +Since you’ve been working with: + +* Whisper / ASR datasets +* multilingual + noisy datasets +* Sunbird TTS fine-tuning + +👉 I’d recommend: + +* Start with **~50k steps** +* Save checkpoints every **2k–5k** +* Evaluate audio every checkpoint +* Expect optimal point around: + + * **30k–80k steps** (likely sweet spot) + +--- + + + +Yes — your current notebook trains with **3 epochs**, but it has **no eval dataset, no early stopping, and no “best checkpoint” loading** before pushing. + +Replace your current **Trainer cell** with this improved version: + +```python +from transformers import TrainingArguments, Trainer, DataCollatorForSeq2Seq, EarlyStoppingCallback + +if tokenizer.pad_token_id is None: + tokenizer.pad_token_id = PAD_TOKEN + +data_collator = DataCollatorForSeq2Seq( + tokenizer=tokenizer, + padding=True, + label_pad_token_id=-100, +) + +# Create tokenized eval set from ds_test +ds_eval = ds_test.map(create_input_ids, remove_columns=["text", "codes_list"]) +ds_eval = ds_eval.remove_columns( + [c for c in ds_eval.column_names if c not in ("input_ids", "labels", "attention_mask")] +) +ds_eval = ds_eval.filter(lambda r: len(r["input_ids"]) <= MAX_SEQ_LEN) + +# Optional: keep eval small enough to avoid slow validation +MAX_EVAL_SAMPLES = 1000 +if len(ds_eval) > MAX_EVAL_SAMPLES: + ds_eval = ds_eval.shuffle(seed=3407).select(range(MAX_EVAL_SAMPLES)) + +trainer = Trainer( + model=model, + train_dataset=ds_train, + eval_dataset=ds_eval, + data_collator=data_collator, + args=TrainingArguments( + output_dir=str(OUTPUT_DIR), + + # Training length + max_steps=-1, + num_train_epochs=5, # allow up to 5, early stopping will stop earlier + + # Batch config + per_device_train_batch_size=1, + per_device_eval_batch_size=1, + gradient_accumulation_steps=4, + + # Optimisation + learning_rate=2e-4, + warmup_ratio=0.03, + weight_decay=0.001, + optim="adamw_8bit", + lr_scheduler_type="cosine", + + # Evaluation + checkpointing + eval_strategy="steps", + eval_steps=500, + save_strategy="steps", + save_steps=500, + save_total_limit=3, + + # Best model selection + load_best_model_at_end=True, + metric_for_best_model="eval_loss", + greater_is_better=False, + + # Logging + logging_steps=25, + report_to="mlflow", + + # Stability + seed=3407, + fp16=not torch.cuda.is_bf16_supported(), + bf16=torch.cuda.is_bf16_supported(), + ), + callbacks=[ + EarlyStoppingCallback( + early_stopping_patience=4, + early_stopping_threshold=0.001, + ) + ], +) +``` + +Then keep: + +```python +trainer_stats = trainer.train() +``` + +After training, push the **best loaded model**, not just the last checkpoint: + +```python +HF_REPO_ID = "sunbird/orpheus-3b-tts-multilingual" + +model.push_to_hub_merged( + HF_REPO_ID, + tokenizer, + save_method="merged_16bit", + token=os.environ["HF_TOKEN"], +) + +print("Pushed best model to https://huggingface.co/" + HF_REPO_ID) +``` + +Best practical epoch setting for your case: + +```python +num_train_epochs=5 +``` + +But because `EarlyStoppingCallback` is active, training may stop around **2–4 epochs** if `eval_loss` stops improving. + +Main improvements: + +1. **Use `ds_test` as `eval_dataset`** +2. **Evaluate every 500 steps** +3. **Save every 500 steps** +4. **Track `eval_loss`** +5. **Load best checkpoint automatically** +6. **Stop early after 4 bad eval rounds** +7. **Push the best model to Hugging Face** + +For TTS, still manually listen to generated samples from checkpoints, because `eval_loss` does not always perfectly match audio quality. + + + +# Question 2: I also see you changed lr_scheduler_type from linear to cosine, why? + +Good catch—that wasn’t arbitrary. I changed from **linear → cosine** because it generally behaves better for **fine-tuning large generative models like TTS (your Orpheus-3B)**. + +Let’s break it down properly 👇 + +--- + +## 🧠 What “linear” vs “cosine” actually do + +### 🔹 Linear scheduler (what you had) + +* LR increases during warmup +* Then **linearly decays to 0** + +📉 Shape: + +``` + /\ + / \ + / \ +/ \____ +``` + +👉 Behavior: + +* Drops LR steadily +* Can **reduce learning too aggressively toward the end** + +--- + +### 🔹 Cosine scheduler (what I switched to) + +* LR increases during warmup +* Then follows a **smooth cosine decay** + +📉 Shape: + +``` + /\ + / \ + / \ +/ \__ + \__ + \__ +``` + +👉 Behavior: + +* Slower decay early +* **More gentle + smoother convergence** + +--- + +## ⚡ Why cosine is better for your TTS fine-tuning + +### 1. TTS models are sensitive to optimization noise + +You're training: + +* autoregressive / sequence model +* audio-token generation (codes) + +👉 These are **very unstable if LR drops too fast** + +Cosine helps by: + +* keeping LR **slightly higher for longer** +* allowing **better fine-grained learning of prosody & alignment** + +--- + +### 2. You’re using early stopping now + +With: + +```python +EarlyStoppingCallback(...) +``` + +👉 You **don’t need LR → 0 aggressively** + +Instead: + +* Let training explore +* Stop when validation stops improving + +👉 Cosine pairs *much better* with early stopping than linear + +--- + +### 3. Small-to-medium dataset (your case) + +Your pipeline: + +* multilingual +* relatively limited hours per language +* noisy / mixed-quality data + +👉 Linear decay often: + +* converges too fast +* → underfits subtle speech patterns + +👉 Cosine: + +* gives more **“exploration time”** +* better voice naturalness + +--- + +### 4. Empirical reality (what people actually see) + +Across: + +* TTS (VITS, Bark-style) +* LLM fine-tuning +* speech models + +👉 Cosine usually gives: + +* slightly better final quality +* smoother loss curves +* less overfitting spikes + +--- + +## ⚖️ When you would KEEP linear + +Linear is still fine if: + +* You want **fast, predictable convergence** +* You train for **fixed short runs (no early stopping)** +* Dataset is **very large + clean** + +--- + +## 🧪 Practical recommendation for your setup + +Keep: + +```python +lr_scheduler_type="cosine" +warmup_ratio=0.03 +``` + +If you want to experiment: + +### Option A — safer (recommended) + +```python +lr_scheduler_type="cosine" +``` + +### Option B — more aggressive (can improve quality) + +```python +lr_scheduler_type="cosine_with_restarts" +``` + +👉 Good for: + +* multilingual TTS +* avoiding local minima + +--- + +## 🔥 One more upgrade (you’ll like this) + +Since you're doing serious TTS work, try: + +```python +learning_rate=1e-4 # slightly lower +``` + +Why: + +* 2e-4 is a bit aggressive for 3B model +* lower LR → better audio stability + +--- + +## 🧩 Bottom line + +I switched to cosine because: + +> It gives smoother, longer, and more stable learning — which is exactly what TTS fine-tuning needs, especially with early stopping. + +--- + +If you want, next step I can help you: + +* add **audio-based evaluation (not just loss)** +* log generated samples in MLflow every eval step +* or tune **LoRA + LR + batch scaling for your dual A5000 setup** + +That’s where you’ll get the real quality gains. + +## Proposed upgrade from above + +Yes — next upgrade should be: + +1. **keep `eval_loss` early stopping** +2. **generate fixed TTS samples every eval** +3. **log audio samples to MLflow** +4. **push only the best model** + +Add a custom callback like this: + +```python +import mlflow +from transformers import TrainerCallback + +class TTSAudioEvalCallback(TrainerCallback): + def __init__(self, eval_prompts, tokenizer, every_n_evals=1): + self.eval_prompts = eval_prompts + self.tokenizer = tokenizer + self.every_n_evals = every_n_evals + self.eval_count = 0 + + def on_evaluate(self, args, state, control, model=None, **kwargs): + self.eval_count += 1 + + if self.eval_count % self.every_n_evals != 0: + return control + + model.eval() + + for i, text in enumerate(self.eval_prompts): + # Replace this with your Orpheus inference function + audio_array, sample_rate = generate_tts_audio( + model=model, + tokenizer=self.tokenizer, + text=text, + ) + + mlflow.log_audio( + audio_array, + artifact_file=f"tts_eval/step_{state.global_step}/sample_{i}.wav", + sample_rate=sample_rate, + ) + + model.train() + return control +``` + +Then attach it to your trainer: + +```python +eval_prompts = [ + "Hello, this is a test of the multilingual TTS model.", + "Omwana alina eddembe okufuna obujjanjabi obulungi.", + "Habari, hii ni sauti ya majaribio.", +] + +trainer = Trainer( + model=model, + train_dataset=ds_train, + eval_dataset=ds_eval, + data_collator=data_collator, + args=training_args, + callbacks=[ + EarlyStoppingCallback( + early_stopping_patience=4, + early_stopping_threshold=0.001, + ), + TTSAudioEvalCallback( + eval_prompts=eval_prompts, + tokenizer=tokenizer, + every_n_evals=1, + ), + ], +) +``` + +Recommended training config: + +```python +training_args = TrainingArguments( + output_dir=str(OUTPUT_DIR), + + num_train_epochs=5, + + per_device_train_batch_size=1, + per_device_eval_batch_size=1, + gradient_accumulation_steps=4, + + learning_rate=1e-4, + warmup_ratio=0.03, + lr_scheduler_type="cosine", + weight_decay=0.001, + optim="adamw_8bit", + + eval_strategy="steps", + eval_steps=500, + + save_strategy="steps", + save_steps=500, + save_total_limit=3, + + load_best_model_at_end=True, + metric_for_best_model="eval_loss", + greater_is_better=False, + + logging_steps=25, + report_to="mlflow", + + bf16=torch.cuda.is_bf16_supported(), + fp16=not torch.cuda.is_bf16_supported(), + + seed=3407, +) +``` + +The missing piece you must connect is this function: + +```python +def generate_tts_audio(model, tokenizer, text): + """ + Use your existing Orpheus inference/decoder logic here. + + Must return: + audio_array: numpy array + sample_rate: int + """ + raise NotImplementedError +``` + +The important idea is: **`eval_loss` decides early stopping, but audio samples decide whether the model actually sounds good.** + +For your Orpheus-3B-TTS fine-tuning, I’d use: + +```python +num_train_epochs=5 +learning_rate=1e-4 +lr_scheduler_type="cosine" +early_stopping_patience=4 +eval_steps=500 +save_steps=500 +``` + +Then manually compare MLflow audio samples from each checkpoint before trusting the final model. + diff --git a/modal-deploy/orpheus-3B/tests/__init__.py b/modal-deploy/orpheus-3B/tests/__init__.py new file mode 100644 index 00000000..1bb8bf6d --- /dev/null +++ b/modal-deploy/orpheus-3B/tests/__init__.py @@ -0,0 +1 @@ +# empty diff --git a/modal-deploy/orpheus-3B/tests/test_chunking.py b/modal-deploy/orpheus-3B/tests/test_chunking.py new file mode 100644 index 00000000..d0b2a173 --- /dev/null +++ b/modal-deploy/orpheus-3B/tests/test_chunking.py @@ -0,0 +1,209 @@ +"""Unit tests for _chunk_text and _concat_wavs in modal_deploy. + +These are pure-Python tests. They import modal_deploy (which imports the +modal SDK at module level), so `pip install modal` must succeed in the +test environment. No Modal services are contacted at import time. +""" + +import logging + +import pytest + +from modal_deploy import ( + CHUNK_HARD_CEILING, + MAX_CHARS_PER_CHUNK, + _chunk_text, +) + + +def test_empty_string_returns_single_empty_chunk(): + assert _chunk_text("") == [""] + + +def test_whitespace_only_returns_single_empty_chunk(): + assert _chunk_text(" \n\t ") == [""] + + +def test_single_short_sentence_yields_one_chunk(): + assert _chunk_text("Hello world.") == ["Hello world."] + + +def test_two_short_sentences_pack_into_one_chunk(): + assert _chunk_text("Hello world. How are you?") == [ + "Hello world. How are you?" + ] + + +def test_whitespace_is_normalized_before_packing(): + # Multiple internal spaces / newlines collapse to single spaces. + assert _chunk_text("Hello.\n\n World.") == ["Hello. World."] + + +def test_long_text_splits_into_multiple_chunks_each_within_limits(): + sentence = "This is a moderately long sentence for testing chunking." # ~56 chars + text = " ".join([sentence] * 8) # ~456 chars, 8 sentences + chunks = _chunk_text(text) + assert len(chunks) >= 2 + for c in chunks: + # Soft target is 220; hard ceiling is 280. Multi-word chunks + # must respect the hard ceiling. + assert len(c) <= CHUNK_HARD_CEILING, c + + +def test_long_sentence_with_clauses_splits_on_clause_boundaries(): + # Single sentence > max_chars, but full of commas. + sent = ( + "Today, even though it was raining heavily outside, the entire team gathered, " + "with notebooks and laptops in hand, in the upstairs meeting room, to carefully " + "discuss the new release schedule, the open bugs, and the overall testing plan, " + "before lunch." + ) + assert len(sent) > MAX_CHARS_PER_CHUNK + chunks = _chunk_text(sent) + assert len(chunks) >= 2 + for c in chunks: + assert len(c) <= CHUNK_HARD_CEILING + + +def test_long_sentence_without_clauses_splits_on_word_boundary_never_mid_word(): + # 60 copies of "word" → ~299 chars, no punctuation at all. + words = ["word"] * 60 + text = " ".join(words) + chunks = _chunk_text(text) + # Every chunk's "tokens" must be intact words. + for c in chunks: + for tok in c.split(): + assert tok == "word", f"word split: {tok!r} in chunk {c!r}" + + +def test_single_word_just_above_max_chars_emitted_as_own_chunk(): + # 250-char single word: > max_chars (220) but <= hard_ceiling (280). + long_word = "a" * 250 + chunks = _chunk_text(long_word) + assert chunks == [long_word] + + +def test_single_word_above_hard_ceiling_emitted_untouched_with_warning(caplog): + pathological = "a" * 300 # > hard_ceiling (280) + with caplog.at_level(logging.WARNING, logger="orpheus.chunker"): + chunks = _chunk_text(pathological) + assert chunks == [pathological] + assert any( + "hard_ceiling" in rec.getMessage() for rec in caplog.records + ), [rec.getMessage() for rec in caplog.records] + + +def test_orphan_tail_merges_into_previous_when_within_hard_ceiling(): + # Two long sentences that fit together under hard ceiling, plus a + # very short tail. + s_a = "a" * 80 + "." + s_b = "b" * 80 + "." + s_c = "End." + text = " ".join([s_a, s_b, s_c]) + # Force packing with max_chars=165: s_a (81) + " " + s_b (81) = 163, + # which fits in 165. s_c (4) cannot join (163+1+4=168 > 165). Without + # merge: ["aaa...", "bbb...", "End."] (s_c is its own chunk, < 30 + # chars). With orphan merge: 163+1+4=168 <= hard_ceiling=280 → merge + # → one chunk total (per spec §4.1). + chunks = _chunk_text(text, max_chars=165, hard_ceiling=280) + # 168 <= hard_ceiling=280, so the orphan-tail merge collapses s_c into + # the prior chunk → one chunk total. + assert len(chunks) == 1 + assert chunks[0].endswith("End.") + assert len(chunks[0]) <= 280 + + +def test_orphan_tail_not_merged_when_would_exceed_hard_ceiling(): + # Force a 2-chunk pack where the tail is short (<30) AND the would-be + # merged length exceeds hard_ceiling. The merge must be skipped. + # + # s_a is exactly max_chars=96 chars. _pack emits it as its own chunk. + # "End." (4 chars) cannot join (96+1+4=101 > 96) so it lands as an + # orphan chunk. The merge check: 96+1+4=101 > hard_ceiling=96, so + # the merge is refused and the orphan tail stays as its own chunk. + s_a = "a" * 95 + "." # 96 chars, exactly max_chars + text = s_a + " End." + chunks = _chunk_text(text, max_chars=96, hard_ceiling=96) + assert chunks[-1] == "End." # orphan stays as its own chunk + assert len(chunks) >= 2 + + +def test_chunker_is_deterministic(): + text = ( + "First sentence here. Second sentence, with a clause, follows. " + "Third sentence ends the test." + ) + assert _chunk_text(text) == _chunk_text(text) + + +def test_chunker_handles_ethiopic_terminator(): + text = "First sentence። Second sentence።" + chunks = _chunk_text(text) + # Both sentences should be preserved verbatim somewhere in the output. + joined = " ".join(chunks) + assert "First sentence።" in joined + assert "Second sentence።" in joined + + +def test_chunker_respects_max_chars_for_normal_multi_atom_chunks(): + # All atoms small. Every emitted chunk must fit max_chars (no + # single-atom-overflow path triggered). + text = " ".join(["short sentence one." for _ in range(40)]) + chunks = _chunk_text(text, max_chars=120, hard_ceiling=160) + for c in chunks: + assert len(c) <= 120, c + + +# -------------------------------------------------------------------------- +# _concat_wavs +# -------------------------------------------------------------------------- + +import numpy as np + +from modal_deploy import _concat_wavs + + +def test_concat_wavs_empty_list_returns_empty_float32_array(): + out = _concat_wavs([]) + assert isinstance(out, np.ndarray) + assert out.dtype == np.float32 + assert out.size == 0 + + +def test_concat_wavs_single_array_returned_unchanged(): + w = np.array([0.1, 0.2, 0.3], dtype=np.float32) + out = _concat_wavs([w]) + np.testing.assert_array_equal(out, w) + assert out.dtype == np.float32 + + +def test_concat_wavs_two_arrays_inserts_silence_pad(): + w1 = np.ones(1000, dtype=np.float32) + w2 = np.full(500, 0.5, dtype=np.float32) + out = _concat_wavs([w1, w2], pad_ms=120) + expected_pad = int(24000 * 120 / 1000) # 2880 samples + assert out.size == 1000 + expected_pad + 500 + np.testing.assert_array_equal(out[:1000], w1) + np.testing.assert_array_equal( + out[1000 : 1000 + expected_pad], + np.zeros(expected_pad, dtype=np.float32), + ) + np.testing.assert_array_equal(out[1000 + expected_pad :], w2) + + +def test_concat_wavs_three_arrays_has_two_pads(): + a = np.array([1.0], dtype=np.float32) + b = np.array([2.0], dtype=np.float32) + c = np.array([3.0], dtype=np.float32) + out = _concat_wavs([a, b, c], pad_ms=10) + pad_len = int(24000 * 10 / 1000) # 240 + assert out.size == 3 + 2 * pad_len + + +def test_concat_wavs_zero_pad_yields_pure_concatenation(): + w1 = np.array([1.0, 2.0], dtype=np.float32) + w2 = np.array([3.0, 4.0], dtype=np.float32) + out = _concat_wavs([w1, w2], pad_ms=0) + np.testing.assert_array_equal( + out, np.array([1.0, 2.0, 3.0, 4.0], dtype=np.float32) + )