From 5b6c6d8070790e7b623bddcfc45413edde6a8f9d Mon Sep 17 00:00:00 2001 From: huwenjie333 Date: Wed, 4 Feb 2026 11:53:15 +0700 Subject: [PATCH 1/8] init from example --- .../batched_whisper.py | 174 ++++++++++++++++++ 1 file changed, 174 insertions(+) create mode 100644 modal-deploy/asr-whisper-large-v3-salt/batched_whisper.py diff --git a/modal-deploy/asr-whisper-large-v3-salt/batched_whisper.py b/modal-deploy/asr-whisper-large-v3-salt/batched_whisper.py new file mode 100644 index 00000000..49af7e16 --- /dev/null +++ b/modal-deploy/asr-whisper-large-v3-salt/batched_whisper.py @@ -0,0 +1,174 @@ +# # Fast Whisper inference using dynamic batching + +# In this example, we demonstrate how to run [dynamically batched inference](https://modal.com/docs/guide/dynamic-batching) +# for OpenAI's speech recognition model, [Whisper](https://openai.com/index/whisper/), on Modal. +# Batching multiple audio samples together or batching chunks of a single audio sample can help to achieve a 2.8x increase +# in inference throughput on an A10G! + +# We will be running the [Whisper Large V3](https://huggingface.co/openai/whisper-large-v3) model. +# To run [any of the other HuggingFace Whisper models](https://huggingface.co/models?search=openai/whisper), +# simply replace the `MODEL_NAME` and `MODEL_REVISION` variables. + +# ## Setup + +# Let's start by importing the Modal client and defining the model that we want to serve. + + +from typing import Optional + +import modal + +MODEL_DIR = "/model" +MODEL_NAME = "openai/whisper-large-v3" +MODEL_REVISION = "afda370583db9c5359511ed5d989400a6199dfe1" + + +# ## Define a container image + +# We’ll start with Modal's baseline `debian_slim` image and install the relevant libraries. + +image = ( + modal.Image.debian_slim(python_version="3.11") + .uv_pip_install( + "torch==2.5.1", + "transformers==4.47.1", + "huggingface-hub==0.36.0", + "librosa==0.10.2", + "soundfile==0.12.1", + "accelerate==1.2.1", + "datasets==3.2.0", + ) + .env({"HF_XET_HIGH_PERFORMANCE": "1", "HF_HUB_CACHE": MODEL_DIR}) +) + +model_cache = modal.Volume.from_name("hf-hub-cache", create_if_missing=True) +app = modal.App( + "example-batched-whisper", + image=image, + volumes={MODEL_DIR: model_cache}, +) + +# ## Caching the model weights + +# We'll define a function to download the model and cache it in a volume. +# You can `modal run` against this function prior to deploying the App. + + +@app.function() +def download_model(): + from huggingface_hub import snapshot_download + from transformers.utils import move_cache + + snapshot_download( + MODEL_NAME, + ignore_patterns=["*.pt", "*.bin"], # Using safetensors + revision=MODEL_REVISION, + ) + move_cache() + + +# ## The model class + +# The inference function is best represented using Modal's [class syntax](https://modal.com/docs/guide/lifecycle-functions). + +# We define a `@modal.enter` method to load the model when the container starts, before it picks up any inputs. +# The weights will be loaded from the Hugging Face cache volume so that we don't need to download them when +# we start a new container. For more on storing model weights on Modal, see +# [this guide](https://modal.com/docs/guide/model-weights). + +# We also define a `transcribe` method that uses the `@modal.batched` decorator to enable dynamic batching. +# This allows us to invoke the function with individual audio samples, and the function will automatically batch them +# together before running inference. Batching is critical for making good use of the GPU, since GPUs are designed +# for running parallel operations at high throughput. + +# The `max_batch_size` parameter limits the maximum number of audio samples combined into a single batch. +# We used a `max_batch_size` of `64`, the largest power-of-2 batch size that can be accommodated by the 24 A10G GPU memory. +# This number will vary depending on the model and the GPU you are using. + +# The `wait_ms` parameter sets the maximum time to wait for more inputs before running the batched transcription. +# To tune this parameter, you can set it to the target latency of your application minus the execution time of an inference batch. +# This allows the latency of any request to stay within your target latency. + + +@app.cls( + gpu="a10g", # Try using an A100 or H100 if you've got a large model or need big batches! + max_containers=10, # default max GPUs for Modal's free tier +) +class Model: + @modal.enter() + def load_model(self): + import torch + from transformers import ( + AutoModelForSpeechSeq2Seq, + AutoProcessor, + pipeline, + ) + + self.processor = AutoProcessor.from_pretrained(MODEL_NAME) + self.model = AutoModelForSpeechSeq2Seq.from_pretrained( + MODEL_NAME, + torch_dtype=torch.float16, + low_cpu_mem_usage=True, + use_safetensors=True, + ).to("cuda") + + self.model.generation_config.language = "<|en|>" + + # Create a pipeline for preprocessing and transcribing speech data + self.pipeline = pipeline( + "automatic-speech-recognition", + model=self.model, + tokenizer=self.processor.tokenizer, + feature_extractor=self.processor.feature_extractor, + torch_dtype=torch.float16, + device="cuda", + ) + + @modal.batched(max_batch_size=64, wait_ms=1000) + def transcribe(self, audio_samples): + import time + + start = time.monotonic_ns() + print(f"Transcribing {len(audio_samples)} audio samples") + transcriptions = self.pipeline(audio_samples, batch_size=len(audio_samples)) + end = time.monotonic_ns() + print( + f"Transcribed {len(audio_samples)} samples in {round((end - start) / 1e9, 2)}s" + ) + return transcriptions + + +# ## Transcribe a dataset + +# In this example, we use the [librispeech_asr_dummy dataset](https://huggingface.co/datasets/hf-internal-testing/librispeech_asr_dummy) +# from Hugging Face's Datasets library to test the model. + +# We use [`map.aio`](https://modal.com/docs/reference/modal.Function#map) to asynchronously map over the audio files. +# This allows us to invoke the batched transcription method on each audio sample in parallel. + + +@app.function() +async def transcribe_hf_dataset(dataset_name): + from datasets import load_dataset + + print("📂 Loading dataset", dataset_name) + ds = load_dataset(dataset_name, "clean", split="validation") + print("📂 Dataset loaded") + batched_whisper = Model() + print("📣 Sending data for transcription") + async for transcription in batched_whisper.transcribe.map.aio(ds["audio"]): + yield transcription + + +# ## Run the model + +# We define a [`local_entrypoint`](https://modal.com/docs/guide/apps#entrypoints-for-ephemeral-apps) +# to run the transcription. You can run this locally with `modal run batched_whisper.py`. + + +@app.local_entrypoint() +async def main(dataset_name: Optional[str] = None): + if dataset_name is None: + dataset_name = "hf-internal-testing/librispeech_asr_dummy" + for result in transcribe_hf_dataset.remote_gen(dataset_name): + print(result["text"]) From cdfb9195bf1aa3de137eca0d7d71021ba98834c2 Mon Sep 17 00:00:00 2001 From: huwenjie333 Date: Wed, 4 Feb 2026 12:38:12 +0700 Subject: [PATCH 2/8] update model and dataset --- .../batched_whisper.py | 55 +++++++++---------- 1 file changed, 25 insertions(+), 30 deletions(-) diff --git a/modal-deploy/asr-whisper-large-v3-salt/batched_whisper.py b/modal-deploy/asr-whisper-large-v3-salt/batched_whisper.py index 49af7e16..853a0a7d 100644 --- a/modal-deploy/asr-whisper-large-v3-salt/batched_whisper.py +++ b/modal-deploy/asr-whisper-large-v3-salt/batched_whisper.py @@ -18,10 +18,11 @@ import modal -MODEL_DIR = "/model" -MODEL_NAME = "openai/whisper-large-v3" -MODEL_REVISION = "afda370583db9c5359511ed5d989400a6199dfe1" +MODEL_NAME = "Sunbird/asr-whisper-large-v3-salt" +# cache model weights with Modal Volumes +HF_CACHE_DIR = "/root/.cache/huggingface" +hf_cache_vol = modal.Volume.from_name("huggingface-cache", create_if_missing=True) # ## Define a container image @@ -37,15 +38,16 @@ "soundfile==0.12.1", "accelerate==1.2.1", "datasets==3.2.0", + "torchaudio==2.5.1", ) - .env({"HF_XET_HIGH_PERFORMANCE": "1", "HF_HUB_CACHE": MODEL_DIR}) + .env({"HF_XET_HIGH_PERFORMANCE": "1", "HF_HUB_CACHE": HF_CACHE_DIR}) ) -model_cache = modal.Volume.from_name("hf-hub-cache", create_if_missing=True) app = modal.App( - "example-batched-whisper", + "asr-whisper-large-v3-salt", image=image, - volumes={MODEL_DIR: model_cache}, + secrets=[modal.Secret.from_name("huggingface-read")], + volumes={HF_CACHE_DIR: hf_cache_vol}, ) # ## Caching the model weights @@ -62,7 +64,6 @@ def download_model(): snapshot_download( MODEL_NAME, ignore_patterns=["*.pt", "*.bin"], # Using safetensors - revision=MODEL_REVISION, ) move_cache() @@ -98,39 +99,33 @@ class Model: @modal.enter() def load_model(self): import torch - from transformers import ( - AutoModelForSpeechSeq2Seq, - AutoProcessor, - pipeline, - ) - - self.processor = AutoProcessor.from_pretrained(MODEL_NAME) - self.model = AutoModelForSpeechSeq2Seq.from_pretrained( - MODEL_NAME, - torch_dtype=torch.float16, - low_cpu_mem_usage=True, - use_safetensors=True, - ).to("cuda") - - self.model.generation_config.language = "<|en|>" + from transformers import pipeline # Create a pipeline for preprocessing and transcribing speech data self.pipeline = pipeline( "automatic-speech-recognition", - model=self.model, - tokenizer=self.processor.tokenizer, - feature_extractor=self.processor.feature_extractor, - torch_dtype=torch.float16, + model=MODEL_NAME, device="cuda", + torch_dtype=torch.float16, ) @modal.batched(max_batch_size=64, wait_ms=1000) def transcribe(self, audio_samples): import time + generate_kwargs = { + "language": 'English', + "task": "transcribe", + "num_beams": 1, + } + start = time.monotonic_ns() print(f"Transcribing {len(audio_samples)} audio samples") - transcriptions = self.pipeline(audio_samples, batch_size=len(audio_samples)) + transcriptions = self.pipeline( + audio_samples, + batch_size=len(audio_samples), + generate_kwargs=generate_kwargs + ) end = time.monotonic_ns() print( f"Transcribed {len(audio_samples)} samples in {round((end - start) / 1e9, 2)}s" @@ -152,7 +147,7 @@ async def transcribe_hf_dataset(dataset_name): from datasets import load_dataset print("📂 Loading dataset", dataset_name) - ds = load_dataset(dataset_name, "clean", split="validation") + ds = load_dataset(dataset_name, "multispeaker-eng", split="test") print("📂 Dataset loaded") batched_whisper = Model() print("📣 Sending data for transcription") @@ -169,6 +164,6 @@ async def transcribe_hf_dataset(dataset_name): @app.local_entrypoint() async def main(dataset_name: Optional[str] = None): if dataset_name is None: - dataset_name = "hf-internal-testing/librispeech_asr_dummy" + dataset_name = "Sunbird/salt" for result in transcribe_hf_dataset.remote_gen(dataset_name): print(result["text"]) From 3d7ce5a1b91468339d5632bec6a74f2b90016395 Mon Sep 17 00:00:00 2001 From: huwenjie333 Date: Wed, 4 Feb 2026 21:39:07 +0700 Subject: [PATCH 3/8] disable batch; add endpoint and client.py --- .../batched_whisper.py | 65 ++++++++++++------- .../asr-whisper-large-v3-salt/client.py | 36 ++++++++++ 2 files changed, 79 insertions(+), 22 deletions(-) create mode 100644 modal-deploy/asr-whisper-large-v3-salt/client.py diff --git a/modal-deploy/asr-whisper-large-v3-salt/batched_whisper.py b/modal-deploy/asr-whisper-large-v3-salt/batched_whisper.py index 853a0a7d..99e0147a 100644 --- a/modal-deploy/asr-whisper-large-v3-salt/batched_whisper.py +++ b/modal-deploy/asr-whisper-large-v3-salt/batched_whisper.py @@ -17,6 +17,7 @@ from typing import Optional import modal +from fastapi import Request MODEL_NAME = "Sunbird/asr-whisper-large-v3-salt" @@ -30,6 +31,7 @@ image = ( modal.Image.debian_slim(python_version="3.11") + .apt_install("ffmpeg") .uv_pip_install( "torch==2.5.1", "transformers==4.47.1", @@ -39,6 +41,8 @@ "accelerate==1.2.1", "datasets==3.2.0", "torchaudio==2.5.1", + "fastapi==0.115.6", + "python-multipart==0.0.20", ) .env({"HF_XET_HIGH_PERFORMANCE": "1", "HF_HUB_CACHE": HF_CACHE_DIR}) ) @@ -53,7 +57,7 @@ # ## Caching the model weights # We'll define a function to download the model and cache it in a volume. -# You can `modal run` against this function prior to deploying the App. +# You can `modal run batched_whisper.py::download_model` against this function prior to deploying the App. @app.function() @@ -77,23 +81,12 @@ def download_model(): # we start a new container. For more on storing model weights on Modal, see # [this guide](https://modal.com/docs/guide/model-weights). -# We also define a `transcribe` method that uses the `@modal.batched` decorator to enable dynamic batching. -# This allows us to invoke the function with individual audio samples, and the function will automatically batch them -# together before running inference. Batching is critical for making good use of the GPU, since GPUs are designed -# for running parallel operations at high throughput. - -# The `max_batch_size` parameter limits the maximum number of audio samples combined into a single batch. -# We used a `max_batch_size` of `64`, the largest power-of-2 batch size that can be accommodated by the 24 A10G GPU memory. -# This number will vary depending on the model and the GPU you are using. - -# The `wait_ms` parameter sets the maximum time to wait for more inputs before running the batched transcription. -# To tune this parameter, you can set it to the target latency of your application minus the execution time of an inference batch. -# This allows the latency of any request to stay within your target latency. @app.cls( gpu="a10g", # Try using an A100 or H100 if you've got a large model or need big batches! max_containers=10, # default max GPUs for Modal's free tier + scaledown_window=60 * 3, ) class Model: @modal.enter() @@ -109,28 +102,56 @@ def load_model(self): torch_dtype=torch.float16, ) - @modal.batched(max_batch_size=64, wait_ms=1000) - def transcribe(self, audio_samples): + # @modal.batched(max_batch_size=64, wait_ms=1000) + # def transcribe(self, audio_samples): + # import time + + # generate_kwargs = { + # "language": 'English', + # "task": "transcribe", + # "num_beams": 1, + # } + + # start = time.monotonic_ns() + # print(f"Transcribing {len(audio_samples)} audio samples") + # transcriptions = self.pipeline( + # audio_samples, + # batch_size=len(audio_samples), + # generate_kwargs=generate_kwargs + # ) + # end = time.monotonic_ns() + # print( + # f"Transcribed {len(audio_samples)} samples in {round((end - start) / 1e9, 2)}s" + # ) + # return transcriptions + + @modal.fastapi_endpoint(docs=True, method="POST") + async def transcribe(self, request: Request): + """ + Web endpoint that accepts audio bytes and returns the transcription. + """ import time + data = await request.body() generate_kwargs = { - "language": 'English', + # "language": 'English', "task": "transcribe", "num_beams": 1, + "return_timestamps": True, } start = time.monotonic_ns() - print(f"Transcribing {len(audio_samples)} audio samples") transcriptions = self.pipeline( - audio_samples, - batch_size=len(audio_samples), - generate_kwargs=generate_kwargs + [data], + batch_size=1, + generate_kwargs=generate_kwargs, ) end = time.monotonic_ns() print( - f"Transcribed {len(audio_samples)} samples in {round((end - start) / 1e9, 2)}s" + f"Transcribed in {round((end - start) / 1e9, 2)}s" ) - return transcriptions + + return {"text": transcriptions[0]["text"]} # ## Transcribe a dataset diff --git a/modal-deploy/asr-whisper-large-v3-salt/client.py b/modal-deploy/asr-whisper-large-v3-salt/client.py new file mode 100644 index 00000000..0295e541 --- /dev/null +++ b/modal-deploy/asr-whisper-large-v3-salt/client.py @@ -0,0 +1,36 @@ +import argparse +import requests +import os +import sys + +def main(): + parser = argparse.ArgumentParser(description="Whisper Client") + parser.add_argument("--audio", type=str, required=True, help="Path to audio file") + parser.add_argument("--url", type=str, required=True, help="URL of the Modal endpoint") + args = parser.parse_args() + + if not os.path.exists(args.audio): + print(f"Error: Audio file not found at {args.audio}") + sys.exit(1) + + with open(args.audio, "rb") as f: + audio_data = f.read() + + print(f"Sending {len(audio_data)} bytes of audio data to {args.url}...") + + # Send audio data as raw request body + response = requests.post( + args.url, + data=audio_data, + headers={"Content-Type": "application/octet-stream"} + ) + + if response.status_code == 200: + print("Success!") + print(response.json()) + else: + print(f"Error: {response.status_code}") + print(response.text) + +if __name__ == "__main__": + main() From 718965fbf387298f01a7765023c67ea277268c92 Mon Sep 17 00:00:00 2001 From: huwenjie333 Date: Wed, 4 Feb 2026 22:48:05 +0700 Subject: [PATCH 4/8] update to vLLM --- .../batched_whisper.py | 133 ++++++++++-------- 1 file changed, 77 insertions(+), 56 deletions(-) diff --git a/modal-deploy/asr-whisper-large-v3-salt/batched_whisper.py b/modal-deploy/asr-whisper-large-v3-salt/batched_whisper.py index 99e0147a..e49355a0 100644 --- a/modal-deploy/asr-whisper-large-v3-salt/batched_whisper.py +++ b/modal-deploy/asr-whisper-large-v3-salt/batched_whisper.py @@ -1,57 +1,61 @@ -# # Fast Whisper inference using dynamic batching - -# In this example, we demonstrate how to run [dynamically batched inference](https://modal.com/docs/guide/dynamic-batching) -# for OpenAI's speech recognition model, [Whisper](https://openai.com/index/whisper/), on Modal. -# Batching multiple audio samples together or batching chunks of a single audio sample can help to achieve a 2.8x increase -# in inference throughput on an A10G! - -# We will be running the [Whisper Large V3](https://huggingface.co/openai/whisper-large-v3) model. -# To run [any of the other HuggingFace Whisper models](https://huggingface.co/models?search=openai/whisper), -# simply replace the `MODEL_NAME` and `MODEL_REVISION` variables. - -# ## Setup - -# Let's start by importing the Modal client and defining the model that we want to serve. - - +# # Fast Whisper inference with vLLM +# +# Deploy: +# ```shell +# modal deploy batched_whisper.py +# ``` +# +# Query the endpoint: +# ```shell +# curl -X POST "https://sb-modal-ws--asr-whisper-large-v3-salt-model-transcribe.modal.run" \ +# --data-binary @audio.wav \ +# -H "Content-Type: application/octet-stream" +# ``` + +import io from typing import Optional - import modal from fastapi import Request MODEL_NAME = "Sunbird/asr-whisper-large-v3-salt" -# cache model weights with Modal Volumes +# Cache model weights with Modal Volumes HF_CACHE_DIR = "/root/.cache/huggingface" +VLLM_CACHE_DIR = "/root/.cache/vllm" hf_cache_vol = modal.Volume.from_name("huggingface-cache", create_if_missing=True) +vllm_cache_vol = modal.Volume.from_name("vllm-cache", create_if_missing=True) # ## Define a container image - -# We’ll start with Modal's baseline `debian_slim` image and install the relevant libraries. - image = ( - modal.Image.debian_slim(python_version="3.11") + modal.Image.debian_slim(python_version="3.12") .apt_install("ffmpeg") .uv_pip_install( - "torch==2.5.1", - "transformers==4.47.1", + "torch==2.9.0", + "transformers==4.56.0", "huggingface-hub==0.36.0", "librosa==0.10.2", "soundfile==0.12.1", "accelerate==1.2.1", "datasets==3.2.0", - "torchaudio==2.5.1", - "fastapi==0.115.6", - "python-multipart==0.0.20", + "torchaudio==2.9.0", + "fastapi[standard]", + "python-multipart", + "vllm==0.12.0", ) .env({"HF_XET_HIGH_PERFORMANCE": "1", "HF_HUB_CACHE": HF_CACHE_DIR}) ) +# Import libraries within image context +with image.imports(): + import time + import librosa + from vllm import LLM, SamplingParams + app = modal.App( "asr-whisper-large-v3-salt", image=image, secrets=[modal.Secret.from_name("huggingface-read")], - volumes={HF_CACHE_DIR: hf_cache_vol}, + volumes={HF_CACHE_DIR: hf_cache_vol, VLLM_CACHE_DIR: vllm_cache_vol}, ) # ## Caching the model weights @@ -87,20 +91,32 @@ def download_model(): gpu="a10g", # Try using an A100 or H100 if you've got a large model or need big batches! max_containers=10, # default max GPUs for Modal's free tier scaledown_window=60 * 3, + # enable_memory_snapshot=True, ) +@modal.concurrent(max_inputs=10) class Model: @modal.enter() def load_model(self): - import torch - from transformers import pipeline - - # Create a pipeline for preprocessing and transcribing speech data - self.pipeline = pipeline( - "automatic-speech-recognition", - model=MODEL_NAME, - device="cuda", - torch_dtype=torch.float16, + # import torch + # from transformers import pipeline + + # # Create a pipeline for preprocessing and transcribing speech data + # self.pipeline = pipeline( + # "automatic-speech-recognition", + # model=MODEL_NAME, + # device="cuda", + # torch_dtype=torch.float16, + # ) + print("Loading Whisper model with vLLM...") + self.model = LLM( + MODEL_NAME, + enforce_eager=True, + gpu_memory_utilization=0.5, + max_model_len=448, + max_num_seqs=5, + limit_mm_per_prompt={"audio": 1}, ) + print("✅ Model loaded successfully!") # @modal.batched(max_batch_size=64, wait_ms=1000) # def transcribe(self, audio_samples): @@ -130,28 +146,33 @@ async def transcribe(self, request: Request): """ Web endpoint that accepts audio bytes and returns the transcription. """ - import time - data = await request.body() - generate_kwargs = { - # "language": 'English', - "task": "transcribe", - "num_beams": 1, - "return_timestamps": True, - } - + + # Load audio from bytes + audio, sr = librosa.load(io.BytesIO(data), sr=16000) + start = time.monotonic_ns() - transcriptions = self.pipeline( - [data], - batch_size=1, - generate_kwargs=generate_kwargs, - ) - end = time.monotonic_ns() - print( - f"Transcribed in {round((end - start) / 1e9, 2)}s" - ) - return {"text": transcriptions[0]["text"]} + # Whisper prompt format + prompt = "<|startoftranscript|>" + + # Prepare input with multimodal data + inputs = { + "prompt": prompt, + "multi_modal_data": { + "audio": [(audio, sr)] + } + } + + sampling_params = SamplingParams(temperature=0.0, max_tokens=256) + + # Use vLLM generate + outputs = self.model.generate([inputs], sampling_params=sampling_params) + + end = time.monotonic_ns() + print(f"Transcribed in {round((end - start) / 1e9, 2)}s") + + return {"text": outputs[0].outputs[0].text} # ## Transcribe a dataset From ffe29577c0d77b31912eae9a875afe0d7dc73a15 Mon Sep 17 00:00:00 2001 From: huwenjie333 Date: Wed, 4 Feb 2026 22:48:33 +0700 Subject: [PATCH 5/8] Revert "update to vLLM" This reverts commit 718965fbf387298f01a7765023c67ea277268c92. --- .../batched_whisper.py | 133 ++++++++---------- 1 file changed, 56 insertions(+), 77 deletions(-) diff --git a/modal-deploy/asr-whisper-large-v3-salt/batched_whisper.py b/modal-deploy/asr-whisper-large-v3-salt/batched_whisper.py index e49355a0..99e0147a 100644 --- a/modal-deploy/asr-whisper-large-v3-salt/batched_whisper.py +++ b/modal-deploy/asr-whisper-large-v3-salt/batched_whisper.py @@ -1,61 +1,57 @@ -# # Fast Whisper inference with vLLM -# -# Deploy: -# ```shell -# modal deploy batched_whisper.py -# ``` -# -# Query the endpoint: -# ```shell -# curl -X POST "https://sb-modal-ws--asr-whisper-large-v3-salt-model-transcribe.modal.run" \ -# --data-binary @audio.wav \ -# -H "Content-Type: application/octet-stream" -# ``` - -import io +# # Fast Whisper inference using dynamic batching + +# In this example, we demonstrate how to run [dynamically batched inference](https://modal.com/docs/guide/dynamic-batching) +# for OpenAI's speech recognition model, [Whisper](https://openai.com/index/whisper/), on Modal. +# Batching multiple audio samples together or batching chunks of a single audio sample can help to achieve a 2.8x increase +# in inference throughput on an A10G! + +# We will be running the [Whisper Large V3](https://huggingface.co/openai/whisper-large-v3) model. +# To run [any of the other HuggingFace Whisper models](https://huggingface.co/models?search=openai/whisper), +# simply replace the `MODEL_NAME` and `MODEL_REVISION` variables. + +# ## Setup + +# Let's start by importing the Modal client and defining the model that we want to serve. + + from typing import Optional + import modal from fastapi import Request MODEL_NAME = "Sunbird/asr-whisper-large-v3-salt" -# Cache model weights with Modal Volumes +# cache model weights with Modal Volumes HF_CACHE_DIR = "/root/.cache/huggingface" -VLLM_CACHE_DIR = "/root/.cache/vllm" hf_cache_vol = modal.Volume.from_name("huggingface-cache", create_if_missing=True) -vllm_cache_vol = modal.Volume.from_name("vllm-cache", create_if_missing=True) # ## Define a container image + +# We’ll start with Modal's baseline `debian_slim` image and install the relevant libraries. + image = ( - modal.Image.debian_slim(python_version="3.12") + modal.Image.debian_slim(python_version="3.11") .apt_install("ffmpeg") .uv_pip_install( - "torch==2.9.0", - "transformers==4.56.0", + "torch==2.5.1", + "transformers==4.47.1", "huggingface-hub==0.36.0", "librosa==0.10.2", "soundfile==0.12.1", "accelerate==1.2.1", "datasets==3.2.0", - "torchaudio==2.9.0", - "fastapi[standard]", - "python-multipart", - "vllm==0.12.0", + "torchaudio==2.5.1", + "fastapi==0.115.6", + "python-multipart==0.0.20", ) .env({"HF_XET_HIGH_PERFORMANCE": "1", "HF_HUB_CACHE": HF_CACHE_DIR}) ) -# Import libraries within image context -with image.imports(): - import time - import librosa - from vllm import LLM, SamplingParams - app = modal.App( "asr-whisper-large-v3-salt", image=image, secrets=[modal.Secret.from_name("huggingface-read")], - volumes={HF_CACHE_DIR: hf_cache_vol, VLLM_CACHE_DIR: vllm_cache_vol}, + volumes={HF_CACHE_DIR: hf_cache_vol}, ) # ## Caching the model weights @@ -91,32 +87,20 @@ def download_model(): gpu="a10g", # Try using an A100 or H100 if you've got a large model or need big batches! max_containers=10, # default max GPUs for Modal's free tier scaledown_window=60 * 3, - # enable_memory_snapshot=True, ) -@modal.concurrent(max_inputs=10) class Model: @modal.enter() def load_model(self): - # import torch - # from transformers import pipeline - - # # Create a pipeline for preprocessing and transcribing speech data - # self.pipeline = pipeline( - # "automatic-speech-recognition", - # model=MODEL_NAME, - # device="cuda", - # torch_dtype=torch.float16, - # ) - print("Loading Whisper model with vLLM...") - self.model = LLM( - MODEL_NAME, - enforce_eager=True, - gpu_memory_utilization=0.5, - max_model_len=448, - max_num_seqs=5, - limit_mm_per_prompt={"audio": 1}, + import torch + from transformers import pipeline + + # Create a pipeline for preprocessing and transcribing speech data + self.pipeline = pipeline( + "automatic-speech-recognition", + model=MODEL_NAME, + device="cuda", + torch_dtype=torch.float16, ) - print("✅ Model loaded successfully!") # @modal.batched(max_batch_size=64, wait_ms=1000) # def transcribe(self, audio_samples): @@ -146,33 +130,28 @@ async def transcribe(self, request: Request): """ Web endpoint that accepts audio bytes and returns the transcription. """ + import time + data = await request.body() - - # Load audio from bytes - audio, sr = librosa.load(io.BytesIO(data), sr=16000) - - start = time.monotonic_ns() - - # Whisper prompt format - prompt = "<|startoftranscript|>" - - # Prepare input with multimodal data - inputs = { - "prompt": prompt, - "multi_modal_data": { - "audio": [(audio, sr)] - } + generate_kwargs = { + # "language": 'English', + "task": "transcribe", + "num_beams": 1, + "return_timestamps": True, } - - sampling_params = SamplingParams(temperature=0.0, max_tokens=256) - - # Use vLLM generate - outputs = self.model.generate([inputs], sampling_params=sampling_params) - - end = time.monotonic_ns() - print(f"Transcribed in {round((end - start) / 1e9, 2)}s") - return {"text": outputs[0].outputs[0].text} + start = time.monotonic_ns() + transcriptions = self.pipeline( + [data], + batch_size=1, + generate_kwargs=generate_kwargs, + ) + end = time.monotonic_ns() + print( + f"Transcribed in {round((end - start) / 1e9, 2)}s" + ) + + return {"text": transcriptions[0]["text"]} # ## Transcribe a dataset From a75c1217e0f6dece5304365be17ea6e04f2c5064 Mon Sep 17 00:00:00 2001 From: huwenjie333 Date: Wed, 4 Feb 2026 23:14:21 +0700 Subject: [PATCH 6/8] clean up and add usage --- .../batched_whisper.py | 70 ++++++++----------- 1 file changed, 31 insertions(+), 39 deletions(-) diff --git a/modal-deploy/asr-whisper-large-v3-salt/batched_whisper.py b/modal-deploy/asr-whisper-large-v3-salt/batched_whisper.py index 99e0147a..d7887800 100644 --- a/modal-deploy/asr-whisper-large-v3-salt/batched_whisper.py +++ b/modal-deploy/asr-whisper-large-v3-salt/batched_whisper.py @@ -1,18 +1,16 @@ -# # Fast Whisper inference using dynamic batching - -# In this example, we demonstrate how to run [dynamically batched inference](https://modal.com/docs/guide/dynamic-batching) -# for OpenAI's speech recognition model, [Whisper](https://openai.com/index/whisper/), on Modal. -# Batching multiple audio samples together or batching chunks of a single audio sample can help to achieve a 2.8x increase -# in inference throughput on an A10G! - -# We will be running the [Whisper Large V3](https://huggingface.co/openai/whisper-large-v3) model. -# To run [any of the other HuggingFace Whisper models](https://huggingface.co/models?search=openai/whisper), -# simply replace the `MODEL_NAME` and `MODEL_REVISION` variables. - -# ## Setup - -# Let's start by importing the Modal client and defining the model that we want to serve. - +# Deploy the asr-whisper-large-v3-salt model with Modal: +# +# ```shell +# modal deploy batched_whisper.py +# ``` +# +# And query the endpoint with: +# +# ```shell +# python client.py \ +# --url https://sb-modal-ws--asr-whisper-large-v3-salt-model-transcribe.modal.run \ +# --audio "../../sunflower-ultravox-vllm/audios/context_eng_6.wav" +# ``` from typing import Optional @@ -26,9 +24,6 @@ hf_cache_vol = modal.Volume.from_name("huggingface-cache", create_if_missing=True) # ## Define a container image - -# We’ll start with Modal's baseline `debian_slim` image and install the relevant libraries. - image = ( modal.Image.debian_slim(python_version="3.11") .apt_install("ffmpeg") @@ -58,8 +53,6 @@ # We'll define a function to download the model and cache it in a volume. # You can `modal run batched_whisper.py::download_model` against this function prior to deploying the App. - - @app.function() def download_model(): from huggingface_hub import snapshot_download @@ -81,8 +74,6 @@ def download_model(): # we start a new container. For more on storing model weights on Modal, see # [this guide](https://modal.com/docs/guide/model-weights). - - @app.cls( gpu="a10g", # Try using an A100 or H100 if you've got a large model or need big batches! max_containers=10, # default max GPUs for Modal's free tier @@ -151,7 +142,8 @@ async def transcribe(self, request: Request): f"Transcribed in {round((end - start) / 1e9, 2)}s" ) - return {"text": transcriptions[0]["text"]} + return transcriptions + # return {"text": transcriptions[0]["text"]} # ## Transcribe a dataset @@ -163,17 +155,17 @@ async def transcribe(self, request: Request): # This allows us to invoke the batched transcription method on each audio sample in parallel. -@app.function() -async def transcribe_hf_dataset(dataset_name): - from datasets import load_dataset +# @app.function() +# async def transcribe_hf_dataset(dataset_name): +# from datasets import load_dataset - print("📂 Loading dataset", dataset_name) - ds = load_dataset(dataset_name, "multispeaker-eng", split="test") - print("📂 Dataset loaded") - batched_whisper = Model() - print("📣 Sending data for transcription") - async for transcription in batched_whisper.transcribe.map.aio(ds["audio"]): - yield transcription +# print("📂 Loading dataset", dataset_name) +# ds = load_dataset(dataset_name, "multispeaker-eng", split="test") +# print("📂 Dataset loaded") +# batched_whisper = Model() +# print("📣 Sending data for transcription") +# async for transcription in batched_whisper.transcribe.map.aio(ds["audio"]): +# yield transcription # ## Run the model @@ -182,9 +174,9 @@ async def transcribe_hf_dataset(dataset_name): # to run the transcription. You can run this locally with `modal run batched_whisper.py`. -@app.local_entrypoint() -async def main(dataset_name: Optional[str] = None): - if dataset_name is None: - dataset_name = "Sunbird/salt" - for result in transcribe_hf_dataset.remote_gen(dataset_name): - print(result["text"]) +# @app.local_entrypoint() +# async def main(dataset_name: Optional[str] = None): +# if dataset_name is None: +# dataset_name = "Sunbird/salt" +# for result in transcribe_hf_dataset.remote_gen(dataset_name): +# print(result["text"]) From 61f8f3304ed517aee57cb24777477159286f32a4 Mon Sep 17 00:00:00 2001 From: huwenjie333 Date: Tue, 10 Feb 2026 22:49:18 +0700 Subject: [PATCH 7/8] add language code option --- .../batched_whisper.py | 73 ++++++++++++++++++- 1 file changed, 71 insertions(+), 2 deletions(-) diff --git a/modal-deploy/asr-whisper-large-v3-salt/batched_whisper.py b/modal-deploy/asr-whisper-large-v3-salt/batched_whisper.py index d7887800..ced5af88 100644 --- a/modal-deploy/asr-whisper-large-v3-salt/batched_whisper.py +++ b/modal-deploy/asr-whisper-large-v3-salt/batched_whisper.py @@ -11,6 +11,14 @@ # --url https://sb-modal-ws--asr-whisper-large-v3-salt-model-transcribe.modal.run \ # --audio "../../sunflower-ultravox-vllm/audios/context_eng_6.wav" # ``` +# +# Or using `curl`: +# +# ```shell +curl -X POST "https://sb-modal-ws--asr-whisper-large-v3-salt-model-transcribe.modal.run?language=eng" \ + --header "Content-Type: application/octet-stream" \ + --data-binary "@../../sunflower-ultravox-vllm/audios/context_eng_6.wav" +# ``` from typing import Optional @@ -83,6 +91,7 @@ class Model: @modal.enter() def load_model(self): import torch + import transformers from transformers import pipeline # Create a pipeline for preprocessing and transcribing speech data @@ -93,6 +102,8 @@ def load_model(self): torch_dtype=torch.float16, ) + self.processor = transformers.WhisperProcessor.from_pretrained(MODEL_NAME) + # @modal.batched(max_batch_size=64, wait_ms=1000) # def transcribe(self, audio_samples): # import time @@ -116,8 +127,65 @@ def load_model(self): # ) # return transcriptions + def get_language_code(self, language: str, processor) -> str: + """ + Returns the correct language code for a given language using the provided processor. + + Parameters: + language (str): The name or code of the language (e.g., "English", "eng", "Luganda", "lug", etc.). + processor: An object that contains a tokenizer used to decode language ID tokens. + + Returns: + str: The corresponding language code. + + Raises: + ValueError: If the language is not supported. + """ + language_codes = { + "English": "eng", + "Luganda": "lug", + "Runyankole": "nyn", + "Acholi": "ach", + "Ateso": "teo", + "Lugbara": "lgg", + "Swahili": "swa", + "Kinyarwanda": "kin", + "Lusoga": "xog", + "Lumasaba": "myx", + } + + code_to_language = {v: k for k, v in language_codes.items()} + standardized_language = ( + language.capitalize() if len(language) > 3 else language.lower() + ) + + if standardized_language in language_codes: + code = language_codes[standardized_language] + elif standardized_language in code_to_language: + code = standardized_language + else: + raise ValueError(f"Language '{language}' is not supported.") + + language_id_tokens = { + "eng": 50259, + "ach": 50357, + "lgg": 50356, + "lug": 50355, + "nyn": 50354, + "teo": 50353, + "xog": 50352, + "kin": 50350, + "myx": 50349, + "swa": 50318, + } + + token = language_id_tokens[code] + language_code = processor.tokenizer.decode(token)[2:-2] + + return language_code + @modal.fastapi_endpoint(docs=True, method="POST") - async def transcribe(self, request: Request): + async def transcribe(self, request: Request, language: Optional[str] = None): """ Web endpoint that accepts audio bytes and returns the transcription. """ @@ -125,11 +193,12 @@ async def transcribe(self, request: Request): data = await request.body() generate_kwargs = { - # "language": 'English', "task": "transcribe", "num_beams": 1, "return_timestamps": True, } + if language: + generate_kwargs["language"] = self.get_language_code(language, self.processor) start = time.monotonic_ns() transcriptions = self.pipeline( From 2b0ef03b95550fbeda339663312393be1f0506db Mon Sep 17 00:00:00 2001 From: Patrick Walukagga Date: Wed, 11 Feb 2026 10:34:41 +0300 Subject: [PATCH 8/8] Add optional language parameter to ASR STT --- .../batched_whisper.py | 23 ++++++++++++++++--- .../asr-whisper-large-v3-salt/client.py | 8 ++++++- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/modal-deploy/asr-whisper-large-v3-salt/batched_whisper.py b/modal-deploy/asr-whisper-large-v3-salt/batched_whisper.py index ced5af88..42c18098 100644 --- a/modal-deploy/asr-whisper-large-v3-salt/batched_whisper.py +++ b/modal-deploy/asr-whisper-large-v3-salt/batched_whisper.py @@ -12,12 +12,29 @@ # --audio "../../sunflower-ultravox-vllm/audios/context_eng_6.wav" # ``` # +# With an optional language argument: +# +# ```shell +# python client.py \ +# --url https://sb-modal-ws--asr-whisper-large-v3-salt-model-transcribe.modal.run \ +# --audio "../../sunflower-ultravox-vllm/audios/context_eng_6.wav" \ +# --language eng +# ``` +# # Or using `curl`: # # ```shell -curl -X POST "https://sb-modal-ws--asr-whisper-large-v3-salt-model-transcribe.modal.run?language=eng" \ - --header "Content-Type: application/octet-stream" \ - --data-binary "@../../sunflower-ultravox-vllm/audios/context_eng_6.wav" +# curl -X POST "https://sb-modal-ws--asr-whisper-large-v3-salt-model-transcribe.modal.run" \ +# --header "Content-Type: application/octet-stream" \ +# --data-binary "@../../sunflower-ultravox-vllm/audios/context_eng_6.wav" +# ``` +# +# With an optional language query parameter: +# +# ```shell +# curl -X POST "https://sb-modal-ws--asr-whisper-large-v3-salt-model-transcribe.modal.run?language=eng" \ +# --header "Content-Type: application/octet-stream" \ +# --data-binary "@../../sunflower-ultravox-vllm/audios/context_eng_6.wav" # ``` from typing import Optional diff --git a/modal-deploy/asr-whisper-large-v3-salt/client.py b/modal-deploy/asr-whisper-large-v3-salt/client.py index 0295e541..6dd0ee7b 100644 --- a/modal-deploy/asr-whisper-large-v3-salt/client.py +++ b/modal-deploy/asr-whisper-large-v3-salt/client.py @@ -7,6 +7,7 @@ def main(): parser = argparse.ArgumentParser(description="Whisper Client") parser.add_argument("--audio", type=str, required=True, help="Path to audio file") parser.add_argument("--url", type=str, required=True, help="URL of the Modal endpoint") + parser.add_argument("--language", type=str, default=None, help="Optional language code for transcription") args = parser.parse_args() if not os.path.exists(args.audio): @@ -19,10 +20,15 @@ def main(): print(f"Sending {len(audio_data)} bytes of audio data to {args.url}...") # Send audio data as raw request body + params = {} + if args.language: + params["language"] = args.language + response = requests.post( args.url, data=audio_data, - headers={"Content-Type": "application/octet-stream"} + headers={"Content-Type": "application/octet-stream"}, + params=params, ) if response.status_code == 200: