From aa10d4a4952bbe595e6f129916fa985ad59aada0 Mon Sep 17 00:00:00 2001 From: huwenjie333 Date: Wed, 3 Dec 2025 15:42:59 +0300 Subject: [PATCH 1/9] init --- modal-deploy/vllm_inference.py | 275 +++++++++++++++++++++++++++++++++ 1 file changed, 275 insertions(+) create mode 100644 modal-deploy/vllm_inference.py diff --git a/modal-deploy/vllm_inference.py b/modal-deploy/vllm_inference.py new file mode 100644 index 00000000..6b59ce8d --- /dev/null +++ b/modal-deploy/vllm_inference.py @@ -0,0 +1,275 @@ +# --- +# pytest: false +# --- + +# # Run OpenAI-compatible LLM inference with Qwen3-8B and vLLM + +# LLMs do more than just model language: they chat, they produce JSON and XML, they run code, and more. +# This has complicated their interface far beyond "text-in, text-out". +# OpenAI's API has emerged as a standard for that interface, +# and it is supported by open source LLM serving frameworks like [vLLM](https://docs.vllm.ai/en/latest/). + +# In this example, we show how to run a vLLM server in OpenAI-compatible mode on Modal. + +# Our examples repository also includes scripts for running clients and load-testing for OpenAI-compatible APIs +# [here](https://github.com/modal-labs/modal-examples/tree/main/06_gpu_and_ml/llm-serving/openai_compatible). + +# You can find a (somewhat out-of-date) video walkthrough of this example and the related scripts on the Modal YouTube channel +# [here](https://www.youtube.com/watch?v=QmY_7ePR1hM). + +# ## Set up the container image + +# Our first order of business is to define the environment our server will run in: +# the [container `Image`](https://modal.com/docs/guide/custom-container). +# vLLM can be installed with `pip`, since Modal [provides the CUDA drivers](https://modal.com/docs/guide/cuda). + +import json +from typing import Any + +import aiohttp +import modal + +vllm_image = ( + modal.Image.from_registry("nvidia/cuda:12.8.0-devel-ubuntu22.04", add_python="3.12") + .entrypoint([]) + .uv_pip_install( + "vllm==0.11.2", + "huggingface-hub==0.36.0", + "flashinfer-python==0.5.2", + ) + .env({"HF_XET_HIGH_PERFORMANCE": "1"}) # faster model transfers +) + +# ## Download the model weights + +# We'll be running a pretrained foundation model -- Qwen's Qwen3-8B. +# It is trained with reasoning capabilities, which allow it to +# enhance the quality of its generated responses. + +# We'll use an FP8 (eight-bit floating-point) post-training-quantized variant: Qwen/Qwen3-8B-FP8. +# Native hardware support for FP8 formats in [Tensor Cores](https://modal.com/gpu-glossary/device-hardware/tensor-core) +# is limited to the latest [Streaming Multiprocessor architectures](https://modal.com/gpu-glossary/device-hardware/streaming-multiprocessor-architecture), +# like those of Modal's [Hopper H100/H200 and Blackwell B200 GPUs](https://modal.com/blog/announcing-h200-b200). + +# You can swap this model out for another by changing the strings below. +# A single H100 GPU has enough VRAM to store an 8,000,000,000 parameter model, +# like Qwen3-8B, in eight bit precision, along with a very large KV cache. + + +MODEL_NAME = "Qwen/Qwen3-8B-FP8" +MODEL_REVISION = "220b46e3b2180893580a4454f21f22d3ebb187d3" # avoid nasty surprises when repos update! + +# Although vLLM will download weights from Hugging Face on-demand, +# we want to cache them so we don't do it every time our server starts. +# We'll use [Modal Volumes](https://modal.com/docs/guide/volumes) for our cache. +# Modal Volumes are essentially a "shared disk" that all Modal Functions can access like it's a regular disk. For more on storing model weights on Modal, see +# [this guide](https://modal.com/docs/guide/model-weights). + + +hf_cache_vol = modal.Volume.from_name("huggingface-cache", create_if_missing=True) + +# We'll also cache some of vLLM's JIT compilation artifacts in a Modal Volume. + +vllm_cache_vol = modal.Volume.from_name("vllm-cache", create_if_missing=True) + +# ## Configuring vLLM + +# ### Trading off fast boots and token generation performance + +# vLLM has embraced dynamic and just-in-time compilation to eke out additional performance without having to write too many custom kernels, +# e.g. via the Torch compiler and CUDA graph capture. +# These compilation features incur latency at startup in exchange for lowered latency and higher throughput during generation. +# We make this trade-off controllable with the `FAST_BOOT` variable below. + +FAST_BOOT = True + +# If you're running an LLM service that frequently scales from 0 (frequent ["cold starts"](https://modal.com/docs/guide/cold-start)) +# then you'll want to set this to `True`. + +# If you're running an LLM service that usually has multiple replicas running, then set this to `False` for improved performance. + +# See the code below for details on the parameters that `FAST_BOOT` controls. + +# For more on the performance you can expect when serving your own LLMs, see +# [our LLM engine performance benchmarks](https://modal.com/llm-almanac). + +# ## Build a vLLM engine and serve it + +# The function below spawns a vLLM instance listening at port 8000, serving requests to our model. +# We wrap it in the [`@modal.web_server` decorator](https://modal.com/docs/guide/webhooks#non-asgi-web-servers) +# to connect it to the Internet. + +# The server runs in an independent process, via `subprocess.Popen`, and only starts accepting requests +# once the model is spun up and the `serve` function returns. + + +app = modal.App("example-vllm-inference") + +N_GPU = 1 +MINUTES = 60 # seconds +VLLM_PORT = 8000 + + +@app.function( + image=vllm_image, + gpu=f"H100:{N_GPU}", + scaledown_window=15 * MINUTES, # how long should we stay up with no requests? + timeout=10 * MINUTES, # how long should we wait for container start? + volumes={ + "/root/.cache/huggingface": hf_cache_vol, + "/root/.cache/vllm": vllm_cache_vol, + }, +) +@modal.concurrent( # how many requests can one replica handle? tune carefully! + max_inputs=32 +) +@modal.web_server(port=VLLM_PORT, startup_timeout=10 * MINUTES) +def serve(): + import subprocess + + cmd = [ + "vllm", + "serve", + "--uvicorn-log-level=info", + MODEL_NAME, + "--revision", + MODEL_REVISION, + "--served-model-name", + MODEL_NAME, + "llm", + "--host", + "0.0.0.0", + "--port", + str(VLLM_PORT), + ] + + # enforce-eager disables both Torch compilation and CUDA graph capture + # default is no-enforce-eager. see the --compilation-config flag for tighter control + cmd += ["--enforce-eager" if FAST_BOOT else "--no-enforce-eager"] + + # assume multiple GPUs are for splitting up large matrix multiplications + cmd += ["--tensor-parallel-size", str(N_GPU)] + + print(cmd) + + subprocess.Popen(" ".join(cmd), shell=True) + + +# ## Deploy the server + +# To deploy the API on Modal, just run +# ```bash +# modal deploy vllm_inference.py +# ``` + +# This will create a new app on Modal, build the container image for it if it hasn't been built yet, +# and deploy the app. + +# ## Interact with the server + +# Once it is deployed, you'll see a URL appear in the command line, +# something like `https://your-workspace-name--example-vllm-inference-serve.modal.run`. + +# You can find [interactive Swagger UI docs](https://swagger.io/tools/swagger-ui/) +# at the `/docs` route of that URL, i.e. `https://your-workspace-name--example-vllm-inference-serve.modal.run/docs`. +# These docs describe each route and indicate the expected input and output +# and translate requests into `curl` commands. + +# For simple routes like `/health`, which checks whether the server is responding, +# you can even send a request directly from the docs. + +# To interact with the API programmatically in Python, we recommend the `openai` library. + +# See the `client.py` script in the examples repository +# [here](https://github.com/modal-labs/modal-examples/tree/main/06_gpu_and_ml/llm-serving/openai_compatible) +# to take it for a spin: + +# ```bash +# # pip install openai==1.76.0 +# python openai_compatible/client.py +# ``` + + +# ## Testing the server + +# To make it easier to test the server setup, we also include a `local_entrypoint` +# that does a healthcheck and then hits the server. + +# If you execute the command + +# ```bash +# modal run vllm_inference.py +# ``` + +# a fresh replica of the server will be spun up on Modal while +# the code below executes on your local machine. + +# Think of this like writing simple tests inside of the `if __name__ == "__main__"` +# block of a Python script, but for cloud deployments! + + +@app.local_entrypoint() +async def test(test_timeout=10 * MINUTES, content=None, twice=True): + url = serve.get_web_url() + + system_prompt = { + "role": "system", + "content": "You are a pirate who can't help but drop sly reminders that he went to Harvard.", + } + if content is None: + content = "Explain the singular value decomposition." + + messages = [ # OpenAI chat format + system_prompt, + {"role": "user", "content": content}, + ] + + async with aiohttp.ClientSession(base_url=url) as session: + print(f"Running health check for server at {url}") + async with session.get("/health", timeout=test_timeout - 1 * MINUTES) as resp: + up = resp.status == 200 + assert up, f"Failed health check for server at {url}" + print(f"Successful health check for server at {url}") + + print(f"Sending messages to {url}:", *messages, sep="\n\t") + await _send_request(session, "llm", messages) + if twice: + messages[0]["content"] = "You are Jar Jar Binks." + print(f"Sending messages to {url}:", *messages, sep="\n\t") + await _send_request(session, "llm", messages) + + +async def _send_request( + session: aiohttp.ClientSession, model: str, messages: list +) -> None: + # `stream=True` tells an OpenAI-compatible backend to stream chunks + payload: dict[str, Any] = {"messages": messages, "model": model, "stream": True} + + headers = {"Content-Type": "application/json", "Accept": "text/event-stream"} + + async with session.post( + "/v1/chat/completions", json=payload, headers=headers, timeout=1 * MINUTES + ) as resp: + async for raw in resp.content: + resp.raise_for_status() + # extract new content and stream it + line = raw.decode().strip() + if not line or line == "data: [DONE]": + continue + if line.startswith("data: "): # SSE prefix + line = line[len("data: ") :] + + chunk = json.loads(line) + assert ( + chunk["object"] == "chat.completion.chunk" + ) # or something went horribly wrong + print(chunk["choices"][0]["delta"]["content"], end="") + print() + + +# We also include a basic example of a load-testing setup using +# `locust` in the `load_test.py` script [here](https://github.com/modal-labs/modal-examples/tree/main/06_gpu_and_ml/llm-serving/openai_compatible): + +# ```bash +# modal run openai_compatible/load_test.py +# ``` From beb694b902f41d1eaa0ae04f4b542f76d304d9f2 Mon Sep 17 00:00:00 2001 From: huwenjie333 Date: Wed, 3 Dec 2025 16:07:41 +0300 Subject: [PATCH 2/9] Qwen3-8B-FP8 --- modal-deploy/vllm_inference.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modal-deploy/vllm_inference.py b/modal-deploy/vllm_inference.py index 6b59ce8d..52390aa4 100644 --- a/modal-deploy/vllm_inference.py +++ b/modal-deploy/vllm_inference.py @@ -103,7 +103,7 @@ # once the model is spun up and the `serve` function returns. -app = modal.App("example-vllm-inference") +app = modal.App("Qwen3-8B-FP8") N_GPU = 1 MINUTES = 60 # seconds @@ -112,8 +112,8 @@ @app.function( image=vllm_image, - gpu=f"H100:{N_GPU}", - scaledown_window=15 * MINUTES, # how long should we stay up with no requests? + gpu=f"L4:{N_GPU}", + scaledown_window=2 * MINUTES, # how long should we stay up with no requests? timeout=10 * MINUTES, # how long should we wait for container start? volumes={ "/root/.cache/huggingface": hf_cache_vol, From 95272fcaa9d3f45e3fd1fce5f5d562419b931d2d Mon Sep 17 00:00:00 2001 From: huwenjie333 Date: Wed, 3 Dec 2025 17:13:38 +0300 Subject: [PATCH 3/9] Sunflower-14B-FP8 --- modal-deploy/vllm_inference.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/modal-deploy/vllm_inference.py b/modal-deploy/vllm_inference.py index 52390aa4..f8fef22c 100644 --- a/modal-deploy/vllm_inference.py +++ b/modal-deploy/vllm_inference.py @@ -56,8 +56,9 @@ # like Qwen3-8B, in eight bit precision, along with a very large KV cache. -MODEL_NAME = "Qwen/Qwen3-8B-FP8" -MODEL_REVISION = "220b46e3b2180893580a4454f21f22d3ebb187d3" # avoid nasty surprises when repos update! +MODEL_NAME = "Sunbird/Sunflower-14B-FP8" +# MODEL_NAME = "Qwen/Qwen3-8B-FP8" +# MODEL_REVISION = "220b46e3b2180893580a4454f21f22d3ebb187d3" # avoid nasty surprises when repos update! # Although vLLM will download weights from Hugging Face on-demand, # we want to cache them so we don't do it every time our server starts. @@ -103,7 +104,7 @@ # once the model is spun up and the `serve` function returns. -app = modal.App("Qwen3-8B-FP8") +app = modal.App("Sunflower-14B-FP8") N_GPU = 1 MINUTES = 60 # seconds @@ -112,13 +113,14 @@ @app.function( image=vllm_image, - gpu=f"L4:{N_GPU}", + gpu=f"A100-40GB:{N_GPU}", scaledown_window=2 * MINUTES, # how long should we stay up with no requests? timeout=10 * MINUTES, # how long should we wait for container start? volumes={ "/root/.cache/huggingface": hf_cache_vol, "/root/.cache/vllm": vllm_cache_vol, }, + secrets=[modal.Secret.from_name("huggingface-read")], ) @modal.concurrent( # how many requests can one replica handle? tune carefully! max_inputs=32 @@ -132,8 +134,8 @@ def serve(): "serve", "--uvicorn-log-level=info", MODEL_NAME, - "--revision", - MODEL_REVISION, + # "--revision", + # MODEL_REVISION, "--served-model-name", MODEL_NAME, "llm", @@ -209,7 +211,7 @@ def serve(): @app.local_entrypoint() -async def test(test_timeout=10 * MINUTES, content=None, twice=True): +async def test(test_timeout=10 * MINUTES, content=None, twice=False): url = serve.get_web_url() system_prompt = { From 6362b0043ca6495fd5b3530fb9809de1a7bc2d6f Mon Sep 17 00:00:00 2001 From: huwenjie333 Date: Fri, 5 Dec 2025 10:44:01 +0300 Subject: [PATCH 4/9] default client script --- modal-deploy/client.py | 235 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 235 insertions(+) create mode 100644 modal-deploy/client.py diff --git a/modal-deploy/client.py b/modal-deploy/client.py new file mode 100644 index 00000000..a5e98ceb --- /dev/null +++ b/modal-deploy/client.py @@ -0,0 +1,235 @@ +"""This simple script shows how to interact with an OpenAI-compatible server from a client.""" + +import argparse + +import modal +from openai import OpenAI + + +class Colors: + """ANSI color codes""" + + GREEN = "\033[0;32m" + RED = "\033[0;31m" + BLUE = "\033[0;34m" + GRAY = "\033[0;90m" + BOLD = "\033[1m" + END = "\033[0m" + + +def get_completion(client, model_id, messages, args): + completion_args = { + "model": model_id, + "messages": messages, + "frequency_penalty": args.frequency_penalty, + "max_tokens": args.max_tokens, + "n": args.n, + "presence_penalty": args.presence_penalty, + "seed": args.seed, + "stop": args.stop, + "stream": args.stream, + "temperature": args.temperature, + "top_p": args.top_p, + } + + completion_args = {k: v for k, v in completion_args.items() if v is not None} + + try: + response = client.chat.completions.create(**completion_args) + return response + except Exception as e: + print(Colors.RED, f"Error during API call: {e}", Colors.END, sep="") + return None + + +def main(): + parser = argparse.ArgumentParser(description="OpenAI Client CLI") + + parser.add_argument( + "--model", + type=str, + default=None, + help="The model to use for completion, defaults to the first available model", + ) + parser.add_argument( + "--workspace", + type=str, + default=None, + help="The workspace where the LLM server app is hosted, defaults to your current Modal workspace", + ) + parser.add_argument( + "--environment", + type=str, + default=None, + help="The environment in your Modal workspace where the LLM server app is hosted, defaults to your current environment", + ) + parser.add_argument( + "--app-name", + type=str, + default="example-vllm-inference", + help="A Modal App serving an OpenAI-compatible API", + ) + parser.add_argument( + "--function-name", + type=str, + default="serve", + help="A Modal Function serving an OpenAI-compatible API. Append `-dev` to use a `modal serve`d Function.", + ) + parser.add_argument( + "--api-key", + type=str, + default="super-secret-key", + help="The API key to use for authentication, set in your api.py", + ) + + # Completion parameters + parser.add_argument("--max-tokens", type=int, default=None) + parser.add_argument("--temperature", type=float, default=0.7) + parser.add_argument("--top-p", type=float, default=0.9) + parser.add_argument("--top-k", type=int, default=0) + parser.add_argument("--frequency-penalty", type=float, default=0) + parser.add_argument("--presence-penalty", type=float, default=0) + parser.add_argument( + "--n", + type=int, + default=1, + help="Number of completions to generate. Streaming and chat mode only support n=1.", + ) + parser.add_argument("--stop", type=str, default=None) + parser.add_argument("--seed", type=int, default=None) + + # Prompting + parser.add_argument( + "--prompt", + type=str, + default="Compose a limerick about baboons and racoons.", + help="The user prompt for the chat completion", + ) + parser.add_argument( + "--system-prompt", + type=str, + default="You are a poetic assistant, skilled in writing satirical doggerel with creative flair.", + help="The system prompt for the chat completion", + ) + + # UI options + parser.add_argument( + "--no-stream", + dest="stream", + action="store_false", + help="Disable streaming of response chunks", + ) + parser.add_argument( + "--chat", action="store_true", help="Enable interactive chat mode" + ) + + args = parser.parse_args() + + client = OpenAI(api_key=args.api_key) + + workspace = args.workspace or modal.config._profile + + environment = args.environment or modal.config.config["environment"] + + prefix = workspace + (f"-{environment}" if environment else "") + + client.base_url = ( + f"https://{prefix}--{args.app_name}-{args.function_name}.modal.run/v1" + ) + + if args.model: + model_id = args.model + print( + Colors.BOLD, + f"šŸ§ : Using model {model_id}. This may trigger a model load on first call!", + Colors.END, + sep="", + ) + else: + print( + Colors.BOLD, + f"šŸ”Ž: Looking up available models on server at {client.base_url}. This may trigger a model load!", + Colors.END, + sep="", + ) + model = client.models.list().data[0] + model_id = model.id + print( + Colors.BOLD, + f"šŸ§ : Using {model_id}", + Colors.END, + sep="", + ) + + messages = [ + { + "role": "system", + "content": args.system_prompt, + } + ] + + print(Colors.BOLD + "šŸ§ : Using system prompt: " + args.system_prompt + Colors.END) + + if args.chat: + print( + Colors.GREEN + + Colors.BOLD + + "\nEntering chat mode. Type 'bye' to end the conversation." + + Colors.END + ) + while True: + user_input = input("\nYou: ") + if user_input.lower() in ["bye"]: + break + + MAX_HISTORY = 10 + if len(messages) > MAX_HISTORY: + messages = messages[:1] + messages[-MAX_HISTORY + 1 :] + + messages.append({"role": "user", "content": user_input}) + + response = get_completion(client, model_id, messages, args) + + if response: + if args.stream: + # only stream assuming n=1 + print(Colors.BLUE + "\nšŸ¤–: ", end="") + assistant_message = "" + for chunk in response: + if chunk.choices[0].delta.content: + content = chunk.choices[0].delta.content + print(content, end="") + assistant_message += content + print(Colors.END) + else: + assistant_message = response.choices[0].message.content + print( + Colors.BLUE + "\nšŸ¤–:" + assistant_message + Colors.END, + sep="", + ) + + messages.append({"role": "assistant", "content": assistant_message}) + else: + messages.append({"role": "user", "content": args.prompt}) + print(Colors.GREEN + f"\nYou: {args.prompt}" + Colors.END) + response = get_completion(client, model_id, messages, args) + if response: + if args.stream: + print(Colors.BLUE + "\nšŸ¤–:", end="") + for chunk in response: + if chunk.choices[0].delta.content: + print(chunk.choices[0].delta.content, end="") + print(Colors.END) + else: + # only case where multiple completions are returned + for i, response in enumerate(response.choices): + print( + Colors.BLUE + + f"\nšŸ¤– Choice {i + 1}:{response.message.content}" + + Colors.END, + sep="", + ) + + +if __name__ == "__main__": + main() From 9700a58fafca76055411cd50dc7a3582020e0b78 Mon Sep 17 00:00:00 2001 From: huwenjie333 Date: Fri, 5 Dec 2025 10:56:50 +0300 Subject: [PATCH 5/9] sunflower client updates --- modal-deploy/client.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modal-deploy/client.py b/modal-deploy/client.py index a5e98ceb..583485d6 100644 --- a/modal-deploy/client.py +++ b/modal-deploy/client.py @@ -66,7 +66,7 @@ def main(): parser.add_argument( "--app-name", type=str, - default="example-vllm-inference", + default="Sunflower-14B-FP8", help="A Modal App serving an OpenAI-compatible API", ) parser.add_argument( @@ -102,13 +102,13 @@ def main(): parser.add_argument( "--prompt", type=str, - default="Compose a limerick about baboons and racoons.", + default="Who are you?", help="The user prompt for the chat completion", ) parser.add_argument( "--system-prompt", type=str, - default="You are a poetic assistant, skilled in writing satirical doggerel with creative flair.", + default="You are Sunflower, a helpful assistant made by Sunbird AI who understands all Ugandan languages. You specialise in accurate translations, explanations, summaries and other language tasks.", help="The system prompt for the chat completion", ) From daacae9f91d46802ccf08e0a6876fee691980d73 Mon Sep 17 00:00:00 2001 From: huwenjie333 Date: Fri, 5 Dec 2025 14:38:01 +0300 Subject: [PATCH 6/9] deployed ultravox --- modal-deploy/client.py | 107 ++++++++++-------------- modal-deploy/vllm_inference.py | 145 +++------------------------------ 2 files changed, 57 insertions(+), 195 deletions(-) diff --git a/modal-deploy/client.py b/modal-deploy/client.py index 583485d6..0aa888b6 100644 --- a/modal-deploy/client.py +++ b/modal-deploy/client.py @@ -4,6 +4,7 @@ import modal from openai import OpenAI +import base64 class Colors: @@ -66,7 +67,7 @@ def main(): parser.add_argument( "--app-name", type=str, - default="Sunflower-14B-FP8", + default="Sunflower32b-Ultravox", help="A Modal App serving an OpenAI-compatible API", ) parser.add_argument( @@ -102,7 +103,7 @@ def main(): parser.add_argument( "--prompt", type=str, - default="Who are you?", + default="Translate to English: ", help="The user prompt for the chat completion", ) parser.add_argument( @@ -111,6 +112,12 @@ def main(): default="You are Sunflower, a helpful assistant made by Sunbird AI who understands all Ugandan languages. You specialise in accurate translations, explanations, summaries and other language tasks.", help="The system prompt for the chat completion", ) + parser.add_argument( + "--audio_file", + type=str, + default="../sunflower-ultravox-vllm/audios/kibuuka_eng.mp3", + help="input audio file for the model.", + ) # UI options parser.add_argument( @@ -119,9 +126,6 @@ def main(): action="store_false", help="Disable streaming of response chunks", ) - parser.add_argument( - "--chat", action="store_true", help="Enable interactive chat mode" - ) args = parser.parse_args() @@ -170,65 +174,42 @@ def main(): print(Colors.BOLD + "šŸ§ : Using system prompt: " + args.system_prompt + Colors.END) - if args.chat: - print( - Colors.GREEN - + Colors.BOLD - + "\nEntering chat mode. Type 'bye' to end the conversation." - + Colors.END - ) - while True: - user_input = input("\nYou: ") - if user_input.lower() in ["bye"]: - break - - MAX_HISTORY = 10 - if len(messages) > MAX_HISTORY: - messages = messages[:1] + messages[-MAX_HISTORY + 1 :] - - messages.append({"role": "user", "content": user_input}) - - response = get_completion(client, model_id, messages, args) - - if response: - if args.stream: - # only stream assuming n=1 - print(Colors.BLUE + "\nšŸ¤–: ", end="") - assistant_message = "" - for chunk in response: - if chunk.choices[0].delta.content: - content = chunk.choices[0].delta.content - print(content, end="") - assistant_message += content - print(Colors.END) - else: - assistant_message = response.choices[0].message.content - print( - Colors.BLUE + "\nšŸ¤–:" + assistant_message + Colors.END, - sep="", - ) - - messages.append({"role": "assistant", "content": assistant_message}) + if args.audio_file: + with open(args.audio_file, 'rb') as f: + audio_bytes = f.read() + audio_b64 = base64.b64encode(audio_bytes).decode("utf-8") + content = [ + { + "type": "text", + "text": args.prompt, + }, + { + "type": "input_audio", + "input_audio": {"data": audio_b64, "format": "wav"}, + }, + ] else: - messages.append({"role": "user", "content": args.prompt}) - print(Colors.GREEN + f"\nYou: {args.prompt}" + Colors.END) - response = get_completion(client, model_id, messages, args) - if response: - if args.stream: - print(Colors.BLUE + "\nšŸ¤–:", end="") - for chunk in response: - if chunk.choices[0].delta.content: - print(chunk.choices[0].delta.content, end="") - print(Colors.END) - else: - # only case where multiple completions are returned - for i, response in enumerate(response.choices): - print( - Colors.BLUE - + f"\nšŸ¤– Choice {i + 1}:{response.message.content}" - + Colors.END, - sep="", - ) + content = args.prompt + + messages.append({"role": "user", "content": content}) + print(Colors.GREEN + f"\nYou: {args.prompt}" + Colors.END) + response = get_completion(client, model_id, messages, args) + if response: + if args.stream: + print(Colors.BLUE + "\nšŸ¤–:", end="") + for chunk in response: + if chunk.choices[0].delta.content: + print(chunk.choices[0].delta.content, end="") + print(Colors.END) + else: + # only case where multiple completions are returned + for i, response in enumerate(response.choices): + print( + Colors.BLUE + + f"\nšŸ¤– Choice {i + 1}:{response.message.content}" + + Colors.END, + sep="", + ) if __name__ == "__main__": diff --git a/modal-deploy/vllm_inference.py b/modal-deploy/vllm_inference.py index f8fef22c..92acdd64 100644 --- a/modal-deploy/vllm_inference.py +++ b/modal-deploy/vllm_inference.py @@ -1,27 +1,5 @@ -# --- -# pytest: false -# --- - -# # Run OpenAI-compatible LLM inference with Qwen3-8B and vLLM - -# LLMs do more than just model language: they chat, they produce JSON and XML, they run code, and more. -# This has complicated their interface far beyond "text-in, text-out". -# OpenAI's API has emerged as a standard for that interface, -# and it is supported by open source LLM serving frameworks like [vLLM](https://docs.vllm.ai/en/latest/). - -# In this example, we show how to run a vLLM server in OpenAI-compatible mode on Modal. - -# Our examples repository also includes scripts for running clients and load-testing for OpenAI-compatible APIs -# [here](https://github.com/modal-labs/modal-examples/tree/main/06_gpu_and_ml/llm-serving/openai_compatible). - -# You can find a (somewhat out-of-date) video walkthrough of this example and the related scripts on the Modal YouTube channel -# [here](https://www.youtube.com/watch?v=QmY_7ePR1hM). - -# ## Set up the container image - -# Our first order of business is to define the environment our server will run in: -# the [container `Image`](https://modal.com/docs/guide/custom-container). -# vLLM can be installed with `pip`, since Modal [provides the CUDA drivers](https://modal.com/docs/guide/cuda). +# Run OpenAI-compatible LLM inference with vLLM +# see documentations in: https://modal.com/docs/examples/vllm_inference#build-a-vllm-engine-and-serve-it import json from typing import Any @@ -29,91 +7,41 @@ import aiohttp import modal +# define container image vllm_image = ( modal.Image.from_registry("nvidia/cuda:12.8.0-devel-ubuntu22.04", add_python="3.12") .entrypoint([]) .uv_pip_install( - "vllm==0.11.2", + "vllm[audio]==0.11.2", "huggingface-hub==0.36.0", "flashinfer-python==0.5.2", ) .env({"HF_XET_HIGH_PERFORMANCE": "1"}) # faster model transfers ) -# ## Download the model weights - -# We'll be running a pretrained foundation model -- Qwen's Qwen3-8B. -# It is trained with reasoning capabilities, which allow it to -# enhance the quality of its generated responses. - -# We'll use an FP8 (eight-bit floating-point) post-training-quantized variant: Qwen/Qwen3-8B-FP8. -# Native hardware support for FP8 formats in [Tensor Cores](https://modal.com/gpu-glossary/device-hardware/tensor-core) -# is limited to the latest [Streaming Multiprocessor architectures](https://modal.com/gpu-glossary/device-hardware/streaming-multiprocessor-architecture), -# like those of Modal's [Hopper H100/H200 and Blackwell B200 GPUs](https://modal.com/blog/announcing-h200-b200). - -# You can swap this model out for another by changing the strings below. -# A single H100 GPU has enough VRAM to store an 8,000,000,000 parameter model, -# like Qwen3-8B, in eight bit precision, along with a very large KV cache. - - -MODEL_NAME = "Sunbird/Sunflower-14B-FP8" -# MODEL_NAME = "Qwen/Qwen3-8B-FP8" -# MODEL_REVISION = "220b46e3b2180893580a4454f21f22d3ebb187d3" # avoid nasty surprises when repos update! - -# Although vLLM will download weights from Hugging Face on-demand, -# we want to cache them so we don't do it every time our server starts. -# We'll use [Modal Volumes](https://modal.com/docs/guide/volumes) for our cache. -# Modal Volumes are essentially a "shared disk" that all Modal Functions can access like it's a regular disk. For more on storing model weights on Modal, see -# [this guide](https://modal.com/docs/guide/model-weights). +# Download the model weights +MODEL_NAME = "huwenjie333/sunflower32b-ultravox-251114-3" +# cache model weights with Modal Volumes hf_cache_vol = modal.Volume.from_name("huggingface-cache", create_if_missing=True) # We'll also cache some of vLLM's JIT compilation artifacts in a Modal Volume. - vllm_cache_vol = modal.Volume.from_name("vllm-cache", create_if_missing=True) -# ## Configuring vLLM - -# ### Trading off fast boots and token generation performance - -# vLLM has embraced dynamic and just-in-time compilation to eke out additional performance without having to write too many custom kernels, -# e.g. via the Torch compiler and CUDA graph capture. -# These compilation features incur latency at startup in exchange for lowered latency and higher throughput during generation. -# We make this trade-off controllable with the `FAST_BOOT` variable below. - +# Trading off fast boots and token generation performance FAST_BOOT = True -# If you're running an LLM service that frequently scales from 0 (frequent ["cold starts"](https://modal.com/docs/guide/cold-start)) -# then you'll want to set this to `True`. - -# If you're running an LLM service that usually has multiple replicas running, then set this to `False` for improved performance. - -# See the code below for details on the parameters that `FAST_BOOT` controls. - -# For more on the performance you can expect when serving your own LLMs, see -# [our LLM engine performance benchmarks](https://modal.com/llm-almanac). - -# ## Build a vLLM engine and serve it - -# The function below spawns a vLLM instance listening at port 8000, serving requests to our model. -# We wrap it in the [`@modal.web_server` decorator](https://modal.com/docs/guide/webhooks#non-asgi-web-servers) -# to connect it to the Internet. - -# The server runs in an independent process, via `subprocess.Popen`, and only starts accepting requests -# once the model is spun up and the `serve` function returns. - - -app = modal.App("Sunflower-14B-FP8") +# Build a vLLM engine and serve it +app = modal.App("Sunflower32b-Ultravox") N_GPU = 1 MINUTES = 60 # seconds VLLM_PORT = 8000 - @app.function( image=vllm_image, - gpu=f"A100-40GB:{N_GPU}", + gpu=f"A100-80GB:{N_GPU}", scaledown_window=2 * MINUTES, # how long should we stay up with no requests? timeout=10 * MINUTES, # how long should we wait for container start? volumes={ @@ -134,8 +62,6 @@ def serve(): "serve", "--uvicorn-log-level=info", MODEL_NAME, - # "--revision", - # MODEL_REVISION, "--served-model-name", MODEL_NAME, "llm", @@ -143,6 +69,8 @@ def serve(): "0.0.0.0", "--port", str(VLLM_PORT), + "--trust-remote-code", + "--max-model-len 4096" ] # enforce-eager disables both Torch compilation and CUDA graph capture @@ -156,7 +84,6 @@ def serve(): subprocess.Popen(" ".join(cmd), shell=True) - # ## Deploy the server # To deploy the API on Modal, just run @@ -164,52 +91,14 @@ def serve(): # modal deploy vllm_inference.py # ``` -# This will create a new app on Modal, build the container image for it if it hasn't been built yet, -# and deploy the app. - -# ## Interact with the server - -# Once it is deployed, you'll see a URL appear in the command line, -# something like `https://your-workspace-name--example-vllm-inference-serve.modal.run`. - -# You can find [interactive Swagger UI docs](https://swagger.io/tools/swagger-ui/) -# at the `/docs` route of that URL, i.e. `https://your-workspace-name--example-vllm-inference-serve.modal.run/docs`. -# These docs describe each route and indicate the expected input and output -# and translate requests into `curl` commands. - -# For simple routes like `/health`, which checks whether the server is responding, -# you can even send a request directly from the docs. - -# To interact with the API programmatically in Python, we recommend the `openai` library. - -# See the `client.py` script in the examples repository -# [here](https://github.com/modal-labs/modal-examples/tree/main/06_gpu_and_ml/llm-serving/openai_compatible) -# to take it for a spin: - -# ```bash -# # pip install openai==1.76.0 -# python openai_compatible/client.py -# ``` - - # ## Testing the server - # To make it easier to test the server setup, we also include a `local_entrypoint` # that does a healthcheck and then hits the server. -# If you execute the command - # ```bash # modal run vllm_inference.py # ``` -# a fresh replica of the server will be spun up on Modal while -# the code below executes on your local machine. - -# Think of this like writing simple tests inside of the `if __name__ == "__main__"` -# block of a Python script, but for cloud deployments! - - @app.local_entrypoint() async def test(test_timeout=10 * MINUTES, content=None, twice=False): url = serve.get_web_url() @@ -267,11 +156,3 @@ async def _send_request( ) # or something went horribly wrong print(chunk["choices"][0]["delta"]["content"], end="") print() - - -# We also include a basic example of a load-testing setup using -# `locust` in the `load_test.py` script [here](https://github.com/modal-labs/modal-examples/tree/main/06_gpu_and_ml/llm-serving/openai_compatible): - -# ```bash -# modal run openai_compatible/load_test.py -# ``` From 78f3b5adf2818f73cdda3e00b2d7f67580d805fa Mon Sep 17 00:00:00 2001 From: huwenjie333 Date: Fri, 5 Dec 2025 15:43:19 +0300 Subject: [PATCH 7/9] readme --- modal-deploy/README.md | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 modal-deploy/README.md diff --git a/modal-deploy/README.md b/modal-deploy/README.md new file mode 100644 index 00000000..1213d7dd --- /dev/null +++ b/modal-deploy/README.md @@ -0,0 +1,41 @@ +# Serverless Deployment with Modal +This readme describes the steps for serverless model deployement with Modal platform and vLLM framework. The detailed documentation is available at [here](https://modal.com/docs/examples/vllm_inference#build-a-vllm-engine-and-serve-it). + +Compares to our current deployment platform Runpod: +| Feature | **RunPod (current)** | **Modal (new)** | +| :--- | :--- | :--- | +| **Support audio vLLM (e.g. Ultravox)** | No | **Yes** | +| **Costs (A100-80GB)** | $0.00076 / s | **$0.00069 / s** | +| **GPU availability** | low when using network volumes | **high** | +| **Deployment methods** | Docker container | **single python script** | +| **serverless cold start time** | 2-3 mins | 2-3 mins | + +## Deployment steps +1. register an account in https://modal.com/ + +2. add your HuggingFace secret in https://modal.com/secrets + +3. install the Modal Python package, and create an API token. +``` +pip install modal +modal setup +``` + +4. `vllm_inference.py` contains all the configuration for a deployment. Here are some important values that you should consider to modify: + - `uv_pip_install`: python packges required + - `MODEL_NAME`: model name in HuggingFace + - `app = modal.App`: deployed model name in Modal platform + - `gpu=f"A100-80GB:{N_GPU}"`: the GPU type and number for deployment + - `scaledown_window`: how long should the instance stay up with no requests? + - `modal.Secret.from_name`: update your HuggingFace secret name if it is different. + - `def serve()`: update the vLLM commands if necessary + +5. run `modal deploy vllm_inference.py` to deploy the model to Modal platform. You can view the deployment on https://modal.com/apps + +6. test the deployed model with the client script, for example: +``` +python client.py \ + --app-name Sunflower32b-Ultravox \ + --prompt "Translate to English: " \ + --audio_file "../sunflower-ultravox-vllm/audios/context_eng_1.wav" +``` \ No newline at end of file From 4acbdd82f801ddf7118fc17f638a20b2355e1f05 Mon Sep 17 00:00:00 2001 From: huwenjie333 Date: Mon, 8 Dec 2025 14:40:38 +0300 Subject: [PATCH 8/9] update temperature --- modal-deploy/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modal-deploy/client.py b/modal-deploy/client.py index 0aa888b6..28cfc0e5 100644 --- a/modal-deploy/client.py +++ b/modal-deploy/client.py @@ -85,7 +85,7 @@ def main(): # Completion parameters parser.add_argument("--max-tokens", type=int, default=None) - parser.add_argument("--temperature", type=float, default=0.7) + parser.add_argument("--temperature", type=float, default=0.6) parser.add_argument("--top-p", type=float, default=0.9) parser.add_argument("--top-k", type=int, default=0) parser.add_argument("--frequency-penalty", type=float, default=0) From 795217764e3a3fe7c2af441297394a6714f5e3d1 Mon Sep 17 00:00:00 2001 From: huwenjie333 Date: Mon, 15 Dec 2025 13:36:08 +0300 Subject: [PATCH 9/9] reorganize files --- modal-deploy/{ => Sunflower32b-Ultravox}/client.py | 0 modal-deploy/{ => Sunflower32b-Ultravox}/vllm_inference.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename modal-deploy/{ => Sunflower32b-Ultravox}/client.py (100%) rename modal-deploy/{ => Sunflower32b-Ultravox}/vllm_inference.py (98%) diff --git a/modal-deploy/client.py b/modal-deploy/Sunflower32b-Ultravox/client.py similarity index 100% rename from modal-deploy/client.py rename to modal-deploy/Sunflower32b-Ultravox/client.py diff --git a/modal-deploy/vllm_inference.py b/modal-deploy/Sunflower32b-Ultravox/vllm_inference.py similarity index 98% rename from modal-deploy/vllm_inference.py rename to modal-deploy/Sunflower32b-Ultravox/vllm_inference.py index 92acdd64..26520ba1 100644 --- a/modal-deploy/vllm_inference.py +++ b/modal-deploy/Sunflower32b-Ultravox/vllm_inference.py @@ -42,7 +42,7 @@ @app.function( image=vllm_image, gpu=f"A100-80GB:{N_GPU}", - scaledown_window=2 * MINUTES, # how long should we stay up with no requests? + scaledown_window=3 * MINUTES, # how long should we stay up with no requests? timeout=10 * MINUTES, # how long should we wait for container start? volumes={ "/root/.cache/huggingface": hf_cache_vol,