diff --git a/modal-deploy/README.md b/modal-deploy/README.md new file mode 100644 index 00000000..1213d7dd --- /dev/null +++ b/modal-deploy/README.md @@ -0,0 +1,41 @@ +# Serverless Deployment with Modal +This readme describes the steps for serverless model deployement with Modal platform and vLLM framework. The detailed documentation is available at [here](https://modal.com/docs/examples/vllm_inference#build-a-vllm-engine-and-serve-it). + +Compares to our current deployment platform Runpod: +| Feature | **RunPod (current)** | **Modal (new)** | +| :--- | :--- | :--- | +| **Support audio vLLM (e.g. Ultravox)** | No | **Yes** | +| **Costs (A100-80GB)** | $0.00076 / s | **$0.00069 / s** | +| **GPU availability** | low when using network volumes | **high** | +| **Deployment methods** | Docker container | **single python script** | +| **serverless cold start time** | 2-3 mins | 2-3 mins | + +## Deployment steps +1. register an account in https://modal.com/ + +2. add your HuggingFace secret in https://modal.com/secrets + +3. install the Modal Python package, and create an API token. +``` +pip install modal +modal setup +``` + +4. `vllm_inference.py` contains all the configuration for a deployment. Here are some important values that you should consider to modify: + - `uv_pip_install`: python packges required + - `MODEL_NAME`: model name in HuggingFace + - `app = modal.App`: deployed model name in Modal platform + - `gpu=f"A100-80GB:{N_GPU}"`: the GPU type and number for deployment + - `scaledown_window`: how long should the instance stay up with no requests? + - `modal.Secret.from_name`: update your HuggingFace secret name if it is different. + - `def serve()`: update the vLLM commands if necessary + +5. run `modal deploy vllm_inference.py` to deploy the model to Modal platform. You can view the deployment on https://modal.com/apps + +6. test the deployed model with the client script, for example: +``` +python client.py \ + --app-name Sunflower32b-Ultravox \ + --prompt "Translate to English: " \ + --audio_file "../sunflower-ultravox-vllm/audios/context_eng_1.wav" +``` \ No newline at end of file diff --git a/modal-deploy/Sunflower32b-Ultravox/client.py b/modal-deploy/Sunflower32b-Ultravox/client.py new file mode 100644 index 00000000..28cfc0e5 --- /dev/null +++ b/modal-deploy/Sunflower32b-Ultravox/client.py @@ -0,0 +1,216 @@ +"""This simple script shows how to interact with an OpenAI-compatible server from a client.""" + +import argparse + +import modal +from openai import OpenAI +import base64 + + +class Colors: + """ANSI color codes""" + + GREEN = "\033[0;32m" + RED = "\033[0;31m" + BLUE = "\033[0;34m" + GRAY = "\033[0;90m" + BOLD = "\033[1m" + END = "\033[0m" + + +def get_completion(client, model_id, messages, args): + completion_args = { + "model": model_id, + "messages": messages, + "frequency_penalty": args.frequency_penalty, + "max_tokens": args.max_tokens, + "n": args.n, + "presence_penalty": args.presence_penalty, + "seed": args.seed, + "stop": args.stop, + "stream": args.stream, + "temperature": args.temperature, + "top_p": args.top_p, + } + + completion_args = {k: v for k, v in completion_args.items() if v is not None} + + try: + response = client.chat.completions.create(**completion_args) + return response + except Exception as e: + print(Colors.RED, f"Error during API call: {e}", Colors.END, sep="") + return None + + +def main(): + parser = argparse.ArgumentParser(description="OpenAI Client CLI") + + parser.add_argument( + "--model", + type=str, + default=None, + help="The model to use for completion, defaults to the first available model", + ) + parser.add_argument( + "--workspace", + type=str, + default=None, + help="The workspace where the LLM server app is hosted, defaults to your current Modal workspace", + ) + parser.add_argument( + "--environment", + type=str, + default=None, + help="The environment in your Modal workspace where the LLM server app is hosted, defaults to your current environment", + ) + parser.add_argument( + "--app-name", + type=str, + default="Sunflower32b-Ultravox", + help="A Modal App serving an OpenAI-compatible API", + ) + parser.add_argument( + "--function-name", + type=str, + default="serve", + help="A Modal Function serving an OpenAI-compatible API. Append `-dev` to use a `modal serve`d Function.", + ) + parser.add_argument( + "--api-key", + type=str, + default="super-secret-key", + help="The API key to use for authentication, set in your api.py", + ) + + # Completion parameters + parser.add_argument("--max-tokens", type=int, default=None) + parser.add_argument("--temperature", type=float, default=0.6) + parser.add_argument("--top-p", type=float, default=0.9) + parser.add_argument("--top-k", type=int, default=0) + parser.add_argument("--frequency-penalty", type=float, default=0) + parser.add_argument("--presence-penalty", type=float, default=0) + parser.add_argument( + "--n", + type=int, + default=1, + help="Number of completions to generate. Streaming and chat mode only support n=1.", + ) + parser.add_argument("--stop", type=str, default=None) + parser.add_argument("--seed", type=int, default=None) + + # Prompting + parser.add_argument( + "--prompt", + type=str, + default="Translate to English: ", + help="The user prompt for the chat completion", + ) + parser.add_argument( + "--system-prompt", + type=str, + default="You are Sunflower, a helpful assistant made by Sunbird AI who understands all Ugandan languages. You specialise in accurate translations, explanations, summaries and other language tasks.", + help="The system prompt for the chat completion", + ) + parser.add_argument( + "--audio_file", + type=str, + default="../sunflower-ultravox-vllm/audios/kibuuka_eng.mp3", + help="input audio file for the model.", + ) + + # UI options + parser.add_argument( + "--no-stream", + dest="stream", + action="store_false", + help="Disable streaming of response chunks", + ) + + args = parser.parse_args() + + client = OpenAI(api_key=args.api_key) + + workspace = args.workspace or modal.config._profile + + environment = args.environment or modal.config.config["environment"] + + prefix = workspace + (f"-{environment}" if environment else "") + + client.base_url = ( + f"https://{prefix}--{args.app_name}-{args.function_name}.modal.run/v1" + ) + + if args.model: + model_id = args.model + print( + Colors.BOLD, + f"šŸ§ : Using model {model_id}. This may trigger a model load on first call!", + Colors.END, + sep="", + ) + else: + print( + Colors.BOLD, + f"šŸ”Ž: Looking up available models on server at {client.base_url}. This may trigger a model load!", + Colors.END, + sep="", + ) + model = client.models.list().data[0] + model_id = model.id + print( + Colors.BOLD, + f"šŸ§ : Using {model_id}", + Colors.END, + sep="", + ) + + messages = [ + { + "role": "system", + "content": args.system_prompt, + } + ] + + print(Colors.BOLD + "šŸ§ : Using system prompt: " + args.system_prompt + Colors.END) + + if args.audio_file: + with open(args.audio_file, 'rb') as f: + audio_bytes = f.read() + audio_b64 = base64.b64encode(audio_bytes).decode("utf-8") + content = [ + { + "type": "text", + "text": args.prompt, + }, + { + "type": "input_audio", + "input_audio": {"data": audio_b64, "format": "wav"}, + }, + ] + else: + content = args.prompt + + messages.append({"role": "user", "content": content}) + print(Colors.GREEN + f"\nYou: {args.prompt}" + Colors.END) + response = get_completion(client, model_id, messages, args) + if response: + if args.stream: + print(Colors.BLUE + "\nšŸ¤–:", end="") + for chunk in response: + if chunk.choices[0].delta.content: + print(chunk.choices[0].delta.content, end="") + print(Colors.END) + else: + # only case where multiple completions are returned + for i, response in enumerate(response.choices): + print( + Colors.BLUE + + f"\nšŸ¤– Choice {i + 1}:{response.message.content}" + + Colors.END, + sep="", + ) + + +if __name__ == "__main__": + main() diff --git a/modal-deploy/Sunflower32b-Ultravox/vllm_inference.py b/modal-deploy/Sunflower32b-Ultravox/vllm_inference.py new file mode 100644 index 00000000..26520ba1 --- /dev/null +++ b/modal-deploy/Sunflower32b-Ultravox/vllm_inference.py @@ -0,0 +1,158 @@ +# Run OpenAI-compatible LLM inference with vLLM +# see documentations in: https://modal.com/docs/examples/vllm_inference#build-a-vllm-engine-and-serve-it + +import json +from typing import Any + +import aiohttp +import modal + +# define container image +vllm_image = ( + modal.Image.from_registry("nvidia/cuda:12.8.0-devel-ubuntu22.04", add_python="3.12") + .entrypoint([]) + .uv_pip_install( + "vllm[audio]==0.11.2", + "huggingface-hub==0.36.0", + "flashinfer-python==0.5.2", + ) + .env({"HF_XET_HIGH_PERFORMANCE": "1"}) # faster model transfers +) + +# Download the model weights + +MODEL_NAME = "huwenjie333/sunflower32b-ultravox-251114-3" + +# cache model weights with Modal Volumes +hf_cache_vol = modal.Volume.from_name("huggingface-cache", create_if_missing=True) + +# We'll also cache some of vLLM's JIT compilation artifacts in a Modal Volume. +vllm_cache_vol = modal.Volume.from_name("vllm-cache", create_if_missing=True) + +# Trading off fast boots and token generation performance +FAST_BOOT = True + +# Build a vLLM engine and serve it +app = modal.App("Sunflower32b-Ultravox") + +N_GPU = 1 +MINUTES = 60 # seconds +VLLM_PORT = 8000 + +@app.function( + image=vllm_image, + gpu=f"A100-80GB:{N_GPU}", + scaledown_window=3 * MINUTES, # how long should we stay up with no requests? + timeout=10 * MINUTES, # how long should we wait for container start? + volumes={ + "/root/.cache/huggingface": hf_cache_vol, + "/root/.cache/vllm": vllm_cache_vol, + }, + secrets=[modal.Secret.from_name("huggingface-read")], +) +@modal.concurrent( # how many requests can one replica handle? tune carefully! + max_inputs=32 +) +@modal.web_server(port=VLLM_PORT, startup_timeout=10 * MINUTES) +def serve(): + import subprocess + + cmd = [ + "vllm", + "serve", + "--uvicorn-log-level=info", + MODEL_NAME, + "--served-model-name", + MODEL_NAME, + "llm", + "--host", + "0.0.0.0", + "--port", + str(VLLM_PORT), + "--trust-remote-code", + "--max-model-len 4096" + ] + + # enforce-eager disables both Torch compilation and CUDA graph capture + # default is no-enforce-eager. see the --compilation-config flag for tighter control + cmd += ["--enforce-eager" if FAST_BOOT else "--no-enforce-eager"] + + # assume multiple GPUs are for splitting up large matrix multiplications + cmd += ["--tensor-parallel-size", str(N_GPU)] + + print(cmd) + + subprocess.Popen(" ".join(cmd), shell=True) + +# ## Deploy the server + +# To deploy the API on Modal, just run +# ```bash +# modal deploy vllm_inference.py +# ``` + +# ## Testing the server +# To make it easier to test the server setup, we also include a `local_entrypoint` +# that does a healthcheck and then hits the server. + +# ```bash +# modal run vllm_inference.py +# ``` + +@app.local_entrypoint() +async def test(test_timeout=10 * MINUTES, content=None, twice=False): + url = serve.get_web_url() + + system_prompt = { + "role": "system", + "content": "You are a pirate who can't help but drop sly reminders that he went to Harvard.", + } + if content is None: + content = "Explain the singular value decomposition." + + messages = [ # OpenAI chat format + system_prompt, + {"role": "user", "content": content}, + ] + + async with aiohttp.ClientSession(base_url=url) as session: + print(f"Running health check for server at {url}") + async with session.get("/health", timeout=test_timeout - 1 * MINUTES) as resp: + up = resp.status == 200 + assert up, f"Failed health check for server at {url}" + print(f"Successful health check for server at {url}") + + print(f"Sending messages to {url}:", *messages, sep="\n\t") + await _send_request(session, "llm", messages) + if twice: + messages[0]["content"] = "You are Jar Jar Binks." + print(f"Sending messages to {url}:", *messages, sep="\n\t") + await _send_request(session, "llm", messages) + + +async def _send_request( + session: aiohttp.ClientSession, model: str, messages: list +) -> None: + # `stream=True` tells an OpenAI-compatible backend to stream chunks + payload: dict[str, Any] = {"messages": messages, "model": model, "stream": True} + + headers = {"Content-Type": "application/json", "Accept": "text/event-stream"} + + async with session.post( + "/v1/chat/completions", json=payload, headers=headers, timeout=1 * MINUTES + ) as resp: + async for raw in resp.content: + resp.raise_for_status() + # extract new content and stream it + line = raw.decode().strip() + if not line or line == "data: [DONE]": + continue + if line.startswith("data: "): # SSE prefix + line = line[len("data: ") :] + + chunk = json.loads(line) + assert ( + chunk["object"] == "chat.completion.chunk" + ) # or something went horribly wrong + print(chunk["choices"][0]["delta"]["content"], end="") + print()