Sunflower-Ultravox Deployment with Modal platform

modal-deploy/README.md

@@ -0,0 +1,41 @@
+# Serverless Deployment with Modal
+This readme describes the steps for serverless model deployement with Modal platform and vLLM framework. The detailed documentation is available at [here](https://modal.com/docs/examples/vllm_inference#build-a-vllm-engine-and-serve-it).
+
+Compares to our current deployment platform Runpod:
+| Feature | **RunPod (current)** | **Modal (new)** |
+| :--- | :--- | :--- |
+| **Support  audio vLLM (e.g. Ultravox)** | No | **Yes** |
+| **Costs (A100-80GB)** | $0.00076 / s |  **$0.00069 / s** |
+| **GPU availability** | low when using network volumes |  **high**  |
+| **Deployment methods** | Docker container | **single python script** |
+| **serverless cold start time** | 2-3 mins | 2-3 mins |
+
+## Deployment steps
+1. register an account in https://modal.com/
+
+2. add your HuggingFace secret in https://modal.com/secrets
+
+3. install the Modal Python package, and create an API token.
+```
+pip install modal
+modal setup
+```
+
+4. `vllm_inference.py` contains all the configuration for a deployment. Here are some important values that you should consider to modify:
+   - `uv_pip_install`: python packges required
+   - `MODEL_NAME`: model name in HuggingFace
+   - `app = modal.App`:  deployed model name in Modal platform
+   - `gpu=f"A100-80GB:{N_GPU}"`: the GPU type and number for deployment
+   - `scaledown_window`: how long should the instance stay up with no requests?
+   - `modal.Secret.from_name`: update your HuggingFace secret name if it is different.
+   - `def serve()`: update the vLLM commands if necessary
+
+5. run `modal deploy vllm_inference.py` to deploy the model to Modal platform. You can view the deployment on https://modal.com/apps
+
+6. test the deployed model with the client script, for example:
+```
+python client.py \
+    --app-name Sunflower32b-Ultravox  \
+    --prompt "Translate to English: " \
+    --audio_file "../sunflower-ultravox-vllm/audios/context_eng_1.wav"
+```

modal-deploy/Sunflower32b-Ultravox/client.py

@@ -0,0 +1,216 @@
+"""This simple script shows how to interact with an OpenAI-compatible server from a client."""
+
+import argparse
+
+import modal
+from openai import OpenAI
+import base64
+
+
+class Colors:
+    """ANSI color codes"""
+
+    GREEN = "\033[0;32m"
+    RED = "\033[0;31m"
+    BLUE = "\033[0;34m"
+    GRAY = "\033[0;90m"
+    BOLD = "\033[1m"
+    END = "\033[0m"
+
+
+def get_completion(client, model_id, messages, args):
+    completion_args = {
+        "model": model_id,
+        "messages": messages,
+        "frequency_penalty": args.frequency_penalty,
+        "max_tokens": args.max_tokens,
+        "n": args.n,
+        "presence_penalty": args.presence_penalty,
+        "seed": args.seed,
+        "stop": args.stop,
+        "stream": args.stream,
+        "temperature": args.temperature,
+        "top_p": args.top_p,
+    }
+
+    completion_args = {k: v for k, v in completion_args.items() if v is not None}
+
+    try:
+        response = client.chat.completions.create(**completion_args)
+        return response
+    except Exception as e:
+        print(Colors.RED, f"Error during API call: {e}", Colors.END, sep="")
+        return None
+
+
+def main():
+    parser = argparse.ArgumentParser(description="OpenAI Client CLI")
+
+    parser.add_argument(
+        "--model",
+        type=str,
+        default=None,
+        help="The model to use for completion, defaults to the first available model",
+    )
+    parser.add_argument(
+        "--workspace",
+        type=str,
+        default=None,
+        help="The workspace where the LLM server app is hosted, defaults to your current Modal workspace",
+    )
+    parser.add_argument(
+        "--environment",
+        type=str,
+        default=None,
+        help="The environment in your Modal workspace where the LLM server app is hosted, defaults to your current environment",
+    )
+    parser.add_argument(
+        "--app-name",
+        type=str,
+        default="Sunflower32b-Ultravox",
+        help="A Modal App serving an OpenAI-compatible API",
+    )
+    parser.add_argument(
+        "--function-name",
+        type=str,
+        default="serve",
+        help="A Modal Function serving an OpenAI-compatible API. Append `-dev` to use a `modal serve`d Function.",
+    )
+    parser.add_argument(
+        "--api-key",
+        type=str,
+        default="super-secret-key",
+        help="The API key to use for authentication, set in your api.py",
+    )
+
+    # Completion parameters
+    parser.add_argument("--max-tokens", type=int, default=None)
+    parser.add_argument("--temperature", type=float, default=0.6)
+    parser.add_argument("--top-p", type=float, default=0.9)
+    parser.add_argument("--top-k", type=int, default=0)
+    parser.add_argument("--frequency-penalty", type=float, default=0)
+    parser.add_argument("--presence-penalty", type=float, default=0)
+    parser.add_argument(
+        "--n",
+        type=int,
+        default=1,
+        help="Number of completions to generate. Streaming and chat mode only support n=1.",
+    )
+    parser.add_argument("--stop", type=str, default=None)
+    parser.add_argument("--seed", type=int, default=None)
+
+    # Prompting
+    parser.add_argument(
+        "--prompt",
+        type=str,
+        default="Translate to English: ",
+        help="The user prompt for the chat completion",
+    )
+    parser.add_argument(
+        "--system-prompt",
+        type=str,
+        default="You are Sunflower, a helpful assistant made by Sunbird AI who understands all Ugandan languages. You specialise in accurate translations, explanations, summaries and other language tasks.",
+        help="The system prompt for the chat completion",
+    )
+    parser.add_argument(
+        "--audio_file",
+        type=str,
+        default="../sunflower-ultravox-vllm/audios/kibuuka_eng.mp3",
+        help="input audio file for the model.",
+    )
+
+    # UI options
+    parser.add_argument(
+        "--no-stream",
+        dest="stream",
+        action="store_false",
+        help="Disable streaming of response chunks",
+    )
+
+    args = parser.parse_args()
+
+    client = OpenAI(api_key=args.api_key)
+
+    workspace = args.workspace or modal.config._profile
+
+    environment = args.environment or modal.config.config["environment"]
+
+    prefix = workspace + (f"-{environment}" if environment else "")
+
+    client.base_url = (
+        f"https://{prefix}--{args.app_name}-{args.function_name}.modal.run/v1"
+    )
+
+    if args.model:
+        model_id = args.model
+        print(
+            Colors.BOLD,
+            f"🧠: Using model {model_id}. This may trigger a model load on first call!",
+            Colors.END,
+            sep="",
+        )
+    else:
+        print(
+            Colors.BOLD,
+            f"🔎: Looking up available models on server at {client.base_url}. This may trigger a model load!",
+            Colors.END,
+            sep="",
+        )
+        model = client.models.list().data[0]
+        model_id = model.id
+        print(
+            Colors.BOLD,
+            f"🧠: Using {model_id}",
+            Colors.END,
+            sep="",
+        )
+
+    messages = [
+        {
+            "role": "system",
+            "content": args.system_prompt,
+        }
+    ]
+
+    print(Colors.BOLD + "🧠: Using system prompt: " + args.system_prompt + Colors.END)
+
+    if args.audio_file:
+        with open(args.audio_file, 'rb') as f:
+            audio_bytes = f.read()
+        audio_b64 = base64.b64encode(audio_bytes).decode("utf-8")
+        content = [
+            {
+                "type": "text",
+                "text": args.prompt,
+            },
+            {
+                "type": "input_audio",
+                "input_audio": {"data": audio_b64, "format": "wav"},
+            },
+        ]
+    else:
+        content = args.prompt
+
+    messages.append({"role": "user", "content": content})
+    print(Colors.GREEN + f"\nYou: {args.prompt}" + Colors.END)
+    response = get_completion(client, model_id, messages, args)
+    if response:
+        if args.stream:
+            print(Colors.BLUE + "\n🤖:", end="")
+            for chunk in response:
+                if chunk.choices[0].delta.content:
+                    print(chunk.choices[0].delta.content, end="")
+            print(Colors.END)
+        else:
+            # only case where multiple completions are returned
+            for i, response in enumerate(response.choices):
+                print(
+                    Colors.BLUE
+                    + f"\n🤖 Choice {i + 1}:{response.message.content}"
+                    + Colors.END,
+                    sep="",
+                )
+
+
+if __name__ == "__main__":
+    main()