diff --git a/modal-deploy/README.md b/modal-deploy/README.md index 1213d7dd..767e613b 100644 --- a/modal-deploy/README.md +++ b/modal-deploy/README.md @@ -34,8 +34,9 @@ modal setup 6. test the deployed model with the client script, for example: ``` +cd Sunflower32b-Ultravox python client.py \ --app-name Sunflower32b-Ultravox \ --prompt "Translate to English: " \ - --audio_file "../sunflower-ultravox-vllm/audios/context_eng_1.wav" + --audio_file "../../sunflower-ultravox-vllm/audios/context_eng_1.wav" ``` \ No newline at end of file diff --git a/modal-deploy/spark-tts-salt/vllm_inference.py b/modal-deploy/spark-tts-salt/vllm_inference.py index 17b28d17..c6fe806a 100644 --- a/modal-deploy/spark-tts-salt/vllm_inference.py +++ b/modal-deploy/spark-tts-salt/vllm_inference.py @@ -60,8 +60,11 @@ # cache model weights with Modal Volumes hf_cache_vol = modal.Volume.from_name("huggingface-cache", create_if_missing=True) -HF_CACHE_DIR = "/root/.cache/huggingface" +# cache some of vLLM's compilation artifacts in a Modal Volume. +vllm_cache_vol = modal.Volume.from_name("vllm-cache", create_if_missing=True) +HF_CACHE_DIR = "/root/.cache/huggingface" +VLLM_CACHE_DIR = "/root/.cache/vllm" # ## The TTS model class @@ -78,6 +81,7 @@ secrets=[modal.Secret.from_name("huggingface-read")], volumes={ HF_CACHE_DIR: hf_cache_vol, + VLLM_CACHE_DIR: vllm_cache_vol, } ) @modal.concurrent(max_inputs=10)