start_server.sh

#!/bin/bash

set -e -o pipefail

WORKSPACE_DIR="${WORKSPACE_DIR:-/workspace}"

SERVER_DIR="$WORKSPACE_DIR/vast-pyworker"
ENV_PATH="$WORKSPACE_DIR/worker-env"
DEBUG_LOG="$WORKSPACE_DIR/debug.log"
PYWORKER_LOG="$WORKSPACE_DIR/pyworker.log"

REPORT_ADDR="${REPORT_ADDR:-https://cloud.vast.ai/api/v0,https://run.vast.ai}"
USE_SSL="${USE_SSL:-true}"
WORKER_PORT="${WORKER_PORT:-3000}"
mkdir -p "$WORKSPACE_DIR"
cd "$WORKSPACE_DIR"

# make all output go to $DEBUG_LOG and stdout without having to add `... | tee -a $DEBUG_LOG` to every command
exec &> >(tee -a "$DEBUG_LOG")

function echo_var(){
    echo "$1: ${!1}"
}

[ -z "$BACKEND" ] && echo "BACKEND must be set!" && exit 1
[ -z "$MODEL_LOG" ] && echo "MODEL_LOG must be set!" && exit 1
[ -z "$HF_TOKEN" ] && echo "HF_TOKEN must be set!" && exit 1
[ "$BACKEND" = "comfyui" ] && [ -z "$COMFY_MODEL" ] && echo "For comfyui backends, COMFY_MODEL must be set!" && exit 1


echo "start_server.sh"
date

echo_var BACKEND
echo_var REPORT_ADDR
echo_var WORKER_PORT
echo_var WORKSPACE_DIR
echo_var SERVER_DIR
echo_var ENV_PATH
echo_var DEBUG_LOG
echo_var PYWORKER_LOG
echo_var MODEL_LOG
echo_var USE_VLLM_BENCHMARK
echo_var AUTO_INSTALL_VLLM_BENCH

# Populate /etc/environment with quoted values
if ! grep -q "VAST" /etc/environment; then
    env -0 | grep -zEv "^(HOME=|SHLVL=)|CONDA" | while IFS= read -r -d '' line; do
            name=${line%%=*}
            value=${line#*=}
            printf '%s="%s"\n' "$name" "$value"
        done > /etc/environment
fi

if [ ! -d "$ENV_PATH" ]
then
    echo "setting up venv"
    if ! which uv; then
        curl -LsSf https://astral.sh/uv/install.sh | sh
        source ~/.local/bin/env
    fi

    # Fork testing
    [[ ! -d $SERVER_DIR ]] && git clone "${PYWORKER_REPO:-https://github.com/vast-ai/pyworker}" "$SERVER_DIR"
    if [[ -n ${PYWORKER_REF:-} ]]; then
        (cd "$SERVER_DIR" && git checkout "$PYWORKER_REF")
    fi

    uv venv --python-preference only-managed "$ENV_PATH" -p 3.10
    source "$ENV_PATH/bin/activate"

    uv pip install -r "${SERVER_DIR}/requirements.txt"

    # Ensure vLLM CLI (bench) is installed if requested
    if [ "${USE_VLLM_BENCHMARK:-false}" = "true" ] || [ "${AUTO_INSTALL_VLLM_BENCH:-false}" = "true" ]; then
        echo "Installing vllm[bench]..."
        echo "python: $(python --version)"
        echo "pip before: $(python -m pip --version || true)"
        set +e
        # Make sure build tooling is current
        python -m pip install --upgrade pip setuptools wheel
        # Install vllm (print full logs to see failures)
        python -m pip install "vllm[bench]"
        install_rc=$?
        echo "pip after: $(python -m pip --version || true)"
        echo "which vllm: $(command -v vllm || true)"

        echo "Verifying vLLM import..."
        python - <<'PY'
import sys
try:
    import vllm
    print("vLLM import OK, version:", getattr(vllm, "__version__", "unknown"))
except Exception as e:
    print("vLLM import failed:", e, file=sys.stderr)
    sys.exit(1)
PY
        import_rc=$?

        echo "Verifying vLLM CLI help..."
        vllm bench serve --help >/dev/null 2>&1 || python -m vllm.entrypoints.cli.main bench serve --help >/dev/null 2>&1
        cli_rc=$?
        set -e
        echo "Install rc=${install_rc} import rc=${import_rc} cli rc=${cli_rc}"

        if [ $install_rc -ne 0 ] || [ $import_rc -ne 0 ] || [ $cli_rc -ne 0 ]; then
            echo "WARNING: vLLM not ready (install/import/CLI). Check logs above."
        else
            echo "vLLM installed and ready."
        fi
    fi

    touch ~/.no_auto_tmux
else
    [[ -f ~/.local/bin/env ]] && source ~/.local/bin/env
    source "$WORKSPACE_DIR/worker-env/bin/activate"
    echo "environment activated"
    echo "venv: $VIRTUAL_ENV"

    # Ensure vLLM CLI (bench) is installed if requested (existing venv path)
    if [ "${USE_VLLM_BENCHMARK:-false}" = "true" ] || [ "${AUTO_INSTALL_VLLM_BENCH:-false}" = "true" ]; then
        echo "Installing vllm[bench] (existing venv path)..."
        echo "python: $(python --version)"
        echo "pip before: $(python -m pip --version || true)"
        set +e
        python -m pip install --upgrade pip setuptools wheel
        python -m pip install "vllm[bench]"
        install_rc=$?
        echo "pip after: $(python -m pip --version || true)"
        echo "which vllm: $(command -v vllm || true)"

        echo "Verifying vLLM import..."
        python - <<'PY'
import sys
try:
    import vllm
    print("vLLM python import OK, version:", getattr(vllm, "__version__", "unknown"))
except Exception as e:
    print("vLLM import failed:", e, file=sys.stderr)
    sys.exit(1)
PY
        import_rc=$?

        echo "Verifying vLLM CLI help..."
        vllm bench serve --help >/dev/null 2>&1 || python -m vllm.entrypoints.cli.main bench serve --help >/dev/null 2>&1
        cli_rc=$?
        set -e
        echo "Install rc=${install_rc} import rc=${import_rc} cli rc=${cli_rc}"

        if [ $install_rc -ne 0 ] || [ $import_rc -ne 0 ] || [ $cli_rc -ne 0 ]; then
            echo "WARNING: vLLM not ready (install/import/CLI). Check logs above."
        else
            echo "vLLM installed and ready."
        fi
    fi
fi

[ ! -d "$SERVER_DIR/workers/$BACKEND" ] && echo "$BACKEND not supported!" && exit 1

if [ "$USE_SSL" = true ]; then

    cat << EOF > /etc/openssl-san.cnf
    [req]
    default_bits       = 2048
    distinguished_name = req_distinguished_name
    req_extensions     = v3_req

    [req_distinguished_name]
    countryName         = US
    stateOrProvinceName = CA
    organizationName    = Vast.ai Inc.
    commonName          = vast.ai

    [v3_req]
    basicConstraints = CA:FALSE
    keyUsage         = nonRepudiation, digitalSignature, keyEncipherment
    subjectAltName   = @alt_names

    [alt_names]
    IP.1   = 0.0.0.0
EOF

    openssl req -newkey rsa:2048 -subj "/C=US/ST=CA/CN=pyworker.vast.ai/" \
        -nodes \
        -sha256 \
        -keyout /etc/instance.key \
        -out /etc/instance.csr \
        -config /etc/openssl-san.cnf

    curl --header 'Content-Type: application/octet-stream' \
        --data-binary @//etc/instance.csr \
        -X \
        POST "https://console.vast.ai/api/v0/sign_cert/?instance_id=$CONTAINER_ID" > /etc/instance.crt;
fi



export REPORT_ADDR WORKER_PORT USE_SSL UNSECURED

cd "$SERVER_DIR"

echo "launching PyWorker server"

# if instance is rebooted, we want to clear out the log file so pyworker doesn't read lines
# from the run prior to reboot. past logs are saved in $MODEL_LOG.old for debugging only
[ -e "$MODEL_LOG" ] && cat "$MODEL_LOG" >> "$MODEL_LOG.old" && : > "$MODEL_LOG"

(python3 -m "workers.$BACKEND.server" |& tee -a "$PYWORKER_LOG") &
echo "launching PyWorker server done"