Skip to content

fix(server): reserve per-device VRAM for LOADING models (#66) #148

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
marksverdhei wants to merge 1 commit into
base: ht
Choose a base branch
from
Draft

fix(server): reserve per-device VRAM for LOADING models (#66) #148

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 48 additions & 3 deletions tools/server/server-models.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -872,11 +872,30 @@ std::vector<common_lora_adapter_info> server_models::get_discovered_adapters() {
return discovered_adapters;
}

void server_models::reserve_apply(const server_model_meta & meta) {
if (meta.per_device_bytes.size() > reserved_per_device.size()) {
reserved_per_device.resize(meta.per_device_bytes.size(), 0);
}
for (size_t i = 0; i < meta.per_device_bytes.size(); i++) {
reserved_per_device[i] += meta.per_device_bytes[i];
}
}

void server_models::reserve_release(const server_model_meta & meta) {
for (size_t i = 0; i < meta.per_device_bytes.size() && i < reserved_per_device.size(); i++) {
reserved_per_device[i] -= meta.per_device_bytes[i];
if (reserved_per_device[i] < 0) {
reserved_per_device[i] = 0; // defensive: an over-release must never leave a phantom reserve
}
}
}

void server_models::unload_lru(const std::string & candidate_name) {
while (true) {
size_t count_active = 0;
bool mem_fits = true;
std::vector<int64_t> candidate_bytes;
std::vector<int64_t> reserved_snapshot;

{
std::unique_lock<std::mutex> lk(mutex);
Expand All @@ -887,16 +906,19 @@ void server_models::unload_lru(const std::string & candidate_name) {
}
}

// 2. Fetch candidate VRAM bytes
// 2. Fetch candidate VRAM bytes and the in-flight reserve (both under the lock)
auto map_it = mapping.find(candidate_name);
if (map_it != mapping.end()) {
candidate_bytes = map_it->second.meta.per_device_bytes;
}
reserved_snapshot = reserved_per_device;
}

bool count_ok = (base_params.models_max <= 0) || (count_active < (size_t)base_params.models_max);

// 3. Query physical free memory for GPUs
// 3. Query physical free memory for GPUs, then subtract the in-flight reserve for
// models that are LOADING but whose cudaMalloc is not yet visible to cudaMemGetInfo.
// Without this subtraction a second same-device admit reads stale-high free and OOMs (#66).
std::vector<int64_t> projected_free;
for (size_t i = 0; i < ggml_backend_dev_count(); i++) {
ggml_backend_dev_t dev = ggml_backend_dev_get(i);
Expand All @@ -907,6 +929,9 @@ void server_models::unload_lru(const std::string & candidate_name) {
projected_free.push_back(free);
}
}
for (size_t id = 0; id < projected_free.size() && id < reserved_snapshot.size(); id++) {
projected_free[id] -= reserved_snapshot[id];
}

// 4. Check if candidate memory footprint fits and collect constrained devices
std::vector<size_t> constrained_devices;
Expand All @@ -931,7 +956,10 @@ void server_models::unload_lru(const std::string & candidate_name) {
{
std::unique_lock<std::mutex> lk(mutex);
for (const auto & m : mapping) {
if (m.second.meta.is_running()) {
// Only LOADED/SLEEPING models are evictable. A model still in LOADING has an
// active reserve and a half-spawned child — evicting it to make room for another
// load thrashes and races the child teardown, so it is never an eviction target.
if (m.second.meta.is_running() && m.second.meta.status != SERVER_MODEL_STATUS_LOADING) {
bool helps_memory = false;
if (!mem_fits) {
for (size_t id : constrained_devices) {
Expand Down Expand Up @@ -1271,6 +1299,13 @@ void server_models::load(const std::string & name, const load_options & opts) {
{"status", server_model_status_to_string(inst.meta.status)},
});

// Reserve this model's per-device VRAM now that the child is spawned and committed in
// LOADING state, but before its cudaMalloc becomes visible to cudaMemGetInfo. Applied here
// (past all throw points, still under `lk`) so it cannot leak on a failed spawn, and so no
// update_status() can release it early — the monitoring thread blocks on `mutex` until this
// load returns. Released in update_status() when the model leaves LOADING. See #66.
reserve_apply(inst.meta);

mapping[name] = std::move(inst);
cv.notify_all();
}
Expand Down Expand Up @@ -1332,6 +1367,7 @@ void server_models::update_status(const std::string & name, const update_status_
auto it = mapping.find(name);
if (it != mapping.end()) {
auto & meta = it->second.meta;
const server_model_status old_status = meta.status;
meta.status = args.status;
meta.exit_code = args.exit_code;
if (!args.loaded_info.is_null()) {
Expand All @@ -1340,6 +1376,15 @@ void server_models::update_status(const std::string & name, const update_status_
if (!args.progress.is_null()) {
meta.progress = args.progress;
}
// Release the in-flight VRAM reserve when the model leaves LOADING — to LOADED on a
// successful boot, or back to UNLOADED on a failed load. update_status() is never called
// with args.status == LOADING (the child's "loading" state is a no-op in handle_child_state),
// so any old_status == LOADING here is a genuine exit. Idempotent against the double-UNLOADED
// path (the crash handler and the monitoring thread can both fire): the second call observes
// old_status != LOADING and does nothing. See heiervang-technologies/ht-llama.cpp#66.
if (old_status == SERVER_MODEL_STATUS_LOADING && args.status != SERVER_MODEL_STATUS_LOADING) {
reserve_release(meta);
}
}
// broadcast status change to SSE
{
Expand Down
14 changes: 14 additions & 0 deletions tools/server/server-models.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -194,6 +194,20 @@ struct server_models {
// unload least recently used models if the limit is reached or memory is tight
void unload_lru(const std::string & candidate_name);

// In-flight VRAM reserve for models in the LOADING state, indexed by system-GPU
// order (identical to server_model_meta::per_device_bytes and to the projected_free
// vector built in unload_lru). A model's per-device footprint is reserved when it
// enters LOADING — before the child process's cudaMalloc becomes visible to
// cudaMemGetInfo — and released when it leaves LOADING (to LOADED or, on a failed
// load, back to UNLOADED). Without this, two same-device admits fired close together
// both read stale-high free memory and co-load onto the same GPU -> OOM.
// See heiervang-technologies/ht-llama.cpp#66. Guarded by `mutex`.
std::vector<int64_t> reserved_per_device;

// apply/release the in-flight reserve for a model. Caller MUST hold `mutex`.
void reserve_apply(const server_model_meta & meta);
void reserve_release(const server_model_meta & meta);

// not thread-safe, caller must hold mutex
void add_model(server_model_meta && meta);

Expand Down
Loading