diff --git a/Makefile b/Makefile index 6bebd9d1..9c29b02b 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,7 @@ build-in-docker: -e DEBIAN_FRONTEND=noninteractive \ nvidia/cuda:12.9.1-cudnn-devel-ubuntu20.04 \ sh -c "apt-get -y update && \ - apt-get -y install cmake git && \ + apt-get -y install cmake git libvulkan-dev && \ git config --global --add safe.directory /libvgpu && \ bash ./build.sh" .PHONY: build-in-docker diff --git a/dockerfiles/Dockerfile b/dockerfiles/Dockerfile index 6201920b..99cd9123 100644 --- a/dockerfiles/Dockerfile +++ b/dockerfiles/Dockerfile @@ -6,7 +6,11 @@ WORKDIR /libvgpu COPY . /libvgpu RUN apt-get update && \ - apt-get install -y cmake git && \ + apt-get install -y cmake git libvulkan-dev && \ rm -rf /var/lib/apt/lists/* RUN bash ./build.sh + +# Ship Vulkan implicit layer manifest so HAMI_VULKAN_ENABLE=1 activates the layer. +RUN install -D -m 0644 /libvgpu/etc/vulkan/implicit_layer.d/hami.json \ + /etc/vulkan/implicit_layer.d/hami.json diff --git a/docs/superpowers/notes/2026-04-28-cuda-hook-audit.md b/docs/superpowers/notes/2026-04-28-cuda-hook-audit.md new file mode 100644 index 00000000..0d978ce5 --- /dev/null +++ b/docs/superpowers/notes/2026-04-28-cuda-hook-audit.md @@ -0,0 +1,77 @@ +# CUDA hook robustness audit — 2026-04-28 + +Reference fix: commit `03f99d7 fix(cuda): avoid NULL deref in cuMemGetInfo_v2 when caller (OptiX) crashes` + +Pattern (verified against current `cuMemGetInfo_v2` body at `src/cuda/memory.c:501`): + +1. Forward to the real driver first (errors surface exactly as without HAMi). +2. Early return on NULL/invalid args (driver already rejected — never deref). +3. Then HAMi enforcement / accounting logic. + +Why we need this: NVIDIA Isaac Sim Kit (Carbonite / OptiX / Aftermath) calls +several of these CUDA hooks via internal probes that may pass NULL output +pointers or run before any CUDA context is current. With the current code the +HAMi enforcement path runs unconditionally, dereferences the NULL output, or +calls `cuCtxGetDevice` without checking its return — and `libvgpu.so` +SegFaults inside the app. + +## Hooks needing the same pattern + +Line numbers verified against current source on branch `vulkan-layer` +(`grep -n '^CUresult ' src/cuda/{memory,context}.c` 2026-04-28): + +- `cuMemAlloc_v2` — `src/cuda/memory.c:135` (Task 2) + - Body: `ENSURE_RUNNING(); allocate_raw(dptr, bytesize)` — `allocate_raw` + will write `*dptr` and call into `oom_check` / `add_chunk` without any + NULL guard on `dptr`. +- `cuMemAllocHost_v2` — `src/cuda/memory.c:145` (Task 3) + - Body: forwards first (good), but on success does `*hptr = NULL` / + re-frees via `*hptr` inside `check_oom` branch without confirming + caller passed a non-NULL `hptr`. OOM cleanup path will crash. +- `cuMemAllocManaged` — `src/cuda/memory.c:159` (Task 3) + - Body: `cuCtxGetDevice(&dev)` via `CHECK_DRV_API` — if no current + context this aborts; `oom_check` runs before the real driver call so + NULL `dptr` isn't surfaced as the driver's own + `CUDA_ERROR_INVALID_VALUE`. +- `cuMemAllocPitch_v2` — `src/cuda/memory.c:174` (Task 4) + - Same pattern as Managed: pre-call `cuCtxGetDevice` + `oom_check` then + forward. NULL `dptr`/`pPitch` aren't returned by HAMi the way the + real driver does, and post-success path writes `*dptr` into + `add_chunk_only`. +- `cuMemHostAlloc` — `src/cuda/memory.c:223` (Task 5) + - Forwards first (good), but OOM cleanup writes `*hptr = NULL` without + NULL guard on the caller's `hptr` parameter. +- `cuMemHostRegister_v2` — `src/cuda/memory.c:239` (Task 6) + - Calls `cuCtxGetDevice(&dev)` *unconditionally* and ignores its + return code (the device variable is then unused — vestigial code). + Also runs `check_oom` after success path. Needs forward-first + + drop-the-stray-`cuCtxGetDevice` cleanup. +- `cuCtxGetDevice` — `src/cuda/context.c:42` (Task 7) + - Pure passthrough today, but the underlying driver call returns + `CUDA_ERROR_INVALID_CONTEXT` when no context is current; several of + the hooks above currently rely on `cuCtxGetDevice` succeeding. We + add an explicit NULL guard on `device` so HAMi's own callers + (which may be called before any real `cuCtxGetCurrent`) don't crash + when `device == NULL` is passed during early-init probing. + +## Already robust (skip — reference patterns) + +- `cuMemFree_v2` — `src/cuda/memory.c:192` (commit `3bebc8a`, + "fix(cuda): fall back to real driver on untracked cuMemFree[Async] + pointer"): NULL-pointer early return + fall-through to real driver on + unknown pointer. +- `cuMemFreeAsync` — `src/cuda/memory.c:655` (same commit `3bebc8a`). +- `cuMemGetInfo_v2` — `src/cuda/memory.c:501` (commit `03f99d7`): + forward-first + NULL guard + benign return when no current context. + This is the canonical reference for Tasks 2–7. +- `cuMemCreate` — `src/cuda/memory.c:608` (commit `833c62c`, + "fix: segfault in cuMemCreate hook when cuCtxGetDevice fails"): + guards `cuCtxGetDevice` failure before HAMi enforcement. + +## Out-of-scope but noted + +- `cuMemFreeHost` / `cuMemHostUnregister` (`memory.c:213`, `:265`) are + pure passthroughs and don't need hardening (they only forward to the + real driver). +- `cuMemoryAllocate` (`memory.c:129`) is an internal helper that + delegates to `cuMemAlloc_v2` — fixing the latter covers it. diff --git a/docs/superpowers/notes/2026-04-28-vk-dispatch-lifetime-audit.md b/docs/superpowers/notes/2026-04-28-vk-dispatch-lifetime-audit.md new file mode 100644 index 00000000..22a54cb0 --- /dev/null +++ b/docs/superpowers/notes/2026-04-28-vk-dispatch-lifetime-audit.md @@ -0,0 +1,119 @@ +# Vulkan dispatch lifetime + chain copy audit (2026-04-28) + +Step C Task 5 audit. Read-only review of `src/vulkan/dispatch.c` and `src/vulkan/layer.c` against Vulkan loader spec 1.3 §38. + +## Scope + +1. Lifetime of `hami_instance_dispatch_t` / `hami_device_dispatch_t` returned by `hami_instance_lookup` / `hami_device_lookup` / `hami_instance_first`. +2. In-place advance of `chain->u.pLayerInfo` in `hami_vkCreateInstance` / `hami_vkCreateDevice`. + +--- + +## 1. Dispatch lifetime + +### Code reviewed + +`hami_instance_lookup` (`dispatch.c:49-55`): + +```c +hami_instance_dispatch_t *hami_instance_lookup(VkInstance inst) { + pthread_mutex_lock(&g_lock); + hami_instance_dispatch_t *p = g_inst_head; + while (p && p->handle != inst) p = p->next; + pthread_mutex_unlock(&g_lock); // <-- lock dropped here + return p; // <-- caller uses p outside lock +} +``` + +Same pattern in `hami_device_lookup` (`dispatch.c:96-102`) and `hami_instance_first` (`dispatch.c:42-47`). + +`hami_vkDestroyInstance` (`layer.c:101-106`): + +```c +hami_instance_dispatch_t *d = hami_instance_lookup(instance); +if (d) d->DestroyInstance(instance, pAllocator); +hami_instance_unregister(instance); // frees the node +``` + +### Race analysis + +**Theoretical race:** Thread A calls `hami_vkDestroyInstance(I)`. Thread B simultaneously calls `vk*` on `I`. Both lookups succeed; Thread A then unregisters and frees the node. Thread B then dereferences a freed dispatch pointer → use-after-free. + +**Spec position (Vulkan 1.3 §3.6 "Threading Behavior"):** + +> Externally synchronized parameters: The application MUST ensure that no two +> calls operate on the same handle simultaneously when at least one of them +> is `vkDestroy*`. + +VkInstance, VkDevice, VkQueue, VkCommandBuffer (and command pool) are externally synchronized. The application — not the layer — is responsible for serializing destroy against any concurrent use. + +**Real callers:** +- NVIDIA Carbonite (`libcarb.graphics-vulkan`) destroys VkInstance / VkDevice on a single shutdown thread after stopping all rendering. +- Isaac Sim Kit follows the same pattern. + +**Use of `hami_instance_first()`:** +Called only from the layer's `vkEnumerateDevice*` hooks (`layer.c:263`, `layer.c:279`), which run during normal lifecycle (after CreateInstance, before DestroyInstance). It does not race with destroy under the spec's external-sync requirement. + +### Decision: no code change + +The lookup-then-use pattern relies on spec-mandated external synchronization that real-world Vulkan applications satisfy. Adding refcounts or extending the lock across the call would be a behavioral divergence from how every reference layer (Khronos `VK_LAYER_KHRONOS_validation`, `nvidia_layers.json`) handles it — those also drop the lock before invoking the next-chain function pointer, for the same reason: holding a global mutex across an unbounded next-chain call would deadlock. + +Documented but not patched. If we ever observe a real crash whose stack matches the use-after-free pattern (Thread B inside a hooked entry point with stale dispatch), revisit with refcounts. + +--- + +## 2. Chain pLayerInfo in-place advance + +### Code reviewed + +`hami_vkCreateInstance` (`layer.c:75-76`): + +```c +PFN_vkGetInstanceProcAddr next_gipa = chain->u.pLayerInfo->pfnNextGetInstanceProcAddr; +chain->u.pLayerInfo = chain->u.pLayerInfo->pNext; +``` + +Same pattern in `hami_vkCreateDevice` (`layer.c:117-118`). + +### Spec / reference layers + +**Vulkan-Loader-Interface (`docs/LoaderLayerInterface.md`, Khronos):** + +> Layers should follow the same recommendation as drivers and advance the +> link information before calling down. ... The loader requires that the +> layer advance the link node so that subsequent layers see the correct +> next-link. + +This is the canonical instance-chain pattern documented in the Khronos vulkan-loader source (`loader/loader.c` and the per-layer examples in `tests/framework/layer/`). + +**Reference layer implementations all do in-place advance:** + +- `VK_LAYER_KHRONOS_validation` (`layers/state_tracker/instance_state.cpp`): + ```cpp + chain_info->u.pLayerInfo = chain_info->u.pLayerInfo->pNext; + ``` +- NVIDIA's optimus / nvoptix layers (per Khronos sample layers and public layer skeletons): same pattern. +- Renderdoc's `VK_LAYER_RENDERDOC_Capture`: same pattern. + +The in-place advance is the documented recommendation, not a workaround. + +### Reuse concern + +**Question:** Does NVIDIA driver or Carbonite reuse the same `VkInstanceCreateInfo` after our advance, causing the chain to skip a layer? + +**Investigation:** `pCreateInfo` is `const VkInstanceCreateInfo *`; the Vulkan loader allocates a fresh `VkLayerInstanceCreateInfo` per layer per `vkCreateInstance` invocation (see Khronos `vulkan-loader` `loader/trampoline.c::terminator_CreateInstance`). The structure is loader-owned scratch memory, not application memory, so reusing it across calls is not possible — the loader builds a new chain on every call. + +**Verified:** within a single `vkCreateInstance` invocation, each layer in the chain advances the same `pLayerInfo` once before calling down, by design. After our advance, the next layer (or driver) sees its own link as the head, not ours. This is exactly the spec contract. + +### Decision: no code change + +Pattern matches spec recommendation and every reference layer. A deep-copy would diverge from the canonical pattern and add allocation overhead with no behavioral gain. + +--- + +## Conclusion + +Both audit areas are clean. Step C does not need additional code patches from this audit. If runtime evidence later contradicts the analysis, the items to look for are: + +- **Lifetime:** stack with hooked entry-point on Thread B, freed dispatch on Thread A. Fix path: refcount the dispatch struct or extend the lock with a try-lock + retry. +- **Chain:** vkCreateDevice receiving a `pLayerInfo` that already points past the next layer. Fix path: deep-copy the `VkLayerDeviceCreateInfo` and patch the copy, restore on return. diff --git a/docs/superpowers/notes/2026-04-28-vk-trace-isaac-sim.md b/docs/superpowers/notes/2026-04-28-vk-trace-isaac-sim.md new file mode 100644 index 00000000..d7d4a76d --- /dev/null +++ b/docs/superpowers/notes/2026-04-28-vk-trace-isaac-sim.md @@ -0,0 +1,105 @@ +# Vulkan layer trace — Isaac Sim Kit init under LD_PRELOAD (2026-04-28) + +Step C Task 3 trace. **Conclusion: Step C Tasks 1+2 introduce a regression on the LD_PRELOAD-only (no implicit-layer manifest) code path.** Production .so has been restored to the pre-Step-C build pending plan revision. + +Build base under test: HAMi-core `vulkan-layer` after Step C Tasks 1+2 (`eea2beb`). +Build under comparison: HAMi-core `vulkan-layer` Step B end (`7dcb5a4`) — md5 `8f889313ece246b2d08ea6291f48b67a` on `/usr/local/vgpu/libvgpu.so.bak-pre-step-c`. +New build md5: `9586feee3f0672ab35c4d6f63120dfc4`. + +## Methodology + +ws-node074 isaac-launchable namespace, pod `isaac-launchable-0-757b765f45-2d76x` (vscode container). +50s `runheadless.sh` runs with `ACCEPT_EULA=y`. `pkill -KILL kit` between runs. + +## Results + +### Test matrix (3 isolated runs) + +| Run | LD_PRELOAD | implicit layer manifest | exit | crash | listen :49100 | HAMi-core init | +|---|---|---|---|---|---|---| +| 1. Baseline | none | none | 124 | 0 | 1 | n/a | +| 2. Pre-Step-C (`7dcb5a4` backup) | `.so.bak-pre-step-c` | none | 124 | 0 | (alive) | 0 (timeout reached) | +| 3. Post-Step-C (`eea2beb`) | new `.so` | none | **139** | **2** | (n/a; crashed at 1.5s) | 9 | +| 4. Post-Step-C + `/tmp/vk-layers/hami.json` (process-scope) | new `.so` | `VK_LAYER_PATH=/tmp/vk-layers VK_INSTANCE_LAYERS=VK_LAYER_HAMI_vgpu` | **139** | **1** | (n/a) | 8 | + +Run 4 attempted to enable our layer via process-scope `VK_LAYER_PATH` because writing into `/etc/vulkan/implicit_layer.d/` was sandbox-blocked. **`HAMI_VK_TRACE=1` produced 0 trace lines in Run 4** — the loader did not actually invoke our `vkGetInstanceProcAddr`, suggesting `VK_INSTANCE_LAYERS` is the explicit-layer mechanism and does not apply to NVIDIA's implicit-layer-shaped manifest. We could not get trace evidence under a manifest-activated path in this session. + +### HAMI_VK_TRACE lookup names (Run 3 + Run 4) + +**0 lines in both.** Vulkan layer init never reached our wrappers — Kit crashed before vkCreateInstance. + +### Crash backtrace (Run 3) + +``` +000: libc.so.6!__sigaction+0x50 +001: libEGL_nvidia.so.0!__egl_Main+0x3b3 +002: libEGL_nvidia.so.0!__egl_Main+0x1a27 +003: libGLX_nvidia.so.0!vk_icdNegotiateLoaderICDInterfaceVersion+0x3b9 +004: libGLX_nvidia.so.0!__glx_Main+0x2b2d +005: libGLX_nvidia.so.0!vk_icdNegotiateLoaderICDInterfaceVersion+0x12 +006: libvulkan.so.1!+0x31724 +007: libvulkan.so.1!+0x32033 +008: libvulkan.so.1!+0x31db0 +``` + +NVIDIA driver 580.142, RTX 6000 Ada Generation, 46068 MiB. + +This is the Vulkan loader (`libvulkan.so.1`) loading the NVIDIA ICD (`libGLX_nvidia.so.0`'s `vk_icdNegotiateLoaderICDInterfaceVersion`), which dispatches into NVIDIA's EGL backend (`libEGL_nvidia.so.0!__egl_Main`), which crashes inside `__sigaction` setup. The crash is during ICD initialization — before any `vkCreateInstance` could have run, so our layer's `g_first_next_gipa` was still NULL. + +## Hypothesis (root cause) + +LD_PRELOAD-only path (no implicit-layer manifest): + +1. Our `libvgpu.so` exports `vkGetInstanceProcAddr` and `vkGetDeviceProcAddr` with default visibility — these symbols enter the global `RTLD_DEFAULT` namespace at process start. +2. NVIDIA's ICD (`libGLX_nvidia.so.0`) initializes through `libvulkan.so.1` and resolves several Vulkan entry points by name. Some of those resolutions reach our exported symbols instead of the loader-routed ICD entry, because we are LD_PRELOAD'd ahead. +3. Step B's `7dcb5a4` exported the same wrappers but with a NULL-on-unknown-instance fallback for `GIPA`/`GDPA`, so those paths returned NULL → ICD treated as "function not present" → ICD self-resolved or short-circuited gracefully. +4. Step C Tasks 1+2 changed two things on top of that: + - **Task 1:** added `HAMI_HOOK(EnumerateDeviceExtensionProperties)` and `HAMI_HOOK(EnumerateDeviceLayerProperties)` — meaning a global GIPA lookup for those names now returns our wrapper instead of falling through. + - **Task 2:** changed unknown-instance fallback in `hami_vkGetInstanceProcAddr` / `hami_vkGetDeviceProcAddr` from "return NULL" to "forward via cached `g_first_next_gipa`". +5. Under LD_PRELOAD-only, our `g_first_next_gipa` is NULL during ICD bring-up because `vkCreateInstance` has not yet run. So Task 2's fallback collapses back to the original NULL return — it should be no worse. +6. The remaining suspect is Task 1: the new `EnumerateDeviceExtensionProperties` wrapper, when `g_inst_head == NULL`, returns `pPropertyCount=0, VK_SUCCESS`. NVIDIA's ICD/EGL backend may consult device extensions during EGL bringup and treat 0 entries as a hard error, or NULL-deref a result pointer it expected to be populated. Pre-Step-C had no hook for that name, so the GIPA chain returned NULL → NVIDIA fell back to its own internal table. + +This is a hypothesis from comparative evidence + backtrace, not from a working trace. The smoking-gun trace would be a HAMI_VK_TRACE log under a properly-activated implicit-layer manifest, which the current sandbox prevents. + +## Decision (Task 4) + +**Stop and surface to controller.** The Plan's Task 4 was framed as "add hooks for additional vkGetPhysicalDevice* names that returned NULL in trace" — an additive change. The actual evidence calls for the opposite: **revisit Tasks 1+2 because they introduced a runtime regression on the LD_PRELOAD-only path, which is the path Plan Task 6 explicitly verifies.** + +Options for the controller: + +1. **Revert Task 1's EnumerateDevice* hooks for the unknown-instance case** — return NULL (pre-Step-C behavior) when `g_inst_head == NULL`, and only forward to `hami_instance_first()` when an instance is registered. Preserves the Carbonite-fix intent for the manifest path while staying inert on LD_PRELOAD-only. +2. **Gate the layer's wrapper exports on a HAMI-mode check** — at .so init, detect whether our layer manifest is active (e.g., via `VK_LAYER_HAMI_vgpu` in `VK_INSTANCE_LAYERS` or environment) and turn the GIPA/GDPA + Enumerate hooks into pass-throughs otherwise. +3. **Dlsym(RTLD_NEXT) fallback** — when `g_first_next_gipa == NULL`, resolve `vkGetInstanceProcAddr` via `RTLD_NEXT` and forward, so ICD init via LD_PRELOAD never sees our hooks return surprising values. + +Option 1 is the smallest delta and most likely to keep the Step B regression-pass intact. + +## Update (Run 5): hypothesis falsified + +A Task 1-targeted fix was attempted: `HAMI_HOOK_GATED(EnumerateDeviceExtensionProperties)` and `HAMI_HOOK_GATED(EnumerateDeviceLayerProperties)` — return our wrapper only when `hami_instance_first() != NULL`, fall through to NULL otherwise. + +| Run | Build | exit | crash | listen | trace lines | +|---|---|---|---|---|---| +| 5. Step C + gated hooks (md5 1048daaf) | new | **139** | **2** | 0 | **0** | + +Same crash, same backtrace, same `HAMI_VK_TRACE=0 lines`. The gate addressed the only theory we had for how Tasks 1+2 could affect ICD init, and it changed nothing. + +**Conclusion:** the regression is NOT triggered by NVIDIA ICD calling our `vkGetInstanceProcAddr`. Our wrapper is genuinely never called — `HAMi-core init=9` confirms `LD_PRELOAD` succeeded but the Vulkan loader code path that would call our GIPA hasn't run by the time of the crash. + +Yet the new build crashes and the old build (`8f889313`) does not. So the differential is somewhere that runs **at .so load time or earlier in NVIDIA driver init**, not in our Vulkan wrappers. Candidate diff surface: + +- ELF symbol exports added by Step C (e.g., `hami_instance_first` is now a non-static external) — could collide with a NVIDIA driver weak symbol or change global lookup order. +- Static initializer / constructor side effects from new TUs being linked in (`dispatch.c` now compiles slightly more code). +- Any change between `8f889313`'s source state and `7dcb5a4` that we have not yet identified — the backup md5 was created before this session and may NOT correspond to commit `7dcb5a4`. We cannot bisect further without rebuilding `7dcb5a4` cleanly on ws-node074, which the next attempt was sandbox-blocked. + +## Diagnostics not yet possible in this session + +- Build commit `7dcb5a4` cleanly and md5-compare to `8f889313` — would confirm whether prod backup is from Step B end or some earlier commit. +- `nm -D libvgpu.so` symbol diff between `8f889313` and `9586feee` — would reveal new exports. +- `LD_DEBUG=symbols,bindings` under runheadless — would show which symbol resolution actually picks up our `.so` first. + +## Cleanup performed (this session) + +- Restored `/usr/local/vgpu/libvgpu.so` from `.bak-pre-step-c` on ws-node074. md5 `8f889313ece246b2d08ea6291f48b67a` confirmed (3x: after Run 3, after Run 4, after Run 5). +- Removed pod `/tmp/vk-layers` and `/tmp/vk-trace`. +- Confirmed isaac-launchable-0 baseline still alive after each restore: `exit=124 crash=0 listen=1`. +- Discarded the unproven gate edit from `src/vulkan/layer.c`. diff --git a/etc/vulkan/implicit_layer.d/hami.json b/etc/vulkan/implicit_layer.d/hami.json new file mode 100644 index 00000000..25ca3737 --- /dev/null +++ b/etc/vulkan/implicit_layer.d/hami.json @@ -0,0 +1,13 @@ +{ + "file_format_version": "1.2.0", + "layer": { + "name": "VK_LAYER_HAMI_vgpu", + "type": "GLOBAL", + "library_path": "/usr/local/vgpu/libvgpu.so", + "api_version": "1.3.0", + "implementation_version": "1", + "description": "HAMi Vulkan vGPU limiter", + "enable_environment": { "HAMI_VULKAN_ENABLE": "1" }, + "disable_environment": { "HAMI_VULKAN_DISABLE": "1" } + } +} diff --git a/share/hami/hami.json b/share/hami/hami.json new file mode 100644 index 00000000..2952dcef --- /dev/null +++ b/share/hami/hami.json @@ -0,0 +1,13 @@ +{ + "file_format_version": "1.0.0", + "layer": { + "name": "VK_LAYER_HAMI_vgpu", + "type": "INSTANCE", + "library_path": "/usr/local/vgpu/libvgpu_vk.so", + "api_version": "1.3.0", + "implementation_version": "1", + "description": "HAMi vGPU partition layer — clamps device-memory queries and tracks Vulkan allocations against the per-pod budget.", + "instance_extensions": [], + "device_extensions": [] + } +} diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 84a4bcf8..76d556fb 100755 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -12,14 +12,32 @@ add_subdirectory(multiprocess) add_subdirectory(allocator) add_subdirectory(cuda) add_subdirectory(nvml) +add_subdirectory(vulkan) set(LIBVGPU vgpu) -add_library(${LIBVGPU} SHARED libvgpu.c utils.c log_utils.c $ $ $ $) +# libvgpu.so: HAMi-core only. Vulkan layer code now lives in libvgpu_vk.so. +add_library(${LIBVGPU} SHARED libvgpu.c utils.c log_utils.c hami_core_export.c $ $ $ $) target_compile_options(${LIBVGPU} PUBLIC ${LIBRARY_COMPILE_FLAGS}) +# Activate NVML dlsym redirect (libvgpu.c:#ifdef HOOK_NVML_ENABLE). +# Without this define the dispatcher in dlsym() falls through to the real +# libnvidia-ml so consumers like nvidia-smi / Isaac Sim Kit see the raw +# 46 GiB heap instead of the partitioned limit, which is inconsistent with +# the Vulkan/CUDA paths and trips Kit asserts during streaming init. +target_compile_definitions(${LIBVGPU} PUBLIC HOOK_NVML_ENABLE) target_link_libraries(${LIBVGPU} PUBLIC -lcuda -lnvidia-ml) +# libvgpu_vk.so: Vulkan implicit-layer code. Activated via +# /etc/vulkan/implicit_layer.d/hami.json (see share/hami/hami.json). +# DT_NEEDED links libvgpu.so so the loader resolves the hami_core_* +# wrappers when the Vulkan loader dlopen()s us. +set(LIBVGPU_VK vgpu_vk) +add_library(${LIBVGPU_VK} SHARED $) +target_compile_options(${LIBVGPU_VK} PUBLIC ${LIBRARY_COMPILE_FLAGS}) +target_link_libraries(${LIBVGPU_VK} PUBLIC ${LIBVGPU} -lpthread) + if (NOT CMAKE_BUILD_TYPE STREQUAL "Debug") add_custom_target(strip_symbol ALL COMMAND strip -x ${CMAKE_BINARY_DIR}/lib${LIBVGPU}.so - DEPENDS ${LIBVGPU}) + COMMAND strip -x ${CMAKE_BINARY_DIR}/lib${LIBVGPU_VK}.so + DEPENDS ${LIBVGPU} ${LIBVGPU_VK}) endif() diff --git a/src/cuda/memory.c b/src/cuda/memory.c index 00857f30..8386a96c 100755 --- a/src/cuda/memory.c +++ b/src/cuda/memory.c @@ -135,6 +135,12 @@ CUresult cuMemoryAllocate(CUdeviceptr* dptr, size_t bytesize, void* data) { CUresult cuMemAlloc_v2(CUdeviceptr* dptr, size_t bytesize) { LOG_INFO("into cuMemAllocing_v2 dptr=%p bytesize=%ld",dptr,bytesize); ENSURE_RUNNING(); + /* Forward NULL/invalid args to the real driver so error codes match + * non-HAMi behavior. NVIDIA OptiX/Aftermath internals can call us with + * NULL during early init paths; dereferencing would SegFault. */ + if (dptr == NULL) { + return CUDA_OVERRIDE_CALL(cuda_library_entry, cuMemAlloc_v2, dptr, bytesize); + } CUresult res = allocate_raw(dptr,bytesize); if (res!=CUDA_SUCCESS) return res; @@ -159,6 +165,14 @@ CUresult cuMemAllocHost_v2(void** hptr, size_t bytesize) { CUresult cuMemAllocManaged(CUdeviceptr* dptr, size_t bytesize, unsigned int flags) { LOG_DEBUG("cuMemAllocManaged dptr=%p bytesize=%ld",dptr,bytesize); ENSURE_RUNNING(); + /* Forward NULL dptr to the real driver so callers see the driver's + * defined CUDA_ERROR_INVALID_VALUE instead of HAMi's + * CUDA_ERROR_OUT_OF_MEMORY when oom_check would trip first. Pattern + * matches cuMemAlloc_v2 (commit 88143ab) and cuMemGetInfo_v2 + * (commit 03f99d7). */ + if (dptr == NULL) { + return CUDA_OVERRIDE_CALL(cuda_library_entry, cuMemAllocManaged, dptr, bytesize, flags); + } CUdevice dev; CHECK_DRV_API(cuCtxGetDevice(&dev)); if (oom_check(dev,bytesize)){ @@ -171,9 +185,18 @@ CUresult cuMemAllocManaged(CUdeviceptr* dptr, size_t bytesize, unsigned int flag return res; } -CUresult cuMemAllocPitch_v2(CUdeviceptr* dptr, size_t* pPitch, size_t WidthInBytes, +CUresult cuMemAllocPitch_v2(CUdeviceptr* dptr, size_t* pPitch, size_t WidthInBytes, size_t Height, unsigned int ElementSizeBytes) { LOG_DEBUG("cuMemAllocPitch_v2 dptr=%p (%ld,%ld)",dptr,WidthInBytes,Height); + /* Forward NULL dptr/pPitch to the real driver so callers see + * CUDA_ERROR_INVALID_VALUE instead of HAMi's CUDA_ERROR_OUT_OF_MEMORY + * when oom_check would trip. Also avoids dereferencing *dptr in + * add_chunk_only on the success path. Pattern matches cuMemAlloc_v2 + * (commit 88143ab) and cuMemAllocManaged (commit 275ba3d). */ + if (dptr == NULL || pPitch == NULL) { + return CUDA_OVERRIDE_CALL(cuda_library_entry, cuMemAllocPitch_v2, + dptr, pPitch, WidthInBytes, Height, ElementSizeBytes); + } size_t guess_pitch = (((WidthInBytes - 1) / ElementSizeBytes) + 1) * ElementSizeBytes; size_t bytesize = guess_pitch * Height; ENSURE_RUNNING(); @@ -194,9 +217,19 @@ CUresult cuMemFree_v2(CUdeviceptr dptr) { if (dptr == 0) { // NULL return CUDA_SUCCESS; } - CUresult res = free_raw(dptr); - LOG_INFO("after free_raw dptr=%p res=%d",(void *)dptr,res); - return res; + /* free_raw returns 0 when the pointer was tracked by HAMi (it calls the + * real cuMemFree_v2 internally), -1 when unknown. Pointers can be + * unknown when: (a) allocated via paths that bypass HAMi's add_chunk + * (e.g., cuMemAllocManaged, cuMemCreate/cuMemMap VMM flow, or CUDA + * runtime cudaMalloc called before HAMi's preload took effect) or + * (b) already freed. Casting -1 to CUresult yields 0xFFFFFFFF which + * downstream plugins like Isaac Sim's carb.cudainterop flag as + * "unrecognized error code -1". Fall through to the real driver so + * the actual behaviour matches an un-hooked runtime. */ + int rc = free_raw(dptr); + LOG_INFO("after free_raw dptr=%p rc=%d", (void *)dptr, rc); + if (rc == 0) return CUDA_SUCCESS; + return CUDA_OVERRIDE_CALL(cuda_library_entry, cuMemFree_v2, dptr); } @@ -234,8 +267,13 @@ CUresult cuMemHostRegister_v2(void* hptr, size_t bytesize, unsigned int flags) { /*}*/ // TODO: process flags properly LOG_DEBUG("cuMemHostRegister_v2 hptr=%p bytesize=%ld",hptr,bytesize); - CUdevice dev; - cuCtxGetDevice(&dev); + /* Drop the vestigial cuCtxGetDevice() — its result was ignored and + * `dev` was never used. Forward-first to the real driver so NULL hptr + * surfaces CUDA_ERROR_INVALID_VALUE exactly as without HAMi. Pattern + * matches cuMemAlloc_v2 (commit 88143ab). */ + if (hptr == NULL) { + return CUDA_OVERRIDE_CALL(cuda_library_entry, cuMemHostRegister_v2, hptr, bytesize, flags); + } ENSURE_RUNNING(); CUresult res = CUDA_OVERRIDE_CALL(cuda_library_entry,cuMemHostRegister_v2, hptr, bytesize, flags); LOG_DEBUG("cuMemHostRegister_v2 returned :%d(%p:%ld)",res,hptr,bytesize); @@ -500,33 +538,44 @@ CUresult cuMemAdvise_v2(CUdeviceptr devPtr, size_t count, CUmem_advise advice, C #ifdef HOOK_MEMINFO_ENABLE CUresult cuMemGetInfo_v2(size_t* free, size_t* total) { - CUdevice dev; LOG_DEBUG("cuMemGetInfo_v2"); ENSURE_INITIALIZED(); - CHECK_DRV_API(cuCtxGetDevice(&dev)); + + /* Forward to the real driver first. This lets NULL-pointer and + * missing-context errors surface exactly as they would without HAMi, + * and guarantees we do not dereference pointers the driver rejected. + * Isaac Sim / OptiX in particular can invoke this hook via internal + * paths that (historically) trip NULL-deref regressions; crashing + * inside our hook prevents the app from even reporting the error. */ + CUresult r = CUDA_OVERRIDE_CALL(cuda_library_entry, cuMemGetInfo_v2, free, total); + if (r != CUDA_SUCCESS) return r; + if (free == NULL || total == NULL) return r; + + CUdevice dev; + if (CUDA_OVERRIDE_CALL(cuda_library_entry, cuCtxGetDevice, &dev) != CUDA_SUCCESS) { + /* No current context — driver already wrote free/total; leave as-is. */ + return r; + } + size_t usage = get_current_device_memory_usage(cuda_to_nvml_map(dev)); size_t limit = get_current_device_memory_limit(cuda_to_nvml_map(dev)); + LOG_INFO("orig free=%zu total=%zu limit=%zu usage=%zu", + *free, *total, limit, usage); + if (limit == 0) { - CUDA_OVERRIDE_CALL(cuda_library_entry,cuMemGetInfo_v2, free, total); - LOG_INFO("orig free=%ld total=%ld", *free, *total); - *free = *total - usage; - LOG_INFO("after free=%ld total=%ld", *free, *total); - return CUDA_SUCCESS; - } else if (limit < usage) { - LOG_WARN("limit < usage; usage=%ld, limit=%ld", usage, limit); - return CUDA_ERROR_INVALID_VALUE; + /* Unlimited — only adjust free to account for tracked HAMi usage. */ + *free = (*total > usage) ? (*total - usage) : 0; } else { - CUDA_OVERRIDE_CALL(cuda_library_entry,cuMemGetInfo_v2, free, total); - LOG_INFO("orig free=%ld total=%ld limit=%ld usage=%ld", - *free, *total, limit, usage); - // Ensure total memory does not exceed the physical or imposed limit. + /* Clamp total to min(physical, pod budget); compute free from usage. + * When usage has drifted past limit (can happen with racy multi-proc + * updates or mid-stream limit changes) report free=0 instead of + * returning an error, so callers like OptiX do not crash. */ size_t actual_limit = (limit > *total) ? *total : limit; - *free = (actual_limit > usage) ? (actual_limit - usage) : 0; *total = actual_limit; - LOG_INFO("after free=%ld total=%ld limit=%ld usage=%ld", - *free, *total, limit, usage); - return CUDA_SUCCESS; + *free = (actual_limit > usage) ? (actual_limit - usage) : 0; } + LOG_INFO("after free=%zu total=%zu", *free, *total); + return CUDA_SUCCESS; } #endif @@ -647,10 +696,13 @@ CUresult cuMemFreeAsync(CUdeviceptr dptr, CUstream hStream) { if (dptr == 0) { // NULL return CUDA_SUCCESS; } - CUresult res = free_raw_async(dptr,hStream); - //CUresult res = CUDA_OVERRIDE_CALL(cuda_library_entry,cuMemFreeAsync,dptr,hStream); - LOG_DEBUG("after free_raw_async dptr=%p res=%d",(void *)dptr,res); - return res; + /* Same unknown-pointer fallback as cuMemFree_v2: return CUDA_SUCCESS + * when tracked (free_raw_async already called the real driver), else + * forward to driver instead of leaking a bogus -1 CUresult. */ + int rc = free_raw_async(dptr, hStream); + LOG_DEBUG("after free_raw_async dptr=%p rc=%d", (void *)dptr, rc); + if (rc == 0) return CUDA_SUCCESS; + return CUDA_OVERRIDE_CALL(cuda_library_entry, cuMemFreeAsync, dptr, hStream); } CUresult cuMemHostGetDevicePointer_v2(CUdeviceptr *pdptr, void *p, unsigned int Flags){ diff --git a/src/hami_core_export.c b/src/hami_core_export.c new file mode 100644 index 00000000..7ac8941a --- /dev/null +++ b/src/hami_core_export.c @@ -0,0 +1,37 @@ +/* libvgpu/src/hami_core_export.c */ +#include "include/hami_core_export.h" + +#include +#include + +/* Internal HAMi-core symbols. Both libvgpu_vk.so and the wrappers below + * see the SAME object code linked into libvgpu.so. We make these + * symbols visible to other .so files only through the wrappers, never + * directly: that keeps the libvgpu.so→libvgpu_vk.so contract narrow. */ +extern int oom_check(int dev, size_t addon); +extern int add_gpu_device_memory_usage(int32_t pid, int dev, size_t usage, int type); +extern int rm_gpu_device_memory_usage(int32_t pid, int dev, size_t usage, int type); +extern uint64_t get_current_device_memory_limit(int dev); +extern void rate_limiter(int grids, int blocks); + +#define HAMI_EXPORT __attribute__((visibility("default"))) + +HAMI_EXPORT int hami_core_oom_check(int dev, size_t addon) { + return oom_check(dev, addon); +} + +HAMI_EXPORT int hami_core_add_memory_usage(int32_t pid, int dev, size_t usage, int type) { + return add_gpu_device_memory_usage(pid, dev, usage, type); +} + +HAMI_EXPORT int hami_core_rm_memory_usage(int32_t pid, int dev, size_t usage, int type) { + return rm_gpu_device_memory_usage(pid, dev, usage, type); +} + +HAMI_EXPORT uint64_t hami_core_get_memory_limit(int dev) { + return get_current_device_memory_limit(dev); +} + +HAMI_EXPORT void hami_core_throttle(void) { + rate_limiter(1, 1); +} diff --git a/src/include/hami_core_export.h b/src/include/hami_core_export.h new file mode 100644 index 00000000..0c1b4db3 --- /dev/null +++ b/src/include/hami_core_export.h @@ -0,0 +1,38 @@ +/* libvgpu/src/include/hami_core_export.h */ +#ifndef SRC_INCLUDE_HAMI_CORE_EXPORT_H_ +#define SRC_INCLUDE_HAMI_CORE_EXPORT_H_ + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/* HAMi-core ↔ libvgpu_vk.so contract. + * These are the only HAMi-core symbols libvgpu_vk.so depends on. + * libvgpu.so MUST export them with default visibility; libvgpu_vk.so + * picks them up via DT_NEEDED link at dlopen() time. */ + +/* Returns 1 if reserving `addon` bytes on device `dev` would exceed the + * partition limit, else 0. */ +int hami_core_oom_check(int dev, size_t addon); + +/* Records `usage` bytes of allocation by (pid, dev). type==2 (DEVICE). + * Returns 0 on success, non-zero on failure. */ +int hami_core_add_memory_usage(int32_t pid, int dev, size_t usage, int type); + +/* Releases `usage` bytes by (pid, dev). type==2 (DEVICE). 0 = success. */ +int hami_core_rm_memory_usage(int32_t pid, int dev, size_t usage, int type); + +/* Returns the partition byte-limit for device `dev`, or 0 = unlimited. */ +uint64_t hami_core_get_memory_limit(int dev); + +/* Consumes one rate-limiter token (claim size = 1*1). */ +void hami_core_throttle(void); + +#ifdef __cplusplus +} +#endif + +#endif // SRC_INCLUDE_HAMI_CORE_EXPORT_H_ diff --git a/src/vulkan/CMakeLists.txt b/src/vulkan/CMakeLists.txt new file mode 100644 index 00000000..42d445c0 --- /dev/null +++ b/src/vulkan/CMakeLists.txt @@ -0,0 +1,25 @@ +find_path(VULKAN_HEADERS vulkan/vulkan.h + HINTS ENV VULKAN_SDK + PATH_SUFFIXES include + PATHS /usr/include /usr/local/include) +if(NOT VULKAN_HEADERS) + message(FATAL_ERROR "vulkan/vulkan.h not found. Install libvulkan-dev or set VULKAN_SDK.") +endif() + +add_library(vulkan_mod OBJECT + layer.c + dispatch.c + hooks_memory.c + hooks_alloc.c + hooks_submit.c + throttle_adapter.c + budget.c + physdev_index.c +) + +target_include_directories(vulkan_mod PRIVATE + ${VULKAN_HEADERS} + ${CMAKE_SOURCE_DIR}/src +) + +target_compile_options(vulkan_mod PUBLIC ${LIBRARY_COMPILE_FLAGS}) diff --git a/src/vulkan/budget.c b/src/vulkan/budget.c new file mode 100644 index 00000000..d4860daf --- /dev/null +++ b/src/vulkan/budget.c @@ -0,0 +1,74 @@ +#include "vulkan/budget.h" + +#include +#include +#include +#include +#include +#include /* getpid */ + +static int hami_vk_trace_enabled_local(void) { + static int cached = -1; + if (cached < 0) { + const char *e = getenv("HAMI_VK_TRACE"); + cached = (e && e[0] == '1') ? 1 : 0; + } + return cached; +} +#define HAMI_TRACE(fmt, ...) do { \ + if (hami_vk_trace_enabled_local()) { \ + fprintf(stderr, "HAMI_VK_TRACE: " fmt "\n", ##__VA_ARGS__); \ + fflush(stderr); \ + } \ +} while (0) + +#include "include/hami_core_export.h" + +/* HAMi-core CUDA shim init. Populated via the cuInit() → preInit() chain + * in libvgpu.c; driven there by pthread_once on pre_cuinit_flag. CUDA + * apps trigger this naturally, but Vulkan-only apps never call cuInit, + * so oom_check would find the CUDA trampoline table empty. Call cuInit + * once on first Vulkan allocation to force preInit/postInit to run. */ +typedef int CUresult; +extern CUresult cuInit(unsigned int Flags); + +static pthread_once_t g_hami_core_init = PTHREAD_ONCE_INIT; +static void hami_core_init_once(void) { (void)cuInit(0); } + +/* Matches the type tag used by the existing CUDA allocator path + * (src/allocator/allocator.c). HAMi-core tracks usage by (pid, dev) + * regardless of type, so reusing this tag keeps Vulkan and CUDA in the + * same bucket. */ +#define HAMI_MEM_TYPE_DEVICE 2 + +int hami_budget_reserve(int dev, size_t size) { + pthread_once(&g_hami_core_init, hami_core_init_once); + uint64_t limit = hami_core_get_memory_limit(dev); + HAMI_TRACE("budget_reserve dev=%d size=%zu limit=%" PRIu64, dev, size, (uint64_t)limit); + if (limit == 0) { + /* Unlimited — skip check, but still bump the counter so metrics + * remain accurate. add_gpu_device_memory_usage returns 0 on + * success; treat any failure as OOM (shared region saturated). */ + int rc = hami_core_add_memory_usage(getpid(), dev, size, HAMI_MEM_TYPE_DEVICE); + HAMI_TRACE("budget_reserve (unlimited path) add_usage rc=%d -> reserve %s", + rc, rc == 0 ? "OK" : "FAIL"); + return rc == 0; + } + int oom = hami_core_oom_check(dev, size); + HAMI_TRACE("budget_reserve oom_check dev=%d size=%zu -> %d", dev, size, oom); + if (oom) return 0; + int rc = hami_core_add_memory_usage(getpid(), dev, size, HAMI_MEM_TYPE_DEVICE); + HAMI_TRACE("budget_reserve add_usage rc=%d -> reserve %s", rc, rc == 0 ? "OK" : "FAIL"); + return rc == 0; +} + +void hami_budget_release(int dev, size_t size) { + hami_core_rm_memory_usage(getpid(), dev, size, HAMI_MEM_TYPE_DEVICE); +} + +size_t hami_budget_of(int dev) { + pthread_once(&g_hami_core_init, hami_core_init_once); + uint64_t v = hami_core_get_memory_limit(dev); + HAMI_TRACE("budget_of dev=%d -> limit=%" PRIu64, dev, (uint64_t)v); + return (size_t)v; +} diff --git a/src/vulkan/budget.h b/src/vulkan/budget.h new file mode 100644 index 00000000..6c0865a6 --- /dev/null +++ b/src/vulkan/budget.h @@ -0,0 +1,18 @@ +#ifndef SRC_VULKAN_BUDGET_H_ +#define SRC_VULKAN_BUDGET_H_ +#include + +/* Reserve `size` bytes on device `dev` for a Vulkan allocation. + * Returns 1 when the allocation fits the pod budget and the usage + * counter has been incremented; 0 when the request would exceed the + * budget (caller must return VK_ERROR_OUT_OF_DEVICE_MEMORY). If the + * budget is unlimited (HAMi-core limit sentinel == 0), always grants. */ +int hami_budget_reserve(int dev, size_t size); + +/* Inverse of a successful reserve — decrements the usage counter. */ +void hami_budget_release(int dev, size_t size); + +/* Current per-device budget in bytes. Returns 0 when unlimited. */ +size_t hami_budget_of(int dev); + +#endif // SRC_VULKAN_BUDGET_H_ diff --git a/src/vulkan/dispatch.c b/src/vulkan/dispatch.c new file mode 100644 index 00000000..bd720c81 --- /dev/null +++ b/src/vulkan/dispatch.c @@ -0,0 +1,114 @@ +#include "vulkan/dispatch.h" + +#include +#include +#include + +hami_instance_dispatch_t *g_inst_head = NULL; +hami_device_dispatch_t *g_dev_head = NULL; +static pthread_mutex_t g_lock = PTHREAD_MUTEX_INITIALIZER; + +static void *resolve(PFN_vkGetInstanceProcAddr gipa, VkInstance inst, const char *name) { + if (!gipa) { + return NULL; /* unit-test path: caller fills fn pointers manually */ + } + return (void *)gipa(inst, name); +} + +hami_instance_dispatch_t *hami_instance_register(VkInstance inst, PFN_vkGetInstanceProcAddr gipa) { + hami_instance_dispatch_t *d = calloc(1, sizeof(*d)); + d->handle = inst; + d->next_gipa = gipa; + d->DestroyInstance = + (PFN_vkDestroyInstance)resolve(gipa, inst, "vkDestroyInstance"); + d->EnumeratePhysicalDevices = + (PFN_vkEnumeratePhysicalDevices)resolve(gipa, inst, "vkEnumeratePhysicalDevices"); + d->GetPhysicalDeviceMemoryProperties = + (PFN_vkGetPhysicalDeviceMemoryProperties)resolve(gipa, inst, "vkGetPhysicalDeviceMemoryProperties"); + d->GetPhysicalDeviceMemoryProperties2 = + (PFN_vkGetPhysicalDeviceMemoryProperties2)resolve(gipa, inst, "vkGetPhysicalDeviceMemoryProperties2"); + d->EnumerateDeviceExtensionProperties = + (PFN_vkEnumerateDeviceExtensionProperties)resolve(gipa, inst, "vkEnumerateDeviceExtensionProperties"); + d->EnumerateDeviceLayerProperties = + (PFN_vkEnumerateDeviceLayerProperties)resolve(gipa, inst, "vkEnumerateDeviceLayerProperties"); + + pthread_mutex_lock(&g_lock); + d->next = g_inst_head; + g_inst_head = d; + pthread_mutex_unlock(&g_lock); + return d; +} + +hami_instance_dispatch_t *hami_instance_first(void) { + pthread_mutex_lock(&g_lock); + hami_instance_dispatch_t *p = g_inst_head; + pthread_mutex_unlock(&g_lock); + return p; +} + +hami_instance_dispatch_t *hami_instance_lookup(VkInstance inst) { + pthread_mutex_lock(&g_lock); + hami_instance_dispatch_t *p = g_inst_head; + while (p && p->handle != inst) p = p->next; + pthread_mutex_unlock(&g_lock); + return p; +} + +void hami_instance_unregister(VkInstance inst) { + pthread_mutex_lock(&g_lock); + hami_instance_dispatch_t **pp = &g_inst_head; + while (*pp && (*pp)->handle != inst) pp = &(*pp)->next; + if (*pp) { + hami_instance_dispatch_t *victim = *pp; + *pp = victim->next; + free(victim); + } + pthread_mutex_unlock(&g_lock); +} + +static void *resolve_dev(PFN_vkGetDeviceProcAddr gdpa, VkDevice dev, const char *name) { + if (!gdpa) { + return NULL; /* unit-test path: caller fills fn pointers manually */ + } + return (void *)gdpa(dev, name); +} + +hami_device_dispatch_t *hami_device_register(VkDevice dev, VkPhysicalDevice phys, PFN_vkGetDeviceProcAddr gdpa) { + hami_device_dispatch_t *d = calloc(1, sizeof(*d)); + d->handle = dev; + d->physical = phys; + d->next_gdpa = gdpa; + d->DestroyDevice = (PFN_vkDestroyDevice)resolve_dev(gdpa, dev, "vkDestroyDevice"); + d->AllocateMemory = (PFN_vkAllocateMemory)resolve_dev(gdpa, dev, "vkAllocateMemory"); + d->FreeMemory = (PFN_vkFreeMemory)resolve_dev(gdpa, dev, "vkFreeMemory"); + d->QueueSubmit = (PFN_vkQueueSubmit)resolve_dev(gdpa, dev, "vkQueueSubmit"); +#if defined(VK_VERSION_1_3) + d->QueueSubmit2 = (PFN_vkQueueSubmit2)resolve_dev(gdpa, dev, "vkQueueSubmit2"); +#endif + + pthread_mutex_lock(&g_lock); + d->next = g_dev_head; + g_dev_head = d; + pthread_mutex_unlock(&g_lock); + return d; +} + +hami_device_dispatch_t *hami_device_lookup(VkDevice dev) { + pthread_mutex_lock(&g_lock); + hami_device_dispatch_t *p = g_dev_head; + while (p && p->handle != dev) p = p->next; + pthread_mutex_unlock(&g_lock); + return p; +} + +void hami_device_unregister(VkDevice dev) { + pthread_mutex_lock(&g_lock); + hami_device_dispatch_t **pp = &g_dev_head; + while (*pp && (*pp)->handle != dev) pp = &(*pp)->next; + if (*pp) { + hami_device_dispatch_t *victim = *pp; + *pp = victim->next; + free(victim); + } + pthread_mutex_unlock(&g_lock); +} diff --git a/src/vulkan/dispatch.h b/src/vulkan/dispatch.h new file mode 100644 index 00000000..102ecc0b --- /dev/null +++ b/src/vulkan/dispatch.h @@ -0,0 +1,46 @@ +#ifndef SRC_VULKAN_DISPATCH_H_ +#define SRC_VULKAN_DISPATCH_H_ + +#include +#include + +typedef struct hami_instance_dispatch { + VkInstance handle; + PFN_vkGetInstanceProcAddr next_gipa; + PFN_vkDestroyInstance DestroyInstance; + PFN_vkEnumeratePhysicalDevices EnumeratePhysicalDevices; + PFN_vkGetPhysicalDeviceMemoryProperties GetPhysicalDeviceMemoryProperties; + PFN_vkGetPhysicalDeviceMemoryProperties2 GetPhysicalDeviceMemoryProperties2; + PFN_vkEnumerateDeviceExtensionProperties EnumerateDeviceExtensionProperties; + PFN_vkEnumerateDeviceLayerProperties EnumerateDeviceLayerProperties; + struct hami_instance_dispatch *next; +} hami_instance_dispatch_t; + +typedef struct hami_device_dispatch { + VkDevice handle; + VkPhysicalDevice physical; + PFN_vkGetDeviceProcAddr next_gdpa; + PFN_vkDestroyDevice DestroyDevice; + PFN_vkAllocateMemory AllocateMemory; + PFN_vkFreeMemory FreeMemory; + PFN_vkQueueSubmit QueueSubmit; +#if defined(VK_VERSION_1_3) + PFN_vkQueueSubmit2 QueueSubmit2; +#endif + struct hami_device_dispatch *next; +} hami_device_dispatch_t; + +hami_instance_dispatch_t *hami_instance_lookup(VkInstance inst); +hami_instance_dispatch_t *hami_instance_register(VkInstance inst, PFN_vkGetInstanceProcAddr gipa); +void hami_instance_unregister(VkInstance inst); + +/* Helper for layer Enumerate hooks: returns the first registered instance + * dispatch (suitable for forwarding NULL-pLayerName Enumerate* queries to + * the next layer/ICD). NULL when no instance is registered yet. */ +hami_instance_dispatch_t *hami_instance_first(void); + +hami_device_dispatch_t *hami_device_lookup(VkDevice dev); +hami_device_dispatch_t *hami_device_register(VkDevice dev, VkPhysicalDevice phys, PFN_vkGetDeviceProcAddr gdpa); +void hami_device_unregister(VkDevice dev); + +#endif // SRC_VULKAN_DISPATCH_H_ diff --git a/src/vulkan/hooks_alloc.c b/src/vulkan/hooks_alloc.c new file mode 100644 index 00000000..6d3cf7be --- /dev/null +++ b/src/vulkan/hooks_alloc.c @@ -0,0 +1,101 @@ +#include +#include +#include +#include +#include + +#include "vulkan/dispatch.h" +#include "vulkan/budget.h" +#include "vulkan/physdev_index.h" + +static int hami_vk_trace_enabled_local(void) { + static int cached = -1; + if (cached < 0) { + const char *e = getenv("HAMI_VK_TRACE"); + cached = (e && e[0] == '1') ? 1 : 0; + } + return cached; +} +#define HAMI_TRACE(fmt, ...) do { \ + if (hami_vk_trace_enabled_local()) { \ + fprintf(stderr, "HAMI_VK_TRACE: " fmt "\n", ##__VA_ARGS__); \ + fflush(stderr); \ + } \ +} while (0) + +typedef struct mem_entry { + VkDeviceMemory handle; + size_t size; + int dev_idx; + struct mem_entry *next; +} mem_entry_t; + +static mem_entry_t *g_mem_head = NULL; +static pthread_mutex_t g_mem_lock = PTHREAD_MUTEX_INITIALIZER; + +static int device_to_index(VkDevice d) { + hami_device_dispatch_t *dd = hami_device_lookup(d); + if (!dd) return -1; + return hami_vk_physdev_index(dd->physical); +} + +VKAPI_ATTR VkResult VKAPI_CALL +hami_vkAllocateMemory(VkDevice device, const VkMemoryAllocateInfo *pInfo, + const VkAllocationCallbacks *pAlloc, VkDeviceMemory *pMem) { + HAMI_TRACE("hami_vkAllocateMemory device=%p size=%" PRIu64, + (void *)device, (uint64_t)pInfo->allocationSize); + hami_device_dispatch_t *d = hami_device_lookup(device); + if (!d || !d->AllocateMemory) { + HAMI_TRACE("hami_vkAllocateMemory: device dispatch missing -> VK_ERROR_INITIALIZATION_FAILED"); + return VK_ERROR_INITIALIZATION_FAILED; + } + + int idx = device_to_index(device); + HAMI_TRACE("hami_vkAllocateMemory: device_to_index -> idx=%d", idx); + if (idx >= 0 && !hami_budget_reserve(idx, pInfo->allocationSize)) { + HAMI_TRACE("hami_vkAllocateMemory: budget reserve REJECTED idx=%d size=%" PRIu64, + idx, (uint64_t)pInfo->allocationSize); + return VK_ERROR_OUT_OF_DEVICE_MEMORY; + } + if (idx < 0) { + HAMI_TRACE("hami_vkAllocateMemory: idx<0 -> SKIP budget enforcement"); + } + + VkResult r = d->AllocateMemory(device, pInfo, pAlloc, pMem); + if (r != VK_SUCCESS) { + if (idx >= 0) hami_budget_release(idx, pInfo->allocationSize); + return r; + } + + mem_entry_t *e = calloc(1, sizeof(*e)); + e->handle = *pMem; + e->size = pInfo->allocationSize; + e->dev_idx = idx; + + pthread_mutex_lock(&g_mem_lock); + e->next = g_mem_head; + g_mem_head = e; + pthread_mutex_unlock(&g_mem_lock); + return VK_SUCCESS; +} + +VKAPI_ATTR void VKAPI_CALL +hami_vkFreeMemory(VkDevice device, VkDeviceMemory mem, const VkAllocationCallbacks *pAlloc) { + hami_device_dispatch_t *d = hami_device_lookup(device); + if (d && d->FreeMemory) d->FreeMemory(device, mem, pAlloc); + + pthread_mutex_lock(&g_mem_lock); + mem_entry_t **pp = &g_mem_head; + while (*pp && (*pp)->handle != mem) pp = &(*pp)->next; + if (*pp) { + mem_entry_t *victim = *pp; + *pp = victim->next; + pthread_mutex_unlock(&g_mem_lock); + if (victim->dev_idx >= 0) hami_budget_release(victim->dev_idx, victim->size); + free(victim); + return; + } + pthread_mutex_unlock(&g_mem_lock); +} + +void hami_vk_hook_device(hami_device_dispatch_t *d) { (void)d; } diff --git a/src/vulkan/hooks_memory.c b/src/vulkan/hooks_memory.c new file mode 100644 index 00000000..0f1ff135 --- /dev/null +++ b/src/vulkan/hooks_memory.c @@ -0,0 +1,145 @@ +#include +#include +#include +#include +#include + +#include + +#include "vulkan/dispatch.h" +#include "vulkan/budget.h" +#include "vulkan/physdev_index.h" + +static int hami_vk_trace_enabled_local(void) { + static int cached = -1; + if (cached < 0) { + const char *e = getenv("HAMI_VK_TRACE"); + cached = (e && e[0] == '1') ? 1 : 0; + } + return cached; +} +#define HAMI_TRACE(fmt, ...) do { \ + if (hami_vk_trace_enabled_local()) { \ + fprintf(stderr, "HAMI_VK_TRACE: " fmt "\n", ##__VA_ARGS__); \ + fflush(stderr); \ + } \ +} while (0) + +static void clamp_heaps(VkPhysicalDevice p, uint32_t *count, VkMemoryHeap *heaps) { + HAMI_TRACE("clamp_heaps ENTER physDev=%p count=%u", (void *)p, (unsigned)*count); + int dev = hami_vk_physdev_index(p); + HAMI_TRACE("clamp_heaps physdev_index -> dev=%d", dev); + if (dev < 0) { + HAMI_TRACE("clamp_heaps EARLY RETURN (dev<0, unresolved physical device)"); + return; + } + size_t budget = hami_budget_of(dev); + HAMI_TRACE("clamp_heaps dev=%d budget=%zu count=%u", dev, budget, (unsigned)*count); + if (budget == 0) { + HAMI_TRACE("clamp_heaps EARLY RETURN (budget=0, unlimited)"); + return; + } + for (uint32_t i = 0; i < *count; ++i) { + if ((heaps[i].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) == 0) continue; + if (heaps[i].size > budget) { + HAMI_TRACE("clamp_heaps[%u] %" PRIu64 " -> %zu", (unsigned)i, + (uint64_t)heaps[i].size, budget); + heaps[i].size = budget; + } + } +} + +VKAPI_ATTR void VKAPI_CALL +hami_vkGetPhysicalDeviceMemoryProperties(VkPhysicalDevice p, + VkPhysicalDeviceMemoryProperties *out) { + HAMI_TRACE("hami_vkGetPhysicalDeviceMemoryProperties physDev=%p", (void *)p); + extern hami_instance_dispatch_t *g_inst_head; + int n = 0; + for (hami_instance_dispatch_t *it = g_inst_head; it; it = it->next) { + n++; + if (it->GetPhysicalDeviceMemoryProperties) { + it->GetPhysicalDeviceMemoryProperties(p, out); + clamp_heaps(p, &out->memoryHeapCount, out->memoryHeaps); + HAMI_TRACE("hami_vkGetPhysicalDeviceMemoryProperties: clamped via dispatch %d", n); + return; + } + } + HAMI_TRACE("hami_vkGetPhysicalDeviceMemoryProperties: g_inst_head walked %d entries," + " no match -> out unmodified", n); +} + +/* Make Carbonite/Kit's "X used / Y available" overlay reflect the pod's + * partition limit. Earlier attempts to clamp heapBudget directly (to the + * partition limit, or to limit-usage) caused omni.physx.tensors to dead- + * lock during plugin initialization, even though heapBudget < heap.size + * was preserved. PhysX/Carbonite appears to consume heapBudget through + * paths that go beyond a simple "available = heapBudget - heapUsage" + * subtraction. + * + * Workaround: leave heapBudget untouched (matches what the ICD reports + * for the host GPU's free memory), and instead inflate heapUsage by the + * delta (icd_budget - partition_limit). The visible "available" computed + * by overlay = heapBudget - heapUsage then matches the partition limit, + * while heapBudget itself stays at the value PhysX expects. */ +static void clamp_budget_pnext(VkPhysicalDevice p, + const VkMemoryHeap *heaps, + uint32_t heap_count, + void *pnext_chain) { + int dev = hami_vk_physdev_index(p); + if (dev < 0) { + HAMI_TRACE("clamp_budget_pnext EARLY RETURN (dev<0)"); + return; + } + size_t budget = hami_budget_of(dev); + if (budget == 0) { + HAMI_TRACE("clamp_budget_pnext EARLY RETURN (budget=0, unlimited)"); + return; + } + VkBaseOutStructure *cur = (VkBaseOutStructure *)pnext_chain; + while (cur) { + if (cur->sType == VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_BUDGET_PROPERTIES_EXT) { + VkPhysicalDeviceMemoryBudgetPropertiesEXT *bud = + (VkPhysicalDeviceMemoryBudgetPropertiesEXT *)cur; + VkDeviceSize budget_vk = (VkDeviceSize)budget; + for (uint32_t i = 0; i < heap_count && i < VK_MAX_MEMORY_HEAPS; ++i) { + if ((heaps[i].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) == 0) continue; + VkDeviceSize icd_budget = bud->heapBudget[i]; + VkDeviceSize icd_usage = bud->heapUsage[i]; + if (icd_budget <= budget_vk) continue; + VkDeviceSize delta = icd_budget - budget_vk; + VkDeviceSize new_usage = icd_usage + delta; + if (new_usage > icd_usage) { + HAMI_TRACE("clamp_budget_pnext[%u] heapUsage %" PRIu64 " -> %" PRIu64 + " (limit=%zu icd_budget=%" PRIu64 ")", + (unsigned)i, + (uint64_t)icd_usage, + (uint64_t)new_usage, + budget, + (uint64_t)icd_budget); + bud->heapUsage[i] = new_usage; + } + } + break; + } + cur = (VkBaseOutStructure *)cur->pNext; + } +} + +VKAPI_ATTR void VKAPI_CALL +hami_vkGetPhysicalDeviceMemoryProperties2(VkPhysicalDevice p, + VkPhysicalDeviceMemoryProperties2 *out) { + extern hami_instance_dispatch_t *g_inst_head; + for (hami_instance_dispatch_t *it = g_inst_head; it; it = it->next) { + if (it->GetPhysicalDeviceMemoryProperties2) { + it->GetPhysicalDeviceMemoryProperties2(p, out); + clamp_heaps(p, &out->memoryProperties.memoryHeapCount, + out->memoryProperties.memoryHeaps); + clamp_budget_pnext(p, out->memoryProperties.memoryHeaps, + out->memoryProperties.memoryHeapCount, + out->pNext); + return; + } + } +} + +void hami_vk_hook_instance(hami_instance_dispatch_t *d) { (void)d; } diff --git a/src/vulkan/hooks_submit.c b/src/vulkan/hooks_submit.c new file mode 100644 index 00000000..a6664dae --- /dev/null +++ b/src/vulkan/hooks_submit.c @@ -0,0 +1,48 @@ +#include +#include + +#include "vulkan/dispatch.h" +#include "vulkan/throttle_adapter.h" + +/* Queue → Device registry populated by layer.c's vkGetDeviceQueue[2] + * wrappers (and by unit tests). */ +typedef struct q_entry { VkQueue q; VkDevice d; struct q_entry *next; } q_entry_t; +static q_entry_t *g_q_head = NULL; +static pthread_mutex_t g_q_lock = PTHREAD_MUTEX_INITIALIZER; + +void hami_vk_register_queue(VkQueue q, VkDevice d) { + q_entry_t *e = calloc(1, sizeof(*e)); + e->q = q; e->d = d; + pthread_mutex_lock(&g_q_lock); + e->next = g_q_head; g_q_head = e; + pthread_mutex_unlock(&g_q_lock); +} + +static VkDevice device_for_queue(VkQueue q) { + pthread_mutex_lock(&g_q_lock); + q_entry_t *p = g_q_head; + while (p && p->q != q) p = p->next; + VkDevice d = p ? p->d : VK_NULL_HANDLE; + pthread_mutex_unlock(&g_q_lock); + return d; +} + +VKAPI_ATTR VkResult VKAPI_CALL +hami_vkQueueSubmit(VkQueue queue, uint32_t n, const VkSubmitInfo *p, VkFence f) { + VkDevice d = device_for_queue(queue); + hami_device_dispatch_t *dd = hami_device_lookup(d); + if (!dd || !dd->QueueSubmit) return VK_ERROR_INITIALIZATION_FAILED; + hami_vulkan_throttle(); + return dd->QueueSubmit(queue, n, p, f); +} + +#if defined(VK_VERSION_1_3) +VKAPI_ATTR VkResult VKAPI_CALL +hami_vkQueueSubmit2(VkQueue queue, uint32_t n, const VkSubmitInfo2 *p, VkFence f) { + VkDevice d = device_for_queue(queue); + hami_device_dispatch_t *dd = hami_device_lookup(d); + if (!dd || !dd->QueueSubmit2) return VK_ERROR_INITIALIZATION_FAILED; + hami_vulkan_throttle(); + return dd->QueueSubmit2(queue, n, p, f); +} +#endif diff --git a/src/vulkan/layer.c b/src/vulkan/layer.c new file mode 100644 index 00000000..4de26e30 --- /dev/null +++ b/src/vulkan/layer.c @@ -0,0 +1,414 @@ +#include "vulkan/layer.h" + +#include +#include +#include + +#include "vulkan/dispatch.h" + +/* Debug trace gated by HAMI_VK_TRACE=1. + * Used to localize where the dispatch chain breaks; safe to leave in + * because it's behind a runtime flag. */ +#define HAMI_VK_TRACE_ENV "HAMI_VK_TRACE" +static int hami_vk_trace_enabled(void) { + static int cached = -1; + if (cached < 0) { + const char *e = getenv(HAMI_VK_TRACE_ENV); + cached = (e && e[0] == '1') ? 1 : 0; + } + return cached; +} +#define HAMI_TRACE(fmt, ...) do { \ + if (hami_vk_trace_enabled()) { \ + fprintf(stderr, "HAMI_VK_TRACE: " fmt "\n", ##__VA_ARGS__); \ + fflush(stderr); \ + } \ +} while (0) + +/* forward declarations for hooks implemented in sibling files */ +extern void hami_vk_hook_instance(hami_instance_dispatch_t *d); +extern void hami_vk_hook_device(hami_device_dispatch_t *d); + +/* Cached next-layer GetInstanceProcAddr from the first vkCreateInstance + * call. Used as a fallback when GIPA is invoked with an unknown instance + * handle (loader probes during init, or instance handles wrapped by an + * upper layer that we haven't seen): we still need to return a valid + * pointer so the loader/driver doesn't dereference NULL. */ +static PFN_vkGetInstanceProcAddr g_first_next_gipa = NULL; +static PFN_vkGetDeviceProcAddr g_first_next_gdpa = NULL; + +static VkLayerInstanceCreateInfo *find_chain_info(const VkInstanceCreateInfo *pCreateInfo, + VkLayerFunction func) { + const VkLayerInstanceCreateInfo *ci = pCreateInfo->pNext; + while (ci) { + if (ci->sType == VK_STRUCTURE_TYPE_LOADER_INSTANCE_CREATE_INFO && ci->function == func) { + return (VkLayerInstanceCreateInfo *)ci; + } + ci = (const VkLayerInstanceCreateInfo *)ci->pNext; + } + return NULL; +} + +static VkLayerDeviceCreateInfo *find_dev_chain_info(const VkDeviceCreateInfo *pCreateInfo, + VkLayerFunction func) { + const VkLayerDeviceCreateInfo *ci = pCreateInfo->pNext; + while (ci) { + if (ci->sType == VK_STRUCTURE_TYPE_LOADER_DEVICE_CREATE_INFO && ci->function == func) { + return (VkLayerDeviceCreateInfo *)ci; + } + ci = (const VkLayerDeviceCreateInfo *)ci->pNext; + } + return NULL; +} + +static VKAPI_ATTR VkResult VKAPI_CALL +hami_vkCreateInstance(const VkInstanceCreateInfo *pCreateInfo, + const VkAllocationCallbacks *pAllocator, + VkInstance *pInstance) { + HAMI_TRACE("hami_vkCreateInstance entered"); + VkLayerInstanceCreateInfo *chain = find_chain_info(pCreateInfo, VK_LAYER_LINK_INFO); + if (!chain || !chain->u.pLayerInfo) { + HAMI_TRACE("hami_vkCreateInstance: no VK_LAYER_LINK_INFO chain -> returning VK_ERROR_INITIALIZATION_FAILED"); + return VK_ERROR_INITIALIZATION_FAILED; + } + + PFN_vkGetInstanceProcAddr next_gipa = chain->u.pLayerInfo->pfnNextGetInstanceProcAddr; + chain->u.pLayerInfo = chain->u.pLayerInfo->pNext; + + /* Cache the next-layer gipa before calling next_create: some drivers + * (NVIDIA) trigger our GIPA from within next_create() to look up + * vkGetPhysicalDevice* entry points on a fresh, not-yet-registered + * instance, and we need to forward those lookups instead of returning + * NULL. */ + if (!g_first_next_gipa) g_first_next_gipa = next_gipa; + + PFN_vkCreateInstance next_create = + (PFN_vkCreateInstance)next_gipa(VK_NULL_HANDLE, "vkCreateInstance"); + HAMI_TRACE("hami_vkCreateInstance: next_create=%p", (void *)next_create); + VkResult r = next_create(pCreateInfo, pAllocator, pInstance); + if (r != VK_SUCCESS) { + HAMI_TRACE("hami_vkCreateInstance: next_create failed r=%d", r); + return r; + } + + hami_instance_dispatch_t *d = hami_instance_register(*pInstance, next_gipa); + hami_vk_hook_instance(d); + HAMI_TRACE("hami_vkCreateInstance: registered instance=%p dispatch=%p", + (void *)*pInstance, (void *)d); + return VK_SUCCESS; +} + +static VKAPI_ATTR void VKAPI_CALL +hami_vkDestroyInstance(VkInstance instance, const VkAllocationCallbacks *pAllocator) { + hami_instance_dispatch_t *d = hami_instance_lookup(instance); + if (d) d->DestroyInstance(instance, pAllocator); + hami_instance_unregister(instance); +} + +static VKAPI_ATTR VkResult VKAPI_CALL +hami_vkCreateDevice(VkPhysicalDevice physicalDevice, + const VkDeviceCreateInfo *pCreateInfo, + const VkAllocationCallbacks *pAllocator, + VkDevice *pDevice) { + VkLayerDeviceCreateInfo *chain = find_dev_chain_info(pCreateInfo, VK_LAYER_LINK_INFO); + if (!chain || !chain->u.pLayerInfo) return VK_ERROR_INITIALIZATION_FAILED; + + PFN_vkGetInstanceProcAddr next_gipa = chain->u.pLayerInfo->pfnNextGetInstanceProcAddr; + PFN_vkGetDeviceProcAddr next_gdpa = chain->u.pLayerInfo->pfnNextGetDeviceProcAddr; + chain->u.pLayerInfo = chain->u.pLayerInfo->pNext; + + if (!g_first_next_gipa) g_first_next_gipa = next_gipa; + if (!g_first_next_gdpa) g_first_next_gdpa = next_gdpa; + + PFN_vkCreateDevice next_create = + (PFN_vkCreateDevice)next_gipa(VK_NULL_HANDLE, "vkCreateDevice"); + VkResult r = next_create(physicalDevice, pCreateInfo, pAllocator, pDevice); + if (r != VK_SUCCESS) return r; + + hami_device_dispatch_t *d = hami_device_register(*pDevice, physicalDevice, next_gdpa); + hami_vk_hook_device(d); + return VK_SUCCESS; +} + +static VKAPI_ATTR void VKAPI_CALL +hami_vkDestroyDevice(VkDevice device, const VkAllocationCallbacks *pAllocator) { + hami_device_dispatch_t *d = hami_device_lookup(device); + if (d) d->DestroyDevice(device, pAllocator); + hami_device_unregister(device); +} + +extern void hami_vk_register_queue(VkQueue q, VkDevice d); + +static VKAPI_ATTR void VKAPI_CALL +hami_vkGetDeviceQueue(VkDevice device, uint32_t family, uint32_t index, VkQueue *pQueue) { + hami_device_dispatch_t *d = hami_device_lookup(device); + if (!d) { + *pQueue = VK_NULL_HANDLE; + return; + } + PFN_vkGetDeviceQueue next = (PFN_vkGetDeviceQueue)d->next_gdpa(device, "vkGetDeviceQueue"); + next(device, family, index, pQueue); + if (*pQueue) { + hami_vk_register_queue(*pQueue, device); + } +} + +static VKAPI_ATTR void VKAPI_CALL +hami_vkGetDeviceQueue2(VkDevice device, const VkDeviceQueueInfo2 *pInfo, VkQueue *pQueue) { + hami_device_dispatch_t *d = hami_device_lookup(device); + if (!d) { + *pQueue = VK_NULL_HANDLE; + return; + } + PFN_vkGetDeviceQueue2 next = (PFN_vkGetDeviceQueue2)d->next_gdpa(device, "vkGetDeviceQueue2"); + next(device, pInfo, pQueue); + if (*pQueue) { + hami_vk_register_queue(*pQueue, device); + } +} + +/* GIPA / GDPA: return our wrappers for hooked names, next-layer for the rest. */ + +/* Hooked functions implemented in other TUs; declarations here. */ +VKAPI_ATTR void VKAPI_CALL hami_vkGetPhysicalDeviceMemoryProperties( + VkPhysicalDevice, VkPhysicalDeviceMemoryProperties*); +VKAPI_ATTR void VKAPI_CALL hami_vkGetPhysicalDeviceMemoryProperties2( + VkPhysicalDevice, VkPhysicalDeviceMemoryProperties2*); +VKAPI_ATTR VkResult VKAPI_CALL hami_vkAllocateMemory( + VkDevice, const VkMemoryAllocateInfo*, const VkAllocationCallbacks*, VkDeviceMemory*); +VKAPI_ATTR void VKAPI_CALL hami_vkFreeMemory( + VkDevice, VkDeviceMemory, const VkAllocationCallbacks*); +VKAPI_ATTR VkResult VKAPI_CALL hami_vkQueueSubmit( + VkQueue, uint32_t, const VkSubmitInfo*, VkFence); +#if defined(VK_VERSION_1_3) +VKAPI_ATTR VkResult VKAPI_CALL hami_vkQueueSubmit2( + VkQueue, uint32_t, const VkSubmitInfo2*, VkFence); +#endif + +/* Vulkan layer name advertised in /etc/vulkan/implicit_layer.d/hami.json. */ +#define HAMI_LAYER_NAME "VK_LAYER_HAMI_vgpu" + +/* Spec-required Enumerate hooks. The Vulkan loader queries layers for + * own-extension and own-layer info via these entry points (often with a + * NULL VkInstance during initialization). The previous implementation only + * exposed CreateInstance/CreateDevice/GIPA via GIPA, so a NULL-instance + * lookup for vkEnumerate*ExtensionProperties / vkEnumerate*LayerProperties + * fell through to `hami_instance_lookup(NULL)` -> NULL and the loader + * dereferenced a NULL function pointer while assembling the enabled + * extension list. That manifested as a SegFault deep in + * libcarb.graphics-vulkan during Carbonite Vulkan plugin startup. + * + * The layer doesn't add any instance/device extensions, so own-name + * queries return zero entries. For non-own queries we MUST return + * VK_SUCCESS with count=0 rather than NULL: the loader will combine our + * answer with results from the next layer/ICD (Vulkan 1.0 spec + * "Layered Implementations" §38.3.1). Returning anything else (or a NULL + * function pointer through GIPA) breaks the chain. */ +/* Vulkan 1.3 §38.3.1 / Layer Documentation: + * - Querying with our own layer name returns our zero own-extensions. + * - Querying with another layer name -> VK_ERROR_LAYER_NOT_PRESENT. + * - Querying with NULL pLayerName -> forward to the next layer/ICD so + * the caller (NVIDIA driver during vkCreateDevice extension + * validation, Carbonite during instance setup) sees the real list. + * + * The original layer omitted these hooks, so the GIPA returned NULL and + * the loader/Carbonite SegFaulted while assembling enabled extensions. + * An earlier draft of this fix returned LAYER_NOT_PRESENT for NULL + * pLayerName, which caused vkCreateDevice to fail because the driver + * could no longer enumerate device extensions through the layer chain. */ + +static VKAPI_ATTR VkResult VKAPI_CALL +hami_vkEnumerateInstanceExtensionProperties(const char *pLayerName, + uint32_t *pPropertyCount, + VkExtensionProperties *pProperties) { + if (pLayerName != NULL && strcmp(pLayerName, HAMI_LAYER_NAME) == 0) { + if (pPropertyCount) *pPropertyCount = 0; + (void)pProperties; + return VK_SUCCESS; + } + /* For NULL pLayerName the loader will already aggregate every layer + * + ICD on its own; we just claim no contribution. For other layer + * names this layer has nothing to say. Both safely map to "0 + * properties, success" so the chain keeps going. */ + if (pPropertyCount) *pPropertyCount = 0; + (void)pProperties; + return VK_SUCCESS; +} + +static VKAPI_ATTR VkResult VKAPI_CALL +hami_vkEnumerateInstanceLayerProperties(uint32_t *pPropertyCount, + VkLayerProperties *pProperties) { + /* Loader assembles the layer list itself from manifests; the layer + * just reports its own count (0 is accepted because the manifest is + * authoritative for our presence). */ + (void)pProperties; + if (pPropertyCount) *pPropertyCount = 0; + return VK_SUCCESS; +} + +static VKAPI_ATTR VkResult VKAPI_CALL +hami_vkEnumerateDeviceExtensionProperties(VkPhysicalDevice physicalDevice, + const char *pLayerName, + uint32_t *pPropertyCount, + VkExtensionProperties *pProperties) { + /* Own-name: zero own device extensions. */ + if (pLayerName != NULL && strcmp(pLayerName, HAMI_LAYER_NAME) == 0) { + if (pPropertyCount) *pPropertyCount = 0; + (void)pProperties; + return VK_SUCCESS; + } + /* For NULL or other names we MUST forward to the next layer/ICD, + * otherwise the NVIDIA driver's vkCreateDevice fails extension + * validation ("ERROR_LAYER_NOT_PRESENT" propagated up the chain). */ + hami_instance_dispatch_t *d = hami_instance_first(); + if (d && d->EnumerateDeviceExtensionProperties) { + return d->EnumerateDeviceExtensionProperties( + physicalDevice, pLayerName, pPropertyCount, pProperties); + } + /* Loader probing before any instance was created: spec allows zero. */ + if (pPropertyCount) *pPropertyCount = 0; + (void)pProperties; + return VK_SUCCESS; +} + +static VKAPI_ATTR VkResult VKAPI_CALL +hami_vkEnumerateDeviceLayerProperties(VkPhysicalDevice physicalDevice, + uint32_t *pPropertyCount, + VkLayerProperties *pProperties) { + /* Deprecated since Vulkan 1.0.13; forward when possible, else 0. */ + hami_instance_dispatch_t *d = hami_instance_first(); + if (d && d->EnumerateDeviceLayerProperties) { + return d->EnumerateDeviceLayerProperties( + physicalDevice, pPropertyCount, pProperties); + } + (void)physicalDevice; + (void)pProperties; + if (pPropertyCount) *pPropertyCount = 0; + return VK_SUCCESS; +} + +#define HAMI_HOOK(name) do { \ + if (strcmp(pName, "vk" #name) == 0) { \ + return (PFN_vkVoidFunction)hami_vk##name; \ + } \ +} while (0) + +/* Same hook function, matched against the KHR-suffixed alias. The + * `*2KHR` names are the original extension form (VK_KHR_get_physical_ + * device_properties2) that many engines still query through + * vkGetInstanceProcAddr even though Vulkan 1.1 promoted them to core. + * Without this alias, the loader returns the next layer's pointer for + * the KHR name and our hook is bypassed — observed in Carbonite/Kit + * where vkGetPhysicalDeviceMemoryProperties2KHR was reading the + * unclamped 45 GiB heap and 32 GiB budget straight from the ICD. */ +#define HAMI_HOOK_KHR_ALIAS(name) do { \ + if (strcmp(pName, "vk" #name "KHR") == 0) { \ + return (PFN_vkVoidFunction)hami_vk##name; \ + } \ +} while (0) + +PFN_vkVoidFunction VKAPI_CALL +hami_vkGetInstanceProcAddr(VkInstance instance, const char *pName) { + HAMI_TRACE("hami_vkGetInstanceProcAddr instance=%p name=%s", (void *)instance, pName); + HAMI_HOOK(CreateInstance); + HAMI_HOOK(DestroyInstance); + HAMI_HOOK(CreateDevice); + HAMI_HOOK(GetInstanceProcAddr); + HAMI_HOOK(GetPhysicalDeviceMemoryProperties); + HAMI_HOOK(GetPhysicalDeviceMemoryProperties2); + HAMI_HOOK_KHR_ALIAS(GetPhysicalDeviceMemoryProperties2); + /* Spec-required global entry points that the loader queries with + * instance=NULL during layer initialization. Returning NULL here + * caused libcarb.graphics-vulkan to SegFault while assembling the + * enabled extension list. */ + HAMI_HOOK(EnumerateInstanceExtensionProperties); + HAMI_HOOK(EnumerateInstanceLayerProperties); + HAMI_HOOK(EnumerateDeviceExtensionProperties); + HAMI_HOOK(EnumerateDeviceLayerProperties); + + hami_instance_dispatch_t *d = hami_instance_lookup(instance); + if (d) return d->next_gipa(instance, pName); + /* Unknown VkInstance handle: NVIDIA driver and Carbonite occasionally + * probe through our GIPA with handles we haven't registered (e.g., + * during vkCreateInstance before our register call returns, or with + * an upper-layer-wrapped handle). Returning NULL would SegFault the + * caller. Forward to the first cached next-layer gipa instead — it + * was set the first time vkCreateInstance ran and is a valid pointer + * into the next layer / driver. */ + if (g_first_next_gipa) { + HAMI_TRACE("hami_vkGetInstanceProcAddr: instance %p not registered," + " forwarding via cached gipa", (void *)instance); + return g_first_next_gipa(instance, pName); + } + /* Pre-CreateInstance loader bootstrap: the only case where the spec + * allows us to return NULL for instance entry points (the loader + * still resolves the global Enumerate* hooks via the same GIPA, but + * those are matched above by HAMI_HOOK before this fall-through). */ + HAMI_TRACE("hami_vkGetInstanceProcAddr: instance %p not registered AND no cached gipa, returning NULL", + (void *)instance); + return NULL; +} + +PFN_vkVoidFunction VKAPI_CALL +hami_vkGetDeviceProcAddr(VkDevice device, const char *pName) { + HAMI_HOOK(DestroyDevice); + HAMI_HOOK(GetDeviceProcAddr); + HAMI_HOOK(AllocateMemory); + HAMI_HOOK(FreeMemory); + HAMI_HOOK(QueueSubmit); +#if defined(VK_VERSION_1_3) + HAMI_HOOK(QueueSubmit2); +#endif + HAMI_HOOK(GetDeviceQueue); + HAMI_HOOK(GetDeviceQueue2); + + hami_device_dispatch_t *d = hami_device_lookup(device); + if (d) return d->next_gdpa(device, pName); + if (g_first_next_gdpa) { + return g_first_next_gdpa(device, pName); + } + return NULL; +} + +/* The Vulkan loader looks up these three entry points by their canonical + * (un-prefixed) names. Some build environments compile this TU with + * -fvisibility=hidden, in which case the upstream VK_LAYER_EXPORT macro + * (which can fall back to empty on older Vulkan-Headers) does not produce + * an exported symbol. Force default visibility here regardless of the + * compile flags so that dlsym from the loader sees them. */ +#define HAMI_LAYER_EXPORT __attribute__((visibility("default"))) + +HAMI_LAYER_EXPORT VkResult VKAPI_CALL +vkNegotiateLoaderLayerInterfaceVersion(VkNegotiateLayerInterface *pVersionStruct) { + HAMI_TRACE("vkNegotiateLoaderLayerInterfaceVersion entered (version=%u)", + pVersionStruct ? pVersionStruct->loaderLayerInterfaceVersion : 0); + if (pVersionStruct->sType != LAYER_NEGOTIATE_INTERFACE_STRUCT) { + HAMI_TRACE("vkNegotiate: sType mismatch -> VK_ERROR_INITIALIZATION_FAILED"); + return VK_ERROR_INITIALIZATION_FAILED; + } + + if (pVersionStruct->loaderLayerInterfaceVersion > 2) { + pVersionStruct->loaderLayerInterfaceVersion = 2; + } + + pVersionStruct->pfnGetInstanceProcAddr = hami_vkGetInstanceProcAddr; + pVersionStruct->pfnGetDeviceProcAddr = hami_vkGetDeviceProcAddr; + pVersionStruct->pfnGetPhysicalDeviceProcAddr = NULL; + HAMI_TRACE("vkNegotiate: success (version=%u)", + pVersionStruct->loaderLayerInterfaceVersion); + return VK_SUCCESS; +} + +/* Fallback wrappers for loader interface version 1: the loader resolves the + * canonical names directly when the manifest does not advertise interface v2. + * Both forms must coexist so the layer works regardless of which path the + * loader picks. */ +HAMI_LAYER_EXPORT PFN_vkVoidFunction VKAPI_CALL +vkGetInstanceProcAddr(VkInstance instance, const char *pName) { + return hami_vkGetInstanceProcAddr(instance, pName); +} + +HAMI_LAYER_EXPORT PFN_vkVoidFunction VKAPI_CALL +vkGetDeviceProcAddr(VkDevice device, const char *pName) { + return hami_vkGetDeviceProcAddr(device, pName); +} diff --git a/src/vulkan/layer.h b/src/vulkan/layer.h new file mode 100644 index 00000000..d395b940 --- /dev/null +++ b/src/vulkan/layer.h @@ -0,0 +1,34 @@ +#ifndef SRC_VULKAN_LAYER_H_ +#define SRC_VULKAN_LAYER_H_ + +#include +#include + +/* Vulkan-Headers 1.3.280+ dropped VK_LAYER_EXPORT. Default visibility on + * ELF/Mach-O is sufficient; Windows would need __declspec(dllexport). */ +#ifndef VK_LAYER_EXPORT +# if defined(_WIN32) +# define VK_LAYER_EXPORT __declspec(dllexport) +# else +# define VK_LAYER_EXPORT __attribute__((visibility("default"))) +# endif +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +VK_LAYER_EXPORT VkResult VKAPI_CALL +vkNegotiateLoaderLayerInterfaceVersion(VkNegotiateLayerInterface *pVersionStruct); + +PFN_vkVoidFunction VKAPI_CALL +hami_vkGetInstanceProcAddr(VkInstance instance, const char *pName); + +PFN_vkVoidFunction VKAPI_CALL +hami_vkGetDeviceProcAddr(VkDevice device, const char *pName); + +#ifdef __cplusplus +} +#endif + +#endif // SRC_VULKAN_LAYER_H_ diff --git a/src/vulkan/physdev_index.c b/src/vulkan/physdev_index.c new file mode 100644 index 00000000..215de472 --- /dev/null +++ b/src/vulkan/physdev_index.c @@ -0,0 +1,206 @@ +#include "vulkan/physdev_index.h" + +#include +#include +#include +#include +#include + +#include "vulkan/dispatch.h" + +static int hami_vk_trace_enabled_local(void) { + static int cached = -1; + if (cached < 0) { + const char *e = getenv("HAMI_VK_TRACE"); + cached = (e && e[0] == '1') ? 1 : 0; + } + return cached; +} +#define HAMI_TRACE(fmt, ...) do { \ + if (hami_vk_trace_enabled_local()) { \ + fprintf(stderr, "HAMI_VK_TRACE: " fmt "\n", ##__VA_ARGS__); \ + fflush(stderr); \ + } \ +} while (0) + +/* Minimal NVML shim — only the symbols we need, resolved from the NVML + * library already linked into libvgpu.so. Keeping this file independent of + * the full NVML header avoids coupling with HAMi-core's NVML-override shim. */ +typedef void *nvmlDevice_t; +typedef int nvmlReturn_t; +#define NVML_SUCCESS 0 +extern nvmlReturn_t nvmlInit_v2(void); +extern nvmlReturn_t nvmlDeviceGetCount_v2(unsigned int *count); +extern nvmlReturn_t nvmlDeviceGetHandleByIndex_v2(unsigned int idx, nvmlDevice_t *dev); +extern nvmlReturn_t nvmlDeviceGetUUID(nvmlDevice_t dev, char *uuid, unsigned int length); + +#define HAMI_VK_UUID_CACHE_SIZE 32 + +typedef struct { + VkPhysicalDevice handle; + int index; +} uuid_cache_entry_t; + +static uuid_cache_entry_t g_cache[HAMI_VK_UUID_CACHE_SIZE]; +static int g_cache_fill = 0; +static pthread_mutex_t g_cache_lock = PTHREAD_MUTEX_INITIALIZER; +static int g_nvml_initialized = 0; + +static int parse_nvml_uuid(const char *s, uint8_t out[16]) { + /* NVML typically formats as "GPU-590c05ea-a735-d6ce-75cb-c6b06baecf21" + * (32 hex chars + 4 hyphens after "GPU-" prefix). Skip the prefix and + * any hyphens, read 16 bytes as hex pairs. */ + if (strncmp(s, "GPU-", 4) == 0) s += 4; + for (int i = 0; i < 16; i++) { + while (*s == '-') s++; + if (!s[0] || !s[1]) return -1; + unsigned int v; + if (sscanf(s, "%2x", &v) != 1) return -1; + out[i] = (uint8_t)v; + s += 2; + } + return 0; +} + +static void hami_log_uuid(const char *tag, const uint8_t u[16]) { + if (!hami_vk_trace_enabled_local()) return; + fprintf(stderr, "HAMI_VK_TRACE: %s = " + "%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x\n", + tag, u[0], u[1], u[2], u[3], u[4], u[5], u[6], u[7], + u[8], u[9], u[10], u[11], u[12], u[13], u[14], u[15]); + fflush(stderr); +} + +static int is_zero_uuid(const uint8_t u[16]) { + for (int i = 0; i < 16; i++) { + if (u[i] != 0) return 0; + } + return 1; +} + +static int nvml_index_for_uuid(const uint8_t vk_uuid[16]) { + if (!g_nvml_initialized) { + nvmlReturn_t r = nvmlInit_v2(); + HAMI_TRACE("nvmlInit_v2 -> %d", (int)r); + if (r != NVML_SUCCESS) return -1; + g_nvml_initialized = 1; + } + unsigned int count = 0; + nvmlReturn_t rc = nvmlDeviceGetCount_v2(&count); + HAMI_TRACE("nvmlDeviceGetCount_v2 -> rc=%d count=%u", (int)rc, count); + if (rc != NVML_SUCCESS) return -1; + hami_log_uuid("vk_uuid (target)", vk_uuid); + + /* Fallback: if Vulkan returned a zero deviceUUID (observed on certain + * NVIDIA driver+container configurations where the VK_KHR_external_memory + * ID extension does not populate the UUID into pNext'd struct) AND only + * one NVML device is visible to this container — which is the standard + * HAMi operating model where the device-plugin assigns one GPU per + * container — we can safely map to NVML index 0. Multi-GPU containers + * fall through to strict UUID matching to avoid mis-binding. */ + if (is_zero_uuid(vk_uuid) && count == 1) { + HAMI_TRACE("nvml_index_for_uuid: vk_uuid all-zero + NVML count==1 " + "-> single-GPU fallback idx=0"); + return 0; + } + + for (unsigned int i = 0; i < count; i++) { + nvmlDevice_t dev = NULL; + nvmlReturn_t rh = nvmlDeviceGetHandleByIndex_v2(i, &dev); + if (rh != NVML_SUCCESS) { + HAMI_TRACE("nvmlDeviceGetHandleByIndex_v2[%u] -> rc=%d (skip)", i, (int)rh); + continue; + } + char uuid_str[96] = {0}; + nvmlReturn_t ru = nvmlDeviceGetUUID(dev, uuid_str, sizeof(uuid_str)); + if (ru != NVML_SUCCESS) { + HAMI_TRACE("nvmlDeviceGetUUID[%u] -> rc=%d (skip)", i, (int)ru); + continue; + } + HAMI_TRACE("nvml[%u] uuid_str='%s'", i, uuid_str); + uint8_t uuid_bin[16]; + if (parse_nvml_uuid(uuid_str, uuid_bin) != 0) { + HAMI_TRACE("nvml[%u] parse_nvml_uuid FAILED (skip)", i); + continue; + } + hami_log_uuid("nvml uuid_bin", uuid_bin); + if (memcmp(uuid_bin, vk_uuid, 16) == 0) { + HAMI_TRACE("nvml[%u] UUID MATCH -> return idx=%d", i, (int)i); + return (int)i; + } + } + HAMI_TRACE("nvml_index_for_uuid: NO MATCH -> -1"); + return -1; +} + +/* Defined in dispatch.c (non-static, exported to sibling TUs). */ +extern hami_instance_dispatch_t *g_inst_head; + +static int resolve_via_vulkan_props(VkPhysicalDevice p, uint8_t out_uuid[16]) { + /* Walk registered instance dispatches and use whichever next-layer + * GetPhysicalDeviceProperties2 is available to read the deviceUUID. */ + int n = 0; + for (hami_instance_dispatch_t *it = g_inst_head; it; it = it->next) { + n++; + if (!it->next_gipa) { + HAMI_TRACE("resolve_via_vulkan_props inst[%d] NO next_gipa (skip)", n); + continue; + } + PFN_vkGetPhysicalDeviceProperties2 get2 = + (PFN_vkGetPhysicalDeviceProperties2) + it->next_gipa(it->handle, "vkGetPhysicalDeviceProperties2"); + if (!get2) { + get2 = (PFN_vkGetPhysicalDeviceProperties2) + it->next_gipa(it->handle, "vkGetPhysicalDeviceProperties2KHR"); + } + if (!get2) { + HAMI_TRACE("resolve_via_vulkan_props inst[%d] NO get2 (skip)", n); + continue; + } + VkPhysicalDeviceIDProperties id = { + .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ID_PROPERTIES, + .pNext = NULL, + }; + VkPhysicalDeviceProperties2 props = { + .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, + .pNext = &id, + }; + get2(p, &props); + memcpy(out_uuid, id.deviceUUID, 16); + HAMI_TRACE("resolve_via_vulkan_props inst[%d] OK", n); + return 0; + } + HAMI_TRACE("resolve_via_vulkan_props walked %d insts, NO match", n); + return -1; +} + +int hami_vk_physdev_index(VkPhysicalDevice p) { + pthread_mutex_lock(&g_cache_lock); + for (int i = 0; i < g_cache_fill; i++) { + if (g_cache[i].handle == p) { + int idx = g_cache[i].index; + pthread_mutex_unlock(&g_cache_lock); + HAMI_TRACE("physdev_index CACHE HIT physDev=%p -> idx=%d", (void *)p, idx); + return idx; + } + } + pthread_mutex_unlock(&g_cache_lock); + + uint8_t vk_uuid[16]; + int idx = -1; + int rv = resolve_via_vulkan_props(p, vk_uuid); + HAMI_TRACE("physdev_index resolve_via_vulkan_props physDev=%p -> rc=%d", (void *)p, rv); + if (rv == 0) { + idx = nvml_index_for_uuid(vk_uuid); + } + HAMI_TRACE("physdev_index physDev=%p -> idx=%d", (void *)p, idx); + + pthread_mutex_lock(&g_cache_lock); + if (g_cache_fill < HAMI_VK_UUID_CACHE_SIZE) { + g_cache[g_cache_fill].handle = p; + g_cache[g_cache_fill].index = idx; + g_cache_fill++; + } + pthread_mutex_unlock(&g_cache_lock); + return idx; +} diff --git a/src/vulkan/physdev_index.h b/src/vulkan/physdev_index.h new file mode 100644 index 00000000..d1639ab7 --- /dev/null +++ b/src/vulkan/physdev_index.h @@ -0,0 +1,14 @@ +#ifndef SRC_VULKAN_PHYSDEV_INDEX_H_ +#define SRC_VULKAN_PHYSDEV_INDEX_H_ + +#include + +/* Resolve the HAMi-core device index for a VkPhysicalDevice by comparing its + * Vulkan deviceUUID (from VK_KHR_get_physical_device_properties2) against + * each NVML device UUID. Returns a cached mapping on subsequent calls. + * Returns -1 if the device could not be resolved (e.g., software rasterizer + * or NVML unavailable); callers should treat this as "no budget enforcement + * for this device". */ +int hami_vk_physdev_index(VkPhysicalDevice p); + +#endif // SRC_VULKAN_PHYSDEV_INDEX_H_ diff --git a/src/vulkan/throttle_adapter.c b/src/vulkan/throttle_adapter.c new file mode 100644 index 00000000..09ca8c9c --- /dev/null +++ b/src/vulkan/throttle_adapter.c @@ -0,0 +1,10 @@ +#include "vulkan/throttle_adapter.h" +#include "include/hami_core_export.h" + +void hami_vulkan_throttle(void) { + /* Consume one token — represents "one queue submission". The + * underlying rate_limiter interprets (grids*blocks) as the claim + * size; the wrapper uses (1,1) so Vulkan submits compete fairly + * with tiny CUDA kernel launches. */ + hami_core_throttle(); +} diff --git a/src/vulkan/throttle_adapter.h b/src/vulkan/throttle_adapter.h new file mode 100644 index 00000000..5c767bb2 --- /dev/null +++ b/src/vulkan/throttle_adapter.h @@ -0,0 +1,10 @@ +#ifndef SRC_VULKAN_THROTTLE_ADAPTER_H_ +#define SRC_VULKAN_THROTTLE_ADAPTER_H_ + +/* Consume one "compute unit" token from the HAMi-core SM rate limiter. + * When the HAMi SM limit is 0 or >= 100 (unlimited), this is a no-op + * inherited from the underlying rate_limiter. Call once per Vulkan + * vkQueueSubmit/vkQueueSubmit2 before forwarding to the next layer. */ +void hami_vulkan_throttle(void); + +#endif // SRC_VULKAN_THROTTLE_ADAPTER_H_ diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 86575d0e..a1487eaf 100755 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -6,7 +6,10 @@ set(TEST_TARGET_NAMES_LIST) file(GLOB_RECURSE TEST_SCRIPTS "${TEST_CPP_SOURCE_DIR}/*.c" "${TEST_CPP_SOURCE_DIR}/*.cu") -foreach(TEST_SCRIPT ${TEST_SCRIPTS}) +# Vulkan unit tests link against vulkan_mod (not CUDA) and have their own +# build harness; exclude them from this CUDA-test glob. +list(FILTER TEST_SCRIPTS EXCLUDE REGEX "/vulkan/") +foreach(TEST_SCRIPT ${TEST_SCRIPTS}) file(RELATIVE_PATH RELATIVE_TEST_PATH ${TEST_CPP_SOURCE_DIR} ${TEST_SCRIPT}) get_filename_component(TEST_TARGET_DIR ${RELATIVE_TEST_PATH} DIRECTORY) get_filename_component(TEST_TARGET_NAME ${RELATIVE_TEST_PATH} NAME_WE) diff --git a/test/test_cuda_null_guards.c b/test/test_cuda_null_guards.c new file mode 100644 index 00000000..bf2ef4e6 --- /dev/null +++ b/test/test_cuda_null_guards.c @@ -0,0 +1,106 @@ +/* Regression test for NULL-pointer guards in CUDA hooks. + * + * NVIDIA OptiX/Aftermath internal init paths historically pass NULL into + * cuMemAlloc_v2 during fallback probes. Without explicit guards in our + * hooks the LD_PRELOAD-injected libvgpu.so would dereference NULL inside + * allocate_raw and SegFault Isaac Sim Kit at startup. This test asserts + * the hook returns a non-success error code and does not crash. + * + * Pattern matches commit 03f99d7 (cuMemGetInfo_v2 NULL forward). + */ + +#include +#include +#include +#include + +extern CUresult cuMemAlloc_v2(CUdeviceptr* dptr, size_t bytesize); +extern CUresult cuMemAllocHost_v2(void** hptr, size_t bytesize); +extern CUresult cuMemAllocManaged(CUdeviceptr* dptr, size_t bytesize, unsigned int flags); +extern CUresult cuMemAllocPitch_v2(CUdeviceptr* dptr, size_t* pPitch, + size_t WidthInBytes, size_t Height, + unsigned int ElementSizeBytes); +extern CUresult cuMemHostAlloc(void** hptr, size_t bytesize, unsigned int flags); +extern CUresult cuMemHostRegister_v2(void* hptr, size_t bytesize, unsigned int flags); +extern CUresult cuCtxGetDevice(CUdevice* device); + +static void test_cuMemAlloc_v2_null_dptr(void) { + CUresult r = cuMemAlloc_v2(NULL, 4096); + assert(r != CUDA_SUCCESS); + printf("[OK] cuMemAlloc_v2(NULL, 4096) returned %d (non-zero, no crash)\n", r); +} + +static void test_cuMemAlloc_v2_zero_size(void) { + CUdeviceptr dptr = 0; + CUresult r = cuMemAlloc_v2(&dptr, 0); + printf("[OK] cuMemAlloc_v2(&dptr, 0) returned %d\n", r); +} + +static void test_cuMemAllocHost_v2_null_hptr(void) { + CUresult r = cuMemAllocHost_v2(NULL, 4096); + assert(r != CUDA_SUCCESS); + printf("[OK] cuMemAllocHost_v2(NULL, 4096) returned %d\n", r); +} + +static void test_cuMemAllocManaged_null_dptr(void) { + CUresult r = cuMemAllocManaged(NULL, 4096, CU_MEM_ATTACH_GLOBAL); + assert(r != CUDA_SUCCESS); + printf("[OK] cuMemAllocManaged(NULL, 4096) returned %d\n", r); +} + +static void test_cuMemAllocPitch_v2_null_dptr(void) { + size_t pitch = 0; + CUresult r = cuMemAllocPitch_v2(NULL, &pitch, 1024, 1024, 4); + assert(r != CUDA_SUCCESS); + printf("[OK] cuMemAllocPitch_v2(NULL, ...) returned %d\n", r); +} + +static void test_cuMemAllocPitch_v2_null_pitch(void) { + CUdeviceptr dptr = 0; + CUresult r = cuMemAllocPitch_v2(&dptr, NULL, 1024, 1024, 4); + assert(r != CUDA_SUCCESS); + printf("[OK] cuMemAllocPitch_v2(&dptr, NULL, ...) returned %d\n", r); +} + +static void test_cuMemHostAlloc_null_hptr(void) { + CUresult r = cuMemHostAlloc(NULL, 4096, 0); + assert(r != CUDA_SUCCESS); + printf("[OK] cuMemHostAlloc(NULL, 4096, 0) returned %d\n", r); +} + +static void test_cuMemHostRegister_v2_null_hptr(void) { + CUresult r = cuMemHostRegister_v2(NULL, 4096, 0); + assert(r != CUDA_SUCCESS); + printf("[OK] cuMemHostRegister_v2(NULL, 4096, 0) returned %d\n", r); +} + +static void test_cuCtxGetDevice_null(void) { + CUresult r = cuCtxGetDevice(NULL); + assert(r != CUDA_SUCCESS); + printf("[OK] cuCtxGetDevice(NULL) returned %d\n", r); +} + +int main(void) { + CUresult r = cuInit(0); + if (r != CUDA_SUCCESS) { + fprintf(stderr, "cuInit failed: %d (skipping - no GPU?)\n", r); + return 0; + } + CUdevice dev; + cuDeviceGet(&dev, 0); + CUcontext ctx; + cuCtxCreate_v2(&ctx, 0, dev); + + test_cuMemAlloc_v2_null_dptr(); + test_cuMemAlloc_v2_zero_size(); + test_cuMemAllocHost_v2_null_hptr(); + test_cuMemAllocManaged_null_dptr(); + test_cuMemAllocPitch_v2_null_dptr(); + test_cuMemAllocPitch_v2_null_pitch(); + test_cuMemHostAlloc_null_hptr(); + test_cuMemHostRegister_v2_null_hptr(); + test_cuCtxGetDevice_null(); + + cuCtxDestroy_v2(ctx); + return 0; +} diff --git a/test/vulkan/test_alloc.c b/test/vulkan/test_alloc.c new file mode 100644 index 00000000..741ca1fc --- /dev/null +++ b/test/vulkan/test_alloc.c @@ -0,0 +1,68 @@ +#include +#include +#include +#include + +#include "../../src/vulkan/dispatch.h" + +/* Budget adapter stubs (real impl arrives in Task 1.6 / src/vulkan/budget.c). */ +static size_t g_used = 0; +static const size_t BUDGET = 1ull << 30; /* 1 GiB */ + +size_t hami_budget_of(int dev) { (void)dev; return BUDGET; } +int hami_budget_reserve(int dev, size_t size) { + (void)dev; + if (g_used + size > BUDGET) { + return 0; + } + g_used += size; + return 1; +} +void hami_budget_release(int dev, size_t size) { (void)dev; g_used -= size; } + +/* Throttle stub — hooks_submit.c references it, but this test does not + * exercise the submit path. */ +void hami_vulkan_throttle(void) {} + +/* Stub the NVML UUID resolver — always report device 0. */ +int hami_vk_physdev_index(VkPhysicalDevice p) { (void)p; return 0; } + +static VkResult VKAPI_CALL fake_alloc(VkDevice d, const VkMemoryAllocateInfo *i, + const VkAllocationCallbacks *a, VkDeviceMemory *m) { + (void)d; + (void)a; + *m = (VkDeviceMemory)(uintptr_t)(i->allocationSize); + return VK_SUCCESS; +} +static void VKAPI_CALL fake_free(VkDevice d, VkDeviceMemory m, const VkAllocationCallbacks *a) { + (void)d; + (void)m; + (void)a; +} + +extern VKAPI_ATTR VkResult VKAPI_CALL +hami_vkAllocateMemory(VkDevice, const VkMemoryAllocateInfo*, const VkAllocationCallbacks*, VkDeviceMemory*); +extern VKAPI_ATTR void VKAPI_CALL +hami_vkFreeMemory(VkDevice, VkDeviceMemory, const VkAllocationCallbacks*); + +int main(void) { + VkDevice dev = (VkDevice)0x1; + hami_device_dispatch_t *d = hami_device_register(dev, (VkPhysicalDevice)0x2, NULL); + d->AllocateMemory = fake_alloc; + d->FreeMemory = fake_free; + + VkMemoryAllocateInfo info = { + .sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, + .allocationSize = (512ull << 20) + }; + VkDeviceMemory m1, m2, m3; + + assert(hami_vkAllocateMemory(dev, &info, NULL, &m1) == VK_SUCCESS); + assert(hami_vkAllocateMemory(dev, &info, NULL, &m2) == VK_SUCCESS); + assert(hami_vkAllocateMemory(dev, &info, NULL, &m3) == VK_ERROR_OUT_OF_DEVICE_MEMORY); + + hami_vkFreeMemory(dev, m1, NULL); + assert(hami_vkAllocateMemory(dev, &info, NULL, &m3) == VK_SUCCESS); + printf("ok: allocate/free budget enforced\n"); + return 0; +} diff --git a/test/vulkan/test_layer.c b/test/vulkan/test_layer.c new file mode 100644 index 00000000..92a025d6 --- /dev/null +++ b/test/vulkan/test_layer.c @@ -0,0 +1,26 @@ +#include +#include +#include +#include +#include + +typedef VkResult (VKAPI_PTR *PFN_vkNegotiateLoaderLayerInterfaceVersion)(VkNegotiateLayerInterface*); + +int main(void) { + void *h = dlopen("./libvgpu.so", RTLD_NOW); + assert(h != NULL); + PFN_vkNegotiateLoaderLayerInterfaceVersion fn = + (PFN_vkNegotiateLoaderLayerInterfaceVersion) + dlsym(h, "vkNegotiateLoaderLayerInterfaceVersion"); + assert(fn != NULL); + + VkNegotiateLayerInterface iface = {0}; + iface.sType = LAYER_NEGOTIATE_INTERFACE_STRUCT; + iface.loaderLayerInterfaceVersion = 2; + VkResult r = fn(&iface); + assert(r == VK_SUCCESS); + assert(iface.pfnGetInstanceProcAddr != NULL); + assert(iface.pfnGetDeviceProcAddr != NULL); + printf("ok: layer entry point negotiates\n"); + return 0; +} diff --git a/test/vulkan/test_memprops.c b/test/vulkan/test_memprops.c new file mode 100644 index 00000000..be05927a --- /dev/null +++ b/test/vulkan/test_memprops.c @@ -0,0 +1,44 @@ +#include +#include +#include +#include +#include + +#include "../../src/vulkan/dispatch.h" + +/* Budget-adapter stubs — real impl in Task 1.6 (src/vulkan/budget.c). */ +size_t hami_budget_of(int dev) { (void)dev; return 1ull << 30; /* 1 GiB */ } +int hami_budget_reserve(int dev, size_t size) { (void)dev; (void)size; return 1; } +void hami_budget_release(int dev, size_t size) { (void)dev; (void)size; } + +/* Throttle stub — hooks_submit.c references it, but this test does not + * exercise the submit path. */ +void hami_vulkan_throttle(void) {} + +/* Stub the NVML UUID resolver — the real version calls into NVML/Vulkan + * which aren't available in this unit test harness. Always report device 0. */ +int hami_vk_physdev_index(VkPhysicalDevice p) { (void)p; return 0; } + +static void VKAPI_CALL fake_next(VkPhysicalDevice p, VkPhysicalDeviceMemoryProperties *out) { + (void)p; + memset(out, 0, sizeof(*out)); + out->memoryHeapCount = 1; + out->memoryHeaps[0].size = 8ull << 30; + out->memoryHeaps[0].flags = VK_MEMORY_HEAP_DEVICE_LOCAL_BIT; +} + +extern VKAPI_ATTR void VKAPI_CALL +hami_vkGetPhysicalDeviceMemoryProperties(VkPhysicalDevice p, VkPhysicalDeviceMemoryProperties *out); + +int main(void) { + VkInstance inst = (VkInstance)0x1; + hami_instance_dispatch_t *d = hami_instance_register(inst, NULL); + d->GetPhysicalDeviceMemoryProperties = fake_next; + + VkPhysicalDeviceMemoryProperties props; + hami_vkGetPhysicalDeviceMemoryProperties((VkPhysicalDevice)0x2, &props); + assert(props.memoryHeapCount == 1); + assert(props.memoryHeaps[0].size == (1ull << 30)); + printf("ok: heap clamped to 1 GiB\n"); + return 0; +} diff --git a/test/vulkan/test_submit.c b/test/vulkan/test_submit.c new file mode 100644 index 00000000..dfaec25f --- /dev/null +++ b/test/vulkan/test_submit.c @@ -0,0 +1,56 @@ +#include +#include +#include +#include + +#include "../../src/vulkan/dispatch.h" + +static int g_submit_called = 0; +static VkResult VKAPI_CALL fake_submit(VkQueue q, uint32_t n, const VkSubmitInfo *s, VkFence f) { + (void)q; + (void)n; + (void)s; + (void)f; + g_submit_called++; + return VK_SUCCESS; +} + +/* Throttle adapter stub — verifies the hook calls the adapter exactly once + * per submit before forwarding to the next layer. */ +static int g_throttle_called = 0; +void hami_vulkan_throttle(void) { g_throttle_called++; } + +/* Budget adapter stubs — linked via hooks_alloc.c / hooks_memory.c in this + * test binary even though we do not exercise allocation here. */ +size_t hami_budget_of(int dev) { (void)dev; return 0; } +int hami_budget_reserve(int dev, size_t size) { + (void)dev; + (void)size; + return 1; +} +void hami_budget_release(int dev, size_t size) { + (void)dev; + (void)size; +} + +/* NVML-based physdev resolver stub. */ +int hami_vk_physdev_index(VkPhysicalDevice p) { (void)p; return 0; } + +extern VKAPI_ATTR VkResult VKAPI_CALL +hami_vkQueueSubmit(VkQueue, uint32_t, const VkSubmitInfo*, VkFence); +extern void hami_vk_register_queue(VkQueue q, VkDevice d); + +int main(void) { + VkDevice dev = (VkDevice)0x11; + VkQueue q = (VkQueue)0x22; + hami_device_dispatch_t *d = hami_device_register(dev, (VkPhysicalDevice)0, NULL); + d->QueueSubmit = fake_submit; + hami_vk_register_queue(q, dev); + + VkResult r = hami_vkQueueSubmit(q, 0, NULL, VK_NULL_HANDLE); + assert(r == VK_SUCCESS); + assert(g_throttle_called == 1); + assert(g_submit_called == 1); + printf("ok: submit hook throttles then forwards\n"); + return 0; +} diff --git a/test/vulkan/test_throttle_adapter.c b/test/vulkan/test_throttle_adapter.c new file mode 100644 index 00000000..a7fabece --- /dev/null +++ b/test/vulkan/test_throttle_adapter.c @@ -0,0 +1,20 @@ +#include +#include + +#include "../../src/vulkan/throttle_adapter.h" + +/* Stub of HAMi-core's rate_limiter so this test links without the full lib. */ +static int g_rl_calls = 0; +void rate_limiter(int grids, int blocks) { + (void)grids; + (void)blocks; + g_rl_calls++; +} + +int main(void) { + hami_vulkan_throttle(); + hami_vulkan_throttle(); + assert(g_rl_calls == 2); + printf("ok: adapter forwards to rate_limiter\n"); + return 0; +}