fix: bundle NVIDIA 580.95.05 userspace libs for Akash CDI GPU access

Author: zJuuu

docker/train_entrypoint.sh

@@ -9,6 +9,24 @@ if [[ "${PSYCHE_MAIN_HOST:-}" != "" ]]; then
     exec /bin/sidecar_entrypoint.sh
 fi
 
+# GPU diagnostics
+echo "=== GPU DIAGNOSTICS ==="
+echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-<not set>}"
+echo "NVIDIA_VISIBLE_DEVICES=${NVIDIA_VISIBLE_DEVICES:-<not set>}"
+echo "NVIDIA_DRIVER_CAPABILITIES=${NVIDIA_DRIVER_CAPABILITIES:-<not set>}"
+ls /usr/local/nvidia/lib64/ 2>/dev/null && echo "Found /usr/local/nvidia/lib64" || echo "No /usr/local/nvidia/lib64"
+ls /dev/nvidia* 2>/dev/null || echo "No /dev/nvidia* devices found"
+echo "Host kernel driver version:"
+cat /proc/driver/nvidia/version 2>/dev/null || \
+  cat /sys/module/nvidia/version 2>/dev/null || \
+  echo "Cannot determine driver version"
+echo "Searching for libcuda in LD_LIBRARY_PATH dirs:"
+IFS=':' read -ra _LIB_DIRS <<< "${LD_LIBRARY_PATH:-}"
+for _dir in "${_LIB_DIRS[@]}"; do
+  ls "$_dir"/libcuda* 2>/dev/null && echo "  ^ in $_dir"
+done
+echo "=== END DIAGNOSTICS ==="
+
 # Some sanity checks before starting
 
 if [[ "${NVIDIA_DRIVER_CAPABILITIES:-}" == "" ]]; then

nix/docker.nix

@@ -102,6 +102,19 @@ let
     ]
   );
 
+  nvidiaUserSpaceLibs = lib.optionals pkgs.config.cudaSupport [
+    ((pkgs.linuxPackages.nvidiaPackages.dc.override {
+      libsOnly = true;
+      kernel = null;
+    }).overrideAttrs (old: let version = "580.95.05"; in {
+      inherit version;
+      src = pkgs.fetchurl {
+        url = "https://us.download.nvidia.com/tesla/${version}/NVIDIA-Linux-x86_64-${version}.run";
+        sha256 = "sha256-hJ7w746EK5gGss3p8RwTA9VPGpp2lGfk5dlhsv4Rgqc=";
+      };
+    }))
+  ];
+
   dockerPackages = {
     docker-psyche-solana-client = pkgs.dockerTools.streamLayeredImage {
       name = "psyche-solana-client";
@@ -115,7 +128,9 @@ let
           coreutils
           stdenv.cc
           rdma-core
-          rustPackages."psyche-solana-client"
+          dockerTools.fakeNss
+          dockerTools.usrBinEnv
+          rustPackages."psyche-solana-client".unwrapped
           rustPackages."psyche-centralized-client"
           rustPackages."inference"
           rustPackages."train"
@@ -128,15 +143,19 @@ let
             cp ${../docker/sidecar_entrypoint.sh} $out/bin/sidecar_entrypoint.sh
             chmod +x $out/bin/train_entrypoint.sh
             chmod +x $out/bin/sidecar_entrypoint.sh
+            ln -s ${bashInteractive}/bin/bash $out/bin/bash
+            ln -s ${bashInteractive}/bin/bash $out/bin/sh
           '')
         ]
-        ++ cudaRuntimeLibs;
+        ++ cudaRuntimeLibs
+        ++ nvidiaUserSpaceLibs;
 
       config = {
         Env = [
           "NVIDIA_DRIVER_CAPABILITIES=all"
           "NVIDIA_VISIBLE_DEVICES=all"
-          "LD_LIBRARY_PATH=${lib.makeLibraryPath cudaRuntimeLibs}:/lib:/usr/lib"
+          "LD_LIBRARY_PATH=${lib.makeLibraryPath (cudaRuntimeLibs ++ nvidiaUserSpaceLibs)}:/usr/local/nvidia/lib64:/lib:/usr/lib"
+          "PATH=${lib.makeBinPath (with pkgs; [ bashInteractive coreutils findutils gnugrep ])}:/bin:/usr/bin"
           "LOGNAME=root"
           "TORCHINDUCTOR_CACHE_DIR=/tmp/torchinductor"
           "PYTHONUNBUFFERED=1"

nix/lib.nix

@@ -531,6 +531,7 @@ let
       {
         nativeBuildInputs = [ pkgs.makeWrapper ];
         meta.mainProgram = package.meta.mainProgram;
+        passthru = { unwrapped = package; };
       }
       ''
         mkdir -p $out/bin

nix/nixpkgs.nix

@@ -46,6 +46,7 @@ in
 
     config = {
       allowUnfree = true;
+      nvidia.acceptLicense = true;
       metalSupport = lib.mkDefault false;
     }
     // lib.optionalAttrs cudaSupported {