Upgrade to use CUDA 13

.github/workflows/pr.yml

@@ -23,32 +23,30 @@ jobs:
           pip install virtualenv
           virtualenv .venv
           source .venv/bin/activate
-          python3 -m pip install nvidia-cuda-runtime-cu12
+          python3 -m pip install "nvidia-cuda-runtime==13.*"
+          CUDA_WHL_LIB_DIR="$(python3 -c 'import nvidia.cu13, os; print(os.path.join(nvidia.cu13.__path__[0], "lib"))')"
+          export LD_LIBRARY_PATH="${CUDA_WHL_LIB_DIR}:${LD_LIBRARY_PATH}"
+          echo "CUDA_WHL_LIB_DIR=${CUDA_WHL_LIB_DIR}" >> "${GITHUB_ENV}"
+          echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> "${GITHUB_ENV}"
           ./run setup
       - name: Check formatting
         run: |
           source .venv/bin/activate
-          python3 -m pip install nvidia-cuda-runtime-cu12
+          python3 -m pip install "nvidia-cuda-runtime==13.*"
           python3 -c 'import sys; print(sys.executable)'
           python3 -c 'import site; print(site.getsitepackages())'
           python3 -m pip freeze
-          export CUDA_WHL_LIB_DIR=$(python3 -c 'import nvidia.cuda_runtime; print(nvidia.cuda_runtime.__path__[0])')/lib
-          export LD_LIBRARY_PATH="$CUDA_WHL_LIB_DIR:$LD_LIBRARY_PATH"
           python3 -c 'from holoscan.core import *'
           ./run check -f
       - name: Run Unit tests
         run: |
           source .venv/bin/activate
-          python3 -m pip install nvidia-cuda-runtime-cu12
-          export CUDA_WHL_LIB_DIR=$(python3 -c 'import nvidia.cuda_runtime; print(nvidia.cuda_runtime.__path__[0])')/lib
-          export LD_LIBRARY_PATH="$CUDA_WHL_LIB_DIR:$LD_LIBRARY_PATH"
+          python3 -m pip install "nvidia-cuda-runtime==13.*"
           ./run test all unit
       - name: Coverage
         run: |
           source .venv/bin/activate
-          python3 -m pip install nvidia-cuda-runtime-cu12
-          export CUDA_WHL_LIB_DIR=$(python3 -c 'import nvidia.cuda_runtime; print(nvidia.cuda_runtime.__path__[0])')/lib
-          export LD_LIBRARY_PATH="$CUDA_WHL_LIB_DIR:$LD_LIBRARY_PATH"
+          python3 -m pip install "nvidia-cuda-runtime==13.*"
           coverage xml
       - name: Upload coverage
         uses: codecov/codecov-action@v2

README.md

@@ -38,10 +38,10 @@ pip install monai-deploy-app-sdk
 
 ### Prerequisites
 
-- This SDK depends on [NVIDIA Holoscan SDK](https://pypi.org/project/holoscan/) for its core implementation as well as its CLI, hence inherits its prerequisites, e.g. Ubuntu 22.04 with glibc 2.35 on X86-64 and NVIDIA dGPU drivers version 535 or above. Important to note that `holoscan` and `holoscan-cli` up to v4.2 are compatible.
-- Key runtime dependencies also include [nvidia-nvimgcodec](https://pypi.org/project/nvidia-nvimgcodec-cu12/) and its own dependencies for GPU accecelerated DICOM image decoding.
-- [CUDA 12.2](https://developer.nvidia.com/cuda-12-2-0-download-archive) or above is required along with a supported NVIDIA GPU with at least 8GB of video RAM.
-- If inference is not used in an example application and a GPU is not installed, at least [CUDA 12 runtime](https://pypi.org/project/nvidia-cuda-runtime-cu12/) is required, as this is one of the requirements of Holoscan SDK. In addition, the `LIB_LIBRARY_PATH` must be set to include the installed shared library, e.g. in a Python 3.10 env, ```export LD_LIBRARY_PATH=`pwd`/.venv/lib/python3.10/site-packages/nvidia/cuda_runtime/lib:$LD_LIBRARY_PATH```
+- This SDK depends on [NVIDIA Holoscan SDK (CUDA 13)](https://pypi.org/project/holoscan-cu13/) for its core implementation as well as its CLI, hence inherits its prerequisites, e.g. Ubuntu 22.04 with glibc 2.35+ (see output of ldd --version) and CUDA Runtime 13.0 or above. It is important to note that `holoscan-cu13` and `holoscan-cli` up to version 4.2 are compatible.
+- Key runtime dependencies also include [nvidia-nvimgcodec](https://pypi.org/project/nvidia-nvimgcodec-cu13/) and its own dependencies for GPU-accelerated DICOM image decoding.
+- [CUDA 13.0](https://developer.nvidia.com/cuda-downloads) or above is required along with a supported NVIDIA GPU with at least 8GB of video RAM.
+- If inference is not used in an example application and a GPU is not installed, at least [CUDA 13 runtime](https://pypi.org/project/nvidia-cuda-runtime/) is required, as this is one of the requirements of Holoscan SDK. In addition, the `LD_LIBRARY_PATH` must be set to include the installed shared library, e.g. in a Python 3.10 env, ```export LD_LIBRARY_PATH=`pwd`/.venv/lib/python3.10/site-packages/nvidia/cu13/lib:$LD_LIBRARY_PATH```
 - Python: 3.10 to 3.13
 
 ## Getting Started

THIRD_PARTY_NOTICES/nvidia-nvimgcodec_Apache2.0_LICENSE.txt

@@ -1,4 +1,4 @@
-nvidia-nvimgcodec (PyPI package: nvidia-nvimgcodec-cu12)
+nvidia-nvimgcodec (PyPI package: nvidia-nvimgcodec-cu13)
 
 The MONAI Deploy App SDK optionally integrates with NVIDIA nvImageCodec for
 GPU-accelerated DICOM compressed pixel data decoding.

monai/deploy/operators/decoder_nvimgcodec.py

@@ -108,7 +108,7 @@ def _decode_frame(src: bytes, runner: DecodeRunner) -> bytearray | bytes
     nvimgcodec = None
 
 # nvimgcodec pypi package name, minimum version required and the label for this decoder plugin.
-NVIMGCODEC_MODULE_NAME = "nvidia.nvimgcodec"  # from nvidia-nvimgcodec-cu12 or other variants
+NVIMGCODEC_MODULE_NAME = "nvidia.nvimgcodec"  # from nvidia-nvimgcodec-cu13 or other variants
 NVIMGCODEC_MIN_VERSION = "0.6"
 NVIMGCODEC_MIN_VERSION_TUPLE = tuple(int(x) for x in NVIMGCODEC_MIN_VERSION.split("."))
 NVIMGCODEC_PLUGIN_LABEL = "0.6+nvimgcodec"  # to be sorted to first in ascending order of plugins
@@ -139,7 +139,12 @@ def _decode_frame(src: bytes, runner: DecodeRunner) -> bytearray | bytes
 
 # Required for decoder plugin
 DECODER_DEPENDENCIES = {
-    x: ("numpy", "cupy", f"{NVIMGCODEC_MODULE_NAME}>={NVIMGCODEC_MIN_VERSION}, nvidia-nvjpeg2k-cu12>=0.9.1,")
+    x: (
+        "numpy",
+        "cupy",
+        f"{NVIMGCODEC_MODULE_NAME}>={NVIMGCODEC_MIN_VERSION}",
+        "nvidia-nvjpeg2k-cu13>=0.9.1",
+    )
     for x in SUPPORTED_TRANSFER_SYNTAXES
 }
 

monai/deploy/operators/dicom_series_to_volume_operator.py

@@ -64,8 +64,7 @@ def __init__(self, fragment: Fragment, *args, affine_lps_to_ras: bool = True, **
         based decoder plugins are available at runtime.
 
         Registering the decoder plugin is all automatic and does not require any additional change in user's application
-        except for adding a dependency on the `nvimgcodec-cu12` and `nvidia-nvjpeg2k-cu12` packages (suffix of cu12 means
-        CUDA 12.0 though cu13 is also supported).
+        except for adding a dependency on the `nvidia-nvimgcodec-cu13` and `all` of its optional dependencies.
 
         Named Input:
             study_selected_series_list: List of StudySelectedSeries.

notebooks/tutorials/01_simple_app.ipynb

@@ -959,7 +959,7 @@
    "source": [
     "tag_prefix = \"simple_imaging_app\"\n",
     "\n",
-    "!monai-deploy package simple_imaging_app -c simple_imaging_app/app.yaml -t {tag_prefix}:1.0 --platform x86_64 --cuda 12 -l DEBUG"
+    "!monai-deploy package simple_imaging_app -c simple_imaging_app/app.yaml -t {tag_prefix}:1.0 --platform x86_64 -l DEBUG"
    ]
   },
   {

notebooks/tutorials/02_mednist_app-prebuilt.ipynb

@@ -161,7 +161,7 @@
    "source": [
     "tag_prefix = \"mednist_app\"\n",
     "\n",
-    "!monai-deploy package \"source/examples/apps/mednist_classifier_monaideploy/mednist_classifier_monaideploy.py\" -m {models_folder} -c \"source/examples/apps/mednist_classifier_monaideploy/app.yaml\" -t {tag_prefix}:1.0 --platform x86_64 -l DEBUG --cuda 12"
+    "!monai-deploy package \"source/examples/apps/mednist_classifier_monaideploy/mednist_classifier_monaideploy.py\" -m {models_folder} -c \"source/examples/apps/mednist_classifier_monaideploy/app.yaml\" -t {tag_prefix}:1.0 --platform x86_64 -l DEBUG"
    ]
   },
   {

notebooks/tutorials/02_mednist_app.ipynb

@@ -1023,7 +1023,7 @@
    "source": [
     "tag_prefix = \"mednist_app\"\n",
     "\n",
-    "!monai-deploy package \"mednist_app/mednist_classifier_monaideploy.py\" -m {models_folder} -c \"mednist_app/app.yaml\" -t {tag_prefix}:1.0 --platform x86_64 -l DEBUG --cuda 12"
+    "!monai-deploy package \"mednist_app/mednist_classifier_monaideploy.py\" -m {models_folder} -c \"mednist_app/app.yaml\" -t {tag_prefix}:1.0 --platform x86_64 -l DEBUG"
    ]
   },
   {

notebooks/tutorials/03_segmentation_app.ipynb

@@ -989,7 +989,7 @@
    "source": [
     "tag_prefix = \"my_app\"\n",
     "\n",
-    "!monai-deploy package my_app -m {models_folder} -c my_app/app.yaml -t {tag_prefix}:1.0 --platform x86_64 -l DEBUG --cuda 12"
+    "!monai-deploy package my_app -m {models_folder} -c my_app/app.yaml -t {tag_prefix}:1.0 --platform x86_64 -l DEBUG"
    ]
   },
   {

notebooks/tutorials/04_monai_bundle_app.ipynb

@@ -740,7 +740,7 @@
       "source": [
         "tag_prefix = \"my_app\"\n",
         "\n",
-        "!monai-deploy package my_app -m {models_folder} -c my_app/app.yaml -t {tag_prefix}:1.0 --platform x86_64 -l DEBUG --cuda 12 "
+        "!monai-deploy package my_app -m {models_folder} -c my_app/app.yaml -t {tag_prefix}:1.0 --platform x86_64 -l DEBUG"
       ]
     },
     {

notebooks/tutorials/05_multi_model_app.ipynb

@@ -931,7 +931,7 @@
    "source": [
     "tag_prefix = \"my_app\"\n",
     "\n",
-    "!monai-deploy package my_app -m {models_folder} -c my_app/app.yaml -t {tag_prefix}:1.0 --platform x86_64 -l DEBUG  --cuda 12"
+    "!monai-deploy package my_app -m {models_folder} -c my_app/app.yaml -t {tag_prefix}:1.0 --platform x86_64 -l DEBUG"
    ]
   },
   {

requirements-dev.txt

@@ -34,7 +34,7 @@ nibabel>=3.2.1
 numpy-stl>=2.12.0
 trimesh>=3.8.11
 torch>=2.6.0
-nvidia-nvimgcodec-cu12[all]>=0.6.1
+nvidia-nvimgcodec-cu13[all]>=0.6.1
 python-gdcm>=3.0.10
 pylibjpeg>=2.0
 pylibjpeg-libjpeg>=2.1

requirements-examples.txt

@@ -10,7 +10,7 @@ numpy-stl>=2.12.0
 trimesh>=3.8.11
 torch>=2.6.0
 monai>=1.3.0
-nvidia-nvimgcodec-cu12[all]>=0.6.1
+nvidia-nvimgcodec-cu13[all]>=0.6.1
 python-gdcm>=3.0.10
 pylibjpeg>=2.0
 pylibjpeg-libjpeg>=2.1

requirements.txt

@@ -1,4 +1,4 @@
-holoscan-cu12>=4.0.0,<4.3.0
+holoscan-cu13>=4.0.0,<4.3.0
 holoscan-cli>=4.0.0,<4.3.0
 numpy>=1.21.6
 colorama>=0.4.1

run

@@ -339,15 +339,15 @@ install_python_dev_deps() {
         run_command ${MONAI_PY_EXE} -m pip install -U PyYAML
 
         # Install cuda runtime dependency
-        run_command ${MONAI_PY_EXE} -m pip install nvidia-cuda-runtime-cu12
+        run_command ${MONAI_PY_EXE} -m pip install "nvidia-cuda-runtime==13.*"
 
         # Copy the cuda runtime library to the fixed location (workaround for readthedocs) so that
         # we can leverage the existing LD_LIBRARY_PATH (configured by the readthedocs UI) to locate the cuda runtime library.
         # (LD_LIBRARY_PATH is set to /home/docs/ for that purpose)
         # Note that 'python3.10' is hard-coded here, it should be updated if the Python version changes by
         # .readthedocs.yml or other configurations.
-        run_command ls -al /home/docs/checkouts/readthedocs.org/user_builds/${READTHEDOCS_PROJECT}/envs/${READTHEDOCS_VERSION}/lib/python3.10/site-packages/nvidia/cuda_runtime/lib/
-        run_command cp /home/docs/checkouts/readthedocs.org/user_builds/${READTHEDOCS_PROJECT}/envs/${READTHEDOCS_VERSION}/lib/python3.10/site-packages/nvidia/cuda_runtime/lib/*.so* /home/docs/
+        run_command ls -al /home/docs/checkouts/readthedocs.org/user_builds/${READTHEDOCS_PROJECT}/envs/${READTHEDOCS_VERSION}/lib/python3.10/site-packages/nvidia/cu13/lib/
+        run_command cp /home/docs/checkouts/readthedocs.org/user_builds/${READTHEDOCS_PROJECT}/envs/${READTHEDOCS_VERSION}/lib/python3.10/site-packages/nvidia/cu13/lib/*.so* /home/docs/
         run_command ls -al /home/docs/
     fi
 }

setup.cfg

@@ -24,7 +24,7 @@ python_requires = >=3.10, <3.14
 #     cucim
 install_requires =
     numpy>=1.21.6
-    holoscan-cu12
+    holoscan-cu13
     holoscan-cli
     colorama>=0.4.1
     tritonclient[all]>=2.53.0

tests/unit/test_decoder_nvimgcodec.py

@@ -1,4 +1,5 @@
 import logging
+import sys
 import time
 from pathlib import Path
 from typing import Any, Iterator, cast
@@ -31,6 +32,49 @@
 _DEFAULT_PLUGIN_CACHE: dict[str, Any] = {}
 _logger = logging.getLogger(__name__)
 
+_LOG_FORMAT = "%(asctime)s.%(msecs)03d [%(levelname)s] %(name)s: %(message)s"
+_LOG_DATE_FORMAT = "%H:%M:%S"
+_DECODER_LOGGER = "monai.deploy.operators.decoder_nvimgcodec"
+
+
+@pytest.fixture(scope="module", autouse=True)
+def _configure_decoder_test_console_logging() -> Iterator[None]:
+    """Emit decoder and test logs to the console while this module runs."""
+    handler = logging.StreamHandler(sys.stderr)
+    handler.setFormatter(logging.Formatter(_LOG_FORMAT, datefmt=_LOG_DATE_FORMAT))
+    handler.setLevel(logging.DEBUG)
+
+    configured_loggers: list[tuple[logging.Logger, int, bool]] = []
+    for name in (_DECODER_LOGGER, __name__):
+        logger = logging.getLogger(name)
+        configured_loggers.append((logger, logger.level, logger.propagate))
+        logger.addHandler(handler)
+        logger.setLevel(logging.DEBUG)
+        logger.propagate = False
+
+    _logger.info("Verbose console logging enabled for nvimgcodec decoder tests.")
+    yield
+
+    for logger, previous_level, previous_propagate in configured_loggers:
+        logger.removeHandler(handler)
+        logger.setLevel(previous_level)
+        logger.propagate = previous_propagate
+    handler.close()
+
+
+def _summarize_array(pixels: np.ndarray) -> str:
+    """Return a compact summary of decoded pixel data for logging."""
+    arr = np.asarray(pixels)
+    summary = f"shape={arr.shape}, dtype={arr.dtype}"
+    if arr.size == 0:
+        return summary
+
+    if np.issubdtype(arr.dtype, np.floating):
+        summary += f", min={float(np.nanmin(arr)):.4g}, max={float(np.nanmax(arr)):.4g}"
+    else:
+        summary += f", min={int(arr.min())}, max={int(arr.max())}"
+    return summary
+
 
 def _iter_frames(pixel_array: np.ndarray) -> Iterator[tuple[int, np.ndarray, bool]]:
     """Yield per-frame arrays and whether they represent color data."""
@@ -161,6 +205,7 @@ def test_nvimgcodec_decoder_matches_default(path: str) -> None:
 
     rtol = 0.01
     atol = 4.0
+    file_name = Path(path).name
 
     baseline_pixels: np.ndarray = np.array([])
     nv_pixels: np.ndarray = np.array([])
@@ -171,24 +216,48 @@ def test_nvimgcodec_decoder_matches_default(path: str) -> None:
     default_decoder_error_message = None
     nvimgcodec_decoder_error_message = None
     transfer_syntax = None
+    _logger.info("Testing %s", path)
     try:
         ds_default = dcmread(path)
         transfer_syntax = ds_default.file_meta.TransferSyntaxUID
+        _logger.info(
+            "Default decoder: file=%s transfer_syntax=%s samples_per_pixel=%s photometric=%s",
+            file_name,
+            transfer_syntax,
+            getattr(ds_default, "SamplesPerPixel", "?"),
+            getattr(ds_default, "PhotometricInterpretation", "?"),
+        )
+        start = time.perf_counter()
         baseline_pixels = ds_default.pixel_array
+        baseline_elapsed = time.perf_counter() - start
+        _logger.info(
+            "Default decoder finished in %.4fs: %s",
+            baseline_elapsed,
+            _summarize_array(baseline_pixels),
+        )
     except Exception as e:
         default_decoder_error_message = f"{e}"
         default_decoder_errored = True
+        _logger.warning("Default decoder failed for %s: %s", file_name, default_decoder_error_message)
 
     # Remove and cache the other default decoder plugins first
     _remove_default_plugins()
     # Register the nvimgcodec decoder plugin and unregister it after each use.
     register_as_decoder_plugin()
     try:
         ds_custom = dcmread(path)
+        start = time.perf_counter()
         nv_pixels = ds_custom.pixel_array
+        nv_elapsed = time.perf_counter() - start
+        _logger.info(
+            "nvimgcodec decoder finished in %.4fs: %s",
+            nv_elapsed,
+            _summarize_array(nv_pixels),
+        )
     except Exception as e:
         nvimgcodec_decoder_error_message = f"{e}"
         nvimgcodec_decoder_errored = True
+        _logger.warning("nvimgcodec decoder failed for %s: %s", file_name, nvimgcodec_decoder_error_message)
     finally:
         unregister_as_decoder_plugin()
         _restore_default_plugins()
@@ -210,10 +279,24 @@ def test_nvimgcodec_decoder_matches_default(path: str) -> None:
     assert baseline_pixels.dtype == nv_pixels.dtype, f"Dtype mismatch with transfer syntax {transfer_syntax}"
     try:
         np.testing.assert_allclose(baseline_pixels, nv_pixels, rtol=rtol, atol=atol)
-        _logger.info(f"Pixels values matched for transfer syntax: {transfer_syntax} in file: {Path(path).name}")
+        _logger.info(
+            "Pixels matched for transfer_syntax=%s file=%s rtol=%s atol=%s",
+            transfer_syntax,
+            file_name,
+            rtol,
+            atol,
+        )
     except AssertionError as e:
+        diff = np.abs(baseline_pixels.astype(np.float64) - nv_pixels.astype(np.float64))
+        _logger.error(
+            "Pixel mismatch for transfer_syntax=%s file=%s max_abs_diff=%.4g mean_abs_diff=%.4g",
+            transfer_syntax,
+            file_name,
+            float(diff.max()) if diff.size else 0.0,
+            float(diff.mean()) if diff.size else 0.0,
+        )
         raise AssertionError(
-            f"Pixels values mismatch for transfer syntax: {transfer_syntax} in file: {Path(path).name}"
+            f"Pixels values mismatch for transfer syntax: {transfer_syntax} in file: {file_name}"
         ) from e
 
 
@@ -350,7 +433,9 @@ def _restore_default_plugins():
 if __name__ == "__main__":
 
     # Use pytest to test the functionality with pydicom embedded DICOM files of supported transfer syntaxes individually
-    # python -m pytest test_decoder_nvimgcodec.py
+    # python -m pytest tests/unit/test_decoder_nvimgcodec.py -vv --log-cli-level=DEBUG \
+    #   --log-cli-format="%(asctime)s [%(levelname)s] %(name)s: %(message)s" \
+    #   --log-cli-date-format="%H:%M:%S.%f"
     #
     # The following compares the performance of the nvimgcodec decoder against the default decoders
     # with DICOM files in pydicom embedded dataset or an optional custom folder