diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 34a63ab7..6144a538 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -23,32 +23,30 @@ jobs: pip install virtualenv virtualenv .venv source .venv/bin/activate - python3 -m pip install nvidia-cuda-runtime-cu12 + python3 -m pip install "nvidia-cuda-runtime==13.*" + CUDA_WHL_LIB_DIR="$(python3 -c 'import nvidia.cu13, os; print(os.path.join(nvidia.cu13.__path__[0], "lib"))')" + export LD_LIBRARY_PATH="${CUDA_WHL_LIB_DIR}:${LD_LIBRARY_PATH}" + echo "CUDA_WHL_LIB_DIR=${CUDA_WHL_LIB_DIR}" >> "${GITHUB_ENV}" + echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> "${GITHUB_ENV}" ./run setup - name: Check formatting run: | source .venv/bin/activate - python3 -m pip install nvidia-cuda-runtime-cu12 + python3 -m pip install "nvidia-cuda-runtime==13.*" python3 -c 'import sys; print(sys.executable)' python3 -c 'import site; print(site.getsitepackages())' python3 -m pip freeze - export CUDA_WHL_LIB_DIR=$(python3 -c 'import nvidia.cuda_runtime; print(nvidia.cuda_runtime.__path__[0])')/lib - export LD_LIBRARY_PATH="$CUDA_WHL_LIB_DIR:$LD_LIBRARY_PATH" python3 -c 'from holoscan.core import *' ./run check -f - name: Run Unit tests run: | source .venv/bin/activate - python3 -m pip install nvidia-cuda-runtime-cu12 - export CUDA_WHL_LIB_DIR=$(python3 -c 'import nvidia.cuda_runtime; print(nvidia.cuda_runtime.__path__[0])')/lib - export LD_LIBRARY_PATH="$CUDA_WHL_LIB_DIR:$LD_LIBRARY_PATH" + python3 -m pip install "nvidia-cuda-runtime==13.*" ./run test all unit - name: Coverage run: | source .venv/bin/activate - python3 -m pip install nvidia-cuda-runtime-cu12 - export CUDA_WHL_LIB_DIR=$(python3 -c 'import nvidia.cuda_runtime; print(nvidia.cuda_runtime.__path__[0])')/lib - export LD_LIBRARY_PATH="$CUDA_WHL_LIB_DIR:$LD_LIBRARY_PATH" + python3 -m pip install "nvidia-cuda-runtime==13.*" coverage xml - name: Upload coverage uses: codecov/codecov-action@v2 diff --git a/README.md b/README.md index 358cb92d..baa9dc01 100644 --- a/README.md +++ b/README.md @@ -38,10 +38,10 @@ pip install monai-deploy-app-sdk ### Prerequisites -- This SDK depends on [NVIDIA Holoscan SDK](https://pypi.org/project/holoscan/) for its core implementation as well as its CLI, hence inherits its prerequisites, e.g. Ubuntu 22.04 with glibc 2.35 on X86-64 and NVIDIA dGPU drivers version 535 or above. Important to note that `holoscan` and `holoscan-cli` up to v4.2 are compatible. -- Key runtime dependencies also include [nvidia-nvimgcodec](https://pypi.org/project/nvidia-nvimgcodec-cu12/) and its own dependencies for GPU accecelerated DICOM image decoding. -- [CUDA 12.2](https://developer.nvidia.com/cuda-12-2-0-download-archive) or above is required along with a supported NVIDIA GPU with at least 8GB of video RAM. -- If inference is not used in an example application and a GPU is not installed, at least [CUDA 12 runtime](https://pypi.org/project/nvidia-cuda-runtime-cu12/) is required, as this is one of the requirements of Holoscan SDK. In addition, the `LIB_LIBRARY_PATH` must be set to include the installed shared library, e.g. in a Python 3.10 env, ```export LD_LIBRARY_PATH=`pwd`/.venv/lib/python3.10/site-packages/nvidia/cuda_runtime/lib:$LD_LIBRARY_PATH``` +- This SDK depends on [NVIDIA Holoscan SDK (CUDA 13)](https://pypi.org/project/holoscan-cu13/) for its core implementation as well as its CLI, hence inherits its prerequisites, e.g. Ubuntu 22.04 with glibc 2.35+ (see output of ldd --version) and CUDA Runtime 13.0 or above. It is important to note that `holoscan-cu13` and `holoscan-cli` up to version 4.2 are compatible. +- Key runtime dependencies also include [nvidia-nvimgcodec](https://pypi.org/project/nvidia-nvimgcodec-cu13/) and its own dependencies for GPU-accelerated DICOM image decoding. +- [CUDA 13.0](https://developer.nvidia.com/cuda-downloads) or above is required along with a supported NVIDIA GPU with at least 8GB of video RAM. +- If inference is not used in an example application and a GPU is not installed, at least [CUDA 13 runtime](https://pypi.org/project/nvidia-cuda-runtime/) is required, as this is one of the requirements of Holoscan SDK. In addition, the `LD_LIBRARY_PATH` must be set to include the installed shared library, e.g. in a Python 3.10 env, ```export LD_LIBRARY_PATH=`pwd`/.venv/lib/python3.10/site-packages/nvidia/cu13/lib:$LD_LIBRARY_PATH``` - Python: 3.10 to 3.13 ## Getting Started diff --git a/THIRD_PARTY_NOTICES/nvidia-nvimgcodec_Apache2.0_LICENSE.txt b/THIRD_PARTY_NOTICES/nvidia-nvimgcodec_Apache2.0_LICENSE.txt index deb4841e..791c0862 100644 --- a/THIRD_PARTY_NOTICES/nvidia-nvimgcodec_Apache2.0_LICENSE.txt +++ b/THIRD_PARTY_NOTICES/nvidia-nvimgcodec_Apache2.0_LICENSE.txt @@ -1,4 +1,4 @@ -nvidia-nvimgcodec (PyPI package: nvidia-nvimgcodec-cu12) +nvidia-nvimgcodec (PyPI package: nvidia-nvimgcodec-cu13) The MONAI Deploy App SDK optionally integrates with NVIDIA nvImageCodec for GPU-accelerated DICOM compressed pixel data decoding. diff --git a/monai/deploy/operators/decoder_nvimgcodec.py b/monai/deploy/operators/decoder_nvimgcodec.py index 736fb9ba..b75bd0c6 100644 --- a/monai/deploy/operators/decoder_nvimgcodec.py +++ b/monai/deploy/operators/decoder_nvimgcodec.py @@ -108,7 +108,7 @@ def _decode_frame(src: bytes, runner: DecodeRunner) -> bytearray | bytes nvimgcodec = None # nvimgcodec pypi package name, minimum version required and the label for this decoder plugin. -NVIMGCODEC_MODULE_NAME = "nvidia.nvimgcodec" # from nvidia-nvimgcodec-cu12 or other variants +NVIMGCODEC_MODULE_NAME = "nvidia.nvimgcodec" # from nvidia-nvimgcodec-cu13 or other variants NVIMGCODEC_MIN_VERSION = "0.6" NVIMGCODEC_MIN_VERSION_TUPLE = tuple(int(x) for x in NVIMGCODEC_MIN_VERSION.split(".")) NVIMGCODEC_PLUGIN_LABEL = "0.6+nvimgcodec" # to be sorted to first in ascending order of plugins @@ -139,7 +139,12 @@ def _decode_frame(src: bytes, runner: DecodeRunner) -> bytearray | bytes # Required for decoder plugin DECODER_DEPENDENCIES = { - x: ("numpy", "cupy", f"{NVIMGCODEC_MODULE_NAME}>={NVIMGCODEC_MIN_VERSION}, nvidia-nvjpeg2k-cu12>=0.9.1,") + x: ( + "numpy", + "cupy", + f"{NVIMGCODEC_MODULE_NAME}>={NVIMGCODEC_MIN_VERSION}", + "nvidia-nvjpeg2k-cu13>=0.9.1", + ) for x in SUPPORTED_TRANSFER_SYNTAXES } diff --git a/monai/deploy/operators/dicom_series_to_volume_operator.py b/monai/deploy/operators/dicom_series_to_volume_operator.py index dacb59ce..751dfc8d 100644 --- a/monai/deploy/operators/dicom_series_to_volume_operator.py +++ b/monai/deploy/operators/dicom_series_to_volume_operator.py @@ -64,8 +64,7 @@ def __init__(self, fragment: Fragment, *args, affine_lps_to_ras: bool = True, ** based decoder plugins are available at runtime. Registering the decoder plugin is all automatic and does not require any additional change in user's application - except for adding a dependency on the `nvimgcodec-cu12` and `nvidia-nvjpeg2k-cu12` packages (suffix of cu12 means - CUDA 12.0 though cu13 is also supported). + except for adding a dependency on the `nvidia-nvimgcodec-cu13` and `all` of its optional dependencies. Named Input: study_selected_series_list: List of StudySelectedSeries. diff --git a/notebooks/tutorials/01_simple_app.ipynb b/notebooks/tutorials/01_simple_app.ipynb index 36c3c7d4..b1a6540b 100644 --- a/notebooks/tutorials/01_simple_app.ipynb +++ b/notebooks/tutorials/01_simple_app.ipynb @@ -959,7 +959,7 @@ "source": [ "tag_prefix = \"simple_imaging_app\"\n", "\n", - "!monai-deploy package simple_imaging_app -c simple_imaging_app/app.yaml -t {tag_prefix}:1.0 --platform x86_64 --cuda 12 -l DEBUG" + "!monai-deploy package simple_imaging_app -c simple_imaging_app/app.yaml -t {tag_prefix}:1.0 --platform x86_64 -l DEBUG" ] }, { diff --git a/notebooks/tutorials/02_mednist_app-prebuilt.ipynb b/notebooks/tutorials/02_mednist_app-prebuilt.ipynb index 6334d118..ff43c838 100644 --- a/notebooks/tutorials/02_mednist_app-prebuilt.ipynb +++ b/notebooks/tutorials/02_mednist_app-prebuilt.ipynb @@ -161,7 +161,7 @@ "source": [ "tag_prefix = \"mednist_app\"\n", "\n", - "!monai-deploy package \"source/examples/apps/mednist_classifier_monaideploy/mednist_classifier_monaideploy.py\" -m {models_folder} -c \"source/examples/apps/mednist_classifier_monaideploy/app.yaml\" -t {tag_prefix}:1.0 --platform x86_64 -l DEBUG --cuda 12" + "!monai-deploy package \"source/examples/apps/mednist_classifier_monaideploy/mednist_classifier_monaideploy.py\" -m {models_folder} -c \"source/examples/apps/mednist_classifier_monaideploy/app.yaml\" -t {tag_prefix}:1.0 --platform x86_64 -l DEBUG" ] }, { diff --git a/notebooks/tutorials/02_mednist_app.ipynb b/notebooks/tutorials/02_mednist_app.ipynb index 1dc32db5..2c9c761d 100644 --- a/notebooks/tutorials/02_mednist_app.ipynb +++ b/notebooks/tutorials/02_mednist_app.ipynb @@ -1023,7 +1023,7 @@ "source": [ "tag_prefix = \"mednist_app\"\n", "\n", - "!monai-deploy package \"mednist_app/mednist_classifier_monaideploy.py\" -m {models_folder} -c \"mednist_app/app.yaml\" -t {tag_prefix}:1.0 --platform x86_64 -l DEBUG --cuda 12" + "!monai-deploy package \"mednist_app/mednist_classifier_monaideploy.py\" -m {models_folder} -c \"mednist_app/app.yaml\" -t {tag_prefix}:1.0 --platform x86_64 -l DEBUG" ] }, { diff --git a/notebooks/tutorials/03_segmentation_app.ipynb b/notebooks/tutorials/03_segmentation_app.ipynb index b9e4d627..339cf292 100644 --- a/notebooks/tutorials/03_segmentation_app.ipynb +++ b/notebooks/tutorials/03_segmentation_app.ipynb @@ -989,7 +989,7 @@ "source": [ "tag_prefix = \"my_app\"\n", "\n", - "!monai-deploy package my_app -m {models_folder} -c my_app/app.yaml -t {tag_prefix}:1.0 --platform x86_64 -l DEBUG --cuda 12" + "!monai-deploy package my_app -m {models_folder} -c my_app/app.yaml -t {tag_prefix}:1.0 --platform x86_64 -l DEBUG" ] }, { diff --git a/notebooks/tutorials/04_monai_bundle_app.ipynb b/notebooks/tutorials/04_monai_bundle_app.ipynb index ca9fbb92..680e1a44 100644 --- a/notebooks/tutorials/04_monai_bundle_app.ipynb +++ b/notebooks/tutorials/04_monai_bundle_app.ipynb @@ -740,7 +740,7 @@ "source": [ "tag_prefix = \"my_app\"\n", "\n", - "!monai-deploy package my_app -m {models_folder} -c my_app/app.yaml -t {tag_prefix}:1.0 --platform x86_64 -l DEBUG --cuda 12 " + "!monai-deploy package my_app -m {models_folder} -c my_app/app.yaml -t {tag_prefix}:1.0 --platform x86_64 -l DEBUG" ] }, { diff --git a/notebooks/tutorials/05_multi_model_app.ipynb b/notebooks/tutorials/05_multi_model_app.ipynb index c68f08a6..0477b1ea 100644 --- a/notebooks/tutorials/05_multi_model_app.ipynb +++ b/notebooks/tutorials/05_multi_model_app.ipynb @@ -931,7 +931,7 @@ "source": [ "tag_prefix = \"my_app\"\n", "\n", - "!monai-deploy package my_app -m {models_folder} -c my_app/app.yaml -t {tag_prefix}:1.0 --platform x86_64 -l DEBUG --cuda 12" + "!monai-deploy package my_app -m {models_folder} -c my_app/app.yaml -t {tag_prefix}:1.0 --platform x86_64 -l DEBUG" ] }, { diff --git a/requirements-dev.txt b/requirements-dev.txt index 8aaab66f..c28b676a 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -34,7 +34,7 @@ nibabel>=3.2.1 numpy-stl>=2.12.0 trimesh>=3.8.11 torch>=2.6.0 -nvidia-nvimgcodec-cu12[all]>=0.6.1 +nvidia-nvimgcodec-cu13[all]>=0.6.1 python-gdcm>=3.0.10 pylibjpeg>=2.0 pylibjpeg-libjpeg>=2.1 diff --git a/requirements-examples.txt b/requirements-examples.txt index 0aeb394a..e510c5cc 100644 --- a/requirements-examples.txt +++ b/requirements-examples.txt @@ -10,7 +10,7 @@ numpy-stl>=2.12.0 trimesh>=3.8.11 torch>=2.6.0 monai>=1.3.0 -nvidia-nvimgcodec-cu12[all]>=0.6.1 +nvidia-nvimgcodec-cu13[all]>=0.6.1 python-gdcm>=3.0.10 pylibjpeg>=2.0 pylibjpeg-libjpeg>=2.1 diff --git a/requirements.txt b/requirements.txt index 70a4812a..4a28717a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -holoscan-cu12>=4.0.0,<4.3.0 +holoscan-cu13>=4.0.0,<4.3.0 holoscan-cli>=4.0.0,<4.3.0 numpy>=1.21.6 colorama>=0.4.1 diff --git a/run b/run index 33cb0041..b9209f50 100755 --- a/run +++ b/run @@ -339,15 +339,15 @@ install_python_dev_deps() { run_command ${MONAI_PY_EXE} -m pip install -U PyYAML # Install cuda runtime dependency - run_command ${MONAI_PY_EXE} -m pip install nvidia-cuda-runtime-cu12 + run_command ${MONAI_PY_EXE} -m pip install "nvidia-cuda-runtime==13.*" # Copy the cuda runtime library to the fixed location (workaround for readthedocs) so that # we can leverage the existing LD_LIBRARY_PATH (configured by the readthedocs UI) to locate the cuda runtime library. # (LD_LIBRARY_PATH is set to /home/docs/ for that purpose) # Note that 'python3.10' is hard-coded here, it should be updated if the Python version changes by # .readthedocs.yml or other configurations. - run_command ls -al /home/docs/checkouts/readthedocs.org/user_builds/${READTHEDOCS_PROJECT}/envs/${READTHEDOCS_VERSION}/lib/python3.10/site-packages/nvidia/cuda_runtime/lib/ - run_command cp /home/docs/checkouts/readthedocs.org/user_builds/${READTHEDOCS_PROJECT}/envs/${READTHEDOCS_VERSION}/lib/python3.10/site-packages/nvidia/cuda_runtime/lib/*.so* /home/docs/ + run_command ls -al /home/docs/checkouts/readthedocs.org/user_builds/${READTHEDOCS_PROJECT}/envs/${READTHEDOCS_VERSION}/lib/python3.10/site-packages/nvidia/cu13/lib/ + run_command cp /home/docs/checkouts/readthedocs.org/user_builds/${READTHEDOCS_PROJECT}/envs/${READTHEDOCS_VERSION}/lib/python3.10/site-packages/nvidia/cu13/lib/*.so* /home/docs/ run_command ls -al /home/docs/ fi } diff --git a/setup.cfg b/setup.cfg index 17c587b9..fe8db6f8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -24,7 +24,7 @@ python_requires = >=3.10, <3.14 # cucim install_requires = numpy>=1.21.6 - holoscan-cu12 + holoscan-cu13 holoscan-cli colorama>=0.4.1 tritonclient[all]>=2.53.0 diff --git a/tests/unit/test_decoder_nvimgcodec.py b/tests/unit/test_decoder_nvimgcodec.py index 1b10bbaf..138c8058 100644 --- a/tests/unit/test_decoder_nvimgcodec.py +++ b/tests/unit/test_decoder_nvimgcodec.py @@ -1,4 +1,5 @@ import logging +import sys import time from pathlib import Path from typing import Any, Iterator, cast @@ -31,6 +32,49 @@ _DEFAULT_PLUGIN_CACHE: dict[str, Any] = {} _logger = logging.getLogger(__name__) +_LOG_FORMAT = "%(asctime)s.%(msecs)03d [%(levelname)s] %(name)s: %(message)s" +_LOG_DATE_FORMAT = "%H:%M:%S" +_DECODER_LOGGER = "monai.deploy.operators.decoder_nvimgcodec" + + +@pytest.fixture(scope="module", autouse=True) +def _configure_decoder_test_console_logging() -> Iterator[None]: + """Emit decoder and test logs to the console while this module runs.""" + handler = logging.StreamHandler(sys.stderr) + handler.setFormatter(logging.Formatter(_LOG_FORMAT, datefmt=_LOG_DATE_FORMAT)) + handler.setLevel(logging.DEBUG) + + configured_loggers: list[tuple[logging.Logger, int, bool]] = [] + for name in (_DECODER_LOGGER, __name__): + logger = logging.getLogger(name) + configured_loggers.append((logger, logger.level, logger.propagate)) + logger.addHandler(handler) + logger.setLevel(logging.DEBUG) + logger.propagate = False + + _logger.info("Verbose console logging enabled for nvimgcodec decoder tests.") + yield + + for logger, previous_level, previous_propagate in configured_loggers: + logger.removeHandler(handler) + logger.setLevel(previous_level) + logger.propagate = previous_propagate + handler.close() + + +def _summarize_array(pixels: np.ndarray) -> str: + """Return a compact summary of decoded pixel data for logging.""" + arr = np.asarray(pixels) + summary = f"shape={arr.shape}, dtype={arr.dtype}" + if arr.size == 0: + return summary + + if np.issubdtype(arr.dtype, np.floating): + summary += f", min={float(np.nanmin(arr)):.4g}, max={float(np.nanmax(arr)):.4g}" + else: + summary += f", min={int(arr.min())}, max={int(arr.max())}" + return summary + def _iter_frames(pixel_array: np.ndarray) -> Iterator[tuple[int, np.ndarray, bool]]: """Yield per-frame arrays and whether they represent color data.""" @@ -161,6 +205,7 @@ def test_nvimgcodec_decoder_matches_default(path: str) -> None: rtol = 0.01 atol = 4.0 + file_name = Path(path).name baseline_pixels: np.ndarray = np.array([]) nv_pixels: np.ndarray = np.array([]) @@ -171,13 +216,29 @@ def test_nvimgcodec_decoder_matches_default(path: str) -> None: default_decoder_error_message = None nvimgcodec_decoder_error_message = None transfer_syntax = None + _logger.info("Testing %s", path) try: ds_default = dcmread(path) transfer_syntax = ds_default.file_meta.TransferSyntaxUID + _logger.info( + "Default decoder: file=%s transfer_syntax=%s samples_per_pixel=%s photometric=%s", + file_name, + transfer_syntax, + getattr(ds_default, "SamplesPerPixel", "?"), + getattr(ds_default, "PhotometricInterpretation", "?"), + ) + start = time.perf_counter() baseline_pixels = ds_default.pixel_array + baseline_elapsed = time.perf_counter() - start + _logger.info( + "Default decoder finished in %.4fs: %s", + baseline_elapsed, + _summarize_array(baseline_pixels), + ) except Exception as e: default_decoder_error_message = f"{e}" default_decoder_errored = True + _logger.warning("Default decoder failed for %s: %s", file_name, default_decoder_error_message) # Remove and cache the other default decoder plugins first _remove_default_plugins() @@ -185,10 +246,18 @@ def test_nvimgcodec_decoder_matches_default(path: str) -> None: register_as_decoder_plugin() try: ds_custom = dcmread(path) + start = time.perf_counter() nv_pixels = ds_custom.pixel_array + nv_elapsed = time.perf_counter() - start + _logger.info( + "nvimgcodec decoder finished in %.4fs: %s", + nv_elapsed, + _summarize_array(nv_pixels), + ) except Exception as e: nvimgcodec_decoder_error_message = f"{e}" nvimgcodec_decoder_errored = True + _logger.warning("nvimgcodec decoder failed for %s: %s", file_name, nvimgcodec_decoder_error_message) finally: unregister_as_decoder_plugin() _restore_default_plugins() @@ -210,10 +279,24 @@ def test_nvimgcodec_decoder_matches_default(path: str) -> None: assert baseline_pixels.dtype == nv_pixels.dtype, f"Dtype mismatch with transfer syntax {transfer_syntax}" try: np.testing.assert_allclose(baseline_pixels, nv_pixels, rtol=rtol, atol=atol) - _logger.info(f"Pixels values matched for transfer syntax: {transfer_syntax} in file: {Path(path).name}") + _logger.info( + "Pixels matched for transfer_syntax=%s file=%s rtol=%s atol=%s", + transfer_syntax, + file_name, + rtol, + atol, + ) except AssertionError as e: + diff = np.abs(baseline_pixels.astype(np.float64) - nv_pixels.astype(np.float64)) + _logger.error( + "Pixel mismatch for transfer_syntax=%s file=%s max_abs_diff=%.4g mean_abs_diff=%.4g", + transfer_syntax, + file_name, + float(diff.max()) if diff.size else 0.0, + float(diff.mean()) if diff.size else 0.0, + ) raise AssertionError( - f"Pixels values mismatch for transfer syntax: {transfer_syntax} in file: {Path(path).name}" + f"Pixels values mismatch for transfer syntax: {transfer_syntax} in file: {file_name}" ) from e @@ -350,7 +433,9 @@ def _restore_default_plugins(): if __name__ == "__main__": # Use pytest to test the functionality with pydicom embedded DICOM files of supported transfer syntaxes individually - # python -m pytest test_decoder_nvimgcodec.py + # python -m pytest tests/unit/test_decoder_nvimgcodec.py -vv --log-cli-level=DEBUG \ + # --log-cli-format="%(asctime)s [%(levelname)s] %(name)s: %(message)s" \ + # --log-cli-date-format="%H:%M:%S.%f" # # The following compares the performance of the nvimgcodec decoder against the default decoders # with DICOM files in pydicom embedded dataset or an optional custom folder