From fcdb792b3648a81684546d460dac06eb78451d99 Mon Sep 17 00:00:00 2001 From: Riley Dixon Date: Mon, 30 Mar 2026 16:30:43 -0600 Subject: [PATCH 01/28] Python: Generate low-level Cython code --- python/CMakeLists.txt | 64 +++++++++++ python/hipfile/__init__.py | 33 ++++++ python/hipfile/_chipfile.pxd | 191 ++++++++++++++++++++++++++++++++ python/hipfile/_hipfile.pyx | 204 +++++++++++++++++++++++++++++++++++ python/pyproject.toml | 14 +++ 5 files changed, 506 insertions(+) create mode 100644 python/CMakeLists.txt create mode 100644 python/hipfile/__init__.py create mode 100644 python/hipfile/_chipfile.pxd create mode 100644 python/hipfile/_hipfile.pyx create mode 100644 python/pyproject.toml diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt new file mode 100644 index 00000000..aa87d54f --- /dev/null +++ b/python/CMakeLists.txt @@ -0,0 +1,64 @@ +cmake_minimum_required(VERSION 3.21) +project(hipfile-python LANGUAGES C) + +# ---- Python & Cython ----------------------------------------------------- + +find_package(Python REQUIRED COMPONENTS Interpreter Development.Module) +find_program(CYTHON_EXECUTABLE cython REQUIRED) + +# ---- hipFile library & headers -------------------------------------------- + +# hipfile.h location (default: sibling include/ directory) +set(HIPFILE_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../include" + CACHE PATH "Path to directory containing hipfile.h") + +# libhipfile.so location +find_library(HIPFILE_LIBRARY hipfile + HINTS + "${CMAKE_CURRENT_SOURCE_DIR}/../build/src/amd_detail" + "/opt/rocm/lib" +) +if(NOT HIPFILE_LIBRARY) + message(FATAL_ERROR + "Could not find libhipfile. Set -DHIPFILE_LIBRARY= or " + "install hipfile to /opt/rocm.") +endif() + +# HIP runtime headers (needed because hipfile.h includes hip/hip_runtime_api.h) +find_path(HIP_INCLUDE_DIR hip/hip_runtime_api.h + HINTS + "/opt/rocm/include" + "/opt/rocm/hip/include" +) +if(NOT HIP_INCLUDE_DIR) + message(FATAL_ERROR + "Could not find hip/hip_runtime_api.h. Set -DHIP_INCLUDE_DIR= " + "or install HIP development headers.") +endif() + +# ---- Cythonize ------------------------------------------------------------ + +set(_PYX_SRC "${CMAKE_CURRENT_SOURCE_DIR}/hipfile/_hipfile.pyx") +set(_C_OUT "${CMAKE_CURRENT_BINARY_DIR}/_hipfile.c") + +add_custom_command( + OUTPUT "${_C_OUT}" + COMMAND "${CYTHON_EXECUTABLE}" "${_PYX_SRC}" -o "${_C_OUT}" -3 + DEPENDS + "${_PYX_SRC}" + "${CMAKE_CURRENT_SOURCE_DIR}/hipfile/_chipfile.pxd" + COMMENT "Cythonizing _hipfile.pyx" +) + +# ---- Build extension module ----------------------------------------------- + +Python_add_library(_hipfile MODULE "${_C_OUT}" WITH_SOABI) + +target_include_directories(_hipfile PRIVATE + "${HIPFILE_INCLUDE_DIR}" + "${HIP_INCLUDE_DIR}" +) + +target_link_libraries(_hipfile PRIVATE "${HIPFILE_LIBRARY}") + +install(TARGETS _hipfile DESTINATION hipfile) diff --git a/python/hipfile/__init__.py b/python/hipfile/__init__.py new file mode 100644 index 00000000..7fa58fdf --- /dev/null +++ b/python/hipfile/__init__.py @@ -0,0 +1,33 @@ +from hipfile._hipfile import * # noqa: F401,F403 +from hipfile._hipfile import ( + # Constants + VERSION_MAJOR, + VERSION_MINOR, + VERSION_PATCH, + BASE_ERR, + # Error helpers + is_hipfile_err, + hipfile_errstr, + is_hip_drv_err, + hip_drv_err, + get_op_error_string, + # Driver lifecycle + driver_open, + driver_close, + use_count, + # Version + get_version, + # File handles + handle_register, + handle_deregister, + # Buffer registration + buf_register, + buf_deregister, + # Synchronous I/O + read, + write, + # Driver properties + driver_get_properties, +) + +__version__ = f"{VERSION_MAJOR}.{VERSION_MINOR}.{VERSION_PATCH}" diff --git a/python/hipfile/_chipfile.pxd b/python/hipfile/_chipfile.pxd new file mode 100644 index 00000000..9803ef3d --- /dev/null +++ b/python/hipfile/_chipfile.pxd @@ -0,0 +1,191 @@ +# cython: language_level=3 +""" +C declarations for hipFile API (extern from hipfile.h). + +This .pxd file declares the subset of the hipFile C API that is +wrapped by the low-level Cython bindings. +""" + +from libc.stdint cimport int64_t, uint64_t +from libc.stddef cimport size_t +from posix.types cimport off_t + + +# --------------------------------------------------------------------------- +# HIP runtime stub — only hipError_t is needed +# --------------------------------------------------------------------------- + +cdef extern from "hip/hip_runtime_api.h": + ctypedef enum hipError_t: + hipSuccess = 0 + + +# --------------------------------------------------------------------------- +# hipFile public API +# --------------------------------------------------------------------------- + +cdef extern from "hipfile.h": + + # -- Version constants -------------------------------------------------- + + int HIPFILE_VERSION_MAJOR + int HIPFILE_VERSION_MINOR + int HIPFILE_VERSION_PATCH + int HIPFILE_BASE_ERR + + # -- Platform-independent types ----------------------------------------- + + ctypedef off_t hoff_t + + # -- Error handling ----------------------------------------------------- + + ctypedef enum hipFileOpError_t: + hipFileSuccess + hipFileDriverNotInitialized + hipFileDriverInvalidProps + hipFileDriverUnsupportedLimit + hipFileDriverVersionMismatch + hipFileDriverVersionReadError + hipFileDriverClosing + hipFilePlatformNotSupported + hipFileIONotSupported + hipFileDeviceNotSupported + hipFileDriverError + hipFileHipDriverError + hipFileHipPointerInvalid + hipFileHipMemoryTypeInvalid + hipFileHipPointerRangeError + hipFileHipContextMismatch + hipFileInvalidMappingSize + hipFileInvalidMappingRange + hipFileInvalidFileType + hipFileInvalidFileOpenFlag + hipFileDIONotSet + # 5021 intentionally unused + hipFileInvalidValue + hipFileMemoryAlreadyRegistered + hipFileMemoryNotRegistered + hipFilePermissionDenied + hipFileDriverAlreadyOpen + hipFileHandleNotRegistered + hipFileHandleAlreadyRegistered + hipFileDeviceNotFound + hipFileInternalError + hipFileGetNewFDFailed + # 5032 intentionally unused + hipFileDriverSetupError + hipFileIODisabled + hipFileBatchSubmitFailed + hipFileGPUMemoryPinningFailed + hipFileBatchFull + hipFileAsyncNotSupported + hipFileIOMaxError + + ctypedef struct hipFileError_t: + hipFileOpError_t err + hipError_t hip_drv_err + + const char *hipFileGetOpErrorString(hipFileOpError_t status) + + # -- Opaque handles ----------------------------------------------------- + + ctypedef void *hipFileHandle_t + + # -- File handle types -------------------------------------------------- + + ctypedef enum hipFileFileHandleType_t: + hipFileHandleTypeOpaqueFD + hipFileHandleTypeOpaqueWin32 + hipFileHandleTypeUserspaceFS + + # -- Userspace FS ops (opaque — only needed as pointer type) ------------ + + ctypedef struct hipFileFSOps_t: + pass + + # -- File descriptor ---------------------------------------------------- + # The anonymous union is accessed via Cython C-name strings. + + ctypedef struct hipFileDescr_t: + hipFileFileHandleType_t type + int fd "handle.fd" + void *hFile "handle.hFile" + const hipFileFSOps_t *fs_ops + + # -- Driver status / control / feature flag enums ----------------------- + + ctypedef enum hipFileDriverStatusFlags_t: + hipFileLustreSupported + hipFileWekaFSSupported + hipFileNFSSupported + hipFileGPFSSupported + hipFileNVMeSupported + hipFileNVMeoFSupported + hipFileSCSISupported + hipFileScaleFluxCSDSupported + hipFileNVMeshSupported + hipFileBeeGFSSupported + # 10 reserved for YRCloudFile + hipFileNVMeP2PSupported + hipFileScatefsSupported + + ctypedef enum hipFileDriverControlFlags_t: + hipFileUsePollMode + hipFileAllowCompatMode + + ctypedef enum hipFileFeatureFlags_t: + hipFileDynRoutingSupported + hipFileBatchIOSupported + hipFileStreamsSupported + hipFileParallelIOSupported + + # -- Driver properties -------------------------------------------------- + # Nested anonymous struct ``nvfs`` is flattened with C-name strings. + + ctypedef struct hipFileDriverProps_t: + unsigned int nvfs_major_version "nvfs.major_version" + unsigned int nvfs_minor_version "nvfs.minor_version" + uint64_t nvfs_poll_thresh_size "nvfs.poll_thresh_size" + uint64_t nvfs_max_direct_io_size "nvfs.max_direct_io_size" + unsigned int nvfs_driver_status_flags "nvfs.driver_status_flags" + unsigned int nvfs_driver_control_flags "nvfs.driver_control_flags" + unsigned int feature_flags + uint64_t max_device_cache_size + uint64_t per_buffer_cache_size + uint64_t max_device_pinned_mem_size + unsigned int max_batch_io_count + unsigned int max_batch_io_timeout_msecs + + # -- Function declarations ---------------------------------------------- + + # Error + const char *hipFileGetOpErrorString(hipFileOpError_t status) + + # File handles + hipFileError_t hipFileHandleRegister(hipFileHandle_t *fh, + hipFileDescr_t *descr) + void hipFileHandleDeregister(hipFileHandle_t fh) + + # Buffer registration + hipFileError_t hipFileBufRegister(const void *buffer_base, + size_t length, int flags) + hipFileError_t hipFileBufDeregister(const void *buffer_base) + + # Synchronous I/O + ssize_t hipFileRead(hipFileHandle_t fh, void *buffer_base, size_t size, + hoff_t file_offset, hoff_t buffer_offset) + ssize_t hipFileWrite(hipFileHandle_t fh, const void *buffer_base, + size_t size, hoff_t file_offset, + hoff_t buffer_offset) + + # Driver lifecycle + hipFileError_t hipFileDriverOpen() + hipFileError_t hipFileDriverClose() + int64_t hipFileUseCount() + + # Driver properties + hipFileError_t hipFileDriverGetProperties(hipFileDriverProps_t *props) + + # Version + hipFileError_t hipFileGetVersion(unsigned *major, unsigned *minor, + unsigned *patch) diff --git a/python/hipfile/_hipfile.pyx b/python/hipfile/_hipfile.pyx new file mode 100644 index 00000000..6658e654 --- /dev/null +++ b/python/hipfile/_hipfile.pyx @@ -0,0 +1,204 @@ +# cython: language_level=3 +""" +Low-level Cython wrappers for the hipFile C API. + +Every function mirrors the C API as closely as possible. +Functions that return ``hipFileError_t`` in C return a +``(hipFileOpError_t, hipError_t)`` 2-tuple here. +""" + +from libc.string cimport memset +from libc.stdint cimport uintptr_t + +from hipfile._chipfile cimport * + + +# --------------------------------------------------------------------------- +# Module-level constants +# --------------------------------------------------------------------------- + +VERSION_MAJOR = HIPFILE_VERSION_MAJOR +VERSION_MINOR = HIPFILE_VERSION_MINOR +VERSION_PATCH = HIPFILE_VERSION_PATCH +BASE_ERR = HIPFILE_BASE_ERR + + +# --------------------------------------------------------------------------- +# Internal helpers +# --------------------------------------------------------------------------- + +cdef inline tuple _err(hipFileError_t e): + return (e.err, e.hip_drv_err) + + +# --------------------------------------------------------------------------- +# Error-handling helpers (replacements for C macros) +# --------------------------------------------------------------------------- + +def is_hipfile_err(int err_code): + """Equivalent of the ``IS_HIPFILE_ERR`` C macro.""" + return abs(err_code) > HIPFILE_BASE_ERR + + +def hipfile_errstr(int err_code): + """Equivalent of the ``HIPFILE_ERRSTR`` C macro.""" + cdef const char *s = hipFileGetOpErrorString(abs(err_code)) + if s == NULL: + return "" + return s.decode("utf-8") + + +def is_hip_drv_err(tuple err): + """Equivalent of the ``IS_HIP_DRV_ERR`` C macro. + + Takes an error tuple as returned by the wrapper functions. + """ + return err[0] == hipFileHipDriverError + + +def hip_drv_err(tuple err): + """Equivalent of the ``HIP_DRV_ERR`` C macro. + + Takes an error tuple and returns the ``hipError_t`` component. + """ + return err[1] + + +def get_op_error_string(int status): + """Wrapper for ``hipFileGetOpErrorString``.""" + cdef const char *s = hipFileGetOpErrorString(status) + if s == NULL: + return "" + return s.decode("utf-8") + + +# --------------------------------------------------------------------------- +# Driver lifecycle +# --------------------------------------------------------------------------- + +def driver_open(): + """Wrapper for ``hipFileDriverOpen``.""" + return _err(hipFileDriverOpen()) + + +def driver_close(): + """Wrapper for ``hipFileDriverClose``.""" + return _err(hipFileDriverClose()) + + +def use_count(): + """Wrapper for ``hipFileUseCount``.""" + return hipFileUseCount() + + +# --------------------------------------------------------------------------- +# Version +# --------------------------------------------------------------------------- + +def get_version(): + """Wrapper for ``hipFileGetVersion``. + + Returns ``((major, minor, patch), error_tuple)``. + """ + cdef unsigned major = 0, minor = 0, patch = 0 + cdef hipFileError_t e = hipFileGetVersion(&major, &minor, &patch) + return ((major, minor, patch), _err(e)) + + +# --------------------------------------------------------------------------- +# File handles +# --------------------------------------------------------------------------- + +def handle_register(int fd): + """Wrapper for ``hipFileHandleRegister`` (POSIX fd path). + + Returns ``(handle_int, error_tuple)``. The handle is an opaque + integer that must be passed back to other hipFile calls. + """ + cdef hipFileHandle_t fh = NULL + cdef hipFileDescr_t descr + memset(&descr, 0, sizeof(descr)) + descr.type = hipFileHandleTypeOpaqueFD + descr.fd = fd + cdef hipFileError_t e = hipFileHandleRegister(&fh, &descr) + return (fh, _err(e)) + + +def handle_deregister(uintptr_t handle): + """Wrapper for ``hipFileHandleDeregister``.""" + hipFileHandleDeregister(handle) + + +# --------------------------------------------------------------------------- +# Buffer registration +# --------------------------------------------------------------------------- + +def buf_register(uintptr_t buffer_base, size_t length, int flags=0): + """Wrapper for ``hipFileBufRegister``.""" + return _err(hipFileBufRegister(buffer_base, length, flags)) + + +def buf_deregister(uintptr_t buffer_base): + """Wrapper for ``hipFileBufDeregister``.""" + return _err(hipFileBufDeregister(buffer_base)) + + +# --------------------------------------------------------------------------- +# Synchronous I/O +# --------------------------------------------------------------------------- + +def read(uintptr_t handle, uintptr_t buffer_base, size_t size, + hoff_t file_offset, hoff_t buffer_offset): + """Wrapper for ``hipFileRead``. + + Returns raw ``ssize_t``: + * ``>= 0`` — number of bytes read + * ``-1`` — system error (check ``errno``) + * ``< -1`` — negate to get a ``hipFileOpError_t`` + """ + return hipFileRead(handle, + buffer_base, size, + file_offset, buffer_offset) + + +def write(uintptr_t handle, uintptr_t buffer_base, size_t size, + hoff_t file_offset, hoff_t buffer_offset): + """Wrapper for ``hipFileWrite``. + + Returns raw ``ssize_t``: + * ``>= 0`` — number of bytes written + * ``-1`` — system error (check ``errno``) + * ``< -1`` — negate to get a ``hipFileOpError_t`` + """ + return hipFileWrite(handle, + buffer_base, size, + file_offset, buffer_offset) + + +# --------------------------------------------------------------------------- +# Driver properties +# --------------------------------------------------------------------------- + +def driver_get_properties(): + """Wrapper for ``hipFileDriverGetProperties``. + + Returns ``(props_dict, error_tuple)``. + """ + cdef hipFileDriverProps_t props + memset(&props, 0, sizeof(props)) + cdef hipFileError_t e = hipFileDriverGetProperties(&props) + d = { + "nvfs_major_version": props.nvfs_major_version, + "nvfs_minor_version": props.nvfs_minor_version, + "nvfs_poll_thresh_size": props.nvfs_poll_thresh_size, + "nvfs_max_direct_io_size": props.nvfs_max_direct_io_size, + "nvfs_driver_status_flags": props.nvfs_driver_status_flags, + "nvfs_driver_control_flags": props.nvfs_driver_control_flags, + "feature_flags": props.feature_flags, + "max_device_cache_size": props.max_device_cache_size, + "per_buffer_cache_size": props.per_buffer_cache_size, + "max_device_pinned_mem_size": props.max_device_pinned_mem_size, + "max_batch_io_count": props.max_batch_io_count, + "max_batch_io_timeout_msecs": props.max_batch_io_timeout_msecs, + } + return (d, _err(e)) diff --git a/python/pyproject.toml b/python/pyproject.toml new file mode 100644 index 00000000..9a334dd8 --- /dev/null +++ b/python/pyproject.toml @@ -0,0 +1,14 @@ +[build-system] +requires = ["scikit-build-core>=0.10", "cython>=3.0"] +build-backend = "scikit_build_core.build" + +[project] +name = "hipfile" +version = "0.2.0" +description = "Low-level Python bindings for hipFile (direct-to-GPU IO)" +requires-python = ">=3.10" +license = {text = "MIT"} + +[tool.scikit-build] +cmake.build-type = "Release" +wheel.packages = ["hipfile"] From 0f7545e8ec840a2a4b1ffcb5feed8e80afa2cab7 Mon Sep 17 00:00:00 2001 From: Riley Dixon Date: Mon, 30 Mar 2026 16:51:13 -0600 Subject: [PATCH 02/28] Ignore: Python venv's --- .gitignore | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitignore b/.gitignore index 5274fea0..ce60b65c 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,9 @@ build/ # Ignore .cache directory generated by clangd .cache/ + +# Ignore any Python Virtual Environments +.venv/ + +# Ignore any pycache files +__pycache__/ From f69ae0ad5cd0bbb2ed9b1e047982f248ac2e48b0 Mon Sep 17 00:00:00 2001 From: Riley Dixon Date: Tue, 31 Mar 2026 10:15:19 -0600 Subject: [PATCH 03/28] Python/CMake: Define __HIP_PLATFORM_AMD__ --- python/CMakeLists.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index aa87d54f..81ef8691 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -61,4 +61,7 @@ target_include_directories(_hipfile PRIVATE target_link_libraries(_hipfile PRIVATE "${HIPFILE_LIBRARY}") +# _hipfile bindings only support AMD for now +target_compile_definitions(_hipfile PRIVATE __HIP_PLATFORM_AMD__) + install(TARGETS _hipfile DESTINATION hipfile) From 4e0609223197ceb11b947f8f7797779ccabe0549 Mon Sep 17 00:00:00 2001 From: Riley Dixon Date: Tue, 31 Mar 2026 10:15:32 -0600 Subject: [PATCH 04/28] Python: Create Driver class Also move _hipfile driver functions out of public __init__.py --- python/hipfile/__init__.py | 7 +++---- python/hipfile/driver.py | 24 ++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 4 deletions(-) create mode 100644 python/hipfile/driver.py diff --git a/python/hipfile/__init__.py b/python/hipfile/__init__.py index 7fa58fdf..0ce81d9b 100644 --- a/python/hipfile/__init__.py +++ b/python/hipfile/__init__.py @@ -11,10 +11,6 @@ is_hip_drv_err, hip_drv_err, get_op_error_string, - # Driver lifecycle - driver_open, - driver_close, - use_count, # Version get_version, # File handles @@ -29,5 +25,8 @@ # Driver properties driver_get_properties, ) +from hipfile.driver import ( + Driver +) __version__ = f"{VERSION_MAJOR}.{VERSION_MINOR}.{VERSION_PATCH}" diff --git a/python/hipfile/driver.py b/python/hipfile/driver.py new file mode 100644 index 00000000..90a83e78 --- /dev/null +++ b/python/hipfile/driver.py @@ -0,0 +1,24 @@ +from hipfile._hipfile import ( + driver_open, + driver_close, + use_count as _use_count +) + +class Driver: + + @staticmethod + def use_count(): + return _use_count() + + def __enter__(self): + self.open() + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.close() + + def close(self): + driver_close() + + def open(self): + driver_open() From cbfb5874e0aaff57e87bb7db506454c37cb65248 Mon Sep 17 00:00:00 2001 From: Riley Dixon Date: Tue, 31 Mar 2026 11:22:51 -0600 Subject: [PATCH 05/28] Python: Create HipFileException class This exception will be thrown by the Python hipFile module rather than returning an error struct. The hipFile C error macros are unused since they aren't very Pythonic in the first place. If the error is a hipFile error (even if it is a wrapped HIP error), we raise HipFileException. If it is a different error, a different Exception will be raised. We also won't be exposing the error struct/tuple to Python users, though it can be accessed via the HipFileException when raised. --- python/hipfile/__init__.py | 6 ++++-- python/hipfile/error.py | 22 ++++++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) create mode 100644 python/hipfile/error.py diff --git a/python/hipfile/__init__.py b/python/hipfile/__init__.py index 0ce81d9b..60970f98 100644 --- a/python/hipfile/__init__.py +++ b/python/hipfile/__init__.py @@ -5,12 +5,11 @@ VERSION_MINOR, VERSION_PATCH, BASE_ERR, - # Error helpers + # Error Macros is_hipfile_err, hipfile_errstr, is_hip_drv_err, hip_drv_err, - get_op_error_string, # Version get_version, # File handles @@ -28,5 +27,8 @@ from hipfile.driver import ( Driver ) +from hipfile.error import ( + HipFileException +) __version__ = f"{VERSION_MAJOR}.{VERSION_MINOR}.{VERSION_PATCH}" diff --git a/python/hipfile/error.py b/python/hipfile/error.py new file mode 100644 index 00000000..efad2712 --- /dev/null +++ b/python/hipfile/error.py @@ -0,0 +1,22 @@ +from hipfile._hipfile import ( + get_op_error_string +) + +class HipFileException(Exception): + def __init__(self, hipfile_err, hip_err): + self._hipfile_err = hipfile_err + self._hip_err = hip_err + + @property + def hipfile_err(self): + return self._hipfile_err + + @property + def hip_err(self): + return self._hip_err + + def __str__(self): + err_msg = f"{self._hipfile_err} - {get_op_error_string(self._hipfile_err)}" + if (self._hipfile_err == 5011): + err_msg += f" {self._hip_err}" + return err_msg From 2bb2d30771a82a6b8fbfbb278ccea12659af673a Mon Sep 17 00:00:00 2001 From: Riley Dixon Date: Tue, 31 Mar 2026 12:54:51 -0600 Subject: [PATCH 06/28] Python: Create a properties module. This module will contain the functions related to querying driver/library configuration/properties in a Python friendly manner. --- python/hipfile/__init__.py | 10 +++++----- python/hipfile/properties.py | 19 +++++++++++++++++++ 2 files changed, 24 insertions(+), 5 deletions(-) create mode 100644 python/hipfile/properties.py diff --git a/python/hipfile/__init__.py b/python/hipfile/__init__.py index 60970f98..c5d92b63 100644 --- a/python/hipfile/__init__.py +++ b/python/hipfile/__init__.py @@ -10,8 +10,6 @@ hipfile_errstr, is_hip_drv_err, hip_drv_err, - # Version - get_version, # File handles handle_register, handle_deregister, @@ -20,9 +18,7 @@ buf_deregister, # Synchronous I/O read, - write, - # Driver properties - driver_get_properties, + write ) from hipfile.driver import ( Driver @@ -30,5 +26,9 @@ from hipfile.error import ( HipFileException ) +from hipfile.properties import ( + driver_get_properties, + get_version +) __version__ = f"{VERSION_MAJOR}.{VERSION_MINOR}.{VERSION_PATCH}" diff --git a/python/hipfile/properties.py b/python/hipfile/properties.py new file mode 100644 index 00000000..0a702547 --- /dev/null +++ b/python/hipfile/properties.py @@ -0,0 +1,19 @@ +from hipfile._hipfile import ( + # Driver properties + driver_get_properties as _driver_get_properties, + # Library version + get_version as _get_version +) +from hipfile.error import HipFileException + +def driver_get_properties(): + (_props, err) = _driver_get_properties() + if (err[0] != 0): + raise HipFileException(err[0], err[1]) + return _props + +def get_version(): + (version_tuple, err) = _get_version() + if (err[0] != 0): + raise HipFileException(err[0], err[1]) + return version_tuple From 013733add8b93e08d3f4106ef087320f485bff92 Mon Sep 17 00:00:00 2001 From: Riley Dixon Date: Tue, 31 Mar 2026 14:48:36 -0600 Subject: [PATCH 07/28] Python: Create a FileHandle class. A fairly rudimentary interface for opening/closing files with hipFile. --- python/hipfile/__init__.py | 6 ++-- python/hipfile/file.py | 66 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 3 deletions(-) create mode 100644 python/hipfile/file.py diff --git a/python/hipfile/__init__.py b/python/hipfile/__init__.py index c5d92b63..ddb17e1d 100644 --- a/python/hipfile/__init__.py +++ b/python/hipfile/__init__.py @@ -10,9 +10,6 @@ hipfile_errstr, is_hip_drv_err, hip_drv_err, - # File handles - handle_register, - handle_deregister, # Buffer registration buf_register, buf_deregister, @@ -26,6 +23,9 @@ from hipfile.error import ( HipFileException ) +from hipfile.file import ( + FileHandle +) from hipfile.properties import ( driver_get_properties, get_version diff --git a/python/hipfile/file.py b/python/hipfile/file.py new file mode 100644 index 00000000..2cdab928 --- /dev/null +++ b/python/hipfile/file.py @@ -0,0 +1,66 @@ +import os +import stat + +from hipfile._hipfile import ( + # File handles + handle_register, + handle_deregister +) +from hipfile.error import HipFileException + +class FileHandle(): + DEFAULT_MODE = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH + + def __init__(self, path, flags, mode = DEFAULT_MODE, io_type = None): + self._fd = None + self._flags = flags + self._handle = None + # io_type currently unsupported - C bindings only use hipFileHandleTypeOpaqueFD + self._io_type = io_type + self._mode = mode + self._path = path + + def __del__(self): + self.close() + + def __enter__(self): + self.open() + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.close() + + @property + def flags(self): + return self._flags + + @property + def handle(self): + return self._handle + + @property + def mode(self): + return self._mode + + @property + def path(self): + return self._path + + def open(self): + if (self._handle is not None): + raise RuntimeError("The FileHandle is already open.") + self._fd = os.open(self._path, self._flags, self._mode) + (handle, err) = handle_register(self._fd) + if (err[0] != 0): + os.close(self._fd) + self._fd = None + raise HipFileException(err[0], err[1]) + self._handle = handle + + def close(self): + if (self._handle is not None): + handle_deregister(self._handle) + self._handle = None + if (self._fd is not None): + os.close(self._fd) + self._fd = None From ac4f6cd850f4a070a42bcaa8663f19bd2e2eebb3 Mon Sep 17 00:00:00 2001 From: Riley Dixon Date: Tue, 31 Mar 2026 16:37:09 -0600 Subject: [PATCH 08/28] Python: Create a buffer module A simple wrapper. Still need to figure out how exactly a hipFile Buffer object should look like and how it is tied to the underlying device memory. --- python/hipfile/__init__.py | 7 ++++--- python/hipfile/buffer.py | 15 +++++++++++++++ 2 files changed, 19 insertions(+), 3 deletions(-) create mode 100644 python/hipfile/buffer.py diff --git a/python/hipfile/__init__.py b/python/hipfile/__init__.py index ddb17e1d..d09ac8bf 100644 --- a/python/hipfile/__init__.py +++ b/python/hipfile/__init__.py @@ -10,13 +10,14 @@ hipfile_errstr, is_hip_drv_err, hip_drv_err, - # Buffer registration - buf_register, - buf_deregister, # Synchronous I/O read, write ) +from hipfile.buffer import( + buf_deregister, + buf_register +) from hipfile.driver import ( Driver ) diff --git a/python/hipfile/buffer.py b/python/hipfile/buffer.py new file mode 100644 index 00000000..f6f501ea --- /dev/null +++ b/python/hipfile/buffer.py @@ -0,0 +1,15 @@ +from hipfile._hipfile import ( + buf_deregister as _buf_deregister, + buf_register as _buf_register +) +from hipfile.error import HipFileException + +def buf_register(device_buffer, length, flags): + err = _buf_register(device_buffer, length, flags) + if (err[0] != 0): + raise HipFileException(err[0], err[1]) + +def buf_deregister(device_buffer): + err = _buf_deregister(device_buffer) + if (err[0] != 0): + raise HipFileException(err[0], err[1]) From fbaf27cbc09cddf52d982e0b38da58b199421775 Mon Sep 17 00:00:00 2001 From: Riley Dixon Date: Tue, 31 Mar 2026 16:41:04 -0600 Subject: [PATCH 09/28] Python: Add error handling to Driver. --- python/hipfile/driver.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/python/hipfile/driver.py b/python/hipfile/driver.py index 90a83e78..9bfa365c 100644 --- a/python/hipfile/driver.py +++ b/python/hipfile/driver.py @@ -3,6 +3,7 @@ driver_close, use_count as _use_count ) +from hipfile.error import HipFileException class Driver: @@ -18,7 +19,11 @@ def __exit__(self, exc_type, exc_value, traceback): self.close() def close(self): - driver_close() + err = driver_close() + if (err[0] != 0): + raise HipFileException(err[0], err[1]) def open(self): - driver_open() + err = driver_open() + if (err[0] != 0): + raise HipFileException(err[0], err[1]) From 66e9988df268c9333195146cf3c9cefe695d9645 Mon Sep 17 00:00:00 2001 From: Riley Dixon Date: Wed, 1 Apr 2026 10:53:33 -0600 Subject: [PATCH 10/28] Python: Hack in hipMalloc & hipFree We need some GPU memory allocated for registered a buffer with hipFile. --- python/hipfile/hipMalloc.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 python/hipfile/hipMalloc.py diff --git a/python/hipfile/hipMalloc.py b/python/hipfile/hipMalloc.py new file mode 100644 index 00000000..2988caf8 --- /dev/null +++ b/python/hipfile/hipMalloc.py @@ -0,0 +1,37 @@ +""" +This is a hack to have some semblance of GPU memory management +without introducing a dependency at this early stage of +development. Do not rely upon anything in this module. +""" +import ctypes +import sys + +# Load the HIP runtime library +if sys.platform.startswith("linux"): + _hip_lib_name = "libamdhip64.so" +elif sys.platform == "win32": + _hip_lib_name = "amdhip64.dll" +else: + raise OSError("Unsupported platform for HIP runtime") + +_hip = ctypes.CDLL(_hip_lib_name) + +# hipError_t hipMalloc(void** ptr, size_t size); +_hip.hipMalloc.argtypes = [ctypes.POINTER(ctypes.c_void_p), ctypes.c_size_t] +_hip.hipMalloc.restype = ctypes.c_int + +# hipError_t hipFree(void* ptr); +_hip.hipFree.argtypes = [ctypes.c_void_p] +_hip.hipFree.restype = ctypes.c_int + +def hipMalloc(size_bytes: int) -> ctypes.c_void_p: + d_ptr = ctypes.c_void_p() + status = _hip.hipMalloc(ctypes.byref(d_ptr), ctypes.c_size_t(size_bytes)) + if (status != 0): + raise RuntimeError(f"hipMalloc failed ({status})") + return d_ptr + +def hipFree(ptr: ctypes.c_void_p) -> None: + status = _hip.hipFree(ptr) + if (status != 0): + raise RuntimeError(f"hipFree failed ({status})") From 2cc15e5c0247ddf6e78fe3f91ad70d1740fcc8b3 Mon Sep 17 00:00:00 2001 From: Riley Dixon Date: Thu, 2 Apr 2026 15:49:21 +0000 Subject: [PATCH 11/28] Python: Add read/write methods to FileHandle This isn't perfect, but it provides a starting point to at least running IO. --- python/hipfile/__init__.py | 3 --- python/hipfile/file.py | 29 ++++++++++++++++++++++++++++- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/python/hipfile/__init__.py b/python/hipfile/__init__.py index d09ac8bf..1d9da690 100644 --- a/python/hipfile/__init__.py +++ b/python/hipfile/__init__.py @@ -10,9 +10,6 @@ hipfile_errstr, is_hip_drv_err, hip_drv_err, - # Synchronous I/O - read, - write ) from hipfile.buffer import( buf_deregister, diff --git a/python/hipfile/file.py b/python/hipfile/file.py index 2cdab928..db25321f 100644 --- a/python/hipfile/file.py +++ b/python/hipfile/file.py @@ -4,7 +4,10 @@ from hipfile._hipfile import ( # File handles handle_register, - handle_deregister + handle_deregister, + # Synchronous I/O + read as _read, + write as _write ) from hipfile.error import HipFileException @@ -64,3 +67,27 @@ def close(self): if (self._fd is not None): os.close(self._fd) self._fd = None + + def read(self, buffer, size, file_offset, buffer_offset): + if (self._handle is None): + raise RuntimeError("The FileHandle is not open.") + bytes_read = _read(self._handle, buffer, size, file_offset, buffer_offset) + # Note: _read should raise an OSError if a system error occured. + if (bytes_read < -1): + # hipFile Error + # Issue: If bytes_read == -hipFileHipDriverError, how do we get the hipError_t? + # Probably like in C and calling hipPeekLastError + raise HipFileException(-bytes_read, 0) + return bytes_read + + def write(self, buffer, size, file_offset, buffer_offset): + if (self._handle is None): + raise RuntimeError("The FileHandle is not open.") + bytes_written = _write(self._handle, buffer, size, file_offset, buffer_offset) + # Note: _write should raise an OSError if a system error occured. + if (bytes_written < -1): + # hipFile Error + # Issue: If bytes_written == -hipFileHipDriverError, how do we get the hipError_t? + # Probably like in C and calling hipPeekLastError + raise HipFileException(-bytes_written, 0) + return bytes_written From 37fd2402641586689fd93be38380b5fd535011fe Mon Sep 17 00:00:00 2001 From: Riley Dixon Date: Thu, 2 Apr 2026 15:57:19 +0000 Subject: [PATCH 12/28] Python: Add a hacked up testing script. Not meant to be permanent. Just a quick and dirty script to verify running IO works. --- python/main.py | 56 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 python/main.py diff --git a/python/main.py b/python/main.py new file mode 100644 index 00000000..22ffd007 --- /dev/null +++ b/python/main.py @@ -0,0 +1,56 @@ +import hashlib +import os +import pathlib +import stat + +from hipfile.buffer import buf_deregister, buf_register +from hipfile.driver import Driver +from hipfile.file import FileHandle +from hipfile.properties import driver_get_properties, get_version +from hipfile.hipMalloc import hipFree, hipMalloc + +hipfile_version = get_version() + +input_path = pathlib.Path("/mnt/ais/ext4/random_2GiB.bin") +output_path = pathlib.Path("/mnt/ais/ext4/output.bin") + +print(f"hipFile Version: {hipfile_version}") +print(f"Driver Use Count Before: {Driver.use_count()}") + +# Max to a 2GiB Buffer +# Note: Max IO in a single transaction is 2GiB - 4KiB as set by the Linux Kernel +# Larger IOs will be quietly truncated. +size = min(input_path.stat().st_size, 2 * 1024 * 1024 * 1024) +buffer = hipMalloc(size) +buffer_ptr = buffer.value +print(f"Buffer located at: {buffer_ptr} | {hex(buffer_ptr)}") +buf_register(buffer.value, size, 0) + +with Driver() as hipfile_driver: + print(f"Driver Use Count After: {hipfile_driver.use_count()}") + with FileHandle(input_path, os.O_RDWR | os.O_DIRECT | os.O_CREAT) as fh_input: + with FileHandle(output_path, os.O_RDWR | os.O_DIRECT | os.O_CREAT | os.O_TRUNC) as fh_output: + print(f"Transferring {size} bytes...") + bytes_read = fh_input.read(buffer_ptr, size, 0, 0) + print(f"Bytes Read: {bytes_read}") + bytes_written = fh_output.write(buffer_ptr, size, 0, 0) + print(f"Bytes Written: {bytes_written}") + +buf_deregister(buffer_ptr) +hipFree(buffer) + +with open(input_path, 'br') as file_in: + hash_in = hashlib.sha256() + chunk = file_in.read(1 * 1024 * 1024) # 1 MiB + while (len(chunk) != 0): + hash_in.update(chunk) + chunk = file_in.read(1 * 1024 * 1024) # 1 MiB + print(f"Input File Hash: {hash_in.hexdigest()}") + +with open(output_path, 'br') as file_out: + hash_out = hashlib.sha256() + chunk = file_out.read(1 * 1024 * 1024) # 1 MiB + while (len(chunk) != 0): + hash_out.update(chunk) + chunk = file_out.read(1 * 1024 * 1024) # 1 MiB + print(f"Output File Hash: {hash_out.hexdigest()}") From 4984f6ce9c4cc77aea3976f3b6fb15181f6c9d05 Mon Sep 17 00:00:00 2001 From: Riley Dixon Date: Thu, 2 Apr 2026 16:34:31 +0000 Subject: [PATCH 13/28] Python: Add a README for building the bindings --- python/README.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 python/README.md diff --git a/python/README.md b/python/README.md new file mode 100644 index 00000000..1769f457 --- /dev/null +++ b/python/README.md @@ -0,0 +1,25 @@ +# hipFile Python Bindings + +> [!CAUTION] +> These bindings in particular are *experimental* and the API will change. + +## Building & Installing + +1. Setup a Python virtual environment. +```bash +$ python3 -m venv .venv +``` +2. Activate the Python virtual environment. +```bash +$ source .venv/bin/activate +``` +3. Build the C hipFile library. See [INSTALL.md](../INSTALL.md). +4. Build & Install the hipFile package. +```bash +(.venv) $ pip install -e python -Ccmake.define.HIPFILE_INCLUDE_DIR=../include -Ccmake.define.HIP_INCLUDE_DIR=/opt/rocm/include +``` + +This will install an editable version of hipFile in your virtual environment. +It is editable in the sense that any changes you make to the hipFile Python +source code will be immediately available for any tests/scripts that use the hipFile library. +Any changes to the Cython source code will require a rebuild step. \ No newline at end of file From bb7fa2a8ddaa66cb55d4d3cbeb7d28d877a01762 Mon Sep 17 00:00:00 2001 From: Riley Dixon Date: Mon, 6 Apr 2026 16:42:22 -0600 Subject: [PATCH 14/28] Python: Turn buffer functions into a class --- python/hipfile/__init__.py | 3 +-- python/hipfile/buffer.py | 41 ++++++++++++++++++++++++++++---------- python/main.py | 27 ++++++++++++------------- 3 files changed, 45 insertions(+), 26 deletions(-) diff --git a/python/hipfile/__init__.py b/python/hipfile/__init__.py index 1d9da690..8c7d2c2a 100644 --- a/python/hipfile/__init__.py +++ b/python/hipfile/__init__.py @@ -12,8 +12,7 @@ hip_drv_err, ) from hipfile.buffer import( - buf_deregister, - buf_register + Buffer ) from hipfile.driver import ( Driver diff --git a/python/hipfile/buffer.py b/python/hipfile/buffer.py index f6f501ea..ae8ef954 100644 --- a/python/hipfile/buffer.py +++ b/python/hipfile/buffer.py @@ -1,15 +1,36 @@ from hipfile._hipfile import ( - buf_deregister as _buf_deregister, - buf_register as _buf_register + buf_deregister, + buf_register ) from hipfile.error import HipFileException -def buf_register(device_buffer, length, flags): - err = _buf_register(device_buffer, length, flags) - if (err[0] != 0): - raise HipFileException(err[0], err[1]) +class Buffer(): + def __init__(self, buffer_ptr, length, flags) -> None: + self._buffer_ptr = buffer_ptr + self._flags = flags + self._length = length + self._registered = False -def buf_deregister(device_buffer): - err = _buf_deregister(device_buffer) - if (err[0] != 0): - raise HipFileException(err[0], err[1]) + def __del__(self): + # We did not create the underlying buffer. Don't try to free it. + self.deregister() + + def __enter__(self): + self.register() + return self + + def __exit__(self, exc_type, exc, tb): + self.deregister() + + def deregister(self): + if (self._registered): + err = buf_deregister(self._buffer_ptr) + if (err[0] != 0): + raise HipFileException(err[0], err[1]) + self._registered = False + + def register(self): + err = buf_register(self._buffer_ptr, self._length, self._flags) + if (err[0] != 0): + raise HipFileException(err[0], err[1]) + self._registered = True diff --git a/python/main.py b/python/main.py index 22ffd007..35e97466 100644 --- a/python/main.py +++ b/python/main.py @@ -3,7 +3,7 @@ import pathlib import stat -from hipfile.buffer import buf_deregister, buf_register +from hipfile.buffer import Buffer from hipfile.driver import Driver from hipfile.file import FileHandle from hipfile.properties import driver_get_properties, get_version @@ -11,32 +11,31 @@ hipfile_version = get_version() -input_path = pathlib.Path("/mnt/ais/ext4/random_2GiB.bin") +input_path = pathlib.Path("/mnt/ais/ext4/random_2MiB.bin") output_path = pathlib.Path("/mnt/ais/ext4/output.bin") print(f"hipFile Version: {hipfile_version}") print(f"Driver Use Count Before: {Driver.use_count()}") -# Max to a 2GiB Buffer +# Max to a 2GiB - 4KiB Buffer # Note: Max IO in a single transaction is 2GiB - 4KiB as set by the Linux Kernel # Larger IOs will be quietly truncated. -size = min(input_path.stat().st_size, 2 * 1024 * 1024 * 1024) +size = min(input_path.stat().st_size, 2 * 1024 * 1024 * 1024 - 4 * 1024) buffer = hipMalloc(size) buffer_ptr = buffer.value print(f"Buffer located at: {buffer_ptr} | {hex(buffer_ptr)}") -buf_register(buffer.value, size, 0) with Driver() as hipfile_driver: print(f"Driver Use Count After: {hipfile_driver.use_count()}") - with FileHandle(input_path, os.O_RDWR | os.O_DIRECT | os.O_CREAT) as fh_input: - with FileHandle(output_path, os.O_RDWR | os.O_DIRECT | os.O_CREAT | os.O_TRUNC) as fh_output: - print(f"Transferring {size} bytes...") - bytes_read = fh_input.read(buffer_ptr, size, 0, 0) - print(f"Bytes Read: {bytes_read}") - bytes_written = fh_output.write(buffer_ptr, size, 0, 0) - print(f"Bytes Written: {bytes_written}") - -buf_deregister(buffer_ptr) + with Buffer(buffer_ptr, size, 0) as registered_buffer: + with FileHandle(input_path, os.O_RDWR | os.O_DIRECT | os.O_CREAT) as fh_input: + with FileHandle(output_path, os.O_RDWR | os.O_DIRECT | os.O_CREAT | os.O_TRUNC) as fh_output: + print(f"Transferring {size} bytes...") + bytes_read = fh_input.read(registered_buffer._buffer_ptr, size, 0, 0) + print(f"Bytes Read: {bytes_read}") + bytes_written = fh_output.write(registered_buffer._buffer_ptr, size, 0, 0) + print(f"Bytes Written: {bytes_written}") + hipFree(buffer) with open(input_path, 'br') as file_in: From 6df21adbf1d772755db48b86da42fa15dc6e3fe2 Mon Sep 17 00:00:00 2001 From: Riley Dixon Date: Tue, 7 Apr 2026 10:01:45 -0600 Subject: [PATCH 15/28] Python: Change read()/write() to accept Buffer --- python/hipfile/buffer.py | 4 ++++ python/hipfile/file.py | 4 ++-- python/main.py | 4 ++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/python/hipfile/buffer.py b/python/hipfile/buffer.py index ae8ef954..46840608 100644 --- a/python/hipfile/buffer.py +++ b/python/hipfile/buffer.py @@ -22,6 +22,10 @@ def __enter__(self): def __exit__(self, exc_type, exc, tb): self.deregister() + @property + def ptr(self): + return self._buffer_ptr + def deregister(self): if (self._registered): err = buf_deregister(self._buffer_ptr) diff --git a/python/hipfile/file.py b/python/hipfile/file.py index db25321f..47e10ad9 100644 --- a/python/hipfile/file.py +++ b/python/hipfile/file.py @@ -71,7 +71,7 @@ def close(self): def read(self, buffer, size, file_offset, buffer_offset): if (self._handle is None): raise RuntimeError("The FileHandle is not open.") - bytes_read = _read(self._handle, buffer, size, file_offset, buffer_offset) + bytes_read = _read(self._handle, buffer.ptr, size, file_offset, buffer_offset) # Note: _read should raise an OSError if a system error occured. if (bytes_read < -1): # hipFile Error @@ -83,7 +83,7 @@ def read(self, buffer, size, file_offset, buffer_offset): def write(self, buffer, size, file_offset, buffer_offset): if (self._handle is None): raise RuntimeError("The FileHandle is not open.") - bytes_written = _write(self._handle, buffer, size, file_offset, buffer_offset) + bytes_written = _write(self._handle, buffer.ptr, size, file_offset, buffer_offset) # Note: _write should raise an OSError if a system error occured. if (bytes_written < -1): # hipFile Error diff --git a/python/main.py b/python/main.py index 35e97466..d1620d54 100644 --- a/python/main.py +++ b/python/main.py @@ -31,9 +31,9 @@ with FileHandle(input_path, os.O_RDWR | os.O_DIRECT | os.O_CREAT) as fh_input: with FileHandle(output_path, os.O_RDWR | os.O_DIRECT | os.O_CREAT | os.O_TRUNC) as fh_output: print(f"Transferring {size} bytes...") - bytes_read = fh_input.read(registered_buffer._buffer_ptr, size, 0, 0) + bytes_read = fh_input.read(registered_buffer, size, 0, 0) print(f"Bytes Read: {bytes_read}") - bytes_written = fh_output.write(registered_buffer._buffer_ptr, size, 0, 0) + bytes_written = fh_output.write(registered_buffer, size, 0, 0) print(f"Bytes Written: {bytes_written}") hipFree(buffer) From 84b181868fe574b1fe95384cfb6189daca547d61 Mon Sep 17 00:00:00 2001 From: Riley Dixon Date: Tue, 7 Apr 2026 11:16:17 -0600 Subject: [PATCH 16/28] Python: Add alternate ctor for Buffer using ctypes.c_void_p Also adds limited typing hint for this method only. This import is only run if TYPE_CHECKING is defined. At runtime, if the ctypes package is not found, no error is raised since __future__.annotations is imported which treats the hint as a string literal and does not try to evaluate the name. --- python/hipfile/buffer.py | 11 +++++++++++ python/main.py | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/python/hipfile/buffer.py b/python/hipfile/buffer.py index 46840608..0cde0214 100644 --- a/python/hipfile/buffer.py +++ b/python/hipfile/buffer.py @@ -1,10 +1,21 @@ +from __future__ import annotations +from typing import TYPE_CHECKING + from hipfile._hipfile import ( buf_deregister, buf_register ) from hipfile.error import HipFileException +if TYPE_CHECKING: + from ctypes import c_void_p + class Buffer(): + + @classmethod + def from_ctypes_void_p(cls, ctypes_void_p: c_void_p, length, flags): + return cls(ctypes_void_p.value, length, flags) + def __init__(self, buffer_ptr, length, flags) -> None: self._buffer_ptr = buffer_ptr self._flags = flags diff --git a/python/main.py b/python/main.py index d1620d54..9cbc5667 100644 --- a/python/main.py +++ b/python/main.py @@ -27,7 +27,7 @@ with Driver() as hipfile_driver: print(f"Driver Use Count After: {hipfile_driver.use_count()}") - with Buffer(buffer_ptr, size, 0) as registered_buffer: + with Buffer.from_ctypes_void_p(buffer, size, 0) as registered_buffer: with FileHandle(input_path, os.O_RDWR | os.O_DIRECT | os.O_CREAT) as fh_input: with FileHandle(output_path, os.O_RDWR | os.O_DIRECT | os.O_CREAT | os.O_TRUNC) as fh_output: print(f"Transferring {size} bytes...") From 02b3915a50829a16cb3ea138b4ac9aaf7b46f8bf Mon Sep 17 00:00:00 2001 From: Riley Dixon Date: Tue, 7 Apr 2026 14:06:53 -0600 Subject: [PATCH 17/28] Python: Expose 2 hipFile C enums to Python hipFileOpError and hipFileFileHandleType are exposed to the Python layer to provide the user direct access to these enum values. --- python/hipfile/__init__.py | 4 ++ python/hipfile/_hipfile.pyx | 13 ++++- python/hipfile/enums.py | 106 ++++++++++++++++++++++++++++++++++++ 3 files changed, 120 insertions(+), 3 deletions(-) create mode 100644 python/hipfile/enums.py diff --git a/python/hipfile/__init__.py b/python/hipfile/__init__.py index 8c7d2c2a..17240728 100644 --- a/python/hipfile/__init__.py +++ b/python/hipfile/__init__.py @@ -17,6 +17,10 @@ from hipfile.driver import ( Driver ) +from hipfile.enums import ( + FileHandleType, + OpError +) from hipfile.error import ( HipFileException ) diff --git a/python/hipfile/_hipfile.pyx b/python/hipfile/_hipfile.pyx index 6658e654..fca36dd5 100644 --- a/python/hipfile/_hipfile.pyx +++ b/python/hipfile/_hipfile.pyx @@ -109,8 +109,15 @@ def get_version(): # File handles # --------------------------------------------------------------------------- -def handle_register(int fd): - """Wrapper for ``hipFileHandleRegister`` (POSIX fd path). +def handle_register(int fd, int handle_type): + """Wrapper for ``hipFileHandleRegister``. + + Parameters + ---------- + fd : int + POSIX file descriptor. + handle_type : int + Value from ``hipFileFileHandleType_t``. Returns ``(handle_int, error_tuple)``. The handle is an opaque integer that must be passed back to other hipFile calls. @@ -118,7 +125,7 @@ def handle_register(int fd): cdef hipFileHandle_t fh = NULL cdef hipFileDescr_t descr memset(&descr, 0, sizeof(descr)) - descr.type = hipFileHandleTypeOpaqueFD + descr.type = handle_type descr.fd = fd cdef hipFileError_t e = hipFileHandleRegister(&fh, &descr) return (fh, _err(e)) diff --git a/python/hipfile/enums.py b/python/hipfile/enums.py new file mode 100644 index 00000000..208e12cd --- /dev/null +++ b/python/hipfile/enums.py @@ -0,0 +1,106 @@ +from enum import IntEnum + +from hipfile._hipfile import ( + # hipFileOpError_t values (resolved from C at build time) + hipFileSuccess, + hipFileDriverNotInitialized, + hipFileDriverInvalidProps, + hipFileDriverUnsupportedLimit, + hipFileDriverVersionMismatch, + hipFileDriverVersionReadError, + hipFileDriverClosing, + hipFilePlatformNotSupported, + hipFileIONotSupported, + hipFileDeviceNotSupported, + hipFileDriverError, + hipFileHipDriverError, + hipFileHipPointerInvalid, + hipFileHipMemoryTypeInvalid, + hipFileHipPointerRangeError, + hipFileHipContextMismatch, + hipFileInvalidMappingSize, + hipFileInvalidMappingRange, + hipFileInvalidFileType, + hipFileInvalidFileOpenFlag, + hipFileDIONotSet, + hipFileInvalidValue, + hipFileMemoryAlreadyRegistered, + hipFileMemoryNotRegistered, + hipFilePermissionDenied, + hipFileDriverAlreadyOpen, + hipFileHandleNotRegistered, + hipFileHandleAlreadyRegistered, + hipFileDeviceNotFound, + hipFileInternalError, + hipFileGetNewFDFailed, + hipFileDriverSetupError, + hipFileIODisabled, + hipFileBatchSubmitFailed, + hipFileGPUMemoryPinningFailed, + hipFileBatchFull, + hipFileAsyncNotSupported, + hipFileIOMaxError, + # hipFileFileHandleType_t values (resolved from C at build time) + hipFileHandleTypeOpaqueFD, + hipFileHandleTypeOpaqueWin32, + hipFileHandleTypeUserspaceFS, +) + + +class OpError(IntEnum): + """Python enum mirroring hipFileOpError_t. + + Values are sourced from the C enum via the Cython layer, not + redefined. Rebuilding the extension picks up any value changes + in hipfile.h automatically. + """ + + Success = hipFileSuccess + DriverNotInitialized = hipFileDriverNotInitialized + DriverInvalidProps = hipFileDriverInvalidProps + DriverUnsupportedLimit = hipFileDriverUnsupportedLimit + DriverVersionMismatch = hipFileDriverVersionMismatch + DriverVersionReadError = hipFileDriverVersionReadError + DriverClosing = hipFileDriverClosing + PlatformNotSupported = hipFilePlatformNotSupported + IONotSupported = hipFileIONotSupported + DeviceNotSupported = hipFileDeviceNotSupported + DriverError = hipFileDriverError + HipDriverError = hipFileHipDriverError + HipPointerInvalid = hipFileHipPointerInvalid + HipMemoryTypeInvalid = hipFileHipMemoryTypeInvalid + HipPointerRangeError = hipFileHipPointerRangeError + HipContextMismatch = hipFileHipContextMismatch + InvalidMappingSize = hipFileInvalidMappingSize + InvalidMappingRange = hipFileInvalidMappingRange + InvalidFileType = hipFileInvalidFileType + InvalidFileOpenFlag = hipFileInvalidFileOpenFlag + DIONotSet = hipFileDIONotSet + InvalidValue = hipFileInvalidValue + MemoryAlreadyRegistered = hipFileMemoryAlreadyRegistered + MemoryNotRegistered = hipFileMemoryNotRegistered + PermissionDenied = hipFilePermissionDenied + DriverAlreadyOpen = hipFileDriverAlreadyOpen + HandleNotRegistered = hipFileHandleNotRegistered + HandleAlreadyRegistered = hipFileHandleAlreadyRegistered + DeviceNotFound = hipFileDeviceNotFound + InternalError = hipFileInternalError + GetNewFDFailed = hipFileGetNewFDFailed + DriverSetupError = hipFileDriverSetupError + IODisabled = hipFileIODisabled + BatchSubmitFailed = hipFileBatchSubmitFailed + GPUMemoryPinningFailed = hipFileGPUMemoryPinningFailed + BatchFull = hipFileBatchFull + AsyncNotSupported = hipFileAsyncNotSupported + IOMaxError = hipFileIOMaxError + + +class FileHandleType(IntEnum): + """Python enum mirroring hipFileFileHandleType_t. + + Values are sourced from the C enum via the Cython layer. + """ + + OpaqueFD = hipFileHandleTypeOpaqueFD + OpaqueWin32 = hipFileHandleTypeOpaqueWin32 + UserspaceFS = hipFileHandleTypeUserspaceFS From 79589b0f3aa21f462eca620b0c071c2a8151ac6f Mon Sep 17 00:00:00 2001 From: Riley Dixon Date: Tue, 7 Apr 2026 15:17:01 -0600 Subject: [PATCH 18/28] Python: Temp fix C->Python enum namespace collisions A proper fix will come next where the C enum values are namespace qualified so that enum values exposed to Python do not need to have their names modified. --- python/hipfile/_hipfile.pyx | 53 ++++++++++++ python/hipfile/enums.py | 164 ++++++++++++++++++------------------ 2 files changed, 135 insertions(+), 82 deletions(-) diff --git a/python/hipfile/_hipfile.pyx b/python/hipfile/_hipfile.pyx index fca36dd5..e0cd8d4c 100644 --- a/python/hipfile/_hipfile.pyx +++ b/python/hipfile/_hipfile.pyx @@ -22,6 +22,59 @@ VERSION_MINOR = HIPFILE_VERSION_MINOR VERSION_PATCH = HIPFILE_VERSION_PATCH BASE_ERR = HIPFILE_BASE_ERR +# --------------------------------------------------------------------------- +# Enum re-exports (C → Python) +# +# ctypedef enum values from _chipfile.pxd are C-level only after cimport. +# These assignments create Python-visible module attributes whose values +# are resolved from the C enum at compile time. +# --------------------------------------------------------------------------- + +# hipFileOpError_t +py_hipFileSuccess = hipFileSuccess +py_hipFileDriverNotInitialized = hipFileDriverNotInitialized +py_hipFileDriverInvalidProps = hipFileDriverInvalidProps +py_hipFileDriverUnsupportedLimit = hipFileDriverUnsupportedLimit +py_hipFileDriverVersionMismatch = hipFileDriverVersionMismatch +py_hipFileDriverVersionReadError = hipFileDriverVersionReadError +py_hipFileDriverClosing = hipFileDriverClosing +py_hipFilePlatformNotSupported = hipFilePlatformNotSupported +py_hipFileIONotSupported = hipFileIONotSupported +py_hipFileDeviceNotSupported = hipFileDeviceNotSupported +py_hipFileDriverError = hipFileDriverError +py_hipFileHipDriverError = hipFileHipDriverError +py_hipFileHipPointerInvalid = hipFileHipPointerInvalid +py_hipFileHipMemoryTypeInvalid = hipFileHipMemoryTypeInvalid +py_hipFileHipPointerRangeError = hipFileHipPointerRangeError +py_hipFileHipContextMismatch = hipFileHipContextMismatch +py_hipFileInvalidMappingSize = hipFileInvalidMappingSize +py_hipFileInvalidMappingRange = hipFileInvalidMappingRange +py_hipFileInvalidFileType = hipFileInvalidFileType +py_hipFileInvalidFileOpenFlag = hipFileInvalidFileOpenFlag +py_hipFileDIONotSet = hipFileDIONotSet +py_hipFileInvalidValue = hipFileInvalidValue +py_hipFileMemoryAlreadyRegistered = hipFileMemoryAlreadyRegistered +py_hipFileMemoryNotRegistered = hipFileMemoryNotRegistered +py_hipFilePermissionDenied = hipFilePermissionDenied +py_hipFileDriverAlreadyOpen = hipFileDriverAlreadyOpen +py_hipFileHandleNotRegistered = hipFileHandleNotRegistered +py_hipFileHandleAlreadyRegistered = hipFileHandleAlreadyRegistered +py_hipFileDeviceNotFound = hipFileDeviceNotFound +py_hipFileInternalError = hipFileInternalError +py_hipFileGetNewFDFailed = hipFileGetNewFDFailed +py_hipFileDriverSetupError = hipFileDriverSetupError +py_hipFileIODisabled = hipFileIODisabled +py_hipFileBatchSubmitFailed = hipFileBatchSubmitFailed +py_hipFileGPUMemoryPinningFailed = hipFileGPUMemoryPinningFailed +py_hipFileBatchFull = hipFileBatchFull +py_hipFileAsyncNotSupported = hipFileAsyncNotSupported +py_hipFileIOMaxError = hipFileIOMaxError + +# hipFileFileHandleType_t +py_hipFileHandleTypeOpaqueFD = hipFileHandleTypeOpaqueFD +py_hipFileHandleTypeOpaqueWin32 = hipFileHandleTypeOpaqueWin32 +py_hipFileHandleTypeUserspaceFS = hipFileHandleTypeUserspaceFS + # --------------------------------------------------------------------------- # Internal helpers diff --git a/python/hipfile/enums.py b/python/hipfile/enums.py index 208e12cd..3ad36eeb 100644 --- a/python/hipfile/enums.py +++ b/python/hipfile/enums.py @@ -2,48 +2,48 @@ from hipfile._hipfile import ( # hipFileOpError_t values (resolved from C at build time) - hipFileSuccess, - hipFileDriverNotInitialized, - hipFileDriverInvalidProps, - hipFileDriverUnsupportedLimit, - hipFileDriverVersionMismatch, - hipFileDriverVersionReadError, - hipFileDriverClosing, - hipFilePlatformNotSupported, - hipFileIONotSupported, - hipFileDeviceNotSupported, - hipFileDriverError, - hipFileHipDriverError, - hipFileHipPointerInvalid, - hipFileHipMemoryTypeInvalid, - hipFileHipPointerRangeError, - hipFileHipContextMismatch, - hipFileInvalidMappingSize, - hipFileInvalidMappingRange, - hipFileInvalidFileType, - hipFileInvalidFileOpenFlag, - hipFileDIONotSet, - hipFileInvalidValue, - hipFileMemoryAlreadyRegistered, - hipFileMemoryNotRegistered, - hipFilePermissionDenied, - hipFileDriverAlreadyOpen, - hipFileHandleNotRegistered, - hipFileHandleAlreadyRegistered, - hipFileDeviceNotFound, - hipFileInternalError, - hipFileGetNewFDFailed, - hipFileDriverSetupError, - hipFileIODisabled, - hipFileBatchSubmitFailed, - hipFileGPUMemoryPinningFailed, - hipFileBatchFull, - hipFileAsyncNotSupported, - hipFileIOMaxError, + py_hipFileSuccess, + py_hipFileDriverNotInitialized, + py_hipFileDriverInvalidProps, + py_hipFileDriverUnsupportedLimit, + py_hipFileDriverVersionMismatch, + py_hipFileDriverVersionReadError, + py_hipFileDriverClosing, + py_hipFilePlatformNotSupported, + py_hipFileIONotSupported, + py_hipFileDeviceNotSupported, + py_hipFileDriverError, + py_hipFileHipDriverError, + py_hipFileHipPointerInvalid, + py_hipFileHipMemoryTypeInvalid, + py_hipFileHipPointerRangeError, + py_hipFileHipContextMismatch, + py_hipFileInvalidMappingSize, + py_hipFileInvalidMappingRange, + py_hipFileInvalidFileType, + py_hipFileInvalidFileOpenFlag, + py_hipFileDIONotSet, + py_hipFileInvalidValue, + py_hipFileMemoryAlreadyRegistered, + py_hipFileMemoryNotRegistered, + py_hipFilePermissionDenied, + py_hipFileDriverAlreadyOpen, + py_hipFileHandleNotRegistered, + py_hipFileHandleAlreadyRegistered, + py_hipFileDeviceNotFound, + py_hipFileInternalError, + py_hipFileGetNewFDFailed, + py_hipFileDriverSetupError, + py_hipFileIODisabled, + py_hipFileBatchSubmitFailed, + py_hipFileGPUMemoryPinningFailed, + py_hipFileBatchFull, + py_hipFileAsyncNotSupported, + py_hipFileIOMaxError, # hipFileFileHandleType_t values (resolved from C at build time) - hipFileHandleTypeOpaqueFD, - hipFileHandleTypeOpaqueWin32, - hipFileHandleTypeUserspaceFS, + py_hipFileHandleTypeOpaqueFD, + py_hipFileHandleTypeOpaqueWin32, + py_hipFileHandleTypeUserspaceFS, ) @@ -55,44 +55,44 @@ class OpError(IntEnum): in hipfile.h automatically. """ - Success = hipFileSuccess - DriverNotInitialized = hipFileDriverNotInitialized - DriverInvalidProps = hipFileDriverInvalidProps - DriverUnsupportedLimit = hipFileDriverUnsupportedLimit - DriverVersionMismatch = hipFileDriverVersionMismatch - DriverVersionReadError = hipFileDriverVersionReadError - DriverClosing = hipFileDriverClosing - PlatformNotSupported = hipFilePlatformNotSupported - IONotSupported = hipFileIONotSupported - DeviceNotSupported = hipFileDeviceNotSupported - DriverError = hipFileDriverError - HipDriverError = hipFileHipDriverError - HipPointerInvalid = hipFileHipPointerInvalid - HipMemoryTypeInvalid = hipFileHipMemoryTypeInvalid - HipPointerRangeError = hipFileHipPointerRangeError - HipContextMismatch = hipFileHipContextMismatch - InvalidMappingSize = hipFileInvalidMappingSize - InvalidMappingRange = hipFileInvalidMappingRange - InvalidFileType = hipFileInvalidFileType - InvalidFileOpenFlag = hipFileInvalidFileOpenFlag - DIONotSet = hipFileDIONotSet - InvalidValue = hipFileInvalidValue - MemoryAlreadyRegistered = hipFileMemoryAlreadyRegistered - MemoryNotRegistered = hipFileMemoryNotRegistered - PermissionDenied = hipFilePermissionDenied - DriverAlreadyOpen = hipFileDriverAlreadyOpen - HandleNotRegistered = hipFileHandleNotRegistered - HandleAlreadyRegistered = hipFileHandleAlreadyRegistered - DeviceNotFound = hipFileDeviceNotFound - InternalError = hipFileInternalError - GetNewFDFailed = hipFileGetNewFDFailed - DriverSetupError = hipFileDriverSetupError - IODisabled = hipFileIODisabled - BatchSubmitFailed = hipFileBatchSubmitFailed - GPUMemoryPinningFailed = hipFileGPUMemoryPinningFailed - BatchFull = hipFileBatchFull - AsyncNotSupported = hipFileAsyncNotSupported - IOMaxError = hipFileIOMaxError + Success = py_hipFileSuccess + DriverNotInitialized = py_hipFileDriverNotInitialized + DriverInvalidProps = py_hipFileDriverInvalidProps + DriverUnsupportedLimit = py_hipFileDriverUnsupportedLimit + DriverVersionMismatch = py_hipFileDriverVersionMismatch + DriverVersionReadError = py_hipFileDriverVersionReadError + DriverClosing = py_hipFileDriverClosing + PlatformNotSupported = py_hipFilePlatformNotSupported + IONotSupported = py_hipFileIONotSupported + DeviceNotSupported = py_hipFileDeviceNotSupported + DriverError = py_hipFileDriverError + HipDriverError = py_hipFileHipDriverError + HipPointerInvalid = py_hipFileHipPointerInvalid + HipMemoryTypeInvalid = py_hipFileHipMemoryTypeInvalid + HipPointerRangeError = py_hipFileHipPointerRangeError + HipContextMismatch = py_hipFileHipContextMismatch + InvalidMappingSize = py_hipFileInvalidMappingSize + InvalidMappingRange = py_hipFileInvalidMappingRange + InvalidFileType = py_hipFileInvalidFileType + InvalidFileOpenFlag = py_hipFileInvalidFileOpenFlag + DIONotSet = py_hipFileDIONotSet + InvalidValue = py_hipFileInvalidValue + MemoryAlreadyRegistered = py_hipFileMemoryAlreadyRegistered + MemoryNotRegistered = py_hipFileMemoryNotRegistered + PermissionDenied = py_hipFilePermissionDenied + DriverAlreadyOpen = py_hipFileDriverAlreadyOpen + HandleNotRegistered = py_hipFileHandleNotRegistered + HandleAlreadyRegistered = py_hipFileHandleAlreadyRegistered + DeviceNotFound = py_hipFileDeviceNotFound + InternalError = py_hipFileInternalError + GetNewFDFailed = py_hipFileGetNewFDFailed + DriverSetupError = py_hipFileDriverSetupError + IODisabled = py_hipFileIODisabled + BatchSubmitFailed = py_hipFileBatchSubmitFailed + GPUMemoryPinningFailed = py_hipFileGPUMemoryPinningFailed + BatchFull = py_hipFileBatchFull + AsyncNotSupported = py_hipFileAsyncNotSupported + IOMaxError = py_hipFileIOMaxError class FileHandleType(IntEnum): @@ -101,6 +101,6 @@ class FileHandleType(IntEnum): Values are sourced from the C enum via the Cython layer. """ - OpaqueFD = hipFileHandleTypeOpaqueFD - OpaqueWin32 = hipFileHandleTypeOpaqueWin32 - UserspaceFS = hipFileHandleTypeUserspaceFS + OpaqueFD = py_hipFileHandleTypeOpaqueFD + OpaqueWin32 = py_hipFileHandleTypeOpaqueWin32 + UserspaceFS = py_hipFileHandleTypeUserspaceFS From 43e86ffec0a8cf7231e4ee97c4afb66c11b4e1c4 Mon Sep 17 00:00:00 2001 From: Riley Dixon Date: Tue, 7 Apr 2026 14:36:39 -0600 Subject: [PATCH 19/28] Python: FileHandle can now choose HandleType --- python/hipfile/file.py | 26 ++++++++++++++++++++++---- python/main.py | 3 ++- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/python/hipfile/file.py b/python/hipfile/file.py index 47e10ad9..dec1cfbf 100644 --- a/python/hipfile/file.py +++ b/python/hipfile/file.py @@ -9,20 +9,28 @@ read as _read, write as _write ) +from hipfile.enums import FileHandleType from hipfile.error import HipFileException +DefaultHandleType = None +if (os.name == 'posix'): + DefaultHandleType = FileHandleType.OpaqueFD +elif (os.name == 'nt'): + DefaultHandleType = FileHandleType.OpaqueWin32 + class FileHandle(): DEFAULT_MODE = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH - def __init__(self, path, flags, mode = DEFAULT_MODE, io_type = None): + def __init__(self, path, flags, mode = DEFAULT_MODE, handle_type = DefaultHandleType): self._fd = None self._flags = flags self._handle = None - # io_type currently unsupported - C bindings only use hipFileHandleTypeOpaqueFD - self._io_type = io_type + self._handle_type = None self._mode = mode self._path = path + self.handle_type = handle_type + def __del__(self): self.close() @@ -41,6 +49,16 @@ def flags(self): def handle(self): return self._handle + @property + def handle_type(self): + return self._handle_type + + @handle_type.setter + def handle_type(self, _handle_type): + if (_handle_type not in FileHandleType): + raise ValueError(f"'{_handle_type}' is not a member of enum FileHandleType") + self._handle_type = _handle_type + @property def mode(self): return self._mode @@ -53,7 +71,7 @@ def open(self): if (self._handle is not None): raise RuntimeError("The FileHandle is already open.") self._fd = os.open(self._path, self._flags, self._mode) - (handle, err) = handle_register(self._fd) + (handle, err) = handle_register(self._fd, self._handle_type) if (err[0] != 0): os.close(self._fd) self._fd = None diff --git a/python/main.py b/python/main.py index 9cbc5667..5919e43f 100644 --- a/python/main.py +++ b/python/main.py @@ -5,6 +5,7 @@ from hipfile.buffer import Buffer from hipfile.driver import Driver +from hipfile.enums import FileHandleType from hipfile.file import FileHandle from hipfile.properties import driver_get_properties, get_version from hipfile.hipMalloc import hipFree, hipMalloc @@ -28,7 +29,7 @@ with Driver() as hipfile_driver: print(f"Driver Use Count After: {hipfile_driver.use_count()}") with Buffer.from_ctypes_void_p(buffer, size, 0) as registered_buffer: - with FileHandle(input_path, os.O_RDWR | os.O_DIRECT | os.O_CREAT) as fh_input: + with FileHandle(input_path, os.O_RDWR | os.O_DIRECT | os.O_CREAT, handle_type = FileHandleType.OpaqueFD) as fh_input: with FileHandle(output_path, os.O_RDWR | os.O_DIRECT | os.O_CREAT | os.O_TRUNC) as fh_output: print(f"Transferring {size} bytes...") bytes_read = fh_input.read(registered_buffer, size, 0, 0) From 539c09614748d7f0b68cf209f35f4752843ffcb5 Mon Sep 17 00:00:00 2001 From: Riley Dixon Date: Tue, 7 Apr 2026 15:44:18 -0600 Subject: [PATCH 20/28] Python: Fix importing C symbols in Cython This puts the C hipFile symbols being imported by Cython into a "_c" namespace to avoid a namespace conflict when trying to set a Python variable/function with the same name. --- python/hipfile/_hipfile.pyx | 144 +++++++++++++++---------------- python/hipfile/enums.py | 164 ++++++++++++++++++------------------ 2 files changed, 154 insertions(+), 154 deletions(-) diff --git a/python/hipfile/_hipfile.pyx b/python/hipfile/_hipfile.pyx index e0cd8d4c..dddd3906 100644 --- a/python/hipfile/_hipfile.pyx +++ b/python/hipfile/_hipfile.pyx @@ -10,17 +10,17 @@ Functions that return ``hipFileError_t`` in C return a from libc.string cimport memset from libc.stdint cimport uintptr_t -from hipfile._chipfile cimport * +cimport hipfile._chipfile as _c # --------------------------------------------------------------------------- # Module-level constants # --------------------------------------------------------------------------- -VERSION_MAJOR = HIPFILE_VERSION_MAJOR -VERSION_MINOR = HIPFILE_VERSION_MINOR -VERSION_PATCH = HIPFILE_VERSION_PATCH -BASE_ERR = HIPFILE_BASE_ERR +VERSION_MAJOR = _c.HIPFILE_VERSION_MAJOR +VERSION_MINOR = _c.HIPFILE_VERSION_MINOR +VERSION_PATCH = _c.HIPFILE_VERSION_PATCH +BASE_ERR = _c.HIPFILE_BASE_ERR # --------------------------------------------------------------------------- # Enum re-exports (C → Python) @@ -31,56 +31,56 @@ BASE_ERR = HIPFILE_BASE_ERR # --------------------------------------------------------------------------- # hipFileOpError_t -py_hipFileSuccess = hipFileSuccess -py_hipFileDriverNotInitialized = hipFileDriverNotInitialized -py_hipFileDriverInvalidProps = hipFileDriverInvalidProps -py_hipFileDriverUnsupportedLimit = hipFileDriverUnsupportedLimit -py_hipFileDriverVersionMismatch = hipFileDriverVersionMismatch -py_hipFileDriverVersionReadError = hipFileDriverVersionReadError -py_hipFileDriverClosing = hipFileDriverClosing -py_hipFilePlatformNotSupported = hipFilePlatformNotSupported -py_hipFileIONotSupported = hipFileIONotSupported -py_hipFileDeviceNotSupported = hipFileDeviceNotSupported -py_hipFileDriverError = hipFileDriverError -py_hipFileHipDriverError = hipFileHipDriverError -py_hipFileHipPointerInvalid = hipFileHipPointerInvalid -py_hipFileHipMemoryTypeInvalid = hipFileHipMemoryTypeInvalid -py_hipFileHipPointerRangeError = hipFileHipPointerRangeError -py_hipFileHipContextMismatch = hipFileHipContextMismatch -py_hipFileInvalidMappingSize = hipFileInvalidMappingSize -py_hipFileInvalidMappingRange = hipFileInvalidMappingRange -py_hipFileInvalidFileType = hipFileInvalidFileType -py_hipFileInvalidFileOpenFlag = hipFileInvalidFileOpenFlag -py_hipFileDIONotSet = hipFileDIONotSet -py_hipFileInvalidValue = hipFileInvalidValue -py_hipFileMemoryAlreadyRegistered = hipFileMemoryAlreadyRegistered -py_hipFileMemoryNotRegistered = hipFileMemoryNotRegistered -py_hipFilePermissionDenied = hipFilePermissionDenied -py_hipFileDriverAlreadyOpen = hipFileDriverAlreadyOpen -py_hipFileHandleNotRegistered = hipFileHandleNotRegistered -py_hipFileHandleAlreadyRegistered = hipFileHandleAlreadyRegistered -py_hipFileDeviceNotFound = hipFileDeviceNotFound -py_hipFileInternalError = hipFileInternalError -py_hipFileGetNewFDFailed = hipFileGetNewFDFailed -py_hipFileDriverSetupError = hipFileDriverSetupError -py_hipFileIODisabled = hipFileIODisabled -py_hipFileBatchSubmitFailed = hipFileBatchSubmitFailed -py_hipFileGPUMemoryPinningFailed = hipFileGPUMemoryPinningFailed -py_hipFileBatchFull = hipFileBatchFull -py_hipFileAsyncNotSupported = hipFileAsyncNotSupported -py_hipFileIOMaxError = hipFileIOMaxError +hipFileSuccess = _c.hipFileSuccess +hipFileDriverNotInitialized = _c.hipFileDriverNotInitialized +hipFileDriverInvalidProps = _c.hipFileDriverInvalidProps +hipFileDriverUnsupportedLimit = _c.hipFileDriverUnsupportedLimit +hipFileDriverVersionMismatch = _c.hipFileDriverVersionMismatch +hipFileDriverVersionReadError = _c.hipFileDriverVersionReadError +hipFileDriverClosing = _c.hipFileDriverClosing +hipFilePlatformNotSupported = _c.hipFilePlatformNotSupported +hipFileIONotSupported = _c.hipFileIONotSupported +hipFileDeviceNotSupported = _c.hipFileDeviceNotSupported +hipFileDriverError = _c.hipFileDriverError +hipFileHipDriverError = _c.hipFileHipDriverError +hipFileHipPointerInvalid = _c.hipFileHipPointerInvalid +hipFileHipMemoryTypeInvalid = _c.hipFileHipMemoryTypeInvalid +hipFileHipPointerRangeError = _c.hipFileHipPointerRangeError +hipFileHipContextMismatch = _c.hipFileHipContextMismatch +hipFileInvalidMappingSize = _c.hipFileInvalidMappingSize +hipFileInvalidMappingRange = _c.hipFileInvalidMappingRange +hipFileInvalidFileType = _c.hipFileInvalidFileType +hipFileInvalidFileOpenFlag = _c.hipFileInvalidFileOpenFlag +hipFileDIONotSet = _c.hipFileDIONotSet +hipFileInvalidValue = _c.hipFileInvalidValue +hipFileMemoryAlreadyRegistered = _c.hipFileMemoryAlreadyRegistered +hipFileMemoryNotRegistered = _c.hipFileMemoryNotRegistered +hipFilePermissionDenied = _c.hipFilePermissionDenied +hipFileDriverAlreadyOpen = _c.hipFileDriverAlreadyOpen +hipFileHandleNotRegistered = _c.hipFileHandleNotRegistered +hipFileHandleAlreadyRegistered = _c.hipFileHandleAlreadyRegistered +hipFileDeviceNotFound = _c.hipFileDeviceNotFound +hipFileInternalError = _c.hipFileInternalError +hipFileGetNewFDFailed = _c.hipFileGetNewFDFailed +hipFileDriverSetupError = _c.hipFileDriverSetupError +hipFileIODisabled = _c.hipFileIODisabled +hipFileBatchSubmitFailed = _c.hipFileBatchSubmitFailed +hipFileGPUMemoryPinningFailed = _c.hipFileGPUMemoryPinningFailed +hipFileBatchFull = _c.hipFileBatchFull +hipFileAsyncNotSupported = _c.hipFileAsyncNotSupported +hipFileIOMaxError = _c.hipFileIOMaxError # hipFileFileHandleType_t -py_hipFileHandleTypeOpaqueFD = hipFileHandleTypeOpaqueFD -py_hipFileHandleTypeOpaqueWin32 = hipFileHandleTypeOpaqueWin32 -py_hipFileHandleTypeUserspaceFS = hipFileHandleTypeUserspaceFS +hipFileHandleTypeOpaqueFD = _c.hipFileHandleTypeOpaqueFD +hipFileHandleTypeOpaqueWin32 = _c.hipFileHandleTypeOpaqueWin32 +hipFileHandleTypeUserspaceFS = _c.hipFileHandleTypeUserspaceFS # --------------------------------------------------------------------------- # Internal helpers # --------------------------------------------------------------------------- -cdef inline tuple _err(hipFileError_t e): +cdef inline tuple _err(_c.hipFileError_t e): return (e.err, e.hip_drv_err) @@ -90,12 +90,12 @@ cdef inline tuple _err(hipFileError_t e): def is_hipfile_err(int err_code): """Equivalent of the ``IS_HIPFILE_ERR`` C macro.""" - return abs(err_code) > HIPFILE_BASE_ERR + return abs(err_code) > _c.HIPFILE_BASE_ERR def hipfile_errstr(int err_code): """Equivalent of the ``HIPFILE_ERRSTR`` C macro.""" - cdef const char *s = hipFileGetOpErrorString(abs(err_code)) + cdef const char *s = _c.hipFileGetOpErrorString(<_c.hipFileOpError_t>abs(err_code)) if s == NULL: return "" return s.decode("utf-8") @@ -106,7 +106,7 @@ def is_hip_drv_err(tuple err): Takes an error tuple as returned by the wrapper functions. """ - return err[0] == hipFileHipDriverError + return err[0] == _c.hipFileHipDriverError def hip_drv_err(tuple err): @@ -119,7 +119,7 @@ def hip_drv_err(tuple err): def get_op_error_string(int status): """Wrapper for ``hipFileGetOpErrorString``.""" - cdef const char *s = hipFileGetOpErrorString(status) + cdef const char *s = _c.hipFileGetOpErrorString(<_c.hipFileOpError_t>status) if s == NULL: return "" return s.decode("utf-8") @@ -131,17 +131,17 @@ def get_op_error_string(int status): def driver_open(): """Wrapper for ``hipFileDriverOpen``.""" - return _err(hipFileDriverOpen()) + return _err(_c.hipFileDriverOpen()) def driver_close(): """Wrapper for ``hipFileDriverClose``.""" - return _err(hipFileDriverClose()) + return _err(_c.hipFileDriverClose()) def use_count(): """Wrapper for ``hipFileUseCount``.""" - return hipFileUseCount() + return _c.hipFileUseCount() # --------------------------------------------------------------------------- @@ -154,7 +154,7 @@ def get_version(): Returns ``((major, minor, patch), error_tuple)``. """ cdef unsigned major = 0, minor = 0, patch = 0 - cdef hipFileError_t e = hipFileGetVersion(&major, &minor, &patch) + cdef _c.hipFileError_t e = _c.hipFileGetVersion(&major, &minor, &patch) return ((major, minor, patch), _err(e)) @@ -175,18 +175,18 @@ def handle_register(int fd, int handle_type): Returns ``(handle_int, error_tuple)``. The handle is an opaque integer that must be passed back to other hipFile calls. """ - cdef hipFileHandle_t fh = NULL - cdef hipFileDescr_t descr + cdef _c.hipFileHandle_t fh = NULL + cdef _c.hipFileDescr_t descr memset(&descr, 0, sizeof(descr)) - descr.type = handle_type + descr.type = <_c.hipFileFileHandleType_t>handle_type descr.fd = fd - cdef hipFileError_t e = hipFileHandleRegister(&fh, &descr) + cdef _c.hipFileError_t e = _c.hipFileHandleRegister(&fh, &descr) return (fh, _err(e)) def handle_deregister(uintptr_t handle): """Wrapper for ``hipFileHandleDeregister``.""" - hipFileHandleDeregister(handle) + _c.hipFileHandleDeregister(<_c.hipFileHandle_t>handle) # --------------------------------------------------------------------------- @@ -195,12 +195,12 @@ def handle_deregister(uintptr_t handle): def buf_register(uintptr_t buffer_base, size_t length, int flags=0): """Wrapper for ``hipFileBufRegister``.""" - return _err(hipFileBufRegister(buffer_base, length, flags)) + return _err(_c.hipFileBufRegister(buffer_base, length, flags)) def buf_deregister(uintptr_t buffer_base): """Wrapper for ``hipFileBufDeregister``.""" - return _err(hipFileBufDeregister(buffer_base)) + return _err(_c.hipFileBufDeregister(buffer_base)) # --------------------------------------------------------------------------- @@ -208,7 +208,7 @@ def buf_deregister(uintptr_t buffer_base): # --------------------------------------------------------------------------- def read(uintptr_t handle, uintptr_t buffer_base, size_t size, - hoff_t file_offset, hoff_t buffer_offset): + _c.hoff_t file_offset, _c.hoff_t buffer_offset): """Wrapper for ``hipFileRead``. Returns raw ``ssize_t``: @@ -216,13 +216,13 @@ def read(uintptr_t handle, uintptr_t buffer_base, size_t size, * ``-1`` — system error (check ``errno``) * ``< -1`` — negate to get a ``hipFileOpError_t`` """ - return hipFileRead(handle, - buffer_base, size, - file_offset, buffer_offset) + return _c.hipFileRead(<_c.hipFileHandle_t>handle, + buffer_base, size, + file_offset, buffer_offset) def write(uintptr_t handle, uintptr_t buffer_base, size_t size, - hoff_t file_offset, hoff_t buffer_offset): + _c.hoff_t file_offset, _c.hoff_t buffer_offset): """Wrapper for ``hipFileWrite``. Returns raw ``ssize_t``: @@ -230,9 +230,9 @@ def write(uintptr_t handle, uintptr_t buffer_base, size_t size, * ``-1`` — system error (check ``errno``) * ``< -1`` — negate to get a ``hipFileOpError_t`` """ - return hipFileWrite(handle, - buffer_base, size, - file_offset, buffer_offset) + return _c.hipFileWrite(<_c.hipFileHandle_t>handle, + buffer_base, size, + file_offset, buffer_offset) # --------------------------------------------------------------------------- @@ -244,9 +244,9 @@ def driver_get_properties(): Returns ``(props_dict, error_tuple)``. """ - cdef hipFileDriverProps_t props + cdef _c.hipFileDriverProps_t props memset(&props, 0, sizeof(props)) - cdef hipFileError_t e = hipFileDriverGetProperties(&props) + cdef _c.hipFileError_t e = _c.hipFileDriverGetProperties(&props) d = { "nvfs_major_version": props.nvfs_major_version, "nvfs_minor_version": props.nvfs_minor_version, diff --git a/python/hipfile/enums.py b/python/hipfile/enums.py index 3ad36eeb..208e12cd 100644 --- a/python/hipfile/enums.py +++ b/python/hipfile/enums.py @@ -2,48 +2,48 @@ from hipfile._hipfile import ( # hipFileOpError_t values (resolved from C at build time) - py_hipFileSuccess, - py_hipFileDriverNotInitialized, - py_hipFileDriverInvalidProps, - py_hipFileDriverUnsupportedLimit, - py_hipFileDriverVersionMismatch, - py_hipFileDriverVersionReadError, - py_hipFileDriverClosing, - py_hipFilePlatformNotSupported, - py_hipFileIONotSupported, - py_hipFileDeviceNotSupported, - py_hipFileDriverError, - py_hipFileHipDriverError, - py_hipFileHipPointerInvalid, - py_hipFileHipMemoryTypeInvalid, - py_hipFileHipPointerRangeError, - py_hipFileHipContextMismatch, - py_hipFileInvalidMappingSize, - py_hipFileInvalidMappingRange, - py_hipFileInvalidFileType, - py_hipFileInvalidFileOpenFlag, - py_hipFileDIONotSet, - py_hipFileInvalidValue, - py_hipFileMemoryAlreadyRegistered, - py_hipFileMemoryNotRegistered, - py_hipFilePermissionDenied, - py_hipFileDriverAlreadyOpen, - py_hipFileHandleNotRegistered, - py_hipFileHandleAlreadyRegistered, - py_hipFileDeviceNotFound, - py_hipFileInternalError, - py_hipFileGetNewFDFailed, - py_hipFileDriverSetupError, - py_hipFileIODisabled, - py_hipFileBatchSubmitFailed, - py_hipFileGPUMemoryPinningFailed, - py_hipFileBatchFull, - py_hipFileAsyncNotSupported, - py_hipFileIOMaxError, + hipFileSuccess, + hipFileDriverNotInitialized, + hipFileDriverInvalidProps, + hipFileDriverUnsupportedLimit, + hipFileDriverVersionMismatch, + hipFileDriverVersionReadError, + hipFileDriverClosing, + hipFilePlatformNotSupported, + hipFileIONotSupported, + hipFileDeviceNotSupported, + hipFileDriverError, + hipFileHipDriverError, + hipFileHipPointerInvalid, + hipFileHipMemoryTypeInvalid, + hipFileHipPointerRangeError, + hipFileHipContextMismatch, + hipFileInvalidMappingSize, + hipFileInvalidMappingRange, + hipFileInvalidFileType, + hipFileInvalidFileOpenFlag, + hipFileDIONotSet, + hipFileInvalidValue, + hipFileMemoryAlreadyRegistered, + hipFileMemoryNotRegistered, + hipFilePermissionDenied, + hipFileDriverAlreadyOpen, + hipFileHandleNotRegistered, + hipFileHandleAlreadyRegistered, + hipFileDeviceNotFound, + hipFileInternalError, + hipFileGetNewFDFailed, + hipFileDriverSetupError, + hipFileIODisabled, + hipFileBatchSubmitFailed, + hipFileGPUMemoryPinningFailed, + hipFileBatchFull, + hipFileAsyncNotSupported, + hipFileIOMaxError, # hipFileFileHandleType_t values (resolved from C at build time) - py_hipFileHandleTypeOpaqueFD, - py_hipFileHandleTypeOpaqueWin32, - py_hipFileHandleTypeUserspaceFS, + hipFileHandleTypeOpaqueFD, + hipFileHandleTypeOpaqueWin32, + hipFileHandleTypeUserspaceFS, ) @@ -55,44 +55,44 @@ class OpError(IntEnum): in hipfile.h automatically. """ - Success = py_hipFileSuccess - DriverNotInitialized = py_hipFileDriverNotInitialized - DriverInvalidProps = py_hipFileDriverInvalidProps - DriverUnsupportedLimit = py_hipFileDriverUnsupportedLimit - DriverVersionMismatch = py_hipFileDriverVersionMismatch - DriverVersionReadError = py_hipFileDriverVersionReadError - DriverClosing = py_hipFileDriverClosing - PlatformNotSupported = py_hipFilePlatformNotSupported - IONotSupported = py_hipFileIONotSupported - DeviceNotSupported = py_hipFileDeviceNotSupported - DriverError = py_hipFileDriverError - HipDriverError = py_hipFileHipDriverError - HipPointerInvalid = py_hipFileHipPointerInvalid - HipMemoryTypeInvalid = py_hipFileHipMemoryTypeInvalid - HipPointerRangeError = py_hipFileHipPointerRangeError - HipContextMismatch = py_hipFileHipContextMismatch - InvalidMappingSize = py_hipFileInvalidMappingSize - InvalidMappingRange = py_hipFileInvalidMappingRange - InvalidFileType = py_hipFileInvalidFileType - InvalidFileOpenFlag = py_hipFileInvalidFileOpenFlag - DIONotSet = py_hipFileDIONotSet - InvalidValue = py_hipFileInvalidValue - MemoryAlreadyRegistered = py_hipFileMemoryAlreadyRegistered - MemoryNotRegistered = py_hipFileMemoryNotRegistered - PermissionDenied = py_hipFilePermissionDenied - DriverAlreadyOpen = py_hipFileDriverAlreadyOpen - HandleNotRegistered = py_hipFileHandleNotRegistered - HandleAlreadyRegistered = py_hipFileHandleAlreadyRegistered - DeviceNotFound = py_hipFileDeviceNotFound - InternalError = py_hipFileInternalError - GetNewFDFailed = py_hipFileGetNewFDFailed - DriverSetupError = py_hipFileDriverSetupError - IODisabled = py_hipFileIODisabled - BatchSubmitFailed = py_hipFileBatchSubmitFailed - GPUMemoryPinningFailed = py_hipFileGPUMemoryPinningFailed - BatchFull = py_hipFileBatchFull - AsyncNotSupported = py_hipFileAsyncNotSupported - IOMaxError = py_hipFileIOMaxError + Success = hipFileSuccess + DriverNotInitialized = hipFileDriverNotInitialized + DriverInvalidProps = hipFileDriverInvalidProps + DriverUnsupportedLimit = hipFileDriverUnsupportedLimit + DriverVersionMismatch = hipFileDriverVersionMismatch + DriverVersionReadError = hipFileDriverVersionReadError + DriverClosing = hipFileDriverClosing + PlatformNotSupported = hipFilePlatformNotSupported + IONotSupported = hipFileIONotSupported + DeviceNotSupported = hipFileDeviceNotSupported + DriverError = hipFileDriverError + HipDriverError = hipFileHipDriverError + HipPointerInvalid = hipFileHipPointerInvalid + HipMemoryTypeInvalid = hipFileHipMemoryTypeInvalid + HipPointerRangeError = hipFileHipPointerRangeError + HipContextMismatch = hipFileHipContextMismatch + InvalidMappingSize = hipFileInvalidMappingSize + InvalidMappingRange = hipFileInvalidMappingRange + InvalidFileType = hipFileInvalidFileType + InvalidFileOpenFlag = hipFileInvalidFileOpenFlag + DIONotSet = hipFileDIONotSet + InvalidValue = hipFileInvalidValue + MemoryAlreadyRegistered = hipFileMemoryAlreadyRegistered + MemoryNotRegistered = hipFileMemoryNotRegistered + PermissionDenied = hipFilePermissionDenied + DriverAlreadyOpen = hipFileDriverAlreadyOpen + HandleNotRegistered = hipFileHandleNotRegistered + HandleAlreadyRegistered = hipFileHandleAlreadyRegistered + DeviceNotFound = hipFileDeviceNotFound + InternalError = hipFileInternalError + GetNewFDFailed = hipFileGetNewFDFailed + DriverSetupError = hipFileDriverSetupError + IODisabled = hipFileIODisabled + BatchSubmitFailed = hipFileBatchSubmitFailed + GPUMemoryPinningFailed = hipFileGPUMemoryPinningFailed + BatchFull = hipFileBatchFull + AsyncNotSupported = hipFileAsyncNotSupported + IOMaxError = hipFileIOMaxError class FileHandleType(IntEnum): @@ -101,6 +101,6 @@ class FileHandleType(IntEnum): Values are sourced from the C enum via the Cython layer. """ - OpaqueFD = py_hipFileHandleTypeOpaqueFD - OpaqueWin32 = py_hipFileHandleTypeOpaqueWin32 - UserspaceFS = py_hipFileHandleTypeUserspaceFS + OpaqueFD = hipFileHandleTypeOpaqueFD + OpaqueWin32 = hipFileHandleTypeOpaqueWin32 + UserspaceFS = hipFileHandleTypeUserspaceFS From a80d1fb334a61f786ba0333a840aabd05c7e01ce Mon Sep 17 00:00:00 2001 From: Riley Dixon Date: Tue, 7 Apr 2026 16:44:41 -0600 Subject: [PATCH 21/28] Python: Return extra error info from read()/write() The C functions hipFileRead()/hipFileWrite() report error conditions through a negative return value. If this return value is -1 or -5011, it indicates additional error information needs to be fetched as it cannot be included in a single return value. However, this error information needs to be fetched prior to control returning to Python. Otherwise there is no guarantee that these error values are still valid for the error that occurred. --- python/hipfile/_chipfile.pxd | 2 ++ python/hipfile/_hipfile.pyx | 45 +++++++++++++++++++++++++----------- python/hipfile/file.py | 30 +++++++++++++----------- 3 files changed, 50 insertions(+), 27 deletions(-) diff --git a/python/hipfile/_chipfile.pxd b/python/hipfile/_chipfile.pxd index 9803ef3d..ed3a1fa8 100644 --- a/python/hipfile/_chipfile.pxd +++ b/python/hipfile/_chipfile.pxd @@ -19,6 +19,8 @@ cdef extern from "hip/hip_runtime_api.h": ctypedef enum hipError_t: hipSuccess = 0 + hipError_t hipPeekAtLastError() + # --------------------------------------------------------------------------- # hipFile public API diff --git a/python/hipfile/_hipfile.pyx b/python/hipfile/_hipfile.pyx index dddd3906..d753da7b 100644 --- a/python/hipfile/_hipfile.pyx +++ b/python/hipfile/_hipfile.pyx @@ -7,6 +7,7 @@ Functions that return ``hipFileError_t`` in C return a ``(hipFileOpError_t, hipError_t)`` 2-tuple here. """ +from libc.errno cimport errno from libc.string cimport memset from libc.stdint cimport uintptr_t @@ -211,28 +212,44 @@ def read(uintptr_t handle, uintptr_t buffer_base, size_t size, _c.hoff_t file_offset, _c.hoff_t buffer_offset): """Wrapper for ``hipFileRead``. - Returns raw ``ssize_t``: - * ``>= 0`` — number of bytes read - * ``-1`` — system error (check ``errno``) - * ``< -1`` — negate to get a ``hipFileOpError_t`` + Returns ``(result, extra)``: + * ``result >= 0`` — number of bytes read, ``extra = 0`` + * ``result == -1`` — system error, ``extra = errno`` + * ``result < -1`` — negated ``hipFileOpError_t``; if + ``-hipFileHipDriverError``, ``extra = hipError_t`` from + ``hipPeekAtLastError()``, otherwise ``extra = 0`` """ - return _c.hipFileRead(<_c.hipFileHandle_t>handle, - buffer_base, size, - file_offset, buffer_offset) + cdef ssize_t ret = _c.hipFileRead(<_c.hipFileHandle_t>handle, + buffer_base, size, + file_offset, buffer_offset) + cdef int extra = 0 + if ret == -1: + extra = errno + elif ret == -_c.hipFileHipDriverError: + extra = _c.hipPeekAtLastError() + return (ret, extra) def write(uintptr_t handle, uintptr_t buffer_base, size_t size, _c.hoff_t file_offset, _c.hoff_t buffer_offset): """Wrapper for ``hipFileWrite``. - Returns raw ``ssize_t``: - * ``>= 0`` — number of bytes written - * ``-1`` — system error (check ``errno``) - * ``< -1`` — negate to get a ``hipFileOpError_t`` + Returns ``(result, extra)``: + * ``result >= 0`` — number of bytes written, ``extra = 0`` + * ``result == -1`` — system error, ``extra = errno`` + * ``result < -1`` — negated ``hipFileOpError_t``; if + ``-hipFileHipDriverError``, ``extra = hipError_t`` from + ``hipPeekAtLastError()``, otherwise ``extra = 0`` """ - return _c.hipFileWrite(<_c.hipFileHandle_t>handle, - buffer_base, size, - file_offset, buffer_offset) + cdef ssize_t ret = _c.hipFileWrite(<_c.hipFileHandle_t>handle, + buffer_base, size, + file_offset, buffer_offset) + cdef int extra = 0 + if ret == -1: + extra = errno + elif ret == -_c.hipFileHipDriverError: + extra = _c.hipPeekAtLastError() + return (ret, extra) # --------------------------------------------------------------------------- diff --git a/python/hipfile/file.py b/python/hipfile/file.py index dec1cfbf..7fa1a77a 100644 --- a/python/hipfile/file.py +++ b/python/hipfile/file.py @@ -9,7 +9,7 @@ read as _read, write as _write ) -from hipfile.enums import FileHandleType +from hipfile.enums import OpError, FileHandleType from hipfile.error import HipFileException DefaultHandleType = None @@ -89,23 +89,27 @@ def close(self): def read(self, buffer, size, file_offset, buffer_offset): if (self._handle is None): raise RuntimeError("The FileHandle is not open.") - bytes_read = _read(self._handle, buffer.ptr, size, file_offset, buffer_offset) - # Note: _read should raise an OSError if a system error occured. - if (bytes_read < -1): + (bytes_read, extra_err) = _read(self._handle, buffer.ptr, size, file_offset, buffer_offset) + if (bytes_read == -1): + # extra_err is errno + raise OSError(extra_err, os.strerror(extra_err)) + elif (bytes_read < -1): # hipFile Error - # Issue: If bytes_read == -hipFileHipDriverError, how do we get the hipError_t? - # Probably like in C and calling hipPeekLastError - raise HipFileException(-bytes_read, 0) + # If -bytes_read == OpError.HipDriverError, extra_err is hipError_t. + # Otherwise, extra_err is 0. + raise HipFileException(-bytes_read, extra_err) return bytes_read def write(self, buffer, size, file_offset, buffer_offset): if (self._handle is None): raise RuntimeError("The FileHandle is not open.") - bytes_written = _write(self._handle, buffer.ptr, size, file_offset, buffer_offset) - # Note: _write should raise an OSError if a system error occured. - if (bytes_written < -1): + (bytes_written, extra_err) = _write(self._handle, buffer.ptr, size, file_offset, buffer_offset) + if (bytes_written == -1): + # extra_err is errno + raise OSError(extra_err, os.strerror(extra_err)) + elif (bytes_written < -1): # hipFile Error - # Issue: If bytes_written == -hipFileHipDriverError, how do we get the hipError_t? - # Probably like in C and calling hipPeekLastError - raise HipFileException(-bytes_written, 0) + # If -bytes_written == OpError.HipDriverError, extra_err is hipError_t. + # Otherwise, extra_err is 0. + raise HipFileException(-bytes_written, extra_err) return bytes_written From d8ecba10d88afdc423fc8ad58fd4573718c6960d Mon Sep 17 00:00:00 2001 From: Riley Dixon Date: Tue, 7 Apr 2026 17:42:41 -0600 Subject: [PATCH 22/28] Python: Cleanup __init__.py for public consumption Cleans up what is imported in __init__.py to not pollute the namespace when a user imports 'hipfile'. --- python/hipfile/__init__.py | 28 ++++++++++++++++------------ python/main.py | 14 +++++++++----- 2 files changed, 25 insertions(+), 17 deletions(-) diff --git a/python/hipfile/__init__.py b/python/hipfile/__init__.py index 17240728..421e1160 100644 --- a/python/hipfile/__init__.py +++ b/python/hipfile/__init__.py @@ -1,17 +1,10 @@ -from hipfile._hipfile import * # noqa: F401,F403 from hipfile._hipfile import ( # Constants - VERSION_MAJOR, - VERSION_MINOR, - VERSION_PATCH, - BASE_ERR, - # Error Macros - is_hipfile_err, - hipfile_errstr, - is_hip_drv_err, - hip_drv_err, + VERSION_MAJOR as _VERSION_MAJOR, + VERSION_MINOR as _VERSION_MINOR, + VERSION_PATCH as _VERSION_PATCH ) -from hipfile.buffer import( +from hipfile.buffer import ( Buffer ) from hipfile.driver import ( @@ -32,4 +25,15 @@ get_version ) -__version__ = f"{VERSION_MAJOR}.{VERSION_MINOR}.{VERSION_PATCH}" +__all__ = [ + "__version__", + "Driver", + "FileHandle", + "Buffer", + "HipFileException", + "FileHandleType", + "OpError", + "driver_get_properties", + "get_version" +] +__version__ = f"{_VERSION_MAJOR}.{_VERSION_MINOR}.{_VERSION_PATCH}" diff --git a/python/main.py b/python/main.py index 5919e43f..26e4d6c8 100644 --- a/python/main.py +++ b/python/main.py @@ -3,13 +3,17 @@ import pathlib import stat -from hipfile.buffer import Buffer -from hipfile.driver import Driver -from hipfile.enums import FileHandleType -from hipfile.file import FileHandle -from hipfile.properties import driver_get_properties, get_version from hipfile.hipMalloc import hipFree, hipMalloc +from hipfile import ( + Driver, + FileHandle, + Buffer, + FileHandleType, + driver_get_properties, + get_version +) + hipfile_version = get_version() input_path = pathlib.Path("/mnt/ais/ext4/random_2MiB.bin") From 834c16fa14c8568155d78f8ce93a7a8b0f50ffc4 Mon Sep 17 00:00:00 2001 From: Riley Dixon Date: Wed, 8 Apr 2026 12:41:54 -0600 Subject: [PATCH 23/28] Python: Formatting with Black --- python/hipfile/__init__.py | 30 ++++----------- python/hipfile/buffer.py | 14 +++---- python/hipfile/driver.py | 11 ++---- python/hipfile/enums.py | 74 ++++++++++++++++++------------------ python/hipfile/error.py | 11 +++--- python/hipfile/file.py | 57 ++++++++++++++------------- python/hipfile/hipMalloc.py | 7 +++- python/hipfile/properties.py | 12 +++--- python/main.py | 28 ++++++++------ 9 files changed, 120 insertions(+), 124 deletions(-) diff --git a/python/hipfile/__init__.py b/python/hipfile/__init__.py index 421e1160..40a8ba2a 100644 --- a/python/hipfile/__init__.py +++ b/python/hipfile/__init__.py @@ -2,28 +2,14 @@ # Constants VERSION_MAJOR as _VERSION_MAJOR, VERSION_MINOR as _VERSION_MINOR, - VERSION_PATCH as _VERSION_PATCH -) -from hipfile.buffer import ( - Buffer -) -from hipfile.driver import ( - Driver -) -from hipfile.enums import ( - FileHandleType, - OpError -) -from hipfile.error import ( - HipFileException -) -from hipfile.file import ( - FileHandle -) -from hipfile.properties import ( - driver_get_properties, - get_version + VERSION_PATCH as _VERSION_PATCH, ) +from hipfile.buffer import Buffer +from hipfile.driver import Driver +from hipfile.enums import FileHandleType, OpError +from hipfile.error import HipFileException +from hipfile.file import FileHandle +from hipfile.properties import driver_get_properties, get_version __all__ = [ "__version__", @@ -34,6 +20,6 @@ "FileHandleType", "OpError", "driver_get_properties", - "get_version" + "get_version", ] __version__ = f"{_VERSION_MAJOR}.{_VERSION_MINOR}.{_VERSION_PATCH}" diff --git a/python/hipfile/buffer.py b/python/hipfile/buffer.py index 0cde0214..875608ca 100644 --- a/python/hipfile/buffer.py +++ b/python/hipfile/buffer.py @@ -1,16 +1,14 @@ from __future__ import annotations from typing import TYPE_CHECKING -from hipfile._hipfile import ( - buf_deregister, - buf_register -) +from hipfile._hipfile import buf_deregister, buf_register from hipfile.error import HipFileException if TYPE_CHECKING: from ctypes import c_void_p -class Buffer(): + +class Buffer: @classmethod def from_ctypes_void_p(cls, ctypes_void_p: c_void_p, length, flags): @@ -38,14 +36,14 @@ def ptr(self): return self._buffer_ptr def deregister(self): - if (self._registered): + if self._registered: err = buf_deregister(self._buffer_ptr) - if (err[0] != 0): + if err[0] != 0: raise HipFileException(err[0], err[1]) self._registered = False def register(self): err = buf_register(self._buffer_ptr, self._length, self._flags) - if (err[0] != 0): + if err[0] != 0: raise HipFileException(err[0], err[1]) self._registered = True diff --git a/python/hipfile/driver.py b/python/hipfile/driver.py index 9bfa365c..04ee52a5 100644 --- a/python/hipfile/driver.py +++ b/python/hipfile/driver.py @@ -1,10 +1,7 @@ -from hipfile._hipfile import ( - driver_open, - driver_close, - use_count as _use_count -) +from hipfile._hipfile import driver_open, driver_close, use_count as _use_count from hipfile.error import HipFileException + class Driver: @staticmethod @@ -20,10 +17,10 @@ def __exit__(self, exc_type, exc_value, traceback): def close(self): err = driver_close() - if (err[0] != 0): + if err[0] != 0: raise HipFileException(err[0], err[1]) def open(self): err = driver_open() - if (err[0] != 0): + if err[0] != 0: raise HipFileException(err[0], err[1]) diff --git a/python/hipfile/enums.py b/python/hipfile/enums.py index 208e12cd..e51fec6b 100644 --- a/python/hipfile/enums.py +++ b/python/hipfile/enums.py @@ -55,44 +55,44 @@ class OpError(IntEnum): in hipfile.h automatically. """ - Success = hipFileSuccess - DriverNotInitialized = hipFileDriverNotInitialized - DriverInvalidProps = hipFileDriverInvalidProps - DriverUnsupportedLimit = hipFileDriverUnsupportedLimit - DriverVersionMismatch = hipFileDriverVersionMismatch - DriverVersionReadError = hipFileDriverVersionReadError - DriverClosing = hipFileDriverClosing - PlatformNotSupported = hipFilePlatformNotSupported - IONotSupported = hipFileIONotSupported - DeviceNotSupported = hipFileDeviceNotSupported - DriverError = hipFileDriverError - HipDriverError = hipFileHipDriverError - HipPointerInvalid = hipFileHipPointerInvalid - HipMemoryTypeInvalid = hipFileHipMemoryTypeInvalid - HipPointerRangeError = hipFileHipPointerRangeError - HipContextMismatch = hipFileHipContextMismatch - InvalidMappingSize = hipFileInvalidMappingSize - InvalidMappingRange = hipFileInvalidMappingRange - InvalidFileType = hipFileInvalidFileType - InvalidFileOpenFlag = hipFileInvalidFileOpenFlag - DIONotSet = hipFileDIONotSet - InvalidValue = hipFileInvalidValue + Success = hipFileSuccess + DriverNotInitialized = hipFileDriverNotInitialized + DriverInvalidProps = hipFileDriverInvalidProps + DriverUnsupportedLimit = hipFileDriverUnsupportedLimit + DriverVersionMismatch = hipFileDriverVersionMismatch + DriverVersionReadError = hipFileDriverVersionReadError + DriverClosing = hipFileDriverClosing + PlatformNotSupported = hipFilePlatformNotSupported + IONotSupported = hipFileIONotSupported + DeviceNotSupported = hipFileDeviceNotSupported + DriverError = hipFileDriverError + HipDriverError = hipFileHipDriverError + HipPointerInvalid = hipFileHipPointerInvalid + HipMemoryTypeInvalid = hipFileHipMemoryTypeInvalid + HipPointerRangeError = hipFileHipPointerRangeError + HipContextMismatch = hipFileHipContextMismatch + InvalidMappingSize = hipFileInvalidMappingSize + InvalidMappingRange = hipFileInvalidMappingRange + InvalidFileType = hipFileInvalidFileType + InvalidFileOpenFlag = hipFileInvalidFileOpenFlag + DIONotSet = hipFileDIONotSet + InvalidValue = hipFileInvalidValue MemoryAlreadyRegistered = hipFileMemoryAlreadyRegistered - MemoryNotRegistered = hipFileMemoryNotRegistered - PermissionDenied = hipFilePermissionDenied - DriverAlreadyOpen = hipFileDriverAlreadyOpen - HandleNotRegistered = hipFileHandleNotRegistered + MemoryNotRegistered = hipFileMemoryNotRegistered + PermissionDenied = hipFilePermissionDenied + DriverAlreadyOpen = hipFileDriverAlreadyOpen + HandleNotRegistered = hipFileHandleNotRegistered HandleAlreadyRegistered = hipFileHandleAlreadyRegistered - DeviceNotFound = hipFileDeviceNotFound - InternalError = hipFileInternalError - GetNewFDFailed = hipFileGetNewFDFailed - DriverSetupError = hipFileDriverSetupError - IODisabled = hipFileIODisabled - BatchSubmitFailed = hipFileBatchSubmitFailed - GPUMemoryPinningFailed = hipFileGPUMemoryPinningFailed - BatchFull = hipFileBatchFull - AsyncNotSupported = hipFileAsyncNotSupported - IOMaxError = hipFileIOMaxError + DeviceNotFound = hipFileDeviceNotFound + InternalError = hipFileInternalError + GetNewFDFailed = hipFileGetNewFDFailed + DriverSetupError = hipFileDriverSetupError + IODisabled = hipFileIODisabled + BatchSubmitFailed = hipFileBatchSubmitFailed + GPUMemoryPinningFailed = hipFileGPUMemoryPinningFailed + BatchFull = hipFileBatchFull + AsyncNotSupported = hipFileAsyncNotSupported + IOMaxError = hipFileIOMaxError class FileHandleType(IntEnum): @@ -101,6 +101,6 @@ class FileHandleType(IntEnum): Values are sourced from the C enum via the Cython layer. """ - OpaqueFD = hipFileHandleTypeOpaqueFD + OpaqueFD = hipFileHandleTypeOpaqueFD OpaqueWin32 = hipFileHandleTypeOpaqueWin32 UserspaceFS = hipFileHandleTypeUserspaceFS diff --git a/python/hipfile/error.py b/python/hipfile/error.py index efad2712..5664f5fe 100644 --- a/python/hipfile/error.py +++ b/python/hipfile/error.py @@ -1,12 +1,11 @@ -from hipfile._hipfile import ( - get_op_error_string -) +from hipfile._hipfile import get_op_error_string + class HipFileException(Exception): def __init__(self, hipfile_err, hip_err): self._hipfile_err = hipfile_err self._hip_err = hip_err - + @property def hipfile_err(self): return self._hipfile_err @@ -14,9 +13,9 @@ def hipfile_err(self): @property def hip_err(self): return self._hip_err - + def __str__(self): err_msg = f"{self._hipfile_err} - {get_op_error_string(self._hipfile_err)}" - if (self._hipfile_err == 5011): + if self._hipfile_err == 5011: err_msg += f" {self._hip_err}" return err_msg diff --git a/python/hipfile/file.py b/python/hipfile/file.py index 7fa1a77a..ebd7f719 100644 --- a/python/hipfile/file.py +++ b/python/hipfile/file.py @@ -7,21 +7,22 @@ handle_deregister, # Synchronous I/O read as _read, - write as _write + write as _write, ) from hipfile.enums import OpError, FileHandleType from hipfile.error import HipFileException DefaultHandleType = None -if (os.name == 'posix'): +if os.name == "posix": DefaultHandleType = FileHandleType.OpaqueFD -elif (os.name == 'nt'): +elif os.name == "nt": DefaultHandleType = FileHandleType.OpaqueWin32 -class FileHandle(): + +class FileHandle: DEFAULT_MODE = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH - def __init__(self, path, flags, mode = DEFAULT_MODE, handle_type = DefaultHandleType): + def __init__(self, path, flags, mode=DEFAULT_MODE, handle_type=DefaultHandleType): self._fd = None self._flags = flags self._handle = None @@ -44,56 +45,58 @@ def __exit__(self, exc_type, exc_value, traceback): @property def flags(self): return self._flags - + @property def handle(self): return self._handle - + @property def handle_type(self): return self._handle_type - + @handle_type.setter def handle_type(self, _handle_type): - if (_handle_type not in FileHandleType): + if _handle_type not in FileHandleType: raise ValueError(f"'{_handle_type}' is not a member of enum FileHandleType") self._handle_type = _handle_type - + @property def mode(self): return self._mode - + @property def path(self): return self._path def open(self): - if (self._handle is not None): + if self._handle is not None: raise RuntimeError("The FileHandle is already open.") self._fd = os.open(self._path, self._flags, self._mode) - (handle, err) = handle_register(self._fd, self._handle_type) - if (err[0] != 0): + handle, err = handle_register(self._fd, self._handle_type) + if err[0] != 0: os.close(self._fd) self._fd = None raise HipFileException(err[0], err[1]) self._handle = handle - + def close(self): - if (self._handle is not None): + if self._handle is not None: handle_deregister(self._handle) self._handle = None - if (self._fd is not None): + if self._fd is not None: os.close(self._fd) self._fd = None - + def read(self, buffer, size, file_offset, buffer_offset): - if (self._handle is None): + if self._handle is None: raise RuntimeError("The FileHandle is not open.") - (bytes_read, extra_err) = _read(self._handle, buffer.ptr, size, file_offset, buffer_offset) - if (bytes_read == -1): + bytes_read, extra_err = _read( + self._handle, buffer.ptr, size, file_offset, buffer_offset + ) + if bytes_read == -1: # extra_err is errno raise OSError(extra_err, os.strerror(extra_err)) - elif (bytes_read < -1): + elif bytes_read < -1: # hipFile Error # If -bytes_read == OpError.HipDriverError, extra_err is hipError_t. # Otherwise, extra_err is 0. @@ -101,13 +104,15 @@ def read(self, buffer, size, file_offset, buffer_offset): return bytes_read def write(self, buffer, size, file_offset, buffer_offset): - if (self._handle is None): + if self._handle is None: raise RuntimeError("The FileHandle is not open.") - (bytes_written, extra_err) = _write(self._handle, buffer.ptr, size, file_offset, buffer_offset) - if (bytes_written == -1): + bytes_written, extra_err = _write( + self._handle, buffer.ptr, size, file_offset, buffer_offset + ) + if bytes_written == -1: # extra_err is errno raise OSError(extra_err, os.strerror(extra_err)) - elif (bytes_written < -1): + elif bytes_written < -1: # hipFile Error # If -bytes_written == OpError.HipDriverError, extra_err is hipError_t. # Otherwise, extra_err is 0. diff --git a/python/hipfile/hipMalloc.py b/python/hipfile/hipMalloc.py index 2988caf8..7018e567 100644 --- a/python/hipfile/hipMalloc.py +++ b/python/hipfile/hipMalloc.py @@ -3,6 +3,7 @@ without introducing a dependency at this early stage of development. Do not rely upon anything in this module. """ + import ctypes import sys @@ -24,14 +25,16 @@ _hip.hipFree.argtypes = [ctypes.c_void_p] _hip.hipFree.restype = ctypes.c_int + def hipMalloc(size_bytes: int) -> ctypes.c_void_p: d_ptr = ctypes.c_void_p() status = _hip.hipMalloc(ctypes.byref(d_ptr), ctypes.c_size_t(size_bytes)) - if (status != 0): + if status != 0: raise RuntimeError(f"hipMalloc failed ({status})") return d_ptr + def hipFree(ptr: ctypes.c_void_p) -> None: status = _hip.hipFree(ptr) - if (status != 0): + if status != 0: raise RuntimeError(f"hipFree failed ({status})") diff --git a/python/hipfile/properties.py b/python/hipfile/properties.py index 0a702547..a6335393 100644 --- a/python/hipfile/properties.py +++ b/python/hipfile/properties.py @@ -2,18 +2,20 @@ # Driver properties driver_get_properties as _driver_get_properties, # Library version - get_version as _get_version + get_version as _get_version, ) from hipfile.error import HipFileException + def driver_get_properties(): - (_props, err) = _driver_get_properties() - if (err[0] != 0): + _props, err = _driver_get_properties() + if err[0] != 0: raise HipFileException(err[0], err[1]) return _props + def get_version(): - (version_tuple, err) = _get_version() - if (err[0] != 0): + version_tuple, err = _get_version() + if err[0] != 0: raise HipFileException(err[0], err[1]) return version_tuple diff --git a/python/main.py b/python/main.py index 26e4d6c8..7475b5d0 100644 --- a/python/main.py +++ b/python/main.py @@ -11,7 +11,7 @@ Buffer, FileHandleType, driver_get_properties, - get_version + get_version, ) hipfile_version = get_version() @@ -33,8 +33,14 @@ with Driver() as hipfile_driver: print(f"Driver Use Count After: {hipfile_driver.use_count()}") with Buffer.from_ctypes_void_p(buffer, size, 0) as registered_buffer: - with FileHandle(input_path, os.O_RDWR | os.O_DIRECT | os.O_CREAT, handle_type = FileHandleType.OpaqueFD) as fh_input: - with FileHandle(output_path, os.O_RDWR | os.O_DIRECT | os.O_CREAT | os.O_TRUNC) as fh_output: + with FileHandle( + input_path, + os.O_RDWR | os.O_DIRECT | os.O_CREAT, + handle_type=FileHandleType.OpaqueFD, + ) as fh_input: + with FileHandle( + output_path, os.O_RDWR | os.O_DIRECT | os.O_CREAT | os.O_TRUNC + ) as fh_output: print(f"Transferring {size} bytes...") bytes_read = fh_input.read(registered_buffer, size, 0, 0) print(f"Bytes Read: {bytes_read}") @@ -43,18 +49,18 @@ hipFree(buffer) -with open(input_path, 'br') as file_in: +with open(input_path, "br") as file_in: hash_in = hashlib.sha256() - chunk = file_in.read(1 * 1024 * 1024) # 1 MiB - while (len(chunk) != 0): + chunk = file_in.read(1 * 1024 * 1024) # 1 MiB + while len(chunk) != 0: hash_in.update(chunk) - chunk = file_in.read(1 * 1024 * 1024) # 1 MiB + chunk = file_in.read(1 * 1024 * 1024) # 1 MiB print(f"Input File Hash: {hash_in.hexdigest()}") -with open(output_path, 'br') as file_out: +with open(output_path, "br") as file_out: hash_out = hashlib.sha256() - chunk = file_out.read(1 * 1024 * 1024) # 1 MiB - while (len(chunk) != 0): + chunk = file_out.read(1 * 1024 * 1024) # 1 MiB + while len(chunk) != 0: hash_out.update(chunk) - chunk = file_out.read(1 * 1024 * 1024) # 1 MiB + chunk = file_out.read(1 * 1024 * 1024) # 1 MiB print(f"Output File Hash: {hash_out.hexdigest()}") From 57546b3921cba0ba9ffdde1c66639e12e0c8e66a Mon Sep 17 00:00:00 2001 From: Riley Dixon Date: Wed, 8 Apr 2026 13:18:17 -0600 Subject: [PATCH 24/28] Python: Formatting with pylint --- python/hipfile/__init__.py | 4 +- python/hipfile/buffer.py | 3 +- python/hipfile/driver.py | 3 +- python/hipfile/enums.py | 85 ++++++++++++++++++------------------ python/hipfile/error.py | 3 +- python/hipfile/file.py | 21 ++++----- python/hipfile/hipMalloc.py | 1 + python/hipfile/properties.py | 3 +- python/main.py | 12 +++-- 9 files changed, 74 insertions(+), 61 deletions(-) diff --git a/python/hipfile/__init__.py b/python/hipfile/__init__.py index 40a8ba2a..e68d4dff 100644 --- a/python/hipfile/__init__.py +++ b/python/hipfile/__init__.py @@ -1,4 +1,6 @@ -from hipfile._hipfile import ( +# pylint: disable=C0114 + +from hipfile._hipfile import ( # pylint: disable=E0401,E0611 # Constants VERSION_MAJOR as _VERSION_MAJOR, VERSION_MINOR as _VERSION_MINOR, diff --git a/python/hipfile/buffer.py b/python/hipfile/buffer.py index 875608ca..831c9308 100644 --- a/python/hipfile/buffer.py +++ b/python/hipfile/buffer.py @@ -1,7 +1,8 @@ +# pylint: disable=C0114,C0115,C0116 from __future__ import annotations from typing import TYPE_CHECKING -from hipfile._hipfile import buf_deregister, buf_register +from hipfile._hipfile import buf_deregister, buf_register # pylint: disable=E0401,E0611 from hipfile.error import HipFileException if TYPE_CHECKING: diff --git a/python/hipfile/driver.py b/python/hipfile/driver.py index 04ee52a5..3813459d 100644 --- a/python/hipfile/driver.py +++ b/python/hipfile/driver.py @@ -1,4 +1,5 @@ -from hipfile._hipfile import driver_open, driver_close, use_count as _use_count +# pylint: disable=C0114,C0115,C0116 +from hipfile._hipfile import driver_open, driver_close, use_count as _use_count # pylint: disable=E0401,E0611 from hipfile.error import HipFileException diff --git a/python/hipfile/enums.py b/python/hipfile/enums.py index e51fec6b..61e7a08c 100644 --- a/python/hipfile/enums.py +++ b/python/hipfile/enums.py @@ -1,6 +1,7 @@ +# pylint: disable=C0114,C0115,C0116 from enum import IntEnum -from hipfile._hipfile import ( +from hipfile._hipfile import ( # pylint: disable=E0401,E0611 # hipFileOpError_t values (resolved from C at build time) hipFileSuccess, hipFileDriverNotInitialized, @@ -55,44 +56,44 @@ class OpError(IntEnum): in hipfile.h automatically. """ - Success = hipFileSuccess - DriverNotInitialized = hipFileDriverNotInitialized - DriverInvalidProps = hipFileDriverInvalidProps - DriverUnsupportedLimit = hipFileDriverUnsupportedLimit - DriverVersionMismatch = hipFileDriverVersionMismatch - DriverVersionReadError = hipFileDriverVersionReadError - DriverClosing = hipFileDriverClosing - PlatformNotSupported = hipFilePlatformNotSupported - IONotSupported = hipFileIONotSupported - DeviceNotSupported = hipFileDeviceNotSupported - DriverError = hipFileDriverError - HipDriverError = hipFileHipDriverError - HipPointerInvalid = hipFileHipPointerInvalid - HipMemoryTypeInvalid = hipFileHipMemoryTypeInvalid - HipPointerRangeError = hipFileHipPointerRangeError - HipContextMismatch = hipFileHipContextMismatch - InvalidMappingSize = hipFileInvalidMappingSize - InvalidMappingRange = hipFileInvalidMappingRange - InvalidFileType = hipFileInvalidFileType - InvalidFileOpenFlag = hipFileInvalidFileOpenFlag - DIONotSet = hipFileDIONotSet - InvalidValue = hipFileInvalidValue - MemoryAlreadyRegistered = hipFileMemoryAlreadyRegistered - MemoryNotRegistered = hipFileMemoryNotRegistered - PermissionDenied = hipFilePermissionDenied - DriverAlreadyOpen = hipFileDriverAlreadyOpen - HandleNotRegistered = hipFileHandleNotRegistered - HandleAlreadyRegistered = hipFileHandleAlreadyRegistered - DeviceNotFound = hipFileDeviceNotFound - InternalError = hipFileInternalError - GetNewFDFailed = hipFileGetNewFDFailed - DriverSetupError = hipFileDriverSetupError - IODisabled = hipFileIODisabled - BatchSubmitFailed = hipFileBatchSubmitFailed - GPUMemoryPinningFailed = hipFileGPUMemoryPinningFailed - BatchFull = hipFileBatchFull - AsyncNotSupported = hipFileAsyncNotSupported - IOMaxError = hipFileIOMaxError + SUCCESS = hipFileSuccess + DRIVER_NOT_INITIALIZED = hipFileDriverNotInitialized + DRIVER_INVALID_PROPS = hipFileDriverInvalidProps + DRIVER_UNSUPPORTED_LIMIT = hipFileDriverUnsupportedLimit + DRIVER_VERSION_MISMATCH = hipFileDriverVersionMismatch + DRIVER_VERSION_READ_ERROR = hipFileDriverVersionReadError + DRIVER_CLOSING = hipFileDriverClosing + PLATFORM_NOT_SUPPORTED = hipFilePlatformNotSupported + IO_NOT_SUPPORTED = hipFileIONotSupported + DEVICE_NOT_SUPPORTED = hipFileDeviceNotSupported + DRIVER_ERROR = hipFileDriverError + HIP_DRIVER_ERROR = hipFileHipDriverError + HIP_POINTER_INVALID = hipFileHipPointerInvalid + HIP_MEMORY_TYPE_INVALID = hipFileHipMemoryTypeInvalid + HIP_POINTER_RANGE_ERROR = hipFileHipPointerRangeError + HIP_CONTEXT_MISMATCH = hipFileHipContextMismatch + INVALID_MAPPING_SIZE = hipFileInvalidMappingSize + INVALID_MAPPING_RANGE = hipFileInvalidMappingRange + INVALID_FILE_TYPE = hipFileInvalidFileType + INVALID_FILE_OPEN_FLAG = hipFileInvalidFileOpenFlag + DIO_NOT_SET = hipFileDIONotSet + INVALID_VALUE = hipFileInvalidValue + MEMORY_ALREADY_REGISTERED = hipFileMemoryAlreadyRegistered + MEMORY_NOT_REGISTERED = hipFileMemoryNotRegistered + PERMISSION_DENIED = hipFilePermissionDenied + DRIVER_ALREADY_OPEN = hipFileDriverAlreadyOpen + HANDLE_NOT_REGISTERED = hipFileHandleNotRegistered + HANDLE_ALREADY_REGISTERED = hipFileHandleAlreadyRegistered + DEVICE_NOT_FOUND = hipFileDeviceNotFound + INTERNAL_ERROR = hipFileInternalError + GET_NEW_FD_FAILED = hipFileGetNewFDFailed + DRIVER_SETUP_ERROR = hipFileDriverSetupError + IO_DISABLED = hipFileIODisabled + BATCH_SUBMIT_FAILED = hipFileBatchSubmitFailed + GPU_MEMORY_PINNING_FAILED = hipFileGPUMemoryPinningFailed + BATCH_FULL = hipFileBatchFull + ASYNC_NOT_SUPPORTED = hipFileAsyncNotSupported + IO_MAX_ERROR = hipFileIOMaxError class FileHandleType(IntEnum): @@ -101,6 +102,6 @@ class FileHandleType(IntEnum): Values are sourced from the C enum via the Cython layer. """ - OpaqueFD = hipFileHandleTypeOpaqueFD - OpaqueWin32 = hipFileHandleTypeOpaqueWin32 - UserspaceFS = hipFileHandleTypeUserspaceFS + OPAQUE_FD = hipFileHandleTypeOpaqueFD + OPAQUE_WIN32 = hipFileHandleTypeOpaqueWin32 + USERSPACE_FS = hipFileHandleTypeUserspaceFS diff --git a/python/hipfile/error.py b/python/hipfile/error.py index 5664f5fe..1d378f2b 100644 --- a/python/hipfile/error.py +++ b/python/hipfile/error.py @@ -1,4 +1,5 @@ -from hipfile._hipfile import get_op_error_string +# pylint: disable=C0114,C0115,C0116 +from hipfile._hipfile import get_op_error_string # pylint: disable=E0401,E0611 class HipFileException(Exception): diff --git a/python/hipfile/file.py b/python/hipfile/file.py index ebd7f719..eb41909a 100644 --- a/python/hipfile/file.py +++ b/python/hipfile/file.py @@ -1,7 +1,8 @@ +# pylint: disable=C0114,C0115,C0116 import os import stat -from hipfile._hipfile import ( +from hipfile._hipfile import ( # pylint: disable=E0401,E0611 # File handles handle_register, handle_deregister, @@ -9,20 +10,20 @@ read as _read, write as _write, ) -from hipfile.enums import OpError, FileHandleType +from hipfile.enums import FileHandleType from hipfile.error import HipFileException -DefaultHandleType = None +default_handle_type = None if os.name == "posix": - DefaultHandleType = FileHandleType.OpaqueFD + default_handle_type = FileHandleType.OPAQUE_FD elif os.name == "nt": - DefaultHandleType = FileHandleType.OpaqueWin32 + default_handle_type = FileHandleType.OPAQUE_WIN32 class FileHandle: DEFAULT_MODE = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH - def __init__(self, path, flags, mode=DEFAULT_MODE, handle_type=DefaultHandleType): + def __init__(self, path, flags, mode=DEFAULT_MODE, handle_type=default_handle_type): self._fd = None self._flags = flags self._handle = None @@ -96,9 +97,9 @@ def read(self, buffer, size, file_offset, buffer_offset): if bytes_read == -1: # extra_err is errno raise OSError(extra_err, os.strerror(extra_err)) - elif bytes_read < -1: + if bytes_read < -1: # hipFile Error - # If -bytes_read == OpError.HipDriverError, extra_err is hipError_t. + # If -bytes_read == OpError.HIP_DRIVER_ERROR, extra_err is hipError_t. # Otherwise, extra_err is 0. raise HipFileException(-bytes_read, extra_err) return bytes_read @@ -112,9 +113,9 @@ def write(self, buffer, size, file_offset, buffer_offset): if bytes_written == -1: # extra_err is errno raise OSError(extra_err, os.strerror(extra_err)) - elif bytes_written < -1: + if bytes_written < -1: # hipFile Error - # If -bytes_written == OpError.HipDriverError, extra_err is hipError_t. + # If -bytes_written == OpError.HIP_DRIVER_ERROR, extra_err is hipError_t. # Otherwise, extra_err is 0. raise HipFileException(-bytes_written, extra_err) return bytes_written diff --git a/python/hipfile/hipMalloc.py b/python/hipfile/hipMalloc.py index 7018e567..803be6b2 100644 --- a/python/hipfile/hipMalloc.py +++ b/python/hipfile/hipMalloc.py @@ -1,3 +1,4 @@ +# pylint: disable=all """ This is a hack to have some semblance of GPU memory management without introducing a dependency at this early stage of diff --git a/python/hipfile/properties.py b/python/hipfile/properties.py index a6335393..241823c5 100644 --- a/python/hipfile/properties.py +++ b/python/hipfile/properties.py @@ -1,4 +1,5 @@ -from hipfile._hipfile import ( +# pylint: disable=C0114,C0116 +from hipfile._hipfile import ( # pylint: disable=E0401,E0611 # Driver properties driver_get_properties as _driver_get_properties, # Library version diff --git a/python/main.py b/python/main.py index 7475b5d0..a80ca6fb 100644 --- a/python/main.py +++ b/python/main.py @@ -1,7 +1,12 @@ +""" +A quick & rough script for testing the Cython bindings to the +hipFile C library. Reads a given file and copies it to an +output file, and then prints the hashes of the files. +""" + import hashlib import os import pathlib -import stat from hipfile.hipMalloc import hipFree, hipMalloc @@ -10,7 +15,6 @@ FileHandle, Buffer, FileHandleType, - driver_get_properties, get_version, ) @@ -27,7 +31,7 @@ # Larger IOs will be quietly truncated. size = min(input_path.stat().st_size, 2 * 1024 * 1024 * 1024 - 4 * 1024) buffer = hipMalloc(size) -buffer_ptr = buffer.value +buffer_ptr = buffer.value # pylint: disable=C0103 # False Positive print(f"Buffer located at: {buffer_ptr} | {hex(buffer_ptr)}") with Driver() as hipfile_driver: @@ -36,7 +40,7 @@ with FileHandle( input_path, os.O_RDWR | os.O_DIRECT | os.O_CREAT, - handle_type=FileHandleType.OpaqueFD, + handle_type=FileHandleType.OPAQUE_FD, ) as fh_input: with FileHandle( output_path, os.O_RDWR | os.O_DIRECT | os.O_CREAT | os.O_TRUNC From 737d1e667231bf62468a14bb0b6ba7b77a827a3a Mon Sep 17 00:00:00 2001 From: Riley Dixon Date: Wed, 8 Apr 2026 14:02:56 -0600 Subject: [PATCH 25/28] Python/CMake: Add single lint ignore for Python_add_library --- python/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 81ef8691..afac9114 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -51,9 +51,9 @@ add_custom_command( ) # ---- Build extension module ----------------------------------------------- - +# lint_cmake: -readability/wonkycase Python_add_library(_hipfile MODULE "${_C_OUT}" WITH_SOABI) - +# lint_cmake: +readability/wonkycase target_include_directories(_hipfile PRIVATE "${HIPFILE_INCLUDE_DIR}" "${HIP_INCLUDE_DIR}" From a337f12e02c76e7ca3b3d29cffff7edac2f13e86 Mon Sep 17 00:00:00 2001 From: Riley Dixon Date: Wed, 8 Apr 2026 13:47:26 -0600 Subject: [PATCH 26/28] Python: Align Cython functions to their C counterpart Originally these functions were given different names when the initial Cython code was generated. In part, this was to avoid namespace conflicts. Now that the C functions are imported into their own "_c" namespace, we can reuse the same function names without conflict. Reusing the names improves consistency of what Cython function calls which C function. --- python/hipfile/_hipfile.pyx | 28 ++++++++++++++-------------- python/hipfile/buffer.py | 9 ++++++--- python/hipfile/driver.py | 12 ++++++++---- python/hipfile/error.py | 4 ++-- python/hipfile/file.py | 18 ++++++++---------- python/hipfile/properties.py | 10 ++++------ 6 files changed, 42 insertions(+), 39 deletions(-) diff --git a/python/hipfile/_hipfile.pyx b/python/hipfile/_hipfile.pyx index d753da7b..78f5a556 100644 --- a/python/hipfile/_hipfile.pyx +++ b/python/hipfile/_hipfile.pyx @@ -118,7 +118,7 @@ def hip_drv_err(tuple err): return err[1] -def get_op_error_string(int status): +def hipFileGetOpErrorString(int status): """Wrapper for ``hipFileGetOpErrorString``.""" cdef const char *s = _c.hipFileGetOpErrorString(<_c.hipFileOpError_t>status) if s == NULL: @@ -130,17 +130,17 @@ def get_op_error_string(int status): # Driver lifecycle # --------------------------------------------------------------------------- -def driver_open(): +def hipFileDriverOpen(): """Wrapper for ``hipFileDriverOpen``.""" return _err(_c.hipFileDriverOpen()) -def driver_close(): +def hipFileDriverClose(): """Wrapper for ``hipFileDriverClose``.""" return _err(_c.hipFileDriverClose()) -def use_count(): +def hipFileUseCount(): """Wrapper for ``hipFileUseCount``.""" return _c.hipFileUseCount() @@ -149,7 +149,7 @@ def use_count(): # Version # --------------------------------------------------------------------------- -def get_version(): +def hipFileGetVersion(): """Wrapper for ``hipFileGetVersion``. Returns ``((major, minor, patch), error_tuple)``. @@ -163,7 +163,7 @@ def get_version(): # File handles # --------------------------------------------------------------------------- -def handle_register(int fd, int handle_type): +def hipFileHandleRegister(int fd, int handle_type): """Wrapper for ``hipFileHandleRegister``. Parameters @@ -185,7 +185,7 @@ def handle_register(int fd, int handle_type): return (fh, _err(e)) -def handle_deregister(uintptr_t handle): +def hipFileHandleDeregister(uintptr_t handle): """Wrapper for ``hipFileHandleDeregister``.""" _c.hipFileHandleDeregister(<_c.hipFileHandle_t>handle) @@ -194,12 +194,12 @@ def handle_deregister(uintptr_t handle): # Buffer registration # --------------------------------------------------------------------------- -def buf_register(uintptr_t buffer_base, size_t length, int flags=0): +def hipFileBufRegister(uintptr_t buffer_base, size_t length, int flags=0): """Wrapper for ``hipFileBufRegister``.""" return _err(_c.hipFileBufRegister(buffer_base, length, flags)) -def buf_deregister(uintptr_t buffer_base): +def hipFileBufDeregister(uintptr_t buffer_base): """Wrapper for ``hipFileBufDeregister``.""" return _err(_c.hipFileBufDeregister(buffer_base)) @@ -208,8 +208,8 @@ def buf_deregister(uintptr_t buffer_base): # Synchronous I/O # --------------------------------------------------------------------------- -def read(uintptr_t handle, uintptr_t buffer_base, size_t size, - _c.hoff_t file_offset, _c.hoff_t buffer_offset): +def hipFileRead(uintptr_t handle, uintptr_t buffer_base, size_t size, + _c.hoff_t file_offset, _c.hoff_t buffer_offset): """Wrapper for ``hipFileRead``. Returns ``(result, extra)``: @@ -230,8 +230,8 @@ def read(uintptr_t handle, uintptr_t buffer_base, size_t size, return (ret, extra) -def write(uintptr_t handle, uintptr_t buffer_base, size_t size, - _c.hoff_t file_offset, _c.hoff_t buffer_offset): +def hipFileWrite(uintptr_t handle, uintptr_t buffer_base, size_t size, + _c.hoff_t file_offset, _c.hoff_t buffer_offset): """Wrapper for ``hipFileWrite``. Returns ``(result, extra)``: @@ -256,7 +256,7 @@ def write(uintptr_t handle, uintptr_t buffer_base, size_t size, # Driver properties # --------------------------------------------------------------------------- -def driver_get_properties(): +def hipFileDriverGetProperties(): """Wrapper for ``hipFileDriverGetProperties``. Returns ``(props_dict, error_tuple)``. diff --git a/python/hipfile/buffer.py b/python/hipfile/buffer.py index 831c9308..2f75d6cc 100644 --- a/python/hipfile/buffer.py +++ b/python/hipfile/buffer.py @@ -2,7 +2,10 @@ from __future__ import annotations from typing import TYPE_CHECKING -from hipfile._hipfile import buf_deregister, buf_register # pylint: disable=E0401,E0611 +from hipfile._hipfile import ( # pylint: disable=E0401,E0611 + hipFileBufDeregister, + hipFileBufRegister, +) from hipfile.error import HipFileException if TYPE_CHECKING: @@ -38,13 +41,13 @@ def ptr(self): def deregister(self): if self._registered: - err = buf_deregister(self._buffer_ptr) + err = hipFileBufDeregister(self._buffer_ptr) if err[0] != 0: raise HipFileException(err[0], err[1]) self._registered = False def register(self): - err = buf_register(self._buffer_ptr, self._length, self._flags) + err = hipFileBufRegister(self._buffer_ptr, self._length, self._flags) if err[0] != 0: raise HipFileException(err[0], err[1]) self._registered = True diff --git a/python/hipfile/driver.py b/python/hipfile/driver.py index 3813459d..16242c58 100644 --- a/python/hipfile/driver.py +++ b/python/hipfile/driver.py @@ -1,5 +1,9 @@ # pylint: disable=C0114,C0115,C0116 -from hipfile._hipfile import driver_open, driver_close, use_count as _use_count # pylint: disable=E0401,E0611 +from hipfile._hipfile import ( # pylint: disable=E0401,E0611 + hipFileDriverOpen, + hipFileDriverClose, + hipFileUseCount, +) from hipfile.error import HipFileException @@ -7,7 +11,7 @@ class Driver: @staticmethod def use_count(): - return _use_count() + return hipFileUseCount() def __enter__(self): self.open() @@ -17,11 +21,11 @@ def __exit__(self, exc_type, exc_value, traceback): self.close() def close(self): - err = driver_close() + err = hipFileDriverClose() if err[0] != 0: raise HipFileException(err[0], err[1]) def open(self): - err = driver_open() + err = hipFileDriverOpen() if err[0] != 0: raise HipFileException(err[0], err[1]) diff --git a/python/hipfile/error.py b/python/hipfile/error.py index 1d378f2b..e1cbc5cb 100644 --- a/python/hipfile/error.py +++ b/python/hipfile/error.py @@ -1,5 +1,5 @@ # pylint: disable=C0114,C0115,C0116 -from hipfile._hipfile import get_op_error_string # pylint: disable=E0401,E0611 +from hipfile._hipfile import hipFileGetOpErrorString # pylint: disable=E0401,E0611 class HipFileException(Exception): @@ -16,7 +16,7 @@ def hip_err(self): return self._hip_err def __str__(self): - err_msg = f"{self._hipfile_err} - {get_op_error_string(self._hipfile_err)}" + err_msg = f"{self._hipfile_err} - {hipFileGetOpErrorString(self._hipfile_err)}" if self._hipfile_err == 5011: err_msg += f" {self._hip_err}" return err_msg diff --git a/python/hipfile/file.py b/python/hipfile/file.py index eb41909a..345303f9 100644 --- a/python/hipfile/file.py +++ b/python/hipfile/file.py @@ -3,12 +3,10 @@ import stat from hipfile._hipfile import ( # pylint: disable=E0401,E0611 - # File handles - handle_register, - handle_deregister, - # Synchronous I/O - read as _read, - write as _write, + hipFileHandleRegister, + hipFileHandleDeregister, + hipFileRead, + hipFileWrite, ) from hipfile.enums import FileHandleType from hipfile.error import HipFileException @@ -73,7 +71,7 @@ def open(self): if self._handle is not None: raise RuntimeError("The FileHandle is already open.") self._fd = os.open(self._path, self._flags, self._mode) - handle, err = handle_register(self._fd, self._handle_type) + handle, err = hipFileHandleRegister(self._fd, self._handle_type) if err[0] != 0: os.close(self._fd) self._fd = None @@ -82,7 +80,7 @@ def open(self): def close(self): if self._handle is not None: - handle_deregister(self._handle) + hipFileHandleDeregister(self._handle) self._handle = None if self._fd is not None: os.close(self._fd) @@ -91,7 +89,7 @@ def close(self): def read(self, buffer, size, file_offset, buffer_offset): if self._handle is None: raise RuntimeError("The FileHandle is not open.") - bytes_read, extra_err = _read( + bytes_read, extra_err = hipFileRead( self._handle, buffer.ptr, size, file_offset, buffer_offset ) if bytes_read == -1: @@ -107,7 +105,7 @@ def read(self, buffer, size, file_offset, buffer_offset): def write(self, buffer, size, file_offset, buffer_offset): if self._handle is None: raise RuntimeError("The FileHandle is not open.") - bytes_written, extra_err = _write( + bytes_written, extra_err = hipFileWrite( self._handle, buffer.ptr, size, file_offset, buffer_offset ) if bytes_written == -1: diff --git a/python/hipfile/properties.py b/python/hipfile/properties.py index 241823c5..a6ae68e2 100644 --- a/python/hipfile/properties.py +++ b/python/hipfile/properties.py @@ -1,22 +1,20 @@ # pylint: disable=C0114,C0116 from hipfile._hipfile import ( # pylint: disable=E0401,E0611 - # Driver properties - driver_get_properties as _driver_get_properties, - # Library version - get_version as _get_version, + hipFileDriverGetProperties, + hipFileGetVersion, ) from hipfile.error import HipFileException def driver_get_properties(): - _props, err = _driver_get_properties() + _props, err = hipFileDriverGetProperties() if err[0] != 0: raise HipFileException(err[0], err[1]) return _props def get_version(): - version_tuple, err = _get_version() + version_tuple, err = hipFileGetVersion() if err[0] != 0: raise HipFileException(err[0], err[1]) return version_tuple From e0039c9d45f5788ab8925a42d7d6e1eebb8b939e Mon Sep 17 00:00:00 2001 From: Riley Dixon Date: Wed, 8 Apr 2026 16:57:05 -0600 Subject: [PATCH 27/28] Python: Address copilot review --- python/hipfile/_chipfile.pxd | 3 --- python/hipfile/buffer.py | 8 +++++++- python/hipfile/error.py | 3 ++- python/hipfile/file.py | 1 - 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/python/hipfile/_chipfile.pxd b/python/hipfile/_chipfile.pxd index ed3a1fa8..cb841e72 100644 --- a/python/hipfile/_chipfile.pxd +++ b/python/hipfile/_chipfile.pxd @@ -7,7 +7,6 @@ wrapped by the low-level Cython bindings. """ from libc.stdint cimport int64_t, uint64_t -from libc.stddef cimport size_t from posix.types cimport off_t @@ -87,8 +86,6 @@ cdef extern from "hipfile.h": hipFileOpError_t err hipError_t hip_drv_err - const char *hipFileGetOpErrorString(hipFileOpError_t status) - # -- Opaque handles ----------------------------------------------------- ctypedef void *hipFileHandle_t diff --git a/python/hipfile/buffer.py b/python/hipfile/buffer.py index 2f75d6cc..e5f9b60a 100644 --- a/python/hipfile/buffer.py +++ b/python/hipfile/buffer.py @@ -1,6 +1,7 @@ # pylint: disable=C0114,C0115,C0116 from __future__ import annotations from typing import TYPE_CHECKING +from sys import stderr from hipfile._hipfile import ( # pylint: disable=E0401,E0611 hipFileBufDeregister, @@ -26,7 +27,12 @@ def __init__(self, buffer_ptr, length, flags) -> None: def __del__(self): # We did not create the underlying buffer. Don't try to free it. - self.deregister() + try: + self.deregister() + except Exception: # pylint: disable=W0718 # Suppress exceptions in a dtor + print( + "Failed to deregister hipFile.Buffer at destruction time.", file=stderr + ) def __enter__(self): self.register() diff --git a/python/hipfile/error.py b/python/hipfile/error.py index e1cbc5cb..f20882aa 100644 --- a/python/hipfile/error.py +++ b/python/hipfile/error.py @@ -1,5 +1,6 @@ # pylint: disable=C0114,C0115,C0116 from hipfile._hipfile import hipFileGetOpErrorString # pylint: disable=E0401,E0611 +from hipfile.enums import OpError class HipFileException(Exception): @@ -17,6 +18,6 @@ def hip_err(self): def __str__(self): err_msg = f"{self._hipfile_err} - {hipFileGetOpErrorString(self._hipfile_err)}" - if self._hipfile_err == 5011: + if self._hipfile_err == OpError.HIP_DRIVER_ERROR: err_msg += f" {self._hip_err}" return err_msg diff --git a/python/hipfile/file.py b/python/hipfile/file.py index 345303f9..f0bb2076 100644 --- a/python/hipfile/file.py +++ b/python/hipfile/file.py @@ -74,7 +74,6 @@ def open(self): handle, err = hipFileHandleRegister(self._fd, self._handle_type) if err[0] != 0: os.close(self._fd) - self._fd = None raise HipFileException(err[0], err[1]) self._handle = handle From 0a384567ef21052267fdf0b4441adaf79fac1bed Mon Sep 17 00:00:00 2001 From: Riley Dixon Date: Wed, 8 Apr 2026 18:11:29 -0600 Subject: [PATCH 28/28] Python: Remove Win32 support from FileHandle Removes supporting Win32 from FileHandle. FileHandle was designed to manage opening & closing its own file reference. Python does not provide a public API to open a file which returns a Win32 handle. This leads to a contrived process of opening a file, getting a file descriptor, and converting it to a Win32 handle. To be clear, os.open() on Windows provides a POSIX FD. FileHandle does not currently support a user providing an already open file resource, which at this time seems to be the only plausible use case of a Win32 handle being used. For the sake of simplicity and getting out an initial release, we will just error if the OPAQUE_WIN32 handle type is set. (This however will leave Win32 support in the Cython layer as that is a trivial switch that gets passed into the C library.) --- python/hipfile/_hipfile.pyx | 11 +++++++---- python/hipfile/file.py | 16 +++++++++------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/python/hipfile/_hipfile.pyx b/python/hipfile/_hipfile.pyx index 78f5a556..e74aeb3e 100644 --- a/python/hipfile/_hipfile.pyx +++ b/python/hipfile/_hipfile.pyx @@ -163,13 +163,13 @@ def hipFileGetVersion(): # File handles # --------------------------------------------------------------------------- -def hipFileHandleRegister(int fd, int handle_type): +def hipFileHandleRegister(uintptr_t handle_value, int handle_type): """Wrapper for ``hipFileHandleRegister``. Parameters ---------- - fd : int - POSIX file descriptor. + handle_value : int + POSIX file descriptor or Win32 HANDLE, depending on *handle_type*. handle_type : int Value from ``hipFileFileHandleType_t``. @@ -180,7 +180,10 @@ def hipFileHandleRegister(int fd, int handle_type): cdef _c.hipFileDescr_t descr memset(&descr, 0, sizeof(descr)) descr.type = <_c.hipFileFileHandleType_t>handle_type - descr.fd = fd + if handle_type == _c.hipFileHandleTypeOpaqueWin32: + descr.hFile = handle_value + else: + descr.fd = handle_value cdef _c.hipFileError_t e = _c.hipFileHandleRegister(&fh, &descr) return (fh, _err(e)) diff --git a/python/hipfile/file.py b/python/hipfile/file.py index f0bb2076..9dfe327d 100644 --- a/python/hipfile/file.py +++ b/python/hipfile/file.py @@ -11,17 +11,13 @@ from hipfile.enums import FileHandleType from hipfile.error import HipFileException -default_handle_type = None -if os.name == "posix": - default_handle_type = FileHandleType.OPAQUE_FD -elif os.name == "nt": - default_handle_type = FileHandleType.OPAQUE_WIN32 - class FileHandle: DEFAULT_MODE = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH - def __init__(self, path, flags, mode=DEFAULT_MODE, handle_type=default_handle_type): + def __init__( + self, path, flags, mode=DEFAULT_MODE, handle_type=FileHandleType.OPAQUE_FD + ): self._fd = None self._flags = flags self._handle = None @@ -55,8 +51,14 @@ def handle_type(self): @handle_type.setter def handle_type(self, _handle_type): + if self._handle is not None: + raise RuntimeError("Cannot modify handle_type while FileHandle is open") if _handle_type not in FileHandleType: raise ValueError(f"'{_handle_type}' is not a member of enum FileHandleType") + if _handle_type == FileHandleType.OPAQUE_WIN32: + raise NotImplementedError( + "FileHandle does not currently support Win32 Handles" + ) self._handle_type = _handle_type @property